diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f2128090..a37f1664c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,12 @@ naming scheme accordingly before upgrading. - Fixed a bug causing `external: false` entries in the Compose file to be printed as `external: true` in the output of `docker-compose config` +- Fixed a bug where issuing a `docker-compose pull` command on services + without a defined image key would cause Compose to crash + +- Volumes and binds are now mounted in the order they're declared in the + service definition + ### Miscellaneous - The `zsh` completion script has been updated with new options, and no diff --git a/compose/__init__.py b/compose/__init__.py index f0e3f3274..1f35b9a35 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.23.0-rc1' +__version__ = '1.23.0-rc2' diff --git a/compose/project.py b/compose/project.py index 4340577c9..92c352050 100644 --- a/compose/project.py +++ b/compose/project.py @@ -34,6 +34,7 @@ from .service import Service from .service import ServiceNetworkMode from .service import ServicePidMode from .utils import microseconds_from_time_nano +from .utils import truncate_string from .volume import ProjectVolumes @@ -554,12 +555,10 @@ class Project(object): if parallel_pull: def pull_service(service): strm = service.pull(ignore_pull_failures, True, stream=True) - writer = parallel.get_stream_writer() + if strm is None: # Attempting to pull service with no `image` key is a no-op + return - def trunc(s): - if len(s) > 35: - return s[:33] + '...' - return s + writer = parallel.get_stream_writer() for event in strm: if 'status' not in event: @@ -572,7 +571,7 @@ class Project(object): status = '{} ({:.1%})'.format(status, percentage) writer.write( - msg, service.name, trunc(status), lambda s: s + msg, service.name, truncate_string(status), lambda s: s ) _, errors = parallel.parallel_execute( diff --git a/compose/service.py b/compose/service.py index aca24ce17..3327c77f8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -56,6 +56,7 @@ from .utils import json_hash from .utils import parse_bytes from .utils import parse_seconds_float from .utils import truncate_id +from .utils import unique_everseen log = logging.getLogger(__name__) @@ -940,8 +941,9 @@ class Service(object): override_options['mounts'] = override_options.get('mounts') or [] override_options['mounts'].extend([build_mount(v) for v in secret_volumes]) - # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885) - override_options['binds'] = list(set(binds)) + # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885). + # unique_everseen preserves order. (see https://github.com/docker/compose/issues/6091). + override_options['binds'] = list(unique_everseen(binds)) return container_options, override_options def _get_container_host_config(self, override_options, one_off=False): @@ -1427,7 +1429,7 @@ def merge_volume_bindings(volumes, tmpfs, previous_container, mounts): """ affinity = {} - volume_bindings = dict( + volume_bindings = OrderedDict( build_volume_binding(volume) for volume in volumes if volume.external diff --git a/compose/utils.py b/compose/utils.py index 8f0b3e549..9f0441d08 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -170,3 +170,19 @@ def truncate_id(value): if len(value) > 12: return value[:12] return value + + +def unique_everseen(iterable, key=lambda x: x): + "List unique elements, preserving order. Remember all elements ever seen." + seen = set() + for element in iterable: + unique_key = key(element) + if unique_key not in seen: + seen.add(unique_key) + yield element + + +def truncate_string(s, max_chars=35): + if len(s) > max_chars: + return s[:max_chars - 2] + '...' + return s diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index f4c42362c..395888d34 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -136,7 +136,18 @@ _docker_compose_bundle() { _docker_compose_config() { - COMPREPLY=( $( compgen -W "--help --quiet -q --resolve-image-digests --services --volumes --hash" -- "$cur" ) ) + case "$prev" in + --hash) + if [[ $cur == \\* ]] ; then + COMPREPLY=( '\*' ) + else + COMPREPLY=( $(compgen -W "$(__docker_compose_services) \\\* " -- "$cur") ) + fi + return + ;; + esac + + COMPREPLY=( $( compgen -W "--hash --help --quiet -q --resolve-image-digests --services --volumes" -- "$cur" ) ) } diff --git a/script/release/release.py b/script/release/release.py index 749ea49d3..9a5af3aa5 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -173,9 +173,10 @@ def distclean(): def pypi_upload(args): print('Uploading to PyPi') try: + rel = args.release.replace('-rc', 'rc') twine_upload([ - 'dist/docker_compose-{}*.whl'.format(args.release), - 'dist/docker-compose-{}*.tar.gz'.format(args.release) + 'dist/docker_compose-{}*.whl'.format(rel), + 'dist/docker-compose-{}*.tar.gz'.format(rel) ]) except HTTPError as e: if e.response.status_code == 400 and 'File already exists' in e.message: diff --git a/script/run/run.sh b/script/run/run.sh index fa2248609..f02135f42 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.23.0-rc1" +VERSION="1.23.0-rc2" IMAGE="docker/compose:$VERSION" diff --git a/script/test/versions.py b/script/test/versions.py index 6d273a9e6..a06c49f20 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -36,6 +36,8 @@ import requests GITHUB_API = 'https://api.github.com/repos' +STAGES = ['tp', 'beta', 'rc'] + class Version(namedtuple('_Version', 'major minor patch stage edition')): @@ -45,7 +47,7 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')): version = version.lstrip('v') version, _, stage = version.partition('-') if stage: - if not any(marker in stage for marker in ['rc', 'tp', 'beta']): + if not any(marker in stage for marker in STAGES): edition = stage stage = None elif '-' in stage: @@ -62,8 +64,16 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')): """Return a representation that allows this object to be sorted correctly with the default comparator. """ - # rc releases should appear before official releases - stage = (0, self.stage) if self.stage else (1, ) + # non-GA releases should appear before GA releases + # Order: tp -> beta -> rc -> GA + if self.stage: + for st in STAGES: + if st in self.stage: + stage = (STAGES.index(st), self.stage) + break + else: + stage = (len(STAGES),) + return (int(self.major), int(self.minor), int(self.patch)) + stage def __str__(self): @@ -124,9 +134,6 @@ def get_versions(tags): v = Version.parse(tag['name']) if v in BLACKLIST: continue - # FIXME: Temporary. Remove once these versions are built on dockerswarm/dind - if v.stage and 'rc' not in v.stage: - continue yield v except ValueError: print("Skipping invalid tag: {name}".format(**tag), file=sys.stderr) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 63939676e..57f3b7074 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -105,6 +105,23 @@ class ProjectTest(DockerClientTestCase): project = Project('composetest', [web, db], self.client) assert set(project.containers(stopped=True)) == set([web_1, db_1]) + def test_parallel_pull_with_no_image(self): + config_data = build_config( + version=V2_3, + services=[{ + 'name': 'web', + 'build': {'context': '.'}, + }], + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, + client=self.client + ) + + project.pull(parallel_pull=True) + def test_volumes_from_service(self): project = Project.from_config( name='composetest', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1d42c10d5..52c89a9e0 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -8,6 +8,7 @@ import os import shutil import tempfile from operator import itemgetter +from random import shuffle import py import pytest @@ -42,7 +43,7 @@ from tests import unittest DEFAULT_VERSION = V2_0 -def make_service_dict(name, service_dict, working_dir, filename=None): +def make_service_dict(name, service_dict, working_dir='.', filename=None): """Test helper function to construct a ServiceExtendsResolver """ resolver = config.ServiceExtendsResolver( @@ -3536,6 +3537,13 @@ class VolumeConfigTest(unittest.TestCase): ).services[0] assert d['volumes'] == [VolumeSpec.parse('/host/path:/container/path')] + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') + def test_volumes_order_is_preserved(self): + volumes = ['/{0}:/{0}'.format(i) for i in range(0, 6)] + shuffle(volumes) + cfg = make_service_dict('foo', {'build': '.', 'volumes': volumes}) + assert cfg['volumes'] == volumes + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') @mock.patch.dict(os.environ) def test_volume_binding_with_home(self): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index d5dbcbea6..af1cd1bea 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -1037,6 +1037,23 @@ class ServiceTest(unittest.TestCase): assert len(override_opts['binds']) == 1 assert override_opts['binds'][0] == 'vol:/data:rw' + def test_volumes_order_is_preserved(self): + service = Service('foo', client=self.mock_client) + volumes = [ + VolumeSpec.parse(cfg) for cfg in [ + '/v{0}:/v{0}:rw'.format(i) for i in range(6) + ] + ] + ctnr_opts, override_opts = service._build_container_volume_options( + previous_container=None, + container_options={ + 'volumes': volumes, + 'environment': {}, + }, + override_options={}, + ) + assert override_opts['binds'] == [vol.repr() for vol in volumes] + class TestServiceNetwork(unittest.TestCase): def setUp(self): diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 84becb975..21b88d962 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -68,3 +68,11 @@ class TestParseBytes(object): assert utils.parse_bytes(123) == 123 assert utils.parse_bytes('foobar') is None assert utils.parse_bytes('123') == 123 + + +class TestMoreItertools(object): + def test_unique_everseen(self): + unique = utils.unique_everseen + assert list(unique([2, 1, 2, 1])) == [2, 1] + assert list(unique([2, 1, 2, 1], hash)) == [2, 1] + assert list(unique([2, 1, 2, 1], lambda x: 'key_%s' % x)) == [2, 1]