From 8471276e73e7fe431e6e47d119d1899775ab1b4c Mon Sep 17 00:00:00 2001 From: fate-grand-order Date: Fri, 17 Mar 2017 11:35:10 +0800 Subject: [PATCH 01/33] fix misspell "compatibility" in script/ci Signed-off-by: fate-grand-order --- script/ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/ci b/script/ci index 7b3489a1b..34bf9a4be 100755 --- a/script/ci +++ b/script/ci @@ -1,6 +1,6 @@ #!/bin/bash # -# Backwards compatiblity for jenkins +# Backwards compatibility for jenkins # # TODO: remove this script after all current PRs and jenkins are updated with # the new script/test/ci change From 69d0c0e3a00df6bf0d4e4701d125436aa34393ad Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 17 Mar 2017 18:22:44 -0700 Subject: [PATCH 02/33] Add support for expanded mount/volume notation Signed-off-by: Joffrey F --- compose/config/config.py | 14 ++++- tests/acceptance/cli_test.py | 7 ++- tests/fixtures/v3-full/docker-compose.yml | 13 ++++- tests/unit/config/config_test.py | 62 +++++++++++++++++++++++ 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 225919415..8cbaae272 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1030,7 +1030,13 @@ def resolve_volume_paths(working_dir, service_dict): def resolve_volume_path(working_dir, volume): - container_path, host_path = split_path_mapping(volume) + if isinstance(volume, dict): + host_path = volume.get('source') + container_path = volume.get('target') + if host_path and volume.get('read_only'): + container_path += ':ro' + else: + container_path, host_path = split_path_mapping(volume) if host_path is not None: if host_path.startswith('.'): @@ -1112,6 +1118,8 @@ def split_path_mapping(volume_path): path. Using splitdrive so windows absolute paths won't cause issues with splitting on ':'. """ + if isinstance(volume_path, dict): + return (volume_path.get('target'), volume_path) drive, volume_config = splitdrive(volume_path) if ':' in volume_config: @@ -1123,7 +1131,9 @@ def split_path_mapping(volume_path): def join_path_mapping(pair): (container, host) = pair - if host is None: + if isinstance(host, dict): + return host + elif host is None: return container else: return ":".join((host, container)) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6a498e250..14e6f7336 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -321,7 +321,7 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['config']) assert yaml.load(result.stdout) == { - 'version': '3.0', + 'version': '3.2', 'networks': {}, 'volumes': { 'foobar': { @@ -371,6 +371,11 @@ class CLITestCase(DockerClientTestCase): 'timeout': '1s', 'retries': 5, }, + 'volumes': [ + '/host/path:/container/path:ro', + 'foobar:/container/volumepath:rw', + '/anonymous' + ], 'stop_grace_period': '20s', }, diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml index a1661ab93..27f3c6e04 100644 --- a/tests/fixtures/v3-full/docker-compose.yml +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3" +version: "3.2" services: web: image: busybox @@ -34,6 +34,17 @@ services: timeout: 1s retries: 5 + volumes: + - source: /host/path + target: /container/path + type: bind + read_only: true + - source: foobar + type: volume + target: /container/volumepath + - type: volume + target: /anonymous + stop_grace_period: 20s volumes: foobar: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c86485d7b..c9eb3796e 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -29,6 +29,7 @@ from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_1 as V2_1 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_1 as V3_1 +from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import IS_WINDOWS_PLATFORM from compose.utils import nanoseconds_from_time_seconds from tests import mock @@ -964,6 +965,44 @@ class ConfigTest(unittest.TestCase): ] assert service_sort(service_dicts) == service_sort(expected) + @mock.patch.dict(os.environ) + def test_load_with_multiple_files_v3_2(self): + os.environ['COMPOSE_CONVERT_WINDOWS_PATHS'] = 'true' + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.2', + 'services': { + 'web': { + 'image': 'example/web', + 'volumes': [ + {'source': '/a', 'target': '/b', 'type': 'bind'}, + {'source': 'vol', 'target': '/x', 'type': 'volume', 'read_only': True} + ] + } + }, + 'volumes': {'vol': {}} + } + ) + + override_file = config.ConfigFile( + 'override.yaml', + { + 'version': '3.2', + 'services': { + 'web': { + 'volumes': ['/c:/b', '/anonymous'] + } + } + } + ) + details = config.ConfigDetails('.', [base_file, override_file]) + service_dicts = config.load(details).services + svc_volumes = map(lambda v: v.repr(), service_dicts[0]['volumes']) + assert sorted(svc_volumes) == sorted( + ['/anonymous', '/c:/b:rw', 'vol:/x:ro'] + ) + def test_undeclared_volume_v2(self): base_file = config.ConfigFile( 'base.yaml', @@ -1544,6 +1583,29 @@ class ConfigTest(unittest.TestCase): 'ports': types.ServicePort.parse('5432') } + def test_merge_service_dicts_heterogeneous_volumes(self): + base = { + 'volumes': ['/a:/b', '/x:/z'], + } + + override = { + 'image': 'alpine:edge', + 'volumes': [ + {'source': '/e', 'target': '/b', 'type': 'bind'}, + {'source': '/c', 'target': '/d', 'type': 'bind'} + ] + } + + actual = config.merge_service_dicts_from_files( + base, override, V3_2 + ) + + assert actual['volumes'] == [ + {'source': '/e', 'target': '/b', 'type': 'bind'}, + {'source': '/c', 'target': '/d', 'type': 'bind'}, + '/x:/z' + ] + def test_merge_logging_v1(self): base = { 'image': 'alpine:edge', From 5fdbd5026a4b4f73f9628e6c81cc11cdded76d2c Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Sun, 19 Mar 2017 08:00:35 -0700 Subject: [PATCH 03/33] Add bash completion for `config --resolve-image-digests|--volumes"` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 4d134b5cb..1cd5a496f 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -142,7 +142,7 @@ _docker_compose_bundle() { _docker_compose_config() { - COMPREPLY=( $( compgen -W "--help --quiet -q --services" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --quiet -q --resolve-image-digests --services --volumes" -- "$cur" ) ) } From 02c294ee287ead04bc3163bca43c0c54c058d62c Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Mon, 20 Mar 2017 16:27:56 +0100 Subject: [PATCH 04/33] Fix bash completion for `up --exit-code-from` `--exit-code-from` requires an argument. Also corrected sort order. Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 4d134b5cb..c8d08ce89 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -498,6 +498,10 @@ _docker_compose_unpause() { _docker_compose_up() { case "$prev" in + --exit-code-from) + __docker_compose_services_all + return + ;; --timeout|-t) return ;; @@ -505,7 +509,7 @@ _docker_compose_up() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--exit-code-from --abort-on-container-exit --build -d --force-recreate --help --no-build --no-color --no-deps --no-recreate --timeout -t --remove-orphans" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--abort-on-container-exit --build -d --exit-code-from --force-recreate --help --no-build --no-color --no-deps --no-recreate --timeout -t --remove-orphans" -- "$cur" ) ) ;; *) __docker_compose_services_all From 0cf5f28f17cd0784f3a331eb1fcfef43846edbfb Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Mon, 20 Mar 2017 16:37:41 +0100 Subject: [PATCH 05/33] Add bash completion for `pull --parallel` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 4d134b5cb..b1d0a393b 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -341,7 +341,7 @@ _docker_compose_ps() { _docker_compose_pull() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --ignore-pull-failures" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --parallel" -- "$cur" ) ) ;; *) __docker_compose_services_from_image From 732bf52a4e40f8a974262896e5fffb5f9c4e80d8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Mar 2017 14:09:50 -0700 Subject: [PATCH 06/33] The interval is too damn small Signed-off-by: Joffrey F --- tests/integration/project_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index e8dbe8fbf..455189851 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1419,7 +1419,7 @@ class ProjectTest(DockerClientTestCase): 'test': 'exit 0', 'retries': 1, 'timeout': '10s', - 'interval': '0.1s' + 'interval': '1s' }, }, 'svc2': { @@ -1456,7 +1456,7 @@ class ProjectTest(DockerClientTestCase): 'test': 'exit 1', 'retries': 1, 'timeout': '10s', - 'interval': '0.1s' + 'interval': '1s' }, }, 'svc2': { From a0add5cc129f3f7a4d43db777155daafccac371b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 21 Mar 2017 17:23:29 -0700 Subject: [PATCH 07/33] Fix ports reparsing for service extends Signed-off-by: Joffrey F --- compose/config/types.py | 4 ++++ tests/unit/config/config_test.py | 24 +++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/compose/config/types.py b/compose/config/types.py index a8d366fba..96846b5ba 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -266,6 +266,10 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext @classmethod def parse(cls, spec): + if isinstance(spec, cls): + # WHen extending a service with ports, the port definitions have already been parsed + return [spec] + if not isinstance(spec, dict): result = [] for k, v in build_port_bindings([spec]).items(): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 195efe3b9..4db87ecb6 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3403,7 +3403,7 @@ class ExtendsTest(unittest.TestCase): self.assertEqual(service[0]['command'], "top") def test_extends_with_depends_on(self): - tmpdir = py.test.ensuretemp('test_extends_with_defined_version') + tmpdir = py.test.ensuretemp('test_extends_with_depends_on') self.addCleanup(tmpdir.remove) tmpdir.join('docker-compose.yml').write(""" version: "2" @@ -3435,6 +3435,28 @@ class ExtendsTest(unittest.TestCase): } }] + def test_extends_with_ports(self): + tmpdir = py.test.ensuretemp('test_extends_with_ports') + self.addCleanup(tmpdir.remove) + tmpdir.join('docker-compose.yml').write(""" + version: '2' + + services: + a: + image: nginx + ports: + - 80 + + b: + extends: + service: a + """) + services = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + + assert len(services) == 2 + for svc in services: + assert svc['ports'] == [types.ServicePort('80', None, None, None, None)] + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From f55c9d42013e8fbb5285bc402d8248a846485217 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Mar 2017 17:27:11 -0700 Subject: [PATCH 08/33] Recognize COMPOSE_TLS_VERSION env var in tls_config_from_options Signed-off-by: Joffrey F --- compose/cli/command.py | 19 +------------------ compose/cli/docker_client.py | 27 ++++++++++++++++++++++++--- tests/unit/cli/command_test.py | 20 -------------------- tests/unit/cli/docker_client_test.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 41 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 4e2722648..ccc76ceb4 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals import logging import os import re -import ssl import six @@ -15,6 +14,7 @@ from ..config.environment import Environment from ..const import API_VERSIONS from ..project import Project from .docker_client import docker_client +from .docker_client import get_tls_version from .docker_client import tls_config_from_options from .utils import get_version_info @@ -60,23 +60,6 @@ def get_config_path_from_options(base_dir, options, environment): return None -def get_tls_version(environment): - compose_tls_version = environment.get('COMPOSE_TLS_VERSION', None) - if not compose_tls_version: - return None - - tls_attr_name = "PROTOCOL_{}".format(compose_tls_version) - if not hasattr(ssl, tls_attr_name): - log.warn( - 'The "{}" protocol is unavailable. You may need to update your ' - 'version of Python or OpenSSL. Falling back to TLSv1 (default).' - .format(compose_tls_version) - ) - return None - - return getattr(ssl, tls_attr_name) - - def get_client(environment, verbose=False, version=None, tls_config=None, host=None, tls_version=None): diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 018d24513..44c7ad91d 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging +import ssl from docker import APIClient from docker.errors import TLSParameterError @@ -16,7 +17,24 @@ from .utils import unquote_path log = logging.getLogger(__name__) -def tls_config_from_options(options): +def get_tls_version(environment): + compose_tls_version = environment.get('COMPOSE_TLS_VERSION', None) + if not compose_tls_version: + return None + + tls_attr_name = "PROTOCOL_{}".format(compose_tls_version) + if not hasattr(ssl, tls_attr_name): + log.warn( + 'The "{}" protocol is unavailable. You may need to update your ' + 'version of Python or OpenSSL. Falling back to TLSv1 (default).' + .format(compose_tls_version) + ) + return None + + return getattr(ssl, tls_attr_name) + + +def tls_config_from_options(options, environment=None): tls = options.get('--tls', False) ca_cert = unquote_path(options.get('--tlscacert')) cert = unquote_path(options.get('--tlscert')) @@ -24,7 +42,9 @@ def tls_config_from_options(options): verify = options.get('--tlsverify') skip_hostname_check = options.get('--skip-hostname-check', False) - advanced_opts = any([ca_cert, cert, key, verify]) + tls_version = get_tls_version(environment or {}) + + advanced_opts = any([ca_cert, cert, key, verify, tls_version]) if tls is True and not advanced_opts: return True @@ -35,7 +55,8 @@ def tls_config_from_options(options): return TLSConfig( client_cert=client_cert, verify=verify, ca_cert=ca_cert, - assert_hostname=False if skip_hostname_check else None + assert_hostname=False if skip_hostname_check else None, + ssl_version=tls_version ) return None diff --git a/tests/unit/cli/command_test.py b/tests/unit/cli/command_test.py index 3655c432e..c64a0401b 100644 --- a/tests/unit/cli/command_test.py +++ b/tests/unit/cli/command_test.py @@ -2,12 +2,10 @@ from __future__ import absolute_import from __future__ import unicode_literals import os -import ssl import pytest from compose.cli.command import get_config_path_from_options -from compose.cli.command import get_tls_version from compose.config.environment import Environment from compose.const import IS_WINDOWS_PLATFORM from tests import mock @@ -57,21 +55,3 @@ class TestGetConfigPathFromOptions(object): def test_no_path(self): environment = Environment.from_env_file('.') assert not get_config_path_from_options('.', {}, environment) - - -class TestGetTlsVersion(object): - def test_get_tls_version_default(self): - environment = {} - assert get_tls_version(environment) is None - - @pytest.mark.skipif(not hasattr(ssl, 'PROTOCOL_TLSv1_2'), reason='TLS v1.2 unsupported') - def test_get_tls_version_upgrade(self): - environment = {'COMPOSE_TLS_VERSION': 'TLSv1_2'} - assert get_tls_version(environment) == ssl.PROTOCOL_TLSv1_2 - - def test_get_tls_version_unavailable(self): - environment = {'COMPOSE_TLS_VERSION': 'TLSv5_5'} - with mock.patch('compose.cli.command.log') as mock_log: - tls_version = get_tls_version(environment) - mock_log.warn.assert_called_once_with(mock.ANY) - assert tls_version is None diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index aaa935afa..482ad9850 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import os import platform +import ssl import docker import pytest @@ -10,6 +11,7 @@ import pytest import compose from compose.cli import errors from compose.cli.docker_client import docker_client +from compose.cli.docker_client import get_tls_version from compose.cli.docker_client import tls_config_from_options from tests import mock from tests import unittest @@ -157,3 +159,29 @@ class TLSConfigTestCase(unittest.TestCase): assert result.cert == (self.client_cert, self.key) assert result.ca_cert == self.ca_cert assert result.verify is True + + def test_tls_simple_with_tls_version(self): + tls_version = 'TLSv1' + options = {'--tls': True} + environment = {'COMPOSE_TLS_VERSION': tls_version} + result = tls_config_from_options(options, environment) + assert isinstance(result, docker.tls.TLSConfig) + assert result.ssl_version == ssl.PROTOCOL_TLSv1 + + +class TestGetTlsVersion(object): + def test_get_tls_version_default(self): + environment = {} + assert get_tls_version(environment) is None + + @pytest.mark.skipif(not hasattr(ssl, 'PROTOCOL_TLSv1_2'), reason='TLS v1.2 unsupported') + def test_get_tls_version_upgrade(self): + environment = {'COMPOSE_TLS_VERSION': 'TLSv1_2'} + assert get_tls_version(environment) == ssl.PROTOCOL_TLSv1_2 + + def test_get_tls_version_unavailable(self): + environment = {'COMPOSE_TLS_VERSION': 'TLSv5_5'} + with mock.patch('compose.cli.docker_client.log') as mock_log: + tls_version = get_tls_version(environment) + mock_log.warn.assert_called_once_with(mock.ANY) + assert tls_version is None From eab1ee0eaf76921ba016e81741ccf57dbc0217b4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Mar 2017 17:04:26 -0700 Subject: [PATCH 09/33] Support 'nocopy' mode for expanded volume syntax Signed-off-by: Joffrey F --- compose/config/config.py | 7 +++++-- tests/acceptance/cli_test.py | 3 ++- tests/fixtures/v3-full/docker-compose.yml | 5 +++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 8cbaae272..72687d756 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1033,8 +1033,11 @@ def resolve_volume_path(working_dir, volume): if isinstance(volume, dict): host_path = volume.get('source') container_path = volume.get('target') - if host_path and volume.get('read_only'): - container_path += ':ro' + if host_path: + if volume.get('read_only'): + container_path += ':ro' + if volume.get('volume', {}).get('nocopy'): + container_path += ':nocopy' else: container_path, host_path = split_path_mapping(volume) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 14e6f7336..bceb102a2 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -374,7 +374,8 @@ class CLITestCase(DockerClientTestCase): 'volumes': [ '/host/path:/container/path:ro', 'foobar:/container/volumepath:rw', - '/anonymous' + '/anonymous', + 'foobar:/container/volumepath2:nocopy' ], 'stop_grace_period': '20s', diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml index 27f3c6e04..2bc0e248d 100644 --- a/tests/fixtures/v3-full/docker-compose.yml +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -44,6 +44,11 @@ services: target: /container/volumepath - type: volume target: /anonymous + - type: volume + source: foobar + target: /container/volumepath2 + volume: + nocopy: true stop_grace_period: 20s volumes: From cd3343440ce3e5a7f91d1e0f76c95a29ebb18031 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 21 Mar 2017 16:28:53 -0700 Subject: [PATCH 10/33] Change docker-py dependency error to a warning, update fix command Signed-off-by: Joffrey F --- compose/cli/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index c5db44558..4061e7091 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -20,16 +20,15 @@ try: list(filter(lambda p: p.startswith(b'docker-py=='), packages)) ) > 0 if dockerpy_installed: - from .colors import red + from .colors import yellow print( - red('ERROR:'), + yellow('WARNING:'), "Dependency conflict: an older version of the 'docker-py' package " - "is polluting the namespace. " - "Run the following command to remedy the issue:\n" - "pip uninstall docker docker-py; pip install docker", + "may be polluting the namespace. " + "If you're experiencing crashes, run the following command to remedy the issue:\n" + "pip uninstall docker-py; pip uninstall docker; pip install docker", file=sys.stderr ) - sys.exit(1) except OSError: # pip command is not available, which indicates it's probably the binary From 962330120dd9294c663ca1a80f263d34c71e58dc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 23 Mar 2017 16:43:15 -0700 Subject: [PATCH 11/33] Ignore unicode error in subprocess call Signed-off-by: Joffrey F --- compose/cli/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index 4061e7091..1fe9aab8d 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -34,3 +34,9 @@ except OSError: # pip command is not available, which indicates it's probably the binary # distribution of Compose which is not affected pass +except UnicodeDecodeError: + # ref: https://github.com/docker/compose/issues/4663 + # This could be caused by a number of things, but it seems to be a + # python 2 + MacOS interaction. It's not ideal to ignore this, but at least + # it doesn't make the program unusable. + pass From 0e5acfa16caa66e7eff6042e2473b2fdca6870af Mon Sep 17 00:00:00 2001 From: King Chung Huang Date: Fri, 24 Mar 2017 13:49:54 -0600 Subject: [PATCH 12/33] Merge deploy key in service dicts Update merge_service_dicts() to merge deploy mappings. Compose file version 3 added the deploy key to service dicts to specify configs related to Docker services. Signed-off-by: King Chung Huang --- compose/config/config.py | 2 ++ tests/unit/config/config_test.py | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index 72687d756..3292845f5 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -879,6 +879,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('depends_on', parse_depends_on) md.merge_sequence('links', ServiceLink.parse) md.merge_sequence('secrets', types.ServiceSecret.parse) + md.merge_mapping('deploy', parse_deploy) for field in ['volumes', 'devices']: md.merge_field(field, merge_path_mappings) @@ -1003,6 +1004,7 @@ parse_sysctls = functools.partial(parse_dict_or_list, split_kv, 'sysctls') parse_depends_on = functools.partial( parse_dict_or_list, lambda k: (k, {'condition': 'service_started'}), 'depends_on' ) +parse_deploy = functools.partial(parse_dict_or_list, split_kv, 'deploy') def parse_ulimits(ulimits): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 4db87ecb6..88a475801 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1950,6 +1950,57 @@ class ConfigTest(unittest.TestCase): actual = config.merge_service_dicts(base, override, V3_1) assert actual['secrets'] == override['secrets'] + def test_merge_deploy(self): + base = { + 'image': 'busybox', + } + override = { + 'deploy': { + 'mode': 'global', + 'restart_policy': { + 'condition': 'on-failure' + } + } + } + actual = config.merge_service_dicts(base, override, V3_0) + assert actual['deploy'] == override['deploy'] + + def test_merge_deploy_override(self): + base = { + 'image': 'busybox', + 'deploy': { + 'mode': 'global', + 'restart_policy': { + 'condition': 'on-failure' + }, + 'placement': { + 'constraints': [ + 'node.role == manager' + ] + } + } + } + override = { + 'deploy': { + 'mode': 'replicated', + 'restart_policy': { + 'condition': 'any' + } + } + } + actual = config.merge_service_dicts(base, override, V3_0) + assert actual['deploy'] == { + 'mode': 'replicated', + 'restart_policy': { + 'condition': 'any' + }, + 'placement': { + 'constraints': [ + 'node.role == manager' + ] + } + } + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From 48831a8d5fd5b76875d4cde909125cc1945d5317 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 21 Mar 2017 12:39:32 -0700 Subject: [PATCH 13/33] Bump docker SDK dependency Update invalid ports test Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- tests/unit/config/config_test.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 53b9294ce..f8061af83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 colorama==0.3.7 -docker==2.1.0 +docker==2.2.1 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/setup.py b/setup.py index 13fe59b22..19a0d4aa0 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.1.0, < 3.0', + 'docker >= 2.2.1, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 195efe3b9..bf4567601 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2249,7 +2249,8 @@ class PortsTest(unittest.TestCase): ] INVALID_PORT_MAPPINGS = [ - ["8000-8001:8000"], + ["8000-8004:8000-8002"], + ["4242:4242-4244"], ] VALID_SINGLE_PORTS = [ From f6fc8b582224fea3823e439d0cf9b6f3435ae680 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Thu, 30 Mar 2017 21:32:33 +0200 Subject: [PATCH 14/33] Add zsh completion for 'docker-compose images' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 66d924f73..b40d440f9 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -3,11 +3,6 @@ # Description # ----------- # zsh completion for docker-compose -# https://github.com/sdurrheimer/docker-compose-zsh-completion -# ------------------------------------------------------------------------- -# Version -# ------- -# 1.5.0 # ------------------------------------------------------------------------- # Authors # ------- @@ -253,6 +248,12 @@ __docker-compose_subcommand() { (help) _arguments ':subcommand:__docker-compose_commands' && ret=0 ;; + (images) + _arguments \ + $opts_help \ + '-q[Only display IDs]' \ + '*:services:__docker-compose_services_all' && ret=0 + ;; (kill) _arguments \ $opts_help \ From 9fc3cc9c3c0bb4cfa85548fed5c98690d741f48b Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Thu, 30 Mar 2017 21:38:14 +0200 Subject: [PATCH 15/33] Add zsh completion for 'docker-compose run -v --volume' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 66d924f73..2a3792fef 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -309,16 +309,17 @@ __docker-compose_subcommand() { (run) _arguments \ $opts_help \ + $opts_no_deps \ '-d[Detached mode: Run container in the background, print new container name.]' \ '*-e[KEY=VAL Set an environment variable (can be used multiple times)]:environment variable KEY=VAL: ' \ '--entrypoint[Overwrite the entrypoint of the image.]:entry point: ' \ '--name=[Assign a name to the container]:name: ' \ - $opts_no_deps \ '(-p --publish)'{-p,--publish=}"[Publish a container's port(s) to the host]" \ '--rm[Remove container after run. Ignored in detached mode.]' \ "--service-ports[Run command with the service's ports enabled and mapped to the host.]" \ '-T[Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY.]' \ '(-u --user)'{-u,--user=}'[Run as specified username or uid]:username or uid:_users' \ + '(-v --volume)*'{-v,--volume=}'[Bind mount a volume]:volume: ' \ '(-w --workdir)'{-w,--workdir=}'[Working directory inside the container]:workdir: ' \ '(-):services:__docker-compose_services' \ '(-):command: _command_names -e' \ From 29d23c27939a5347c3be3e11d9f8c2b679032b4f Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Thu, 30 Mar 2017 22:05:36 +0200 Subject: [PATCH 16/33] Add zsh completion for 'docker-compose build --build-arg' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 66d924f73..50ba589d8 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -199,6 +199,7 @@ __docker-compose_subcommand() { (build) _arguments \ $opts_help \ + "*--build-arg=[Set build-time variables for one service.]:=: " \ '--force-rm[Always remove intermediate containers.]' \ '--no-cache[Do not use cache when building the image.]' \ '--pull[Always attempt to pull a newer version of the image.]' \ From be8fd5810e9d0557bc8f2d03241ea35d793a5466 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Thu, 30 Mar 2017 22:10:20 +0200 Subject: [PATCH 17/33] Add zsh completion for 'docker-compose config --resolve-image-digests --volumes' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 66d924f73..b6f34897a 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -214,7 +214,9 @@ __docker-compose_subcommand() { _arguments \ $opts_help \ '(--quiet -q)'{--quiet,-q}"[Only validate the configuration, don't print anything.]" \ - '--services[Print the service names, one per line.]' && ret=0 + '--resolve-image-digests[Pin image tags to digests.]' \ + '--services[Print the service names, one per line.]' \ + '--volumes[Print the volume names, one per line.]' && ret=0 ;; (create) _arguments \ From 65b919cf08d1c760fe8c4a6aa7057377667183e8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 3 Apr 2017 11:54:12 -0700 Subject: [PATCH 18/33] Add 3.2 schema to docker-compose.spec Signed-off-by: Joffrey F --- compose/config/errors.py | 4 ++-- docker-compose.spec | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/compose/config/errors.py b/compose/config/errors.py index 0f78d4a94..9b82df0ab 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -4,8 +4,8 @@ from __future__ import unicode_literals VERSION_EXPLANATION = ( 'You might be seeing this error because you\'re using the wrong Compose file version. ' - 'Either specify a supported version ("2.0", "2.1", "3.0", "3.1") and place your ' - 'service definitions under the `services` key, or omit the `version` key ' + 'Either specify a supported version ("2.0", "2.1", "3.0", "3.1", "3.2") and place ' + 'your service definitions under the `services` key, or omit the `version` key ' 'and place your service definitions at the root of the file to use ' 'version 1.\nFor more on the Compose file format versions, see ' 'https://docs.docker.com/compose/compose-file/') diff --git a/docker-compose.spec b/docker-compose.spec index ef0e2593e..f4280dd42 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -42,6 +42,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.1.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.2.json', + 'compose/config/config_schema_v3.2.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', From e7e159076b3ffd6d624467b7d6d999701fef662d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 3 Apr 2017 14:15:36 -0700 Subject: [PATCH 19/33] Prevent pip version checks when calling `pip freeze` Signed-off-by: Joffrey F --- compose/cli/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index 1fe9aab8d..379059c1a 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import os import subprocess import sys @@ -12,8 +13,12 @@ try: # https://github.com/docker/compose/issues/4425 # https://github.com/docker/compose/issues/4481 # https://github.com/pypa/pip/blob/master/pip/_vendor/__init__.py + env = os.environ.copy() + env[str('PIP_DISABLE_PIP_VERSION_CHECK')] = str('1') + s_cmd = subprocess.Popen( - ['pip', 'freeze'], stderr=subprocess.PIPE, stdout=subprocess.PIPE + ['pip', 'freeze'], stderr=subprocess.PIPE, stdout=subprocess.PIPE, + env=env ) packages = s_cmd.communicate()[0].splitlines() dockerpy_installed = len( From dd862b28c035cf5a165575241f2607c5f7dc0abc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 5 Apr 2017 13:57:56 -0700 Subject: [PATCH 20/33] 1.13.0dev Signed-off-by: Joffrey F --- CHANGELOG.md | 126 ++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c07e9ca2..8d49ddea7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,132 @@ Change log ========== +1.12.0 (2017-04-04) +------------------- + +### New features + +#### Compose file version 3.2 + +- Introduced version 3.2 of the `docker-compose.yml` specification. + +- Added support for `cache_from` in the `build` section of services + +- Added support for the new expanded ports syntax in service definitions + +- Added support for the new expanded volumes syntax in service definitions + +#### Compose file version 2.1 + +- Added support for `pids_limit` in service definitions + +#### Compose file version 2.0 and up + +- Added `--volumes` option to `docker-compose config` that lists named + volumes declared for that project + +- Added support for `mem_reservation` in service definitions (2.x only) + +- Added support for `dns_opt` in service definitions (2.x only) + +#### All formats + +- Added a new `docker-compose images` command that lists images used by + the current project's containers + +- Added a `--stop` (shorthand `-s`) option to `docker-compose rm` that stops + the running containers before removing them + +- Added a `--resolve-image-digests` option to `docker-compose config` that + pins the image version for each service to a permanent digest + +- Added a `--exit-code-from SERVICE` option to `docker-compose up`. When + used, `docker-compose` will exit on any container's exit with the code + corresponding to the specified service's exit code + +- Added a `--parallel` option to `docker-compose pull` that enables images + for multiple services to be pulled simultaneously + +- Added a `--build-arg` option to `docker-compose build` + +- Added a `--volume ` (shorthand `-v`) option to + `docker-compose run` to declare runtime volumes to be mounted + +- Added a `--project-directory PATH` option to `docker-compose` that will + affect path resolution for the project + +- When using `--abort-on-container-exit` in `docker-compose up`, the exit + code for the container that caused the abort will be the exit code of + the `docker-compose up` command + +- Users can now configure which path separator character they want to use + to separate the `COMPOSE_FILE` environment value using the + `COMPOSE_PATH_SEPARATOR` environment variable + +- Added support for port range to single port in port mappings + (e.g. `8000-8010:80`) + +### Bugfixes + +- `docker-compose run --rm` now removes anonymous volumes after execution, + matching the behavior of `docker run --rm`. + +- Fixed a bug where override files containing port lists would cause a + TypeError to be raised + +- Fixed a bug where the `deploy` key would be missing from the output of + `docker-compose config` + +- Fixed a bug where scaling services up or down would sometimes re-use + obsolete containers + +- Fixed a bug where the output of `docker-compose config` would be invalid + if the project declared anonymous volumes + +- Variable interpolation now properly occurs in the `secrets` section of + the Compose file + +- The `secrets` section now properly appears in the output of + `docker-compose config` + +- Fixed a bug where changes to some networks properties would not be + detected against previously created networks + +- Fixed a bug where `docker-compose` would crash when trying to write into + a closed pipe + +- Fixed an issue where Compose would not pick up on the value of + COMPOSE_TLS_VERSION when used in combination with command-line TLS flags + +1.11.2 (2017-02-17) +------------------- + +### Bugfixes + +- Fixed a bug that was preventing secrets configuration from being + loaded properly + +- Fixed a bug where the `docker-compose config` command would fail + if the config file contained secrets definitions + +- Fixed an issue where Compose on some linux distributions would + pick up and load an outdated version of the requests library + +- Fixed an issue where socket-type files inside a build folder + would cause `docker-compose` to crash when trying to build that + service + +- Fixed an issue where recursive wildcard patterns `**` were not being + recognized in `.dockerignore` files. + +1.11.1 (2017-02-09) +------------------- + +### Bugfixes + +- Fixed a bug where the 3.1 file format was not being recognized as valid + by the Compose parser + 1.11.0 (2017-02-08) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index b2ca86f86..d387af24e 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.12.0dev' +__version__ = '1.13.0dev' From 3b02426236787032185feafb507da480e6d2df62 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 7 Apr 2017 12:19:37 -0700 Subject: [PATCH 21/33] Fix Config.find for Compose files in nested folders Signed-off-by: Joffrey F --- compose/config/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 3292845f5..0c514763a 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -234,10 +234,10 @@ class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name conf config) -def find(base_dir, filenames, environment, override_dir='.'): +def find(base_dir, filenames, environment, override_dir=None): if filenames == ['-']: return ConfigDetails( - os.path.abspath(override_dir), + os.path.abspath(override_dir) if override_dir else os.getcwd(), [ConfigFile(None, yaml.safe_load(sys.stdin))], environment ) @@ -249,7 +249,7 @@ def find(base_dir, filenames, environment, override_dir='.'): log.debug("Using configuration files: {}".format(",".join(filenames))) return ConfigDetails( - override_dir or os.path.dirname(filenames[0]), + override_dir if override_dir else os.path.dirname(filenames[0]), [ConfigFile.from_filename(f) for f in filenames], environment ) From 72a2ea9d8605b6b1b204c2347e3a4ed687d63730 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 7 Apr 2017 16:52:46 -0700 Subject: [PATCH 22/33] Fix serializer bug (python 3) Signed-off-by: Joffrey F --- compose/config/serialize.py | 8 ++++---- tests/unit/config/config_test.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 5b36124d0..e364117e7 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -111,9 +111,9 @@ def denormalize_service_dict(service_dict, version, image_digest=None): ) if 'ports' in service_dict and version not in (V3_2,): - service_dict['ports'] = map( - lambda p: p.legacy_repr() if isinstance(p, types.ServicePort) else p, - service_dict['ports'] - ) + service_dict['ports'] = [ + p.legacy_repr() if isinstance(p, types.ServicePort) else p + for p in service_dict['ports'] + ] return service_dict diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index b7e4cc9bf..6bf4986ff 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3837,3 +3837,15 @@ class SerializeTest(unittest.TestCase): serialized_service = serialized_config['services']['web'] assert secret_sort(serialized_service['secrets']) == secret_sort(service_dict['secrets']) assert 'secrets' in serialized_config + + def test_serialize_ports(self): + config_dict = config.Config(version='2.0', services=[ + { + 'ports': [types.ServicePort('80', '8080', None, None, None)], + 'image': 'alpine', + 'name': 'web' + } + ], volumes={}, networks={}, secrets={}) + + serialized_config = yaml.load(serialize_config(config_dict)) + assert '8080:80/tcp' in serialized_config['services']['web']['ports'] From 5a4293848cb970dac1ca17074a3bd14b2eb2268b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 7 Apr 2017 19:17:11 -0700 Subject: [PATCH 23/33] Add 2.2 schema for API 1.25 features support Signed-off-by: Joffrey F --- compose/config/config_schema_v2.2.json | 386 +++++++++++++++++++++++++ compose/config/serialize.py | 3 +- compose/const.py | 3 + docker-compose.spec | 5 + 4 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 compose/config/config_schema_v2.2.json diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json new file mode 100644 index 000000000..daa07d149 --- /dev/null +++ b/compose/config/config_schema_v2.2.json @@ -0,0 +1,386 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v2.2.json", + "type": "object", + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "cpu_shares": {"type": ["number", "string"]}, + "cpu_quota": {"type": ["number", "string"]}, + "cpuset": {"type": "string"}, + "depends_on": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "condition": { + "type": "string", + "enum": ["service_started", "service_healthy"] + } + }, + "required": ["condition"] + } + } + } + ] + }, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns_opt": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "extends": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + + "properties": { + "service": {"type": "string"}, + "file": {"type": "string"} + }, + "required": ["service"], + "additionalProperties": false + } + ] + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "init": {"type": ["boolean", "string"]}, + "ipc": {"type": "string"}, + "isolation": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": {"type": "object"} + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "mem_limit": {"type": ["number", "string"]}, + "mem_reservation": {"type": ["string", "integer"]}, + "mem_swappiness": {"type": "integer"}, + "memswap_limit": {"type": ["number", "string"]}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"}, + "link_local_ips": {"$ref": "#/definitions/list_of_strings"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, + "group_add": { + "type": "array", + "items": { + "type": ["string", "number"] + }, + "uniqueItems": true + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "pids_limit": {"type": ["number", "string"]}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "volume_driver": {"type": "string"}, + "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} + }, + + "dependencies": { + "memswap_limit": ["mem_limit"] + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + + "network": { + "id": "#/definitions/network", + "type": "object", + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array" + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "enable_ipv6": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 5b36124d0..979de4bf7 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -7,6 +7,7 @@ import yaml from compose.config import types from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 +from compose.const import COMPOSEFILE_V2_1 as V2_2 from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import COMPOSEFILE_V3_1 as V3_2 @@ -95,7 +96,7 @@ def denormalize_service_dict(service_dict, version, image_digest=None): if version == V1 and 'network_mode' not in service_dict: service_dict['network_mode'] = 'bridge' - if 'depends_on' in service_dict and version != V2_1: + if 'depends_on' in service_dict and version not in (V2_1, V2_2): service_dict['depends_on'] = sorted([ svc for svc in service_dict['depends_on'].keys() ]) diff --git a/compose/const.py b/compose/const.py index 8de693445..573136d5d 100644 --- a/compose/const.py +++ b/compose/const.py @@ -21,6 +21,7 @@ SECRETS_PATH = '/run/secrets' COMPOSEFILE_V1 = '1' COMPOSEFILE_V2_0 = '2.0' COMPOSEFILE_V2_1 = '2.1' +COMPOSEFILE_V2_2 = '2.2' COMPOSEFILE_V3_0 = '3.0' COMPOSEFILE_V3_1 = '3.1' @@ -30,6 +31,7 @@ API_VERSIONS = { COMPOSEFILE_V1: '1.21', COMPOSEFILE_V2_0: '1.22', COMPOSEFILE_V2_1: '1.24', + COMPOSEFILE_V2_2: '1.25', COMPOSEFILE_V3_0: '1.25', COMPOSEFILE_V3_1: '1.25', COMPOSEFILE_V3_2: '1.25', @@ -39,6 +41,7 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V1]: '1.9.0', API_VERSIONS[COMPOSEFILE_V2_0]: '1.10.0', API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0', + API_VERSIONS[COMPOSEFILE_V2_2]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0', diff --git a/docker-compose.spec b/docker-compose.spec index f4280dd42..21b3c1742 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -32,6 +32,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v2.1.json', 'DATA' ), + ( + 'compose/config/config_schema_v2.2.json', + 'compose/config/config_schema_v2.2.json', + 'DATA' + ), ( 'compose/config/config_schema_v3.0.json', 'compose/config/config_schema_v3.0.json', From cc966a7e19f288fb1f91e61ec76670e72b885d4d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 7 Apr 2017 20:01:33 -0700 Subject: [PATCH 24/33] Add support for init and init_path Signed-off-by: Joffrey F --- compose/config/config.py | 1 + compose/service.py | 24 +++++++++++++----------- tests/integration/service_test.py | 16 ++++++++++++++++ tests/integration/testcases.py | 6 +++--- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 3292845f5..e8fbe7083 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -108,6 +108,7 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'log_opt', 'logging', 'network_mode', + 'init', ] DOCKER_VALID_URL_PREFIXES = ( diff --git a/compose/service.py b/compose/service.py index b9f77beb9..f8c549381 100644 --- a/compose/service.py +++ b/compose/service.py @@ -48,7 +48,7 @@ from .utils import parse_seconds_float log = logging.getLogger(__name__) -DOCKER_START_KEYS = [ +HOST_CONFIG_KEYS = [ 'cap_add', 'cap_drop', 'cgroup_parent', @@ -60,6 +60,7 @@ DOCKER_START_KEYS = [ 'env_file', 'extra_hosts', 'group_add', + 'init', 'ipc', 'read_only', 'log_driver', @@ -729,8 +730,8 @@ class Service(object): number, self.config_hash if add_config_hash else None) - # Delete options which are only used when starting - for key in DOCKER_START_KEYS: + # Delete options which are only used in HostConfig + for key in HOST_CONFIG_KEYS: container_options.pop(key, None) container_options['host_config'] = self._get_container_host_config( @@ -750,8 +751,12 @@ class Service(object): logging_dict = options.get('logging', None) log_config = get_log_config(logging_dict) + init_path = None + if isinstance(options.get('init'), six.string_types): + init_path = options.get('init') + options['init'] = True - host_config = self.client.create_host_config( + return self.client.create_host_config( links=self._get_links(link_to_self=one_off), port_bindings=build_port_bindings( formatted_ports(options.get('ports', [])) @@ -786,15 +791,12 @@ class Service(object): oom_score_adj=options.get('oom_score_adj'), mem_swappiness=options.get('mem_swappiness'), group_add=options.get('group_add'), - userns_mode=options.get('userns_mode') + userns_mode=options.get('userns_mode'), + init=options.get('init', None), + init_path=init_path, + isolation=options.get('isolation'), ) - # TODO: Add as an argument to create_host_config once it's supported - # in docker-py - host_config['Isolation'] = options.get('isolation') - - return host_config - def get_secret_volumes(self): def build_spec(secret): target = '{}/{}'.format( diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 12ec8a993..636071755 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import os import shutil import tempfile +from distutils.spawn import find_executable from os import path import pytest @@ -115,6 +116,21 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(container.get('HostConfig.ShmSize'), 67108864) + def test_create_container_with_init_bool(self): + self.require_api_version('1.25') + service = self.create_service('db', init=True) + container = service.create_container() + service.start_container(container) + assert container.get('HostConfig.Init') is True + + def test_create_container_with_init_path(self): + self.require_api_version('1.25') + docker_init_path = find_executable('docker-init') + service = self.create_service('db', init=docker_init_path) + container = service.create_container() + service.start_container(container) + assert container.get('HostConfig.InitPath') == docker_init_path + @pytest.mark.xfail(True, reason='Some kernels/configs do not support pids_limit') def test_create_container_with_pids_limit(self): self.require_api_version('1.23') diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 38fdcc660..a5fe999d9 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -15,7 +15,7 @@ from compose.const import API_VERSIONS from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_0 as V2_1 -from compose.const import COMPOSEFILE_V3_0 as V3_0 +from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service @@ -37,7 +37,7 @@ def get_links(container): def engine_max_version(): if 'DOCKER_VERSION' not in os.environ: - return V3_0 + return V3_2 version = os.environ['DOCKER_VERSION'].partition('-')[0] if version_lt(version, '1.10'): return V1 @@ -45,7 +45,7 @@ def engine_max_version(): return V2_0 if version_lt(version, '1.13'): return V2_1 - return V3_0 + return V3_2 def build_version_required_decorator(ignored_versions): From 843a0d82a0a0664ebb9cff8a221bde81f6fe8eb6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 7 Apr 2017 19:02:39 -0700 Subject: [PATCH 25/33] Add support for IPAM options in v2 format Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 7 ++++++ compose/config/config_schema_v2.1.json | 7 ++++++ compose/network.py | 7 ++++++ tests/integration/project_test.py | 35 ++++++++++++++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 54e9b3314..f3688685b 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -253,6 +253,13 @@ "driver": {"type": "string"}, "config": { "type": "array" + }, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": "string"} + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 0f87be24e..aa59d181e 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -298,6 +298,13 @@ "driver": {"type": "string"}, "config": { "type": "array" + }, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": "string"} + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/compose/network.py b/compose/network.py index 053fdacd8..4aeff2d1e 100644 --- a/compose/network.py +++ b/compose/network.py @@ -123,6 +123,7 @@ def create_ipam_config_from_dict(ipam_dict): ) for config in ipam_dict.get('config', []) ], + options=ipam_dict.get('options') ) @@ -157,6 +158,12 @@ def check_remote_ipam_config(remote, local): if sorted(lc.get('AuxiliaryAddresses')) != sorted(rc.get('AuxiliaryAddresses')): raise NetworkConfigChangedError(local.full_name, 'IPAM config aux_addresses') + remote_opts = remote_ipam.get('Options', {}) + local_opts = local.ipam.get('options', {}) + for k in set.union(set(remote_opts.keys()), set(local_opts.keys())): + if remote_opts.get(k) != local_opts.get(k): + raise NetworkConfigChangedError(local.full_name, 'IPAM option "{}"'.format(k)) + def check_remote_network_config(remote, local): if local.driver and remote.get('Driver') != local.driver: diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 455189851..2eae88ec5 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -681,6 +681,41 @@ class ProjectTest(DockerClientTestCase): }], } + @v2_only() + def test_up_with_ipam_options(self): + config_data = build_config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'networks': {'front': None}, + }], + networks={ + 'front': { + 'driver': 'bridge', + 'ipam': { + 'driver': 'default', + 'options': { + "com.docker.compose.network.test": "9-29-045" + } + }, + }, + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up() + + network = self.client.networks(names=['composetest_front'])[0] + + assert network['IPAM']['Options'] == { + "com.docker.compose.network.test": "9-29-045" + } + @v2_only() def test_up_with_network_static_addresses(self): config_data = build_config( From 0f00aa409861f4f7820a21055d25ec1f312db7d0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 10 Apr 2017 17:32:10 -0700 Subject: [PATCH 26/33] Convert paths to unicode in get_config_path_from_options if needed Signed-off-by: Joffrey F --- compose/cli/command.py | 7 +++++-- tests/unit/cli/command_test.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index ccc76ceb4..e1ae690c0 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -49,14 +49,17 @@ def get_config_from_options(base_dir, options): def get_config_path_from_options(base_dir, options, environment): + def unicode_paths(paths): + return [p.decode('utf-8') if isinstance(p, six.binary_type) else p for p in paths] + file_option = options.get('--file') if file_option: - return file_option + return unicode_paths(file_option) config_files = environment.get('COMPOSE_FILE') if config_files: pathsep = environment.get('COMPOSE_PATH_SEPARATOR', os.pathsep) - return config_files.split(pathsep) + return unicode_paths(config_files.split(pathsep)) return None diff --git a/tests/unit/cli/command_test.py b/tests/unit/cli/command_test.py index c64a0401b..3a9844c4f 100644 --- a/tests/unit/cli/command_test.py +++ b/tests/unit/cli/command_test.py @@ -1,9 +1,11 @@ +# ~*~ encoding: utf-8 ~*~ from __future__ import absolute_import from __future__ import unicode_literals import os import pytest +import six from compose.cli.command import get_config_path_from_options from compose.config.environment import Environment @@ -55,3 +57,20 @@ class TestGetConfigPathFromOptions(object): def test_no_path(self): environment = Environment.from_env_file('.') assert not get_config_path_from_options('.', {}, environment) + + def test_unicode_path_from_options(self): + paths = [b'\xe5\xb0\xb1\xe5\x90\x83\xe9\xa5\xad/docker-compose.yml'] + opts = {'--file': paths} + environment = Environment.from_env_file('.') + assert get_config_path_from_options( + '.', opts, environment + ) == ['就吃饭/docker-compose.yml'] + + @pytest.mark.skipif(six.PY3, reason='Env values in Python 3 are already Unicode') + def test_unicode_path_from_env(self): + with mock.patch.dict(os.environ): + os.environ['COMPOSE_FILE'] = b'\xe5\xb0\xb1\xe5\x90\x83\xe9\xa5\xad/docker-compose.yml' + environment = Environment.from_env_file('.') + assert get_config_path_from_options( + '.', {}, environment + ) == ['就吃饭/docker-compose.yml'] From 1891b2b78ce977512885b289fe310221ce88efae Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 11 Apr 2017 14:45:14 -0700 Subject: [PATCH 27/33] Fix ServicePort.legacy_repr bug for `ext_ip::target` notation Signed-off-by: Joffrey F --- compose/config/types.py | 4 ++-- tests/unit/config/types_test.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index 96846b5ba..dd61a8796 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -267,7 +267,7 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext @classmethod def parse(cls, spec): if isinstance(spec, cls): - # WHen extending a service with ports, the port definitions have already been parsed + # When extending a service with ports, the port definitions have already been parsed return [spec] if not isinstance(spec, dict): @@ -316,7 +316,7 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext def normalize_port_dict(port): return '{external_ip}{has_ext_ip}{published}{is_pub}{target}/{protocol}'.format( published=port.get('published', ''), - is_pub=(':' if port.get('published') else ''), + is_pub=(':' if port.get('published') or port.get('external_ip') else ''), target=port.get('target'), protocol=port.get('protocol', 'tcp'), external_ip=port.get('external_ip', ''), diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 66588d629..83d6270d2 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -71,6 +71,16 @@ class TestServicePort(object): } assert ports[0].legacy_repr() == port_def + def test_parse_ext_ip_no_published_port(self): + port_def = '1.1.1.1::3000' + ports = ServicePort.parse(port_def) + assert len(ports) == 1 + assert ports[0].legacy_repr() == port_def + '/tcp' + assert ports[0].repr() == { + 'target': '3000', + 'external_ip': '1.1.1.1', + } + def test_parse_port_range(self): ports = ServicePort.parse('25000-25001:4000-4001') assert len(ports) == 2 From c817dedef77f1741a3e9361b525184e26f5ec4d9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 13 Apr 2017 16:52:40 -0700 Subject: [PATCH 28/33] Repair bad imports Signed-off-by: Joffrey F --- compose/config/serialize.py | 4 ++-- tests/acceptance/cli_test.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index caf4f1fbe..aaaf05399 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -7,9 +7,9 @@ import yaml from compose.config import types from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 -from compose.const import COMPOSEFILE_V2_1 as V2_2 +from compose.const import COMPOSEFILE_V2_2 as V2_2 from compose.const import COMPOSEFILE_V3_1 as V3_1 -from compose.const import COMPOSEFILE_V3_1 as V3_2 +from compose.const import COMPOSEFILE_V3_2 as V3_2 def serialize_config_type(dumper, data): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index bceb102a2..bfc963402 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -323,6 +323,7 @@ class CLITestCase(DockerClientTestCase): assert yaml.load(result.stdout) == { 'version': '3.2', 'networks': {}, + 'secrets': {}, 'volumes': { 'foobar': { 'labels': { From 1bd9083de670f8402d5a066a1c84025e81bcdf76 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 14 Apr 2017 15:30:40 -0700 Subject: [PATCH 29/33] Do not wait for exec output when using detached mode Signed-off-by: Joffrey F --- compose/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 84cae9f53..53ff84f45 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -439,7 +439,7 @@ class TopLevelCommand(object): exec_id = container.create_exec(command, **create_exec_options) if detach: - container.start_exec(exec_id, tty=tty) + container.start_exec(exec_id, tty=tty, stream=True) return signals.set_signal_handler_to_shutdown() From c8a7891cc8a635367d595b844496f9807c1f610c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 17 Apr 2017 19:03:56 -0700 Subject: [PATCH 30/33] Implement --scale option on up command, allow scale config in v2.2 format docker-compose scale modified to reuse code between up and scale Signed-off-by: Joffrey F --- compose/cli/main.py | 36 +++-- compose/config/config_schema_v2.2.json | 1 + compose/parallel.py | 4 - compose/project.py | 20 ++- compose/service.py | 196 +++++++++++++----------- tests/acceptance/cli_test.py | 27 ++++ tests/fixtures/scale/docker-compose.yml | 9 ++ tests/integration/project_test.py | 28 ++++ 8 files changed, 213 insertions(+), 108 deletions(-) create mode 100644 tests/fixtures/scale/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 53ff84f45..e018a0174 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -771,15 +771,7 @@ class TopLevelCommand(object): """ timeout = timeout_from_opts(options) - for s in options['SERVICE=NUM']: - if '=' not in s: - raise UserError('Arguments to scale should be in the form service=num') - service_name, num = s.split('=', 1) - try: - num = int(num) - except ValueError: - raise UserError('Number of containers for service "%s" is not a ' - 'number' % service_name) + for service_name, num in parse_scale_args(options['SERVICE=NUM']).items(): self.project.get_service(service_name).scale(num, timeout=timeout) def start(self, options): @@ -875,7 +867,7 @@ class TopLevelCommand(object): If you want to force Compose to stop and recreate all containers, use the `--force-recreate` flag. - Usage: up [options] [SERVICE...] + Usage: up [options] [--scale SERVICE=NUM...] [SERVICE...] Options: -d Detached mode: Run containers in the background, @@ -898,7 +890,9 @@ class TopLevelCommand(object): --remove-orphans Remove containers for services not defined in the Compose file --exit-code-from SERVICE Return the exit code of the selected service container. - Requires --abort-on-container-exit. + Implies --abort-on-container-exit. + --scale SERVICE=NUM Scale SERVICE to NUM instances. Overrides the `scale` + setting in the Compose file if present. """ start_deps = not options['--no-deps'] exit_value_from = exitval_from_opts(options, self.project) @@ -919,7 +913,9 @@ class TopLevelCommand(object): do_build=build_action_from_opts(options), timeout=timeout, detached=detached, - remove_orphans=remove_orphans) + remove_orphans=remove_orphans, + scale_override=parse_scale_args(options['--scale']), + ) if detached: return @@ -1238,3 +1234,19 @@ def call_docker(args): log.debug(" ".join(map(pipes.quote, args))) return subprocess.call(args) + + +def parse_scale_args(options): + res = {} + for s in options: + if '=' not in s: + raise UserError('Arguments to scale should be in the form service=num') + service_name, num = s.split('=', 1) + try: + num = int(num) + except ValueError: + raise UserError( + 'Number of containers for service "%s" is not a number' % service_name + ) + res[service_name] = num + return res diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index daa07d149..390d3efa9 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -222,6 +222,7 @@ "privileged": {"type": "boolean"}, "read_only": {"type": "boolean"}, "restart": {"type": "string"}, + "scale": {"type": "integer"}, "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "shm_size": {"type": ["number", "string"]}, "sysctls": {"$ref": "#/definitions/list_or_dict"}, diff --git a/compose/parallel.py b/compose/parallel.py index fde723f33..34fef71db 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -260,10 +260,6 @@ def parallel_remove(containers, options): parallel_operation(stopped_containers, 'remove', options, 'Removing') -def parallel_start(containers, options): - parallel_operation(containers, 'start', options, 'Starting') - - def parallel_pause(containers, options): parallel_operation(containers, 'pause', options, 'Pausing') diff --git a/compose/project.py b/compose/project.py index a75d71efc..853228764 100644 --- a/compose/project.py +++ b/compose/project.py @@ -380,13 +380,17 @@ class Project(object): do_build=BuildAction.none, timeout=None, detached=False, - remove_orphans=False): + remove_orphans=False, + scale_override=None): warn_for_swarm_mode(self.client) self.initialize() self.find_orphan_containers(remove_orphans) + if scale_override is None: + scale_override = {} + services = self.get_services_without_duplicate( service_names, include_deps=start_deps) @@ -399,7 +403,8 @@ class Project(object): return service.execute_convergence_plan( plans[service.name], timeout=timeout, - detached=detached + detached=detached, + scale_override=scale_override.get(service.name) ) def get_deps(service): @@ -589,10 +594,13 @@ def get_secrets(service, service_secrets, secret_defs): continue if secret.uid or secret.gid or secret.mode: - log.warn("Service \"{service}\" uses secret \"{secret}\" with uid, " - "gid, or mode. These fields are not supported by this " - "implementation of the Compose file".format( - service=service, secret=secret.source)) + log.warn( + "Service \"{service}\" uses secret \"{secret}\" with uid, " + "gid, or mode. These fields are not supported by this " + "implementation of the Compose file".format( + service=service, secret=secret.source + ) + ) secrets.append({'secret': secret, 'file': secret_def.get('file')}) diff --git a/compose/service.py b/compose/service.py index f8c549381..4fa3aeabf 100644 --- a/compose/service.py +++ b/compose/service.py @@ -38,7 +38,6 @@ from .errors import HealthCheckFailed from .errors import NoHealthCheckConfigured from .errors import OperationFailedError from .parallel import parallel_execute -from .parallel import parallel_start from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash @@ -148,6 +147,7 @@ class Service(object): network_mode=None, networks=None, secrets=None, + scale=None, **options ): self.name = name @@ -159,6 +159,7 @@ class Service(object): self.network_mode = network_mode or NetworkMode(None) self.networks = networks or {} self.secrets = secrets or [] + self.scale_num = scale or 1 self.options = options def __repr__(self): @@ -189,16 +190,7 @@ class Service(object): self.start_container_if_stopped(c, **options) return containers - def scale(self, desired_num, timeout=None): - """ - Adjusts the number of containers to the specified number and ensures - they are running. - - - creates containers until there are at least `desired_num` - - stops containers until there are at most `desired_num` running - - starts containers until there are at least `desired_num` running - - removes all stopped containers - """ + def show_scale_warnings(self, desired_num): if self.custom_container_name and desired_num > 1: log.warn('The "%s" service is using the custom container name "%s". ' 'Docker requires each container to have a unique name. ' @@ -210,14 +202,18 @@ class Service(object): 'for this service are created on a single host, the port will clash.' % self.name) - def create_and_start(service, number): - container = service.create_container(number=number, quiet=True) - service.start_container(container) - return container + def scale(self, desired_num, timeout=None): + """ + Adjusts the number of containers to the specified number and ensures + they are running. - def stop_and_remove(container): - container.stop(timeout=self.stop_timeout(timeout)) - container.remove() + - creates containers until there are at least `desired_num` + - stops containers until there are at most `desired_num` running + - starts containers until there are at least `desired_num` running + - removes all stopped containers + """ + + self.show_scale_warnings(desired_num) running_containers = self.containers(stopped=False) num_running = len(running_containers) @@ -228,11 +224,10 @@ class Service(object): return if desired_num > num_running: - # we need to start/create until we have desired_num all_containers = self.containers(stopped=True) if num_running != len(all_containers): - # we have some stopped containers, let's start them up again + # we have some stopped containers, check for divergences stopped_containers = [ c for c in all_containers if not c.is_running ] @@ -241,38 +236,14 @@ class Service(object): divergent_containers = [ c for c in stopped_containers if self._containers_have_diverged([c]) ] - stopped_containers = sorted( - set(stopped_containers) - set(divergent_containers), - key=attrgetter('number') - ) for c in divergent_containers: c.remove() - num_stopped = len(stopped_containers) + all_containers = list(set(all_containers) - set(divergent_containers)) - if num_stopped + num_running > desired_num: - num_to_start = desired_num - num_running - containers_to_start = stopped_containers[:num_to_start] - else: - containers_to_start = stopped_containers - - parallel_start(containers_to_start, {}) - - num_running += len(containers_to_start) - - num_to_create = desired_num - num_running - next_number = self._next_container_number() - container_numbers = [ - number for number in range( - next_number, next_number + num_to_create - ) - ] - - parallel_execute( - container_numbers, - lambda n: create_and_start(service=self, number=n), - lambda n: self.get_container_name(n), - "Creating and starting" + sorted_containers = sorted(all_containers, key=attrgetter('number')) + self._execute_convergence_start( + sorted_containers, desired_num, timeout, True, True ) if desired_num < num_running: @@ -282,12 +253,7 @@ class Service(object): running_containers, key=attrgetter('number')) - parallel_execute( - sorted_running_containers[-num_to_stop:], - stop_and_remove, - lambda c: c.name, - "Stopping and removing", - ) + self._downscale(sorted_running_containers[-num_to_stop:], timeout) def create_container(self, one_off=False, @@ -400,51 +366,109 @@ class Service(object): return has_diverged - def execute_convergence_plan(self, - plan, - timeout=None, - detached=False, - start=True): - (action, containers) = plan - should_attach_logs = not detached + def _execute_convergence_create(self, scale, detached, start): + i = self._next_container_number() - if action == 'create': - container = self.create_container() + def create_and_start(service, n): + container = service.create_container(number=n) + if not detached: + container.attach_log_stream() + if start: + self.start_container(container) + return container - if should_attach_logs: - container.attach_log_stream() + return parallel_execute( + range(i, i + scale), + lambda n: create_and_start(self, n), + lambda n: self.get_container_name(n), + "Creating" + )[0] - if start: - self.start_container(container) + def _execute_convergence_recreate(self, containers, scale, timeout, detached, start): + if len(containers) > scale: + self._downscale(containers[scale:], timeout) + containers = containers[:scale] - return [container] - - elif action == 'recreate': - return [ - self.recreate_container( - container, - timeout=timeout, - attach_logs=should_attach_logs, + def recreate(container): + return self.recreate_container( + container, timeout=timeout, attach_logs=not detached, start_new_container=start ) - for container in containers - ] - - elif action == 'start': - if start: - for container in containers: - self.start_container_if_stopped(container, attach_logs=should_attach_logs) - + containers = parallel_execute( + containers, + recreate, + lambda c: c.name, + "Recreating" + )[0] + if len(containers) < scale: + containers.extend(self._execute_convergence_create( + scale - len(containers), detached, start + )) return containers - elif action == 'noop': + def _execute_convergence_start(self, containers, scale, timeout, detached, start): + if len(containers) > scale: + self._downscale(containers[scale:], timeout) + containers = containers[:scale] + if start: + parallel_execute( + containers, + lambda c: self.start_container_if_stopped(c, attach_logs=not detached), + lambda c: c.name, + "Starting" + ) + if len(containers) < scale: + containers.extend(self._execute_convergence_create( + scale - len(containers), detached, start + )) + return containers + + def _downscale(self, containers, timeout=None): + def stop_and_remove(container): + container.stop(timeout=self.stop_timeout(timeout)) + container.remove() + + parallel_execute( + containers, + stop_and_remove, + lambda c: c.name, + "Stopping and removing", + ) + + def execute_convergence_plan(self, plan, timeout=None, detached=False, + start=True, scale_override=None): + (action, containers) = plan + scale = scale_override if scale_override is not None else self.scale_num + containers = sorted(containers, key=attrgetter('number')) + + self.show_scale_warnings(scale) + + if action == 'create': + return self._execute_convergence_create( + scale, detached, start + ) + + if action == 'recreate': + return self._execute_convergence_recreate( + containers, scale, timeout, detached, start + ) + + if action == 'start': + return self._execute_convergence_start( + containers, scale, timeout, detached, start + ) + + if action == 'noop': + if scale != len(containers): + return self._execute_convergence_start( + containers, scale, timeout, detached, start + ) for c in containers: log.info("%s is up-to-date" % c.name) return containers - else: - raise Exception("Invalid action: {}".format(action)) + raise Exception("Invalid action: {}".format(action)) def recreate_container( self, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index bfc963402..43dc216ba 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1866,6 +1866,33 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(project.get_service('simple').containers()), 0) self.assertEqual(len(project.get_service('another').containers()), 0) + def test_up_scale(self): + self.base_dir = 'tests/fixtures/scale' + project = self.project + self.dispatch(['up', '-d']) + assert len(project.get_service('web').containers()) == 2 + assert len(project.get_service('db').containers()) == 1 + + self.dispatch(['up', '-d', '--scale', 'web=1']) + assert len(project.get_service('web').containers()) == 1 + assert len(project.get_service('db').containers()) == 1 + + self.dispatch(['up', '-d', '--scale', 'web=3']) + assert len(project.get_service('web').containers()) == 3 + assert len(project.get_service('db').containers()) == 1 + + self.dispatch(['up', '-d', '--scale', 'web=1', '--scale', 'db=2']) + assert len(project.get_service('web').containers()) == 1 + assert len(project.get_service('db').containers()) == 2 + + self.dispatch(['up', '-d']) + assert len(project.get_service('web').containers()) == 2 + assert len(project.get_service('db').containers()) == 1 + + self.dispatch(['up', '-d', '--scale', 'web=0', '--scale', 'db=0']) + assert len(project.get_service('web').containers()) == 0 + assert len(project.get_service('db').containers()) == 0 + def test_port(self): self.base_dir = 'tests/fixtures/ports-composefile' self.dispatch(['up', '-d'], None) diff --git a/tests/fixtures/scale/docker-compose.yml b/tests/fixtures/scale/docker-compose.yml new file mode 100644 index 000000000..a0d3b771f --- /dev/null +++ b/tests/fixtures/scale/docker-compose.yml @@ -0,0 +1,9 @@ +version: '2.2' +services: + web: + image: busybox + command: top + scale: 2 + db: + image: busybox + command: top diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 2eae88ec5..afb408c83 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -19,6 +19,7 @@ from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_1 as V2_1 +from compose.const import COMPOSEFILE_V2_2 as V2_2 from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE @@ -1137,6 +1138,33 @@ class ProjectTest(DockerClientTestCase): containers = project.containers() self.assertEqual(len(containers), 1) + def test_project_up_config_scale(self): + config_data = build_config( + version=V2_2, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top', + 'scale': 3 + }] + ) + + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + project.up() + assert len(project.containers()) == 3 + + project.up(scale_override={'web': 2}) + assert len(project.containers()) == 2 + + project.up(scale_override={'web': 4}) + assert len(project.containers()) == 4 + + project.stop() + project.up() + assert len(project.containers()) == 3 + @v2_only() def test_initialize_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) From 457c16a7b1248ed177c5f9fa444c65f36329917e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 18 Apr 2017 12:53:43 -0700 Subject: [PATCH 31/33] Properly relay errors in execute_convergence_plan Signed-off-by: Joffrey F --- compose/service.py | 20 +++++++++++++++----- tests/acceptance/cli_test.py | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/compose/service.py b/compose/service.py index 4fa3aeabf..65eded8ec 100644 --- a/compose/service.py +++ b/compose/service.py @@ -377,12 +377,16 @@ class Service(object): self.start_container(container) return container - return parallel_execute( + containers, errors = parallel_execute( range(i, i + scale), lambda n: create_and_start(self, n), lambda n: self.get_container_name(n), "Creating" - )[0] + ) + if errors: + raise OperationFailedError(errors.values()[0]) + + return containers def _execute_convergence_recreate(self, containers, scale, timeout, detached, start): if len(containers) > scale: @@ -394,12 +398,14 @@ class Service(object): container, timeout=timeout, attach_logs=not detached, start_new_container=start ) - containers = parallel_execute( + containers, errors = parallel_execute( containers, recreate, lambda c: c.name, "Recreating" - )[0] + ) + if errors: + raise OperationFailedError(errors.values()[0]) if len(containers) < scale: containers.extend(self._execute_convergence_create( scale - len(containers), detached, start @@ -411,12 +417,16 @@ class Service(object): self._downscale(containers[scale:], timeout) containers = containers[:scale] if start: - parallel_execute( + _, errors = parallel_execute( containers, lambda c: self.start_container_if_stopped(c, attach_logs=not detached), lambda c: c.name, "Starting" ) + + if errors: + raise OperationFailedError(errors.values()[0]) + if len(containers) < scale: containers.extend(self._execute_convergence_create( scale - len(containers), detached, start diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 43dc216ba..c4b24b4b5 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -151,7 +151,7 @@ class CLITestCase(DockerClientTestCase): def test_help(self): self.base_dir = 'tests/fixtures/no-composefile' result = self.dispatch(['help', 'up'], returncode=0) - assert 'Usage: up [options] [SERVICE...]' in result.stdout + assert 'Usage: up [options] [--scale SERVICE=NUM...] [SERVICE...]' in result.stdout # Prevent tearDown from trying to create a project self.base_dir = None From ef40e3c6b99e24c580eabd57580778d60ae79d99 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 19 Apr 2017 16:47:43 -0700 Subject: [PATCH 32/33] Prevent `docker-compose scale` to be used with a v2.2 config file Signed-off-by: Joffrey F --- compose/cli/main.py | 7 ++++++ compose/project.py | 5 +++-- compose/service.py | 13 +++++------ tests/acceptance/cli_test.py | 36 ++++++++++++++++++++++++++----- tests/integration/project_test.py | 6 +++--- tests/integration/service_test.py | 18 +++++++++------- 6 files changed, 61 insertions(+), 24 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index e018a0174..0fdf3c28a 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -26,6 +26,7 @@ from ..config import resolve_build_args from ..config.environment import Environment from ..config.serialize import serialize_config from ..config.types import VolumeSpec +from ..const import COMPOSEFILE_V2_2 as V2_2 from ..const import IS_WINDOWS_PLATFORM from ..errors import StreamParseError from ..progress_stream import StreamOutputError @@ -771,6 +772,12 @@ class TopLevelCommand(object): """ timeout = timeout_from_opts(options) + if self.project.config_version == V2_2: + raise UserError( + 'The scale command is incompatible with the v2.2 format. ' + 'Use the up command with the --scale flag instead.' + ) + for service_name, num in parse_scale_args(options['SERVICE=NUM']).items(): self.project.get_service(service_name).scale(num, timeout=timeout) diff --git a/compose/project.py b/compose/project.py index 853228764..e80b10455 100644 --- a/compose/project.py +++ b/compose/project.py @@ -57,12 +57,13 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client, networks=None, volumes=None): + def __init__(self, name, services, client, networks=None, volumes=None, config_version=None): self.name = name self.services = services self.client = client self.volumes = volumes or ProjectVolumes({}) self.networks = networks or ProjectNetworks({}, False) + self.config_version = config_version def labels(self, one_off=OneOffFilter.exclude): labels = ['{0}={1}'.format(LABEL_PROJECT, self.name)] @@ -82,7 +83,7 @@ class Project(object): networks, use_networking) volumes = ProjectVolumes.from_config(name, config_data, client) - project = cls(name, [], client, project_networks, volumes) + project = cls(name, [], client, project_networks, volumes, config_data.version) for service_dict in config_data.services: service_dict = dict(service_dict) diff --git a/compose/service.py b/compose/service.py index 65eded8ec..e903115af 100644 --- a/compose/service.py +++ b/compose/service.py @@ -383,8 +383,8 @@ class Service(object): lambda n: self.get_container_name(n), "Creating" ) - if errors: - raise OperationFailedError(errors.values()[0]) + for error in errors.values(): + raise OperationFailedError(error) return containers @@ -404,8 +404,9 @@ class Service(object): lambda c: c.name, "Recreating" ) - if errors: - raise OperationFailedError(errors.values()[0]) + for error in errors.values(): + raise OperationFailedError(error) + if len(containers) < scale: containers.extend(self._execute_convergence_create( scale - len(containers), detached, start @@ -424,8 +425,8 @@ class Service(object): "Starting" ) - if errors: - raise OperationFailedError(errors.values()[0]) + for error in errors.values(): + raise OperationFailedError(error) if len(containers) < scale: containers.extend(self._execute_convergence_create( diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index c4b24b4b5..75b15ae65 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1866,9 +1866,27 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(project.get_service('simple').containers()), 0) self.assertEqual(len(project.get_service('another').containers()), 0) - def test_up_scale(self): + def test_scale_v2_2(self): + self.base_dir = 'tests/fixtures/scale' + result = self.dispatch(['scale', 'web=1'], returncode=1) + assert 'incompatible with the v2.2 format' in result.stderr + + def test_up_scale_scale_up(self): self.base_dir = 'tests/fixtures/scale' project = self.project + + self.dispatch(['up', '-d']) + assert len(project.get_service('web').containers()) == 2 + assert len(project.get_service('db').containers()) == 1 + + self.dispatch(['up', '-d', '--scale', 'web=3']) + assert len(project.get_service('web').containers()) == 3 + assert len(project.get_service('db').containers()) == 1 + + def test_up_scale_scale_down(self): + self.base_dir = 'tests/fixtures/scale' + project = self.project + self.dispatch(['up', '-d']) assert len(project.get_service('web').containers()) == 2 assert len(project.get_service('db').containers()) == 1 @@ -1877,13 +1895,21 @@ class CLITestCase(DockerClientTestCase): assert len(project.get_service('web').containers()) == 1 assert len(project.get_service('db').containers()) == 1 - self.dispatch(['up', '-d', '--scale', 'web=3']) + def test_up_scale_reset(self): + self.base_dir = 'tests/fixtures/scale' + project = self.project + + self.dispatch(['up', '-d', '--scale', 'web=3', '--scale', 'db=3']) assert len(project.get_service('web').containers()) == 3 + assert len(project.get_service('db').containers()) == 3 + + self.dispatch(['up', '-d']) + assert len(project.get_service('web').containers()) == 2 assert len(project.get_service('db').containers()) == 1 - self.dispatch(['up', '-d', '--scale', 'web=1', '--scale', 'db=2']) - assert len(project.get_service('web').containers()) == 1 - assert len(project.get_service('db').containers()) == 2 + def test_up_scale_to_zero(self): + self.base_dir = 'tests/fixtures/scale' + project = self.project self.dispatch(['up', '-d']) assert len(project.get_service('web').containers()) == 2 diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index afb408c83..b69b04565 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -565,12 +565,12 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(service.containers()), 3) project.up() service = project.get_service('web') - self.assertEqual(len(service.containers()), 3) + self.assertEqual(len(service.containers()), 1) service.scale(1) self.assertEqual(len(service.containers()), 1) - project.up() + project.up(scale_override={'web': 3}) service = project.get_service('web') - self.assertEqual(len(service.containers()), 1) + self.assertEqual(len(service.containers()), 3) # does scale=0 ,makes any sense? after recreating at least 1 container is running service.scale(0) project.up() diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 636071755..87549c506 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -26,6 +26,7 @@ from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION from compose.container import Container +from compose.errors import OperationFailedError from compose.project import OneOffFilter from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy @@ -777,15 +778,15 @@ class ServiceTest(DockerClientTestCase): message="testing", response={}, explanation="Boom")): - with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: - service.scale(3) + with pytest.raises(OperationFailedError): + service.scale(3) - self.assertEqual(len(service.containers()), 1) - self.assertTrue(service.containers()[0].is_running) - self.assertIn( - "ERROR: for composetest_web_2 Cannot create container for service web: Boom", - mock_stderr.getvalue() + assert len(service.containers()) == 1 + assert service.containers()[0].is_running + assert ( + "ERROR: for composetest_web_2 Cannot create container for service" + " web: Boom" in mock_stderr.getvalue() ) def test_scale_with_unexpected_exception(self): @@ -837,7 +838,8 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('app', container_name='custom-container') self.assertEqual(service.custom_container_name, 'custom-container') - service.scale(3) + with pytest.raises(OperationFailedError): + service.scale(3) captured_output = mock_log.warn.call_args[0][0] From 38af51314e295ad14d24c2a323a164a787e77bf7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 21 Apr 2017 14:43:03 -0700 Subject: [PATCH 33/33] Bump 1.13.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 49 ++++++++++++++++++++++++++++++++++++++++++++- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d49ddea7..a8f64d757 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,53 @@ Change log ========== +1.13.0 (2017-05-01) +------------------- + +### Breaking changes + +- `docker-compose up` now resets a service's scaling to its default value. + You can use the newly introduced `--scale` option to specify a custom + scale value + +### New features + +#### Compose file version 2.2 + +- Introduced version 2.2 of the `docker-compose.yml` specification. This + version requires to be used with Docker Engine 1.13.0 or above + +- Added support for `init` in service definitions. + +- Added support for `scale` in service definitions. The configuration's value + can be overridden using the `--scale` flag in `docker-compose up`. + Please note that the `scale` command is disabled for this file format + +#### Compose file version 2.x + +- Added support for `options` in the `ipam` section of network definitions + +### Bugfixes + +- Fixed a bug where paths provided to compose via the `-f` option were not + being resolved properly + +- Fixed a bug where the `ext_ip::target_port` notation in the ports section + was incorrectly marked as invalid + +- Fixed an issue where the `exec` command would sometimes not return control + to the terminal when using the `-d` flag + +- Fixed a bug where secrets were missing from the output of the `config` + command for v3.2 files + +- Fixed an issue where `docker-compose` would hang if no internet connection + was available + +- Fixed an issue where paths containing unicode characters passed via the `-f` + flag were causing Compose to crash + + 1.12.0 (2017-04-04) ------------------- @@ -8,7 +55,7 @@ Change log #### Compose file version 3.2 -- Introduced version 3.2 of the `docker-compose.yml` specification. +- Introduced version 3.2 of the `docker-compose.yml` specification - Added support for `cache_from` in the `build` section of services diff --git a/compose/__init__.py b/compose/__init__.py index d387af24e..f80467115 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.13.0dev' +__version__ = '1.13.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 31c5d3151..beff0c6b9 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.12.0dev" +VERSION="1.13.0-rc1" IMAGE="docker/compose:$VERSION"