From da32c44bce15a83b1d504db3b6e6fb2ed8f4d584 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Wed, 1 Nov 2017 12:15:00 -0300 Subject: [PATCH 01/47] Terminate containers on SIGHUP (fixes #4909) Signed-off-by: Guillermo Arribas --- compose/cli/main.py | 5 +++-- compose/cli/signals.py | 14 ++++++++++++++ tests/acceptance/cli_test.py | 13 +++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index c3e30919d..e89a0fe13 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1178,6 +1178,7 @@ def run_one_off_container(container_options, project, service, options): project.client.remove_container(container.id, force=True, v=True) signals.set_signal_handler_to_shutdown() + signals.set_signal_handler_to_hang_up() try: try: if IS_WINDOWS_PLATFORM: @@ -1195,10 +1196,10 @@ def run_one_off_container(container_options, project, service, options): service.start_container(container) pty.start(sockets) exit_code = container.wait() - except signals.ShutdownException: + except (signals.ShutdownException, signals.HangUpException): project.client.stop(container.id) exit_code = 1 - except signals.ShutdownException: + except (signals.ShutdownException, signals.HangUpException): project.client.kill(container.id) remove_container(force=True) sys.exit(2) diff --git a/compose/cli/signals.py b/compose/cli/signals.py index 9b360c44e..44def2ece 100644 --- a/compose/cli/signals.py +++ b/compose/cli/signals.py @@ -10,6 +10,10 @@ class ShutdownException(Exception): pass +class HangUpException(Exception): + pass + + def shutdown(signal, frame): raise ShutdownException() @@ -23,6 +27,16 @@ def set_signal_handler_to_shutdown(): set_signal_handler(shutdown) +def hang_up(signal, frame): + raise HangUpException() + + +def set_signal_handler_to_hang_up(): + # on Windows a ValueError will be raised if trying to set signal handler for SIGHUP + if not IS_WINDOWS_PLATFORM: + signal.signal(signal.SIGHUP, hang_up) + + def ignore_sigpipe(): # Restore default behavior for SIGPIPE instead of raising # an exception when encountered. diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6c18a175c..65b96733e 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1795,6 +1795,19 @@ class CLITestCase(DockerClientTestCase): 'simplecomposefile_simple_run_1', 'exited')) + def test_run_handles_sighup(self): + proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) + wait_on_condition(ContainerStateCondition( + self.project.client, + 'simplecomposefile_simple_run_1', + 'running')) + + os.kill(proc.pid, signal.SIGHUP) + wait_on_condition(ContainerStateCondition( + self.project.client, + 'simplecomposefile_simple_run_1', + 'exited')) + @mock.patch.dict(os.environ) def test_run_unicode_env_values_from_system(self): value = 'ą, ć, ę, ł, ń, ó, ś, ź, ż' From b27c2453956209225f2e89dd9f5da8d0ca9086e2 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Wed, 24 Jan 2018 23:07:27 +0100 Subject: [PATCH 02/47] Add bash completion for `up {--always-recreate-deps,--renew-anon-volumes}` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 017b0192f..faa537cb3 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -550,7 +550,7 @@ _docker_compose_up() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--abort-on-container-exit --build -d --exit-code-from --force-recreate --help --no-build --no-color --no-deps --no-recreate --no-start --remove-orphans --scale --timeout -t" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--abort-on-container-exit --always-recreate-deps --build -d --exit-code-from --force-recreate --help --no-build --no-color --no-deps --no-recreate --no-start --renew-anon-volumes -V --remove-orphans --scale --timeout -t" -- "$cur" ) ) ;; *) __docker_compose_services_all From 5d1554c9dfd1096ecbb6ede5f92148e605d5989b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 25 Jan 2018 17:50:20 -0800 Subject: [PATCH 03/47] Trigger remote build for OSX 10.11 Signed-off-by: Joffrey F --- .circleci/config.yml | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11239e25f..748cbdd0d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,10 +32,10 @@ jobs: - store_artifacts: path: dist/docker-compose-Darwin-x86_64 destination: docker-compose-Darwin-x86_64 - - deploy: - name: Deploy binary to bintray - command: | - OS_NAME=Darwin PKG_NAME=osx ./script/circle/bintray-deploy.sh + # - deploy: + # name: Deploy binary to bintray + # command: | + # OS_NAME=Darwin PKG_NAME=osx ./script/circle/bintray-deploy.sh build-linux-binary: @@ -53,6 +53,30 @@ jobs: name: Deploy binary to bintray command: | OS_NAME=Linux PKG_NAME=linux ./script/circle/bintray-deploy.sh + + trigger-osx-binary-deploy: + # We use a separate repo to build OSX binaries meant for distribution + # with support for OSSX 10.11 (xcode 7). This job triggers a build on + # that repo. + docker: + - image: alpine:3.6 + + steps: + - run: + name: install curl + command: apk update && apk add curl + + - run: + name: API trigger + command: | + curl -X POST -H "Content-Type: application/json" \ + -u ${OSX_RELEASE_TOKEN} -d "{\ + \"build_parameters\": {\ + \"COMPOSE_BRANCH\": \"${CIRCLE_BRANCH}\"\ + }\ + }" https://circleci.com/api/v1.1/project/github/docker/compose-osx-release + + workflows: version: 2 all: @@ -60,3 +84,9 @@ workflows: - test - build-linux-binary - build-osx-binary + - trigger-osx-binary-deploy: + filters: + branches: + only: + - master + - /bump-.*/ From b9939c97e576c1a24e1bbd907e8807ed6eed76e9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 25 Jan 2018 18:10:56 -0800 Subject: [PATCH 04/47] Don't leak circle API auth info Signed-off-by: Joffrey F --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 748cbdd0d..4be1557b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,7 +74,7 @@ jobs: \"build_parameters\": {\ \"COMPOSE_BRANCH\": \"${CIRCLE_BRANCH}\"\ }\ - }" https://circleci.com/api/v1.1/project/github/docker/compose-osx-release + }" https://circleci.com/api/v1.1/project/github/docker/compose-osx-release > /dev/null workflows: From 520fad331fa498862ebb3baf928820cb37b10aa6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 25 Jan 2018 18:13:49 -0800 Subject: [PATCH 05/47] Circle token Signed-off-by: Joffrey F --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4be1557b0..4ac6d4135 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,12 +69,12 @@ jobs: - run: name: API trigger command: | - curl -X POST -H "Content-Type: application/json" \ - -u ${OSX_RELEASE_TOKEN} -d "{\ + curl -X POST -H "Content-Type: application/json" -d "{\ \"build_parameters\": {\ \"COMPOSE_BRANCH\": \"${CIRCLE_BRANCH}\"\ }\ - }" https://circleci.com/api/v1.1/project/github/docker/compose-osx-release > /dev/null + }" https://circleci.com/api/v1.1/project/github/docker/compose-osx-release?circle-token=${OSX_RELEASE_TOKEN} \ + > /dev/null workflows: From 6cfbb7ed8a4a9ac022144465c9544f92ab6357b5 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Tue, 30 Jan 2018 00:57:07 +0100 Subject: [PATCH 06/47] Add missing `-q|--quiet` options to increase consistency Signed-off-by: Harald Albers --- compose/cli/main.py | 14 +++++++------- contrib/completion/bash/docker-compose | 6 +++--- tests/acceptance/cli_test.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 380257dbf..6f8cc47c8 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -503,14 +503,14 @@ class TopLevelCommand(object): Usage: images [options] [SERVICE...] Options: - -q Only display IDs + -q, --quiet Only display IDs """ containers = sorted( self.project.containers(service_names=options['SERVICE'], stopped=True) + self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only), key=attrgetter('name')) - if options['-q']: + if options['--quiet']: for image in set(c.image for c in containers): print(image.split(':')[1]) else: @@ -624,12 +624,12 @@ class TopLevelCommand(object): Usage: ps [options] [SERVICE...] Options: - -q Only display IDs + -q, --quiet Only display IDs --services Display services --filter KEY=VAL Filter services by a property """ - if options['-q'] and options['--services']: - raise UserError('-q and --services cannot be combined') + if options['--quiet'] and options['--services']: + raise UserError('--quiet and --services cannot be combined') if options['--services']: filt = build_filter(options.get('--filter')) @@ -644,7 +644,7 @@ class TopLevelCommand(object): self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only), key=attrgetter('name')) - if options['-q']: + if options['--quiet']: for container in containers: print(container.id) else: @@ -676,7 +676,7 @@ class TopLevelCommand(object): Options: --ignore-pull-failures Pull what it can and ignores images with pull failures. --parallel Pull multiple images in parallel. - --quiet Pull without printing progress information + -q, --quiet Pull without printing progress information """ self.project.pull( service_names=options['SERVICE'], diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index faa537cb3..98e1d6c0d 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -259,7 +259,7 @@ _docker_compose_help() { _docker_compose_images() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help -q" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --quiet -q" -- "$cur" ) ) ;; *) __docker_compose_services_all @@ -361,7 +361,7 @@ _docker_compose_ps() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help -q --services --filter" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --quiet -q --services --filter" -- "$cur" ) ) ;; *) __docker_compose_services_all @@ -373,7 +373,7 @@ _docker_compose_ps() { _docker_compose_pull() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --parallel --quiet" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --parallel --quiet -q" -- "$cur" ) ) ;; *) __docker_compose_services_from_image diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ade7d10a9..8fb6dffec 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -207,13 +207,13 @@ class CLITestCase(DockerClientTestCase): self.base_dir = None result = self.dispatch([ '-f', 'tests/fixtures/invalid-composefile/invalid.yml', - 'config', '-q' + 'config', '--quiet' ], returncode=1) assert "'notaservice' must be a mapping" in result.stderr def test_config_quiet(self): self.base_dir = 'tests/fixtures/v2-full' - assert self.dispatch(['config', '-q']).stdout == '' + assert self.dispatch(['config', '--quiet']).stdout == '' def test_config_default(self): self.base_dir = 'tests/fixtures/v2-full' From d8d484e0e19db5326afeb4cdf56864eceb81566c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Jan 2018 18:54:58 -0800 Subject: [PATCH 07/47] Bump python SDK to 3.0.0 Signed-off-by: Joffrey F --- compose/container.py | 2 +- compose/service.py | 1 - requirements.txt | 2 +- setup.py | 2 +- tests/helpers.py | 2 +- tests/unit/service_test.py | 2 -- 6 files changed, 4 insertions(+), 7 deletions(-) diff --git a/compose/container.py b/compose/container.py index 4ab99ffa8..9323b1192 100644 --- a/compose/container.py +++ b/compose/container.py @@ -243,7 +243,7 @@ class Container(object): self.inspect() def wait(self): - return self.client.wait(self.id) + return self.client.wait(self.id).get('StatusCode', 127) def logs(self, *args, **kwargs): return self.client.logs(self.id, *args, **kwargs) diff --git a/compose/service.py b/compose/service.py index b1f7d707b..b3d911135 100644 --- a/compose/service.py +++ b/compose/service.py @@ -972,7 +972,6 @@ class Service(object): build_output = self.client.build( path=path, tag=self.image_name, - stream=True, rm=True, forcerm=force_rm, pull=pull, diff --git a/requirements.txt b/requirements.txt index bc483b4b7..100e72117 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==2.7.0 +docker==3.0.0 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index a75e0cb7f..a85bcdf72 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.7.0, < 3.0', + 'docker >= 3.0.0, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', diff --git a/tests/helpers.py b/tests/helpers.py index f151f9cde..dd1299811 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -32,7 +32,7 @@ def create_custom_host_file(client, filename, content): ) try: client.start(container) - exitcode = client.wait(container) + exitcode = client.wait(container)['StatusCode'] if exitcode != 0: output = client.logs(container) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 92d7f08d5..21bac8b83 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -470,7 +470,6 @@ class ServiceTest(unittest.TestCase): self.mock_client.build.assert_called_once_with( tag='default_foo', dockerfile=None, - stream=True, path='.', pull=False, forcerm=False, @@ -513,7 +512,6 @@ class ServiceTest(unittest.TestCase): self.mock_client.build.assert_called_once_with( tag='default_foo', dockerfile=None, - stream=True, path='.', pull=False, forcerm=False, From 4d4e066fbc605b75996ea135139f13f3b1853712 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 1 Feb 2018 12:16:32 -0800 Subject: [PATCH 08/47] Fix DOCKER_TLS_VERIFY bug Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 9 ++++-- tests/unit/cli/docker_client_test.py | 42 +++++++++++++++++++++++----- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index a581ae672..818fe63ad 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -10,6 +10,7 @@ from docker.errors import TLSParameterError from docker.tls import TLSConfig from docker.utils import kwargs_from_env +from ..config.environment import Environment from ..const import HTTP_TIMEOUT from .errors import UserError from .utils import generate_user_agent @@ -36,14 +37,18 @@ def get_tls_version(environment): def tls_config_from_options(options, environment=None): - environment = environment or {} + environment = environment or Environment() cert_path = environment.get('DOCKER_CERT_PATH') or None tls = options.get('--tls', False) ca_cert = unquote_path(options.get('--tlscacert')) cert = unquote_path(options.get('--tlscert')) key = unquote_path(options.get('--tlskey')) - verify = options.get('--tlsverify', environment.get('DOCKER_TLS_VERIFY')) + # verify is a special case - with docopt `--tlsverify` = False means it + # wasn't used, so we set it if either the environment or the flag is True + # see https://github.com/docker/compose/issues/5632 + verify = options.get('--tlsverify') or environment.get_boolean('DOCKER_TLS_VERIFY') + skip_hostname_check = options.get('--skip-hostname-check', False) if cert_path is not None and not any((ca_cert, cert, key)): # FIXME: Modify TLSConfig to take a cert_path argument and do this internally diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index 62a537ba5..d8ce31fba 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -13,6 +13,7 @@ from compose.cli import errors from compose.cli.docker_client import docker_client from compose.cli.docker_client import get_tls_version from compose.cli.docker_client import tls_config_from_options +from compose.config.environment import Environment from tests import mock from tests import unittest @@ -163,14 +164,14 @@ class TLSConfigTestCase(unittest.TestCase): def test_tls_simple_with_tls_version(self): tls_version = 'TLSv1' options = {'--tls': True} - environment = {'COMPOSE_TLS_VERSION': tls_version} + environment = Environment({'COMPOSE_TLS_VERSION': tls_version}) result = tls_config_from_options(options, environment) assert isinstance(result, docker.tls.TLSConfig) assert result.ssl_version == ssl.PROTOCOL_TLSv1 def test_tls_mixed_environment_and_flags(self): options = {'--tls': True, '--tlsverify': False} - environment = {'DOCKER_CERT_PATH': 'tests/fixtures/tls/'} + environment = Environment({'DOCKER_CERT_PATH': 'tests/fixtures/tls/'}) result = tls_config_from_options(options, environment) assert isinstance(result, docker.tls.TLSConfig) assert result.cert == (self.client_cert, self.key) @@ -178,15 +179,42 @@ class TLSConfigTestCase(unittest.TestCase): assert result.verify is False def test_tls_flags_override_environment(self): - environment = {'DOCKER_TLS_VERIFY': True} - options = {'--tls': True, '--tlsverify': False} - assert tls_config_from_options(options, environment) is True + environment = Environment({ + 'DOCKER_CERT_PATH': '/completely/wrong/path', + 'DOCKER_TLS_VERIFY': 'false' + }) + options = { + '--tlscacert': '"{0}"'.format(self.ca_cert), + '--tlscert': '"{0}"'.format(self.client_cert), + '--tlskey': '"{0}"'.format(self.key), + '--tlsverify': True + } + + result = tls_config_from_options(options, environment) + assert isinstance(result, docker.tls.TLSConfig) + assert result.cert == (self.client_cert, self.key) + assert result.ca_cert == self.ca_cert + assert result.verify is True + + def test_tls_verify_flag_no_override(self): + environment = Environment({ + 'DOCKER_TLS_VERIFY': 'true', + 'COMPOSE_TLS_VERSION': 'TLSv1' + }) + options = {'--tls': True, '--tlsverify': False} - environment['COMPOSE_TLS_VERSION'] = 'TLSv1' result = tls_config_from_options(options, environment) assert isinstance(result, docker.tls.TLSConfig) assert result.ssl_version == ssl.PROTOCOL_TLSv1 - assert result.verify is False + # verify is a special case - since `--tlsverify` = False means it + # wasn't used, we set it if either the environment or the flag is True + # see https://github.com/docker/compose/issues/5632 + assert result.verify is True + + def test_tls_verify_env_falsy_value(self): + environment = Environment({'DOCKER_TLS_VERIFY': '0'}) + options = {'--tls': True} + assert tls_config_from_options(options, environment) is True class TestGetTlsVersion(object): From a0f78539b64ee81eb5e43feac85c4ed4b449e102 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 18 Jan 2018 17:19:22 -0800 Subject: [PATCH 09/47] Test and build on 3.6 (replaces 3.4) ; dist 3.6-compiled binaries Signed-off-by: Joffrey F --- .circleci/config.yml | 8 ++++---- Dockerfile | 16 ++++++++-------- Dockerfile.armhf | 8 ++++---- Jenkinsfile | 2 +- appveyor.yml | 6 +++--- requirements-build.txt | 2 +- requirements-dev.txt | 6 +++--- requirements.txt | 3 ++- script/build/linux-entrypoint | 2 +- script/build/osx | 2 +- script/build/windows.ps1 | 13 +++++++++---- script/circle/bintray-deploy.sh | 2 ++ script/clean | 1 + script/setup/osx | 27 ++++++++++++++++++++++++++- script/test/all | 2 +- setup.py | 1 + tests/unit/cli/docker_client_test.py | 5 ++++- tox.ini | 5 ++++- 18 files changed, 76 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ac6d4135..7661c6470 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,15 +5,15 @@ jobs: xcode: "8.3.3" steps: - checkout -# - run: -# name: install python3 -# command: brew install python3 + - run: + name: install python3 + command: brew update > /dev/null && brew install python3 - run: name: install tox command: sudo pip install --upgrade tox==2.1.1 - run: name: unit tests - command: tox -e py27 -- tests/unit + command: tox -e py27,py36 -- tests/unit build-osx-binary: macos: diff --git a/Dockerfile b/Dockerfile index c5ae9e739..6e36fddb4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,19 +39,19 @@ RUN set -ex; \ rm -rf /Python-2.7.13; \ rm Python-2.7.13.tgz -# Build python 3.4 from source +# Build python 3.6 from source RUN set -ex; \ - curl -LO https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz && \ - SHA256=fe59daced99549d1d452727c050ae486169e9716a890cffb0d468b376d916b48; \ - echo "${SHA256} Python-3.4.6.tgz" | sha256sum -c - && \ - tar -xzf Python-3.4.6.tgz; \ - cd Python-3.4.6; \ + curl -LO https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz && \ + SHA256=9de6494314ea199e3633211696735f65; \ + echo "${SHA256} Python-3.6.4.tgz" | md5sum -c - && \ + tar -xzf Python-3.6.4.tgz; \ + cd Python-3.6.4; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-3.4.6; \ - rm Python-3.4.6.tgz + rm -rf /Python-3.6.4; \ + rm Python-3.6.4.tgz # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib diff --git a/Dockerfile.armhf b/Dockerfile.armhf index b7be8cd36..ce4ab7c13 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -33,15 +33,15 @@ RUN set -ex; \ cd ..; \ rm -rf /Python-2.7.13 -# Build python 3.4 from source +# Build python 3.6 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz | tar -xz; \ - cd Python-3.4.6; \ + curl -L https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz | tar -xz; \ + cd Python-3.6.4; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-3.4.6 + rm -rf /Python-3.6.4 # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib diff --git a/Jenkinsfile b/Jenkinsfile index 51136b1f7..eb86ea326 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -60,5 +60,5 @@ buildImage() parallel( failFast: true, all_py27: runTests(pythonVersions: "py27", dockerVersions: "all"), - all_py34: runTests(pythonVersions: "py34", dockerVersions: "all"), + all_py36: runTests(pythonVersions: "py36", dockerVersions: "all"), ) diff --git a/appveyor.yml b/appveyor.yml index e4f39544a..f027a1180 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,15 +2,15 @@ version: '{branch}-{build}' install: - - "SET PATH=C:\\Python27-x64;C:\\Python27-x64\\Scripts;%PATH%" + - "SET PATH=C:\\Python36-x64;C:\\Python36-x64\\Scripts;%PATH%" - "python --version" - - "pip install tox==2.1.1 virtualenv==13.1.2" + - "pip install tox==2.9.1 virtualenv==15.1.0" # Build the binary after tests build: false test_script: - - "tox -e py27,py34 -- tests/unit" + - "tox -e py27,py36 -- tests/unit" - ps: ".\\script\\build\\windows.ps1" artifacts: diff --git a/requirements-build.txt b/requirements-build.txt index 27f610ca9..e5a77e794 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -pyinstaller==3.2.1 +pyinstaller==3.3.1 diff --git a/requirements-dev.txt b/requirements-dev.txt index e06cad45c..32c5c23a1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ -coverage==3.7.1 +coverage==4.4.2 flake8==3.5.0 mock>=1.0.1 -pytest==2.7.2 -pytest-cov==2.1.0 +pytest==2.9.2 +pytest-cov==2.5.1 diff --git a/requirements.txt b/requirements.txt index 100e72117..0aad2ea28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,8 @@ git+git://github.com/tartley/colorama.git@bd378c725b45eba0b8e5cc091c3ca76a954c92 idna==2.5 ipaddress==1.0.18 jsonschema==2.6.0 -pypiwin32==219; sys_platform == 'win32' +pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' +pypiwin32==220; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.6.7 PyYAML==3.12 requests==2.18.4 diff --git a/script/build/linux-entrypoint b/script/build/linux-entrypoint index bf515060a..0e3c7ec1e 100755 --- a/script/build/linux-entrypoint +++ b/script/build/linux-entrypoint @@ -3,7 +3,7 @@ set -ex TARGET=dist/docker-compose-$(uname -s)-$(uname -m) -VENV=/code/.tox/py27 +VENV=/code/.tox/py36 mkdir -p `pwd`/dist chmod 777 `pwd`/dist diff --git a/script/build/osx b/script/build/osx index 3de345762..0c4b062bb 100755 --- a/script/build/osx +++ b/script/build/osx @@ -5,7 +5,7 @@ PATH="/usr/local/bin:$PATH" rm -rf venv -virtualenv -p /usr/local/bin/python venv +virtualenv -p /usr/local/bin/python3 venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-build.txt venv/bin/pip install --no-deps . diff --git a/script/build/windows.ps1 b/script/build/windows.ps1 index db643274c..98a748158 100644 --- a/script/build/windows.ps1 +++ b/script/build/windows.ps1 @@ -6,17 +6,17 @@ # # http://git-scm.com/download/win # -# 2. Install Python 2.7.10: +# 2. Install Python 3.6.4: # # https://www.python.org/downloads/ # -# 3. Append ";C:\Python27;C:\Python27\Scripts" to the "Path" environment variable: +# 3. Append ";C:\Python36;C:\Python36\Scripts" to the "Path" environment variable: # # https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx?mfr=true # # 4. In Powershell, run the following commands: # -# $ pip install virtualenv +# $ pip install 'virtualenv>=15.1.0' # $ Set-ExecutionPolicy -Scope CurrentUser RemoteSigned # # 5. Clone the repository: @@ -45,7 +45,12 @@ virtualenv .\venv $ErrorActionPreference = "Continue" # Install dependencies -.\venv\Scripts\pip install pypiwin32==219 +# Fix for https://github.com/pypa/pip/issues/3964 +# Remove-Item -Recurse -Force .\venv\Lib\site-packages\pip +# .\venv\Scripts\easy_install pip==9.0.1 +# .\venv\Scripts\pip install --upgrade pip setuptools +# End fix +.\venv\Scripts\pip install pypiwin32==220 .\venv\Scripts\pip install -r requirements.txt .\venv\Scripts\pip install --no-deps . .\venv\Scripts\pip install --allow-external pyinstaller -r requirements-build.txt diff --git a/script/circle/bintray-deploy.sh b/script/circle/bintray-deploy.sh index d508da365..8c8871aa6 100755 --- a/script/circle/bintray-deploy.sh +++ b/script/circle/bintray-deploy.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -x + curl -f -u$BINTRAY_USERNAME:$BINTRAY_API_KEY -X GET \ https://api.bintray.com/repos/docker-compose/${CIRCLE_BRANCH} diff --git a/script/clean b/script/clean index fb7ba3be2..2e1994df3 100755 --- a/script/clean +++ b/script/clean @@ -2,6 +2,7 @@ set -e find . -type f -name '*.pyc' -delete +rm -rf .coverage-binfiles find . -name .coverage.* -delete find . -name __pycache__ -delete rm -rf docs/_site build dist docker-compose.egg-info diff --git a/script/setup/osx b/script/setup/osx index 407524cba..972e79efb 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -6,11 +6,36 @@ python_version() { python -V 2>&1 } +python3_version() { + python3 -V 2>&1 +} + openssl_version() { python -c "import ssl; print ssl.OPENSSL_VERSION" } -echo "*** Using $(python_version)" +desired_python3_version="3.6.4" +desired_python3_brew_version="3.6.4_2" +python3_formula="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e69a9a592232fa5a82741f6acecffc2f1d198d/Formula/python3.rb" + +PATH="/usr/local/bin:$PATH" + +if !(which brew); then + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +fi + +brew update > /dev/null + +if !(python3_version | grep "$desired_python3_version"); then + if brew list | grep python3; then + brew unlink python3 + fi + + brew install "$python3_formula" + brew switch python3 "$desired_python3_brew_version" +fi + +echo "*** Using $(python3_version) ; $(python_version)" echo "*** Using $(openssl_version)" if !(which virtualenv); then diff --git a/script/test/all b/script/test/all index 1200c496e..e48f73bba 100755 --- a/script/test/all +++ b/script/test/all @@ -24,7 +24,7 @@ fi BUILD_NUMBER=${BUILD_NUMBER-$USER} -PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py27,py34} +PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py27,py36} for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" diff --git a/setup.py b/setup.py index a85bcdf72..fbf34e465 100644 --- a/setup.py +++ b/setup.py @@ -99,5 +99,6 @@ setup( 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.6', ], ) diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index d8ce31fba..5bb4564ef 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -22,7 +22,10 @@ class DockerClientTestCase(unittest.TestCase): def test_docker_client_no_home(self): with mock.patch.dict(os.environ): - del os.environ['HOME'] + try: + del os.environ['HOME'] + except KeyError: + pass docker_client(os.environ) @mock.patch.dict(os.environ) diff --git a/tox.ini b/tox.ini index 749be3faa..33347df20 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,9 @@ [tox] -envlist = py27,py34,pre-commit +envlist = py27,py36,pre-commit [testenv] usedevelop=True +whitelist_externals=mkdir passenv = LD_LIBRARY_PATH DOCKER_HOST @@ -17,6 +18,7 @@ deps = -rrequirements.txt -rrequirements-dev.txt commands = + mkdir -p .coverage-binfiles py.test -v \ --cov=compose \ --cov-report html \ @@ -35,6 +37,7 @@ commands = # Coverage configuration [run] branch = True +data_file = .coverage-binfiles/.coverage [report] show_missing = true From 9dde4fff0e2ad2180e256b44bac3287bdd9ae213 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 2 Feb 2018 11:57:15 -0800 Subject: [PATCH 10/47] Add support for 3.6 schema and tmpfs mount size Signed-off-by: Joffrey F --- compose/config/config_schema_v2.3.json | 6 + compose/config/config_schema_v3.6.json | 582 ++++++++++++++++++++++++ compose/config/interpolation.py | 9 + compose/config/types.py | 6 + compose/const.py | 3 + docker-compose.spec | 5 + tests/integration/service_test.py | 17 + tests/unit/config/interpolation_test.py | 7 + 8 files changed, 635 insertions(+) create mode 100644 compose/config/config_schema_v3.6.json diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 42e2afdad..2d28df77a 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -321,6 +321,12 @@ "properties": { "nocopy": {"type": "boolean"} } + }, + "tmpfs": { + "type": "object", + "properties": { + "size": {"type": ["integer", "string"]} + } } } } diff --git a/compose/config/config_schema_v3.6.json b/compose/config/config_schema_v3.6.json new file mode 100644 index 000000000..8e718780b --- /dev/null +++ b/compose/config/config_schema_v3.6.json @@ -0,0 +1,582 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.5.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secret" + } + }, + "additionalProperties": false + }, + + "configs": { + "id": "#/properties/configs", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/config" + } + }, + "additionalProperties": false + } + }, + + "patternProperties": {"^x-": {}}, + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"}, + "network": {"type": "string"}, + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "configs": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "container_name": {"type": "string"}, + "credential_spec": {"type": "object", "properties": { + "file": {"type": "string"}, + "registry": {"type": "string"} + }}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "isolation": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "additionalProperties": false + } + ] + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + }, + "tmpfs": { + "type": "object", + "properties": { + "size": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "additionalProperties": false + } + ], + "uniqueItems": true + } + }, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string", "format": "duration"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string", "format": "duration"}, + "start_period": {"type": "string", "format": "duration"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "endpoint_mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"}, + "order": {"type": "string", "enum": [ + "start-first", "stop-first" + ]} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + "reservations": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"}, + "generic_resources": {"$ref": "#/definitions/generic_resources"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}}, + "preferences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "spread": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "generic_resources": { + "id": "#/definitions/generic_resources", + "type": "array", + "items": { + "type": "object", + "properties": { + "discrete_resource_spec": { + "type": "object", + "properties": { + "kind": {"type": "string"}, + "value": {"type": "number"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "name": {"type": "string"}, + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "attachable": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "name": {"type": "string"}, + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "name": {"type": "string"}, + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "config": { + "id": "#/definitions/config", + "type": "object", + "properties": { + "name": {"type": "string"}, + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 9d52d2edb..b1143d66c 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -9,6 +9,7 @@ import six from .errors import ConfigurationError from compose.const import COMPOSEFILE_V2_0 as V2_0 +from compose.utils import parse_bytes log = logging.getLogger(__name__) @@ -215,6 +216,13 @@ def to_str(o): return o +def bytes_to_int(s): + v = parse_bytes(s) + if v is None: + raise ValueError('"{}" is not a valid byte value'.format(s)) + return v + + class ConversionMap(object): map = { service_path('blkio_config', 'weight'): to_int, @@ -247,6 +255,7 @@ class ConversionMap(object): service_path('tty'): to_boolean, service_path('volumes', 'read_only'): to_boolean, service_path('volumes', 'volume', 'nocopy'): to_boolean, + service_path('volumes', 'tmpfs', 'size'): bytes_to_int, re_path_basic('network', 'attachable'): to_boolean, re_path_basic('network', 'external'): to_boolean, re_path_basic('network', 'internal'): to_boolean, diff --git a/compose/config/types.py b/compose/config/types.py index b896b883f..d84491d0a 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -141,6 +141,9 @@ class MountSpec(object): }, 'bind': { 'propagation': 'propagation' + }, + 'tmpfs': { + 'size': 'tmpfs_size' } } _fields = ['type', 'source', 'target', 'read_only', 'consistency'] @@ -149,6 +152,9 @@ class MountSpec(object): def parse(cls, mount_dict, normalize=False, win_host=False): normpath = ntpath.normpath if win_host else os.path.normpath if mount_dict.get('source'): + if mount_dict['type'] == 'tmpfs': + raise ConfigurationError('tmpfs mounts can not specify a source') + mount_dict['source'] = normpath(mount_dict['source']) if normalize: mount_dict['source'] = normalize_path_for_engine(mount_dict['source']) diff --git a/compose/const.py b/compose/const.py index 6e5902cad..495539fb0 100644 --- a/compose/const.py +++ b/compose/const.py @@ -34,6 +34,7 @@ COMPOSEFILE_V3_2 = ComposeVersion('3.2') COMPOSEFILE_V3_3 = ComposeVersion('3.3') COMPOSEFILE_V3_4 = ComposeVersion('3.4') COMPOSEFILE_V3_5 = ComposeVersion('3.5') +COMPOSEFILE_V3_6 = ComposeVersion('3.6') API_VERSIONS = { COMPOSEFILE_V1: '1.21', @@ -47,6 +48,7 @@ API_VERSIONS = { COMPOSEFILE_V3_3: '1.30', COMPOSEFILE_V3_4: '1.30', COMPOSEFILE_V3_5: '1.30', + COMPOSEFILE_V3_6: '1.36', } API_VERSION_TO_ENGINE_VERSION = { @@ -61,4 +63,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V3_3]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_4]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_5]: '17.06.0', + API_VERSIONS[COMPOSEFILE_V3_6]: '18.02.0', } diff --git a/docker-compose.spec b/docker-compose.spec index 83d7389f3..b2b4f5f18 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -72,6 +72,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.5.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.6.json', + 'compose/config/config_schema_v3.6.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index c12724c85..0bc902aea 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -315,6 +315,23 @@ class ServiceTest(DockerClientTestCase): assert mount assert mount['Type'] == 'tmpfs' + @v2_3_only() + def test_create_container_with_tmpfs_mount_tmpfs_size(self): + container_path = '/container-tmpfs' + service = self.create_service( + 'db', + volumes=[MountSpec(type='tmpfs', target=container_path, tmpfs={'size': 5368709})] + ) + container = service.create_container() + service.start_container(container) + mount = container.get_mount(container_path) + assert mount + print(container.dictionary) + assert mount['Type'] == 'tmpfs' + assert container.get('HostConfig.Mounts')[0]['TmpfsOptions'] == { + 'SizeBytes': 5368709 + } + @v2_3_only() def test_create_container_with_volume_mount(self): container_path = '/container-volume' diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index fe5ef2490..2ba698fbf 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -27,6 +27,7 @@ def mock_env(): 'NEGINT': '-200', 'FLOAT': '0.145', 'MODE': '0600', + 'BYTES': '512m', }) @@ -147,6 +148,9 @@ def test_interpolate_environment_services_convert_types_v2(mock_env): 'read_only': '${DEFAULT:-no}', 'tty': '${DEFAULT:-N}', 'stdin_open': '${DEFAULT-on}', + 'volumes': [ + {'type': 'tmpfs', 'target': '/target', 'tmpfs': {'size': '$BYTES'}} + ] } } @@ -177,6 +181,9 @@ def test_interpolate_environment_services_convert_types_v2(mock_env): 'read_only': False, 'tty': False, 'stdin_open': True, + 'volumes': [ + {'type': 'tmpfs', 'target': '/target', 'tmpfs': {'size': 536870912}} + ] } } From dce62c81d5c3c801f39065fccd45223f11c0d21c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 2 Feb 2018 15:18:55 -0800 Subject: [PATCH 11/47] Remove obsolete code that slows down test execution Signed-off-by: Joffrey F --- Dockerfile | 55 +++-------------------------------------- compose/cli/__init__.py | 49 ------------------------------------ 2 files changed, 4 insertions(+), 100 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6e36fddb4..9df78a826 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,12 @@ -FROM debian:wheezy +FROM python:3.6 RUN set -ex; \ apt-get update -qq; \ apt-get install -y \ locales \ - gcc \ - make \ - zlib1g \ - zlib1g-dev \ - libssl-dev \ - git \ - ca-certificates \ curl \ - libsqlite3-dev \ - libbz2-dev \ - ; \ - rm -rf /var/lib/apt/lists/* + python-dev \ + git RUN curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.0-ce.tgz" && \ SHA256=692e1c72937f6214b1038def84463018d8e320c8eaf8530546c84c2f8f9c767d; \ @@ -25,44 +16,6 @@ RUN curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stabl chmod +x /usr/local/bin/docker && \ rm dockerbins.tgz -# Build Python 2.7.13 from source -RUN set -ex; \ - curl -LO https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz && \ - SHA256=a4f05a0720ce0fd92626f0278b6b433eee9a6173ddf2bced7957dfb599a5ece1; \ - echo "${SHA256} Python-2.7.13.tgz" | sha256sum -c - && \ - tar -xzf Python-2.7.13.tgz; \ - cd Python-2.7.13; \ - ./configure --enable-shared; \ - make; \ - make install; \ - cd ..; \ - rm -rf /Python-2.7.13; \ - rm Python-2.7.13.tgz - -# Build python 3.6 from source -RUN set -ex; \ - curl -LO https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz && \ - SHA256=9de6494314ea199e3633211696735f65; \ - echo "${SHA256} Python-3.6.4.tgz" | md5sum -c - && \ - tar -xzf Python-3.6.4.tgz; \ - cd Python-3.6.4; \ - ./configure --enable-shared; \ - make; \ - make install; \ - cd ..; \ - rm -rf /Python-3.6.4; \ - rm Python-3.6.4.tgz - -# Make libpython findable -ENV LD_LIBRARY_PATH /usr/local/lib - -# Install pip -RUN set -ex; \ - curl -LO https://bootstrap.pypa.io/get-pip.py && \ - SHA256=19dae841a150c86e2a09d475b5eb0602861f2a5b7761ec268049a662dbd2bd0c; \ - echo "${SHA256} get-pip.py" | sha256sum -c - && \ - python get-pip.py - # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen ENV LANG en_US.UTF-8 @@ -83,4 +36,4 @@ RUN tox --notest ADD . /code/ RUN chown -R user /code/ -ENTRYPOINT ["/code/.tox/py27/bin/docker-compose"] +ENTRYPOINT ["/code/.tox/py36/bin/docker-compose"] diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index 2574a311f..e69de29bb 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -1,49 +0,0 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -import os -import subprocess -import sys - -# Attempt to detect https://github.com/docker/compose/issues/4344 -try: - # We don't try importing pip because it messes with package imports - # on some Linux distros (Ubuntu, Fedora) - # https://github.com/docker/compose/issues/4425 - # https://github.com/docker/compose/issues/4481 - # https://github.com/pypa/pip/blob/master/pip/_vendor/__init__.py - env = os.environ.copy() - env[str('PIP_DISABLE_PIP_VERSION_CHECK')] = str('1') - - s_cmd = subprocess.Popen( - # DO NOT replace this call with a `sys.executable` call. It breaks the binary - # distribution (with the binary calling itself recursively over and over). - ['pip', 'freeze'], stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env - ) - packages = s_cmd.communicate()[0].splitlines() - dockerpy_installed = len( - list(filter(lambda p: p.startswith(b'docker-py=='), packages)) - ) > 0 - if dockerpy_installed: - from .colors import yellow - print( - yellow('WARNING:'), - "Dependency conflict: an older version of the 'docker-py' package " - "may be polluting the namespace. " - "If you're experiencing crashes, run the following command to remedy the issue:\n" - "pip uninstall docker-py; pip uninstall docker; pip install docker", - file=sys.stderr - ) - -except OSError: - # pip command is not available, which indicates it's probably the binary - # distribution of Compose which is not affected - pass -except UnicodeDecodeError: - # ref: https://github.com/docker/compose/issues/4663 - # This could be caused by a number of things, but it seems to be a - # python 2 + MacOS interaction. It's not ideal to ignore this, but at least - # it doesn't make the program unusable. - pass From 632abe94c010455a588127efa99bc8f3f10bea2d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 2 Feb 2018 16:24:17 -0800 Subject: [PATCH 12/47] Parallelize Docker versions Signed-off-by: Joffrey F --- Jenkinsfile | 35 +++++++++++++++++++++++++++-------- script/test/ci | 2 +- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index eb86ea326..44cd7c3c2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,12 +18,26 @@ def buildImage = { -> } } +def get_versions = { int number -> + def docker_versions + wrappedNode(label: "ubuntu && !zfs") { + def result = sh(script: """docker run --rm \\ + --entrypoint=/code/.tox/py27/bin/python \\ + ${image.id} \\ + /code/script/test/versions.py -n ${number} docker/docker-ce recent + """, returnStdout: true + ) + docker_versions = result.split() + } + return docker_versions +} + def runTests = { Map settings -> def dockerVersions = settings.get("dockerVersions", null) def pythonVersions = settings.get("pythonVersions", null) if (!pythonVersions) { - throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py34')`") + throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py36')`") } if (!dockerVersions) { throw new Exception("Need Docker versions to test. e.g.: `runTests(dockerVersions: 'all')`") @@ -46,7 +60,7 @@ def runTests = { Map settings -> -e "DOCKER_VERSIONS=${dockerVersions}" \\ -e "BUILD_NUMBER=\$BUILD_TAG" \\ -e "PY_TEST_VERSIONS=${pythonVersions}" \\ - --entrypoint="script/ci" \\ + --entrypoint="script/test/ci" \\ ${image.id} \\ --verbose """ @@ -56,9 +70,14 @@ def runTests = { Map settings -> } buildImage() -// TODO: break this out into meaningful "DOCKER_VERSIONS" values instead of all -parallel( - failFast: true, - all_py27: runTests(pythonVersions: "py27", dockerVersions: "all"), - all_py36: runTests(pythonVersions: "py36", dockerVersions: "all"), -) + +def testMatrix = [failFast: true] +def docker_versions = get_versions(2) + +for (int i = 0 ;i < docker_versions.length ; i++) { + def dockerVersion = docker_versions[i] + testMatrix["${dockerVersion}_py27"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py27"]) + testMatrix["${dockerVersion}_py36"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py36"]) +} + +parallel(testMatrix) diff --git a/script/test/ci b/script/test/ci index c5927b2c9..8d3aa56cb 100755 --- a/script/test/ci +++ b/script/test/ci @@ -14,7 +14,7 @@ set -ex docker version -export DOCKER_VERSIONS=all +export DOCKER_VERSIONS=${DOCKER_VERSIONS:-all} STORAGE_DRIVER=${STORAGE_DRIVER:-overlay} export DOCKER_DAEMON_ARGS="--storage-driver=$STORAGE_DRIVER" From 95005e6c03565acc33758719987a11192d9255a7 Mon Sep 17 00:00:00 2001 From: Daniel Schildt Date: Sat, 3 Feb 2018 17:22:16 +0200 Subject: [PATCH 13/47] Improve spelling in the README.md Improve spelling of the brand names: - GitHub instead of Github Signed-off-by: Daniel Schildt --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e3ca8f833..ea07f6a7d 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Installation and documentation - Full documentation is available on [Docker's website](https://docs.docker.com/compose/). - If you have any questions, you can talk in real-time with other developers in the #docker-compose IRC channel on Freenode. [Click here to join using IRCCloud.](https://www.irccloud.com/invite?hostname=irc.freenode.net&channel=%23docker-compose) -- Code repository for Compose is on [Github](https://github.com/docker/compose) +- Code repository for Compose is on [GitHub](https://github.com/docker/compose) - If you find any problems please fill out an [issue](https://github.com/docker/compose/issues/new) Contributing From 649604d88d94c73acb0860ab53ed1b55a6bd61b1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Feb 2018 14:49:13 -0800 Subject: [PATCH 14/47] Bump Docker Python SDK version Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0aad2ea28..6b1df1916 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.0.0 +docker==3.0.1 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index fbf34e465..264d647f7 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.0.0, < 4.0', + 'docker >= 3.0.1, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 8c297f267e5f986485961a4d9b9fc24a46d4c4a8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Feb 2018 17:31:17 -0800 Subject: [PATCH 15/47] Implement compatibility mode, translating deploy keys to equivalent v2 config if available Enabled using `--compatibility` CLI flag Signed-off-by: Joffrey F --- compose/cli/command.py | 9 ++- compose/cli/main.py | 2 + compose/config/config.py | 94 ++++++++++++++++++++--- tests/fixtures/v3-full/docker-compose.yml | 15 ++-- 4 files changed, 101 insertions(+), 19 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 6977195a0..9fd941bb6 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -38,6 +38,7 @@ def project_from_options(project_dir, options): tls_config=tls_config_from_options(options, environment), environment=environment, override_dir=options.get('--project-directory'), + compatibility=options.get('--compatibility'), ) @@ -63,7 +64,8 @@ def get_config_from_options(base_dir, options): base_dir, options, environment ) return config.load( - config.find(base_dir, config_path, environment) + config.find(base_dir, config_path, environment), + options.get('--compatibility') ) @@ -100,14 +102,15 @@ def get_client(environment, verbose=False, version=None, tls_config=None, host=N def get_project(project_dir, config_path=None, project_name=None, verbose=False, - host=None, tls_config=None, environment=None, override_dir=None): + host=None, tls_config=None, environment=None, override_dir=None, + compatibility=False): if not environment: environment = Environment.from_env_file(project_dir) config_details = config.find(project_dir, config_path, environment, override_dir) project_name = get_project_name( config_details.working_dir, project_name, environment ) - config_data = config.load(config_details) + config_data = config.load(config_details, compatibility) api_version = environment.get( 'COMPOSE_API_VERSION', diff --git a/compose/cli/main.py b/compose/cli/main.py index 380257dbf..9c76a561b 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -185,6 +185,8 @@ class TopLevelCommand(object): is an IP address) --project-directory PATH Specify an alternate working directory (default: the path of the Compose file) + --compatibility If set, Compose will attempt to convert deploy keys in v3 + files to their non-Swarm equivalent Commands: build Build or rebuild services diff --git a/compose/config/config.py b/compose/config/config.py index 960c3c678..58420d157 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -16,6 +16,7 @@ from . import types from .. import const from ..const import COMPOSEFILE_V1 as V1 from ..const import COMPOSEFILE_V2_1 as V2_1 +from ..const import COMPOSEFILE_V2_3 as V2_3 from ..const import COMPOSEFILE_V3_0 as V3_0 from ..const import COMPOSEFILE_V3_4 as V3_4 from ..utils import build_string_dict @@ -341,7 +342,7 @@ def find_candidates_in_parent_dirs(filenames, path): return (candidates, path) -def check_swarm_only_config(service_dicts): +def check_swarm_only_config(service_dicts, compatibility=False): warning_template = ( "Some services ({services}) use the '{key}' key, which will be ignored. " "Compose does not support '{key}' configuration - use " @@ -357,13 +358,13 @@ def check_swarm_only_config(service_dicts): key=key ) ) - - check_swarm_only_key(service_dicts, 'deploy') + if not compatibility: + check_swarm_only_key(service_dicts, 'deploy') check_swarm_only_key(service_dicts, 'credential_spec') check_swarm_only_key(service_dicts, 'configs') -def load(config_details): +def load(config_details, compatibility=False): """Load the configuration from a working directory and a list of configuration files. Files are loaded in order, and merged on top of each other to create the final configuration. @@ -391,15 +392,17 @@ def load(config_details): configs = load_mapping( config_details.config_files, 'get_configs', 'Config', config_details.working_dir ) - service_dicts = load_services(config_details, main_file) + service_dicts = load_services(config_details, main_file, compatibility) if main_file.version != V1: for service_dict in service_dicts: match_named_volumes(service_dict, volumes) - check_swarm_only_config(service_dicts) + check_swarm_only_config(service_dicts, compatibility) - return Config(main_file.version, service_dicts, volumes, networks, secrets, configs) + version = V2_3 if compatibility and main_file.version >= V3_0 else main_file.version + + return Config(version, service_dicts, volumes, networks, secrets, configs) def load_mapping(config_files, get_func, entity_type, working_dir=None): @@ -441,7 +444,7 @@ def validate_external(entity_type, name, config, version): entity_type, name, ', '.join(k for k in config if k != 'external'))) -def load_services(config_details, config_file): +def load_services(config_details, config_file, compatibility=False): def build_service(service_name, service_dict, service_names): service_config = ServiceConfig.with_abs_paths( config_details.working_dir, @@ -459,7 +462,9 @@ def load_services(config_details, config_file): service_config, service_names, config_file.version, - config_details.environment) + config_details.environment, + compatibility + ) return service_dict def build_services(service_config): @@ -827,7 +832,7 @@ def finalize_service_volumes(service_dict, environment): return service_dict -def finalize_service(service_config, service_names, version, environment): +def finalize_service(service_config, service_names, version, environment, compatibility): service_dict = dict(service_config.config) if 'environment' in service_dict or 'env_file' in service_dict: @@ -868,10 +873,79 @@ def finalize_service(service_config, service_names, version, environment): normalize_build(service_dict, service_config.working_dir, environment) + if compatibility: + service_dict, ignored_keys = translate_deploy_keys_to_container_config( + service_dict + ) + if ignored_keys: + log.warn( + 'The following deploy sub-keys are not supported in compatibility mode and have' + ' been ignored: {}'.format(', '.join(ignored_keys)) + ) + service_dict['name'] = service_config.name return normalize_v1_service_format(service_dict) +def translate_resource_keys_to_container_config(resources_dict, service_dict): + if 'limits' in resources_dict: + service_dict['mem_limit'] = resources_dict['limits'].get('memory') + if 'cpus' in resources_dict['limits']: + service_dict['cpus'] = float(resources_dict['limits']['cpus']) + if 'reservations' in resources_dict: + service_dict['mem_reservation'] = resources_dict['reservations'].get('memory') + if 'cpus' in resources_dict['reservations']: + return ['resources.reservations.cpus'] + + +def convert_restart_policy(name): + try: + return { + 'any': 'always', + 'none': 'no', + 'on-failure': 'on-failure' + }[name] + except KeyError: + raise ConfigurationError('Invalid restart policy "{}"'.format(name)) + + +def translate_deploy_keys_to_container_config(service_dict): + if 'deploy' not in service_dict: + return service_dict, [] + + deploy_dict = service_dict['deploy'] + ignored_keys = [ + k for k in ['endpoint_mode', 'labels', 'update_config', 'placement'] + if k in deploy_dict + ] + + if 'replicas' in deploy_dict and deploy_dict.get('mode') == 'replicated': + service_dict['scale'] = deploy_dict['replicas'] + + if 'restart_policy' in deploy_dict: + service_dict['restart'] = { + 'Name': convert_restart_policy(deploy_dict['restart_policy'].get('condition', 'any')), + 'MaximumRetryCount': deploy_dict['restart_policy'].get('max_attempts', 0) + } + for k in deploy_dict['restart_policy'].keys(): + if k != 'condition' and k != 'max_attempts': + ignored_keys.append('restart_policy.{}'.format(k)) + + ignored_keys.extend( + translate_resource_keys_to_container_config( + deploy_dict.get('resources', {}), service_dict + ) + ) + + del service_dict['deploy'] + if 'credential_spec' in service_dict: + del service_dict['credential_spec'] + if 'configs' in service_dict: + del service_dict['configs'] + + return service_dict, ignored_keys + + def normalize_v1_service_format(service_dict): if 'log_driver' in service_dict or 'log_opt' in service_dict: if 'logging' not in service_dict: diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml index 2bc0e248d..3a7ac25c9 100644 --- a/tests/fixtures/v3-full/docker-compose.yml +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -1,8 +1,7 @@ -version: "3.2" +version: "3.5" services: web: image: busybox - deploy: mode: replicated replicas: 6 @@ -15,18 +14,22 @@ services: max_failure_ratio: 0.3 resources: limits: - cpus: '0.001' + cpus: '0.05' memory: 50M reservations: - cpus: '0.0001' + cpus: '0.01' memory: 20M restart_policy: - condition: on_failure + condition: on-failure delay: 5s max_attempts: 3 window: 120s placement: - constraints: [node=foo] + constraints: + - node.hostname==foo + - node.role != manager + preferences: + - spread: node.labels.datacenter healthcheck: test: cat /etc/passwd From df6e300081667688bbee9e457558ad9e5e4145df Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Feb 2018 14:50:52 -0800 Subject: [PATCH 16/47] Update dockerfile.run to compile the latest (2.27) version of glibc Signed-off-by: Joffrey F --- Dockerfile.run | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/Dockerfile.run b/Dockerfile.run index 5d246e9e6..a09e57a09 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,13 +1,33 @@ -FROM alpine:3.4 +FROM sgerrand/glibc-builder as glibc +RUN apt-get install -yq bison -ENV GLIBC 2.23-r3 +ENV PKGDIR /pkgdata -RUN apk update && apk add --no-cache openssl ca-certificates && \ - wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \ - wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ - apk add --no-cache glibc-$GLIBC.apk && rm glibc-$GLIBC.apk && \ - ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ - ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib +RUN mkdir -p /usr/glibc-compat/etc && touch /usr/glibc-compat/etc/ld.so.conf +RUN /builder 2.27 /usr/glibc-compat || true +RUN mkdir -p $PKGDIR +RUN tar -xf /glibc-bin-2.27.tar.gz -C $PKGDIR +RUN rm "$PKGDIR"/usr/glibc-compat/etc/rpc && \ + rm -rf "$PKGDIR"/usr/glibc-compat/bin && \ + rm -rf "$PKGDIR"/usr/glibc-compat/sbin && \ + rm -rf "$PKGDIR"/usr/glibc-compat/lib/gconv && \ + rm -rf "$PKGDIR"/usr/glibc-compat/lib/getconf && \ + rm -rf "$PKGDIR"/usr/glibc-compat/lib/audit && \ + rm -rf "$PKGDIR"/usr/glibc-compat/share && \ + rm -rf "$PKGDIR"/usr/glibc-compat/var + + +FROM alpine:3.6 + +RUN apk update && apk add --no-cache openssl ca-certificates +COPY --from=glibc /pkgdata/ / + +RUN mkdir -p /lib /lib64 /usr/glibc-compat/lib/locale /etc && \ + ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ + ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ + ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 && \ + ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 && \ + ln -s /usr/glibc-compat/etc/ld.so.cache /etc/ld.so.cache COPY dist/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose From 3a0ed8cbbae36a197697e0876fb932ae10604d6a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Feb 2018 16:58:15 -0800 Subject: [PATCH 17/47] Modified options dict leads to a mismatched config hash for API < 1.30 Signed-off-by: Joffrey F --- compose/service.py | 4 ++-- tests/unit/service_test.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index b3d911135..f79a46962 100644 --- a/compose/service.py +++ b/compose/service.py @@ -849,10 +849,10 @@ class Service(object): override_options['mounts'] = [build_mount(v) for v in container_mounts] or None else: # Workaround for 3.2 format - self.options['tmpfs'] = self.options.get('tmpfs') or [] + override_options['tmpfs'] = self.options.get('tmpfs') or [] for m in container_mounts: if m.is_tmpfs: - self.options['tmpfs'].append(m.target) + override_options['tmpfs'].append(m.target) else: override_options['binds'].append(m.legacy_repr()) container_options['volumes'][m.target] = {} diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 21bac8b83..002ae0c0d 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -13,6 +13,7 @@ from compose.config.types import ServicePort from compose.config.types import ServiceSecret from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec +from compose.const import API_VERSIONS from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT @@ -599,6 +600,25 @@ class ServiceTest(unittest.TestCase): } assert config_dict == expected + def test_config_hash_matches_label(self): + self.mock_client.inspect_image.return_value = {'Id': 'abcd'} + service = Service( + 'foo', + image='example.com/foo', + client=self.mock_client, + network_mode=NetworkMode('bridge'), + networks={'bridge': {}}, + links=[(Service('one', client=self.mock_client), 'one')], + volumes_from=[VolumeFromSpec(Service('two', client=self.mock_client), 'rw', 'service')] + ) + config_hash = service.config_hash + + for api_version in set(API_VERSIONS.values()): + self.mock_client.api_version = api_version + assert service._get_container_create_options({}, 1)['labels'][LABEL_CONFIG_HASH] == ( + config_hash + ) + def test_remove_image_none(self): web = Service('web', image='example', client=self.mock_client) assert not web.remove_image(ImageType.none) From 8ea89efdd86da53c0d1b6ac20b1fc4a5e7c9d3cf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 7 Feb 2018 12:20:45 -0800 Subject: [PATCH 18/47] 1.20.0-dev Signed-off-by: Joffrey F --- CHANGELOG.md | 2 +- compose/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea970a100..4287de493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ Change log ========== -1.19.0 (2018-01-31) +1.19.0 (2018-02-07) ------------------- ### Breaking changes diff --git a/compose/__init__.py b/compose/__init__.py index e5e83434f..f0ad67347 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.19.0-rc3' +__version__ = '1.20.0dev' From 75572f38609b57d637d79c67660a9c0899527f34 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 7 Feb 2018 14:03:31 -0800 Subject: [PATCH 19/47] Immediately kill / force-rm one-off container when receiving SIGHUP Signed-off-by: Joffrey F --- compose/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 9359ec703..4319d0ada 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1274,7 +1274,7 @@ def run_one_off_container(container_options, project, service, options, project_ service.start_container(container) pty.start(sockets) exit_code = container.wait() - except (signals.ShutdownException, signals.HangUpException): + except (signals.ShutdownException): project.client.stop(container.id) exit_code = 1 except (signals.ShutdownException, signals.HangUpException): From cd87d88882f6b10d259f5618e98e5110eea2ba88 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Feb 2018 13:59:27 -0800 Subject: [PATCH 20/47] Add `docker` CLI to the `docker/compose` image Signed-off-by: Joffrey F --- Dockerfile.run | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Dockerfile.run b/Dockerfile.run index a09e57a09..6c532fc14 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -29,6 +29,16 @@ RUN mkdir -p /lib /lib64 /usr/glibc-compat/lib/locale /etc && \ ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 && \ ln -s /usr/glibc-compat/etc/ld.so.cache /etc/ld.so.cache +RUN apk add --no-cache curl && \ + curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.0-ce.tgz" && \ + SHA256=692e1c72937f6214b1038def84463018d8e320c8eaf8530546c84c2f8f9c767d; \ + echo "${SHA256} dockerbins.tgz" | sha256sum -c - && \ + tar xvf dockerbins.tgz docker/docker --strip-components 1 && \ + mv docker /usr/local/bin/docker && \ + chmod +x /usr/local/bin/docker && \ + rm dockerbins.tgz && \ + apk del curl + COPY dist/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose ENTRYPOINT ["docker-compose"] From bb8c2e1f456b1b846dbb8d87c1cb6c83a4ae55b8 Mon Sep 17 00:00:00 2001 From: Brian de Alwis Date: Fri, 9 Feb 2018 11:22:36 -0500 Subject: [PATCH 21/47] Fix bash completion on systems where extglob is not set Signed-off-by: Brian de Alwis --- contrib/completion/bash/docker-compose | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 98e1d6c0d..fc51f604e 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -16,6 +16,8 @@ # below to your .bashrc after bash completion features are loaded # . ~/.docker-compose-completion.sh +__docker_compose_previous_extglob_setting=$(shopt -p extglob) +shopt -s extglob __docker_compose_q() { docker-compose 2>/dev/null "${top_level_options[@]}" "$@" @@ -658,4 +660,7 @@ _docker_compose() { return 0 } +eval "$__docker_compose_previous_extglob_setting" +unset __docker_compose_previous_extglob_setting + complete -F _docker_compose docker-compose docker-compose.exe From 515ea20f25de739d496f2931fe9fd1899301781b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 12 Feb 2018 11:54:08 -0800 Subject: [PATCH 22/47] Fix Dockerfile.run indentation Signed-off-by: Joffrey F --- Dockerfile.run | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Dockerfile.run b/Dockerfile.run index 6c532fc14..b3f9a01f6 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -8,13 +8,13 @@ RUN /builder 2.27 /usr/glibc-compat || true RUN mkdir -p $PKGDIR RUN tar -xf /glibc-bin-2.27.tar.gz -C $PKGDIR RUN rm "$PKGDIR"/usr/glibc-compat/etc/rpc && \ - rm -rf "$PKGDIR"/usr/glibc-compat/bin && \ - rm -rf "$PKGDIR"/usr/glibc-compat/sbin && \ - rm -rf "$PKGDIR"/usr/glibc-compat/lib/gconv && \ - rm -rf "$PKGDIR"/usr/glibc-compat/lib/getconf && \ - rm -rf "$PKGDIR"/usr/glibc-compat/lib/audit && \ - rm -rf "$PKGDIR"/usr/glibc-compat/share && \ - rm -rf "$PKGDIR"/usr/glibc-compat/var + rm -rf "$PKGDIR"/usr/glibc-compat/bin && \ + rm -rf "$PKGDIR"/usr/glibc-compat/sbin && \ + rm -rf "$PKGDIR"/usr/glibc-compat/lib/gconv && \ + rm -rf "$PKGDIR"/usr/glibc-compat/lib/getconf && \ + rm -rf "$PKGDIR"/usr/glibc-compat/lib/audit && \ + rm -rf "$PKGDIR"/usr/glibc-compat/share && \ + rm -rf "$PKGDIR"/usr/glibc-compat/var FROM alpine:3.6 @@ -23,11 +23,11 @@ RUN apk update && apk add --no-cache openssl ca-certificates COPY --from=glibc /pkgdata/ / RUN mkdir -p /lib /lib64 /usr/glibc-compat/lib/locale /etc && \ - ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ + ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ - ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 && \ - ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 && \ - ln -s /usr/glibc-compat/etc/ld.so.cache /etc/ld.so.cache + ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 && \ + ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 && \ + ln -s /usr/glibc-compat/etc/ld.so.cache /etc/ld.so.cache RUN apk add --no-cache curl && \ curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.0-ce.tgz" && \ From 981df93f121ee4a8342f26d5b6441728427c1e5d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 12 Feb 2018 12:05:45 -0800 Subject: [PATCH 23/47] Keep CONTRIBUTING.md information up to date Signed-off-by: Joffrey F --- CONTRIBUTING.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a031e2d68..5bf7cb131 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,11 @@ To run the style checks at any time run `tox -e pre-commit`. ## Submitting a pull request -See Docker's [basic contribution workflow](https://docs.docker.com/opensource/workflow/make-a-contribution/#the-basic-contribution-workflow) for a guide on how to submit a pull request for code or documentation. +See Docker's [basic contribution workflow](https://docs.docker.com/v17.06/opensource/code/#code-contribution-workflow) for a guide on how to submit a pull request for code. + +## Documentation changes + +Issues and pull requests to update the documentation should be submitted to the [docs repo](https://github.com/docker/docker.github.io). You can learn more about contributing to the documentation [here](https://docs.docker.com/opensource/#how-to-contribute-to-the-docs). ## Running the test suite @@ -69,6 +73,4 @@ you can specify a test directory, file, module, class or method: ## Finding things to work on -We use a [ZenHub board](https://www.zenhub.io/) to keep track of specific things we are working on and planning to work on. If you're looking for things to work on, stuff in the backlog is a great place to start. - -For more information about our project planning, take a look at our [GitHub wiki](https://github.com/docker/compose/wiki). +[Issues marked with the `exp/beginner` label](https://github.com/docker/compose/issues?q=is%3Aopen+is%3Aissue+label%3Aexp%2Fbeginner) are a good starting point for people looking to make their first contribution to the project. From bb16d9e951afbd62a9f22d02b2eb613c1ac3ef4c Mon Sep 17 00:00:00 2001 From: Sorawis Nilparuk Date: Sat, 3 Feb 2018 13:14:27 -0800 Subject: [PATCH 24/47] Add health string generator to match `docker ps` output --- compose/container.py | 14 +++++++- tests/unit/container_test.py | 67 ++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/compose/container.py b/compose/container.py index 9323b1192..0c2ca9902 100644 --- a/compose/container.py +++ b/compose/container.py @@ -129,7 +129,7 @@ class Container(object): if self.is_restarting: return 'Restarting' if self.is_running: - return 'Ghost' if self.get('State.Ghost') else 'Up' + return 'Ghost' if self.get('State.Ghost') else self.human_readable_health_status else: return 'Exit %s' % self.get('State.ExitCode') @@ -172,6 +172,18 @@ class Container(object): log_type = self.log_driver return not log_type or log_type in ('json-file', 'journald') + @property + def human_readable_health_status(self): + """ Generate UP status string with up time and health + """ + status_string = 'Up' + container_status = self.get('State.Health.Status') + if container_status == 'starting': + status_string += ' (health: starting)' + elif container_status is not None: + status_string += ' (%s)' % container_status + return status_string + def attach_log_stream(self): """A log stream can only be attached if the container uses a json-file log driver. diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 0fcf23fa6..d64263c1f 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -129,6 +129,73 @@ class ContainerTest(unittest.TestCase): assert container.get_local_port(45454, protocol='tcp') == '0.0.0.0:49197' + def test_human_readable_states_no_health(self): + container = Container(None, { + "State": { + "Status": "running", + "Running": True, + "Paused": False, + "Restarting": False, + "OOMKilled": False, + "Dead": False, + "Pid": 7623, + "ExitCode": 0, + "Error": "", + "StartedAt": "2018-01-29T00:34:25.2052414Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + }, has_been_inspected=True) + expected = "Up" + assert container.human_readable_state == expected + + def test_human_readable_states_starting(self): + container = Container(None, { + "State": { + "Status": "running", + "Running": True, + "Paused": False, + "Restarting": False, + "OOMKilled": False, + "Dead": False, + "Pid": 11744, + "ExitCode": 0, + "Error": "", + "StartedAt": "2018-02-03T07:56:20.3591233Z", + "FinishedAt": "2018-01-31T08:56:11.0505228Z", + "Health": { + "Status": "starting", + "FailingStreak": 0, + "Log": [] + } + } + }, has_been_inspected=True) + expected = "Up (health: starting)" + assert container.human_readable_state == expected + + def test_human_readable_states_healthy(self): + container = Container(None, { + "State": { + "Status": "running", + "Running": True, + "Paused": False, + "Restarting": False, + "OOMKilled": False, + "Dead": False, + "Pid": 5674, + "ExitCode": 0, + "Error": "", + "StartedAt": "2018-02-03T08:32:05.3281831Z", + "FinishedAt": "2018-02-03T08:11:35.7872706Z", + "Health": { + "Status": "healthy", + "FailingStreak": 0, + "Log": [] + } + } + }, has_been_inspected=True) + expected = "Up (healthy)" + assert container.human_readable_state == expected + def test_get(self): container = Container(None, { "Status": "Up 8 seconds", From 64b466c0bc56a871aee6c34e527789d4a5a658ed Mon Sep 17 00:00:00 2001 From: kcboschert Date: Wed, 27 Jan 2016 05:02:33 +0000 Subject: [PATCH 25/47] Add optional argument to pull dependencies on docker-compose pull. Signed-off-by: Kevin Boschert --- compose/cli/main.py | 2 ++ compose/project.py | 5 +++-- tests/acceptance/cli_test.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 4319d0ada..edc07d426 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -677,12 +677,14 @@ class TopLevelCommand(object): --ignore-pull-failures Pull what it can and ignores images with pull failures. --parallel Pull multiple images in parallel. -q, --quiet Pull without printing progress information + --include-deps Also pull services declared as dependencies """ self.project.pull( service_names=options['SERVICE'], ignore_pull_failures=options.get('--ignore-pull-failures'), parallel_pull=options.get('--parallel'), silent=options.get('--quiet'), + include_deps=options.get('--include-deps'), ) def push(self, options): diff --git a/compose/project.py b/compose/project.py index 1880f39ad..5c7244493 100644 --- a/compose/project.py +++ b/compose/project.py @@ -537,8 +537,9 @@ class Project(object): return plans - def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False): - services = self.get_services(service_names, include_deps=False) + def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False, + include_deps=False): + services = self.get_services(service_names, include_deps) if parallel_pull: def pull_service(service): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 9dd03b72f..134485c40 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -567,6 +567,21 @@ class CLITestCase(DockerClientTestCase): result.stderr ) + def test_pull_with_no_deps(self): + self.base_dir = 'tests/fixtures/links-composefile' + result = self.dispatch(['pull', 'web']) + assert sorted(result.stderr.split('\n'))[1:] == [ + 'Pulling web (busybox:latest)...', + ] + + def test_pull_with_include_deps(self): + self.base_dir = 'tests/fixtures/links-composefile' + result = self.dispatch(['pull', '--include-deps', 'web']) + assert sorted(result.stderr.split('\n'))[1:] == [ + 'Pulling db (busybox:latest)...', + 'Pulling web (busybox:latest)...', + ] + def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) From 51076b5e1280aa316b4a3eaf7ab636357ed467e1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 16 Feb 2018 16:15:55 -0800 Subject: [PATCH 26/47] Tests for compatibility mode Signed-off-by: Joffrey F --- compose/cli/main.py | 15 ++-- compose/config/config.py | 3 +- compose/config/config_schema_v3.6.json | 2 +- tests/acceptance/cli_test.py | 34 +++++++-- .../compatibility-mode/docker-compose.yml | 22 ++++++ tests/unit/config/config_test.py | 76 +++++++++++++++++++ 6 files changed, 138 insertions(+), 14 deletions(-) create mode 100644 tests/fixtures/compatibility-mode/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 9c76a561b..56abf4b48 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -168,8 +168,10 @@ class TopLevelCommand(object): docker-compose -h|--help Options: - -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) - -p, --project-name NAME Specify an alternate project name (default: directory name) + -f, --file FILE Specify an alternate compose file + (default: docker-compose.yml) + -p, --project-name NAME Specify an alternate project name + (default: directory name) --verbose Show more output --no-ansi Do not print ANSI control characters -v, --version Print version and exit @@ -180,13 +182,12 @@ class TopLevelCommand(object): --tlscert CLIENT_CERT_PATH Path to TLS certificate file --tlskey TLS_KEY_PATH Path to TLS key file --tlsverify Use TLS and verify the remote - --skip-hostname-check Don't check the daemon's hostname against the name specified - in the client certificate (for example if your docker host - is an IP address) + --skip-hostname-check Don't check the daemon's hostname against the + name specified in the client certificate --project-directory PATH Specify an alternate working directory (default: the path of the Compose file) - --compatibility If set, Compose will attempt to convert deploy keys in v3 - files to their non-Swarm equivalent + --compatibility If set, Compose will attempt to convert deploy + keys in v3 files to their non-Swarm equivalent Commands: build Build or rebuild services diff --git a/compose/config/config.py b/compose/config/config.py index 58420d157..b7764dd3b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -896,6 +896,7 @@ def translate_resource_keys_to_container_config(resources_dict, service_dict): service_dict['mem_reservation'] = resources_dict['reservations'].get('memory') if 'cpus' in resources_dict['reservations']: return ['resources.reservations.cpus'] + return [] def convert_restart_policy(name): @@ -919,7 +920,7 @@ def translate_deploy_keys_to_container_config(service_dict): if k in deploy_dict ] - if 'replicas' in deploy_dict and deploy_dict.get('mode') == 'replicated': + if 'replicas' in deploy_dict and deploy_dict.get('mode', 'replicated') == 'replicated': service_dict['scale'] = deploy_dict['replicas'] if 'restart_policy' in deploy_dict: diff --git a/compose/config/config_schema_v3.6.json b/compose/config/config_schema_v3.6.json index 8e718780b..95a552b34 100644 --- a/compose/config/config_schema_v3.6.json +++ b/compose/config/config_schema_v3.6.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "id": "config_schema_v3.5.json", + "id": "config_schema_v3.6.json", "type": "object", "required": ["version"], diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ade7d10a9..67d953483 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -395,7 +395,7 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['config']) assert yaml.load(result.stdout) == { - 'version': '3.2', + 'version': '3.5', 'volumes': { 'foobar': { 'labels': { @@ -419,22 +419,25 @@ class CLITestCase(DockerClientTestCase): }, 'resources': { 'limits': { - 'cpus': '0.001', + 'cpus': '0.05', 'memory': '50M', }, 'reservations': { - 'cpus': '0.0001', + 'cpus': '0.01', 'memory': '20M', }, }, 'restart_policy': { - 'condition': 'on_failure', + 'condition': 'on-failure', 'delay': '5s', 'max_attempts': 3, 'window': '120s', }, 'placement': { - 'constraints': ['node=foo'], + 'constraints': [ + 'node.hostname==foo', 'node.role != manager' + ], + 'preferences': [{'spread': 'node.labels.datacenter'}] }, }, @@ -464,6 +467,27 @@ class CLITestCase(DockerClientTestCase): }, } + def test_config_compatibility_mode(self): + self.base_dir = 'tests/fixtures/compatibility-mode' + result = self.dispatch(['--compatibility', 'config']) + + assert yaml.load(result.stdout) == { + 'version': '2.3', + 'volumes': {'foo': {'driver': 'default'}}, + 'services': { + 'foo': { + 'command': '/bin/true', + 'image': 'alpine:3.7', + 'scale': 3, + 'restart': 'always:7', + 'mem_limit': '300M', + 'mem_reservation': '100M', + 'cpus': 0.7, + 'volumes': ['foo:/bar:rw'] + } + } + } + def test_ps(self): self.project.get_service('simple').create_container() result = self.dispatch(['ps']) diff --git a/tests/fixtures/compatibility-mode/docker-compose.yml b/tests/fixtures/compatibility-mode/docker-compose.yml new file mode 100644 index 000000000..aac6fd4cb --- /dev/null +++ b/tests/fixtures/compatibility-mode/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3.5' +services: + foo: + image: alpine:3.7 + command: /bin/true + deploy: + replicas: 3 + restart_policy: + condition: any + max_attempts: 7 + resources: + limits: + memory: 300M + cpus: '0.7' + reservations: + memory: 100M + volumes: + - foo:/bar + +volumes: + foo: + driver: default diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index a33080726..d72fae2f5 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3303,6 +3303,82 @@ class InterpolationTest(unittest.TestCase): assert 'BAR' in warnings[0] assert 'FOO' in warnings[1] + def test_compatibility_mode_warnings(self): + config_details = build_config_details({ + 'version': '3.5', + 'services': { + 'web': { + 'deploy': { + 'labels': ['abc=def'], + 'endpoint_mode': 'dnsrr', + 'update_config': {'max_failure_ratio': 0.4}, + 'placement': {'constraints': ['node.id==deadbeef']}, + 'resources': { + 'reservations': {'cpus': '0.2'} + }, + 'restart_policy': { + 'delay': '2s', + 'window': '12s' + } + }, + 'image': 'busybox' + } + } + }) + + with mock.patch('compose.config.config.log') as log: + config.load(config_details, compatibility=True) + + assert log.warn.call_count == 1 + warn_message = log.warn.call_args[0][0] + assert warn_message.startswith( + 'The following deploy sub-keys are not supported in compatibility mode' + ) + assert 'labels' in warn_message + assert 'endpoint_mode' in warn_message + assert 'update_config' in warn_message + assert 'placement' in warn_message + assert 'resources.reservations.cpus' in warn_message + assert 'restart_policy.delay' in warn_message + assert 'restart_policy.window' in warn_message + + def test_compatibility_mode_load(self): + config_details = build_config_details({ + 'version': '3.5', + 'services': { + 'foo': { + 'image': 'alpine:3.7', + 'deploy': { + 'replicas': 3, + 'restart_policy': { + 'condition': 'any', + 'max_attempts': 7, + }, + 'resources': { + 'limits': {'memory': '300M', 'cpus': '0.7'}, + 'reservations': {'memory': '100M'}, + }, + }, + }, + }, + }) + + with mock.patch('compose.config.config.log') as log: + cfg = config.load(config_details, compatibility=True) + + assert log.warn.call_count == 0 + + service_dict = cfg.services[0] + assert service_dict == { + 'image': 'alpine:3.7', + 'scale': 3, + 'restart': {'MaximumRetryCount': 7, 'Name': 'always'}, + 'mem_limit': '300M', + 'mem_reservation': '100M', + 'cpus': 0.7, + 'name': 'foo' + } + @mock.patch.dict(os.environ) def test_invalid_interpolation(self): with pytest.raises(config.ConfigurationError) as cm: From cd7ccad81ee527582992bbc225d5f485cb5e12bb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Feb 2018 13:24:25 -0800 Subject: [PATCH 27/47] Retrieve certs from default path if not provided explicitly Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 11 +++++++++++ tests/unit/cli/docker_client_test.py | 21 +++++++++++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 818fe63ad..cc8993d7f 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -9,6 +9,7 @@ from docker import APIClient from docker.errors import TLSParameterError from docker.tls import TLSConfig from docker.utils import kwargs_from_env +from docker.utils.config import home_dir from ..config.environment import Environment from ..const import HTTP_TIMEOUT @@ -19,6 +20,10 @@ from .utils import unquote_path log = logging.getLogger(__name__) +def default_cert_path(): + return os.path.join(home_dir(), '.docker') + + def get_tls_version(environment): compose_tls_version = environment.get('COMPOSE_TLS_VERSION', None) if not compose_tls_version: @@ -56,6 +61,12 @@ def tls_config_from_options(options, environment=None): key = os.path.join(cert_path, 'key.pem') ca_cert = os.path.join(cert_path, 'ca.pem') + if verify and not any((ca_cert, cert, key)): + # Default location for cert files is ~/.docker + ca_cert = os.path.join(default_cert_path(), 'ca.pem') + cert = os.path.join(default_cert_path(), 'cert.pem') + key = os.path.join(default_cert_path(), 'key.pem') + tls_version = get_tls_version(environment) advanced_opts = any([ca_cert, cert, key, verify, tls_version]) diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index 5bb4564ef..be91ea31d 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -68,9 +68,10 @@ class DockerClientTestCase(unittest.TestCase): class TLSConfigTestCase(unittest.TestCase): - ca_cert = os.path.join('tests/fixtures/tls/', 'ca.pem') - client_cert = os.path.join('tests/fixtures/tls/', 'cert.pem') - key = os.path.join('tests/fixtures/tls/', 'key.pem') + cert_path = 'tests/fixtures/tls/' + ca_cert = os.path.join(cert_path, 'ca.pem') + client_cert = os.path.join(cert_path, 'cert.pem') + key = os.path.join(cert_path, 'key.pem') def test_simple_tls(self): options = {'--tls': True} @@ -202,7 +203,8 @@ class TLSConfigTestCase(unittest.TestCase): def test_tls_verify_flag_no_override(self): environment = Environment({ 'DOCKER_TLS_VERIFY': 'true', - 'COMPOSE_TLS_VERSION': 'TLSv1' + 'COMPOSE_TLS_VERSION': 'TLSv1', + 'DOCKER_CERT_PATH': self.cert_path }) options = {'--tls': True, '--tlsverify': False} @@ -219,6 +221,17 @@ class TLSConfigTestCase(unittest.TestCase): options = {'--tls': True} assert tls_config_from_options(options, environment) is True + def test_tls_verify_default_cert_path(self): + environment = Environment({'DOCKER_TLS_VERIFY': '1'}) + options = {'--tls': True} + with mock.patch('compose.cli.docker_client.default_cert_path') as dcp: + dcp.return_value = 'tests/fixtures/tls/' + result = tls_config_from_options(options, environment) + assert isinstance(result, docker.tls.TLSConfig) + assert result.verify is True + assert result.ca_cert == self.ca_cert + assert result.cert == (self.client_cert, self.key) + class TestGetTlsVersion(object): def test_get_tls_version_default(self): From ac209a2485296d8b54ad794cd8a307d4b1374a51 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Fri, 22 Sep 2017 12:01:22 -0700 Subject: [PATCH 28/47] [cli] Lift artificial limitation on --build-arg Currently, `docker-compose --build-arg` requires that a service be specified as part of the command-line invocation. So, $ docker-compose build --build-arg nocache=`git rev-parse @` foom works. However, when using out-of-band scripts to automate the build process of several Docker containers (in a CI system, for instance), it becomes difficult to specify exactly which service requires the build-arg. Docker has supported Dockerfiles that ignore build-args for a long time, so there is no problem is specifying spurious build-args to builds that don't consume it. The limitation on `docker-compose build` today is artificial, and there are no other commands that require specifying a service. Allow `--build-arg` to also match all services so this is possible: $ docker-compose build --build-arg nocache=`git rev-parse @` Please refer to #3790 for discussion on the original feature. Signed-off-by: Ramkumar Ramachandra --- compose/cli/main.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index edc07d426..164769bbf 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -234,19 +234,15 @@ class TopLevelCommand(object): --no-cache Do not use cache when building the image. --pull Always attempt to pull a newer version of the image. -m, --memory MEM Sets memory limit for the build container. - --build-arg key=val Set build-time variables for one service. + --build-arg key=val Set build-time variables for services. """ - service_names = options['SERVICE'] build_args = options.get('--build-arg', None) if build_args: environment = Environment.from_env_file(self.project_dir) build_args = resolve_build_args(build_args, environment) - if not service_names and build_args: - raise UserError("Need service name for --build-arg option") - self.project.build( - service_names=service_names, + service_names=options['SERVICE'], no_cache=bool(options.get('--no-cache', False)), pull=bool(options.get('--pull', False)), force_rm=bool(options.get('--force-rm', False)), @@ -1030,7 +1026,8 @@ class TopLevelCommand(object): if cascade_stop: print("Aborting on container exit...") - all_containers = self.project.containers(service_names=options['SERVICE'], stopped=True) + all_containers = self.project.containers( + service_names=options['SERVICE'], stopped=True) exit_code = compute_exit_code( exit_value_from, attached_containers, cascade_starter, all_containers ) From 8e268afc936321753789a28817eb25f0ad619e85 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Feb 2018 15:56:02 -0800 Subject: [PATCH 29/47] Add version guard for multi-service buildarg Add buildarg tests Signed-off-by: Joffrey F --- compose/cli/main.py | 9 +++++-- tests/acceptance/cli_test.py | 27 ++++++++++++++++++++ tests/fixtures/build-args/Dockerfile | 4 +++ tests/fixtures/build-args/docker-compose.yml | 7 +++++ 4 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/build-args/Dockerfile create mode 100644 tests/fixtures/build-args/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 164769bbf..fd7f92a81 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -236,8 +236,14 @@ class TopLevelCommand(object): -m, --memory MEM Sets memory limit for the build container. --build-arg key=val Set build-time variables for services. """ + service_names = options['SERVICE'] build_args = options.get('--build-arg', None) if build_args: + if not service_names and docker.utils.version_lt(self.project.client.api_version, '1.25'): + raise UserError( + '--build-arg is only supported when services are specified for API version < 1.25.' + ' Please use a Compose file version > 2.2 or specify which services to build.' + ) environment = Environment.from_env_file(self.project_dir) build_args = resolve_build_args(build_args, environment) @@ -1026,8 +1032,7 @@ class TopLevelCommand(object): if cascade_stop: print("Aborting on container exit...") - all_containers = self.project.containers( - service_names=options['SERVICE'], stopped=True) + all_containers = self.project.containers(service_names=options['SERVICE'], stopped=True) exit_code = compute_exit_code( exit_value_from, attached_containers, cascade_starter, all_containers ) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 134485c40..125501b0d 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -658,6 +658,33 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['build', '--no-cache', '--memory', '96m', 'service'], None) assert 'memory: 100663296' in result.stdout # 96 * 1024 * 1024 + def test_build_with_buildarg_from_compose_file(self): + pull_busybox(self.client) + self.base_dir = 'tests/fixtures/build-args' + result = self.dispatch(['build'], None) + assert 'Favorite Touhou Character: mariya.kirisame' in result.stdout + + def test_build_with_buildarg_cli_override(self): + pull_busybox(self.client) + self.base_dir = 'tests/fixtures/build-args' + result = self.dispatch(['build', '--build-arg', 'favorite_th_character=sakuya.izayoi'], None) + assert 'Favorite Touhou Character: sakuya.izayoi' in result.stdout + + @mock.patch.dict(os.environ) + def test_build_with_buildarg_old_api_version(self): + pull_busybox(self.client) + self.base_dir = 'tests/fixtures/build-args' + os.environ['COMPOSE_API_VERSION'] = '1.24' + result = self.dispatch( + ['build', '--build-arg', 'favorite_th_character=reimu.hakurei'], None, returncode=1 + ) + assert '--build-arg is only supported when services are specified' in result.stderr + + result = self.dispatch( + ['build', '--build-arg', 'favorite_th_character=hong.meiling', 'web'], None + ) + assert 'Favorite Touhou Character: hong.meiling' in result.stdout + def test_bundle_with_digests(self): self.base_dir = 'tests/fixtures/bundle-with-digests/' tmpdir = pytest.ensuretemp('cli_test_bundle') diff --git a/tests/fixtures/build-args/Dockerfile b/tests/fixtures/build-args/Dockerfile new file mode 100644 index 000000000..93ebcb9cd --- /dev/null +++ b/tests/fixtures/build-args/Dockerfile @@ -0,0 +1,4 @@ +FROM busybox:latest +LABEL com.docker.compose.test_image=true +ARG favorite_th_character +RUN echo "Favorite Touhou Character: ${favorite_th_character}" diff --git a/tests/fixtures/build-args/docker-compose.yml b/tests/fixtures/build-args/docker-compose.yml new file mode 100644 index 000000000..ed60a337b --- /dev/null +++ b/tests/fixtures/build-args/docker-compose.yml @@ -0,0 +1,7 @@ +version: '2.2' +services: + web: + build: + context: . + args: + - favorite_th_character=mariya.kirisame From 18338606143971b5ba64250cb0eca49d1a516c1d Mon Sep 17 00:00:00 2001 From: Gustavo Pantuza Coelho Pinto Date: Tue, 24 Oct 2017 19:24:06 -0200 Subject: [PATCH 30/47] CLI: Add --detach option for up, run and exec commands This contribution allows the usage of --detach option when running: docker-compose run --detach docker-compose exec --detach docker-compose up --detach The behavior is the same as the -d option. Signed-off-by: Gustavo Pantuza Coelho Pinto --- compose/cli/main.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index edc07d426..7b6ca98c4 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -424,7 +424,7 @@ class TopLevelCommand(object): Usage: exec [options] [-e KEY=VAL...] SERVICE COMMAND [ARGS...] Options: - -d Detached mode: Run command in the background. + -d, --detach Detached mode: Run command in the background. --privileged Give extended privileges to the process. -u, --user USER Run the command as this user. -T Disable pseudo-tty allocation. By default `docker-compose exec` @@ -438,7 +438,7 @@ class TopLevelCommand(object): use_cli = not environment.get_boolean('COMPOSE_INTERACTIVE_NO_CLI') index = int(options.get('--index')) service = self.project.get_service(options['SERVICE']) - detach = options['-d'] + detach = options.get('-d') or options.get('--detach') if options['--env'] and docker.utils.version_lt(self.project.client.api_version, '1.25'): raise UserError("Setting environment for exec is not supported in API < 1.25'") @@ -762,7 +762,7 @@ class TopLevelCommand(object): SERVICE [COMMAND] [ARGS...] Options: - -d Detached mode: Run container in the background, print + -d --detach Detached mode: Run container in the background, print new container name. --name NAME Assign a name to the container --entrypoint CMD Override the entrypoint of the image. @@ -780,7 +780,7 @@ class TopLevelCommand(object): -w, --workdir="" Working directory inside the container """ service = self.project.get_service(options['SERVICE']) - detach = options['-d'] + detach = options.get('-d') or options.get('--detach') if options['--publish'] and options['--service-ports']: raise UserError( @@ -928,7 +928,7 @@ class TopLevelCommand(object): Usage: up [options] [--scale SERVICE=NUM...] [SERVICE...] Options: - -d Detached mode: Run containers in the background, + -d, --detach Detached mode: Run containers in the background, print new container names. Incompatible with --abort-on-container-exit. --no-color Produce monochrome output. @@ -963,7 +963,7 @@ class TopLevelCommand(object): service_names = options['SERVICE'] timeout = timeout_from_opts(options) remove_orphans = options['--remove-orphans'] - detached = options.get('-d') + detached = options.get('-d') or options.get('--detach') no_start = options.get('--no-start') if detached and (cascade_stop or exit_value_from): @@ -975,7 +975,7 @@ class TopLevelCommand(object): if ignore_orphans and remove_orphans: raise UserError("COMPOSE_IGNORE_ORPHANS and --remove-orphans cannot be combined.") - opts = ['-d', '--abort-on-container-exit', '--exit-code-from'] + opts = ['--detach', '--abort-on-container-exit', '--exit-code-from'] for excluded in [x for x in opts if options.get(x) and no_start]: raise UserError('--no-start and {} cannot be combined.'.format(excluded)) @@ -1245,7 +1245,7 @@ def run_one_off_container(container_options, project, service, options, project_ one_off=True, **container_options) - if options['-d']: + if options.get('-d') or options.get('--detach'): service.start_container(container) print(container.name) return From c6fe564ed579d8150efa2ec5613734de09d8dc45 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Feb 2018 16:42:49 -0800 Subject: [PATCH 31/47] Add --detach tests Signed-off-by: Joffrey F --- compose/cli/main.py | 12 ++++++------ tests/acceptance/cli_test.py | 22 ++++++++++++++++++++++ tests/unit/cli_test.py | 8 ++++---- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 7b6ca98c4..460786489 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -438,7 +438,7 @@ class TopLevelCommand(object): use_cli = not environment.get_boolean('COMPOSE_INTERACTIVE_NO_CLI') index = int(options.get('--index')) service = self.project.get_service(options['SERVICE']) - detach = options.get('-d') or options.get('--detach') + detach = options.get('--detach') if options['--env'] and docker.utils.version_lt(self.project.client.api_version, '1.25'): raise UserError("Setting environment for exec is not supported in API < 1.25'") @@ -762,7 +762,7 @@ class TopLevelCommand(object): SERVICE [COMMAND] [ARGS...] Options: - -d --detach Detached mode: Run container in the background, print + -d, --detach Detached mode: Run container in the background, print new container name. --name NAME Assign a name to the container --entrypoint CMD Override the entrypoint of the image. @@ -780,7 +780,7 @@ class TopLevelCommand(object): -w, --workdir="" Working directory inside the container """ service = self.project.get_service(options['SERVICE']) - detach = options.get('-d') or options.get('--detach') + detach = options.get('--detach') if options['--publish'] and options['--service-ports']: raise UserError( @@ -963,7 +963,7 @@ class TopLevelCommand(object): service_names = options['SERVICE'] timeout = timeout_from_opts(options) remove_orphans = options['--remove-orphans'] - detached = options.get('-d') or options.get('--detach') + detached = options.get('--detach') no_start = options.get('--no-start') if detached and (cascade_stop or exit_value_from): @@ -1245,7 +1245,7 @@ def run_one_off_container(container_options, project, service, options, project_ one_off=True, **container_options) - if options.get('-d') or options.get('--detach'): + if options.get('--detach'): service.start_container(container) print(container.name) return @@ -1372,7 +1372,7 @@ def parse_scale_args(options): def build_exec_command(options, container_id, command): args = ["exec"] - if options["-d"]: + if options["--detach"]: args += ["--detach"] else: args += ["--interactive"] diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 134485c40..63122e743 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -884,6 +884,19 @@ class CLITestCase(DockerClientTestCase): assert not container.get('Config.AttachStdout') assert not container.get('Config.AttachStdin') + def test_up_detached_long_form(self): + self.dispatch(['up', '--detach']) + service = self.project.get_service('simple') + another = self.project.get_service('another') + assert len(service.containers()) == 1 + assert len(another.containers()) == 1 + + # Ensure containers don't have stdin and stdout connected in -d mode + container, = service.containers() + assert not container.get('Config.AttachStderr') + assert not container.get('Config.AttachStdout') + assert not container.get('Config.AttachStdin') + def test_up_attached(self): self.base_dir = 'tests/fixtures/echo-services' result = self.dispatch(['up', '--no-color']) @@ -1463,6 +1476,15 @@ class CLITestCase(DockerClientTestCase): assert stderr == "" assert stdout == "/\n" + def test_exec_detach_long_form(self): + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['up', '--detach', 'console']) + assert len(self.project.containers()) == 1 + + stdout, stderr = self.dispatch(['exec', '-T', 'console', 'ls', '-1d', '/']) + assert stderr == "" + assert stdout == "/\n" + def test_exec_custom_user(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', 'console']) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index d078614e6..6399bef89 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -119,7 +119,7 @@ class CLITestCase(unittest.TestCase): '--label': [], '--user': None, '--no-deps': None, - '-d': False, + '--detach': False, '-T': None, '--entrypoint': None, '--service-ports': None, @@ -156,7 +156,7 @@ class CLITestCase(unittest.TestCase): '--label': [], '--user': None, '--no-deps': None, - '-d': True, + '--detach': True, '-T': None, '--entrypoint': None, '--service-ports': None, @@ -177,7 +177,7 @@ class CLITestCase(unittest.TestCase): '--label': [], '--user': None, '--no-deps': None, - '-d': True, + '--detach': True, '-T': None, '--entrypoint': None, '--service-ports': None, @@ -208,7 +208,7 @@ class CLITestCase(unittest.TestCase): '--label': [], '--user': None, '--no-deps': None, - '-d': True, + '--detach': True, '-T': None, '--entrypoint': None, '--service-ports': True, From c16820eca093f2bbf573c31f7390220565aa5937 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Feb 2018 16:45:29 -0800 Subject: [PATCH 32/47] Update bash completion Signed-off-by: Joffrey F --- contrib/completion/bash/docker-compose | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index fc51f604e..29853b089 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -245,7 +245,7 @@ _docker_compose_exec() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --help --index --privileged -T --user -u" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "-d --detach --help --index --privileged -T --user -u" -- "$cur" ) ) ;; *) __docker_compose_services_running @@ -444,7 +444,7 @@ _docker_compose_run() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --label -l --name --no-deps --publish -p --rm --service-ports -T --user -u --volume -v --workdir -w" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "-d --detach --entrypoint -e --help --label -l --name --no-deps --publish -p --rm --service-ports -T --user -u --volume -v --workdir -w" -- "$cur" ) ) ;; *) __docker_compose_services_all @@ -552,7 +552,7 @@ _docker_compose_up() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--abort-on-container-exit --always-recreate-deps --build -d --exit-code-from --force-recreate --help --no-build --no-color --no-deps --no-recreate --no-start --renew-anon-volumes -V --remove-orphans --scale --timeout -t" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--abort-on-container-exit --always-recreate-deps --build -d --detach --exit-code-from --force-recreate --help --no-build --no-color --no-deps --no-recreate --no-start --renew-anon-volumes -V --remove-orphans --scale --timeout -t" -- "$cur" ) ) ;; *) __docker_compose_services_all From ad683b2d8d83f6c4df9500fef243c0b5d2bfe3cb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Feb 2018 15:18:18 -0800 Subject: [PATCH 33/47] Bump SDK version Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6b1df1916..da05e4212 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.0.1 +docker==3.1.0 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index 264d647f7..d1788df0a 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.0.1, < 4.0', + 'docker >= 3.1.0, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From d4106679a6883addea940dcc8bbf424b90eed023 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Feb 2018 16:18:44 -0800 Subject: [PATCH 34/47] Use configfile-provided proxy values to populate buildargs and env values Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 5 ++++- compose/service.py | 36 ++++++++++++++++++++++++++++++++++-- tests/unit/cli_test.py | 2 ++ tests/unit/project_test.py | 1 + tests/unit/service_test.py | 31 ++++++++++++++++++++----------- 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index cc8993d7f..a4609780f 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -117,4 +117,7 @@ def docker_client(environment, version=None, tls_config=None, host=None, kwargs['user_agent'] = generate_user_agent() - return APIClient(**kwargs) + client = APIClient(**kwargs) + client._original_base_url = kwargs.get('base_url') + + return client diff --git a/compose/service.py b/compose/service.py index f79a46962..50f9fd71b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -793,8 +793,12 @@ class Service(object): )) container_options['environment'] = merge_environment( - self.options.get('environment'), - override_options.get('environment')) + self._parse_proxy_config(), + merge_environment( + self.options.get('environment'), + override_options.get('environment') + ) + ) container_options['labels'] = merge_labels( self.options.get('labels'), @@ -963,6 +967,9 @@ class Service(object): if build_args_override: build_args.update(build_args_override) + for k, v in self._parse_proxy_config().items(): + build_args.setdefault(k, v) + # python2 os.stat() doesn't support unicode on some UNIX, so we # encode it to a bytestring to be safe path = build_opts.get('context') @@ -1142,6 +1149,31 @@ class Service(object): raise HealthCheckFailed(ctnr.short_id) return result + def _parse_proxy_config(self): + client = self.client + if 'proxies' not in client._general_configs: + return {} + docker_host = getattr(client, '_original_base_url', client.base_url) + proxy_config = client._general_configs['proxies'].get( + docker_host, client._general_configs['proxies'].get('default') + ) or {} + + permitted = { + 'ftpProxy': 'FTP_PROXY', + 'httpProxy': 'HTTP_PROXY', + 'httpsProxy': 'HTTPS_PROXY', + 'noProxy': 'NO_PROXY', + } + + result = {} + + for k, v in proxy_config.items(): + if k not in permitted: + continue + result[permitted[k]] = result[permitted[k].lower()] = v + + return result + def short_id_alias_exists(container, network): aliases = container.get( diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 6399bef89..cef53740d 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -102,6 +102,7 @@ class CLITestCase(unittest.TestCase): os.environ['COMPOSE_INTERACTIVE_NO_CLI'] = 'true' mock_client = mock.create_autospec(docker.APIClient) mock_client.api_version = DEFAULT_DOCKER_API_VERSION + mock_client._general_configs = {} project = Project.from_config( name='composetest', client=mock_client, @@ -136,6 +137,7 @@ class CLITestCase(unittest.TestCase): def test_run_service_with_restart_always(self): mock_client = mock.create_autospec(docker.APIClient) mock_client.api_version = DEFAULT_DOCKER_API_VERSION + mock_client._general_configs = {} project = Project.from_config( name='composetest', diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index a0291d9f9..b4994a99e 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -24,6 +24,7 @@ from compose.service import Service class ProjectTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client._general_configs = {} def test_from_config_v1(self): config = Config( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 002ae0c0d..884598b60 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -43,6 +43,7 @@ class ServiceTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION + self.mock_client._general_configs = {} def test_containers(self): service = Service('db', self.mock_client, 'myproject', image='foo') @@ -743,13 +744,16 @@ class ServiceTest(unittest.TestCase): 'for this service are created on a single host, the port will clash.'.format(name)) -class TestServiceNetwork(object): +class TestServiceNetwork(unittest.TestCase): + def setUp(self): + self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION + self.mock_client._general_configs = {} def test_connect_container_to_networks_short_aliase_exists(self): - mock_client = mock.create_autospec(docker.APIClient) service = Service( 'db', - mock_client, + self.mock_client, 'myproject', image='foo', networks={'project_default': {}}) @@ -768,8 +772,8 @@ class TestServiceNetwork(object): True) service.connect_container_to_networks(container) - assert not mock_client.disconnect_container_from_network.call_count - assert not mock_client.connect_container_to_network.call_count + assert not self.mock_client.disconnect_container_from_network.call_count + assert not self.mock_client.connect_container_to_network.call_count def sort_by_name(dictionary_list): @@ -814,6 +818,10 @@ class BuildUlimitsTestCase(unittest.TestCase): class NetTestCase(unittest.TestCase): + def setUp(self): + self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION + self.mock_client._general_configs = {} def test_network_mode(self): network_mode = NetworkMode('host') @@ -831,12 +839,11 @@ class NetTestCase(unittest.TestCase): def test_network_mode_service(self): container_id = 'bbbb' service_name = 'web' - mock_client = mock.create_autospec(docker.APIClient) - mock_client.containers.return_value = [ + self.mock_client.containers.return_value = [ {'Id': container_id, 'Name': container_id, 'Image': 'abcd'}, ] - service = Service(name=service_name, client=mock_client) + service = Service(name=service_name, client=self.mock_client) network_mode = ServiceNetworkMode(service) assert network_mode.id == service_name @@ -845,10 +852,9 @@ class NetTestCase(unittest.TestCase): def test_network_mode_service_no_containers(self): service_name = 'web' - mock_client = mock.create_autospec(docker.APIClient) - mock_client.containers.return_value = [] + self.mock_client.containers.return_value = [] - service = Service(name=service_name, client=mock_client) + service = Service(name=service_name, client=self.mock_client) network_mode = ServiceNetworkMode(service) assert network_mode.id == service_name @@ -884,6 +890,7 @@ class ServiceVolumesTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION + self.mock_client._general_configs = {} def test_build_volume_binding(self): binding = build_volume_binding(VolumeSpec.parse('/outside:/inside', True)) @@ -1118,6 +1125,8 @@ class ServiceVolumesTest(unittest.TestCase): class ServiceSecretTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION + self.mock_client._general_configs = {} def test_get_secret_volumes(self): secret1 = { From 5eefa81f9e23e98d02e2e5df8e74378c4713a9ec Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Thu, 22 Feb 2018 11:30:51 +0100 Subject: [PATCH 35/47] Add '--quiet' option to 'up' to pull silently. Signed-off-by: Matthieu Nottale --- compose/cli/main.py | 4 +++- compose/project.py | 6 ++++-- compose/service.py | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 39178fb3c..b0ffd2f6c 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -934,6 +934,7 @@ class TopLevelCommand(object): print new container names. Incompatible with --abort-on-container-exit. --no-color Produce monochrome output. + --quiet-pull Pull without printing progress information --no-deps Don't start linked services. --force-recreate Recreate containers even if their configuration and image haven't changed. @@ -998,7 +999,8 @@ class TopLevelCommand(object): start=not no_start, always_recreate_deps=always_recreate_deps, reset_container_image=rebuild, - renew_anonymous_volumes=options.get('--renew-anon-volumes') + renew_anonymous_volumes=options.get('--renew-anon-volumes'), + silent=options.get('--quiet-pull'), ) try: diff --git a/compose/project.py b/compose/project.py index 5c7244493..2cbc4aeea 100644 --- a/compose/project.py +++ b/compose/project.py @@ -446,7 +446,9 @@ class Project(object): start=True, always_recreate_deps=False, reset_container_image=False, - renew_anonymous_volumes=False): + renew_anonymous_volumes=False, + silent=False, + ): self.initialize() if not ignore_orphans: @@ -460,7 +462,7 @@ class Project(object): include_deps=start_deps) for svc in services: - svc.ensure_image_exists(do_build=do_build) + svc.ensure_image_exists(do_build=do_build, silent=silent) plans = self._get_convergence_plans( services, strategy, always_recreate_deps=always_recreate_deps) scaled_services = self.get_scaled_services(services, scale_override) diff --git a/compose/service.py b/compose/service.py index f79a46962..3918a19e8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -305,7 +305,7 @@ class Service(object): raise OperationFailedError("Cannot create container for service %s: %s" % (self.name, ex.explanation)) - def ensure_image_exists(self, do_build=BuildAction.none): + def ensure_image_exists(self, do_build=BuildAction.none, silent=False): if self.can_be_built() and do_build == BuildAction.force: self.build() return @@ -317,7 +317,7 @@ class Service(object): pass if not self.can_be_built(): - self.pull() + self.pull(silent=silent) return if do_build == BuildAction.skip: From 59b08c7d1db4b1105481ab74bbf788b7d7f24fd2 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Thu, 22 Feb 2018 13:47:51 +0100 Subject: [PATCH 36/47] New --log-level option. Signed-off-by: Matthieu Nottale --- compose/cli/main.py | 27 +++++++++++++++++++++++---- tests/acceptance/cli_test.py | 14 ++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 39178fb3c..71aa58c5d 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -100,7 +100,10 @@ def dispatch(): {'options_first': True, 'version': get_version_info('compose')}) options, handler, command_options = dispatcher.parse(sys.argv[1:]) - setup_console_handler(console_handler, options.get('--verbose'), options.get('--no-ansi')) + setup_console_handler(console_handler, + options.get('--verbose'), + options.get('--no-ansi'), + options.get("--log-level")) setup_parallel_logger(options.get('--no-ansi')) if options.get('--no-ansi'): command_options['--no-color'] = True @@ -139,7 +142,7 @@ def setup_parallel_logger(noansi): compose.parallel.ParallelStreamWriter.set_noansi() -def setup_console_handler(handler, verbose, noansi=False): +def setup_console_handler(handler, verbose, noansi=False, level=None): if handler.stream.isatty() and noansi is False: format_class = ConsoleWarningFormatter else: @@ -147,10 +150,25 @@ def setup_console_handler(handler, verbose, noansi=False): if verbose: handler.setFormatter(format_class('%(name)s.%(funcName)s: %(message)s')) - handler.setLevel(logging.DEBUG) + loglevel = logging.DEBUG else: handler.setFormatter(format_class()) - handler.setLevel(logging.INFO) + loglevel = logging.INFO + + if level is not None: + levels = { + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRITICAL': logging.CRITICAL, + } + loglevel = levels.get(level.upper()) + if loglevel is None: + raise UserError('Invalid value for --log-level. Expected one of ' + + 'DEBUG, INFO, WARNING, ERROR, CRITICAL.') + + handler.setLevel(loglevel) # stolen from docopt master @@ -171,6 +189,7 @@ class TopLevelCommand(object): -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) -p, --project-name NAME Specify an alternate project name (default: directory name) --verbose Show more output + --log-level LEVEL Set log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) --no-ansi Do not print ANSI control characters -v, --version Print version and exit -H, --host HOST Daemon socket to connect to diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 66857307b..796861577 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -619,6 +619,20 @@ class CLITestCase(DockerClientTestCase): assert BUILD_CACHE_TEXT not in result.stdout assert BUILD_PULL_TEXT in result.stdout + def test_build_log_level(self): + self.base_dir = 'tests/fixtures/simple-dockerfile' + result = self.dispatch(['--log-level', 'warning', 'build', 'simple']) + assert result.stderr == '' + result = self.dispatch(['--log-level', 'debug', 'build', 'simple']) + assert 'Building simple' in result.stderr + assert 'Using configuration file' in result.stderr + self.base_dir = 'tests/fixtures/simple-failing-dockerfile' + result = self.dispatch(['--log-level', 'critical', 'build', 'simple'], returncode=1) + assert result.stderr == '' + result = self.dispatch(['--log-level', 'debug', 'build', 'simple'], returncode=1) + assert 'Building simple' in result.stderr + assert 'non-zero code' in result.stderr + def test_build_failed(self): self.base_dir = 'tests/fixtures/simple-failing-dockerfile' self.dispatch(['build', 'simple'], returncode=1) From 5e4700b176bf36eaece1cfb6bb5c75a03d069d3f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Feb 2018 14:32:43 -0800 Subject: [PATCH 37/47] Add proxy_config tests Signed-off-by: Joffrey F --- tests/unit/service_test.py | 143 +++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 884598b60..c315dcc4d 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -25,6 +25,7 @@ from compose.service import build_ulimits from compose.service import build_volume_binding from compose.service import BuildAction from compose.service import ContainerNetworkMode +from compose.service import format_environment from compose.service import formatted_ports from compose.service import get_container_data_volumes from compose.service import ImageType @@ -743,6 +744,148 @@ class ServiceTest(unittest.TestCase): 'The "{}" service specifies a port on the host. If multiple containers ' 'for this service are created on a single host, the port will clash.'.format(name)) + def test_parse_proxy_config(self): + default_proxy_config = { + 'httpProxy': 'http://proxy.mycorp.com:3128', + 'httpsProxy': 'https://user:password@proxy.mycorp.com:3129', + 'ftpProxy': 'http://ftpproxy.mycorp.com:21', + 'noProxy': '*.intra.mycorp.com', + } + + self.mock_client.base_url = 'http+docker://localunixsocket' + self.mock_client._general_configs = { + 'proxies': { + 'default': default_proxy_config, + } + } + + service = Service('foo', client=self.mock_client) + + assert service._parse_proxy_config() == { + 'HTTP_PROXY': default_proxy_config['httpProxy'], + 'http_proxy': default_proxy_config['httpProxy'], + 'HTTPS_PROXY': default_proxy_config['httpsProxy'], + 'https_proxy': default_proxy_config['httpsProxy'], + 'FTP_PROXY': default_proxy_config['ftpProxy'], + 'ftp_proxy': default_proxy_config['ftpProxy'], + 'NO_PROXY': default_proxy_config['noProxy'], + 'no_proxy': default_proxy_config['noProxy'], + } + + def test_parse_proxy_config_per_host(self): + default_proxy_config = { + 'httpProxy': 'http://proxy.mycorp.com:3128', + 'httpsProxy': 'https://user:password@proxy.mycorp.com:3129', + 'ftpProxy': 'http://ftpproxy.mycorp.com:21', + 'noProxy': '*.intra.mycorp.com', + } + host_specific_proxy_config = { + 'httpProxy': 'http://proxy.example.com:3128', + 'httpsProxy': 'https://user:password@proxy.example.com:3129', + 'ftpProxy': 'http://ftpproxy.example.com:21', + 'noProxy': '*.intra.example.com' + } + + self.mock_client.base_url = 'http+docker://localunixsocket' + self.mock_client._general_configs = { + 'proxies': { + 'default': default_proxy_config, + 'tcp://example.docker.com:2376': host_specific_proxy_config, + } + } + + service = Service('foo', client=self.mock_client) + + assert service._parse_proxy_config() == { + 'HTTP_PROXY': default_proxy_config['httpProxy'], + 'http_proxy': default_proxy_config['httpProxy'], + 'HTTPS_PROXY': default_proxy_config['httpsProxy'], + 'https_proxy': default_proxy_config['httpsProxy'], + 'FTP_PROXY': default_proxy_config['ftpProxy'], + 'ftp_proxy': default_proxy_config['ftpProxy'], + 'NO_PROXY': default_proxy_config['noProxy'], + 'no_proxy': default_proxy_config['noProxy'], + } + + self.mock_client._original_base_url = 'tcp://example.docker.com:2376' + + assert service._parse_proxy_config() == { + 'HTTP_PROXY': host_specific_proxy_config['httpProxy'], + 'http_proxy': host_specific_proxy_config['httpProxy'], + 'HTTPS_PROXY': host_specific_proxy_config['httpsProxy'], + 'https_proxy': host_specific_proxy_config['httpsProxy'], + 'FTP_PROXY': host_specific_proxy_config['ftpProxy'], + 'ftp_proxy': host_specific_proxy_config['ftpProxy'], + 'NO_PROXY': host_specific_proxy_config['noProxy'], + 'no_proxy': host_specific_proxy_config['noProxy'], + } + + def test_build_service_with_proxy_config(self): + default_proxy_config = { + 'httpProxy': 'http://proxy.mycorp.com:3128', + 'httpsProxy': 'https://user:password@proxy.example.com:3129', + } + buildargs = { + 'HTTPS_PROXY': 'https://rdcf.th08.jp:8911', + 'https_proxy': 'https://rdcf.th08.jp:8911', + } + self.mock_client._general_configs = { + 'proxies': { + 'default': default_proxy_config, + } + } + self.mock_client.base_url = 'http+docker://localunixsocket' + self.mock_client.build.return_value = [ + b'{"stream": "Successfully built 12345"}', + ] + + service = Service('foo', client=self.mock_client, build={'context': '.', 'args': buildargs}) + service.build() + + assert self.mock_client.build.call_count == 1 + assert self.mock_client.build.call_args[1]['buildargs'] == { + 'HTTP_PROXY': default_proxy_config['httpProxy'], + 'http_proxy': default_proxy_config['httpProxy'], + 'HTTPS_PROXY': buildargs['HTTPS_PROXY'], + 'https_proxy': buildargs['HTTPS_PROXY'], + } + + def test_get_create_options_with_proxy_config(self): + default_proxy_config = { + 'httpProxy': 'http://proxy.mycorp.com:3128', + 'httpsProxy': 'https://user:password@proxy.mycorp.com:3129', + 'ftpProxy': 'http://ftpproxy.mycorp.com:21', + } + self.mock_client._general_configs = { + 'proxies': { + 'default': default_proxy_config, + } + } + self.mock_client.base_url = 'http+docker://localunixsocket' + + override_options = { + 'environment': { + 'FTP_PROXY': 'ftp://xdge.exo.au:21', + 'ftp_proxy': 'ftp://xdge.exo.au:21', + } + } + environment = { + 'HTTPS_PROXY': 'https://rdcf.th08.jp:8911', + 'https_proxy': 'https://rdcf.th08.jp:8911', + } + + service = Service('foo', client=self.mock_client, environment=environment) + + create_opts = service._get_container_create_options(override_options, 1) + assert set(create_opts['environment']) == set(format_environment({ + 'HTTP_PROXY': default_proxy_config['httpProxy'], + 'http_proxy': default_proxy_config['httpProxy'], + 'HTTPS_PROXY': environment['HTTPS_PROXY'], + 'https_proxy': environment['HTTPS_PROXY'], + 'FTP_PROXY': override_options['environment']['FTP_PROXY'], + 'ftp_proxy': override_options['environment']['FTP_PROXY'], + })) + class TestServiceNetwork(unittest.TestCase): def setUp(self): From a6c31b80fefc68e64c3ad57abb6f64541460453d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Feb 2018 18:32:45 -0800 Subject: [PATCH 38/47] Add support for seccomp files Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 2 +- compose/cli/utils.py | 8 ------- compose/config/config.py | 15 +++++++++++-- compose/config/serialize.py | 1 + compose/config/types.py | 29 +++++++++++++++++++++++++ compose/service.py | 6 +++++- compose/utils.py | 8 +++++++ tests/integration/project_test.py | 36 ++++++++++++++++++++++++++++++- tests/integration/service_test.py | 5 +++-- tests/unit/cli/utils_test.py | 2 +- 10 files changed, 96 insertions(+), 16 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index cc8993d7f..73a7b7e10 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -13,9 +13,9 @@ from docker.utils.config import home_dir from ..config.environment import Environment from ..const import HTTP_TIMEOUT +from ..utils import unquote_path from .errors import UserError from .utils import generate_user_agent -from .utils import unquote_path log = logging.getLogger(__name__) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index a171d6678..4cc055cc9 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -131,14 +131,6 @@ def generate_user_agent(): return " ".join(parts) -def unquote_path(s): - if not s: - return s - if s[0] == '"' and s[-1] == '"': - return s[1:-1] - return s - - def human_readable_file_size(size): suffixes = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', ] order = int(math.log(size, 2) / 10) if size else 0 diff --git a/compose/config/config.py b/compose/config/config.py index b7764dd3b..38e887417 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -40,6 +40,7 @@ from .sort_services import sort_service_dicts from .types import MountSpec from .types import parse_extra_hosts from .types import parse_restart_spec +from .types import SecurityOpt from .types import ServiceLink from .types import ServicePort from .types import VolumeFromSpec @@ -734,9 +735,9 @@ def process_service(service_config): if field in service_dict: service_dict[field] = to_list(service_dict[field]) - service_dict = process_blkio_config(process_ports( + service_dict = process_security_opt(process_blkio_config(process_ports( process_healthcheck(service_dict) - )) + ))) return service_dict @@ -1376,6 +1377,16 @@ def split_path_mapping(volume_path): return (volume_path, None) +def process_security_opt(service_dict): + security_opts = service_dict.get('security_opt', []) + result = [] + for value in security_opts: + result.append(SecurityOpt.parse(value)) + if result: + service_dict['security_opt'] = result + return service_dict + + def join_path_mapping(pair): (container, host) = pair if isinstance(host, dict): diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 7fb9360a2..7de7f41e8 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -42,6 +42,7 @@ def serialize_string(dumper, data): yaml.SafeDumper.add_representer(types.MountSpec, serialize_dict_type) yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) +yaml.SafeDumper.add_representer(types.SecurityOpt, serialize_config_type) yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type) yaml.SafeDumper.add_representer(types.ServiceConfig, serialize_dict_type) yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type) diff --git a/compose/config/types.py b/compose/config/types.py index d84491d0a..47e7222a3 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -4,6 +4,7 @@ Types for objects parsed from the configuration. from __future__ import absolute_import from __future__ import unicode_literals +import json import ntpath import os import re @@ -13,6 +14,7 @@ import six from docker.utils.ports import build_port_bindings from ..const import COMPOSEFILE_V1 as V1 +from ..utils import unquote_path from .errors import ConfigurationError from compose.const import IS_WINDOWS_PLATFORM from compose.utils import splitdrive @@ -457,3 +459,30 @@ def normalize_port_dict(port): external_ip=port.get('external_ip', ''), has_ext_ip=(':' if port.get('external_ip') else ''), ) + + +class SecurityOpt(namedtuple('_SecurityOpt', 'value src_file')): + @classmethod + def parse(cls, value): + # based on https://github.com/docker/cli/blob/9de1b162f/cli/command/container/opts.go#L673-L697 + con = value.split('=', 2) + if len(con) == 1 and con[0] != 'no-new-privileges': + if ':' not in value: + raise ConfigurationError('Invalid security_opt: {}'.format(value)) + con = value.split(':', 2) + + if con[0] == 'seccomp' and con[1] != 'unconfined': + try: + with open(unquote_path(con[1]), 'r') as f: + seccomp_data = json.load(f) + except (IOError, ValueError) as e: + raise ConfigurationError('Error reading seccomp profile: {}'.format(e)) + return cls( + 'seccomp={}'.format(json.dumps(seccomp_data)), con[1] + ) + return cls(value, None) + + def repr(self): + if self.src_file is not None: + return 'seccomp:{}'.format(self.src_file) + return self.value diff --git a/compose/service.py b/compose/service.py index 3918a19e8..37368d64d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -881,6 +881,10 @@ class Service(object): init_path = options.get('init') options['init'] = True + security_opt = [ + o.value for o in options.get('security_opt') + ] if options.get('security_opt') else None + nano_cpus = None if 'cpus' in options: nano_cpus = int(options.get('cpus') * NANOCPUS_SCALE) @@ -910,7 +914,7 @@ class Service(object): extra_hosts=options.get('extra_hosts'), read_only=options.get('read_only'), pid_mode=self.pid_mode.mode, - security_opt=options.get('security_opt'), + security_opt=security_opt, ipc_mode=options.get('ipc'), cgroup_parent=options.get('cgroup_parent'), cpu_quota=options.get('cpu_quota'), diff --git a/compose/utils.py b/compose/utils.py index 00b01df2e..956673b4b 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -143,3 +143,11 @@ def parse_bytes(n): return sdk_parse_bytes(n) except DockerException: return None + + +def unquote_path(s): + if not s: + return s + if s[0] == '"' and s[-1] == '"': + return s[1:-1] + return s diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index b0e55f2d0..0acb80284 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,8 +1,10 @@ from __future__ import absolute_import from __future__ import unicode_literals -import os.path +import json +import os import random +import tempfile import py import pytest @@ -1834,3 +1836,35 @@ class ProjectTest(DockerClientTestCase): assert 'svc1' in svc2.get_dependency_names() with pytest.raises(NoHealthCheckConfigured): svc1.is_healthy() + + def test_project_up_seccomp_profile(self): + seccomp_data = { + 'defaultAction': 'SCMP_ACT_ALLOW', + 'syscalls': [] + } + fd, profile_path = tempfile.mkstemp('_seccomp.json') + self.addCleanup(os.remove, profile_path) + with os.fdopen(fd, 'w') as f: + json.dump(seccomp_data, f) + + config_dict = { + 'version': '2.3', + 'services': { + 'svc1': { + 'image': 'busybox:latest', + 'command': 'top', + 'security_opt': ['seccomp:"{}"'.format(profile_path)] + } + } + } + + config_data = load_config(config_dict) + project = Project.from_config(name='composetest', config_data=config_data, client=self.client) + project.up() + containers = project.containers() + assert len(containers) == 1 + + remote_secopts = containers[0].get('HostConfig.SecurityOpt') + assert len(remote_secopts) == 1 + assert remote_secopts[0].startswith('seccomp=') + assert json.loads(remote_secopts[0].lstrip('seccomp=')) == seccomp_data diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 0bc902aea..d1a704199 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -23,6 +23,7 @@ from .testcases import SWARM_SKIP_CONTAINERS_ALL from .testcases import SWARM_SKIP_CPU_SHARES from compose import __version__ from compose.config.types import MountSpec +from compose.config.types import SecurityOpt from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM @@ -238,11 +239,11 @@ class ServiceTest(DockerClientTestCase): }] def test_create_container_with_security_opt(self): - security_opt = ['label:disable'] + security_opt = [SecurityOpt.parse('label:disable')] service = self.create_service('db', security_opt=security_opt) container = service.create_container() service.start_container(container) - assert set(container.get('HostConfig.SecurityOpt')) == set(security_opt) + assert set(container.get('HostConfig.SecurityOpt')) == set([o.repr() for o in security_opt]) @pytest.mark.xfail(True, reason='Not supported on most drivers') def test_create_container_with_storage_opt(self): diff --git a/tests/unit/cli/utils_test.py b/tests/unit/cli/utils_test.py index 066fb3595..26524ff37 100644 --- a/tests/unit/cli/utils_test.py +++ b/tests/unit/cli/utils_test.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import unittest -from compose.cli.utils import unquote_path +from compose.utils import unquote_path class UnquotePathTest(unittest.TestCase): From a35335a75c66bf9cd8c85d02d880afc674f0da5c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Feb 2018 14:43:44 -0800 Subject: [PATCH 39/47] Add support for device_cgroup_rules in v2.3 files Signed-off-by: Joffrey F --- compose/config/config.py | 3 ++- compose/config/config_schema_v2.3.json | 15 ++++++++------- compose/service.py | 2 ++ tests/integration/service_test.py | 5 +++++ tests/unit/config/config_test.py | 15 +++++++++++++++ 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 38e887417..0b5c7df0a 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -72,6 +72,7 @@ DOCKER_CONFIG_KEYS = [ 'cpus', 'cpuset', 'detach', + 'device_cgroup_rules', 'devices', 'dns', 'dns_search', @@ -1045,7 +1046,7 @@ def merge_service_dicts(base, override, version): for field in [ 'cap_add', 'cap_drop', 'expose', 'external_links', - 'security_opt', 'volumes_from', + 'security_opt', 'volumes_from', 'device_cgroup_rules', ]: md.merge_field(field, merge_unique_items_lists, default=[]) diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 2d28df77a..33840dba5 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -99,8 +99,8 @@ } ] }, - "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_add": {"$ref": "#/definitions/list_of_strings"}, + "cap_drop": {"$ref": "#/definitions/list_of_strings"}, "cgroup_parent": {"type": "string"}, "command": { "oneOf": [ @@ -137,7 +137,8 @@ } ] }, - "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "device_cgroup_rules": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"$ref": "#/definitions/list_of_strings"}, "dns_opt": { "type": "array", "items": { @@ -184,7 +185,7 @@ ] }, - "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "external_links": {"$ref": "#/definitions/list_of_strings"}, "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, "healthcheck": {"$ref": "#/definitions/healthcheck"}, "hostname": {"type": "string"}, @@ -193,7 +194,7 @@ "ipc": {"type": "string"}, "isolation": {"type": "string"}, "labels": {"$ref": "#/definitions/labels"}, - "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "links": {"$ref": "#/definitions/list_of_strings"}, "logging": { "type": "object", @@ -264,7 +265,7 @@ "restart": {"type": "string"}, "runtime": {"type": "string"}, "scale": {"type": "integer"}, - "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "security_opt": {"$ref": "#/definitions/list_of_strings"}, "shm_size": {"type": ["number", "string"]}, "sysctls": {"$ref": "#/definitions/list_or_dict"}, "pids_limit": {"type": ["number", "string"]}, @@ -335,7 +336,7 @@ } }, "volume_driver": {"type": "string"}, - "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "volumes_from": {"$ref": "#/definitions/list_of_strings"}, "working_dir": {"type": "string"} }, diff --git a/compose/service.py b/compose/service.py index 37368d64d..4a1cde88d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -66,6 +66,7 @@ HOST_CONFIG_KEYS = [ 'cpu_shares', 'cpus', 'cpuset', + 'device_cgroup_rules', 'devices', 'dns', 'dns_search', @@ -944,6 +945,7 @@ class Service(object): device_write_bps=blkio_config.get('device_write_bps'), device_write_iops=blkio_config.get('device_write_iops'), mounts=options.get('mounts'), + device_cgroup_rules=options.get('device_cgroup_rules'), ) def get_secret_volumes(self): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index d1a704199..2b6b7711e 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -265,6 +265,11 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) assert container.inspect()['Config']['MacAddress'] == '02:42:ac:11:65:43' + def test_create_container_with_device_cgroup_rules(self): + service = self.create_service('db', device_cgroup_rules=['c 7:128 rwm']) + container = service.create_container() + assert container.get('HostConfig.DeviceCgroupRules') == ['c 7:128 rwm'] + def test_create_container_with_specified_volume(self): host_path = '/tmp/host-path' container_path = '/container-path' diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d72fae2f5..e03298221 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2558,6 +2558,21 @@ class ConfigTest(unittest.TestCase): actual = config.merge_service_dicts(base, override, V2_3) assert actual['healthcheck'] == override['healthcheck'] + def test_merge_device_cgroup_rules(self): + base = { + 'image': 'bar', + 'device_cgroup_rules': ['c 7:128 rwm', 'x 3:244 rw'] + } + + override = { + 'device_cgroup_rules': ['c 7:128 rwm', 'f 0:128 n'] + } + + actual = config.merge_service_dicts(base, override, V2_3) + assert sorted(actual['device_cgroup_rules']) == sorted( + ['c 7:128 rwm', 'x 3:244 rw', 'f 0:128 n'] + ) + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From 7ce5766f6a02f013b64f3611af36deccf5109500 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Feb 2018 11:14:28 -0800 Subject: [PATCH 40/47] Re-use TLS and host options when shelling out to docker CLI Signed-off-by: Joffrey F --- compose/cli/main.py | 62 ++++++++++++++++++++++++++++--------- tests/unit/cli/main_test.py | 42 +++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 624b007a8..972dffe20 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -122,7 +122,7 @@ def perform_command(options, handler, command_options): return project = project_from_options('.', options) - command = TopLevelCommand(project) + command = TopLevelCommand(project, options=options) with errors.handle_connection_errors(project.client): handler(command, command_options) @@ -157,16 +157,17 @@ def setup_console_handler(handler, verbose, noansi=False, level=None): if level is not None: levels = { - 'DEBUG': logging.DEBUG, - 'INFO': logging.INFO, - 'WARNING': logging.WARNING, - 'ERROR': logging.ERROR, - 'CRITICAL': logging.CRITICAL, + 'DEBUG': logging.DEBUG, + 'INFO': logging.INFO, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRITICAL': logging.CRITICAL, } loglevel = levels.get(level.upper()) if loglevel is None: - raise UserError('Invalid value for --log-level. Expected one of ' - + 'DEBUG, INFO, WARNING, ERROR, CRITICAL.') + raise UserError( + 'Invalid value for --log-level. Expected one of DEBUG, INFO, WARNING, ERROR, CRITICAL.' + ) handler.setLevel(loglevel) @@ -237,9 +238,10 @@ class TopLevelCommand(object): version Show the Docker-Compose version information """ - def __init__(self, project, project_dir='.'): + def __init__(self, project, project_dir='.', options=None): self.project = project self.project_dir = '.' + self.toplevel_options = options or {} def build(self, options): """ @@ -475,7 +477,10 @@ class TopLevelCommand(object): tty = not options["-T"] if IS_WINDOWS_PLATFORM or use_cli and not detach: - sys.exit(call_docker(build_exec_command(options, container.id, command))) + sys.exit(call_docker( + build_exec_command(options, container.id, command), + self.toplevel_options) + ) create_exec_options = { "privileged": options["--privileged"], @@ -820,7 +825,10 @@ class TopLevelCommand(object): command = service.options.get('command') container_options = build_container_options(options, detach, command) - run_one_off_container(container_options, self.project, service, options, self.project_dir) + run_one_off_container( + container_options, self.project, service, options, + self.toplevel_options, self.project_dir + ) def scale(self, options): """ @@ -1253,7 +1261,8 @@ def build_container_options(options, detach, command): return container_options -def run_one_off_container(container_options, project, service, options, project_dir='.'): +def run_one_off_container(container_options, project, service, options, toplevel_options, + project_dir='.'): if not options['--no-deps']: deps = service.get_dependency_names() if deps: @@ -1289,7 +1298,10 @@ def run_one_off_container(container_options, project, service, options, project_ try: if IS_WINDOWS_PLATFORM or use_cli: service.connect_container_to_networks(container) - exit_code = call_docker(["start", "--attach", "--interactive", container.id]) + exit_code = call_docker( + ["start", "--attach", "--interactive", container.id], + toplevel_options + ) else: operation = RunOperation( project.client, @@ -1368,12 +1380,32 @@ def exit_if(condition, message, exit_code): raise SystemExit(exit_code) -def call_docker(args): +def call_docker(args, dockeropts): executable_path = find_executable('docker') if not executable_path: raise UserError(errors.docker_not_found_msg("Couldn't find `docker` binary.")) - args = [executable_path] + args + tls = dockeropts.get('--tls', False) + ca_cert = dockeropts.get('--tlscacert') + cert = dockeropts.get('--tlscert') + key = dockeropts.get('--tlskey') + verify = dockeropts.get('--tlsverify') + host = dockeropts.get('--host') + tls_options = [] + if tls: + tls_options.append('--tls') + if ca_cert: + tls_options.extend(['--tlscacert', ca_cert]) + if cert: + tls_options.extend(['--tlscert', cert]) + if key: + tls_options.extend(['--tlskey', key]) + if verify: + tls_options.append('--tlsverify') + if host: + tls_options.extend(['--host', host]) + + args = [executable_path] + tls_options + args log.debug(" ".join(map(pipes.quote, args))) return subprocess.call(args) diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index b1546d6f3..b46a3ee22 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -9,6 +9,7 @@ import pytest from compose import container from compose.cli.errors import UserError from compose.cli.formatter import ConsoleWarningFormatter +from compose.cli.main import call_docker from compose.cli.main import convergence_strategy_from_opts from compose.cli.main import filter_containers_to_service_names from compose.cli.main import setup_console_handler @@ -112,3 +113,44 @@ class TestConvergeStrategyFromOptsTestCase(object): convergence_strategy_from_opts(options) == ConvergenceStrategy.changed ) + + +def mock_find_executable(exe): + return exe + + +@mock.patch('compose.cli.main.find_executable', mock_find_executable) +class TestCallDocker(object): + def test_simple_no_options(self): + with mock.patch('subprocess.call') as fake_call: + call_docker(['ps'], {}) + + assert fake_call.call_args[0][0] == ['docker', 'ps'] + + def test_simple_tls_option(self): + with mock.patch('subprocess.call') as fake_call: + call_docker(['ps'], {'--tls': True}) + + assert fake_call.call_args[0][0] == ['docker', '--tls', 'ps'] + + def test_advanced_tls_options(self): + with mock.patch('subprocess.call') as fake_call: + call_docker(['ps'], { + '--tls': True, + '--tlscacert': './ca.pem', + '--tlscert': './cert.pem', + '--tlskey': './key.pem', + }) + + assert fake_call.call_args[0][0] == [ + 'docker', '--tls', '--tlscacert', './ca.pem', '--tlscert', + './cert.pem', '--tlskey', './key.pem', 'ps' + ] + + def test_with_host_option(self): + with mock.patch('subprocess.call') as fake_call: + call_docker(['ps'], {'--host': 'tcp://mydocker.net:2333'}) + + assert fake_call.call_args[0][0] == [ + 'docker', '--host', 'tcp://mydocker.net:2333', 'ps' + ] From a7d1fada5216fee774619cfa07e3e644e464460a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Feb 2018 11:24:40 -0800 Subject: [PATCH 41/47] Unify toplevel command handlers Signed-off-by: Joffrey F --- compose/cli/main.py | 83 ++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 972dffe20..aff69c2d3 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -116,9 +116,9 @@ def perform_command(options, handler, command_options): handler(command_options) return - if options['COMMAND'] in ('config', 'bundle'): - command = TopLevelCommand(None) - handler(command, options, command_options) + if options['COMMAND'] == 'config': + command = TopLevelCommand(None, options=options) + handler(command, command_options) return project = project_from_options('.', options) @@ -279,7 +279,7 @@ class TopLevelCommand(object): memory=options.get('--memory'), build_args=build_args) - def bundle(self, config_options, options): + def bundle(self, options): """ Generate a Distributed Application Bundle (DAB) from the Compose file. @@ -298,8 +298,7 @@ class TopLevelCommand(object): -o, --output PATH Path to write the bundle file to. Defaults to ".dab". """ - self.project = project_from_options('.', config_options) - compose_config = get_config_from_options(self.project_dir, config_options) + compose_config = get_config_from_options(self.project_dir, self.toplevel_options) output = options["--output"] if not output: @@ -312,7 +311,7 @@ class TopLevelCommand(object): log.info("Wrote bundle to {}".format(output)) - def config(self, config_options, options): + def config(self, options): """ Validate and view the Compose file. @@ -327,12 +326,13 @@ class TopLevelCommand(object): """ - compose_config = get_config_from_options(self.project_dir, config_options) + compose_config = get_config_from_options(self.project_dir, self.toplevel_options) image_digests = None if options['--resolve-image-digests']: - self.project = project_from_options('.', config_options) - image_digests = image_digests_for_project(self.project) + self.project = project_from_options('.', self.toplevel_options) + with errors.handle_connection_errors(self.project.client): + image_digests = image_digests_for_project(self.project) if options['--quiet']: return @@ -1144,42 +1144,41 @@ def timeout_from_opts(options): def image_digests_for_project(project, allow_push=False): - with errors.handle_connection_errors(project.client): - try: - return get_image_digests( - project, - allow_push=allow_push + try: + return get_image_digests( + project, + allow_push=allow_push + ) + except MissingDigests as e: + def list_images(images): + return "\n".join(" {}".format(name) for name in sorted(images)) + + paras = ["Some images are missing digests."] + + if e.needs_push: + command_hint = ( + "Use `docker-compose push {}` to push them. " + .format(" ".join(sorted(e.needs_push))) ) - except MissingDigests as e: - def list_images(images): - return "\n".join(" {}".format(name) for name in sorted(images)) + paras += [ + "The following images can be pushed:", + list_images(e.needs_push), + command_hint, + ] - paras = ["Some images are missing digests."] + if e.needs_pull: + command_hint = ( + "Use `docker-compose pull {}` to pull them. " + .format(" ".join(sorted(e.needs_pull))) + ) - if e.needs_push: - command_hint = ( - "Use `docker-compose push {}` to push them. " - .format(" ".join(sorted(e.needs_push))) - ) - paras += [ - "The following images can be pushed:", - list_images(e.needs_push), - command_hint, - ] + paras += [ + "The following images need to be pulled:", + list_images(e.needs_pull), + command_hint, + ] - if e.needs_pull: - command_hint = ( - "Use `docker-compose pull {}` to pull them. " - .format(" ".join(sorted(e.needs_pull))) - ) - - paras += [ - "The following images need to be pulled:", - list_images(e.needs_pull), - command_hint, - ] - - raise UserError("\n\n".join(paras)) + raise UserError("\n\n".join(paras)) def exitval_from_opts(options, project): From 59c8ed77e4e8a54c5a7a7d645a73feed43ef140d Mon Sep 17 00:00:00 2001 From: Ganesh Satpute Date: Sun, 28 Jan 2018 16:45:38 +0530 Subject: [PATCH 42/47] Allow unset of entrypoint (resolves #5582) When an empty string is passed to the 'entrypoint' parameter, for example `docker-compose run --entrypoint='' ...` OR `docker-compose run --entrypoint '' ...` It allows the default entrypoint to be overriden by empty value. Signed-off-by: Ganesh Satpute --- compose/cli/main.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index aff69c2d3..056d8c46f 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1232,8 +1232,7 @@ def build_container_options(options, detach, command): if options['--label']: container_options['labels'] = parse_labels(options['--label']) - if options['--entrypoint']: - container_options['entrypoint'] = options.get('--entrypoint') + _build_container_entrypoint_options(container_options, options) if options['--rm']: container_options['restart'] = None @@ -1260,6 +1259,16 @@ def build_container_options(options, detach, command): return container_options +def _build_container_entrypoint_options(container_options, options): + if options['--entrypoint']: + if options['--entrypoint'].strip() == '': + # Set an empty entry point. Refer https://github.com/moby/moby/pull/23718 + log.info("Overriding the entrypoint") + container_options['entrypoint'] = [""] + else: + container_options['entrypoint'] = options.get('--entrypoint') + + def run_one_off_container(container_options, project, service, options, toplevel_options, project_dir='.'): if not options['--no-deps']: From 5b6e02d13ac929efdf99c145f351429040493940 Mon Sep 17 00:00:00 2001 From: Thomas Scholtes Date: Wed, 10 May 2017 10:36:05 +0200 Subject: [PATCH 43/47] 'run' command can use network aliases for service It is now possible for the 'run' command to use the network aliases defined for the service. Fixes #3492 Signed-off-by: Thomas Scholtes --- compose/cli/main.py | 10 +++++++--- compose/service.py | 26 ++++++++++++-------------- tests/unit/cli_test.py | 4 ++++ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index aff69c2d3..c79e4f7e4 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -803,6 +803,8 @@ class TopLevelCommand(object): -p, --publish=[] Publish a container's port(s) to the host --service-ports Run command with the service's ports enabled and mapped to the host. + --use-aliases Use the service's network aliases in the network(s) the + container connects to. -v, --volume=[] Bind mount a volume (default []) -T Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY. @@ -1279,8 +1281,10 @@ def run_one_off_container(container_options, project, service, options, toplevel one_off=True, **container_options) + use_network_aliases = options['--use-aliases'] + if options.get('--detach'): - service.start_container(container) + service.start_container(container, use_network_aliases) print(container.name) return @@ -1296,7 +1300,7 @@ def run_one_off_container(container_options, project, service, options, toplevel try: try: if IS_WINDOWS_PLATFORM or use_cli: - service.connect_container_to_networks(container) + service.connect_container_to_networks(container, use_network_aliases) exit_code = call_docker( ["start", "--attach", "--interactive", container.id], toplevel_options @@ -1310,7 +1314,7 @@ def run_one_off_container(container_options, project, service, options, toplevel ) pty = PseudoTerminal(project.client, operation) sockets = pty.sockets() - service.start_container(container) + service.start_container(container, use_network_aliases) pty.start(sockets) exit_code = container.wait() except (signals.ShutdownException): diff --git a/compose/service.py b/compose/service.py index 85aa74f26..cd1f77587 100644 --- a/compose/service.py +++ b/compose/service.py @@ -557,8 +557,8 @@ class Service(object): container.attach_log_stream() return self.start_container(container) - def start_container(self, container): - self.connect_container_to_networks(container) + def start_container(self, container, use_network_aliases=True): + self.connect_container_to_networks(container, use_network_aliases) try: container.start() except APIError as ex: @@ -574,7 +574,7 @@ class Service(object): ) ) - def connect_container_to_networks(self, container): + def connect_container_to_networks(self, container, use_network_aliases=True): connected_networks = container.get('NetworkSettings.Networks') for network, netdefs in self.prioritized_networks.items(): @@ -583,10 +583,15 @@ class Service(object): continue self.client.disconnect_container_from_network(container.id, network) - log.debug('Connecting to {}'.format(network)) + self.client.disconnect_container_from_network( + container.id, + network) + + aliases = self._get_aliases(netdefs) if use_network_aliases else [] + self.client.connect_container_to_network( container.id, network, - aliases=self._get_aliases(netdefs, container), + aliases=aliases, ipv4_address=netdefs.get('ipv4_address', None), ipv6_address=netdefs.get('ipv6_address', None), links=self._get_links(False), @@ -691,15 +696,8 @@ class Service(object): numbers = [c.number for c in containers] return 1 if not numbers else max(numbers) + 1 - def _get_aliases(self, network, container=None): - if container and container.labels.get(LABEL_ONE_OFF) == "True": - return [] - - return list( - {self.name} | - ({container.short_id} if container else set()) | - set(network.get('aliases', ())) - ) + def _get_aliases(self, network): + return list({self.name} | set(network.get('aliases', ()))) def build_default_networking_config(self): if not self.networks: diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index cef53740d..47eaabf9d 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -124,6 +124,7 @@ class CLITestCase(unittest.TestCase): '-T': None, '--entrypoint': None, '--service-ports': None, + '--use-aliases': None, '--publish': [], '--volume': [], '--rm': None, @@ -162,6 +163,7 @@ class CLITestCase(unittest.TestCase): '-T': None, '--entrypoint': None, '--service-ports': None, + '--use-aliases': None, '--publish': [], '--volume': [], '--rm': None, @@ -183,6 +185,7 @@ class CLITestCase(unittest.TestCase): '-T': None, '--entrypoint': None, '--service-ports': None, + '--use-aliases': None, '--publish': [], '--volume': [], '--rm': True, @@ -214,6 +217,7 @@ class CLITestCase(unittest.TestCase): '-T': None, '--entrypoint': None, '--service-ports': True, + '--use-aliases': None, '--publish': ['80:80'], '--rm': None, '--name': None, From e78c0bf533280ee526e1b5cc52cc7e7db139fb36 Mon Sep 17 00:00:00 2001 From: Jim Dalton Date: Wed, 10 May 2017 13:28:40 +0200 Subject: [PATCH 44/47] Add acceptance test for use-aliases feature Signed-off-by: Jim Dalton --- tests/acceptance/cli_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 42b487aab..404cd8c48 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1903,6 +1903,28 @@ class CLITestCase(DockerClientTestCase): container = service.containers(stopped=True, one_off=True)[0] assert workdir == container.get('Config.WorkingDir') + @v2_only() + def test_run_service_with_use_aliases(self): + filename = 'network-aliases.yml' + self.base_dir = 'tests/fixtures/networks' + self.dispatch(['-f', filename, 'run', '-d', '--use-aliases', 'web', 'top']) + + back_name = '{}_back'.format(self.project.name) + front_name = '{}_front'.format(self.project.name) + + web_container = self.project.get_service('web').containers(one_off=OneOffFilter.only)[0] + + back_aliases = web_container.get( + 'NetworkSettings.Networks.{}.Aliases'.format(back_name) + ) + assert 'web' in back_aliases + front_aliases = web_container.get( + 'NetworkSettings.Networks.{}.Aliases'.format(front_name) + ) + assert 'web' in front_aliases + assert 'forward_facing' in front_aliases + assert 'ahead' in front_aliases + @v2_only() def test_run_interactive_connects_to_network(self): self.base_dir = 'tests/fixtures/networks' From 1096a903be0b5897af8995cec4cccac9d20d880c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Feb 2018 12:57:58 -0800 Subject: [PATCH 45/47] unset entrypoint test Signed-off-by: Joffrey F --- compose/cli/main.py | 15 ++++----------- tests/acceptance/cli_test.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 056d8c46f..62de35fec 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1232,7 +1232,10 @@ def build_container_options(options, detach, command): if options['--label']: container_options['labels'] = parse_labels(options['--label']) - _build_container_entrypoint_options(container_options, options) + if options.get('--entrypoint') is not None: + container_options['entrypoint'] = ( + [""] if options['--entrypoint'] == '' else options['--entrypoint'] + ) if options['--rm']: container_options['restart'] = None @@ -1259,16 +1262,6 @@ def build_container_options(options, detach, command): return container_options -def _build_container_entrypoint_options(container_options, options): - if options['--entrypoint']: - if options['--entrypoint'].strip() == '': - # Set an empty entry point. Refer https://github.com/moby/moby/pull/23718 - log.info("Overriding the entrypoint") - container_options['entrypoint'] = [""] - else: - container_options['entrypoint'] = options.get('--entrypoint') - - def run_one_off_container(container_options, project, service, options, toplevel_options, project_dir='.'): if not options['--no-deps']: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 42b487aab..a8d93bfe8 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1697,6 +1697,18 @@ class CLITestCase(DockerClientTestCase): assert container.get('Config.Entrypoint') == ['printf'] assert container.get('Config.Cmd') == ['default', 'args'] + def test_run_service_with_unset_entrypoint(self): + self.base_dir = 'tests/fixtures/entrypoint-dockerfile' + self.dispatch(['run', '--entrypoint=""', 'test', 'true']) + container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0] + assert container.get('Config.Entrypoint') is None + assert container.get('Config.Cmd') == ['true'] + + self.dispatch(['run', '--entrypoint', '""', 'test', 'true']) + container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0] + assert container.get('Config.Entrypoint') is None + assert container.get('Config.Cmd') == ['true'] + def test_run_service_with_dockerfile_entrypoint_overridden(self): self.base_dir = 'tests/fixtures/entrypoint-dockerfile' self.dispatch(['run', '--entrypoint', 'echo', 'test']) From 07199fac374c427dcd949507e282abd2082d7564 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Feb 2018 13:17:28 -0800 Subject: [PATCH 46/47] Restore container ID alias Signed-off-by: Joffrey F --- compose/service.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/compose/service.py b/compose/service.py index cd1f77587..b9f9af2cd 100644 --- a/compose/service.py +++ b/compose/service.py @@ -583,11 +583,7 @@ class Service(object): continue self.client.disconnect_container_from_network(container.id, network) - self.client.disconnect_container_from_network( - container.id, - network) - - aliases = self._get_aliases(netdefs) if use_network_aliases else [] + aliases = self._get_aliases(netdefs, container) if use_network_aliases else [] self.client.connect_container_to_network( container.id, network, @@ -696,8 +692,12 @@ class Service(object): numbers = [c.number for c in containers] return 1 if not numbers else max(numbers) + 1 - def _get_aliases(self, network): - return list({self.name} | set(network.get('aliases', ()))) + def _get_aliases(self, network, container=None): + return list( + {self.name} | + ({container.short_id} if container else set()) | + set(network.get('aliases', ())) + ) def build_default_networking_config(self): if not self.networks: From 86428af5bcb96975e4379074c54c1e6c088603d5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Feb 2018 14:44:21 -0800 Subject: [PATCH 47/47] Bump 1.20.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 77 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4287de493..e662d40a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,83 @@ Change log ========== +1.20.0 (2018-03-07) +------------------- + +### New features + +#### Compose file version 3.6 + +- Introduced version 3.6 of the `docker-compose.yml` specification. + This version requires to be used with Docker Engine 18.02.0 or above. + +- Added support for the `tmpfs.size` property in volume mappings + +#### Compose file version 3.2 and up + +- The `--build-arg` option can now be used without specifying a service + in `docker-compose build` + +#### Compose file version 2.3 + +- Added support for `device_cgroup_rules` in service definitions + +- Added support for the `tmpfs.size` property in long-form volume mappings + +- The `--build-arg` option can now be used without specifying a service + in `docker-compose build` + +#### All formats + +- Added a `--log-level` option to the top-level `docker-compose` command. + Accepted values are `debug`, `info`, `warning`, `error`, `critical`. + Default log level is `info` + +- `docker-compose run` now allows users to unset the container's entrypoint + +- Proxy configuration found in the `~/.docker/config.json` file now populates + environment and build args for containers created by Compose + +- Added a `--use-aliases` flag to `docker-compose run`, indicating that + network aliases declared in the service's config should be used for the + running container + +- `docker-compose run` now kills and removes the running container upon + receiving `SIGHUP` + +- `docker-compose ps` now shows the containers' health status if available + +- Added the long-form `--detach` option to the `exec`, `run` and `up` + commands + +### Bugfixes + +- Fixed `.dockerignore` handling, notably with regard to absolute paths + and last-line precedence rules + +- Fixed a bug introduced in 1.19.0 which caused the default certificate path + to not be honored by Compose + +- Fixed a bug where Compose would incorrectly check whether a symlink's + destination was accessible when part of a build context + +- Fixed a bug where `.dockerignore` files containing lines of whitespace + caused Compose to error out on Windows + +- Fixed a bug where `--tls*` and `--host` options wouldn't be properly honored + for interactive `run` and `exec` commands + +- A `seccomp:` entry in the `security_opt` config now correctly + sends the contents of the file to the engine + +- Improved support for non-unicode locales + +- Fixed a crash occurring on Windows when the user's home directory name + contained non-ASCII characters + +- Fixed a bug occurring during builds caused by files with a negative `mtime` + values in the build context + 1.19.0 (2018-02-07) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index f0ad67347..2090f10c8 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.20.0dev' +__version__ = '1.20.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index ae55ff759..2adcc98f8 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.19.0" +VERSION="1.20.0-rc1" IMAGE="docker/compose:$VERSION"