From cb3bf869f46e2aa2ead695b942a9d1d7b07b23f3 Mon Sep 17 00:00:00 2001 From: Andreas Kohn Date: Tue, 20 Sep 2016 13:57:34 +0200 Subject: [PATCH 01/43] Fix typo Signed-off-by: Andreas Kohn --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ec7d5b57..cd4a23705 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ Bug Fixes - Fixed a bug in Windows environment where volume mappings of the host's root directory would be parsed incorrectly. -- Fixed a bug where `docker-compose config` would ouput an invalid +- Fixed a bug where `docker-compose config` would output an invalid Compose file if external networks were specified. - Fixed an issue where unset buildargs would be assigned a string From 90356b7040be49e402b1e176f4db7461659fbbd5 Mon Sep 17 00:00:00 2001 From: Matthew Bray Date: Wed, 28 Sep 2016 12:04:13 +0100 Subject: [PATCH 02/43] Zsh completion: permit multiple --file arguments Before this change: ``` $ docker-compose --file docker-compose.yml - -- option -- --help -h -- Get help --host -H -- Daemon socket to connect to --project-name -p -- Specify an alternate project name (default: directory name) --skip-hostname-check -- Don't check the daemon's hostname against the name specified in the client certificate (for example if your docker host is an IP address) --tls -- Use TLS; implied by --tlsverify --tlscacert -- Trust certs signed only by this CA --tlscert -- Path to TLS certificate file --tlskey -- Path to TLS key file --tlsverify -- Use TLS and verify the remote --verbose -- Show more output --version -v -- Print version and exit ``` (Note the `--file` argument is no longer available to complete.) After this change: ``` docker-compose --file docker-compose.yml - -- option -- --file -f -- Specify an alternate docker-compose file (default: docker-compose.yml) --help -h -- Get help --host -H -- Daemon socket to connect to --project-name -p -- Specify an alternate project name (default: directory name) --skip-hostname-check -- Don't check the daemon's hostname against the name specified in the client certificate (for example if your docker host is an IP address) --tls -- Use TLS; implied by --tlsverify --tlscacert -- Trust certs signed only by this CA --tlscert -- Path to TLS certificate file --tlskey -- Path to TLS key file --tlsverify -- Use TLS and verify the remote --verbose -- Show more output --version -v -- Print version and exit ``` Signed-off-by: Matt Bray --- contrib/completion/zsh/_docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 928e28def..fae758426 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -388,7 +388,7 @@ _docker-compose() { _arguments -C \ '(- :)'{-h,--help}'[Get help]' \ - '(-f --file)'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ + '*'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ '--verbose[Show more output]' \ '(- :)'{-v,--version}'[Print version and exit]' \ From a37d99f20114efced6381336990252f2e8238850 Mon Sep 17 00:00:00 2001 From: Matt Bray Date: Fri, 30 Sep 2016 00:38:48 +0100 Subject: [PATCH 03/43] Zsh completion: change --file description text Signed-off-by: Matt Bray --- contrib/completion/zsh/_docker-compose | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index fae758426..ceb7d0f58 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -386,9 +386,17 @@ _docker-compose() { integer ret=1 typeset -A opt_args + local file_description + + if [[ -n ${words[(r)-f]} || -n ${words[(r)--file]} ]] ; then + file_description="Specify an override docker-compose file (default: docker-compose.override.yml)" + else + file_description="Specify an alternate docker-compose file (default: docker-compose.yml)" + fi + _arguments -C \ '(- :)'{-h,--help}'[Get help]' \ - '*'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ + '*'{-f,--file}"[${file_description}]:file:_files -g '*.yml'" \ '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ '--verbose[Show more output]' \ '(- :)'{-v,--version}'[Print version and exit]' \ From a74b2f2f70a82bc56fb463d46480ea1baad9d4ab Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Dec 2016 15:16:53 -0500 Subject: [PATCH 04/43] Fix schema typo. Signed-off-by: Daniel Nephin --- compose/config/config_schema_v3.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 1f93347f9..6212058c4 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -270,7 +270,7 @@ "cpus": {"type": "string"}, "memory": {"type": "string"} }, - "additionaProperties": false + "additionalProperties": false }, "network": { From c73fc26824f2bffa791472a27039da0d2dd1ccbe Mon Sep 17 00:00:00 2001 From: Jun Guo Date: Wed, 4 Jan 2017 15:31:12 +0800 Subject: [PATCH 05/43] Fix 404 issue, change APIError to more accureate ImageNotFound Signed-off-by: Jun Guo --- compose/service.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index e3862b6ea..ee8c88a92 100644 --- a/compose/service.py +++ b/compose/service.py @@ -10,6 +10,7 @@ from operator import attrgetter import enum import six from docker.errors import APIError +from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig from docker.utils.ports import build_port_bindings @@ -318,11 +319,8 @@ class Service(object): def image(self): try: return self.client.inspect_image(self.image_name) - except APIError as e: - if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation): - raise NoSuchImageError("Image '{}' not found".format(self.image_name)) - else: - raise + except ImageNotFound: + raise NoSuchImageError("Image '{}' not found".format(self.image_name)) @property def image_name(self): From 2648af6807f83f0dd85b236e89e4bc3ee5db15fc Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 22 Nov 2016 11:09:56 +0000 Subject: [PATCH 06/43] enable universal wheels Signed-off-by: Thomas Grainger --- Dockerfile.run | 2 +- script/build/image | 4 ++-- script/release/push-release | 6 +++--- setup.cfg | 2 ++ setup.py | 23 ++++++++++++++++++++++- 5 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 setup.cfg diff --git a/Dockerfile.run b/Dockerfile.run index 4e76d64ff..c6852af14 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -7,7 +7,7 @@ RUN apk -U add \ COPY requirements.txt /code/requirements.txt RUN pip install -r /code/requirements.txt -ADD dist/docker-compose-release.tar.gz /code/docker-compose +ADD dist/docker-compose-release-py2.py3-none-any.whl /code/docker-compose RUN pip install --no-deps /code/docker-compose/docker-compose-* ENTRYPOINT ["/usr/bin/docker-compose"] diff --git a/script/build/image b/script/build/image index bdd98f03e..28aa2047a 100755 --- a/script/build/image +++ b/script/build/image @@ -11,6 +11,6 @@ TAG=$1 VERSION="$(python setup.py --version)" ./script/build/write-git-sha -python setup.py sdist -cp dist/docker-compose-$VERSION.tar.gz dist/docker-compose-release.tar.gz +python setup.py sdist bdist_wheel +cp dist/docker-compose-$VERSION-py2.py3-none-any.whl dist/docker-compose-release-py2.py3-none-any.whl docker build -t docker/compose:$TAG -f Dockerfile.run . diff --git a/script/release/push-release b/script/release/push-release index d5ae3de9d..d1a9e3f66 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -54,13 +54,13 @@ git push $GITHUB_REPO $VERSION echo "Uploading the docker image" docker push docker/compose:$VERSION -echo "Uploading sdist to PyPI" +echo "Uploading package to PyPI" pandoc -f markdown -t rst README.md -o README.rst sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst ./script/build/write-git-sha -python setup.py sdist +python setup.py sdist bdist_wheel if [ "$(command -v twine 2> /dev/null)" ]; then - twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz + twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz ./dist/docker-compose-${VERSION/-/}-py2.py3-none-any.whl else python setup.py upload fi diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..3c6e79cf3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index 8b4cf709e..00ca9f4c7 100644 --- a/setup.py +++ b/setup.py @@ -4,10 +4,12 @@ from __future__ import absolute_import from __future__ import unicode_literals import codecs +import logging import os import re import sys +import pkg_resources from setuptools import find_packages from setuptools import setup @@ -49,7 +51,25 @@ tests_require = [ if sys.version_info[:2] < (3, 4): tests_require.append('mock >= 1.0.1') - install_requires.append('enum34 >= 1.0.4, < 2') + +extras_require = { + ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'] +} + + +try: + if 'bdist_wheel' not in sys.argv: + for key, value in extras_require.items(): + if key.startswith(':') and pkg_resources.evaluate_marker(key[1:]): + install_requires.extend(value) +except Exception: + logging.getLogger(__name__).exception( + 'Something went wrong calculating platform specific dependencies, so ' + "you're getting them all!" + ) + for key, value in extras_require.items(): + if key.startswith(':'): + install_requires.extend(value) setup( @@ -63,6 +83,7 @@ setup( include_package_data=True, test_suite='nose.collector', install_requires=install_requires, + extras_require=extras_require, tests_require=tests_require, entry_points=""" [console_scripts] From 8145429399346a8d800369aad17f5fe69237c2ad Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 4 Jan 2017 13:14:23 -0800 Subject: [PATCH 07/43] Unify healthcheck spec definition in v2 and v3 Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- compose/config/config_schema_v2.1.json | 2 +- compose/config/config_schema_v3.0.json | 12 ++++++------ tests/integration/project_test.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 73a340172..fd935591f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -675,7 +675,7 @@ def process_healthcheck(service_dict, service_name): hc = {} raw = service_dict['healthcheck'] - if raw.get('disable') or raw.get('disabled'): + if raw.get('disable'): if len(raw) > 1: raise ConfigurationError( 'Service "{}" defines an invalid healthcheck: ' diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 5ab9f71fb..d0d5233a5 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -258,7 +258,7 @@ "type": "object", "additionalProperties": false, "properties": { - "disabled": {"type": "boolean"}, + "disable": {"type": "boolean"}, "interval": {"type": "string"}, "retries": {"type": "number"}, "test": { diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 1f93347f9..8d075d47a 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -202,10 +202,11 @@ "healthcheck": { "id": "#/definitions/healthcheck", - "type": ["object", "null"], + "type": "object", + "additionalProperties": false, "properties": { - "interval": {"type":"string"}, - "timeout": {"type":"string"}, + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, "retries": {"type": "number"}, "test": { "oneOf": [ @@ -213,9 +214,8 @@ {"type": "array", "items": {"type": "string"}} ] }, - "disable": {"type": "boolean"} - }, - "additionalProperties": false + "timeout": {"type": "string"} + } }, "deployment": { "id": "#/definitions/deployment", diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 855974de1..c5e3cf50f 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1463,7 +1463,7 @@ class ProjectTest(DockerClientTestCase): 'image': 'busybox:latest', 'command': 'top', 'healthcheck': { - 'disabled': True + 'disable': True }, }, 'svc2': { From 1be41f59c9119c72b3c39045e4e4031608fa18df Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 4 Jan 2017 14:30:20 -0800 Subject: [PATCH 08/43] Add support for stop_grace_period in v2 Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 1 + compose/config/config_schema_v2.1.json | 1 + compose/config/config_schema_v3.0.json | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 766889169..77494715c 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -192,6 +192,7 @@ "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "shm_size": {"type": ["number", "string"]}, "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, "stop_signal": {"type": "string"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index d0d5233a5..97ec5fa1d 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -217,6 +217,7 @@ "shm_size": {"type": ["number", "string"]}, "sysctls": {"$ref": "#/definitions/list_or_dict"}, "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, "stop_signal": {"type": "string"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 8d075d47a..2b410446a 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -169,8 +169,8 @@ "shm_size": {"type": ["number", "string"]}, "sysctls": {"$ref": "#/definitions/list_or_dict"}, "stdin_open": {"type": "boolean"}, - "stop_signal": {"type": "string"}, "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, "ulimits": { From 534b4ed820ec2b2e2f7b296e0065c42ebf3489cd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 4 Jan 2017 15:26:11 -0800 Subject: [PATCH 09/43] Falsy values in COMPOSE_CONVERT_WINDOWS_PATHS are properly recognized Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- compose/config/environment.py | 11 ++++++++ tests/unit/config/environment_test.py | 40 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/unit/config/environment_test.py diff --git a/compose/config/config.py b/compose/config/config.py index fd935591f..c11460fa5 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -712,7 +712,7 @@ def finalize_service(service_config, service_names, version, environment): if 'volumes' in service_dict: service_dict['volumes'] = [ VolumeSpec.parse( - v, environment.get('COMPOSE_CONVERT_WINDOWS_PATHS') + v, environment.get_boolean('COMPOSE_CONVERT_WINDOWS_PATHS') ) for v in service_dict['volumes'] ] diff --git a/compose/config/environment.py b/compose/config/environment.py index 5d6b5af69..7b9269300 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -105,3 +105,14 @@ class Environment(dict): super(Environment, self).get(key.upper(), *args, **kwargs) ) return super(Environment, self).get(key, *args, **kwargs) + + def get_boolean(self, key): + # Convert a value to a boolean using "common sense" rules. + # Unset, empty, "0" and "false" (i-case) yield False. + # All other values yield True. + value = self.get(key) + if not value: + return False + if value.lower() in ['0', 'false']: + return False + return True diff --git a/tests/unit/config/environment_test.py b/tests/unit/config/environment_test.py new file mode 100644 index 000000000..20446d2bf --- /dev/null +++ b/tests/unit/config/environment_test.py @@ -0,0 +1,40 @@ +# encoding: utf-8 +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from compose.config.environment import Environment +from tests import unittest + + +class EnvironmentTest(unittest.TestCase): + def test_get_simple(self): + env = Environment({ + 'FOO': 'bar', + 'BAR': '1', + 'BAZ': '' + }) + + assert env.get('FOO') == 'bar' + assert env.get('BAR') == '1' + assert env.get('BAZ') == '' + + def test_get_undefined(self): + env = Environment({ + 'FOO': 'bar' + }) + assert env.get('FOOBAR') is None + + def test_get_boolean(self): + env = Environment({ + 'FOO': '', + 'BAR': '0', + 'BAZ': 'FALSE', + 'FOOBAR': 'true', + }) + + assert env.get_boolean('FOO') is False + assert env.get_boolean('BAR') is False + assert env.get_boolean('BAZ') is False + assert env.get_boolean('FOOBAR') is True + assert env.get_boolean('UNDEFINED') is False From e063c5739fedeb56450075920451e3fd8b57a826 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 5 Jan 2017 11:15:24 -0800 Subject: [PATCH 10/43] Fix config schemas (misplaced "additionalProperties") Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 6 +++--- compose/config/config_schema_v2.1.json | 6 +++--- compose/config/config_schema_v3.0.json | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 77494715c..59c7b30c9 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -276,9 +276,9 @@ "type": ["boolean", "object"], "properties": { "name": {"type": "string"} - } - }, - "additionalProperties": false + }, + "additionalProperties": false + } }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 97ec5fa1d..d1ffff89a 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -322,10 +322,10 @@ "type": ["boolean", "object"], "properties": { "name": {"type": "string"} - } + }, + "additionalProperties": false }, - "labels": {"$ref": "#/definitions/list_or_dict"}, - "additionalProperties": false + "labels": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 2b410446a..194fd8e6e 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -328,7 +328,8 @@ "type": ["boolean", "object"], "properties": { "name": {"type": "string"} - } + }, + "additionalProperties": false } }, "labels": {"$ref": "#/definitions/list_or_dict"}, From 2c157e8fa9a94d71637643a6ec807db8b21a9d29 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 6 Jan 2017 17:45:57 -0800 Subject: [PATCH 11/43] Use docker SDK 2.0.1 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bae5d9ea1..4b7c7b760 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 colorama==0.3.7 -docker==2.0.0 +docker==2.0.1 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/setup.py b/setup.py index 8b4cf709e..7954d92bb 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.0.0, < 3.0', + 'docker >= 2.0.1, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 52792b7a963af9c593e61c78c7f0c7f62550a85b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 10 Jan 2017 14:57:32 -0800 Subject: [PATCH 12/43] Update setup.py extra_requires Signed-off-by: Joffrey F --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 0ceb2a22a..2f2ba7429 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,9 @@ if sys.version_info[:2] < (3, 4): tests_require.append('mock >= 1.0.1') extras_require = { - ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'] + ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], + ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5'], + ':python_version < "3.3"': ['ipaddress >= 1.0.16'], } @@ -64,8 +66,8 @@ try: install_requires.extend(value) except Exception: logging.getLogger(__name__).exception( - 'Something went wrong calculating platform specific dependencies, so ' - "you're getting them all!" + 'Failed to compute platform dependencies. All dependencies will be ' + 'installed as a result.' ) for key, value in extras_require.items(): if key.startswith(':'): From 19190ea0df43978d1a9c9f0fefd644ca5b08aee3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 10 Jan 2017 16:43:26 -0800 Subject: [PATCH 13/43] Fix docker image build script when using universal wheels Signed-off-by: Joffrey F --- Dockerfile.run | 5 +++-- script/build/image | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.run b/Dockerfile.run index c6852af14..de46e35e5 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,5 +1,6 @@ FROM alpine:3.4 +ARG version RUN apk -U add \ python \ py-pip @@ -7,7 +8,7 @@ RUN apk -U add \ COPY requirements.txt /code/requirements.txt RUN pip install -r /code/requirements.txt -ADD dist/docker-compose-release-py2.py3-none-any.whl /code/docker-compose -RUN pip install --no-deps /code/docker-compose/docker-compose-* +COPY dist/docker_compose-${version}-py2.py3-none-any.whl /code/ +RUN pip install --no-deps /code/docker_compose-${version}-py2.py3-none-any.whl ENTRYPOINT ["/usr/bin/docker-compose"] diff --git a/script/build/image b/script/build/image index 28aa2047a..3590ce14e 100755 --- a/script/build/image +++ b/script/build/image @@ -12,5 +12,4 @@ VERSION="$(python setup.py --version)" ./script/build/write-git-sha python setup.py sdist bdist_wheel -cp dist/docker-compose-$VERSION-py2.py3-none-any.whl dist/docker-compose-release-py2.py3-none-any.whl -docker build -t docker/compose:$TAG -f Dockerfile.run . +docker build --build-arg version=$VERSION -t docker/compose:$TAG -f Dockerfile.run . From 29b46d5b26055ade86a2e0e608342dd298e5a8c3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Jan 2017 15:39:48 -0800 Subject: [PATCH 14/43] Use correct wheel file name in twine upload command Signed-off-by: Joffrey F --- script/release/push-release | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/release/push-release b/script/release/push-release index d1a9e3f66..9db6f6894 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -60,12 +60,13 @@ sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/maste ./script/build/write-git-sha python setup.py sdist bdist_wheel if [ "$(command -v twine 2> /dev/null)" ]; then - twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz ./dist/docker-compose-${VERSION/-/}-py2.py3-none-any.whl + twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz ./dist/docker_compose-${VERSION/-/}-py2.py3-none-any.whl else python setup.py upload fi echo "Testing pip package" +deactivate || true virtualenv venv-test source venv-test/bin/activate pip install docker-compose==$VERSION From 2df31bb13c9a6820aba1d9b5a827329eded2b9cd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Jan 2017 16:25:40 -0800 Subject: [PATCH 15/43] Provide valid serialization of depends_on when format is not 2.1 Signed-off-by: Joffrey F --- compose/config/serialize.py | 9 ++++++++- tests/unit/config/config_test.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 768f3d473..05ac0d60d 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -56,9 +56,16 @@ def denormalize_service_dict(service_dict, version): service_dict = service_dict.copy() if 'restart' in service_dict: - service_dict['restart'] = types.serialize_restart_spec(service_dict['restart']) + service_dict['restart'] = types.serialize_restart_spec( + service_dict['restart'] + ) if version == V1 and 'network_mode' not in service_dict: service_dict['network_mode'] = 'bridge' + if 'depends_on' in service_dict and version != V2_1: + service_dict['depends_on'] = sorted([ + svc for svc in service_dict['depends_on'].keys() + ]) + return service_dict diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 31a888ed0..ca7c61683 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -22,6 +22,7 @@ from compose.config.config import V3_0 from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION +from compose.config.serialize import denormalize_service_dict from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from compose.utils import nanoseconds_from_time_seconds @@ -3269,3 +3270,33 @@ def get_config_filename_for_files(filenames, subdir=None): return os.path.basename(filename) finally: shutil.rmtree(project_dir) + + +class SerializeTest(unittest.TestCase): + def test_denormalize_depends_on_v3(self): + service_dict = { + 'image': 'busybox', + 'command': 'true', + 'depends_on': { + 'service2': {'condition': 'service_started'}, + 'service3': {'condition': 'service_started'}, + } + } + + assert denormalize_service_dict(service_dict, V3_0) == { + 'image': 'busybox', + 'command': 'true', + 'depends_on': ['service2', 'service3'] + } + + def test_denormalize_depends_on_v2_1(self): + service_dict = { + 'image': 'busybox', + 'command': 'true', + 'depends_on': { + 'service2': {'condition': 'service_started'}, + 'service3': {'condition': 'service_started'}, + } + } + + assert denormalize_service_dict(service_dict, V2_1) == service_dict From 931027c59828f242391de41cbeadfd6d1664588a Mon Sep 17 00:00:00 2001 From: muicoder Date: Mon, 16 Jan 2017 10:43:29 +0800 Subject: [PATCH 16/43] add IMAGE_EVENTS: load/save Signed-off-by: muicoder --- compose/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/const.py b/compose/const.py index 1b1be5c76..354c6d76a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -5,7 +5,7 @@ import sys DEFAULT_TIMEOUT = 10 HTTP_TIMEOUT = 60 -IMAGE_EVENTS = ['delete', 'import', 'pull', 'push', 'tag', 'untag'] +IMAGE_EVENTS = ['delete', 'import', 'load', 'pull', 'push', 'save', 'tag', 'untag'] IS_WINDOWS_PLATFORM = (sys.platform == "win32") LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' LABEL_ONE_OFF = 'com.docker.compose.oneoff' From 56a1b02aac33d09ec7761729a8d6ddcb0fbdea0e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 17 Jan 2017 13:22:16 -0800 Subject: [PATCH 17/43] Catch healthcheck exceptions in parallel_execute Signed-off-by: Joffrey F --- compose/parallel.py | 40 ++++++++++++++++++------------- tests/integration/project_test.py | 4 ++-- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index b2654dcfd..e495410cf 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -12,6 +12,8 @@ from six.moves.queue import Empty from six.moves.queue import Queue from compose.cli.signals import ShutdownException +from compose.errors import HealthCheckFailed +from compose.errors import NoHealthCheckConfigured from compose.errors import OperationFailedError from compose.utils import get_output_stream @@ -48,7 +50,7 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): elif isinstance(exception, APIError): errors[get_name(obj)] = exception.explanation writer.write(get_name(obj), 'error') - elif isinstance(exception, OperationFailedError): + elif isinstance(exception, (OperationFailedError, HealthCheckFailed, NoHealthCheckConfigured)): errors[get_name(obj)] = exception.msg writer.write(get_name(obj), 'error') elif isinstance(exception, UpstreamError): @@ -164,21 +166,27 @@ def feed_queue(objects, func, get_deps, results, state): for obj in pending: deps = get_deps(obj) - - if any(dep[0] in state.failed for dep in deps): - log.debug('{} has upstream errors - not processing'.format(obj)) - results.put((obj, None, UpstreamError())) - state.failed.add(obj) - elif all( - dep not in objects or ( - dep in state.finished and (not ready_check or ready_check(dep)) - ) for dep, ready_check in deps - ): - log.debug('Starting producer thread for {}'.format(obj)) - t = Thread(target=producer, args=(obj, func, results)) - t.daemon = True - t.start() - state.started.add(obj) + try: + if any(dep[0] in state.failed for dep in deps): + log.debug('{} has upstream errors - not processing'.format(obj)) + results.put((obj, None, UpstreamError())) + state.failed.add(obj) + elif all( + dep not in objects or ( + dep in state.finished and (not ready_check or ready_check(dep)) + ) for dep, ready_check in deps + ): + log.debug('Starting producer thread for {}'.format(obj)) + t = Thread(target=producer, args=(obj, func, results)) + t.daemon = True + t.start() + state.started.add(obj) + except (HealthCheckFailed, NoHealthCheckConfigured) as e: + log.debug( + 'Healthcheck for service(s) upstream of {} failed - ' + 'not processing'.format(obj) + ) + results.put((obj, None, e)) if state.is_done(): results.put(STOP) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index c5e3cf50f..ee2b7817b 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1443,7 +1443,7 @@ class ProjectTest(DockerClientTestCase): project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) - with pytest.raises(HealthCheckFailed): + with pytest.raises(ProjectError): project.up() containers = project.containers() assert len(containers) == 1 @@ -1479,7 +1479,7 @@ class ProjectTest(DockerClientTestCase): project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) - with pytest.raises(NoHealthCheckConfigured): + with pytest.raises(ProjectError): project.up() containers = project.containers() assert len(containers) == 1 From 1a02121ab55876f92bcceb62d1e81b7f114f0c79 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Jan 2017 17:52:03 -0800 Subject: [PATCH 18/43] depends_on merge now retains condition information when present Signed-off-by: Joffrey F --- compose/config/config.py | 6 +++++- tests/unit/config/config_test.py | 34 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index c11460fa5..7e77421e5 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -818,6 +818,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('ulimits', parse_ulimits) md.merge_mapping('networks', parse_networks) md.merge_mapping('sysctls', parse_sysctls) + md.merge_mapping('depends_on', parse_depends_on) md.merge_sequence('links', ServiceLink.parse) for field in ['volumes', 'devices']: @@ -825,7 +826,7 @@ def merge_service_dicts(base, override, version): for field in [ 'ports', 'cap_add', 'cap_drop', 'expose', 'external_links', - 'security_opt', 'volumes_from', 'depends_on', + 'security_opt', 'volumes_from', ]: md.merge_field(field, merge_unique_items_lists, default=[]) @@ -920,6 +921,9 @@ parse_environment = functools.partial(parse_dict_or_list, split_env, 'environmen parse_labels = functools.partial(parse_dict_or_list, split_kv, 'labels') parse_networks = functools.partial(parse_dict_or_list, lambda k: (k, None), 'networks') parse_sysctls = functools.partial(parse_dict_or_list, split_kv, 'sysctls') +parse_depends_on = functools.partial( + parse_dict_or_list, lambda k: (k, {'condition': 'service_started'}), 'depends_on' +) def parse_ulimits(ulimits): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ca7c61683..ab8bfcfcc 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1713,6 +1713,40 @@ class ConfigTest(unittest.TestCase): } } + def test_merge_depends_on_no_override(self): + base = { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'} + } + } + override = {} + actual = config.merge_service_dicts(base, override, V2_1) + assert actual == base + + def test_merge_depends_on_mixed_syntax(self): + base = { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'} + } + } + override = { + 'depends_on': ['app3'] + } + + actual = config.merge_service_dicts(base, override, V2_1) + assert actual == { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'}, + 'app3': {'condition': 'service_started'} + } + } + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From 169289c8b66dcab760cdc7e1534fc0bece326d44 Mon Sep 17 00:00:00 2001 From: "Aaron.L.Xu" Date: Fri, 20 Jan 2017 00:52:13 +0800 Subject: [PATCH 19/43] find a fishbone Signed-off-by: Aaron.L.Xu --- script/test/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test/versions.py b/script/test/versions.py index 45ead1438..0c3b8162d 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -5,7 +5,7 @@ version tags for recent releases, or the default release. The default release is the most recent non-RC version. -Recent is a list of unqiue major.minor versions, where each is the most +Recent is a list of unique major.minor versions, where each is the most recent version in the series. For example, if the list of versions is: From 1c46525c2baf8532434c320bf0443a520381431d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 14:47:31 -0800 Subject: [PATCH 20/43] 1.11.0dev Signed-off-by: Joffrey F --- CHANGELOG.md | 48 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c645ca25..6699f8807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,54 @@ Change log ========== +1.10.0 (2017-01-18) +------------------- + +### New Features + +#### Compose file version 3.0 + +- Introduced version 3.0 of the `docker-compose.yml` specification. This + version requires to be used with Docker Engine 1.13 or above and is + specifically designed to work with the `docker stack` commands. + +#### Compose file version 2.1 and up + +- Healthcheck configuration can now be done in the service definition using + the `healthcheck` parameter + +- Containers dependencies can now be set up to wait on positive healthchecks + when declared using `depends_on`. See the documentation for the updated + syntax. + **Note:** This feature will not be ported to version 3 Compose files. + +- Added support for the `sysctls` parameter in service definitions + +- Added support for the `userns_mode` parameter in service definitions + +- Compose now adds identifying labels to networks and volumes it creates + +#### Compose file version 2.0 and up + +- Added support for the `stop_grace_period` option in service definitions. + +### Bugfixes + +- Colored output now works properly on Windows. + +- Fixed a bug where docker-compose run would fail to set up link aliases + in interactive mode on Windows. + +- Networks created by Compose are now always made attachable + (Compose files v2.1 and up). + +- Fixed a bug where falsy values of `COMPOSE_CONVERT_WINDOWS_PATHS` + (`0`, `false`, empty value) were being interpreted as true. + +- Fixed a bug where forward slashes in some .dockerignore patterns weren't + being parsed correctly on Windows + + 1.9.0 (2016-11-16) ----------------- diff --git a/compose/__init__.py b/compose/__init__.py index 6f05b282f..384178364 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.10.0dev' +__version__ = '1.11.0dev' From 5c2165eaafbb625eb2058b199b571a228e86df03 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 15:41:31 -0800 Subject: [PATCH 21/43] Fix volume definition in v3 schema Signed-off-by: Joffrey F --- compose/config/config_schema_v3.0.json | 4 ++-- tests/acceptance/cli_test.py | 8 +++++++- tests/fixtures/v3-full/docker-compose.yml | 4 ++++ tests/integration/testcases.py | 7 +++++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index ae4c05300..584b6ef5d 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -330,9 +330,9 @@ "name": {"type": "string"} }, "additionalProperties": false - } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} }, - "labels": {"$ref": "#/definitions/list_or_dict"}, "additionalProperties": false }, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b9766226d..ce31dd18f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -295,7 +295,13 @@ class CLITestCase(DockerClientTestCase): assert yaml.load(result.stdout) == { 'version': '3.0', 'networks': {}, - 'volumes': {}, + 'volumes': { + 'foobar': { + 'labels': { + 'com.docker.compose.test': 'true', + }, + }, + }, 'services': { 'web': { 'image': 'busybox', diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml index b4d1b6422..a1661ab93 100644 --- a/tests/fixtures/v3-full/docker-compose.yml +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -35,3 +35,7 @@ services: retries: 5 stop_grace_period: 20s +volumes: + foobar: + labels: + com.docker.compose.test: 'true' diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index f6bc402bf..230bd2d92 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -13,6 +13,7 @@ from compose.config.config import resolve_environment from compose.config.config import V1 from compose.config.config import V2_0 from compose.config.config import V2_1 +from compose.config.config import V3_0 from compose.config.environment import Environment from compose.const import API_VERSIONS from compose.const import LABEL_PROJECT @@ -36,13 +37,15 @@ def get_links(container): def engine_max_version(): if 'DOCKER_VERSION' not in os.environ: - return V2_1 + return V3_0 version = os.environ['DOCKER_VERSION'].partition('-')[0] if version_lt(version, '1.10'): return V1 elif version_lt(version, '1.12'): return V2_0 - return V2_1 + elif version_lt(version, '1.13'): + return V2_1 + return V3_0 def build_version_required_decorator(ignored_versions): From d83d31889ea937524db798aa8260638036503764 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 16:05:13 -0800 Subject: [PATCH 22/43] Remove external_name from volume def in config output Signed-off-by: Joffrey F --- compose/config/serialize.py | 7 ++++++- tests/acceptance/cli_test.py | 14 ++++++++++++++ tests/fixtures/volumes/docker-compose.yml | 2 ++ tests/fixtures/volumes/external-volumes.yml | 16 ++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/volumes/docker-compose.yml create mode 100644 tests/fixtures/volumes/external-volumes.yml diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 05ac0d60d..9ea287a46 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -32,6 +32,11 @@ def denormalize_config(config): if 'external_name' in net_conf: del net_conf['external_name'] + volumes = config.volumes.copy() + for vol_name, vol_conf in volumes.items(): + if 'external_name' in vol_conf: + del vol_conf['external_name'] + version = config.version if version == V1: version = V2_1 @@ -40,7 +45,7 @@ def denormalize_config(config): 'version': version, 'services': services, 'networks': networks, - 'volumes': config.volumes, + 'volumes': volumes, } diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b9766226d..287c043c0 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -262,6 +262,20 @@ class CLITestCase(DockerClientTestCase): } } + def test_config_external_volume(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True + }, + 'bar': { + 'external': {'name': 'some_bar'} + } + } + def test_config_v1(self): self.base_dir = 'tests/fixtures/v1-config' result = self.dispatch(['config']) diff --git a/tests/fixtures/volumes/docker-compose.yml b/tests/fixtures/volumes/docker-compose.yml new file mode 100644 index 000000000..da711ac42 --- /dev/null +++ b/tests/fixtures/volumes/docker-compose.yml @@ -0,0 +1,2 @@ +version: '2.1' +services: {} diff --git a/tests/fixtures/volumes/external-volumes.yml b/tests/fixtures/volumes/external-volumes.yml new file mode 100644 index 000000000..05c6c4844 --- /dev/null +++ b/tests/fixtures/volumes/external-volumes.yml @@ -0,0 +1,16 @@ +version: "2.1" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + bar: + external: + name: some_bar From 644e1716c33e0b3a3dcb4e5227b7dac0a289cffd Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 20 Jan 2017 12:55:59 -0500 Subject: [PATCH 23/43] Add missing network.internal to v3 schema. Signed-off-by: Daniel Nephin --- compose/config/config_schema_v3.0.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 584b6ef5d..fbcd8bb85 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -308,6 +308,7 @@ }, "additionalProperties": false }, + "internal": {"type": "boolean"}, "labels": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false From 20d6f450b5e37e5c634fdf517df32d7484a2b3ff Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Jan 2017 15:05:53 -0800 Subject: [PATCH 24/43] Don't encode build context path on Windows Signed-off-by: Joffrey F --- compose/service.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 20a40c684..724e05652 100644 --- a/compose/service.py +++ b/compose/service.py @@ -22,6 +22,7 @@ from .config import DOCKER_CONFIG_KEYS from .config import merge_environment from .config.types import VolumeSpec from .const import DEFAULT_TIMEOUT +from .const import IS_WINDOWS_PLATFORM from .const import LABEL_CONFIG_HASH from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_ONE_OFF @@ -769,9 +770,9 @@ class Service(object): build_opts = self.options.get('build', {}) path = build_opts.get('context') - # python2 os.path() doesn't support unicode, so we need to encode it to - # a byte string - if not six.PY3: + # python2 os.stat() doesn't support unicode on some UNIX, so we + # encode it to a bytestring to be safe + if not six.PY3 and not IS_WINDOWS_PLATFORM: path = path.encode('utf8') build_output = self.client.build( From e10d1140b95d33a589b1a971e0edda703bfb1e9b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Jan 2017 18:00:09 -0800 Subject: [PATCH 25/43] Convert time data back to string values when serializing config Signed-off-by: Joffrey F --- compose/config/serialize.py | 29 +++++++++++++++++++++++++ tests/acceptance/cli_test.py | 4 ++-- tests/unit/config/config_test.py | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 9ea287a46..3745de82d 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -57,6 +57,25 @@ def serialize_config(config): width=80) +def serialize_ns_time_value(value): + result = (value, 'ns') + table = [ + (1000., 'us'), + (1000., 'ms'), + (1000., 's'), + (60., 'm'), + (60., 'h') + ] + for stage in table: + tmp = value / stage[0] + if tmp == int(value / stage[0]): + value = tmp + result = (int(value), stage[1]) + else: + break + return '{0}{1}'.format(*result) + + def denormalize_service_dict(service_dict, version): service_dict = service_dict.copy() @@ -73,4 +92,14 @@ def denormalize_service_dict(service_dict, version): svc for svc in service_dict['depends_on'].keys() ]) + if 'healthcheck' in service_dict: + if 'interval' in service_dict['healthcheck']: + service_dict['healthcheck']['interval'] = serialize_ns_time_value( + service_dict['healthcheck']['interval'] + ) + if 'timeout' in service_dict['healthcheck']: + service_dict['healthcheck']['timeout'] = serialize_ns_time_value( + service_dict['healthcheck']['timeout'] + ) + return service_dict diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 41a06c950..58160c802 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -353,8 +353,8 @@ class CLITestCase(DockerClientTestCase): 'healthcheck': { 'test': 'cat /etc/passwd', - 'interval': 10000000000, - 'timeout': 1000000000, + 'interval': '10s', + 'timeout': '1s', 'retries': 5, }, diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ab8bfcfcc..d7947a4e8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -23,6 +23,7 @@ from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION from compose.config.serialize import denormalize_service_dict +from compose.config.serialize import serialize_ns_time_value from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from compose.utils import nanoseconds_from_time_seconds @@ -3334,3 +3335,38 @@ class SerializeTest(unittest.TestCase): } assert denormalize_service_dict(service_dict, V2_1) == service_dict + + def test_serialize_time(self): + data = { + 9: '9ns', + 9000: '9us', + 9000000: '9ms', + 90000000: '90ms', + 900000000: '900ms', + 999999999: '999999999ns', + 1000000000: '1s', + 60000000000: '1m', + 60000000001: '60000000001ns', + 9000000000000: '150m', + 90000000000000: '25h', + } + + for k, v in data.items(): + assert serialize_ns_time_value(k) == v + + def test_denormalize_healthcheck(self): + service_dict = { + 'image': 'test', + 'healthcheck': { + 'test': 'exit 1', + 'interval': '1m40s', + 'timeout': '30s', + 'retries': 5 + } + } + processed_service = config.process_service(config.ServiceConfig( + '.', 'test', 'test', service_dict + )) + denormalized_service = denormalize_service_dict(processed_service, V2_1) + assert denormalized_service['healthcheck']['interval'] == '100s' + assert denormalized_service['healthcheck']['timeout'] == '30s' From 5895d8bbc9939524b449e296ac93d8e98aa70eb0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Jan 2017 15:02:27 -0800 Subject: [PATCH 26/43] Detect conflicting version of the docker python SDK and prevent execution until issue is fixed Signed-off-by: Joffrey F --- compose/cli/main.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index c25ccbfa4..db068272a 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -14,6 +14,30 @@ from distutils.spawn import find_executable from inspect import getdoc from operator import attrgetter + +# Attempt to detect https://github.com/docker/compose/issues/4344 +try: + # A regular import statement causes PyInstaller to freak out while + # trying to load pip. This way it is simply ignored. + pip = __import__('pip') + pip_packages = pip.get_installed_distributions() + if 'docker-py' in [pkg.project_name for pkg in pip_packages]: + from .colors import red + print( + red('ERROR:'), + "Dependency conflict: an older version of the 'docker-py' package " + "is polluting the namespace. " + "Run the following command to remedy the issue:\n" + "pip uninstall docker docker-py; pip install docker", + file=sys.stderr + ) + sys.exit(1) +except ImportError: + # pip is not available, which indicates it's probably the binary + # distribution of Compose which is not affected + pass + + from . import errors from . import signals from .. import __version__ From 2593366a3ef1fb0673049687f0ca6733a28cf03f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 16:26:35 -0800 Subject: [PATCH 27/43] Bump docker SDK version Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4b7c7b760..3b06bff45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 colorama==0.3.7 -docker==2.0.1 +docker==2.0.2 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/setup.py b/setup.py index 2f2ba7429..0b1d4e08f 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.0.1, < 3.0', + 'docker >= 2.0.2, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From a82de8863ebdc586a45f54aef348cd17340089e3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Dec 2016 15:44:33 -0500 Subject: [PATCH 28/43] Add v3.1 with secrets. Signed-off-by: Daniel Nephin --- compose/config/config_schema_v3.1.json | 426 +++++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 compose/config/config_schema_v3.1.json diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json new file mode 100644 index 000000000..16616498e --- /dev/null +++ b/compose/config/config_schema_v3.1.json @@ -0,0 +1,426 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.1.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secrets" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "number"}, + "gid": {"type": "number"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_signal": {"type": "string"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": ["object", "null"], + "properties": { + "interval": {"type":"string"}, + "timeout": {"type":"string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "disable": {"type": "boolean"} + }, + "additionalProperties": false + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + } + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} From add56ce8182328fefd7fbe4500360a929ea511df Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Jan 2017 15:58:14 -0500 Subject: [PATCH 29/43] Read service secrets as a type. Signed-off-by: Daniel Nephin --- compose/config/config.py | 13 +++++++++++-- compose/config/config_schema_v3.1.json | 6 +++--- compose/config/types.py | 23 +++++++++++++++++++++-- compose/const.py | 3 +++ 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 7e77421e5..3ca994a79 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -12,10 +12,12 @@ import six import yaml from cached_property import cached_property +from . import types from ..const import COMPOSEFILE_V1 as V1 from ..const import COMPOSEFILE_V2_0 as V2_0 from ..const import COMPOSEFILE_V2_1 as V2_1 from ..const import COMPOSEFILE_V3_0 as V3_0 +from ..const import COMPOSEFILE_V3_1 as V3_1 from ..utils import build_string_dict from ..utils import parse_nanoseconds_int from ..utils import splitdrive @@ -82,6 +84,7 @@ DOCKER_CONFIG_KEYS = [ 'privileged', 'read_only', 'restart', + 'secrets', 'security_opt', 'shm_size', 'stdin_open', @@ -202,8 +205,11 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): def get_networks(self): return {} if self.version == V1 else self.config.get('networks', {}) + def get_secrets(self): + return {} if self.version < V3_1 else self.config.get('secrets', {}) -class Config(namedtuple('_Config', 'version services volumes networks')): + +class Config(namedtuple('_Config', 'version services volumes networks secrets')): """ :param version: configuration version :type version: int @@ -328,6 +334,8 @@ def load(config_details): networks = load_mapping( config_details.config_files, 'get_networks', 'Network' ) + secrets = load_mapping( + config_details.config_files, 'get_secrets', 'Secrets') service_dicts = load_services(config_details, main_file) if main_file.version != V1: @@ -342,7 +350,7 @@ def load(config_details): "`docker stack deploy` to deploy to a swarm." .format(", ".join(sorted(s['name'] for s in services_using_deploy)))) - return Config(main_file.version, service_dicts, volumes, networks) + return Config(main_file.version, service_dicts, volumes, networks, secrets) def load_mapping(config_files, get_func, entity_type): @@ -820,6 +828,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('sysctls', parse_sysctls) md.merge_mapping('depends_on', parse_depends_on) md.merge_sequence('links', ServiceLink.parse) + md.merge_sequence('secrets', types.ServiceSecret.parse) for field in ['volumes', 'devices']: md.merge_field(field, merge_path_mappings) diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index 16616498e..c43f296b5 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -46,7 +46,7 @@ "type": "object", "patternProperties": { "^[a-zA-Z0-9._-]+$": { - "$ref": "#/definitions/secrets" + "$ref": "#/definitions/secret" } }, "additionalProperties": false @@ -188,8 +188,8 @@ "properties": { "source": {"type": "string"}, "target": {"type": "string"}, - "uid": {"type": "number"}, - "gid": {"type": "number"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, "mode": {"type": "number"} } } diff --git a/compose/config/types.py b/compose/config/types.py index 4c106747f..17d5c8b37 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -10,8 +10,8 @@ from collections import namedtuple import six -from compose.config.config import V1 -from compose.config.errors import ConfigurationError +from ..const import COMPOSEFILE_V1 as V1 +from .errors import ConfigurationError from compose.const import IS_WINDOWS_PLATFORM from compose.utils import splitdrive @@ -234,3 +234,22 @@ class ServiceLink(namedtuple('_ServiceLink', 'target alias')): @property def merge_field(self): return self.alias + + +class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): + + @classmethod + def parse(cls, spec): + if isinstance(spec, six.string_types): + return cls(spec, None, None, None, None) + return cls( + spec.get('source'), + spec.get('target'), + spec.get('uid'), + spec.get('gid'), + spec.get('mode'), + ) + + @property + def merge_field(self): + return self.source diff --git a/compose/const.py b/compose/const.py index 1b1be5c76..0f2b00c48 100644 --- a/compose/const.py +++ b/compose/const.py @@ -20,12 +20,14 @@ COMPOSEFILE_V1 = '1' COMPOSEFILE_V2_0 = '2.0' COMPOSEFILE_V2_1 = '2.1' COMPOSEFILE_V3_0 = '3.0' +COMPOSEFILE_V3_1 = '3.1' API_VERSIONS = { COMPOSEFILE_V1: '1.21', COMPOSEFILE_V2_0: '1.22', COMPOSEFILE_V2_1: '1.24', COMPOSEFILE_V3_0: '1.25', + COMPOSEFILE_V3_1: '1.25', } API_VERSION_TO_ENGINE_VERSION = { @@ -33,4 +35,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V2_0]: '1.10.0', API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0', API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0', + API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0', } From e0c6397999464dfe94f7e738dc36b2225f88972f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Jan 2017 17:18:04 -0500 Subject: [PATCH 30/43] Implement secrets using bind mounts Signed-off-by: Daniel Nephin --- compose/config/config.py | 48 +++++++++++++++++++++++++++----------- compose/const.py | 2 ++ compose/project.py | 27 +++++++++++++++++++++ compose/service.py | 23 +++++++++++++++--- tests/unit/bundle_test.py | 3 ++- tests/unit/project_test.py | 12 ++++++++++ 6 files changed, 98 insertions(+), 17 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 3ca994a79..0e8b52e79 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -334,8 +334,7 @@ def load(config_details): networks = load_mapping( config_details.config_files, 'get_networks', 'Network' ) - secrets = load_mapping( - config_details.config_files, 'get_secrets', 'Secrets') + secrets = load_secrets(config_details.config_files, config_details.working_dir) service_dicts = load_services(config_details, main_file) if main_file.version != V1: @@ -364,22 +363,12 @@ def load_mapping(config_files, get_func, entity_type): external = config.get('external') if external: - if len(config.keys()) > 1: - raise ConfigurationError( - '{} {} declared as external but specifies' - ' additional attributes ({}). '.format( - entity_type, - name, - ', '.join([k for k in config.keys() if k != 'external']) - ) - ) + validate_external(entity_type, name, config) if isinstance(external, dict): config['external_name'] = external.get('name') else: config['external_name'] = name - mapping[name] = config - if 'driver_opts' in config: config['driver_opts'] = build_string_dict( config['driver_opts'] @@ -391,6 +380,39 @@ def load_mapping(config_files, get_func, entity_type): return mapping +def validate_external(entity_type, name, config): + if len(config.keys()) <= 1: + return + + raise ConfigurationError( + "{} {} declared as external but specifies additional attributes " + "({}).".format( + entity_type, name, ', '.join(k for k in config if k != 'external'))) + + +def load_secrets(config_files, working_dir): + mapping = {} + + for config_file in config_files: + for name, config in config_file.get_secrets().items(): + mapping[name] = config or {} + if not config: + continue + + external = config.get('external') + if external: + validate_external('Secret', name, config) + if isinstance(external, dict): + config['external_name'] = external.get('name') + else: + config['external_name'] = name + + if 'file' in config: + config['file'] = expand_path(working_dir, config['file']) + + return mapping + + def load_services(config_details, config_file): def build_service(service_name, service_dict, service_names): service_config = ServiceConfig.with_abs_paths( diff --git a/compose/const.py b/compose/const.py index 0f2b00c48..3f8f90ab5 100644 --- a/compose/const.py +++ b/compose/const.py @@ -16,6 +16,8 @@ LABEL_VERSION = 'com.docker.compose.version' LABEL_VOLUME = 'com.docker.compose.volume' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' +SECRETS_PATH = '/run/secrets' + COMPOSEFILE_V1 = '1' COMPOSEFILE_V2_0 = '2.0' COMPOSEFILE_V2_1 = '2.1' diff --git a/compose/project.py b/compose/project.py index d99ef7c93..22576e868 100644 --- a/compose/project.py +++ b/compose/project.py @@ -104,6 +104,11 @@ class Project(object): for volume_spec in service_dict.get('volumes', []) ] + secrets = get_secrets( + service_dict['name'], + service_dict.get('secrets') or [], + config_data.secrets) + project.services.append( Service( service_dict.pop('name'), @@ -114,6 +119,7 @@ class Project(object): links=links, network_mode=network_mode, volumes_from=volumes_from, + secrets=secrets, **service_dict) ) @@ -553,6 +559,27 @@ def get_volumes_from(project, service_dict): return [build_volume_from(vf) for vf in volumes_from] +def get_secrets(service, service_secrets, secret_defs): + secrets = [] + + for secret in service_secrets: + secret_def = secret_defs.get(secret.source) + if not secret_def: + raise ConfigurationError( + "Service \"{service}\" uses an undefined secret \"{secret}\" " + .format(service=service, secret=secret.source)) + + if secret_def.get('external_name'): + log.warn("Service \"{service}\" uses secret \"{secret}\" which is external. " + "External secrets are not available to containers created by " + "docker-compose.".format(service=service, secret=secret.source)) + continue + + secrets.append({'secret': secret, 'file': secret_def.get('file')}) + + return secrets + + def warn_for_swarm_mode(client): info = client.info() if info.get('Swarm', {}).get('LocalNodeState') == 'active': diff --git a/compose/service.py b/compose/service.py index 724e05652..9f2fc68b4 100644 --- a/compose/service.py +++ b/compose/service.py @@ -17,6 +17,7 @@ from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port from . import __version__ +from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment @@ -139,6 +140,7 @@ class Service(object): volumes_from=None, network_mode=None, networks=None, + secrets=None, **options ): self.name = name @@ -149,6 +151,7 @@ class Service(object): self.volumes_from = volumes_from or [] self.network_mode = network_mode or NetworkMode(None) self.networks = networks or {} + self.secrets = secrets or [] self.options = options def __repr__(self): @@ -692,9 +695,14 @@ class Service(object): override_options['binds'] = binds container_options['environment'].update(affinity) - if 'volumes' in container_options: - container_options['volumes'] = dict( - (v.internal, {}) for v in container_options['volumes']) + container_options['volumes'] = dict( + (v.internal, {}) for v in container_options.get('volumes') or {}) + + secret_volumes = self.get_secret_volumes() + if secret_volumes: + override_options['binds'].extend(v.repr() for v in secret_volumes) + container_options['volumes'].update( + (v.internal, {}) for v in secret_volumes) container_options['image'] = self.image_name @@ -765,6 +773,15 @@ class Service(object): return host_config + def get_secret_volumes(self): + def build_spec(secret): + target = '{}/{}'.format( + const.SECRETS_PATH, + secret['secret'].target or secret['secret'].source) + return VolumeSpec(secret['file'], target, 'ro') + + return [build_spec(secret) for secret in self.secrets] + def build(self, no_cache=False, pull=False, force_rm=False): log.info('Building %s' % self.name) diff --git a/tests/unit/bundle_test.py b/tests/unit/bundle_test.py index a279cab05..21bdb31b0 100644 --- a/tests/unit/bundle_test.py +++ b/tests/unit/bundle_test.py @@ -77,7 +77,8 @@ def test_to_bundle(): version=2, services=services, volumes={'special': {}}, - networks={'extra': {}}) + networks={'extra': {}}, + secrets={}) with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: output = bundle.to_bundle(config, image_digests) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 9a12438f2..32d0adfaf 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -36,6 +36,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ) project = Project.from_config( name='composetest', @@ -64,6 +65,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ) project = Project.from_config('composetest', config, None) self.assertEqual(len(project.services), 2) @@ -170,6 +172,7 @@ class ProjectTest(unittest.TestCase): }], networks=None, volumes=None, + secrets=None, ), ) assert project.get_service('test')._get_volumes_from() == [container_id + ":rw"] @@ -202,6 +205,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"] @@ -227,6 +231,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) with mock.patch.object(Service, 'containers') as mock_return: @@ -360,6 +365,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) service = project.get_service('test') @@ -384,6 +390,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) service = project.get_service('test') @@ -417,6 +424,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) @@ -437,6 +445,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) @@ -457,6 +466,7 @@ class ProjectTest(unittest.TestCase): ], networks={'custom': {}}, volumes=None, + secrets=None, ), ) @@ -487,6 +497,7 @@ class ProjectTest(unittest.TestCase): }], networks=None, volumes=None, + secrets=None, ), ) self.assertEqual([c.id for c in project.containers()], ['1']) @@ -503,6 +514,7 @@ class ProjectTest(unittest.TestCase): }], networks={'default': {}}, volumes={'data': {}}, + secrets=None, ), ) self.mock_client.remove_network.side_effect = NotFound(None, None, 'oops') From 4053adc7d356270143f8389d41f857a128d9febb Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 5 Jan 2017 14:54:35 -0500 Subject: [PATCH 31/43] Add an integration test for secrets using bind mounts. Signed-off-by: Daniel Nephin --- compose/project.py | 2 +- tests/fixtures/secrets/default | 1 + tests/integration/project_test.py | 132 ++++++++++++++++++------------ tests/integration/testcases.py | 9 +- 4 files changed, 86 insertions(+), 58 deletions(-) create mode 100644 tests/fixtures/secrets/default diff --git a/compose/project.py b/compose/project.py index 22576e868..e522e2ecf 100644 --- a/compose/project.py +++ b/compose/project.py @@ -106,7 +106,7 @@ class Project(object): secrets = get_secrets( service_dict['name'], - service_dict.get('secrets') or [], + service_dict.pop('secrets', None) or [], config_data.secrets) project.services.append( diff --git a/tests/fixtures/secrets/default b/tests/fixtures/secrets/default new file mode 100644 index 000000000..f9dc20149 --- /dev/null +++ b/tests/fixtures/secrets/default @@ -0,0 +1 @@ +This is the secret diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ee2b7817b..30b107e87 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import os.path import random import py @@ -8,12 +9,14 @@ import pytest from docker.errors import NotFound from .. import mock -from ..helpers import build_config +from ..helpers import build_config as load_config from .testcases import DockerClientTestCase from compose.config import config from compose.config import ConfigurationError +from compose.config import types from compose.config.config import V2_0 from compose.config.config import V2_1 +from compose.config.config import V3_1 from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import LABEL_PROJECT @@ -26,6 +29,16 @@ from compose.project import ProjectError from compose.service import ConvergenceStrategy from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_only +from tests.integration.testcases import v3_only + + +def build_config(**kwargs): + return config.Config( + version=kwargs.get('version'), + services=kwargs.get('services'), + volumes=kwargs.get('volumes'), + networks=kwargs.get('networks'), + secrets=kwargs.get('secrets')) class ProjectTest(DockerClientTestCase): @@ -70,7 +83,7 @@ class ProjectTest(DockerClientTestCase): def test_volumes_from_service(self): project = Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'data': { 'image': 'busybox:latest', 'volumes': ['/var/data'], @@ -96,7 +109,7 @@ class ProjectTest(DockerClientTestCase): ) project = Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'db': { 'image': 'busybox:latest', 'volumes_from': ['composetest_data_container'], @@ -112,7 +125,7 @@ class ProjectTest(DockerClientTestCase): project = Project.from_config( name='composetest', client=self.client, - config_data=build_config({ + config_data=load_config({ 'version': V2_0, 'services': { 'net': { @@ -139,7 +152,7 @@ class ProjectTest(DockerClientTestCase): def get_project(): return Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'version': V2_0, 'services': { 'web': { @@ -174,7 +187,7 @@ class ProjectTest(DockerClientTestCase): def test_net_from_service_v1(self): project = Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'net': { 'image': 'busybox:latest', 'command': ["top"] @@ -198,7 +211,7 @@ class ProjectTest(DockerClientTestCase): def get_project(): return Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'web': { 'image': 'busybox:latest', 'net': 'container:composetest_net_container' @@ -469,7 +482,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_starts_depends(self): project = Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -504,7 +517,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_with_no_deps(self): project = Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -564,7 +577,7 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_project_up_networks(self): - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -576,7 +589,6 @@ class ProjectTest(DockerClientTestCase): 'baz': {'aliases': ['extra']}, }, }], - volumes={}, networks={ 'foo': {'driver': 'bridge'}, 'bar': {'driver': None}, @@ -610,14 +622,13 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_up_with_ipam_config(self): - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'networks': {'front': None}, }], - volumes={}, networks={ 'front': { 'driver': 'bridge', @@ -671,7 +682,7 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_up_with_network_static_addresses(self): - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -684,7 +695,6 @@ class ProjectTest(DockerClientTestCase): } }, }], - volumes={}, networks={ 'static_test': { 'driver': 'bridge', @@ -726,7 +736,7 @@ class ProjectTest(DockerClientTestCase): @v2_1_only() def test_up_with_enable_ipv6(self): self.require_api_version('1.23') - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -738,7 +748,6 @@ class ProjectTest(DockerClientTestCase): } }, }], - volumes={}, networks={ 'static_test': { 'driver': 'bridge', @@ -770,7 +779,7 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_up_with_network_static_addresses_missing_subnet(self): - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -782,7 +791,6 @@ class ProjectTest(DockerClientTestCase): } }, }], - volumes={}, networks={ 'static_test': { 'driver': 'bridge', @@ -807,7 +815,7 @@ class ProjectTest(DockerClientTestCase): @v2_1_only() def test_up_with_network_link_local_ips(self): - config_data = config.Config( + config_data = build_config( version=V2_1, services=[{ 'name': 'web', @@ -818,7 +826,6 @@ class ProjectTest(DockerClientTestCase): } } }], - volumes={}, networks={ 'linklocaltest': {'driver': 'bridge'} } @@ -844,15 +851,13 @@ class ProjectTest(DockerClientTestCase): @v2_1_only() def test_up_with_isolation(self): self.require_api_version('1.24') - config_data = config.Config( + config_data = build_config( version=V2_1, services=[{ 'name': 'web', 'image': 'busybox:latest', 'isolation': 'default' }], - volumes={}, - networks={} ) project = Project.from_config( client=self.client, @@ -866,15 +871,13 @@ class ProjectTest(DockerClientTestCase): @v2_1_only() def test_up_with_invalid_isolation(self): self.require_api_version('1.24') - config_data = config.Config( + config_data = build_config( version=V2_1, services=[{ 'name': 'web', 'image': 'busybox:latest', 'isolation': 'foobar' }], - volumes={}, - networks={} ) project = Project.from_config( client=self.client, @@ -887,14 +890,13 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_project_up_with_network_internal(self): self.require_api_version('1.23') - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'networks': {'internal': None}, }], - volumes={}, networks={ 'internal': {'driver': 'bridge', 'internal': True}, }, @@ -917,14 +919,13 @@ class ProjectTest(DockerClientTestCase): network_name = 'network_with_label' - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'networks': {network_name: None} }], - volumes={}, networks={ network_name: {'labels': {'label_key': 'label_val'}} } @@ -951,7 +952,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -959,7 +960,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {'driver': 'local'}}, - networks={}, ) project = Project.from_config( @@ -979,7 +979,7 @@ class ProjectTest(DockerClientTestCase): volume_name = 'volume_with_label' - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -993,7 +993,6 @@ class ProjectTest(DockerClientTestCase): } } }, - networks={}, ) project = Project.from_config( @@ -1106,7 +1105,7 @@ class ProjectTest(DockerClientTestCase): def test_initialize_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1114,7 +1113,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {}}, - networks={}, ) project = Project.from_config( @@ -1124,14 +1122,14 @@ class ProjectTest(DockerClientTestCase): project.volumes.initialize() volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) - self.assertEqual(volume_data['Driver'], 'local') + assert volume_data['Name'] == full_vol_name + assert volume_data['Driver'] == 'local' @v2_only() def test_project_up_implicit_volume_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1139,7 +1137,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {}}, - networks={}, ) project = Project.from_config( @@ -1152,11 +1149,45 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') + @v3_only() + def test_project_up_with_secrets(self): + config_data = build_config( + version=V3_1, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'cat /run/secrets/special', + 'secrets': [ + types.ServiceSecret.parse({'source': 'super', 'target': 'special'}), + ], + }], + secrets={ + 'super': { + 'file': os.path.abspath('tests/fixtures/secrets/default'), + }, + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up() + project.stop() + + containers = project.containers(stopped=True) + assert len(containers) == 1 + container, = containers + + output = container.logs() + assert output == "This is the secret\n" + @v2_only() def test_initialize_volumes_invalid_volume_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1164,7 +1195,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {'driver': 'foobar'}}, - networks={}, ) project = Project.from_config( @@ -1179,7 +1209,7 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1187,7 +1217,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {'driver': 'local'}}, - networks={}, ) project = Project.from_config( name='composetest', @@ -1218,7 +1247,7 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1226,7 +1255,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {'driver': 'local'}}, - networks={}, ) project = Project.from_config( name='composetest', @@ -1257,7 +1285,7 @@ class ProjectTest(DockerClientTestCase): vol_name = 'composetest_{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) self.client.create_volume(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1267,7 +1295,6 @@ class ProjectTest(DockerClientTestCase): volumes={ vol_name: {'external': True, 'external_name': vol_name} }, - networks=None, ) project = Project.from_config( name='composetest', @@ -1282,7 +1309,7 @@ class ProjectTest(DockerClientTestCase): def test_initialize_volumes_inexistent_external_volume(self): vol_name = '{0:x}'.format(random.getrandbits(32)) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1292,7 +1319,6 @@ class ProjectTest(DockerClientTestCase): volumes={ vol_name: {'external': True, 'external_name': vol_name} }, - networks=None, ) project = Project.from_config( name='composetest', @@ -1349,7 +1375,7 @@ class ProjectTest(DockerClientTestCase): } } - config_data = build_config(config_dict) + config_data = load_config(config_dict) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) @@ -1357,7 +1383,7 @@ class ProjectTest(DockerClientTestCase): config_dict['service2'] = config_dict['service1'] del config_dict['service1'] - config_data = build_config(config_dict) + config_data = load_config(config_dict) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 230bd2d92..efc1551b4 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -41,9 +41,9 @@ def engine_max_version(): version = os.environ['DOCKER_VERSION'].partition('-')[0] if version_lt(version, '1.10'): return V1 - elif version_lt(version, '1.12'): + if version_lt(version, '1.12'): return V2_0 - elif version_lt(version, '1.13'): + if version_lt(version, '1.13'): return V2_1 return V3_0 @@ -52,8 +52,9 @@ def build_version_required_decorator(ignored_versions): def decorator(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): - if engine_max_version() in ignored_versions: - skip("Engine version is too low") + max_version = engine_max_version() + if max_version in ignored_versions: + skip("Engine version %s is too low" % max_version) return return f(self, *args, **kwargs) return wrapper From 0d609b68acd12948f181200b3dac85b24c9e1441 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 5 Jan 2017 15:00:33 -0500 Subject: [PATCH 32/43] Add a warning for unsupported secret fields. Signed-off-by: Daniel Nephin --- compose/project.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compose/project.py b/compose/project.py index e522e2ecf..0330ab80f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -575,6 +575,12 @@ def get_secrets(service, service_secrets, secret_defs): "docker-compose.".format(service=service, secret=secret.source)) continue + if secret.uid or secret.gid or secret.mode: + log.warn("Service \"{service}\" uses secret \"{secret}\" with uid, " + "gid, or mode. These fields are not supported by this " + "implementation of the Compose file".format( + service=service, secret=secret.source)) + secrets.append({'secret': secret, 'file': secret_def.get('file')}) return secrets From 3a2735abb933fc8f067e888e6009eac9e2be3132 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 20 Jan 2017 17:10:23 -0500 Subject: [PATCH 33/43] Rebase compose v3.1 on the latest v3 Signed-off-by: Daniel Nephin --- compose/config/config_schema_v3.1.json | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index c43f296b5..b7037485f 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -198,8 +198,8 @@ }, "sysctls": {"$ref": "#/definitions/list_or_dict"}, "stdin_open": {"type": "boolean"}, - "stop_signal": {"type": "string"}, "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, "ulimits": { @@ -231,10 +231,11 @@ "healthcheck": { "id": "#/definitions/healthcheck", - "type": ["object", "null"], + "type": "object", + "additionalProperties": false, "properties": { - "interval": {"type":"string"}, - "timeout": {"type":"string"}, + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, "retries": {"type": "number"}, "test": { "oneOf": [ @@ -242,9 +243,8 @@ {"type": "array", "items": {"type": "string"}} ] }, - "disable": {"type": "boolean"} - }, - "additionalProperties": false + "timeout": {"type": "string"} + } }, "deployment": { "id": "#/definitions/deployment", @@ -337,6 +337,7 @@ }, "additionalProperties": false }, + "internal": {"type": "boolean"}, "labels": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false @@ -357,10 +358,11 @@ "type": ["boolean", "object"], "properties": { "name": {"type": "string"} - } - } + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} }, - "labels": {"$ref": "#/definitions/list_or_dict"}, "additionalProperties": false }, @@ -374,9 +376,9 @@ "properties": { "name": {"type": "string"} } - } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} }, - "labels": {"$ref": "#/definitions/list_or_dict"}, "additionalProperties": false }, From 59d1847d9bc88f9b4248267e93fe0435ce973da9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 27 Jan 2017 12:51:05 -0500 Subject: [PATCH 34/43] Fix some test failures. Signed-off-by: Daniel Nephin --- tests/integration/project_test.py | 37 +++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 30b107e87..28762cd20 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1151,6 +1151,8 @@ class ProjectTest(DockerClientTestCase): @v3_only() def test_project_up_with_secrets(self): + create_host_file(self.client, os.path.abspath('tests/fixtures/secrets/default')) + config_data = build_config( version=V3_1, services=[{ @@ -1181,7 +1183,7 @@ class ProjectTest(DockerClientTestCase): container, = containers output = container.logs() - assert output == "This is the secret\n" + assert output == b"This is the secret\n" @v2_only() def test_initialize_volumes_invalid_volume_driver(self): @@ -1428,7 +1430,7 @@ class ProjectTest(DockerClientTestCase): } } } - config_data = build_config(config_dict) + config_data = load_config(config_dict) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) @@ -1465,7 +1467,7 @@ class ProjectTest(DockerClientTestCase): } } } - config_data = build_config(config_dict) + config_data = load_config(config_dict) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) @@ -1501,7 +1503,7 @@ class ProjectTest(DockerClientTestCase): } } } - config_data = build_config(config_dict) + config_data = load_config(config_dict) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) @@ -1515,3 +1517,30 @@ class ProjectTest(DockerClientTestCase): assert 'svc1' in svc2.get_dependency_names() with pytest.raises(NoHealthCheckConfigured): svc1.is_healthy() + + +def create_host_file(client, filename): + dirname = os.path.dirname(filename) + + with open(filename, 'r') as fh: + content = fh.read() + + container = client.create_container( + 'busybox:latest', + ['sh', '-c', 'echo -n "{}" > {}'.format(content, filename)], + volumes={dirname: {}}, + host_config=client.create_host_config( + binds={dirname: {'bind': dirname, 'ro': False}}, + network_mode='none', + ), + ) + try: + client.start(container) + exitcode = client.wait(container) + + if exitcode != 0: + output = client.logs(container) + raise Exception( + "Container exited with code {}:\n{}".format(exitcode, output)) + finally: + client.remove_container(container, force=True) From 8efb7e6e8bbd1542db768fc1b90c6c7282f0944b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 31 Jan 2017 12:51:46 -0800 Subject: [PATCH 35/43] Don't strip ANSI color codes when output is not a TTY Signed-off-by: Joffrey F --- compose/cli/colors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/colors.py b/compose/cli/colors.py index 6677a376a..f1251e431 100644 --- a/compose/cli/colors.py +++ b/compose/cli/colors.py @@ -33,7 +33,7 @@ def make_color_fn(code): return lambda s: ansi_color(code, s) -colorama.init() +colorama.init(strip=False) for (name, code) in get_pairs(): globals()[name] = make_color_fn(code) From a67500ee5728032aa902a640014bc35b6ab4d715 Mon Sep 17 00:00:00 2001 From: Peter Urda Date: Fri, 14 Oct 2016 00:44:52 -0700 Subject: [PATCH 36/43] Added `top` to `docker-compose` to display running processes. This commit allows `docker-compose` to access `top` for containers much like running `docker top` directly on a given container. This commit includes: * `docker-compose` CLI changes to expose `top` * Completions for `bash` and `zsh` * Required testing for the new `top` command Signed-off-by: Peter Urda --- compose/cli/main.py | 28 ++++++++++++++++++++++++++ contrib/completion/bash/docker-compose | 13 ++++++++++++ contrib/completion/zsh/_docker-compose | 5 +++++ tests/acceptance/cli_test.py | 20 ++++++++++++++++++ tests/fixtures/top/docker-compose.yml | 6 ++++++ 5 files changed, 72 insertions(+) create mode 100644 tests/fixtures/top/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index db068272a..e2ebce48e 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -215,6 +215,7 @@ class TopLevelCommand(object): scale Set number of containers for a service start Start services stop Stop services + top Display the running processes unpause Unpause services up Create and start containers version Show the Docker-Compose version information @@ -800,6 +801,33 @@ class TopLevelCommand(object): containers = self.project.restart(service_names=options['SERVICE'], timeout=timeout) exit_if(not containers, 'No containers to restart', 1) + def top(self, options): + """ + Display the running processes + + Usage: top [SERVICE...] + + """ + containers = sorted( + self.project.containers(service_names=options['SERVICE'], stopped=False) + + self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only), + key=attrgetter('name') + ) + + for idx, container in enumerate(containers): + if idx > 0: + print() + + top_data = self.project.client.top(container.name) + headers = top_data.get("Titles") + rows = [] + + for process in top_data.get("Processes", []): + rows.append(process) + + print(container.name) + print(Formatter().table(headers, rows)) + def unpause(self, options): """ Unpause services. diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 991f65729..77d02b428 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -434,6 +434,18 @@ _docker_compose_stop() { } +_docker_compose_top() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + ;; + *) + __docker_compose_services_running + ;; + esac +} + + _docker_compose_unpause() { case "$cur" in -*) @@ -499,6 +511,7 @@ _docker_compose() { scale start stop + top unpause up version diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index ceb7d0f58..66d924f73 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -341,6 +341,11 @@ __docker-compose_subcommand() { $opts_timeout \ '*:running services:__docker-compose_runningservices' && ret=0 ;; + (top) + _arguments \ + $opts_help \ + '*:running services:__docker-compose_runningservices' && ret=0 + ;; (unpause) _arguments \ $opts_help \ diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 58160c802..160e1913d 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1907,3 +1907,23 @@ class CLITestCase(DockerClientTestCase): "BAZ=2", ]) self.assertTrue(expected_env <= set(web.get('Config.Env'))) + + def test_top_services_not_running(self): + self.base_dir = 'tests/fixtures/top' + result = self.dispatch(['top']) + assert len(result.stdout) == 0 + + def test_top_services_running(self): + self.base_dir = 'tests/fixtures/top' + self.dispatch(['up', '-d']) + result = self.dispatch(['top']) + + self.assertIn('top_service_a', result.stdout) + self.assertIn('top_service_b', result.stdout) + self.assertNotIn('top_not_a_service', result.stdout) + + def test_top_processes_running(self): + self.base_dir = 'tests/fixtures/top' + self.dispatch(['up', '-d']) + result = self.dispatch(['top']) + assert result.stdout.count("top") == 4 diff --git a/tests/fixtures/top/docker-compose.yml b/tests/fixtures/top/docker-compose.yml new file mode 100644 index 000000000..d632a836e --- /dev/null +++ b/tests/fixtures/top/docker-compose.yml @@ -0,0 +1,6 @@ +service_a: + image: busybox:latest + command: top +service_b: + image: busybox:latest + command: top From 7e8958e6cab8edbfabd732e29a1c01b375a8bc02 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Feb 2017 15:43:20 -0800 Subject: [PATCH 37/43] Add missing comma in DOCKER_CONFIG_KEYS list Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- tests/unit/config/config_test.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0e8b52e79..746f63d53 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -78,7 +78,7 @@ DOCKER_CONFIG_KEYS = [ 'memswap_limit', 'mem_swappiness', 'net', - 'oom_score_adj' + 'oom_score_adj', 'pid', 'ports', 'privileged', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d7947a4e8..860e58353 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1748,6 +1748,24 @@ class ConfigTest(unittest.TestCase): } } + def test_merge_pid(self): + # Regression: https://github.com/docker/compose/issues/4184 + base = { + 'image': 'busybox', + 'pid': 'host' + } + + override = { + 'labels': {'com.docker.compose.test': 'yes'} + } + + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'busybox', + 'pid': 'host', + 'labels': {'com.docker.compose.test': 'yes'} + } + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From cf43e6edf7c734c3a98306bf9b4a01eb7f516005 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Feb 2017 14:22:50 -0800 Subject: [PATCH 38/43] Don't re-parse healthcheck values coming from extended services Signed-off-by: Joffrey F --- compose/config/config.py | 10 ++++++++-- tests/fixtures/extends/healthcheck-1.yml | 9 +++++++++ tests/fixtures/extends/healthcheck-2.yml | 6 ++++++ tests/unit/config/config_test.py | 13 +++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/extends/healthcheck-1.yml create mode 100644 tests/fixtures/extends/healthcheck-2.yml diff --git a/compose/config/config.py b/compose/config/config.py index 0e8b52e79..63ee25ab2 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -716,9 +716,15 @@ def process_healthcheck(service_dict, service_name): hc['test'] = raw['test'] if 'interval' in raw: - hc['interval'] = parse_nanoseconds_int(raw['interval']) + if not isinstance(raw['interval'], six.integer_types): + hc['interval'] = parse_nanoseconds_int(raw['interval']) + else: # Conversion has been done previously + hc['interval'] = raw['interval'] if 'timeout' in raw: - hc['timeout'] = parse_nanoseconds_int(raw['timeout']) + if not isinstance(raw['timeout'], six.integer_types): + hc['timeout'] = parse_nanoseconds_int(raw['timeout']) + else: # Conversion has been done previously + hc['timeout'] = raw['timeout'] if 'retries' in raw: hc['retries'] = raw['retries'] diff --git a/tests/fixtures/extends/healthcheck-1.yml b/tests/fixtures/extends/healthcheck-1.yml new file mode 100644 index 000000000..4c311e62c --- /dev/null +++ b/tests/fixtures/extends/healthcheck-1.yml @@ -0,0 +1,9 @@ +version: '2.1' +services: + demo: + image: foobar:latest + healthcheck: + test: ["CMD", "/health.sh"] + interval: 10s + timeout: 5s + retries: 36 diff --git a/tests/fixtures/extends/healthcheck-2.yml b/tests/fixtures/extends/healthcheck-2.yml new file mode 100644 index 000000000..11bc9f09d --- /dev/null +++ b/tests/fixtures/extends/healthcheck-2.yml @@ -0,0 +1,6 @@ +version: '2.1' +services: + demo: + extends: + file: healthcheck-1.yml + service: demo diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d7947a4e8..a3be6df8a 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3098,6 +3098,19 @@ class ExtendsTest(unittest.TestCase): 'other': {'condition': 'service_started'} } + def test_extends_with_healthcheck(self): + service_dicts = load_from_filename('tests/fixtures/extends/healthcheck-2.yml') + assert service_sort(service_dicts) == [{ + 'name': 'demo', + 'image': 'foobar:latest', + 'healthcheck': { + 'test': ['CMD', '/health.sh'], + 'interval': 10000000000, + 'timeout': 5000000000, + 'retries': 36, + } + }] + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From 9a59a9c3ff6dd7b27530d51c53008219525609d8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 16:20:59 -0800 Subject: [PATCH 39/43] Bump 1.10.1 Signed-off-by: Joffrey F --- CHANGELOG.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6699f8807..d0681e8af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,53 @@ Change log ========== +1.11.0 (2017-02-08) +------------------- + +### New Features + +#### Compose file version 3.1 + +- Introduced version 3.1 of the `docker-compose.yml` specification. This + version requires Docker Engine 1.13.0 or above. It introduces support + for secrets. See the documentation for more information + +#### Compose file version 2.0 and up + +- Introduced the `docker-compose top` command that displays processes running + for the different services managed by Compose. + +### Bugfixes + +- Fixed a bug where extending a service defining a healthcheck dictionary + would cause `docker-compose` to error out. + +- Fixed an issue where the `pid` entry in a service definition was being + ignored when using multiple Compose files. + +1.10.1 (2017-02-01) +------------------ + +### Bugfixes + +- Fixed an issue where presence of older versions of the docker-py + package would cause unexpected crashes while running Compose + +- Fixed an issue where healthcheck dependencies would be lost when + using multiple compose files for a project + +- Fixed a few issues that made the output of the `config` command + invalid + +- Fixed an issue where adding volume labels to v3 Compose files would + result in an error + +- Fixed an issue on Windows where build context paths containing unicode + characters were being improperly encoded + +- Fixed a bug where Compose would occasionally crash while streaming logs + when containers would stop or restart + 1.10.0 (2017-01-18) ------------------- From 8f72dadd75b7ebee311dd4da65a3c7f7216b9f28 Mon Sep 17 00:00:00 2001 From: Kevin Jing Qiu Date: Thu, 26 Jan 2017 14:23:12 -0500 Subject: [PATCH 40/43] Close the open file handle using context manager Signed-off-by: Kevin Jing Qiu --- compose/config/environment.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/config/environment.py b/compose/config/environment.py index 7b9269300..4ba228c8a 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import codecs +import contextlib import logging import os @@ -31,11 +32,12 @@ def env_vars_from_file(filename): elif not os.path.isfile(filename): raise ConfigurationError("%s is not a file." % (filename)) env = {} - for line in codecs.open(filename, 'r', 'utf-8'): - line = line.strip() - if line and not line.startswith('#'): - k, v = split_env(line) - env[k] = v + with contextlib.closing(codecs.open(filename, 'r', 'utf-8')) as fileobj: + for line in fileobj: + line = line.strip() + if line and not line.startswith('#'): + k, v = split_env(line) + env[k] = v return env From 0519afd5d38b692648239eb36dbebadf4dc74a4a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 6 Jun 2016 15:48:47 -0700 Subject: [PATCH 41/43] Use newer version of PyInstaller to fix prelinking issues Signed-off-by: Joffrey F --- requirements-build.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-build.txt b/requirements-build.txt index 3f1dbd75b..27f610ca9 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -pyinstaller==3.1.1 +pyinstaller==3.2.1 From 6e9a894ccfcf0a92460a6b86a60e0c4ad8b73318 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 15:16:09 -0800 Subject: [PATCH 42/43] Upgrade python and pip versions in Dockerfile Add libbz2 dependency Signed-off-by: Joffrey F --- Dockerfile | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 63fac3eb3..a03e15106 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN set -ex; \ ca-certificates \ curl \ libsqlite3-dev \ + libbz2-dev \ ; \ rm -rf /var/lib/apt/lists/* @@ -20,40 +21,32 @@ RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.8.3 \ -o /usr/local/bin/docker && \ chmod +x /usr/local/bin/docker -# Build Python 2.7.9 from source +# Build Python 2.7.13 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz | tar -xz; \ - cd Python-2.7.9; \ + curl -L https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz | tar -xz; \ + cd Python-2.7.13; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-2.7.9 + rm -rf /Python-2.7.13 # Build python 3.4 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tgz | tar -xz; \ - cd Python-3.4.3; \ + curl -L https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz | tar -xz; \ + cd Python-3.4.6; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-3.4.3 + rm -rf /Python-3.4.6 # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib -# Install setuptools -RUN set -ex; \ - curl -L https://bootstrap.pypa.io/ez_setup.py | python - # Install pip RUN set -ex; \ - curl -L https://pypi.python.org/packages/source/p/pip/pip-8.1.1.tar.gz | tar -xz; \ - cd pip-8.1.1; \ - python setup.py install; \ - cd ..; \ - rm -rf pip-8.1.1 + curl -L https://bootstrap.pypa.io/get-pip.py | python # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen From 0ea24e7a805a84f82111de7224961b4ac0c3ee3d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Feb 2017 15:26:34 -0800 Subject: [PATCH 43/43] Bump 1.11.0-rc1 Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/__init__.py b/compose/__init__.py index 384178364..ae8d759d4 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.11.0dev' +__version__ = '1.11.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 5872b081a..9de11d5fa 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.8.0" +VERSION="1.11.0-rc1" IMAGE="docker/compose:$VERSION"