From 2fed3998814b8a4b35d5d61e2c637810577ba1ba Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Fri, 23 Feb 2018 16:33:01 +0100 Subject: [PATCH 01/42] Add '--workdir' option to 'exec'. Signed-off-by: Matthieu Nottale --- compose/cli/main.py | 12 +++++++++++- tests/acceptance/cli_test.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 39178fb3c..be3a52f20 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -435,6 +435,7 @@ class TopLevelCommand(object): instances of a service [default: 1] -e, --env KEY=VAL Set environment variables (can be used multiple times, not supported in API < 1.25) + -w, --workdir DIR Path to workdir directory for this command. """ environment = Environment.from_env_file(self.project_dir) use_cli = not environment.get_boolean('COMPOSE_INTERACTIVE_NO_CLI') @@ -443,7 +444,12 @@ class TopLevelCommand(object): detach = options.get('--detach') if options['--env'] and docker.utils.version_lt(self.project.client.api_version, '1.25'): - raise UserError("Setting environment for exec is not supported in API < 1.25'") + raise UserError("Setting environment for exec is not supported in API < 1.25 (%s)" + % self.project.client.api_version) + + if options['--workdir'] and docker.utils.version_lt(self.project.client.api_version, '1.35'): + raise UserError("Setting workdir for exec is not supported in API < 1.35 (%s)" + % self.project.client.api_version) try: container = service.get_container(number=index) @@ -460,6 +466,7 @@ class TopLevelCommand(object): "user": options["--user"], "tty": tty, "stdin": True, + "workdir": options["--workdir"], } if docker.utils.version_gte(self.project.client.api_version, '1.25'): @@ -1392,6 +1399,9 @@ def build_exec_command(options, container_id, command): for env_variable in options["--env"]: args += ["--env", env_variable] + if options["--workdir"]: + args += ["--workdir", options["--workdir"]] + args += [container_id] args += command return args diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 66857307b..e1fbaea56 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1521,6 +1521,16 @@ class CLITestCase(DockerClientTestCase): assert stdout == "operator\n" assert stderr == "" + @v3_only() + def test_exec_workdir(self): + self.base_dir = 'tests/fixtures/links-composefile' + os.environ['COMPOSE_API_VERSION'] = '1.35' + self.dispatch(['up', '-d', 'console']) + assert len(self.project.containers()) == 1 + + stdout, stderr = self.dispatch(['exec', '-T', '--workdir', '/etc', 'console', 'ls']) + assert 'passwd' in stdout + @v2_2_only() def test_exec_service_with_environment_overridden(self): name = 'service' From 7049bea1bb932e96309ef71f6fe21a1b4f22a805 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 1 Mar 2018 15:32:41 +0100 Subject: [PATCH 02/42] Add support for options added in 1.20.0 to bash completion New options: - `docker-compose --log-level` - `docker-compose pull --include-deps` - `docker-compose run --use-aliases` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 30 +++++++++++++++++--------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 29853b089..90c9ce5fc 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -179,18 +179,22 @@ _docker_compose_docker_compose() { _filedir "y?(a)ml" return ;; + --log-level) + COMPREPLY=( $( compgen -W "debug info warning error critical" -- "$cur" ) ) + return + ;; --project-directory) _filedir -d return ;; - $(__docker_compose_to_extglob "$top_level_options_with_args") ) + $(__docker_compose_to_extglob "$daemon_options_with_args") ) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "$top_level_boolean_options $top_level_options_with_args --help -h --no-ansi --verbose --version -v" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "$daemon_boolean_options $daemon_options_with_args $top_level_options_with_args --help -h --no-ansi --verbose --version -v" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -375,7 +379,7 @@ _docker_compose_ps() { _docker_compose_pull() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --parallel --quiet -q" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --include-deps --parallel --quiet -q" -- "$cur" ) ) ;; *) __docker_compose_services_from_image @@ -444,7 +448,7 @@ _docker_compose_run() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --detach --entrypoint -e --help --label -l --name --no-deps --publish -p --rm --service-ports -T --user -u --volume -v --workdir -w" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--detach -d --entrypoint -e --help --label -l --name --no-deps --publish -p --rm --service-ports -T --use-aliases --user -u --volume -v --workdir -w" -- "$cur" ) ) ;; *) __docker_compose_services_all @@ -605,14 +609,12 @@ _docker_compose() { # Options for the docker daemon that have to be passed to secondary calls to # docker-compose executed by this script. - # Other global otions that are not relevant for secondary calls are defined in - # `_docker_compose_docker_compose`. - local top_level_boolean_options=" + local daemon_boolean_options=" --skip-hostname-check --tls --tlsverify " - local top_level_options_with_args=" + local daemon_options_with_args=" --file -f --host -H --project-directory @@ -622,6 +624,11 @@ _docker_compose() { --tlskey " + # These options are require special treatment when searching the command. + local top_level_options_with_args=" + --log-level + " + COMPREPLY=() local cur prev words cword _get_comp_words_by_ref -n : cur prev words cword @@ -634,15 +641,18 @@ _docker_compose() { while [ $counter -lt $cword ]; do case "${words[$counter]}" in - $(__docker_compose_to_extglob "$top_level_boolean_options") ) + $(__docker_compose_to_extglob "$daemon_boolean_options") ) local opt=${words[counter]} top_level_options+=($opt) ;; - $(__docker_compose_to_extglob "$top_level_options_with_args") ) + $(__docker_compose_to_extglob "$daemon_options_with_args") ) local opt=${words[counter]} local arg=${words[++counter]} top_level_options+=($opt $arg) ;; + $(__docker_compose_to_extglob "$top_level_options_with_args") ) + (( counter++ )) + ;; -*) ;; *) From 17610e8d19216610dfc8111671ab84535cbcc2d3 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Thu, 1 Mar 2018 14:09:11 +0100 Subject: [PATCH 03/42] Fix a race condition in ParallelStreamWriter. Signed-off-by: Matthieu Nottale --- compose/parallel.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compose/parallel.py b/compose/parallel.py index 341ca2f5e..dd83c70cd 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import logging import operator import sys +from threading import Lock from threading import Semaphore from threading import Thread @@ -251,6 +252,7 @@ class ParallelStreamWriter(object): """ noansi = False + lock = Lock() @classmethod def set_noansi(cls, value=True): @@ -274,6 +276,7 @@ class ParallelStreamWriter(object): self.stream.flush() def _write_ansi(self, obj_index, status): + self.lock.acquire() position = self.lines.index(obj_index) diff = len(self.lines) - position # move up @@ -285,6 +288,7 @@ class ParallelStreamWriter(object): # move back down self.stream.write("%c[%dB" % (27, diff)) self.stream.flush() + self.lock.release() def _write_noansi(self, obj_index, status): self.stream.write("{} {:<{width}} ... {}\r\n".format(self.msg, obj_index, From efd3d1db064689093ac5bc4a730820f59faa21d7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 2 Mar 2018 15:45:37 -0800 Subject: [PATCH 04/42] Install both versions of python Signed-off-by: Joffrey F --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7661c6470..6ee2a60e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: - checkout - run: name: install python3 - command: brew update > /dev/null && brew install python3 + command: brew update > /dev/null && brew upgrade python - run: name: install tox command: sudo pip install --upgrade tox==2.1.1 From d9e023f79f5e9cc5d5e4934255f14f09ae327661 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Mar 2018 15:38:42 -0800 Subject: [PATCH 05/42] SDK version 3.1.1 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index da05e4212..33462d496 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.1.0 +docker==3.1.1 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index d1788df0a..cf8f6dc13 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.1.0, < 4.0', + 'docker >= 3.1.1, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 31dcfcff2ad9124d028e642e5dd61530714b15c7 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Mon, 5 Mar 2018 14:28:46 +0100 Subject: [PATCH 06/42] Revamp ParallelStreamWriter to fix display issues. Signed-off-by: Matthieu Nottale --- compose/parallel.py | 112 ++++++++++++++++-------------- compose/service.py | 3 +- tests/integration/service_test.py | 3 + tests/unit/parallel_test.py | 3 + tests/unit/service_test.py | 2 + 5 files changed, 69 insertions(+), 54 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index dd83c70cd..5d4791f97 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -43,7 +43,36 @@ class GlobalLimit(object): cls.global_limiter = Semaphore(value) -def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, parent_objects=None): +def parallel_execute_watch(events, writer, errors, results, msg, get_name): + """ Watch events from a parallel execution, update status and fill errors and results. + Returns exception to re-raise. + """ + error_to_reraise = None + for obj, result, exception in events: + if exception is None: + writer.write(msg, get_name(obj), 'done', green) + results.append(result) + elif isinstance(exception, ImageNotFound): + # This is to bubble up ImageNotFound exceptions to the client so we + # can prompt the user if they want to rebuild. + errors[get_name(obj)] = exception.explanation + writer.write(msg, get_name(obj), 'error', red) + error_to_reraise = exception + elif isinstance(exception, APIError): + errors[get_name(obj)] = exception.explanation + writer.write(msg, get_name(obj), 'error', red) + elif isinstance(exception, (OperationFailedError, HealthCheckFailed, NoHealthCheckConfigured)): + errors[get_name(obj)] = exception.msg + writer.write(msg, get_name(obj), 'error', red) + elif isinstance(exception, UpstreamError): + writer.write(msg, get_name(obj), 'error', red) + else: + errors[get_name(obj)] = exception + error_to_reraise = exception + return error_to_reraise + + +def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): """Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. @@ -53,45 +82,21 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, pa objects = list(objects) stream = get_output_stream(sys.stderr) - writer = ParallelStreamWriter(stream, msg) + if ParallelStreamWriter.instance: + writer = ParallelStreamWriter.instance + else: + writer = ParallelStreamWriter(stream) - display_objects = list(parent_objects) if parent_objects else objects - - for obj in display_objects: - writer.add_object(get_name(obj)) - - # write data in a second loop to consider all objects for width alignment - # and avoid duplicates when parent_objects exists for obj in objects: - writer.write_initial(get_name(obj)) + writer.add_object(msg, get_name(obj)) + for obj in objects: + writer.write_initial(msg, get_name(obj)) events = parallel_execute_iter(objects, func, get_deps, limit) errors = {} results = [] - error_to_reraise = None - - for obj, result, exception in events: - if exception is None: - writer.write(get_name(obj), 'done', green) - results.append(result) - elif isinstance(exception, ImageNotFound): - # This is to bubble up ImageNotFound exceptions to the client so we - # can prompt the user if they want to rebuild. - errors[get_name(obj)] = exception.explanation - writer.write(get_name(obj), 'error', red) - error_to_reraise = exception - elif isinstance(exception, APIError): - errors[get_name(obj)] = exception.explanation - writer.write(get_name(obj), 'error', red) - elif isinstance(exception, (OperationFailedError, HealthCheckFailed, NoHealthCheckConfigured)): - errors[get_name(obj)] = exception.msg - writer.write(get_name(obj), 'error', red) - elif isinstance(exception, UpstreamError): - writer.write(get_name(obj), 'error', red) - else: - errors[get_name(obj)] = exception - error_to_reraise = exception + error_to_reraise = parallel_execute_watch(events, writer, errors, results, msg, get_name) for obj_name, error in errors.items(): stream.write("\nERROR: for {} {}\n".format(obj_name, error)) @@ -253,55 +258,58 @@ class ParallelStreamWriter(object): noansi = False lock = Lock() + instance = None @classmethod def set_noansi(cls, value=True): cls.noansi = value - def __init__(self, stream, msg): + def __init__(self, stream): self.stream = stream - self.msg = msg self.lines = [] self.width = 0 + ParallelStreamWriter.instance = self - def add_object(self, obj_index): - self.lines.append(obj_index) - self.width = max(self.width, len(obj_index)) - - def write_initial(self, obj_index): - if self.msg is None: + def add_object(self, msg, obj_index): + if msg is None: return - self.stream.write("{} {:<{width}} ... \r\n".format( - self.msg, self.lines[self.lines.index(obj_index)], width=self.width)) + self.lines.append(msg + obj_index) + self.width = max(self.width, len(msg + ' ' + obj_index)) + + def write_initial(self, msg, obj_index): + if msg is None: + return + self.stream.write("{:<{width}} ... \r\n".format( + msg + ' ' + obj_index, width=self.width)) self.stream.flush() - def _write_ansi(self, obj_index, status): + def _write_ansi(self, msg, obj_index, status): self.lock.acquire() - position = self.lines.index(obj_index) + position = self.lines.index(msg + obj_index) diff = len(self.lines) - position # move up self.stream.write("%c[%dA" % (27, diff)) # erase self.stream.write("%c[2K\r" % 27) - self.stream.write("{} {:<{width}} ... {}\r".format(self.msg, obj_index, + self.stream.write("{:<{width}} ... {}\r".format(msg + ' ' + obj_index, status, width=self.width)) # move back down self.stream.write("%c[%dB" % (27, diff)) self.stream.flush() self.lock.release() - def _write_noansi(self, obj_index, status): - self.stream.write("{} {:<{width}} ... {}\r\n".format(self.msg, obj_index, + def _write_noansi(self, msg, obj_index, status): + self.stream.write("{:<{width}} ... {}\r\n".format(msg + ' ' + obj_index, status, width=self.width)) self.stream.flush() - def write(self, obj_index, status, color_func): - if self.msg is None: + def write(self, msg, obj_index, status, color_func): + if msg is None: return if self.noansi: - self._write_noansi(obj_index, status) + self._write_noansi(msg, obj_index, status) else: - self._write_ansi(obj_index, color_func(status)) + self._write_ansi(msg, obj_index, color_func(status)) def parallel_operation(containers, operation, options, message): diff --git a/compose/service.py b/compose/service.py index b9f9af2cd..652080478 100644 --- a/compose/service.py +++ b/compose/service.py @@ -402,8 +402,7 @@ class Service(object): [ServiceName(self.project, self.name, index) for index in range(i, i + scale)], lambda service_name: create_and_start(self, service_name.number), lambda service_name: self.get_container_name(service_name.service, service_name.number), - "Creating", - parent_objects=project_services + "Creating" ) for error in errors.values(): raise OperationFailedError(error) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2b6b7711e..6e86a02d4 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -35,6 +35,7 @@ from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION from compose.container import Container from compose.errors import OperationFailedError +from compose.parallel import ParallelStreamWriter from compose.project import OneOffFilter from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy @@ -1197,6 +1198,7 @@ class ServiceTest(DockerClientTestCase): service.create_container(number=next_number) service.create_container(number=next_number + 1) + ParallelStreamWriter.instance = None with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: service.scale(2) for container in service.containers(): @@ -1220,6 +1222,7 @@ class ServiceTest(DockerClientTestCase): for container in service.containers(): assert not container.is_running + ParallelStreamWriter.instance = None with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: service.scale(2) diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 4ebc24d8c..0735bfccb 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -143,6 +143,7 @@ class ParallelTest(unittest.TestCase): def test_parallel_execute_alignment(capsys): + ParallelStreamWriter.instance = None results, errors = parallel_execute( objects=["short", "a very long name"], func=lambda x: x, @@ -158,6 +159,7 @@ def test_parallel_execute_alignment(capsys): def test_parallel_execute_ansi(capsys): + ParallelStreamWriter.instance = None ParallelStreamWriter.set_noansi(value=False) results, errors = parallel_execute( objects=["something", "something more"], @@ -173,6 +175,7 @@ def test_parallel_execute_ansi(capsys): def test_parallel_execute_noansi(capsys): + ParallelStreamWriter.instance = None ParallelStreamWriter.set_noansi() results, errors = parallel_execute( objects=["something", "something more"], diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c315dcc4d..9128b9550 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -20,6 +20,7 @@ from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import SECRETS_PATH from compose.container import Container +from compose.parallel import ParallelStreamWriter from compose.project import OneOffFilter from compose.service import build_ulimits from compose.service import build_volume_binding @@ -727,6 +728,7 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.log', autospec=True) def test_only_log_warning_when_host_ports_clash(self, mock_log): self.mock_client.inspect_image.return_value = {'Id': 'abcd'} + ParallelStreamWriter.instance = None name = 'foo' service = Service( name, From b9f9643d24bd358677e39c2d198560b5ccb2b022 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Mon, 5 Mar 2018 11:15:15 +0100 Subject: [PATCH 07/42] Add support for 'cpu_period' for compose v2.1-v2.3. Signed-off-by: Matthieu Nottale --- compose/config/config.py | 1 + compose/config/config_schema_v2.1.json | 1 + compose/config/config_schema_v2.2.json | 1 + compose/config/config_schema_v2.3.json | 1 + compose/config/interpolation.py | 9 +++++++++ compose/service.py | 2 ++ tests/integration/service_test.py | 3 ++- 7 files changed, 17 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0b5c7df0a..3b8490a0b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -67,6 +67,7 @@ DOCKER_CONFIG_KEYS = [ 'command', 'cpu_count', 'cpu_percent', + 'cpu_period', 'cpu_quota', 'cpu_shares', 'cpus', diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 984c46c53..87a730dd8 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -106,6 +106,7 @@ "container_name": {"type": "string"}, "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, + "cpu_period": {"type": ["number", "string"]}, "cpuset": {"type": "string"}, "depends_on": { "oneOf": [ diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 8e3927e1d..e15223fc3 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -110,6 +110,7 @@ "cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100}, "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, + "cpu_period": {"type": ["number", "string"]}, "cpus": {"type": "number", "minimum": 0}, "cpuset": {"type": "string"}, "depends_on": { diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 33840dba5..c2e860be2 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -113,6 +113,7 @@ "cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100}, "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, + "cpu_period": {"type": ["number", "string"]}, "cpus": {"type": "number", "minimum": 0}, "cpuset": {"type": "string"}, "depends_on": { diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index b1143d66c..7e50f9892 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -10,6 +10,7 @@ import six from .errors import ConfigurationError from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.utils import parse_bytes +from compose.utils import parse_nanoseconds_int log = logging.getLogger(__name__) @@ -223,6 +224,12 @@ def bytes_to_int(s): return v +def to_microseconds(v): + if not isinstance(v, six.string_types): + return v + return int(parse_nanoseconds_int(v) / 1000) + + class ConversionMap(object): map = { service_path('blkio_config', 'weight'): to_int, @@ -230,6 +237,8 @@ class ConversionMap(object): service_path('build', 'labels', FULL_JOKER): to_str, service_path('cpus'): to_float, service_path('cpu_count'): to_int, + service_path('cpu_quota'): to_microseconds, + service_path('cpu_period'): to_microseconds, service_path('configs', 'mode'): to_int, service_path('secrets', 'mode'): to_int, service_path('healthcheck', 'retries'): to_int, diff --git a/compose/service.py b/compose/service.py index b9f9af2cd..501664c7c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -62,6 +62,7 @@ HOST_CONFIG_KEYS = [ 'cgroup_parent', 'cpu_count', 'cpu_percent', + 'cpu_period', 'cpu_quota', 'cpu_shares', 'cpus', @@ -948,6 +949,7 @@ class Service(object): device_write_iops=blkio_config.get('device_write_iops'), mounts=options.get('mounts'), device_cgroup_rules=options.get('device_cgroup_rules'), + cpu_period=options.get('cpu_period'), ) def get_secret_volumes(self): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2b6b7711e..6752364b2 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -121,10 +121,11 @@ class ServiceTest(DockerClientTestCase): assert container.get('HostConfig.CpuShares') == 73 def test_create_container_with_cpu_quota(self): - service = self.create_service('db', cpu_quota=40000) + service = self.create_service('db', cpu_quota=40000, cpu_period=150000) container = service.create_container() container.start() assert container.get('HostConfig.CpuQuota') == 40000 + assert container.get('HostConfig.CpuPeriod') == 150000 @v2_2_only() def test_create_container_with_cpu_count(self): From c2052d0370263407ed78cfa6f44ef24b09a67945 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 7 Mar 2018 11:39:24 -0800 Subject: [PATCH 08/42] Add blacklist to versions.py CI script Signed-off-by: Joffrey F --- script/test/versions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/script/test/versions.py b/script/test/versions.py index 46872ed9a..f699f2681 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -73,6 +73,11 @@ class Version(namedtuple('_Version', 'major minor patch rc edition')): return '.'.join(map(str, self[:3])) + edition + rc +BLACKLIST = [ # List of versions known to be broken and should not be used + Version.parse('18.03.0-ce-rc2'), +] + + def group_versions(versions): """Group versions by `major.minor` releases. @@ -117,7 +122,9 @@ def get_default(versions): def get_versions(tags): for tag in tags: try: - yield Version.parse(tag['name']) + v = Version.parse(tag['name']) + if v not in BLACKLIST: + yield v except ValueError: print("Skipping invalid tag: {name}".format(**tag), file=sys.stderr) From 4181d231313632a5b84e603dbf0c8aa37774c98a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Mar 2018 14:00:03 -0800 Subject: [PATCH 09/42] Add new maintainers and move inactive maintainers to alumni Signed-off-by: Joffrey F --- MAINTAINERS | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 89f5b4124..ff8408874 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11,12 +11,26 @@ [Org] [Org."Core maintainers"] people = [ - "aanand", - "bfirsh", "dnephin", - "mnowster", + "mefyl", + "mnottale", "shin-", ] + [Org.Alumni] + people = [ + # Aanand Prasad is one of the two creators of the fig project + # which later went on to become docker-compose, and a longtime + # maintainer responsible for several keystone features + "aanand", + # Ben Firshman is also one of the fig creators and contributed + # heavily to the project's design and UX as well as the + # day-to-day maintenance + "bfirsh", + # Mazz Mosley made significant contributions to the project + # in 2015 with solid bugfixes and improved error handling + # among them + "mnowster", + ] [people] @@ -41,6 +55,16 @@ Email = "dnephin@gmail.com" GitHub = "dnephin" + [people.mefyl] + Name = "Quentin Hocquet" + Email = "quentin.hocquet@docker.com" + GitHub = "mefyl" + + [people.mnottale] + Name = "Matthieu Nottale" + Email = "matthieu.nottale@docker.com" + GitHub = "mnottale" + [people.mnowster] Name = "Mazz Mosley" Email = "mazz@houseofmnowster.com" From 08f71a8d3d73f32d890b7485d0c3449ffb9e97f8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Mar 2018 15:44:42 -0800 Subject: [PATCH 10/42] Update Dockerfile.run to produce smaller image Signed-off-by: Joffrey F --- Dockerfile.run | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/Dockerfile.run b/Dockerfile.run index b3f9a01f6..04acf01c3 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,42 +1,20 @@ -FROM sgerrand/glibc-builder as glibc -RUN apt-get install -yq bison - -ENV PKGDIR /pkgdata - -RUN mkdir -p /usr/glibc-compat/etc && touch /usr/glibc-compat/etc/ld.so.conf -RUN /builder 2.27 /usr/glibc-compat || true -RUN mkdir -p $PKGDIR -RUN tar -xf /glibc-bin-2.27.tar.gz -C $PKGDIR -RUN rm "$PKGDIR"/usr/glibc-compat/etc/rpc && \ - rm -rf "$PKGDIR"/usr/glibc-compat/bin && \ - rm -rf "$PKGDIR"/usr/glibc-compat/sbin && \ - rm -rf "$PKGDIR"/usr/glibc-compat/lib/gconv && \ - rm -rf "$PKGDIR"/usr/glibc-compat/lib/getconf && \ - rm -rf "$PKGDIR"/usr/glibc-compat/lib/audit && \ - rm -rf "$PKGDIR"/usr/glibc-compat/share && \ - rm -rf "$PKGDIR"/usr/glibc-compat/var - - FROM alpine:3.6 -RUN apk update && apk add --no-cache openssl ca-certificates -COPY --from=glibc /pkgdata/ / +ENV GLIBC 2.27-r0 +ENV DOCKERBINS_SHA 1270dce1bd7e1838d62ae21d2505d87f16efc1d9074645571daaefdfd0c14054 -RUN mkdir -p /lib /lib64 /usr/glibc-compat/lib/locale /etc && \ +RUN apk update && apk add --no-cache openssl ca-certificates curl && \ + curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \ + curl -fsSL -o glibc-$GLIBC.apk https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ + apk add --no-cache glibc-$GLIBC.apk && \ ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ - ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 && \ - ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 && \ - ln -s /usr/glibc-compat/etc/ld.so.cache /etc/ld.so.cache - -RUN apk add --no-cache curl && \ - curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.0-ce.tgz" && \ - SHA256=692e1c72937f6214b1038def84463018d8e320c8eaf8530546c84c2f8f9c767d; \ - echo "${SHA256} dockerbins.tgz" | sha256sum -c - && \ + curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.1-ce.tgz" && \ + echo "${DOCKERBINS_SHA} dockerbins.tgz" | sha256sum -c - && \ tar xvf dockerbins.tgz docker/docker --strip-components 1 && \ mv docker /usr/local/bin/docker && \ chmod +x /usr/local/bin/docker && \ - rm dockerbins.tgz && \ + rm dockerbins.tgz /etc/apk/keys/sgerrand.rsa.pub glibc-$GLIBC.apk && \ apk del curl COPY dist/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose From 7e3bbef4369d0bcb9ed1e64a13ddbbb2961874a4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Mar 2018 18:03:54 -0800 Subject: [PATCH 11/42] Preserve security_opt values in extends Signed-off-by: Joffrey F --- compose/config/config.py | 3 ++- compose/config/types.py | 6 ++++++ tests/unit/config/config_test.py | 23 +++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0b5c7df0a..508ceafbf 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1039,6 +1039,7 @@ def merge_service_dicts(base, override, version): md.merge_sequence('links', ServiceLink.parse) md.merge_sequence('secrets', types.ServiceSecret.parse) md.merge_sequence('configs', types.ServiceConfig.parse) + md.merge_sequence('security_opt', types.SecurityOpt.parse) md.merge_mapping('extra_hosts', parse_extra_hosts) for field in ['volumes', 'devices']: @@ -1046,7 +1047,7 @@ def merge_service_dicts(base, override, version): for field in [ 'cap_add', 'cap_drop', 'expose', 'external_links', - 'security_opt', 'volumes_from', 'device_cgroup_rules', + 'volumes_from', 'device_cgroup_rules', ]: md.merge_field(field, merge_unique_items_lists, default=[]) diff --git a/compose/config/types.py b/compose/config/types.py index 47e7222a3..ff9875218 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -464,6 +464,8 @@ def normalize_port_dict(port): class SecurityOpt(namedtuple('_SecurityOpt', 'value src_file')): @classmethod def parse(cls, value): + if not isinstance(value, six.string_types): + return value # based on https://github.com/docker/cli/blob/9de1b162f/cli/command/container/opts.go#L673-L697 con = value.split('=', 2) if len(con) == 1 and con[0] != 'no-new-privileges': @@ -486,3 +488,7 @@ class SecurityOpt(namedtuple('_SecurityOpt', 'value src_file')): if self.src_file is not None: return 'seccomp:{}'.format(self.src_file) return self.value + + @property + def merge_field(self): + return self.value diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e03298221..7a9bb944d 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -4508,6 +4508,29 @@ class ExtendsTest(unittest.TestCase): for svc in services: assert svc['ports'] == [types.ServicePort('80', None, None, None, None)] + def test_extends_with_security_opt(self): + tmpdir = py.test.ensuretemp('test_extends_with_ports') + self.addCleanup(tmpdir.remove) + tmpdir.join('docker-compose.yml').write(""" + version: '2' + + services: + a: + image: nginx + security_opt: + - apparmor:unconfined + - seccomp:unconfined + + b: + extends: + service: a + """) + services = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + assert len(services) == 2 + for svc in services: + assert types.SecurityOpt.parse('apparmor:unconfined') in svc['security_opt'] + assert types.SecurityOpt.parse('seccomp:unconfined') in svc['security_opt'] + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From 0112c740ad6764df47d46b7bd3d1f93bafab32d2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 12 Mar 2018 16:46:59 -0700 Subject: [PATCH 12/42] Check volume config against remote and error out if diverged Signed-off-by: Joffrey F --- .circleci/config.yml | 4 +-- compose/volume.py | 54 +++++++++++++++++++++++-------- tests/integration/project_test.py | 47 +++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6ee2a60e5..d422fdcc5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,8 +6,8 @@ jobs: steps: - checkout - run: - name: install python3 - command: brew update > /dev/null && brew upgrade python + name: setup script + command: ./script/setup/osx - run: name: install tox command: sudo pip install --upgrade tox==2.1.1 diff --git a/compose/volume.py b/compose/volume.py index 0b148620f..6bf184045 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -124,19 +124,7 @@ class ProjectVolumes(object): ) volume.create() else: - driver = volume.inspect()['Driver'] - if volume.driver is not None and driver != volume.driver: - raise ConfigurationError( - 'Configuration for volume {0} specifies driver ' - '{1}, but a volume with the same name uses a ' - 'different driver ({3}). If you wish to use the ' - 'new configuration, please remove the existing ' - 'volume "{2}" first:\n' - '$ docker volume rm {2}'.format( - volume.name, volume.driver, volume.full_name, - volume.inspect()['Driver'] - ) - ) + check_remote_volume_config(volume.inspect(), volume) except NotFound: raise ConfigurationError( 'Volume %s specifies nonexistent driver %s' % (volume.name, volume.driver) @@ -152,3 +140,43 @@ class ProjectVolumes(object): else: volume_spec.source = self.volumes[volume_spec.source].full_name return volume_spec + + +class VolumeConfigChangedError(ConfigurationError): + def __init__(self, local, property_name, local_value, remote_value): + super(VolumeConfigChangedError, self).__init__( + 'Configuration for volume {vol_name} specifies {property_name} ' + '{local_value}, but a volume with the same name uses a different ' + '{property_name} ({remote_value}). If you wish to use the new ' + 'configuration, please remove the existing volume "{full_name}" ' + 'first:\n$ docker volume rm {full_name}'.format( + vol_name=local.name, property_name=property_name, + local_value=local_value, remote_value=remote_value, + full_name=local.full_name + ) + ) + + +def check_remote_volume_config(remote, local): + if local.driver and remote.get('Driver') != local.driver: + raise VolumeConfigChangedError(local, 'driver', local.driver, remote.get('Driver')) + local_opts = local.driver_opts or {} + remote_opts = remote.get('Options') or {} + for k in set.union(set(remote_opts.keys()), set(local_opts.keys())): + if k.startswith('com.docker.'): # These options are set internally + continue + if remote_opts.get(k) != local_opts.get(k): + raise VolumeConfigChangedError( + local, '"{}" driver_opt'.format(k), local_opts.get(k), remote_opts.get(k), + ) + + local_labels = local.labels or {} + remote_labels = remote.get('Labels') or {} + for k in set.union(set(remote_labels.keys()), set(local_labels.keys())): + if k.startswith('com.docker.'): # We are only interested in user-specified labels + continue + if remote_labels.get(k) != local_labels.get(k): + log.warn( + 'Volume {}: label "{}" has changed. It may need to be' + ' recreated.'.format(local.name, k) + ) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 0acb80284..3960d12e5 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import json import os import random +import shutil import tempfile import py @@ -1537,6 +1538,52 @@ class ProjectTest(DockerClientTestCase): vol_name ) in str(e.value) + @v2_only() + @no_cluster('inspect volume by name defect on Swarm Classic') + def test_initialize_volumes_updated_driver_opts(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + tmpdir = tempfile.mkdtemp(prefix='compose_test_') + self.addCleanup(shutil.rmtree, tmpdir) + driver_opts = {'o': 'bind', 'device': tmpdir, 'type': 'none'} + + config_data = build_config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], + volumes={ + vol_name: { + 'driver': 'local', + 'driver_opts': driver_opts + } + }, + ) + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.volumes.initialize() + + volume_data = self.get_volume_data(full_vol_name) + assert volume_data['Name'].split('/')[-1] == full_vol_name + assert volume_data['Driver'] == 'local' + assert volume_data['Options'] == driver_opts + + driver_opts['device'] = '/opt/data/localdata' + project = Project.from_config( + name='composetest', + config_data=config_data, + client=self.client + ) + with pytest.raises(config.ConfigurationError) as e: + project.volumes.initialize() + assert 'Configuration for volume {0} specifies "device" driver_opt {1}'.format( + vol_name, driver_opts['device'] + ) in str(e.value) + @v2_only() def test_initialize_volumes_updated_blank_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) From a43ec0aa2e762853756b9a1d9e78f279f1f508ac Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 13 Mar 2018 14:41:20 -0400 Subject: [PATCH 13/42] Remove myself as maintainer Signed-off-by: Daniel Nephin --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index ff8408874..d552aa3ae 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11,7 +11,6 @@ [Org] [Org."Core maintainers"] people = [ - "dnephin", "mefyl", "mnottale", "shin-", @@ -30,6 +29,7 @@ # in 2015 with solid bugfixes and improved error handling # among them "mnowster", + "dnephin", ] [people] From e7f0ab04a1eb75d7e0573e5aeb5d55f55cea9477 Mon Sep 17 00:00:00 2001 From: Wes Higbee Date: Wed, 14 Mar 2018 21:08:55 -0400 Subject: [PATCH 14/42] Fix docker-compose zsh running service name completion This applies to commands that operate on running services. For example: top, stop, restart, etc. Configuring a custom psFormat in ~/.docker/config.json can break zsh running service name completion that depends upon default `docker ps` output. This breaks when you don't include the output needed by completion. The fix specifies the explicit format needed for completion and is based on a previous fix in the docker CLI completion: https://github.com/docker/cli/commit/8b38343e46e46b589bf556651c10dc3dbee3f88b Signed-off-by: Wes Higbee --- contrib/completion/zsh/_docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index c0a54cced..aba367706 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -88,7 +88,7 @@ __docker-compose_get_services() { shift [[ $kind =~ (stopped|all) ]] && args=($args -a) - lines=(${(f)"$(_call_program commands docker $docker_options ps $args)"}) + lines=(${(f)"$(_call_program commands docker $docker_options ps --format 'table' $args)"}) services=(${(f)"$(_call_program commands docker-compose 2>/dev/null $compose_options ps -q)"}) # Parse header line to find columns From 5fe3aff1c310d6b8684404b19a8246736cfe8f1a Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Thu, 15 Mar 2018 15:41:36 +0100 Subject: [PATCH 15/42] Allow dash and underscore in project name. Signed-off-by: Matthieu Nottale --- compose/cli/command.py | 2 +- tests/acceptance/cli_test.py | 66 ++++++++++++++++++------------------ tests/unit/cli_test.py | 10 +++--- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 9fd941bb6..022bc8576 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -127,7 +127,7 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, def get_project_name(working_dir, project_name=None, environment=None): def normalize_name(name): - return re.sub(r'[^a-z0-9]', '', name.lower()) + return re.sub(r'[^-_a-z0-9]', '', name.lower()) if not environment: environment = Environment.from_env_file(working_dir) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 45e9a54e5..b8d6c9978 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -491,16 +491,16 @@ class CLITestCase(DockerClientTestCase): def test_ps(self): self.project.get_service('simple').create_container() result = self.dispatch(['ps']) - assert 'simplecomposefile_simple_1' in result.stdout + assert 'simple-composefile_simple_1' in result.stdout def test_ps_default_composefile(self): self.base_dir = 'tests/fixtures/multiple-composefiles' self.dispatch(['up', '-d']) result = self.dispatch(['ps']) - assert 'multiplecomposefiles_simple_1' in result.stdout - assert 'multiplecomposefiles_another_1' in result.stdout - assert 'multiplecomposefiles_yetanother_1' not in result.stdout + assert 'multiple-composefiles_simple_1' in result.stdout + assert 'multiple-composefiles_another_1' in result.stdout + assert 'multiple-composefiles_yetanother_1' not in result.stdout def test_ps_alternate_composefile(self): config_path = os.path.abspath( @@ -511,9 +511,9 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['-f', 'compose2.yml', 'up', '-d']) result = self.dispatch(['-f', 'compose2.yml', 'ps']) - assert 'multiplecomposefiles_simple_1' not in result.stdout - assert 'multiplecomposefiles_another_1' not in result.stdout - assert 'multiplecomposefiles_yetanother_1' in result.stdout + assert 'multiple-composefiles_simple_1' not in result.stdout + assert 'multiple-composefiles_another_1' not in result.stdout + assert 'multiple-composefiles_yetanother_1' in result.stdout def test_ps_services_filter_option(self): self.base_dir = 'tests/fixtures/ps-services-filter' @@ -902,18 +902,18 @@ class CLITestCase(DockerClientTestCase): assert len(self.project.containers(one_off=OneOffFilter.only, stopped=True)) == 2 result = self.dispatch(['down', '--rmi=local', '--volumes']) - assert 'Stopping v2full_web_1' in result.stderr - assert 'Stopping v2full_other_1' in result.stderr - assert 'Stopping v2full_web_run_2' in result.stderr - assert 'Removing v2full_web_1' in result.stderr - assert 'Removing v2full_other_1' in result.stderr - assert 'Removing v2full_web_run_1' in result.stderr - assert 'Removing v2full_web_run_2' in result.stderr - assert 'Removing volume v2full_data' in result.stderr - assert 'Removing image v2full_web' in result.stderr + assert 'Stopping v2-full_web_1' in result.stderr + assert 'Stopping v2-full_other_1' in result.stderr + assert 'Stopping v2-full_web_run_2' in result.stderr + assert 'Removing v2-full_web_1' in result.stderr + assert 'Removing v2-full_other_1' in result.stderr + assert 'Removing v2-full_web_run_1' in result.stderr + assert 'Removing v2-full_web_run_2' in result.stderr + assert 'Removing volume v2-full_data' in result.stderr + assert 'Removing image v2-full_web' in result.stderr assert 'Removing image busybox' not in result.stderr - assert 'Removing network v2full_default' in result.stderr - assert 'Removing network v2full_front' in result.stderr + assert 'Removing network v2-full_default' in result.stderr + assert 'Removing network v2-full_front' in result.stderr def test_down_timeout(self): self.dispatch(['up', '-d'], None) @@ -2000,39 +2000,39 @@ class CLITestCase(DockerClientTestCase): proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) wait_on_condition(ContainerStateCondition( self.project.client, - 'simplecomposefile_simple_run_1', + 'simple-composefile_simple_run_1', 'running')) os.kill(proc.pid, signal.SIGINT) wait_on_condition(ContainerStateCondition( self.project.client, - 'simplecomposefile_simple_run_1', + 'simple-composefile_simple_run_1', 'exited')) def test_run_handles_sigterm(self): proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) wait_on_condition(ContainerStateCondition( self.project.client, - 'simplecomposefile_simple_run_1', + 'simple-composefile_simple_run_1', 'running')) os.kill(proc.pid, signal.SIGTERM) wait_on_condition(ContainerStateCondition( self.project.client, - 'simplecomposefile_simple_run_1', + 'simple-composefile_simple_run_1', 'exited')) def test_run_handles_sighup(self): proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) wait_on_condition(ContainerStateCondition( self.project.client, - 'simplecomposefile_simple_run_1', + 'simple-composefile_simple_run_1', 'running')) os.kill(proc.pid, signal.SIGHUP) wait_on_condition(ContainerStateCondition( self.project.client, - 'simplecomposefile_simple_run_1', + 'simple-composefile_simple_run_1', 'exited')) @mock.patch.dict(os.environ) @@ -2234,7 +2234,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up', '-d', 'another']) wait_on_condition(ContainerStateCondition( self.project.client, - 'logscomposefile_another_1', + 'logs-composefile_another_1', 'exited')) self.dispatch(['kill', 'simple']) @@ -2243,8 +2243,8 @@ class CLITestCase(DockerClientTestCase): assert 'hello' in result.stdout assert 'test' in result.stdout - assert 'logscomposefile_another_1 exited with code 0' in result.stdout - assert 'logscomposefile_simple_1 exited with code 137' in result.stdout + assert 'logs-composefile_another_1 exited with code 0' in result.stdout + assert 'logs-composefile_simple_1 exited with code 137' in result.stdout def test_logs_default(self): self.base_dir = 'tests/fixtures/logs-composefile' @@ -2491,7 +2491,7 @@ class CLITestCase(DockerClientTestCase): container, = self.project.containers() expected_template = ' container {} {}' - expected_meta_info = ['image=busybox:latest', 'name=simplecomposefile_simple_1'] + expected_meta_info = ['image=busybox:latest', 'name=simple-composefile_simple_1'] assert expected_template.format('create', container.id) in lines[0] assert expected_template.format('start', container.id) in lines[1] @@ -2611,13 +2611,13 @@ class CLITestCase(DockerClientTestCase): result = wait_on_process(proc, returncode=1) - assert 'exitcodefrom_another_1 exited with code 1' in result.stdout + assert 'exit-code-from_another_1 exited with code 1' in result.stdout def test_images(self): self.project.get_service('simple').create_container() result = self.dispatch(['images']) assert 'busybox' in result.stdout - assert 'simplecomposefile_simple_1' in result.stdout + assert 'simple-composefile_simple_1' in result.stdout def test_images_default_composefile(self): self.base_dir = 'tests/fixtures/multiple-composefiles' @@ -2625,8 +2625,8 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['images']) assert 'busybox' in result.stdout - assert 'multiplecomposefiles_another_1' in result.stdout - assert 'multiplecomposefiles_simple_1' in result.stdout + assert 'multiple-composefiles_another_1' in result.stdout + assert 'multiple-composefiles_simple_1' in result.stdout @mock.patch.dict(os.environ) def test_images_tagless_image(self): @@ -2646,7 +2646,7 @@ class CLITestCase(DockerClientTestCase): self.project.get_service('foo').create_container() result = self.dispatch(['images']) assert '' in result.stdout - assert 'taglessimage_foo_1' in result.stdout + assert 'tagless-image_foo_1' in result.stdout def test_up_with_override_yaml(self): self.base_dir = 'tests/fixtures/override-yaml-files' diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 47eaabf9d..7c8a1423c 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -30,12 +30,12 @@ class CLITestCase(unittest.TestCase): test_dir = py._path.local.LocalPath('tests/fixtures/simple-composefile') with test_dir.as_cwd(): project_name = get_project_name('.') - assert 'simplecomposefile' == project_name + assert 'simple-composefile' == project_name def test_project_name_with_explicit_base_dir(self): base_dir = 'tests/fixtures/simple-composefile' project_name = get_project_name(base_dir) - assert 'simplecomposefile' == project_name + assert 'simple-composefile' == project_name def test_project_name_with_explicit_uppercase_base_dir(self): base_dir = 'tests/fixtures/UpperCaseDir' @@ -45,7 +45,7 @@ class CLITestCase(unittest.TestCase): def test_project_name_with_explicit_project_name(self): name = 'explicit-project-name' project_name = get_project_name(None, project_name=name) - assert 'explicitprojectname' == project_name + assert 'explicit-project-name' == project_name @mock.patch.dict(os.environ) def test_project_name_from_environment_new_var(self): @@ -59,7 +59,7 @@ class CLITestCase(unittest.TestCase): with mock.patch.dict(os.environ): os.environ['COMPOSE_PROJECT_NAME'] = '' project_name = get_project_name(base_dir) - assert 'simplecomposefile' == project_name + assert 'simple-composefile' == project_name @mock.patch.dict(os.environ) def test_project_name_with_environment_file(self): @@ -80,7 +80,7 @@ class CLITestCase(unittest.TestCase): def test_get_project(self): base_dir = 'tests/fixtures/longer-filename-composefile' project = get_project(base_dir) - assert project.name == 'longerfilenamecomposefile' + assert project.name == 'longer-filename-composefile' assert project.client assert project.services From c7b76b1d128a3c4d4bb5deb50633062698b3df3e Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Thu, 15 Mar 2018 15:40:13 +0100 Subject: [PATCH 16/42] pull: Honor --quiet in parallel mode. Signed-off-by: Matthieu Nottale --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 2cbc4aeea..48095e938 100644 --- a/compose/project.py +++ b/compose/project.py @@ -551,7 +551,7 @@ class Project(object): services, pull_service, operator.attrgetter('name'), - 'Pulling', + not silent and 'Pulling' or None, limit=5, ) if len(errors): From 83364078300340cabaad95d5d0e37954b52e3c30 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Thu, 15 Mar 2018 14:10:20 +0100 Subject: [PATCH 17/42] pull: Deprecate '--parallel' and enable it by default. Signed-off-by: Matthieu Nottale --- compose/cli/main.py | 7 +++++-- tests/acceptance/cli_test.py | 16 +++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 59f111184..9e49e2974 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -711,14 +711,17 @@ class TopLevelCommand(object): Options: --ignore-pull-failures Pull what it can and ignores images with pull failures. - --parallel Pull multiple images in parallel. + --parallel Deprecated, pull multiple images in parallel (enabled by default). + --no-parallel Disable parallel pulling. -q, --quiet Pull without printing progress information --include-deps Also pull services declared as dependencies """ + if options.get('--parallel'): + log.warn('--parallel option is deprecated and will be removed in future versions.') self.project.pull( service_names=options['SERVICE'], ignore_pull_failures=options.get('--ignore-pull-failures'), - parallel_pull=options.get('--parallel'), + parallel_pull=not options.get('--no-parallel'), silent=options.get('--quiet'), include_deps=options.get('--include-deps'), ) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 45e9a54e5..0366df51e 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -545,13 +545,11 @@ class CLITestCase(DockerClientTestCase): def test_pull(self): result = self.dispatch(['pull']) - assert sorted(result.stderr.split('\n'))[1:] == [ - 'Pulling another (busybox:latest)...', - 'Pulling simple (busybox:latest)...', - ] + assert 'Pulling simple' in result.stderr + assert 'Pulling another' in result.stderr def test_pull_with_digest(self): - result = self.dispatch(['-f', 'digest.yml', 'pull']) + result = self.dispatch(['-f', 'digest.yml', 'pull', '--no-parallel']) assert 'Pulling simple (busybox:latest)...' in result.stderr assert ('Pulling digest (busybox@' @@ -561,7 +559,7 @@ class CLITestCase(DockerClientTestCase): def test_pull_with_ignore_pull_failures(self): result = self.dispatch([ '-f', 'ignore-pull-failures.yml', - 'pull', '--ignore-pull-failures'] + 'pull', '--ignore-pull-failures', '--no-parallel'] ) assert 'Pulling simple (busybox:latest)...' in result.stderr @@ -576,7 +574,7 @@ class CLITestCase(DockerClientTestCase): def test_pull_with_parallel_failure(self): result = self.dispatch([ - '-f', 'ignore-pull-failures.yml', 'pull', '--parallel'], + '-f', 'ignore-pull-failures.yml', 'pull'], returncode=1 ) @@ -593,14 +591,14 @@ class CLITestCase(DockerClientTestCase): def test_pull_with_no_deps(self): self.base_dir = 'tests/fixtures/links-composefile' - result = self.dispatch(['pull', 'web']) + result = self.dispatch(['pull', '--no-parallel', 'web']) assert sorted(result.stderr.split('\n'))[1:] == [ 'Pulling web (busybox:latest)...', ] def test_pull_with_include_deps(self): self.base_dir = 'tests/fixtures/links-composefile' - result = self.dispatch(['pull', '--include-deps', 'web']) + result = self.dispatch(['pull', '--no-parallel', '--include-deps', 'web']) assert sorted(result.stderr.split('\n'))[1:] == [ 'Pulling db (busybox:latest)...', 'Pulling web (busybox:latest)...', From 06593110dc04d5f14ff4c56b8fbb1921b0aaf8cf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Mar 2018 12:20:00 -0700 Subject: [PATCH 18/42] Fix indentation + HOF comment Signed-off-by: Joffrey F --- MAINTAINERS | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index d552aa3ae..7aedd46e9 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -29,7 +29,11 @@ # in 2015 with solid bugfixes and improved error handling # among them "mnowster", - "dnephin", + # Daniel Nephin is one of the longest-running maitainers on + # the Compose project, and has contributed several major features + # including muti-file support, variable interpolation, secrets + # emulation and many more + "dnephin", ] [people] From 5184f4f78dd083cf2afe4569de3ec0c90743a0b1 Mon Sep 17 00:00:00 2001 From: Dakota Hawkins Date: Thu, 15 Mar 2018 17:33:15 -0400 Subject: [PATCH 19/42] Add update-docker-compose.ps1 to contrib/update/ Updates Windows' installed version of docker-compose to the latest release. Fixes #5790 Signed-off-by: Dakota Hawkins --- contrib/update/update-docker-compose.ps1 | 116 +++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 contrib/update/update-docker-compose.ps1 diff --git a/contrib/update/update-docker-compose.ps1 b/contrib/update/update-docker-compose.ps1 new file mode 100644 index 000000000..bb033b464 --- /dev/null +++ b/contrib/update/update-docker-compose.ps1 @@ -0,0 +1,116 @@ +# Self-elevate the script if required +# http://www.expta.com/2017/03/how-to-self-elevate-powershell-script.html +If (-Not ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] 'Administrator')) { + If ([int](Get-CimInstance -Class Win32_OperatingSystem | Select-Object -ExpandProperty BuildNumber) -ge 6000) { + $CommandLine = "-File `"" + $MyInvocation.MyCommand.Path + "`" " + $MyInvocation.UnboundArguments + Start-Process -FilePath PowerShell.exe -Verb Runas -ArgumentList $CommandLine + Exit + } +} + +$SectionSeparator = "--------------------------------------------------" + +# Update docker-compose if required +Function UpdateDockerCompose() { + Write-Host "Updating docker-compose if required..." + Write-Host $SectionSeparator + + # Find the installed docker-compose.exe location + Try { + $DockerComposePath = Get-Command docker-compose.exe -ErrorAction Stop | ` + Select-Object -First 1 -ExpandProperty Definition + } + Catch { + Write-Host "Error: Could not find path to docker-compose.exe" ` + -ForegroundColor Red + Return $false + } + + # Prefer/enable TLS 1.2 + # https://stackoverflow.com/a/48030563/153079 + [Net.ServicePointManager]::SecurityProtocol = "tls12, tls11, tls" + + # Query for the latest release version + Try { + $URI = "https://api.github.com/repos/docker/compose/releases/latest" + $LatestComposeVersion = [System.Version](Invoke-RestMethod -Method Get -Uri $URI).tag_name + } + Catch { + Write-Host "Error: Query for the latest docker-compose release version failed" ` + -ForegroundColor Red + Return $false + } + + # Check the installed version and compare with latest release + $UpdateDockerCompose = $false + Try { + $InstalledComposeVersion = ` + [System.Version]((docker-compose.exe version --short) | Out-String) + + If ($InstalledComposeVersion -eq $LatestComposeVersion) { + Write-Host ("Installed docker-compose version ({0}) same as latest ({1})." ` + -f $InstalledComposeVersion.ToString(), $LatestComposeVersion.ToString()) + } + ElseIf ($InstalledComposeVersion -lt $LatestComposeVersion) { + Write-Host ("Installed docker-compose version ({0}) older than latest ({1})." ` + -f $InstalledComposeVersion.ToString(), $LatestComposeVersion.ToString()) + $UpdateDockerCompose = $true + } + Else { + Write-Host ("Installed docker-compose version ({0}) newer than latest ({1})." ` + -f $InstalledComposeVersion.ToString(), $LatestComposeVersion.ToString()) ` + -ForegroundColor Yellow + } + } + Catch { + Write-Host ` + "Warning: Couldn't get docker-compose version, assuming an update is required..." ` + -ForegroundColor Yellow + $UpdateDockerCompose = $true + } + + If (-Not $UpdateDockerCompose) { + # Nothing to do! + Return $false + } + + # Download the latest version of docker-compose.exe + Try { + $RemoteFileName = "docker-compose-Windows-x86_64.exe" + $URI = ("https://github.com/docker/compose/releases/download/{0}/{1}" ` + -f $LatestComposeVersion.ToString(), $RemoteFileName) + Invoke-WebRequest -UseBasicParsing -Uri $URI ` + -OutFile $DockerComposePath + Return $true + } + Catch { + Write-Host ("Error: Failed to download the latest version of docker-compose`n{0}" ` + -f $_.Exception.Message) -ForegroundColor Red + Return $false + } + + Return $false +} + +If (UpdateDockerCompose) { + Write-Host "Updated to latest-version of docker-compose, running update again to verify.`n" + If (UpdateDockerCompose) { + Write-Host "Error: Should not have updated twice." -ForegroundColor Red + } +} + +# Assuming elevation popped up a new powershell window, pause so the user can see what happened +# https://stackoverflow.com/a/22362868/153079 +Function Pause ($Message = "Press any key to continue . . . ") { + If ((Test-Path variable:psISE) -and $psISE) { + $Shell = New-Object -ComObject "WScript.Shell" + $Shell.Popup("Click OK to continue.", 0, "Script Paused", 0) + } + Else { + Write-Host "`n$SectionSeparator" + Write-Host -NoNewline $Message + [void][System.Console]::ReadKey($true) + Write-Host + } +} +Pause From b1ade31c67bf57945d867237273d4cc1caae6850 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Mar 2018 16:22:28 -0700 Subject: [PATCH 20/42] Manage encoding errors in progress_stream Signed-off-by: Joffrey F --- compose/progress_stream.py | 32 +++++++++++++++++++----------- tests/unit/progress_stream_test.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 5314f89fd..5e709770a 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -8,6 +8,14 @@ class StreamOutputError(Exception): pass +def write_to_stream(s, stream): + try: + stream.write(s) + except UnicodeEncodeError: + encoding = getattr(stream, 'encoding', 'ascii') + stream.write(s.encode(encoding, errors='replace').decode(encoding)) + + def stream_output(output, stream): is_terminal = hasattr(stream, 'isatty') and stream.isatty() stream = utils.get_output_stream(stream) @@ -34,18 +42,18 @@ def stream_output(output, stream): if image_id not in lines: lines[image_id] = len(lines) - stream.write("\n") + write_to_stream("\n", stream) diff = len(lines) - lines[image_id] # move cursor up `diff` rows - stream.write("%c[%dA" % (27, diff)) + write_to_stream("%c[%dA" % (27, diff), stream) print_output_event(event, stream, is_terminal) if 'id' in event: # move cursor back down - stream.write("%c[%dB" % (27, diff)) + write_to_stream("%c[%dB" % (27, diff), stream) stream.flush() @@ -60,36 +68,36 @@ def print_output_event(event, stream, is_terminal): if is_terminal and 'stream' not in event: # erase current line - stream.write("%c[2K\r" % 27) + write_to_stream("%c[2K\r" % 27, stream) terminator = "\r" elif 'progressDetail' in event: return if 'time' in event: - stream.write("[%s] " % event['time']) + write_to_stream("[%s] " % event['time'], stream) if 'id' in event: - stream.write("%s: " % event['id']) + write_to_stream("%s: " % event['id'], stream) if 'from' in event: - stream.write("(from %s) " % event['from']) + write_to_stream("(from %s) " % event['from'], stream) status = event.get('status', '') if 'progress' in event: - stream.write("%s %s%s" % (status, event['progress'], terminator)) + write_to_stream("%s %s%s" % (status, event['progress'], terminator), stream) elif 'progressDetail' in event: detail = event['progressDetail'] total = detail.get('total') if 'current' in detail and total: percentage = float(detail['current']) / float(total) * 100 - stream.write('%s (%.1f%%)%s' % (status, percentage, terminator)) + write_to_stream('%s (%.1f%%)%s' % (status, percentage, terminator), stream) else: - stream.write('%s%s' % (status, terminator)) + write_to_stream('%s%s' % (status, terminator), stream) elif 'stream' in event: - stream.write("%s%s" % (event['stream'], terminator)) + write_to_stream("%s%s" % (event['stream'], terminator), stream) else: - stream.write("%s%s\n" % (status, terminator)) + write_to_stream("%s%s\n" % (status, terminator), stream) def get_digest_from_pull(events): diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index 22a6e081b..f4a0ab063 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -1,6 +1,13 @@ +# ~*~ encoding: utf-8 ~*~ from __future__ import absolute_import from __future__ import unicode_literals +import io +import os +import random +import shutil +import tempfile + from six import StringIO from compose import progress_stream @@ -66,6 +73,30 @@ class ProgressStreamTestCase(unittest.TestCase): events = progress_stream.stream_output(events, output) assert len(output.getvalue()) > 0 + def test_mismatched_encoding_stream_write(self): + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir, True) + + def mktempfile(encoding): + fname = os.path.join(tmpdir, hex(random.getrandbits(128))[2:-1]) + return io.open(fname, mode='w+', encoding=encoding) + + text = '就吃饭' + with mktempfile(encoding='utf-8') as tf: + progress_stream.write_to_stream(text, tf) + tf.seek(0) + assert tf.read() == text + + with mktempfile(encoding='utf-32') as tf: + progress_stream.write_to_stream(text, tf) + tf.seek(0) + assert tf.read() == text + + with mktempfile(encoding='ascii') as tf: + progress_stream.write_to_stream(text, tf) + tf.seek(0) + assert tf.read() == '???' + def test_get_digest_from_push(): digest = "sha256:abcd" From 16ea49ac8cb71771f52cafc327e864285b6d39ef Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Fri, 16 Mar 2018 15:56:30 +0100 Subject: [PATCH 21/42] Add support for cpu_rt_period and cpu_rt_runtime. Signed-off-by: Matthieu Nottale --- compose/config/config.py | 2 ++ compose/config/config_schema_v2.2.json | 2 ++ compose/config/config_schema_v2.3.json | 2 ++ compose/config/interpolation.py | 2 ++ compose/service.py | 4 ++++ tests/integration/service_test.py | 8 ++++++++ 6 files changed, 20 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index 59f9d35ba..7bc220e5f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -69,6 +69,8 @@ DOCKER_CONFIG_KEYS = [ 'cpu_percent', 'cpu_period', 'cpu_quota', + 'cpu_rt_period', + 'cpu_rt_runtime', 'cpu_shares', 'cpus', 'cpuset', diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index e15223fc3..dc7076745 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -111,6 +111,8 @@ "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, "cpu_period": {"type": ["number", "string"]}, + "cpu_rt_period": {"type": ["number", "string"]}, + "cpu_rt_runtime": {"type": ["number", "string"]}, "cpus": {"type": "number", "minimum": 0}, "cpuset": {"type": "string"}, "depends_on": { diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index c2e860be2..bd7ce166d 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -114,6 +114,8 @@ "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, "cpu_period": {"type": ["number", "string"]}, + "cpu_rt_period": {"type": ["number", "string"]}, + "cpu_rt_runtime": {"type": ["number", "string"]}, "cpus": {"type": "number", "minimum": 0}, "cpuset": {"type": "string"}, "depends_on": { diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 7e50f9892..59a567bb2 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -239,6 +239,8 @@ class ConversionMap(object): service_path('cpu_count'): to_int, service_path('cpu_quota'): to_microseconds, service_path('cpu_period'): to_microseconds, + service_path('cpu_rt_period'): to_microseconds, + service_path('cpu_rt_runtime'): to_microseconds, service_path('configs', 'mode'): to_int, service_path('secrets', 'mode'): to_int, service_path('healthcheck', 'retries'): to_int, diff --git a/compose/service.py b/compose/service.py index bcd21d02f..a4e5d9b82 100644 --- a/compose/service.py +++ b/compose/service.py @@ -64,6 +64,8 @@ HOST_CONFIG_KEYS = [ 'cpu_percent', 'cpu_period', 'cpu_quota', + 'cpu_rt_period', + 'cpu_rt_runtime', 'cpu_shares', 'cpus', 'cpuset', @@ -949,6 +951,8 @@ class Service(object): mounts=options.get('mounts'), device_cgroup_rules=options.get('device_cgroup_rules'), cpu_period=options.get('cpu_period'), + cpu_rt_period=options.get('cpu_rt_period'), + cpu_rt_runtime=options.get('cpu_rt_runtime'), ) def get_secret_volumes(self): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index befc0204a..7c1be24b9 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -128,6 +128,14 @@ class ServiceTest(DockerClientTestCase): assert container.get('HostConfig.CpuQuota') == 40000 assert container.get('HostConfig.CpuPeriod') == 150000 + @pytest.mark.xfail(raises=OperationFailedError, reason='not supported by kernel') + def test_create_container_with_cpu_rt(self): + service = self.create_service('db', cpu_rt_runtime=40000, cpu_rt_period=150000) + container = service.create_container() + container.start() + assert container.get('HostConfig.CpuRealtimeRuntime') == 40000 + assert container.get('HostConfig.CpuRealtimePeriod') == 150000 + @v2_2_only() def test_create_container_with_cpu_count(self): self.require_api_version('1.25') From 25c048fd0ada78796d13f6010a93b25efee02bde Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Mar 2018 12:15:51 +0100 Subject: [PATCH 22/42] Bump Docker SDK -> 3.1.3 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 33462d496..743272d4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.1.1 +docker==3.1.3 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index cf8f6dc13..e24736df0 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.1.1, < 4.0', + 'docker >= 3.1.3, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 679d3555a2c8c5ad0b0f491f5340ba60be34b977 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Mar 2018 16:10:08 +0100 Subject: [PATCH 23/42] 1.21.0-dev Signed-off-by: Joffrey F --- CHANGELOG.md | 2 +- compose/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab0a2c354..25e04d1b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ Change log ========== -1.20.0 (2018-03-07) +1.20.0 (2018-03-20) ------------------- ### New features diff --git a/compose/__init__.py b/compose/__init__.py index 1c5a409f1..a5c1364a6 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.20.0' +__version__ = '1.21.0dev' From db10ef2624aaac2d8f9bea6a22579755c4abcddc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 21 Mar 2018 14:40:21 +0100 Subject: [PATCH 24/42] Bump Docker python SDK -> 3.1.4 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 743272d4e..e5a26bba2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.1.3 +docker==3.1.4 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index e24736df0..965307264 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.1.3, < 4.0', + 'docker >= 3.1.4, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 166f355efb2054786892c2a033164d1c8ea7595e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Mar 2018 14:08:06 +0100 Subject: [PATCH 25/42] Add libgcc dependency for Compose in a container Signed-off-by: Joffrey F --- Dockerfile.run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.run b/Dockerfile.run index 04acf01c3..c403ac230 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -3,12 +3,13 @@ FROM alpine:3.6 ENV GLIBC 2.27-r0 ENV DOCKERBINS_SHA 1270dce1bd7e1838d62ae21d2505d87f16efc1d9074645571daaefdfd0c14054 -RUN apk update && apk add --no-cache openssl ca-certificates curl && \ +RUN apk update && apk add --no-cache openssl ca-certificates curl libgcc && \ curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \ curl -fsSL -o glibc-$GLIBC.apk https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ apk add --no-cache glibc-$GLIBC.apk && \ ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ + ln -s /usr/lib/libgcc_s.so.1 /usr/glibc-compat/lib && \ curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.1-ce.tgz" && \ echo "${DOCKERBINS_SHA} dockerbins.tgz" | sha256sum -c - && \ tar xvf dockerbins.tgz docker/docker --strip-components 1 && \ From 255d16d7fa2b85260237b6ebd74b98da68110c03 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Mar 2018 16:54:12 +0100 Subject: [PATCH 26/42] Add --compress option to build command Signed-off-by: Joffrey F --- compose/cli/main.py | 5 ++++- compose/project.py | 4 ++-- compose/service.py | 4 +++- tests/integration/service_test.py | 18 ++++++++++++++++++ tests/unit/service_test.py | 2 ++ 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 9e49e2974..3932e4eae 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -254,6 +254,7 @@ class TopLevelCommand(object): Usage: build [options] [--build-arg key=val...] [SERVICE...] Options: + --compress Compress the build context using gzip. --force-rm Always remove intermediate containers. --no-cache Do not use cache when building the image. --pull Always attempt to pull a newer version of the image. @@ -277,7 +278,9 @@ class TopLevelCommand(object): pull=bool(options.get('--pull', False)), force_rm=bool(options.get('--force-rm', False)), memory=options.get('--memory'), - build_args=build_args) + build_args=build_args, + gzip=options.get('--compress', False), + ) def bundle(self, options): """ diff --git a/compose/project.py b/compose/project.py index 48095e938..f66e415fe 100644 --- a/compose/project.py +++ b/compose/project.py @@ -366,10 +366,10 @@ class Project(object): return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None, - build_args=None): + build_args=None, gzip=False): for service in self.get_services(service_names): if service.can_be_built(): - service.build(no_cache, pull, force_rm, memory, build_args) + service.build(no_cache, pull, force_rm, memory, build_args, gzip) else: log.info('%s uses an image, skipping' % service.name) diff --git a/compose/service.py b/compose/service.py index a4e5d9b82..cd9b4b568 100644 --- a/compose/service.py +++ b/compose/service.py @@ -967,7 +967,8 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] - def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None): + def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None, + gzip=False): log.info('Building %s' % self.name) build_opts = self.options.get('build', {}) @@ -1003,6 +1004,7 @@ class Service(object): container_limits={ 'memory': parse_bytes(memory) if memory else None }, + gzip=gzip ) try: diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 7c1be24b9..260867fe0 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -1105,6 +1105,24 @@ class ServiceTest(DockerClientTestCase): service.build() assert service.image() + def test_build_with_gzip(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('\n'.join([ + 'FROM busybox', + 'COPY . /src', + 'RUN cat /src/hello.txt' + ])) + with open(os.path.join(base_dir, 'hello.txt'), 'w') as f: + f.write('hello world\n') + + service = self.create_service('build_gzip', build={ + 'context': text_type(base_dir), + }) + service.build(gzip=True) + assert service.image() + def test_start_container_stays_unprivileged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 9128b9550..1f322ce4e 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -487,6 +487,7 @@ class ServiceTest(unittest.TestCase): shmsize=None, extra_hosts=None, container_limits={'memory': None}, + gzip=False, ) def test_ensure_image_exists_no_build(self): @@ -529,6 +530,7 @@ class ServiceTest(unittest.TestCase): shmsize=None, extra_hosts=None, container_limits={'memory': None}, + gzip=False ) def test_build_does_not_pull(self): From 4e97e0fd41efea16fae1d870918095e8a86fa283 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Mar 2018 17:59:22 +0100 Subject: [PATCH 27/42] Add issue / PR templates (fixes #5795) Signed-off-by: Joffrey F --- docs/issue_template.md | 50 ++++++++++++++++++++++++++++++++++++++++++ docs/pull_request.md | 13 +++++++++++ 2 files changed, 63 insertions(+) create mode 100644 docs/issue_template.md create mode 100644 docs/pull_request.md diff --git a/docs/issue_template.md b/docs/issue_template.md new file mode 100644 index 000000000..774f27e20 --- /dev/null +++ b/docs/issue_template.md @@ -0,0 +1,50 @@ + + +## Description of the issue + +## Context information (for bug reports) + +``` +Output of "docker-compose version" +``` + +``` +Output of "docker version" +``` + +``` +Output of "docker-compose config" +``` + + +## Steps to reproduce the issue + +1. +2. +3. + +### Observed result + +### Expected result + +### Stacktrace / full error message + +``` +(if applicable) +``` + +## Additional information + +OS version / distribution, `docker-compose` install method, etc. diff --git a/docs/pull_request.md b/docs/pull_request.md new file mode 100644 index 000000000..ffe751fb3 --- /dev/null +++ b/docs/pull_request.md @@ -0,0 +1,13 @@ + + + +Resolves # From 36e646685501ba66a2093c2cd85655d1d146341d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Mar 2018 18:01:55 +0100 Subject: [PATCH 28/42] pull_request_template.md Signed-off-by: Joffrey F --- docs/{pull_request.md => pull_request_template.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{pull_request.md => pull_request_template.md} (100%) diff --git a/docs/pull_request.md b/docs/pull_request_template.md similarity index 100% rename from docs/pull_request.md rename to docs/pull_request_template.md From 7d68d4bb442eec920864a560fdfa523eb1d0398f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Mar 2018 18:03:08 +0100 Subject: [PATCH 29/42] missing "is" Signed-off-by: Joffrey F --- docs/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pull_request_template.md b/docs/pull_request_template.md index ffe751fb3..15526af05 100644 --- a/docs/pull_request_template.md +++ b/docs/pull_request_template.md @@ -7,7 +7,7 @@ do not comply and contributions with failing tests will not be reviewed! Resolves # From cbb9bff9243a8e2de24e74327caa21779a12e075 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Mar 2018 14:25:31 -0700 Subject: [PATCH 30/42] Support -H=host notation for interactive run/execs Signed-off-by: Joffrey F --- compose/cli/main.py | 2 +- tests/acceptance/cli_test.py | 7 +++++++ tests/unit/cli/main_test.py | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 9e49e2974..c05ba4988 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1418,7 +1418,7 @@ def call_docker(args, dockeropts): if verify: tls_options.append('--tlsverify') if host: - tls_options.extend(['--host', host]) + tls_options.extend(['--host', host.lstrip('=')]) args = [executable_path] + tls_options + args log.debug(" ".join(map(pipes.quote, args))) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 14c96b24d..075705804 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -177,6 +177,13 @@ class CLITestCase(DockerClientTestCase): returncode=0 ) + def test_shorthand_host_opt_interactive(self): + self.dispatch( + ['-H={0}'.format(os.environ.get('DOCKER_HOST', 'unix://')), + 'run', 'another', 'ls'], + returncode=0 + ) + def test_host_not_reachable(self): result = self.dispatch(['-H=tcp://doesnotexist:8000', 'ps'], returncode=1) assert "Couldn't connect to Docker daemon" in result.stderr diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index b46a3ee22..1a2dfbcf3 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -154,3 +154,11 @@ class TestCallDocker(object): assert fake_call.call_args[0][0] == [ 'docker', '--host', 'tcp://mydocker.net:2333', 'ps' ] + + def test_with_host_option_shorthand_equal(self): + with mock.patch('subprocess.call') as fake_call: + call_docker(['ps'], {'--host': '=tcp://mydocker.net:2333'}) + + assert fake_call.call_args[0][0] == [ + 'docker', '--host', 'tcp://mydocker.net:2333', 'ps' + ] From 394c8efe98fb13aaaca8c1f276642445889a73b2 Mon Sep 17 00:00:00 2001 From: Alberto Piai Date: Sun, 25 Mar 2018 22:58:59 +0200 Subject: [PATCH 31/42] fix race condition in Service.create_container() The Service.create_container() method fetches a list of the current containers in order to determine the next container number. In doing so, it makes several API calls: one to fetch the list of containers, then one per container in order to inspect it. In some situations it can happen that a container is removed after having been listed: in that case, the call to inspect will get a 404 and raise a NotFound. One situation in which this has been observed is when trying to concurrently create multiple one-off containers for the same service (using `docker-compose run` and a unique `--name`), as described in more detail in gh-5179. This patch adds a unit test that simulates the race between the calls to list and to inspect, and changes Service._next_container_number to skip removed containers instead of blowing up. Fixes gh-5179 Signed-off-by: Alberto Piai --- compose/service.py | 24 ++++++++++++++++++------ tests/unit/service_test.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/compose/service.py b/compose/service.py index a4e5d9b82..1fa35f45a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -685,15 +685,27 @@ class Service(object): # TODO: this would benefit from github.com/docker/docker/pull/14699 # to remove the need to inspect every container def _next_container_number(self, one_off=False): - containers = filter(None, [ - Container.from_ps(self.client, container) - for container in self.client.containers( - all=True, - filters={'label': self.labels(one_off=one_off)}) - ]) + containers = self._fetch_containers( + all=True, + filters={'label': self.labels(one_off=one_off)} + ) numbers = [c.number for c in containers] return 1 if not numbers else max(numbers) + 1 + def _fetch_containers(self, **fetch_options): + # Account for containers that might have been removed since we fetched + # the list. + def soft_inspect(container): + try: + return Container.from_id(self.client, container['Id']) + except NotFound: + return None + + return filter(None, [ + soft_inspect(container) + for container in self.client.containers(**fetch_options) + ]) + def _get_aliases(self, network, container=None): return list( {self.name} | diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 9128b9550..ca8cd2e6d 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -5,6 +5,7 @@ import docker import pytest from docker.constants import DEFAULT_DOCKER_API_VERSION from docker.errors import APIError +from docker.errors import NotFound from .. import mock from .. import unittest @@ -888,6 +889,38 @@ class ServiceTest(unittest.TestCase): 'ftp_proxy': override_options['environment']['FTP_PROXY'], })) + def test_create_when_removed_containers_are_listed(self): + # This is aimed at simulating a race between the API call to list the + # containers, and the ones to inspect each of the listed containers. + # It can happen that a container has been removed after we listed it. + + # containers() returns a container that is about to be removed + self.mock_client.containers.return_value = [ + {'Id': 'rm_cont_id', 'Name': 'rm_cont', 'Image': 'img_id'}, + ] + + # inspect_container() will raise a NotFound when trying to inspect + # rm_cont_id, which at this point has been removed + def inspect(name): + if name == 'rm_cont_id': + raise NotFound(message='Not Found') + + if name == 'new_cont_id': + return {'Id': 'new_cont_id'} + + raise NotImplementedError("incomplete mock") + + self.mock_client.inspect_container.side_effect = inspect + + self.mock_client.inspect_image.return_value = {'Id': 'imageid'} + + self.mock_client.create_container.return_value = {'Id': 'new_cont_id'} + + # We should nonetheless be able to create a new container + service = Service('foo', client=self.mock_client) + + assert service.create_container().id == 'new_cont_id' + class TestServiceNetwork(unittest.TestCase): def setUp(self): From 71d40c2a9b346bb5e19d443f6ab5553a80a7b692 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Mar 2018 11:10:30 -0700 Subject: [PATCH 32/42] Avoid fallthrough with empty defaults Signed-off-by: Joffrey F --- compose/config/interpolation.py | 5 ++--- tests/unit/config/interpolation_test.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 59a567bb2..8845d73b5 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -133,9 +133,8 @@ class TemplateWithDefaults(Template): braced = mo.group('braced') if braced is not None: sep = mo.group('sep') - result = self.process_braced_group(braced, sep, mapping) - if result: - return result + if sep: + return self.process_braced_group(braced, sep, mapping) if named is not None: val = mapping[named] diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index 2ba698fbf..0d0e7d28d 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -420,3 +420,15 @@ def test_interpolate_unicode_values(): interpol("$FOO") == '十六夜　咲夜' interpol("${BAR}") == '十六夜　咲夜' + + +def test_interpolate_no_fallthrough(): + # Test regression on docker/compose#5829 + variable_mapping = { + 'TEST:-': 'hello', + 'TEST-': 'hello', + } + interpol = Interpolator(TemplateWithDefaults, variable_mapping).interpolate + + assert interpol('${TEST:-}') == '' + assert interpol('${TEST-}') == '' From 90c57f99e88c5caf4e4959b3740df7235a0e1220 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Mar 2018 12:37:21 -0700 Subject: [PATCH 33/42] On load error, retry reading the file with UTF-8 encoding Signed-off-by: Joffrey F --- compose/config/config.py | 12 +++++++++--- tests/unit/config/config_test.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 7bc220e5f..147c1d31d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import functools +import io import logging import os import string @@ -1431,10 +1432,15 @@ def has_uppercase(name): return any(char in string.ascii_uppercase for char in name) -def load_yaml(filename): +def load_yaml(filename, encoding=None): try: - with open(filename, 'r') as fh: + with io.open(filename, 'r', encoding=encoding) as fh: return yaml.safe_load(fh) - except (IOError, yaml.YAMLError) as e: + except (IOError, yaml.YAMLError, UnicodeDecodeError) as e: + if encoding is None: + # Sometimes the user's locale sets an encoding that doesn't match + # the YAML files. Im such cases, retry once with the "default" + # UTF-8 encoding + return load_yaml(filename, encoding='utf-8') error_name = getattr(e, '__module__', '') + '.' + e.__class__.__name__ raise ConfigurationError(u"{}: {}".format(error_name, e)) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 7a9bb944d..0574b215c 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3,6 +3,7 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import codecs import os import shutil import tempfile @@ -1623,6 +1624,21 @@ class ConfigTest(unittest.TestCase): assert 'line 3, column 32' in exc.exconly() + def test_load_yaml_with_bom(self): + tmpdir = py.test.ensuretemp('bom_yaml') + self.addCleanup(tmpdir.remove) + bom_yaml = tmpdir.join('docker-compose.yml') + with codecs.open(str(bom_yaml), 'w', encoding='utf-8') as f: + f.write('''\ufeff + version: '2.3' + volumes: + park_bom: + ''') + assert config.load_yaml(str(bom_yaml)) == { + 'version': '2.3', + 'volumes': {'park_bom': None} + } + def test_validate_extra_hosts_invalid(self): with pytest.raises(ConfigurationError) as exc: config.load(build_config_details({ From 344003a2f9cf1068652e4e9a919f469503068010 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Mar 2018 16:47:25 -0700 Subject: [PATCH 34/42] Ignore NotFound for external overlay networks Signed-off-by: Joffrey F --- compose/network.py | 5 +++++ tests/integration/network_test.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/compose/network.py b/compose/network.py index 027e7d5a5..1a080c40c 100644 --- a/compose/network.py +++ b/compose/network.py @@ -42,6 +42,11 @@ class Network(object): def ensure(self): if self.external: + if self.driver == 'overlay': + # Swarm nodes do not register overlay networks that were + # created on a different node unless they're in use. + # See docker/compose#4399 + return try: self.inspect() log.debug( diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py index 2ff610fbf..a2493fda1 100644 --- a/tests/integration/network_test.py +++ b/tests/integration/network_test.py @@ -1,7 +1,10 @@ from __future__ import absolute_import from __future__ import unicode_literals +import pytest + from .testcases import DockerClientTestCase +from compose.config.errors import ConfigurationError from compose.const import LABEL_NETWORK from compose.const import LABEL_PROJECT from compose.network import Network @@ -15,3 +18,20 @@ class NetworkTest(DockerClientTestCase): labels = net_data['Labels'] assert labels[LABEL_NETWORK] == net.name assert labels[LABEL_PROJECT] == net.project + + def test_network_external_default_ensure(self): + net = Network( + self.client, 'composetest', 'foonet', + external=True + ) + + with pytest.raises(ConfigurationError): + net.ensure() + + def test_network_external_overlay_ensure(self): + net = Network( + self.client, 'composetest', 'foonet', + driver='overlay', external=True + ) + + assert net.ensure() is None From e19fa1ad4d34075709884a38cdd2dd0d46f7b22b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 29 Mar 2018 16:32:15 -0700 Subject: [PATCH 35/42] Bump docker SDK -> 3.2.0 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e5a26bba2..8db120a6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.1.4 +docker==3.2.0 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index 965307264..2ddd606b1 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.1.4, < 4.0', + 'docker >= 3.2.0, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 2e100353d38a6fb5cf0ad37d0403305ab6c54f65 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 29 Mar 2018 16:24:56 -0700 Subject: [PATCH 36/42] Add support for build isolation parameter Signed-off-by: Joffrey F --- compose/config/config.py | 1 + compose/config/config_schema_v2.1.json | 3 ++- compose/config/config_schema_v2.2.json | 3 ++- compose/config/config_schema_v2.3.json | 3 ++- compose/service.py | 3 ++- tests/integration/service_test.py | 14 +++++++++++++ tests/unit/service_test.py | 27 ++++++++++++++++++++++++++ 7 files changed, 50 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 147c1d31d..84d2a86a6 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1119,6 +1119,7 @@ def merge_build(output, base, override): md.merge_scalar('network') md.merge_scalar('target') md.merge_scalar('shm_size') + md.merge_scalar('isolation') md.merge_mapping('args', parse_build_arguments) md.merge_field('cache_from', merge_unique_items_lists, default=[]) md.merge_mapping('labels', parse_labels) diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 87a730dd8..5ad5a20ea 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -88,7 +88,8 @@ "context": {"type": "string"}, "dockerfile": {"type": "string"}, "args": {"$ref": "#/definitions/list_or_dict"}, - "labels": {"$ref": "#/definitions/labels"} + "labels": {"$ref": "#/definitions/labels"}, + "isolation": {"type": "string"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index dc7076745..26044b651 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -90,7 +90,8 @@ "args": {"$ref": "#/definitions/list_or_dict"}, "labels": {"$ref": "#/definitions/labels"}, "cache_from": {"$ref": "#/definitions/list_of_strings"}, - "network": {"type": "string"} + "network": {"type": "string"}, + "isolation": {"type": "string"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index bd7ce166d..ac0778f2a 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -93,7 +93,8 @@ "network": {"type": "string"}, "target": {"type": "string"}, "shm_size": {"type": ["integer", "string"]}, - "extra_hosts": {"$ref": "#/definitions/list_or_dict"} + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "isolation": {"type": "string"} }, "additionalProperties": false } diff --git a/compose/service.py b/compose/service.py index 9b03a3991..9e5725e0a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1016,7 +1016,8 @@ class Service(object): container_limits={ 'memory': parse_bytes(memory) if memory else None }, - gzip=gzip + gzip=gzip, + isolation=build_opts.get('isolation', self.options.get('isolation', None)), ) try: diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 260867fe0..d8f4d094a 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -1123,6 +1123,20 @@ class ServiceTest(DockerClientTestCase): service.build(gzip=True) assert service.image() + @v2_1_only() + def test_build_with_isolation(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\n') + + service = self.create_service('build_isolation', build={ + 'context': text_type(base_dir), + 'isolation': 'default', + }) + service.build() + assert service.image() + def test_start_container_stays_unprivileged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 5002954b5..c4d53a2c0 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -562,6 +562,33 @@ class ServiceTest(unittest.TestCase): assert called_build_args['arg1'] == build_args['arg1'] assert called_build_args['arg2'] == 'arg2' + def test_build_with_isolation_from_service_config(self): + self.mock_client.build.return_value = [ + b'{"stream": "Successfully built 12345"}', + ] + + service = Service('foo', client=self.mock_client, build={'context': '.'}, isolation='hyperv') + service.build() + + assert self.mock_client.build.call_count == 1 + called_build_args = self.mock_client.build.call_args[1] + assert called_build_args['isolation'] == 'hyperv' + + def test_build_isolation_from_build_override_service_config(self): + self.mock_client.build.return_value = [ + b'{"stream": "Successfully built 12345"}', + ] + + service = Service( + 'foo', client=self.mock_client, build={'context': '.', 'isolation': 'default'}, + isolation='hyperv' + ) + service.build() + + assert self.mock_client.build.call_count == 1 + called_build_args = self.mock_client.build.call_args[1] + assert called_build_args['isolation'] == 'default' + def test_config_dict(self): self.mock_client.inspect_image.return_value = {'Id': 'abcd'} service = Service( From 070474acae92b037dda0966ea3e90f1281beca52 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 29 Mar 2018 17:21:47 -0700 Subject: [PATCH 37/42] Fix unit tests Signed-off-by: Joffrey F --- tests/unit/service_test.py | 40 ++++---------------------------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c4d53a2c0..012bfd5e3 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -472,24 +472,8 @@ class ServiceTest(unittest.TestCase): _, args, _ = mock_log.warn.mock_calls[0] assert 'was built because it did not already exist' in args[0] - self.mock_client.build.assert_called_once_with( - tag='default_foo', - dockerfile=None, - path='.', - pull=False, - forcerm=False, - nocache=False, - rm=True, - buildargs={}, - labels=None, - cache_from=None, - network_mode=None, - target=None, - shmsize=None, - extra_hosts=None, - container_limits={'memory': None}, - gzip=False, - ) + assert self.mock_client.build.call_count == 1 + self.mock_client.build.call_args[1]['tag'] == 'default_foo' def test_ensure_image_exists_no_build(self): service = Service('foo', client=self.mock_client, build={'context': '.'}) @@ -515,24 +499,8 @@ class ServiceTest(unittest.TestCase): service.ensure_image_exists(do_build=BuildAction.force) assert not mock_log.warn.called - self.mock_client.build.assert_called_once_with( - tag='default_foo', - dockerfile=None, - path='.', - pull=False, - forcerm=False, - nocache=False, - rm=True, - buildargs={}, - labels=None, - cache_from=None, - network_mode=None, - target=None, - shmsize=None, - extra_hosts=None, - container_limits={'memory': None}, - gzip=False - ) + assert self.mock_client.build.call_count == 1 + self.mock_client.build.call_args[1]['tag'] == 'default_foo' def test_build_does_not_pull(self): self.mock_client.build.return_value = [ From 24a4e312dc5e12f3c71a76c272d6bbbc483ca289 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Mar 2018 11:30:13 -0700 Subject: [PATCH 38/42] Bump SDK -> 3.2.1 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8db120a6e..7dce40246 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.2.0 +docker==3.2.1 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index 2ddd606b1..a7a333634 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.2.0, < 4.0', + 'docker >= 3.2.1, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 520f5d0fdece26862c622c33007775400a9b9000 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Mar 2018 17:28:43 -0800 Subject: [PATCH 39/42] Add 2.4 file format with platform support. Also reads DOCKER_DEFAULT_PLATFORM env Signed-off-by: Joffrey F --- compose/cli/command.py | 4 +- compose/config/config.py | 3 +- compose/config/config_schema_v2.4.json | 509 +++++++++++++++++++++++++ compose/const.py | 3 + compose/project.py | 3 +- compose/service.py | 20 +- docker-compose.spec | 5 + tests/unit/project_test.py | 27 ++ tests/unit/service_test.py | 43 ++- 9 files changed, 609 insertions(+), 8 deletions(-) create mode 100644 compose/config/config_schema_v2.4.json diff --git a/compose/cli/command.py b/compose/cli/command.py index 022bc8576..8a32a93a2 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -122,7 +122,9 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, ) with errors.handle_connection_errors(client): - return Project.from_config(project_name, config_data, client) + return Project.from_config( + project_name, config_data, client, environment.get('DOCKER_DEFAULT_PLATFORM') + ) def get_project_name(working_dir, project_name=None, environment=None): diff --git a/compose/config/config.py b/compose/config/config.py index 84d2a86a6..9f8a50c62 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -129,11 +129,12 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'container_name', 'credential_spec', 'dockerfile', + 'init', 'log_driver', 'log_opt', 'logging', 'network_mode', - 'init', + 'platform', 'scale', 'stop_grace_period', ] diff --git a/compose/config/config_schema_v2.4.json b/compose/config/config_schema_v2.4.json new file mode 100644 index 000000000..7cf0c006e --- /dev/null +++ b/compose/config/config_schema_v2.4.json @@ -0,0 +1,509 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v2.4.json", + "type": "object", + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + } + }, + + "patternProperties": {"^x-": {}}, + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "blkio_config": { + "type": "object", + "properties": { + "device_read_bps": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_read_iops": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_write_bps": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "device_write_iops": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_limit"} + }, + "weight": {"type": "integer"}, + "weight_device": { + "type": "array", + "items": {"$ref": "#/definitions/blkio_weight"} + } + }, + "additionalProperties": false + }, + + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/labels"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"}, + "network": {"type": "string"}, + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"$ref": "#/definitions/list_of_strings"}, + "cap_drop": {"$ref": "#/definitions/list_of_strings"}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "cpu_count": {"type": "integer", "minimum": 0}, + "cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100}, + "cpu_shares": {"type": ["number", "string"]}, + "cpu_quota": {"type": ["number", "string"]}, + "cpus": {"type": "number", "minimum": 0}, + "cpuset": {"type": "string"}, + "depends_on": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "condition": { + "type": "string", + "enum": ["service_started", "service_healthy"] + } + }, + "required": ["condition"] + } + } + } + ] + }, + "device_cgroup_rules": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"$ref": "#/definitions/list_of_strings"}, + "dns_opt": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "extends": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + + "properties": { + "service": {"type": "string"}, + "file": {"type": "string"} + }, + "required": ["service"], + "additionalProperties": false + } + ] + }, + + "external_links": {"$ref": "#/definitions/list_of_strings"}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "group_add": { + "type": "array", + "items": { + "type": ["string", "number"] + }, + "uniqueItems": true + }, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "init": {"type": ["boolean", "string"]}, + "ipc": {"type": "string"}, + "isolation": {"type": "string"}, + "labels": {"$ref": "#/definitions/labels"}, + "links": {"$ref": "#/definitions/list_of_strings"}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": {"type": "object"} + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "mem_limit": {"type": ["number", "string"]}, + "mem_reservation": {"type": ["string", "integer"]}, + "mem_swappiness": {"type": "integer"}, + "memswap_limit": {"type": ["number", "string"]}, + "network_mode": {"type": "string"}, + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"}, + "link_local_ips": {"$ref": "#/definitions/list_of_strings"}, + "priority": {"type": "number"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "oom_kill_disable": {"type": "boolean"}, + "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, + "pid": {"type": ["string", "null"]}, + "platform": {"type": "string"}, + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "runtime": {"type": "string"}, + "scale": {"type": "integer"}, + "security_opt": {"$ref": "#/definitions/list_of_strings"}, + "shm_size": {"type": ["number", "string"]}, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "pids_limit": {"type": ["number", "string"]}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "storage_opt": {"type": "object"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "additionalProperties": false, + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + }, + "tmpfs": { + "type": "object", + "properties": { + "size": {"type": ["integer", "string"]} + } + } + } + } + ], + "uniqueItems": true + } + }, + "volume_driver": {"type": "string"}, + "volumes_from": {"$ref": "#/definitions/list_of_strings"}, + "working_dir": {"type": "string"} + }, + + "dependencies": { + "memswap_limit": ["mem_limit"] + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "start_period": {"type": "string"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + + "network": { + "id": "#/definitions/network", + "type": "object", + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array" + }, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "enable_ipv6": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/labels"}, + "name": {"type": "string"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/labels"}, + "name": {"type": "string"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "labels": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": "string" + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "blkio_limit": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "rate": {"type": ["integer", "string"]} + }, + "additionalProperties": false + }, + "blkio_weight": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "weight": {"type": "integer"} + }, + "additionalProperties": false + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/const.py b/compose/const.py index 495539fb0..200a458a1 100644 --- a/compose/const.py +++ b/compose/const.py @@ -27,6 +27,7 @@ COMPOSEFILE_V2_0 = ComposeVersion('2.0') COMPOSEFILE_V2_1 = ComposeVersion('2.1') COMPOSEFILE_V2_2 = ComposeVersion('2.2') COMPOSEFILE_V2_3 = ComposeVersion('2.3') +COMPOSEFILE_V2_4 = ComposeVersion('2.4') COMPOSEFILE_V3_0 = ComposeVersion('3.0') COMPOSEFILE_V3_1 = ComposeVersion('3.1') @@ -42,6 +43,7 @@ API_VERSIONS = { COMPOSEFILE_V2_1: '1.24', COMPOSEFILE_V2_2: '1.25', COMPOSEFILE_V2_3: '1.30', + COMPOSEFILE_V2_4: '1.35', COMPOSEFILE_V3_0: '1.25', COMPOSEFILE_V3_1: '1.25', COMPOSEFILE_V3_2: '1.25', @@ -57,6 +59,7 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0', API_VERSIONS[COMPOSEFILE_V2_2]: '1.13.0', API_VERSIONS[COMPOSEFILE_V2_3]: '17.06.0', + API_VERSIONS[COMPOSEFILE_V2_4]: '17.12.0', API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0', diff --git a/compose/project.py b/compose/project.py index f66e415fe..afbec183f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -77,7 +77,7 @@ class Project(object): return labels @classmethod - def from_config(cls, name, config_data, client): + def from_config(cls, name, config_data, client, default_platform=None): """ Construct a Project from a config.Config object. """ @@ -128,6 +128,7 @@ class Project(object): volumes_from=volumes_from, secrets=secrets, pid_mode=pid_mode, + platform=service_dict.pop('platform', default_platform), **service_dict) ) diff --git a/compose/service.py b/compose/service.py index 9e5725e0a..bb9e26baa 100644 --- a/compose/service.py +++ b/compose/service.py @@ -998,6 +998,12 @@ class Service(object): if not six.PY3 and not IS_WINDOWS_PLATFORM: path = path.encode('utf8') + platform = self.options.get('platform') + if platform and version_lt(self.client.api_version, '1.35'): + raise OperationFailedError( + 'Impossible to perform platform-targeted builds for API version < 1.35' + ) + build_output = self.client.build( path=path, tag=self.image_name, @@ -1018,6 +1024,7 @@ class Service(object): }, gzip=gzip, isolation=build_opts.get('isolation', self.options.get('isolation', None)), + platform=platform, ) try: @@ -1119,11 +1126,20 @@ class Service(object): return repo, tag, separator = parse_repository_tag(self.options['image']) - tag = tag or 'latest' + kwargs = { + 'tag': tag or 'latest', + 'stream': True, + 'platform': self.options.get('platform'), + } if not silent: log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag)) + + if kwargs['platform'] and version_lt(self.client.api_version, '1.35'): + raise OperationFailedError( + 'Impossible to perform platform-targeted builds for API version < 1.35' + ) try: - output = self.client.pull(repo, tag=tag, stream=True) + output = self.client.pull(repo, **kwargs) if silent: with open(os.devnull, 'w') as devnull: return progress_stream.get_digest_from_pull( diff --git a/docker-compose.spec b/docker-compose.spec index b2b4f5f18..b8c3a4191 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -42,6 +42,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v2.3.json', 'DATA' ), + ( + 'compose/config/config_schema_v2.4.json', + 'compose/config/config_schema_v2.4.json', + 'DATA' + ), ( 'compose/config/config_schema_v3.0.json', 'compose/config/config_schema_v3.0.json', diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index b4994a99e..eb6209723 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -13,6 +13,7 @@ from compose.config.config import Config from compose.config.types import VolumeFromSpec from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_0 as V2_0 +from compose.const import COMPOSEFILE_V2_4 as V2_4 from compose.const import LABEL_SERVICE from compose.container import Container from compose.project import NoSuchService @@ -561,3 +562,29 @@ class ProjectTest(unittest.TestCase): def test_no_such_service_unicode(self): assert NoSuchService('十六夜　咲夜'.encode('utf-8')).msg == 'No such service: 十六夜　咲夜' assert NoSuchService('十六夜　咲夜').msg == 'No such service: 十六夜　咲夜' + + def test_project_platform_value(self): + service_config = { + 'name': 'web', + 'image': 'busybox:latest', + } + config_data = Config( + version=V2_4, services=[service_config], networks={}, volumes={}, secrets=None, configs=None + ) + + project = Project.from_config(name='test', client=self.mock_client, config_data=config_data) + assert project.get_service('web').options.get('platform') is None + + project = Project.from_config( + name='test', client=self.mock_client, config_data=config_data, default_platform='windows' + ) + assert project.get_service('web').options.get('platform') == 'windows' + + service_config['platform'] = 'linux/s390x' + project = Project.from_config(name='test', client=self.mock_client, config_data=config_data) + assert project.get_service('web').options.get('platform') == 'linux/s390x' + + project = Project.from_config( + name='test', client=self.mock_client, config_data=config_data, default_platform='windows' + ) + assert project.get_service('web').options.get('platform') == 'linux/s390x' diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 012bfd5e3..4ccc48653 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -21,6 +21,7 @@ from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import SECRETS_PATH from compose.container import Container +from compose.errors import OperationFailedError from compose.parallel import ParallelStreamWriter from compose.project import OneOffFilter from compose.service import build_ulimits @@ -400,7 +401,8 @@ class ServiceTest(unittest.TestCase): self.mock_client.pull.assert_called_once_with( 'someimage', tag='sometag', - stream=True) + stream=True, + platform=None) mock_log.info.assert_called_once_with('Pulling foo (someimage:sometag)...') def test_pull_image_no_tag(self): @@ -409,7 +411,8 @@ class ServiceTest(unittest.TestCase): self.mock_client.pull.assert_called_once_with( 'ababab', tag='latest', - stream=True) + stream=True, + platform=None) @mock.patch('compose.service.log', autospec=True) def test_pull_image_digest(self, mock_log): @@ -418,9 +421,30 @@ class ServiceTest(unittest.TestCase): self.mock_client.pull.assert_called_once_with( 'someimage', tag='sha256:1234', - stream=True) + stream=True, + platform=None) mock_log.info.assert_called_once_with('Pulling foo (someimage@sha256:1234)...') + @mock.patch('compose.service.log', autospec=True) + def test_pull_image_with_platform(self, mock_log): + self.mock_client.api_version = '1.35' + service = Service( + 'foo', client=self.mock_client, image='someimage:sometag', platform='windows/x86_64' + ) + service.pull() + assert self.mock_client.pull.call_count == 1 + call_args = self.mock_client.pull.call_args + assert call_args[1]['platform'] == 'windows/x86_64' + + @mock.patch('compose.service.log', autospec=True) + def test_pull_image_with_platform_unsupported_api(self, mock_log): + self.mock_client.api_version = '1.33' + service = Service( + 'foo', client=self.mock_client, image='someimage:sometag', platform='linux/arm' + ) + with pytest.raises(OperationFailedError): + service.pull() + @mock.patch('compose.service.Container', autospec=True) def test_recreate_container(self, _): mock_container = mock.create_autospec(Container) @@ -513,6 +537,19 @@ class ServiceTest(unittest.TestCase): assert self.mock_client.build.call_count == 1 assert not self.mock_client.build.call_args[1]['pull'] + def test_build_does_with_platform(self): + self.mock_client.api_version = '1.35' + self.mock_client.build.return_value = [ + b'{"stream": "Successfully built 12345"}', + ] + + service = Service('foo', client=self.mock_client, build={'context': '.'}, platform='linux') + service.build() + + assert self.mock_client.build.call_count == 1 + call_args = self.mock_client.build.call_args + assert call_args[1]['platform'] == 'linux' + def test_build_with_override_build_args(self): self.mock_client.build.return_value = [ b'{"stream": "Successfully built 12345"}', From e6420bd011d4dbb5593981fbb9ff80dda50ff98a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Mar 2018 12:41:19 -0700 Subject: [PATCH 40/42] Update 2.4 schema Signed-off-by: Joffrey F --- compose/config/config_schema_v2.4.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/compose/config/config_schema_v2.4.json b/compose/config/config_schema_v2.4.json index 7cf0c006e..731fa2f9b 100644 --- a/compose/config/config_schema_v2.4.json +++ b/compose/config/config_schema_v2.4.json @@ -93,7 +93,8 @@ "network": {"type": "string"}, "target": {"type": "string"}, "shm_size": {"type": ["integer", "string"]}, - "extra_hosts": {"$ref": "#/definitions/list_or_dict"} + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "isolation": {"type": "string"} }, "additionalProperties": false } @@ -113,6 +114,9 @@ "cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100}, "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, + "cpu_period": {"type": ["number", "string"]}, + "cpu_rt_period": {"type": ["number", "string"]}, + "cpu_rt_runtime": {"type": ["number", "string"]}, "cpus": {"type": "number", "minimum": 0}, "cpuset": {"type": "string"}, "depends_on": { @@ -219,6 +223,7 @@ "mem_swappiness": {"type": "integer"}, "memswap_limit": {"type": ["number", "string"]}, "network_mode": {"type": "string"}, + "networks": { "oneOf": [ {"$ref": "#/definitions/list_of_strings"}, @@ -258,7 +263,6 @@ }, "uniqueItems": true }, - "privileged": {"type": "boolean"}, "read_only": {"type": "boolean"}, "restart": {"type": "string"}, From e69b9a21ca4fd0f36f111f9a6968a3d854e0f28c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Mar 2018 18:02:06 -0700 Subject: [PATCH 41/42] Fix port serialization with external IP Signed-off-by: Joffrey F --- compose/config/serialize.py | 5 +++-- tests/unit/config/config_test.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 7de7f41e8..c0cf35c1b 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -151,9 +151,10 @@ def denormalize_service_dict(service_dict, version, image_digest=None): service_dict['healthcheck']['start_period'] = serialize_ns_time_value( service_dict['healthcheck']['start_period'] ) - if 'ports' in service_dict and version < V3_2: + + if 'ports' in service_dict: service_dict['ports'] = [ - p.legacy_repr() if isinstance(p, types.ServicePort) else p + p.legacy_repr() if p.external_ip or version < V3_2 else p for p in service_dict['ports'] ] if 'volumes' in service_dict and (version < V2_3 or (version > V3_0 and version < V3_2)): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 0574b215c..8a75648ac 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -4943,6 +4943,18 @@ class SerializeTest(unittest.TestCase): serialized_config = yaml.load(serialize_config(config_dict)) assert '8080:80/tcp' in serialized_config['services']['web']['ports'] + def test_serialize_ports_with_ext_ip(self): + config_dict = config.Config(version=V3_5, services=[ + { + 'ports': [types.ServicePort('80', '8080', None, None, '127.0.0.1')], + 'image': 'alpine', + 'name': 'web' + } + ], volumes={}, networks={}, secrets={}, configs={}) + + serialized_config = yaml.load(serialize_config(config_dict)) + assert '127.0.0.1:8080:80/tcp' in serialized_config['services']['web']['ports'] + def test_serialize_configs(self): service_dict = { 'image': 'example/web', From 1d329808cc5c8b6f97a7fd23503f0a48c432f302 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Mar 2018 16:04:48 -0700 Subject: [PATCH 42/42] Bump 1.21.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 74 ++++++++++++++++++++++++++++++++++++++++++++- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25e04d1b2..a8e0b8df5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,78 @@ Change log ========== +1.21.0 (2018-04-11) +------------------- + +### New features + +#### Compose file version 2.4 + +- Introduced version 2.4 of the `docker-compose.yml` specification. + This version requires Docker Engine 17.12.0 or above. + +- Added support for the `platform` parameter in service definitions. + If supplied, the parameter is also used when performing build for the + service. + +#### Compose file version 2.2 and up + +- Added support for the `cpu_rt_period` and `cpu_rt_runtime` parameters + in service definitions (2.x only). + +#### Compose file version 2.1 and up + +- Added support for the `cpu_period` parameter in service definitions + (2.x only). + +- Added support for the `isolation` parameter in service build configurations. + Additionally, the `isolation` parameter is used for builds as well if no + `build.isolation` parameter is defined. (2.x only) + +#### All formats + +- Added support for the `--workdir` flag in `docker-compose exec`. + +- Added support for the `--compress` flag in `docker-compose build`. + +- `docker-compose pull` is now performed in parallel by default. You can + opt out using the `--no-parallel` flag. The `--parallel` flag is now + deprecated and will be removed in a future version. + +- Dashes and underscores in project names are no longer stripped out. + +- `docker-compose build` now supports the use of Dockerfile from outside + the build context. + +### Bugfixes + +- Compose now checks that the volume's configuration matches the remote + volume, and errors out if a mismatch is detected. + +- Fixed a bug that caused Compose to raise unexpected errors when attempting + to create several one-off containers in parallel. + +- Fixed a bug with argument parsing when using `docker-machine config` to + generate TLS flags for `exec` and `run` commands. + +- Fixed a bug where variable substitution with an empty default value + (e.g. `${VAR:-}`) would print an incorrect warning. + +- Improved resilience when encoding of the Compose file doesn't match the + system's. Users are encouraged to use UTF-8 when possible. + +- Fixed a bug where external overlay networks in Swarm would be incorrectly + recognized as inexistent by Compose, interrupting otherwise valid + operations. + +1.20.1 (2018-03-21) +------------------- + +### Bugfixes + +- Fixed an issue where `docker-compose build` would error out if the + build context contained directory symlinks + 1.20.0 (2018-03-20) ------------------- @@ -9,7 +81,7 @@ Change log #### Compose file version 3.6 - Introduced version 3.6 of the `docker-compose.yml` specification. - This version requires to be used with Docker Engine 18.02.0 or above. + This version requires Docker Engine 18.02.0 or above. - Added support for the `tmpfs.size` property in volume mappings diff --git a/compose/__init__.py b/compose/__init__.py index a5c1364a6..d7ed9198c 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.21.0dev' +__version__ = '1.21.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index a739edb35..fb4cb8b4b 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.20.0" +VERSION="1.21.0-rc1" IMAGE="docker/compose:$VERSION"