From 700d6aca545fb32eda9cbdb6448f2f37dd66f9e8 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 15:21:35 +0300 Subject: [PATCH 01/48] Fix testcases.py formatting Signed-off-by: Alexey Rokhin --- tests/integration/testcases.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index b72fb53a8..8435f97dd 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -75,6 +75,7 @@ def v2_1_only(): return min_version_skip(V2_1) + def v2_2_only(): return min_version_skip(V2_2) @@ -83,6 +84,7 @@ def v2_3_only(): return min_version_skip(V2_3) + def v3_only(): return min_version_skip(V3_0) From 9b91f3431b6190e03cd512cc49d49f996f075e2a Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:18:28 +0300 Subject: [PATCH 02/48] skip cpu_percent test for Linux Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3ddf991b3..2583e39e8 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -28,6 +28,7 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION +from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 3d57f702f18f29c80ae90391b43b94c1c19402e7 Mon Sep 17 00:00:00 2001 From: Alexey Rokhin Date: Wed, 17 May 2017 16:42:43 +0300 Subject: [PATCH 03/48] service_test.py reorder imports Signed-off-by: Alexey Rokhin --- tests/integration/service_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2583e39e8..3ddf991b3 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -28,7 +28,6 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION -from compose.const import IS_WINDOWS_PLATFORM from compose.container import Container from compose.errors import OperationFailedError from compose.project import OneOffFilter From 5844dbb38e9e7cc0835cabbf000c3937b80c5c04 Mon Sep 17 00:00:00 2001 From: Joel Barciauskas Date: Wed, 12 Apr 2017 17:45:09 -0400 Subject: [PATCH 04/48] Add --quiet parameter to docker-compose pull, using existing silent flag Signed-off-by: Joel Barciauskas --- compose/project.py | 2 +- tests/acceptance/cli_test.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index c8b57edd2..fa536f02c 100644 --- a/compose/project.py +++ b/compose/project.py @@ -498,7 +498,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True) + service.pull(ignore_pull_failures, True, silent=silent) _, errors = parallel.parallel_execute( services, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index bba2238e7..746973a2a 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -511,6 +511,10 @@ class CLITestCase(DockerClientTestCase): assert self.dispatch(['pull', '--quiet']).stderr == '' assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_pull_with_quiet(self): + assert self.dispatch(['pull', '--quiet']).stderr == '' + assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) From 4286315bc907178efb0ce9d53bb28414ebbcd477 Mon Sep 17 00:00:00 2001 From: NikitaVlaznev Date: Mon, 19 Jun 2017 17:05:19 +0300 Subject: [PATCH 05/48] Fix double silent argument value Fix for "TypeError: pull() got multiple values for keyword argument 'silent'." This change https://github.com/docker/compose/commit/e9b6cc23fcf01d4768c7e082b7bc91b43ff84e7e caused additional value to be passed for the 'silent' argument, that was already passed there: https://github.com/docker/compose/commit/f85da99ef3273794e855afda8678174419d3bf4f Signed-off-by: Nikita Vlaznev --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index fa536f02c..9e0a7b02f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -498,7 +498,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True, silent=silent) + service.pull(ignore_pull_failures, silent=silent) _, errors = parallel.parallel_execute( services, From d1289554d505793d9ffc327df81990c475d482bf Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sat, 1 Jul 2017 13:40:02 +1200 Subject: [PATCH 06/48] Always silence pull output with --parallel This is how things were prior to the addition of the --quiet flag. Making it not silent produces output that's weird and difficult to read. Signed-off-by: Evan Shaw --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 9e0a7b02f..c8b57edd2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -498,7 +498,7 @@ class Project(object): if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, silent=silent) + service.pull(ignore_pull_failures, True) _, errors = parallel.parallel_execute( services, From 2daf3628e9dda4b58e5c38cd5c2590654ba93329 Mon Sep 17 00:00:00 2001 From: aronahl Date: Wed, 9 Aug 2017 19:44:12 -0400 Subject: [PATCH 07/48] Fix exit code 0 upon parallel pull failure. Signed-off-by: Aaron Nall --- tests/acceptance/cli_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 746973a2a..22756bd3d 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -515,6 +515,20 @@ class CLITestCase(DockerClientTestCase): assert self.dispatch(['pull', '--quiet']).stderr == '' assert self.dispatch(['pull', '--quiet']).stdout == '' + def test_pull_with_parallel_failure(self): + result = self.dispatch([ + '-f', 'ignore-pull-failures.yml', 'pull', '--parallel'], + returncode=1 + ) + + self.assertRegexpMatches(result.stderr, re.compile('^Pulling simple', re.MULTILINE)) + self.assertRegexpMatches(result.stderr, re.compile('^Pulling another', re.MULTILINE)) + self.assertRegexpMatches(result.stderr, + re.compile('^ERROR: for another .*does not exist.*', re.MULTILINE)) + self.assertRegexpMatches(result.stderr, + re.compile('''^(ERROR: )?(b')?.* nonexisting-image''', + re.MULTILINE)) + def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) From c8b2dd2fb1346e8efcdcdb6434d6ec860c9581f9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 25 Aug 2017 18:09:06 -0700 Subject: [PATCH 08/48] Add support for extension fields in v2.x and v3.4 Signed-off-by: Joffrey F --- compose/config/config_schema_v3.5.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json index fa95d6a24..5400cd99f 100644 --- a/compose/config/config_schema_v3.5.json +++ b/compose/config/config_schema_v3.5.json @@ -64,6 +64,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { From e1db4f6e191df1abe5c6d4f6ecd85e9dd3d6fbe8 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Fri, 6 Oct 2017 19:12:59 -0300 Subject: [PATCH 09/48] Build labels option: array form produces unmarshal error (fixes #5183) Signed-off-by: Guillermo Arribas --- compose/service.py | 3 ++- tests/integration/service_test.py | 19 ++++++++++++++++++- tests/unit/service_test.py | 4 ++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/compose/service.py b/compose/service.py index 1a18c6654..e2f72aa5a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,6 +23,7 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config.config import parse_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -916,7 +917,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=build_opts.get('labels', None), + labels=parse_labels(build_opts.get('labels', None)), buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3ddf991b3..6cf8ddaa9 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -761,7 +761,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] - def test_build_with_build_labels(self): + def test_build_with_build_labels_dict(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -778,6 +778,23 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + def test_build_with_build_labels_list(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\n') + + service = self.create_service('buildlabels', build={ + 'context': text_type(base_dir), + 'labels': ['com.docker.compose.test=true'] + }) + service.build() + self.addCleanup(self.client.remove_image, service.image_name) + + assert service.image() + assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' + @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 7d61807ba..5c5c2bf67 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,7 +473,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels=None, + labels={}, cache_from=None, network_mode=None, target=None, @@ -514,7 +514,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels=None, + labels={}, cache_from=None, network_mode=None, target=None, From 6dfd4693548520b3ca4d1fb284984d9781433fc5 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Tue, 10 Oct 2017 11:55:14 -0300 Subject: [PATCH 10/48] Progress markers are not shown correctly for docker-compose up (fixes #4801) Signed-off-by: Guillermo Arribas --- compose/parallel.py | 23 ++++++++++++++++------- compose/project.py | 25 ++++++++++++++++++++++++- compose/service.py | 23 ++++++++++++----------- tests/unit/service_test.py | 2 +- 4 files changed, 53 insertions(+), 20 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index d455711dd..f271561ff 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -26,7 +26,7 @@ log = logging.getLogger(__name__) STOP = object() -def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): +def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, parent_objects=None): """Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. @@ -37,9 +37,19 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): stream = get_output_stream(sys.stderr) writer = ParallelStreamWriter(stream, msg) - for obj in objects: + + if parent_objects: + display_objects = list(parent_objects) + else: + display_objects = objects + + for obj in display_objects: writer.add_object(get_name(obj)) - writer.write_initial() + + # write data in a second loop to consider all objects for width alignment + # and avoid duplicates when parent_objects exists + for obj in objects: + writer.write_initial(get_name(obj)) events = parallel_execute_iter(objects, func, get_deps, limit) @@ -237,12 +247,11 @@ class ParallelStreamWriter(object): self.lines.append(obj_index) self.width = max(self.width, len(obj_index)) - def write_initial(self): + def write_initial(self, obj_index): if self.msg is None: return - for line in self.lines: - self.stream.write("{} {:<{width}} ... \r\n".format(self.msg, line, - width=self.width)) + self.stream.write("{} {:<{width}} ... \r\n".format( + self.msg, self.lines[self.lines.index(obj_index)], width=self.width)) self.stream.flush() def _write_ansi(self, obj_index, status): diff --git a/compose/project.py b/compose/project.py index c8b57edd2..f6bd30a88 100644 --- a/compose/project.py +++ b/compose/project.py @@ -29,6 +29,7 @@ from .service import ConvergenceStrategy from .service import NetworkMode from .service import PidMode from .service import Service +from .service import ServiceName from .service import ServiceNetworkMode from .service import ServicePidMode from .utils import microseconds_from_time_nano @@ -190,6 +191,25 @@ class Project(object): service.remove_duplicate_containers() return services + def get_scaled_services(self, services, scale_override): + """ + Returns a list of this project's services as scaled ServiceName objects. + + services: a list of Service objects + scale_override: a dict with the scale to apply to each service (k: service_name, v: scale) + """ + service_names = [] + for service in services: + if service.name in scale_override: + scale = scale_override[service.name] + else: + scale = service.scale_num + + for i in range(1, scale + 1): + service_names.append(ServiceName(self.name, service.name, i)) + + return service_names + def get_links(self, service_dict): links = [] if 'links' in service_dict: @@ -430,15 +450,18 @@ class Project(object): for svc in services: svc.ensure_image_exists(do_build=do_build) plans = self._get_convergence_plans(services, strategy) + scaled_services = self.get_scaled_services(services, scale_override) def do(service): + return service.execute_convergence_plan( plans[service.name], timeout=timeout, detached=detached, scale_override=scale_override.get(service.name), rescale=rescale, - start=start + start=start, + project_services=scaled_services ) def get_deps(service): diff --git a/compose/service.py b/compose/service.py index e2f72aa5a..22a7ca53a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -379,11 +379,11 @@ class Service(object): return has_diverged - def _execute_convergence_create(self, scale, detached, start): + def _execute_convergence_create(self, scale, detached, start, project_services=None): i = self._next_container_number() def create_and_start(service, n): - container = service.create_container(number=n) + container = service.create_container(number=n, quiet=True) if not detached: container.attach_log_stream() if start: @@ -391,10 +391,11 @@ class Service(object): return container containers, errors = parallel_execute( - range(i, i + scale), - lambda n: create_and_start(self, n), - lambda n: self.get_container_name(n), + [ServiceName(self.project, self.name, index) for index in range(i, i + scale)], + lambda service_name: create_and_start(self, service_name.number), + lambda service_name: self.get_container_name(service_name.service, service_name.number), "Creating", + parent_objects=project_services ) for error in errors.values(): raise OperationFailedError(error) @@ -433,7 +434,7 @@ class Service(object): if start: _, errors = parallel_execute( containers, - lambda c: self.start_container_if_stopped(c, attach_logs=not detached), + lambda c: self.start_container_if_stopped(c, attach_logs=not detached, quiet=True), lambda c: c.name, "Starting", ) @@ -460,7 +461,7 @@ class Service(object): ) def execute_convergence_plan(self, plan, timeout=None, detached=False, - start=True, scale_override=None, rescale=True): + start=True, scale_override=None, rescale=True, project_services=None): (action, containers) = plan scale = scale_override if scale_override is not None else self.scale_num containers = sorted(containers, key=attrgetter('number')) @@ -469,7 +470,7 @@ class Service(object): if action == 'create': return self._execute_convergence_create( - scale, detached, start + scale, detached, start, project_services ) # The create action needs always needs an initial scale, but otherwise, @@ -742,7 +743,7 @@ class Service(object): container_options.update(override_options) if not container_options.get('name'): - container_options['name'] = self.get_container_name(number, one_off) + container_options['name'] = self.get_container_name(self.name, number, one_off) container_options.setdefault('detach', True) @@ -961,12 +962,12 @@ class Service(object): def custom_container_name(self): return self.options.get('container_name') - def get_container_name(self, number, one_off=False): + def get_container_name(self, service_name, number, one_off=False): if self.custom_container_name and not one_off: return self.custom_container_name container_name = build_container_name( - self.project, self.name, number, one_off, + self.project, service_name, number, one_off, ) ext_links_origins = [l.split(':')[0] for l in self.options.get('external_links', [])] if container_name in ext_links_origins: diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 5c5c2bf67..50b09c87f 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -179,7 +179,7 @@ class ServiceTest(unittest.TestCase): external_links=['default_foo_1'] ) with self.assertRaises(DependencyError): - service.get_container_name(1) + service.get_container_name('foo', 1) def test_mem_reservation(self): self.mock_client.create_host_config.return_value = {} From 8a08eb668876e73d5f18983fb591cce626ca4b27 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 11:43:06 -0700 Subject: [PATCH 11/48] Move build labels parsing to config module Signed-off-by: Joffrey F --- compose/service.py | 3 +-- tests/integration/service_test.py | 19 +------------------ tests/unit/service_test.py | 4 ++-- 3 files changed, 4 insertions(+), 22 deletions(-) diff --git a/compose/service.py b/compose/service.py index 22a7ca53a..48d428cb8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,7 +23,6 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment -from .config.config import parse_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -918,7 +917,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - labels=parse_labels(build_opts.get('labels', None)), + labels=build_opts.get('labels', None), buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 6cf8ddaa9..3ddf991b3 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -761,7 +761,7 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] - def test_build_with_build_labels_dict(self): + def test_build_with_build_labels(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) @@ -778,23 +778,6 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' - def test_build_with_build_labels_list(self): - base_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, base_dir) - - with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: - f.write('FROM busybox\n') - - service = self.create_service('buildlabels', build={ - 'context': text_type(base_dir), - 'labels': ['com.docker.compose.test=true'] - }) - service.build() - self.addCleanup(self.client.remove_image, service.image_name) - - assert service.image() - assert service.image()['Config']['Labels']['com.docker.compose.test'] == 'true' - @no_cluster('Container networks not on Swarm') def test_build_with_network(self): base_dir = tempfile.mkdtemp() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 50b09c87f..0bf0280de 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -473,7 +473,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels={}, + labels=None, cache_from=None, network_mode=None, target=None, @@ -514,7 +514,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs={}, - labels={}, + labels=None, cache_from=None, network_mode=None, target=None, From dfa7380f3781f182be236f2ba932afc7b19e6acf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Oct 2017 16:34:54 -0700 Subject: [PATCH 12/48] Add missing test constraint Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 22756bd3d..5398f0bb2 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -737,12 +737,13 @@ class CLITestCase(DockerClientTestCase): def test_run_one_off_with_volume_merge(self): self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' volume_path = os.path.abspath(os.path.join(os.getcwd(), self.base_dir, 'files')) - create_host_file(self.client, os.path.join(volume_path, 'example.txt')) + node = create_host_file(self.client, os.path.join(volume_path, 'example.txt')) self.dispatch([ '-f', 'docker-compose.merge.yml', 'run', '-v', '{}:/data'.format(volume_path), + '-e', 'constraint:node=={}'.format(node if node is not None else '*'), 'simple', 'test', '-f', '/data/example.txt' ], returncode=0) From ee6a293ae022877f8113e1fb517cb64b587f0a75 Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Tue, 17 Oct 2017 11:54:06 -0300 Subject: [PATCH 13/48] Placing dots in hostname no longer populates domainname if api >= 1.23 (fixes #4128) Signed-off-by: Guillermo Arribas --- compose/service.py | 8 +++++--- tests/unit/cli_test.py | 3 +++ tests/unit/service_test.py | 22 ++++++++++++++++------ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/compose/service.py b/compose/service.py index 48d428cb8..923c3d944 100644 --- a/compose/service.py +++ b/compose/service.py @@ -14,6 +14,7 @@ from docker.errors import APIError from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig +from docker.utils import version_lt from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port from docker.utils.utils import convert_tmpfs_mounts @@ -748,9 +749,10 @@ class Service(object): # If a qualified hostname was given, split it into an # unqualified hostname and a domainname unless domainname - # was also given explicitly. This matches the behavior of - # the official Docker CLI in that scenario. - if ('hostname' in container_options and + # was also given explicitly. This matches behavior + # until Docker Engine 1.11.0 - Docker API 1.23. + if (version_lt(self.client.api_version, '1.23') and + 'hostname' in container_options and 'domainname' not in container_options and '.' in container_options['hostname']): parts = container_options['hostname'].partition('.') diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index f9ce240a3..1a324f50a 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -10,6 +10,7 @@ from io import StringIO import docker import py import pytest +from docker.constants import DEFAULT_DOCKER_API_VERSION from .. import mock from .. import unittest @@ -98,6 +99,7 @@ class CLITestCase(unittest.TestCase): @mock.patch('compose.cli.main.PseudoTerminal', autospec=True) def test_run_interactive_passes_logs_false(self, mock_pseudo_terminal, mock_run_operation): mock_client = mock.create_autospec(docker.APIClient) + mock_client.api_version = DEFAULT_DOCKER_API_VERSION project = Project.from_config( name='composetest', client=mock_client, @@ -130,6 +132,7 @@ class CLITestCase(unittest.TestCase): def test_run_service_with_restart_always(self): mock_client = mock.create_autospec(docker.APIClient) + mock_client.api_version = DEFAULT_DOCKER_API_VERSION project = Project.from_config( name='composetest', diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0bf0280de..02b4f6223 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import docker import pytest +from docker.constants import DEFAULT_DOCKER_API_VERSION from docker.errors import APIError from .. import mock @@ -40,6 +41,7 @@ class ServiceTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION def test_containers(self): service = Service('db', self.mock_client, 'myproject', image='foo') @@ -145,12 +147,6 @@ class ServiceTest(unittest.TestCase): self.assertEqual(service._get_volumes_from(), [container_id + ':rw']) from_service.create_container.assert_called_once_with() - def test_split_domainname_none(self): - service = Service('foo', image='foo', hostname='name', client=self.mock_client) - opts = service._get_container_create_options({'image': 'foo'}, 1) - self.assertEqual(opts['hostname'], 'name', 'hostname') - self.assertFalse('domainname' in opts, 'domainname') - def test_memory_swap_limit(self): self.mock_client.create_host_config.return_value = {} @@ -232,7 +228,18 @@ class ServiceTest(unittest.TestCase): {'Type': 'syslog', 'Config': {'syslog-address': 'tcp://192.168.0.42:123'}} ) + def test_split_domainname_none(self): + service = Service( + 'foo', + image='foo', + hostname='name.domain.tld', + client=self.mock_client) + opts = service._get_container_create_options({'image': 'foo'}, 1) + self.assertEqual(opts['hostname'], 'name.domain.tld', 'hostname') + self.assertFalse('domainname' in opts, 'domainname') + def test_split_domainname_fqdn(self): + self.mock_client.api_version = '1.22' service = Service( 'foo', hostname='name.domain.tld', @@ -243,6 +250,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') def test_split_domainname_both(self): + self.mock_client.api_version = '1.22' service = Service( 'foo', hostname='name', @@ -254,6 +262,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') def test_split_domainname_weird(self): + self.mock_client.api_version = '1.22' service = Service( 'foo', hostname='name.sub', @@ -857,6 +866,7 @@ class ServiceVolumesTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION def test_build_volume_binding(self): binding = build_volume_binding(VolumeSpec.parse('/outside:/inside', True)) From 8cd46cd54de66300453b81881087b54e213472ea Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Thu, 19 Oct 2017 22:19:05 -0300 Subject: [PATCH 14/48] Allow empty default values in variable interpolation (fixes #5185) Signed-off-by: Guillermo Arribas --- compose/config/interpolation.py | 2 +- .../docker-compose.yml | 13 +++++++++++ tests/unit/config/config_test.py | 22 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/environment-interpolation-with-defaults/docker-compose.yml diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index b13ac591a..df9c988e7 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -71,7 +71,7 @@ def recursive_interpolate(obj, interpolator): class TemplateWithDefaults(Template): - idpattern = r'[_a-z][_a-z0-9]*(?::?-[^}]+)?' + idpattern = r'[_a-z][_a-z0-9]*(?::?-[^}]*)?' # Modified from python2.7/string.py def substitute(self, mapping): diff --git a/tests/fixtures/environment-interpolation-with-defaults/docker-compose.yml b/tests/fixtures/environment-interpolation-with-defaults/docker-compose.yml new file mode 100644 index 000000000..42e7cbb6a --- /dev/null +++ b/tests/fixtures/environment-interpolation-with-defaults/docker-compose.yml @@ -0,0 +1,13 @@ +version: "2.1" + +services: + web: + # set value with default, default must be ignored + image: ${IMAGE:-alpine} + + # unset value with default value + ports: + - "${HOST_PORT:-80}:8000" + + # unset value with empty default + hostname: "host-${UNSET_VALUE:-}" diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 8e3d4e2ee..1c01e52df 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2894,6 +2894,28 @@ class InterpolationTest(unittest.TestCase): } ]) + @mock.patch.dict(os.environ) + def test_config_file_with_environment_variable_with_defaults(self): + project_dir = 'tests/fixtures/environment-interpolation-with-defaults' + os.environ.update( + IMAGE="busybox", + ) + + service_dicts = config.load( + config.find( + project_dir, None, Environment.from_env_file(project_dir) + ) + ).services + + self.assertEqual(service_dicts, [ + { + 'name': 'web', + 'image': 'busybox', + 'ports': types.ServicePort.parse('80:8000'), + 'hostname': 'host-', + } + ]) + @mock.patch.dict(os.environ) def test_unset_variable_produces_warning(self): os.environ.pop('FOO', None) From eb51f0fae8d28bccd0bb339472a8c1755d602c4f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Oct 2017 17:55:32 -0700 Subject: [PATCH 15/48] Add type converter to interpolation module Signed-off-by: Joffrey F --- compose/config/config.py | 4 +- compose/config/interpolation.py | 95 +++++++++++- tests/unit/config/interpolation_test.py | 198 +++++++++++++++++++++++- 3 files changed, 287 insertions(+), 10 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index d5aaf9538..a9f82a29d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -519,13 +519,13 @@ def process_config_file(config_file, environment, service_name=None): processed_config['secrets'] = interpolate_config_section( config_file, config_file.get_secrets(), - 'secrets', + 'secret', environment) if config_file.version >= const.COMPOSEFILE_V3_3: processed_config['configs'] = interpolate_config_section( config_file, config_file.get_configs(), - 'configs', + 'config', environment ) else: diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index df9c988e7..9d7e428c9 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging +import re from string import Template import six @@ -44,9 +45,13 @@ def interpolate_environment_variables(version, config, section, environment): ) +def get_config_path(config_key, section, name): + return '{}.{}.{}'.format(section, name, config_key) + + def interpolate_value(name, config_key, value, section, interpolator): try: - return recursive_interpolate(value, interpolator) + return recursive_interpolate(value, interpolator, get_config_path(config_key, section, name)) except InvalidInterpolation as e: raise ConfigurationError( 'Invalid interpolation format for "{config_key}" option ' @@ -57,16 +62,19 @@ def interpolate_value(name, config_key, value, section, interpolator): string=e.string)) -def recursive_interpolate(obj, interpolator): +def recursive_interpolate(obj, interpolator, config_path): + def append(config_path, key): + return '{}.{}'.format(config_path, key) + if isinstance(obj, six.string_types): - return interpolator.interpolate(obj) + return converter.convert(config_path, interpolator.interpolate(obj)) if isinstance(obj, dict): return dict( - (key, recursive_interpolate(val, interpolator)) + (key, recursive_interpolate(val, interpolator, append(config_path, key))) for (key, val) in obj.items() ) if isinstance(obj, list): - return [recursive_interpolate(val, interpolator) for val in obj] + return [recursive_interpolate(val, interpolator, config_path) for val in obj] return obj @@ -100,3 +108,80 @@ class TemplateWithDefaults(Template): class InvalidInterpolation(Exception): def __init__(self, string): self.string = string + + +PATH_JOKER = '[^.]+' + + +def re_path(*args): + return re.compile('^{}$'.format('.'.join(args))) + + +def re_path_basic(section, name): + return re_path(section, PATH_JOKER, name) + + +def service_path(*args): + return re_path('service', PATH_JOKER, *args) + + +def to_boolean(s): + s = s.lower() + if s in ['y', 'yes', 'true', 'on']: + return True + elif s in ['n', 'no', 'false', 'off']: + return False + raise ValueError('"{}" is not a valid boolean value'.format(s)) + + +def to_int(s): + # We must be able to handle octal representation for `mode` values notably + if six.PY3 and re.match('^0[0-9]+$', s.strip()): + s = '0o' + s[1:] + return int(s, base=0) + + +class ConversionMap(object): + map = { + service_path('blkio_config', 'weight'): to_int, + service_path('blkio_config', 'weight_device', 'weight'): to_int, + service_path('cpus'): float, + service_path('cpu_count'): to_int, + service_path('configs', 'mode'): to_int, + service_path('secrets', 'mode'): to_int, + service_path('healthcheck', 'retries'): to_int, + service_path('healthcheck', 'disable'): to_boolean, + service_path('deploy', 'replicas'): to_int, + service_path('deploy', 'update_config', 'parallelism'): to_int, + service_path('deploy', 'update_config', 'max_failure_ratio'): float, + service_path('deploy', 'restart_policy', 'max_attempts'): to_int, + service_path('mem_swappiness'): to_int, + service_path('oom_score_adj'): to_int, + service_path('ports', 'target'): to_int, + service_path('ports', 'published'): to_int, + service_path('scale'): to_int, + service_path('ulimits', PATH_JOKER): to_int, + service_path('ulimits', PATH_JOKER, 'soft'): to_int, + service_path('ulimits', PATH_JOKER, 'hard'): to_int, + service_path('privileged'): to_boolean, + service_path('read_only'): to_boolean, + service_path('stdin_open'): to_boolean, + service_path('tty'): to_boolean, + service_path('volumes', 'read_only'): to_boolean, + service_path('volumes', 'volume', 'nocopy'): to_boolean, + re_path_basic('network', 'attachable'): to_boolean, + re_path_basic('network', 'external'): to_boolean, + re_path_basic('network', 'internal'): to_boolean, + re_path_basic('volume', 'external'): to_boolean, + re_path_basic('secret', 'external'): to_boolean, + re_path_basic('config', 'external'): to_boolean, + } + + def convert(self, path, value): + for rexp in self.map.keys(): + if rexp.match(path): + return self.map[rexp](value) + return value + + +converter = ConversionMap() diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index 018a5621a..516f5c9e9 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -9,12 +9,22 @@ from compose.config.interpolation import Interpolator from compose.config.interpolation import InvalidInterpolation from compose.config.interpolation import TemplateWithDefaults from compose.const import COMPOSEFILE_V2_0 as V2_0 -from compose.const import COMPOSEFILE_V3_1 as V3_1 +from compose.const import COMPOSEFILE_V2_3 as V2_3 +from compose.const import COMPOSEFILE_V3_4 as V3_4 @pytest.fixture def mock_env(): - return Environment({'USER': 'jenny', 'FOO': 'bar'}) + return Environment({ + 'USER': 'jenny', + 'FOO': 'bar', + 'TRUE': 'True', + 'FALSE': 'OFF', + 'POSINT': '50', + 'NEGINT': '-200', + 'FLOAT': '0.145', + 'MODE': '0600', + }) @pytest.fixture @@ -102,7 +112,189 @@ def test_interpolate_environment_variables_in_secrets(mock_env): }, 'other': {}, } - value = interpolate_environment_variables(V3_1, secrets, 'volume', mock_env) + value = interpolate_environment_variables(V3_4, secrets, 'secret', mock_env) + assert value == expected + + +def test_interpolate_environment_services_convert_types_v2(mock_env): + entry = { + 'service1': { + 'blkio_config': { + 'weight': '${POSINT}', + 'weight_device': [{'file': '/dev/sda1', 'weight': '${POSINT}'}] + }, + 'cpus': '${FLOAT}', + 'cpu_count': '$POSINT', + 'healthcheck': { + 'retries': '${POSINT:-3}', + 'disable': '${FALSE}', + 'command': 'true' + }, + 'mem_swappiness': '${DEFAULT:-127}', + 'oom_score_adj': '${NEGINT}', + 'scale': '${POSINT}', + 'ulimits': { + 'nproc': '${POSINT}', + 'nofile': { + 'soft': '${POSINT}', + 'hard': '${DEFAULT:-40000}' + }, + }, + 'privileged': '${TRUE}', + 'read_only': '${DEFAULT:-no}', + 'tty': '${DEFAULT:-N}', + 'stdin_open': '${DEFAULT-on}', + } + } + + expected = { + 'service1': { + 'blkio_config': { + 'weight': 50, + 'weight_device': [{'file': '/dev/sda1', 'weight': 50}] + }, + 'cpus': 0.145, + 'cpu_count': 50, + 'healthcheck': { + 'retries': 50, + 'disable': False, + 'command': 'true' + }, + 'mem_swappiness': 127, + 'oom_score_adj': -200, + 'scale': 50, + 'ulimits': { + 'nproc': 50, + 'nofile': { + 'soft': 50, + 'hard': 40000 + }, + }, + 'privileged': True, + 'read_only': False, + 'tty': False, + 'stdin_open': True, + } + } + + value = interpolate_environment_variables(V2_3, entry, 'service', mock_env) + assert value == expected + + +def test_interpolate_environment_services_convert_types_v3(mock_env): + entry = { + 'service1': { + 'healthcheck': { + 'retries': '${POSINT:-3}', + 'disable': '${FALSE}', + 'command': 'true' + }, + 'ulimits': { + 'nproc': '${POSINT}', + 'nofile': { + 'soft': '${POSINT}', + 'hard': '${DEFAULT:-40000}' + }, + }, + 'privileged': '${TRUE}', + 'read_only': '${DEFAULT:-no}', + 'tty': '${DEFAULT:-N}', + 'stdin_open': '${DEFAULT-on}', + 'deploy': { + 'update_config': { + 'parallelism': '${DEFAULT:-2}', + 'max_failure_ratio': '${FLOAT}', + }, + 'restart_policy': { + 'max_attempts': '$POSINT', + }, + 'replicas': '${DEFAULT-3}' + }, + 'ports': [{'target': '${POSINT}', 'published': '${DEFAULT:-5000}'}], + 'configs': [{'mode': '${MODE}', 'source': 'config1'}], + 'secrets': [{'mode': '${MODE}', 'source': 'secret1'}], + } + } + + expected = { + 'service1': { + 'healthcheck': { + 'retries': 50, + 'disable': False, + 'command': 'true' + }, + 'ulimits': { + 'nproc': 50, + 'nofile': { + 'soft': 50, + 'hard': 40000 + }, + }, + 'privileged': True, + 'read_only': False, + 'tty': False, + 'stdin_open': True, + 'deploy': { + 'update_config': { + 'parallelism': 2, + 'max_failure_ratio': 0.145, + }, + 'restart_policy': { + 'max_attempts': 50, + }, + 'replicas': 3 + }, + 'ports': [{'target': 50, 'published': 5000}], + 'configs': [{'mode': 0o600, 'source': 'config1'}], + 'secrets': [{'mode': 0o600, 'source': 'secret1'}], + } + } + + value = interpolate_environment_variables(V3_4, entry, 'service', mock_env) + assert value == expected + + +def test_interpolate_environment_network_convert_types(mock_env): + entry = { + 'network1': { + 'external': '${FALSE}', + 'attachable': '${TRUE}', + 'internal': '${DEFAULT:-false}' + } + } + + expected = { + 'network1': { + 'external': False, + 'attachable': True, + 'internal': False, + } + } + + value = interpolate_environment_variables(V3_4, entry, 'network', mock_env) + assert value == expected + + +def test_interpolate_environment_external_resource_convert_types(mock_env): + entry = { + 'resource1': { + 'external': '${TRUE}', + } + } + + expected = { + 'resource1': { + 'external': True, + } + } + + value = interpolate_environment_variables(V3_4, entry, 'network', mock_env) + assert value == expected + value = interpolate_environment_variables(V3_4, entry, 'volume', mock_env) + assert value == expected + value = interpolate_environment_variables(V3_4, entry, 'secret', mock_env) + assert value == expected + value = interpolate_environment_variables(V3_4, entry, 'config', mock_env) assert value == expected From 7dfb856244fb5bb5690c33db12cb2ad2e072f058 Mon Sep 17 00:00:00 2001 From: Reut Sharabani Date: Mon, 23 Oct 2017 23:21:16 +0300 Subject: [PATCH 16/48] Better installation instruction in release notes Changed sample download script to use the built in `-o` optoin in `curl` instead of redicrecting stdout's output. This allows users to prepend `sudo` to the snippet to make it work in common use cases where root permissions are needed to create the output file. From `curl`: -o, --output Write output to instead of stdout. Signed-off-by: Reut Sharabani --- project/RELEASE-PROCESS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 5b30545f4..d4afb87b9 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -89,7 +89,7 @@ When prompted build the non-linux binaries and test them. Alternatively, you can use the usual commands to install or upgrade Compose: ``` - curl -L https://github.com/docker/compose/releases/download/1.16.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.16.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose ``` From e022f32ee99ffa67d6f224d3ac77151c8371774d Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Mon, 23 Oct 2017 12:48:44 -0300 Subject: [PATCH 17/48] Wrong format in the healthcheck test does not issue a warning (fixes #4424) Signed-off-by: Guillermo Arribas --- compose/config/config.py | 33 ++++------ compose/config/validation.py | 24 +++++++ tests/unit/config/config_test.py | 110 ++++++++++++++++++++++--------- 3 files changed, 115 insertions(+), 52 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index a9f82a29d..8a2b2a776 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -47,6 +47,7 @@ from .validation import validate_config_section from .validation import validate_cpu from .validation import validate_depends_on from .validation import validate_extends_file_path +from .validation import validate_healthcheck from .validation import validate_links from .validation import validate_network_mode from .validation import validate_pid_mode @@ -686,6 +687,7 @@ def validate_service(service_config, service_names, config_file): validate_pid_mode(service_config, service_names) validate_depends_on(service_config, service_names) validate_links(service_config, service_names) + validate_healthcheck(service_config) if not service_dict.get('image') and has_uppercase(service_name): raise ConfigurationError( @@ -724,7 +726,7 @@ def process_service(service_config): service_dict[field] = to_list(service_dict[field]) service_dict = process_blkio_config(process_ports( - process_healthcheck(service_dict, service_config.name) + process_healthcheck(service_dict) )) return service_dict @@ -788,33 +790,20 @@ def process_blkio_config(service_dict): return service_dict -def process_healthcheck(service_dict, service_name): +def process_healthcheck(service_dict): if 'healthcheck' not in service_dict: return service_dict - hc = {} - raw = service_dict['healthcheck'] - - if raw.get('disable'): - if len(raw) > 1: - raise ConfigurationError( - 'Service "{}" defines an invalid healthcheck: ' - '"disable: true" cannot be combined with other options' - .format(service_name)) - hc['test'] = ['NONE'] - elif 'test' in raw: - hc['test'] = raw['test'] + if 'disable' in service_dict['healthcheck']: + del service_dict['healthcheck']['disable'] + service_dict['healthcheck']['test'] = ['NONE'] for field in ['interval', 'timeout', 'start_period']: - if field in raw: - if not isinstance(raw[field], six.integer_types): - hc[field] = parse_nanoseconds_int(raw[field]) - else: # Conversion has been done previously - hc[field] = raw[field] - if 'retries' in raw: - hc['retries'] = raw['retries'] + if field in service_dict['healthcheck']: + if not isinstance(service_dict['healthcheck'][field], six.integer_types): + service_dict['healthcheck'][field] = parse_nanoseconds_int( + service_dict['healthcheck'][field]) - service_dict['healthcheck'] = hc return service_dict diff --git a/compose/config/validation.py b/compose/config/validation.py index 940775a20..8247cf150 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -465,3 +465,27 @@ def handle_errors(errors, format_error_func, filename): "The Compose file{file_msg} is invalid because:\n{error_msg}".format( file_msg=" '{}'".format(filename) if filename else "", error_msg=error_msg)) + + +def validate_healthcheck(service_config): + healthcheck = service_config.config.get('healthcheck', {}) + + if 'test' in healthcheck and isinstance(healthcheck['test'], list): + if len(healthcheck['test']) == 0: + raise ConfigurationError( + 'Service "{}" defines an invalid healthcheck: ' + '"test" is an empty list' + .format(service_config.name)) + + # when disable is true config.py::process_healthcheck adds "test: ['NONE']" to service_config + elif healthcheck['test'][0] == 'NONE' and len(healthcheck) > 1: + raise ConfigurationError( + 'Service "{}" defines an invalid healthcheck: ' + '"disable: true" cannot be combined with other options' + .format(service_config.name)) + + elif healthcheck['test'][0] not in ('NONE', 'CMD', 'CMD-SHELL'): + raise ConfigurationError( + 'Service "{}" defines an invalid healthcheck: ' + 'when "test" is a list the first item must be either NONE, CMD or CMD-SHELL' + .format(service_config.name)) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1c01e52df..a758154c0 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -34,7 +34,6 @@ from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_3 as V3_3 from compose.const import IS_WINDOWS_PLATFORM -from compose.utils import nanoseconds_from_time_seconds from tests import mock from tests import unittest @@ -4210,52 +4209,103 @@ class BuildPathTest(unittest.TestCase): class HealthcheckTest(unittest.TestCase): def test_healthcheck(self): - service_dict = make_service_dict( - 'test', - {'healthcheck': { - 'test': ['CMD', 'true'], - 'interval': '1s', - 'timeout': '1m', - 'retries': 3, - 'start_period': '10s' - }}, - '.', + config_dict = config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'test': { + 'image': 'busybox', + 'healthcheck': { + 'test': ['CMD', 'true'], + 'interval': '1s', + 'timeout': '1m', + 'retries': 3, + 'start_period': '10s', + } + } + } + + }) ) - assert service_dict['healthcheck'] == { + serialized_config = yaml.load(serialize_config(config_dict)) + serialized_service = serialized_config['services']['test'] + + assert serialized_service['healthcheck'] == { 'test': ['CMD', 'true'], - 'interval': nanoseconds_from_time_seconds(1), - 'timeout': nanoseconds_from_time_seconds(60), + 'interval': '1s', + 'timeout': '1m', 'retries': 3, - 'start_period': nanoseconds_from_time_seconds(10) + 'start_period': '10s' } def test_disable(self): - service_dict = make_service_dict( - 'test', - {'healthcheck': { - 'disable': True, - }}, - '.', + config_dict = config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'test': { + 'image': 'busybox', + 'healthcheck': { + 'disable': True, + } + } + } + + }) ) - assert service_dict['healthcheck'] == { + serialized_config = yaml.load(serialize_config(config_dict)) + serialized_service = serialized_config['services']['test'] + + assert serialized_service['healthcheck'] == { 'test': ['NONE'], } def test_disable_with_other_config_is_invalid(self): with pytest.raises(ConfigurationError) as excinfo: - make_service_dict( - 'invalid-healthcheck', - {'healthcheck': { - 'disable': True, - 'interval': '1s', - }}, - '.', + config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'invalid-healthcheck': { + 'image': 'busybox', + 'healthcheck': { + 'disable': True, + 'interval': '1s', + } + } + } + + }) ) assert 'invalid-healthcheck' in excinfo.exconly() - assert 'disable' in excinfo.exconly() + assert '"disable: true" cannot be combined with other options' in excinfo.exconly() + + def test_healthcheck_with_invalid_test(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details({ + 'version': '2.3', + 'services': { + 'invalid-healthcheck': { + 'image': 'busybox', + 'healthcheck': { + 'test': ['true'], + 'interval': '1s', + 'timeout': '1m', + 'retries': 3, + 'start_period': '10s', + } + } + } + + }) + ) + + assert 'invalid-healthcheck' in excinfo.exconly() + assert 'the first item must be either NONE, CMD or CMD-SHELL' in excinfo.exconly() class GetDefaultConfigFilesTestCase(unittest.TestCase): From 947e98be387a2c534710a46f49679b5499733581 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 23 Oct 2017 14:49:36 -0700 Subject: [PATCH 18/48] Improve process_healthcheck readability Signed-off-by: Joffrey F --- compose/config/config.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 8a2b2a776..af4b69ce7 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -794,15 +794,16 @@ def process_healthcheck(service_dict): if 'healthcheck' not in service_dict: return service_dict - if 'disable' in service_dict['healthcheck']: - del service_dict['healthcheck']['disable'] - service_dict['healthcheck']['test'] = ['NONE'] + hc = service_dict['healthcheck'] + + if 'disable' in hc: + del hc['disable'] + hc['test'] = ['NONE'] for field in ['interval', 'timeout', 'start_period']: - if field in service_dict['healthcheck']: - if not isinstance(service_dict['healthcheck'][field], six.integer_types): - service_dict['healthcheck'][field] = parse_nanoseconds_int( - service_dict['healthcheck'][field]) + if field not in hc or isinstance(hc[field], six.integer_types): + continue + hc[field] = parse_nanoseconds_int(hc[field]) return service_dict From 558df8fe2f3903c06b58e354a1a749ab09c5ebea Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 23 Oct 2017 17:18:45 -0700 Subject: [PATCH 19/48] Add support for BOM-signed env files Signed-off-by: Joffrey F --- compose/config/environment.py | 2 +- tests/unit/config/environment_test.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/compose/config/environment.py b/compose/config/environment.py index 4ba228c8a..0087b6128 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -32,7 +32,7 @@ def env_vars_from_file(filename): elif not os.path.isfile(filename): raise ConfigurationError("%s is not a file." % (filename)) env = {} - with contextlib.closing(codecs.open(filename, 'r', 'utf-8')) as fileobj: + with contextlib.closing(codecs.open(filename, 'r', 'utf-8-sig')) as fileobj: for line in fileobj: line = line.strip() if line and not line.startswith('#'): diff --git a/tests/unit/config/environment_test.py b/tests/unit/config/environment_test.py index 20446d2bf..854aee5a3 100644 --- a/tests/unit/config/environment_test.py +++ b/tests/unit/config/environment_test.py @@ -3,6 +3,11 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import codecs + +import pytest + +from compose.config.environment import env_vars_from_file from compose.config.environment import Environment from tests import unittest @@ -38,3 +43,12 @@ class EnvironmentTest(unittest.TestCase): assert env.get_boolean('BAZ') is False assert env.get_boolean('FOOBAR') is True assert env.get_boolean('UNDEFINED') is False + + def test_env_vars_from_file_bom(self): + tmpdir = pytest.ensuretemp('env_file') + self.addCleanup(tmpdir.remove) + with codecs.open('{}/bom.env'.format(str(tmpdir)), 'w', encoding='utf-8') as f: + f.write('\ufeffPARK_BOM=박봄\n') + assert env_vars_from_file(str(tmpdir.join('bom.env'))) == { + 'PARK_BOM': '박봄' + } From a1a6fb485b40cc2f4fff19fc7f5067fcf0292bfa Mon Sep 17 00:00:00 2001 From: Guillermo Arribas Date: Thu, 19 Oct 2017 22:07:30 -0300 Subject: [PATCH 20/48] docker-compose exec doesn't have -e option (fixes #4551) Signed-off-by: Guillermo Arribas --- compose/cli/main.py | 59 ++++++++++++------- tests/acceptance/cli_test.py | 26 ++++++++ .../environment-exec/docker-compose.yml | 10 ++++ 3 files changed, 74 insertions(+), 21 deletions(-) create mode 100644 tests/fixtures/environment-exec/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index face38e6d..c3e30919d 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -14,6 +14,8 @@ from distutils.spawn import find_executable from inspect import getdoc from operator import attrgetter +import docker + from . import errors from . import signals from .. import __version__ @@ -402,7 +404,7 @@ class TopLevelCommand(object): """ Execute a command in a running container - Usage: exec [options] SERVICE COMMAND [ARGS...] + Usage: exec [options] [-e KEY=VAL...] SERVICE COMMAND [ARGS...] Options: -d Detached mode: Run command in the background. @@ -412,11 +414,16 @@ class TopLevelCommand(object): allocates a TTY. --index=index index of the container if there are multiple instances of a service [default: 1] + -e, --env KEY=VAL Set environment variables (can be used multiple times, + not supported in API < 1.25) """ index = int(options.get('--index')) service = self.project.get_service(options['SERVICE']) detach = options['-d'] + if options['--env'] and docker.utils.version_lt(self.project.client.api_version, '1.25'): + raise UserError("Setting environment for exec is not supported in API < 1.25'") + try: container = service.get_container(number=index) except ValueError as e: @@ -425,26 +432,7 @@ class TopLevelCommand(object): tty = not options["-T"] if IS_WINDOWS_PLATFORM and not detach: - args = ["exec"] - - if options["-d"]: - args += ["--detach"] - else: - args += ["--interactive"] - - if not options["-T"]: - args += ["--tty"] - - if options["--privileged"]: - args += ["--privileged"] - - if options["--user"]: - args += ["--user", options["--user"]] - - args += [container.id] - args += command - - sys.exit(call_docker(args)) + sys.exit(call_docker(build_exec_command(options, container.id, command))) create_exec_options = { "privileged": options["--privileged"], @@ -453,6 +441,9 @@ class TopLevelCommand(object): "stdin": tty, } + if docker.utils.version_gte(self.project.client.api_version, '1.25'): + create_exec_options["environment"] = options["--env"] + exec_id = container.create_exec(command, **create_exec_options) if detach: @@ -1295,3 +1286,29 @@ def parse_scale_args(options): ) res[service_name] = num return res + + +def build_exec_command(options, container_id, command): + args = ["exec"] + + if options["-d"]: + args += ["--detach"] + else: + args += ["--interactive"] + + if not options["-T"]: + args += ["--tty"] + + if options["--privileged"]: + args += ["--privileged"] + + if options["--user"]: + args += ["--user", options["--user"]] + + if options["--env"]: + for env_variable in options["--env"]: + args += ["--env", env_variable] + + args += [container_id] + args += command + return args diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 5398f0bb2..0fcf866ff 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -33,6 +33,7 @@ from tests.integration.testcases import no_cluster from tests.integration.testcases import pull_busybox from tests.integration.testcases import SWARM_SKIP_RM_VOLUMES from tests.integration.testcases import v2_1_only +from tests.integration.testcases import v2_2_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only @@ -1393,6 +1394,31 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(stdout, "operator\n") self.assertEqual(stderr, "") + @v2_2_only() + def test_exec_service_with_environment_overridden(self): + name = 'service' + self.base_dir = 'tests/fixtures/environment-exec' + self.dispatch(['up', '-d']) + self.assertEqual(len(self.project.containers()), 1) + + stdout, stderr = self.dispatch([ + 'exec', + '-T', + '-e', 'foo=notbar', + '--env', 'alpha=beta', + name, + 'env', + ]) + + # env overridden + assert 'foo=notbar' in stdout + # keep environment from yaml + assert 'hello=world' in stdout + # added option from command line + assert 'alpha=beta' in stdout + + self.assertEqual(stderr, '') + def test_run_service_without_links(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['run', 'console', '/bin/true']) diff --git a/tests/fixtures/environment-exec/docker-compose.yml b/tests/fixtures/environment-exec/docker-compose.yml new file mode 100644 index 000000000..813606eb8 --- /dev/null +++ b/tests/fixtures/environment-exec/docker-compose.yml @@ -0,0 +1,10 @@ +version: "2.2" + +services: + service: + image: busybox:latest + command: top + + environment: + foo: bar + hello: world From f89a55e4881b096e8cfcbf84fc9ae900eef89798 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Oct 2017 15:07:00 -0700 Subject: [PATCH 21/48] Add support for oom_kill_disable in service config Signed-off-by: Joffrey F --- compose/config/config.py | 1 + compose/config/config_schema_v2.1.json | 1 + compose/config/config_schema_v2.2.json | 1 + compose/config/config_schema_v2.3.json | 1 + compose/config/interpolation.py | 1 + compose/service.py | 2 ++ tests/integration/service_test.py | 9 +++++++-- 7 files changed, 14 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index af4b69ce7..adfb53d8f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -91,6 +91,7 @@ DOCKER_CONFIG_KEYS = [ 'mem_swappiness', 'net', 'oom_score_adj', + 'oom_kill_disable', 'pid', 'ports', 'privileged', diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 24e6ba02c..6b74f0ed6 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -229,6 +229,7 @@ } ] }, + "oom_kill_disable": {"type": "boolean"}, "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, "group_add": { "type": "array", diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 86fc5df95..21343b893 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -235,6 +235,7 @@ } ] }, + "oom_kill_disable": {"type": "boolean"}, "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, "group_add": { "type": "array", diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index ceaf44954..0e709e9d9 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -237,6 +237,7 @@ } ] }, + "oom_kill_disable": {"type": "boolean"}, "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, "group_add": { "type": "array", diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 9d7e428c9..45a5f9fc2 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -156,6 +156,7 @@ class ConversionMap(object): service_path('deploy', 'update_config', 'max_failure_ratio'): float, service_path('deploy', 'restart_policy', 'max_attempts'): to_int, service_path('mem_swappiness'): to_int, + service_path('oom_kill_disable'): to_boolean, service_path('oom_score_adj'): to_int, service_path('ports', 'target'): to_int, service_path('ports', 'published'): to_int, diff --git a/compose/service.py b/compose/service.py index 923c3d944..8839c6cfd 100644 --- a/compose/service.py +++ b/compose/service.py @@ -77,6 +77,7 @@ HOST_CONFIG_KEYS = [ 'mem_reservation', 'memswap_limit', 'mem_swappiness', + 'oom_kill_disable', 'oom_score_adj', 'pid', 'pids_limit', @@ -860,6 +861,7 @@ class Service(object): sysctls=options.get('sysctls'), pids_limit=options.get('pids_limit'), tmpfs=options.get('tmpfs'), + oom_kill_disable=options.get('oom_kill_disable'), oom_score_adj=options.get('oom_score_adj'), mem_swappiness=options.get('mem_swappiness'), group_add=options.get('group_add'), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3ddf991b3..deced2742 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -239,8 +239,7 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(set(container.get('HostConfig.SecurityOpt')), set(security_opt)) - # @pytest.mark.xfail(True, reason='Not supported on most drivers') - @pytest.mark.skipif(True, reason='https://github.com/moby/moby/issues/34270') + @pytest.mark.xfail(True, reason='Not supported on most drivers') def test_create_container_with_storage_opt(self): storage_opt = {'size': '1G'} service = self.create_service('db', storage_opt=storage_opt) @@ -248,6 +247,12 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(container.get('HostConfig.StorageOpt'), storage_opt) + def test_create_container_with_oom_kill_disable(self): + self.require_api_version('1.20') + service = self.create_service('db', oom_kill_disable=True) + container = service.create_container() + assert container.get('HostConfig.OomKillDisable') is True + def test_create_container_with_mac_address(self): service = self.create_service('db', mac_address='02:42:ac:11:65:43') container = service.create_container() From 574ac9f124f4d5048feb21c0131fdb13138e31c0 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Thu, 26 Oct 2017 11:42:57 -0400 Subject: [PATCH 22/48] Have stop_grace_period also set StopTimeout on create Signed-off-by: Andy Neff --- compose/service.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compose/service.py b/compose/service.py index 8839c6cfd..14027a1cc 100644 --- a/compose/service.py +++ b/compose/service.py @@ -15,6 +15,7 @@ from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig from docker.utils import version_lt +from docker.utils import version_gte from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port from docker.utils.utils import convert_tmpfs_mounts @@ -760,6 +761,11 @@ class Service(object): container_options['hostname'] = parts[0] container_options['domainname'] = parts[2] + if (version_gte(self.client.api_version, '1.25') and + 'stop_grace_period' in self.options): + container_options['stop_timeout'] = parse_seconds_float( + self.options.pop('stop_grace_period')) + if 'ports' in container_options or 'expose' in self.options: container_options['ports'] = build_container_ports( formatted_ports(container_options.get('ports', [])), From 41d7d6e45ba56fb02e250ac70cab26110541dd42 Mon Sep 17 00:00:00 2001 From: Andy Neff Date: Fri, 27 Oct 2017 17:44:17 -0400 Subject: [PATCH 23/48] Added unit test and used stop_timeout Signed-off-by: Andy Neff --- compose/service.py | 5 ++--- tests/unit/service_test.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 14027a1cc..245d5f7c7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -14,8 +14,8 @@ from docker.errors import APIError from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig -from docker.utils import version_lt from docker.utils import version_gte +from docker.utils import version_lt from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port from docker.utils.utils import convert_tmpfs_mounts @@ -763,8 +763,7 @@ class Service(object): if (version_gte(self.client.api_version, '1.25') and 'stop_grace_period' in self.options): - container_options['stop_timeout'] = parse_seconds_float( - self.options.pop('stop_grace_period')) + container_options['stop_timeout'] = self.stop_timeout(None) if 'ports' in container_options or 'expose' in self.options: container_options['ports'] = build_container_ports( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 02b4f6223..4c879cae7 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -228,6 +228,17 @@ class ServiceTest(unittest.TestCase): {'Type': 'syslog', 'Config': {'syslog-address': 'tcp://192.168.0.42:123'}} ) + def test_stop_grace_period(self): + self.mock_client.api_version = '1.25' + self.mock_client.create_host_config.return_value = {} + service = Service( + 'foo', + image='foo', + client=self.mock_client, + stop_grace_period="1m35s") + opts = service._get_container_create_options({'image': 'foo'}, 1) + self.assertEqual(opts['stop_timeout'], 95) + def test_split_domainname_none(self): service = Service( 'foo', From 0f978642380c593f46448c4fcd91c23649bf3451 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Nov 2017 12:26:31 -0700 Subject: [PATCH 24/48] Add shasum computation to download-binaries script Signed-off-by: Joffrey F --- script/release/download-binaries | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/release/download-binaries b/script/release/download-binaries index 5d01f5f75..bef5430f4 100755 --- a/script/release/download-binaries +++ b/script/release/download-binaries @@ -30,3 +30,8 @@ mkdir $DESTINATION wget -O $DESTINATION/docker-compose-Darwin-x86_64 $BASE_BINTRAY_URL/docker-compose-Darwin-x86_64 wget -O $DESTINATION/docker-compose-Linux-x86_64 $BASE_BINTRAY_URL/docker-compose-Linux-x86_64 wget -O $DESTINATION/docker-compose-Windows-x86_64.exe $APPVEYOR_URL + +echo -e "\n\nCopy the following lines into the integrity check table in the release notes:\n\n" +cd $DESTINATION +ls | xargs sha256sum | sed 's/ / | /g' | sed -r 's/([^ |]+)/`\1`/g' +cd - From 985010b88707b6b13ec7694d6eb06ca6f6e9d3dc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Nov 2017 12:36:53 -0700 Subject: [PATCH 25/48] 1.18.0dev Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index 20392ec99..7b954eb4f 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.17.1' +__version__ = '1.18.0dev' From 183110e0b07453a1826b70f18da0b174d43ef237 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 7 Nov 2017 17:29:23 -0800 Subject: [PATCH 26/48] Bump SDK version to latest Signed-off-by: Joffrey F --- requirements.txt | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index beeaa2851..0207b1938 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.3.9; sys_platform == 'win32' -docker==2.5.1 +docker==2.6.0 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 @@ -15,7 +15,7 @@ jsonschema==2.6.0 pypiwin32==219; sys_platform == 'win32' PySocks==1.6.7 PyYAML==3.12 -requests==2.11.1 +requests==2.18.4 six==1.10.0 texttable==0.9.1 urllib3==1.21.1 diff --git a/setup.py b/setup.py index 192a0f6af..08d708e95 100644 --- a/setup.py +++ b/setup.py @@ -33,10 +33,10 @@ install_requires = [ 'cached-property >= 1.2.0, < 2', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', - 'requests >= 2.6.1, != 2.11.0, < 2.12', + 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.5.1, < 3.0', + 'docker >= 2.6.0, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 8ecd15e5680b18491f7eb90403baafa6733c0501 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Nov 2017 16:48:41 -0800 Subject: [PATCH 27/48] Include SDK attach bugfix Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0207b1938..8d86b7d3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.3.9; sys_platform == 'win32' -docker==2.6.0 +docker==2.6.1 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index 08d708e95..bc760c3ef 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.6.0, < 3.0', + 'docker >= 2.6.1, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From b2c13e15343f9a44106eb5a85ba0c17c1de4c19e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Nov 2017 14:25:14 -0800 Subject: [PATCH 28/48] Remove redundant log message Signed-off-by: Joffrey F --- compose/service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 245d5f7c7..366bb3746 100644 --- a/compose/service.py +++ b/compose/service.py @@ -514,7 +514,6 @@ class Service(object): volumes can be copied to the new container, before the original container is removed. """ - log.info("Recreating %s" % container.name) container.stop(timeout=self.stop_timeout(timeout)) container.rename_to_tmp_name() From 67dfcd6951add2460973fe4180459e4076a0f41f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Oct 2017 14:51:11 -0700 Subject: [PATCH 29/48] Add support for extra_hosts in build config Signed-off-by: Joffrey F --- compose/config/config.py | 1 + compose/config/config_schema_v2.3.json | 3 ++- compose/service.py | 1 + tests/integration/service_test.py | 23 +++++++++++++++++++++++ tests/unit/service_test.py | 4 +++- tox.ini | 1 - 6 files changed, 30 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index adfb53d8f..4c3f93ddb 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1023,6 +1023,7 @@ def merge_build(output, base, override): md.merge_mapping('args', parse_build_arguments) md.merge_field('cache_from', merge_unique_items_lists, default=[]) md.merge_mapping('labels', parse_labels) + md.merge_mapping('extra_hosts', parse_extra_hosts) return dict(md) diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 0e709e9d9..6f923871b 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -92,7 +92,8 @@ "cache_from": {"$ref": "#/definitions/list_of_strings"}, "network": {"type": "string"}, "target": {"type": "string"}, - "shm_size": {"type": ["integer", "string"]} + "shm_size": {"type": ["integer", "string"]}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false } diff --git a/compose/service.py b/compose/service.py index 366bb3746..0b6561d99 100644 --- a/compose/service.py +++ b/compose/service.py @@ -930,6 +930,7 @@ class Service(object): network_mode=build_opts.get('network', None), target=build_opts.get('target', None), shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, + extra_hosts=build_opts.get('extra_hosts', None), ) try: diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index deced2742..00bacebf5 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -833,6 +833,29 @@ class ServiceTest(DockerClientTestCase): assert service.image() assert service.image()['Config']['Labels']['com.docker.compose.test.target'] == 'one' + @v2_3_only() + def test_build_with_extra_hosts(self): + self.require_api_version('1.27') + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('\n'.join([ + 'FROM busybox', + 'RUN ping -c1 foobar', + 'RUN ping -c1 baz', + ])) + + service = self.create_service('build_extra_hosts', build={ + 'context': text_type(base_dir), + 'extra_hosts': { + 'foobar': '127.0.0.1', + 'baz': '127.0.0.1' + } + }) + service.build() + assert service.image() + def test_start_container_stays_unprivileged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 4c879cae7..8e8f60203 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -498,6 +498,7 @@ class ServiceTest(unittest.TestCase): network_mode=None, target=None, shmsize=None, + extra_hosts=None, ) def test_ensure_image_exists_no_build(self): @@ -538,7 +539,8 @@ class ServiceTest(unittest.TestCase): cache_from=None, network_mode=None, target=None, - shmsize=None + shmsize=None, + extra_hosts=None, ) def test_build_does_not_pull(self): diff --git a/tox.ini b/tox.ini index e4f31ec85..749be3faa 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,6 @@ deps = -rrequirements-dev.txt commands = py.test -v \ - --full-trace \ --cov=compose \ --cov-report html \ --cov-report term \ From fb43b8b6b7a0411f15124f50752e5343b2080d00 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Oct 2017 16:56:46 -0700 Subject: [PATCH 30/48] Bump colorama (use unreleased fix) Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8d86b7d3a..889f87a5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,13 +2,13 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -colorama==0.3.9; sys_platform == 'win32' docker==2.6.1 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 enum34==1.1.6; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' +git+git://github.com/tartley/colorama.git@bd378c725b45eba0b8e5cc091c3ca76a954c92ff; sys_platform == 'win32' idna==2.5 ipaddress==1.0.18 jsonschema==2.6.0 diff --git a/setup.py b/setup.py index bc760c3ef..d03534040 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ extras_require = { ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5'], ':python_version < "3.3"': ['ipaddress >= 1.0.16'], - ':sys_platform == "win32"': ['colorama >= 0.3.7, < 0.4'], + ':sys_platform == "win32"': ['colorama >= 0.3.9, < 0.4'], 'socks': ['PySocks >= 1.5.6, != 1.5.7, < 2'], } From cf782a3dbbe82ccabce8cddfd89ae6b00d6b50ac Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Thu, 9 Nov 2017 17:53:27 -0600 Subject: [PATCH 31/48] Implement subnet config validation (fixes #4552) Signed-off-by: Drew Romanyk --- compose/config/config_schema_v3.5.json | 2 +- compose/config/validation.py | 30 +++++++++- tests/unit/config/config_test.py | 82 ++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json index 5400cd99f..c3ac559ee 100644 --- a/compose/config/config_schema_v3.5.json +++ b/compose/config/config_schema_v3.5.json @@ -419,7 +419,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/validation.py b/compose/config/validation.py index 8247cf150..a8061a5a4 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -5,6 +5,7 @@ import json import logging import os import re +import socket import sys import six @@ -43,6 +44,9 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' +VALID_IPV4_FORMAT = r'^(\d{1,3}.){3}\d{1,3}$' +VALID_IPV4_CIDR_FORMAT = r'^(\d|[1-2]\d|3[0-2])$' +VALID_IPV6_CIDR_FORMAT = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' @FormatChecker.cls_checks(format="ports", raises=ValidationError) @@ -64,6 +68,30 @@ def format_expose(instance): return True +@FormatChecker.cls_checks("subnet_ip_address", raises=ValidationError) +def format_subnet_ip_address(instance): + if isinstance(instance, six.string_types): + if '/' not in instance: + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + + ip_address, cidr = instance.split('/') + + if re.match(VALID_IPV4_FORMAT, ip_address): + if not (re.match(VALID_IPV4_CIDR_FORMAT, cidr) and + all(0 <= int(component) <= 255 for component in ip_address.split("."))): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + elif re.match(VALID_IPV6_CIDR_FORMAT, cidr) and hasattr(socket, "inet_pton"): + try: + if not (socket.inet_pton(socket.AF_INET6, ip_address)): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + except socket.error as e: + raise ValidationError(six.text_type(e)) + else: + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + + return True + + def match_named_volumes(service_dict, project_volumes): service_volumes = service_dict.get('volumes', []) for volume_spec in service_volumes: @@ -391,7 +419,7 @@ def process_config_schema_errors(error): def validate_against_config_schema(config_file): schema = load_jsonschema(config_file) - format_checker = FormatChecker(["ports", "expose"]) + format_checker = FormatChecker(["ports", "expose", "subnet_ip_address"]) validator = Draft4Validator( schema, resolver=RefResolver(get_resolver_path(), schema), diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index a758154c0..819d8f5be 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2846,6 +2846,88 @@ class PortsTest(unittest.TestCase): ) +class SubnetTest(unittest.TestCase): + INVALID_SUBNET_TYPES = [ + None, + False, + 10, + ] + + INVALID_SUBNET_MAPPINGS = [ + "", + "192.168.0.1/sdfsdfs", + "192.168.0.1/", + "192.168.0.1/33", + "192.168.0.1/01", + "192.168.0.1", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/sdfsdfs", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/129", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/01", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156", + ] + + ILLEGAL_SUBNET_MAPPINGS = [ + "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128" + ] + + VALID_SUBNET_MAPPINGS = [ + "192.168.0.1/0", + "192.168.0.1/32", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/0", + "fe80:0000:0000:0000:0204:61ff:fe9d:f156/128", + ] + + def test_config_invalid_subnet_type_validation(self): + for invalid_subnet in self.INVALID_SUBNET_TYPES: + with pytest.raises(ConfigurationError) as exc: + self.check_config(invalid_subnet) + + assert "contains an invalid type" in exc.value.msg + + def test_config_invalid_subnet_format_validation(self): + for invalid_subnet in self.INVALID_SUBNET_MAPPINGS: + with pytest.raises(ConfigurationError) as exc: + self.check_config(invalid_subnet) + + assert "should be of the format 'IP_ADDRESS/CIDR'" in exc.value.msg + + def test_config_illegal_subnet_type_validation(self): + for invalid_subnet in self.ILLEGAL_SUBNET_MAPPINGS: + with pytest.raises(ConfigurationError) as exc: + self.check_config(invalid_subnet) + + assert "illegal IP address string" in exc.value.msg + + def test_config_valid_subnet_format_validation(self): + for valid_subnet in self.VALID_SUBNET_MAPPINGS: + self.check_config(valid_subnet) + + def check_config(self, subnet): + config.load( + build_config_details({ + 'version': '3.5', + 'services': { + 'web': { + 'image': 'busybox' + } + }, + 'networks': { + 'default': { + 'ipam': { + 'config': [ + { + 'subnet': subnet + } + ], + 'driver': 'default' + } + } + } + }) + ) + + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From fa61a91cb5056767ba4b72faf99c295a4372e25b Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Thu, 9 Nov 2017 22:57:47 -0600 Subject: [PATCH 32/48] Fix subnet config test for windows Signed-off-by: Drew Romanyk --- compose/config/validation.py | 10 ++++++---- tests/unit/config/config_test.py | 7 +++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index a8061a5a4..c2256804b 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -72,22 +72,24 @@ def format_expose(instance): def format_subnet_ip_address(instance): if isinstance(instance, six.string_types): if '/' not in instance: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError("'{0}' 75 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) ip_address, cidr = instance.split('/') if re.match(VALID_IPV4_FORMAT, ip_address): if not (re.match(VALID_IPV4_CIDR_FORMAT, cidr) and all(0 <= int(component) <= 255 for component in ip_address.split("."))): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError( + "'{0}' 83 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) elif re.match(VALID_IPV6_CIDR_FORMAT, cidr) and hasattr(socket, "inet_pton"): try: if not (socket.inet_pton(socket.AF_INET6, ip_address)): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError( + "'{0}' 88 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) except socket.error as e: raise ValidationError(six.text_type(e)) else: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + raise ValidationError("'{0}' 92 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) return True diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 819d8f5be..51323cd32 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2896,8 +2896,11 @@ class SubnetTest(unittest.TestCase): for invalid_subnet in self.ILLEGAL_SUBNET_MAPPINGS: with pytest.raises(ConfigurationError) as exc: self.check_config(invalid_subnet) - - assert "illegal IP address string" in exc.value.msg + if IS_WINDOWS_PLATFORM: + assert "An invalid argument was supplied" in exc.value.msg or \ + "illegal IP address string" in exc.value.msg + else: + assert "illegal IP address string" in exc.value.msg def test_config_valid_subnet_format_validation(self): for valid_subnet in self.VALID_SUBNET_MAPPINGS: From df0f7e17d3dc5e806d89865eed060db78a8a0b97 Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Fri, 10 Nov 2017 18:04:11 -0600 Subject: [PATCH 33/48] Add format to other v3 configs & remove unix dependency Signed-off-by: Drew Romanyk --- compose/config/config_schema_v3.0.json | 2 +- compose/config/config_schema_v3.1.json | 2 +- compose/config/config_schema_v3.2.json | 2 +- compose/config/config_schema_v3.3.json | 2 +- compose/config/config_schema_v3.4.json | 2 +- compose/config/validation.py | 52 +++++++++++++++++--------- tests/unit/config/config_test.py | 30 ++++++++------- 7 files changed, 55 insertions(+), 37 deletions(-) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index f39344cfb..fa601bed2 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -294,7 +294,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index 719c0fa7a..41da89650 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -323,7 +323,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index 2ca8e92db..a74e2c66b 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -369,7 +369,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.3.json b/compose/config/config_schema_v3.3.json index f1eb9a661..96dc1d7d0 100644 --- a/compose/config/config_schema_v3.3.json +++ b/compose/config/config_schema_v3.3.json @@ -412,7 +412,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.4.json b/compose/config/config_schema_v3.4.json index dae7d7d23..8089c7e6d 100644 --- a/compose/config/config_schema_v3.4.json +++ b/compose/config/config_schema_v3.4.json @@ -420,7 +420,7 @@ "items": { "type": "object", "properties": { - "subnet": {"type": "string"} + "subnet": {"type": "string", "format": "subnet_ip_address"} }, "additionalProperties": false } diff --git a/compose/config/validation.py b/compose/config/validation.py index c2256804b..f97069935 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -5,7 +5,6 @@ import json import logging import os import re -import socket import sys import six @@ -44,9 +43,32 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' -VALID_IPV4_FORMAT = r'^(\d{1,3}.){3}\d{1,3}$' -VALID_IPV4_CIDR_FORMAT = r'^(\d|[1-2]\d|3[0-2])$' -VALID_IPV6_CIDR_FORMAT = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' + +VALID_IPV4_SEG = r'(\d{1,2}|1\d{2}|2[0-4]\d|25[0-5])' +VALID_REGEX_IPV4_CIDR = r'^(\d|[1-2]\d|3[0-2])$' +VALID_IPV4_ADDR = "({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) +VALID_REGEX_IPV4_ADDR = "^{IPV4_ADDR}$".format(IPV4_ADDR=VALID_IPV4_ADDR) + +VALID_IPV6_SEG = r'[0-9a-fA-F]{1,4}' +VALID_REGEX_IPV6_CIDR = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' +VALID_REGEX_IPV6_ADDR = "".join(""" +^ +( + (({IPV6_SEG}:){{7}}{IPV6_SEG})| + (({IPV6_SEG}:){{1,7}}:)| + (({IPV6_SEG}:){{1,6}}(:{IPV6_SEG}){{1,1}})| + (({IPV6_SEG}:){{1,5}}(:{IPV6_SEG}){{1,2}})| + (({IPV6_SEG}:){{1,4}}(:{IPV6_SEG}){{1,3}})| + (({IPV6_SEG}:){{1,3}}(:{IPV6_SEG}){{1,4}})| + (({IPV6_SEG}:){{1,2}}(:{IPV6_SEG}){{1,5}})| + (({IPV6_SEG}:){{1,1}}(:{IPV6_SEG}){{1,6}})| + (:((:{IPV6_SEG}){{1,7}}|:))| + (fe80:(:{IPV6_SEG}){{0,4}}%[0-9a-zA-Z]{{1,}})| + (::(ffff(:0{{1,4}}){{0,1}}:){{0,1}}{IPV4_ADDR})| + (({IPV6_SEG}:){{1,4}}:{IPV4_ADDR}) +) +$ +""".format(IPV6_SEG=VALID_IPV6_SEG, IPV4_ADDR=VALID_IPV4_ADDR).split()) @FormatChecker.cls_checks(format="ports", raises=ValidationError) @@ -72,24 +94,18 @@ def format_expose(instance): def format_subnet_ip_address(instance): if isinstance(instance, six.string_types): if '/' not in instance: - raise ValidationError("'{0}' 75 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") ip_address, cidr = instance.split('/') - if re.match(VALID_IPV4_FORMAT, ip_address): - if not (re.match(VALID_IPV4_CIDR_FORMAT, cidr) and - all(0 <= int(component) <= 255 for component in ip_address.split("."))): - raise ValidationError( - "'{0}' 83 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) - elif re.match(VALID_IPV6_CIDR_FORMAT, cidr) and hasattr(socket, "inet_pton"): - try: - if not (socket.inet_pton(socket.AF_INET6, ip_address)): - raise ValidationError( - "'{0}' 88 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) - except socket.error as e: - raise ValidationError(six.text_type(e)) + if re.match(VALID_REGEX_IPV4_ADDR, ip_address): + if not re.match(VALID_REGEX_IPV4_CIDR, cidr): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + elif re.match(VALID_REGEX_IPV6_ADDR, ip_address): + if not re.match(VALID_REGEX_IPV6_CIDR, cidr): + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") else: - raise ValidationError("'{0}' 92 should be of the format 'IP_ADDRESS/CIDR'".format(instance)) + raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") return True diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 51323cd32..1cf783c77 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2865,10 +2865,7 @@ class SubnetTest(unittest.TestCase): "fe80:0000:0000:0000:0204:61ff:fe9d:f156/129", "fe80:0000:0000:0000:0204:61ff:fe9d:f156/01", "fe80:0000:0000:0000:0204:61ff:fe9d:f156", - ] - - ILLEGAL_SUBNET_MAPPINGS = [ - "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128" + "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128", ] VALID_SUBNET_MAPPINGS = [ @@ -2876,6 +2873,21 @@ class SubnetTest(unittest.TestCase): "192.168.0.1/32", "fe80:0000:0000:0000:0204:61ff:fe9d:f156/0", "fe80:0000:0000:0000:0204:61ff:fe9d:f156/128", + "1:2:3:4:5:6:7:8/0", + "1::/0", + "1:2:3:4:5:6:7::/0", + "1::8/0", + "1:2:3:4:5:6::8/0", + "::/0", + "::8/0", + "::2:3:4:5:6:7:8/0", + "fe80::7:8%eth0/0", + "fe80::7:8%1/0", + "::255.255.255.255/0", + "::ffff:255.255.255.255/0", + "::ffff:0:255.255.255.255/0", + "2001:db8:3:4::192.0.2.33/0", + "64:ff9b::192.0.2.33/0", ] def test_config_invalid_subnet_type_validation(self): @@ -2892,16 +2904,6 @@ class SubnetTest(unittest.TestCase): assert "should be of the format 'IP_ADDRESS/CIDR'" in exc.value.msg - def test_config_illegal_subnet_type_validation(self): - for invalid_subnet in self.ILLEGAL_SUBNET_MAPPINGS: - with pytest.raises(ConfigurationError) as exc: - self.check_config(invalid_subnet) - if IS_WINDOWS_PLATFORM: - assert "An invalid argument was supplied" in exc.value.msg or \ - "illegal IP address string" in exc.value.msg - else: - assert "illegal IP address string" in exc.value.msg - def test_config_valid_subnet_format_validation(self): for valid_subnet in self.VALID_SUBNET_MAPPINGS: self.check_config(valid_subnet) From 76e9076cb714242212a428fe7cfd84159425b5bc Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Mon, 13 Nov 2017 21:53:14 -0600 Subject: [PATCH 34/48] Refactor subnet cidr validator & add new test Signed-off-by: Drew Romanyk --- compose/config/validation.py | 23 ++++++----------------- tests/unit/config/config_test.py | 3 ++- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index f97069935..0fdcb37e7 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -45,13 +45,11 @@ VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' VALID_IPV4_SEG = r'(\d{1,2}|1\d{2}|2[0-4]\d|25[0-5])' -VALID_REGEX_IPV4_CIDR = r'^(\d|[1-2]\d|3[0-2])$' VALID_IPV4_ADDR = "({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) -VALID_REGEX_IPV4_ADDR = "^{IPV4_ADDR}$".format(IPV4_ADDR=VALID_IPV4_ADDR) +VALID_REGEX_IPV4_CIDR = "^{IPV4_ADDR}/(\d|[1-2]\d|3[0-2])$".format(IPV4_ADDR=VALID_IPV4_ADDR) VALID_IPV6_SEG = r'[0-9a-fA-F]{1,4}' -VALID_REGEX_IPV6_CIDR = r'^(\d|[1-9]\d|1[0-1]\d|12[0-8])$' -VALID_REGEX_IPV6_ADDR = "".join(""" +VALID_REGEX_IPV6_CIDR = "".join(""" ^ ( (({IPV6_SEG}:){{7}}{IPV6_SEG})| @@ -67,6 +65,7 @@ VALID_REGEX_IPV6_ADDR = "".join(""" (::(ffff(:0{{1,4}}){{0,1}}:){{0,1}}{IPV4_ADDR})| (({IPV6_SEG}:){{1,4}}:{IPV4_ADDR}) ) +/(\d|[1-9]\d|1[0-1]\d|12[0-8]) $ """.format(IPV6_SEG=VALID_IPV6_SEG, IPV4_ADDR=VALID_IPV4_ADDR).split()) @@ -93,19 +92,9 @@ def format_expose(instance): @FormatChecker.cls_checks("subnet_ip_address", raises=ValidationError) def format_subnet_ip_address(instance): if isinstance(instance, six.string_types): - if '/' not in instance: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") - - ip_address, cidr = instance.split('/') - - if re.match(VALID_REGEX_IPV4_ADDR, ip_address): - if not re.match(VALID_REGEX_IPV4_CIDR, cidr): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") - elif re.match(VALID_REGEX_IPV6_ADDR, ip_address): - if not re.match(VALID_REGEX_IPV6_CIDR, cidr): - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") - else: - raise ValidationError("should be of the format 'IP_ADDRESS/CIDR'") + if not re.match(VALID_REGEX_IPV4_CIDR, instance) and \ + not re.match(VALID_REGEX_IPV6_CIDR, instance): + raise ValidationError("should use the CIDR format") return True diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1cf783c77..32ccf1cec 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2866,6 +2866,7 @@ class SubnetTest(unittest.TestCase): "fe80:0000:0000:0000:0204:61ff:fe9d:f156/01", "fe80:0000:0000:0000:0204:61ff:fe9d:f156", "ge80:0000:0000:0000:0204:61ff:fe9d:f156/128", + "192.168.0.1/31/31", ] VALID_SUBNET_MAPPINGS = [ @@ -2902,7 +2903,7 @@ class SubnetTest(unittest.TestCase): with pytest.raises(ConfigurationError) as exc: self.check_config(invalid_subnet) - assert "should be of the format 'IP_ADDRESS/CIDR'" in exc.value.msg + assert "should use the CIDR format" in exc.value.msg def test_config_valid_subnet_format_validation(self): for valid_subnet in self.VALID_SUBNET_MAPPINGS: From 6b0138d70f430b6ace9cc57287066ee9ef7a4942 Mon Sep 17 00:00:00 2001 From: Madeline Stager Date: Wed, 22 Nov 2017 16:21:47 -0600 Subject: [PATCH 35/48] implement --timeout flag for docker-compose down Fix #3370 Signed-off-by: Madeline Stager --- compose/cli/main.py | 5 ++++- compose/project.py | 4 ++-- tests/acceptance/cli_test.py | 21 +++++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index c3e30919d..f866d5809 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -371,9 +371,12 @@ class TopLevelCommand(object): attached to containers. --remove-orphans Remove containers for services not defined in the Compose file + -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. + (default: 10) """ image_type = image_type_from_opt('--rmi', options['--rmi']) - self.project.down(image_type, options['--volumes'], options['--remove-orphans']) + timeout = timeout_from_opts(options) + self.project.down(image_type, options['--volumes'], options['--remove-orphans'], timeout=timeout) def events(self, options): """ diff --git a/compose/project.py b/compose/project.py index f6bd30a88..9cc726e42 100644 --- a/compose/project.py +++ b/compose/project.py @@ -330,8 +330,8 @@ class Project(object): service_names, stopped=True, one_off=one_off ), options) - def down(self, remove_image_type, include_volumes, remove_orphans=False): - self.stop(one_off=OneOffFilter.include) + def down(self, remove_image_type, include_volumes, remove_orphans=False, timeout=None): + self.stop(one_off=OneOffFilter.include, timeout=timeout) self.find_orphan_containers(remove_orphans) self.remove_stopped(v=include_volumes, one_off=OneOffFilter.include) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 0fcf866ff..9d4ae3255 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -794,6 +794,27 @@ class CLITestCase(DockerClientTestCase): assert 'Removing network v2full_default' in result.stderr assert 'Removing network v2full_front' in result.stderr + def test_down_timeout(self): + self.dispatch(['up', '-d'], None) + service = self.project.get_service('simple') + self.assertEqual(len(service.containers()), 1) + self.assertTrue(service.containers()[0].is_running) + "" + + self.dispatch(['down', '-t', '1'], None) + + self.assertEqual(len(service.containers(stopped=True)), 0) + + def test_down_signal(self): + self.base_dir = 'tests/fixtures/stop-signal-composefile' + self.dispatch(['up', '-d'], None) + service = self.project.get_service('simple') + self.assertEqual(len(service.containers()), 1) + self.assertTrue(service.containers()[0].is_running) + + self.dispatch(['down', '-t', '1'], None) + self.assertEqual(len(service.containers(stopped=True)), 0) + def test_up_detached(self): self.dispatch(['up', '-d']) service = self.project.get_service('simple') From a99dd9f2dc51b1f25145c7638933e66817dcd4b3 Mon Sep 17 00:00:00 2001 From: Madeline Stager Date: Wed, 22 Nov 2017 17:32:51 -0600 Subject: [PATCH 36/48] Fixed example in instructions for running tests. Fix #5394 Signed-off-by: Madeline Stager --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 16bccf98b..a031e2d68 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,8 +64,8 @@ you can specify a test directory, file, module, class or method: $ script/test/default tests/unit $ script/test/default tests/unit/cli_test.py - $ script/test/default tests/unit/config_test.py::ConfigTest - $ script/test/default tests/unit/config_test.py::ConfigTest::test_load + $ script/test/default tests/unit/config/config_test.py::ConfigTest + $ script/test/default tests/unit/config/config_test.py::ConfigTest::test_load ## Finding things to work on From 7835a0755091fd5886cc33276d2a6457151ac981 Mon Sep 17 00:00:00 2001 From: Samantha Miller Date: Sun, 12 Nov 2017 11:33:34 -0600 Subject: [PATCH 37/48] Added a label option to 'docker-compose run' and test. Signed-off-by: Samantha Miller --- compose/cli/main.py | 9 ++++++++- compose/config/__init__.py | 2 ++ compose/config/config.py | 6 ++++++ compose/service.py | 5 +++++ contrib/completion/bash/docker-compose | 4 ++-- tests/acceptance/cli_test.py | 11 +++++++++++ tests/fixtures/run-labels/docker-compose.yml | 7 +++++++ tests/unit/cli_test.py | 4 ++++ 8 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/run-labels/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index f866d5809..79f663096 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -24,6 +24,7 @@ from ..bundle import MissingDigests from ..bundle import serialize_bundle from ..config import ConfigurationError from ..config import parse_environment +from ..config import parse_labels from ..config import resolve_build_args from ..config.environment import Environment from ..config.serialize import serialize_config @@ -723,7 +724,9 @@ class TopLevelCommand(object): running. If you do not want to start linked services, use `docker-compose run --no-deps SERVICE COMMAND [ARGS...]`. - Usage: run [options] [-v VOLUME...] [-p PORT...] [-e KEY=VAL...] SERVICE [COMMAND] [ARGS...] + Usage: + run [options] [-v VOLUME...] [-p PORT...] [-e KEY=VAL...] [-l KEY=VALUE...] + SERVICE [COMMAND] [ARGS...] Options: -d Detached mode: Run container in the background, print @@ -731,6 +734,7 @@ class TopLevelCommand(object): --name NAME Assign a name to the container --entrypoint CMD Override the entrypoint of the image. -e KEY=VAL Set an environment variable (can be used multiple times) + -l, --label KEY=VAL Add or override a label (can be used multiple times) -u, --user="" Run as specified username or uid --no-deps Don't start linked services. --rm Remove container after run. Ignored in detached mode. @@ -1125,6 +1129,9 @@ def build_container_options(options, detach, command): parse_environment(options['-e']) ) + if options['--label']: + container_options['labels'] = parse_labels(options['--label']) + if options['--entrypoint']: container_options['entrypoint'] = options.get('--entrypoint') diff --git a/compose/config/__init__.py b/compose/config/__init__.py index b629edf66..e1032f3de 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -8,5 +8,7 @@ from .config import DOCKER_CONFIG_KEYS from .config import find from .config import load from .config import merge_environment +from .config import merge_labels from .config import parse_environment +from .config import parse_labels from .config import resolve_build_args diff --git a/compose/config/config.py b/compose/config/config.py index 4c3f93ddb..864bc7e90 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1076,6 +1076,12 @@ def merge_environment(base, override): return env +def merge_labels(base, override): + labels = parse_labels(base) + labels.update(parse_labels(override)) + return labels + + def split_kv(kvpair): if '=' in kvpair: return kvpair.split('=', 1) diff --git a/compose/service.py b/compose/service.py index 0b6561d99..b696fd664 100644 --- a/compose/service.py +++ b/compose/service.py @@ -25,6 +25,7 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config import merge_labels from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec @@ -778,6 +779,10 @@ class Service(object): self.options.get('environment'), override_options.get('environment')) + container_options['labels'] = merge_labels( + self.options.get('labels'), + override_options.get('labels')) + binds, affinity = merge_volume_bindings( container_options.get('volumes') or [], self.options.get('tmpfs') or [], diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 1fdb27705..af0368177 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -403,14 +403,14 @@ _docker_compose_run() { __docker_compose_nospace return ;; - --entrypoint|--name|--user|-u|--volume|-v|--workdir|-w) + --entrypoint|--label|-l|--name|--user|-u|--volume|-v|--workdir|-w) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --name --no-deps --publish -p --rm --service-ports -T --user -u --volume -v --workdir -w" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --label -l --name --no-deps --publish -p --rm --service-ports -T --user -u --volume -v --workdir -w" -- "$cur" ) ) ;; *) __docker_compose_services_all diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 9d4ae3255..0ea5f5a6f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1869,6 +1869,17 @@ class CLITestCase(DockerClientTestCase): assert 'FOO=bar' in environment assert 'BAR=baz' not in environment + def test_run_label_flag(self): + self.base_dir = 'tests/fixtures/run-labels' + name = 'service' + self.dispatch(['run', '-l', 'default', '--label', 'foo=baz', name, '/bin/true']) + service = self.project.get_service(name) + container, = service.containers(stopped=True, one_off=OneOffFilter.only) + labels = container.labels + assert labels['default'] == '' + assert labels['foo'] == 'baz' + assert labels['hello'] == 'world' + def test_rm(self): service = self.project.get_service('simple') service.create_container() diff --git a/tests/fixtures/run-labels/docker-compose.yml b/tests/fixtures/run-labels/docker-compose.yml new file mode 100644 index 000000000..e8cd50065 --- /dev/null +++ b/tests/fixtures/run-labels/docker-compose.yml @@ -0,0 +1,7 @@ +service: + image: busybox:latest + command: top + + labels: + foo: bar + hello: world diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 1a324f50a..c6aa75b26 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -114,6 +114,7 @@ class CLITestCase(unittest.TestCase): 'SERVICE': 'service', 'COMMAND': None, '-e': [], + '--label': [], '--user': None, '--no-deps': None, '-d': False, @@ -150,6 +151,7 @@ class CLITestCase(unittest.TestCase): 'SERVICE': 'service', 'COMMAND': None, '-e': [], + '--label': [], '--user': None, '--no-deps': None, '-d': True, @@ -173,6 +175,7 @@ class CLITestCase(unittest.TestCase): 'SERVICE': 'service', 'COMMAND': None, '-e': [], + '--label': [], '--user': None, '--no-deps': None, '-d': True, @@ -205,6 +208,7 @@ class CLITestCase(unittest.TestCase): 'SERVICE': 'service', 'COMMAND': None, '-e': [], + '--label': [], '--user': None, '--no-deps': None, '-d': True, From 3ce2f03d70d9d5688d6a76e6ad7993f994292ad1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 29 Nov 2017 12:21:56 -0800 Subject: [PATCH 38/48] Use mounts for secrets instead of volumes Signed-off-by: Joffrey F --- compose/config/types.py | 42 ++++++++++++++++++++++++++++++++++++++ compose/service.py | 27 ++++++++++++++++++++---- tests/unit/service_test.py | 12 +++++------ 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index c410343b8..548f2c1cd 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -133,6 +133,48 @@ def normalize_path_for_engine(path): return path.replace('\\', '/') +class MountSpec(object): + options_map = { + 'volume': { + 'nocopy': 'no_copy' + }, + 'bind': { + 'propagation': 'propagation' + } + } + _fields = ['type', 'source', 'target', 'read_only', 'consistency'] + + def __init__(self, type, source=None, target=None, read_only=None, consistency=None, **kwargs): + self.type = type + self.source = source + self.target = target + self.read_only = read_only + self.consistency = consistency + self.options = None + if self.type in kwargs: + self.options = kwargs[self.type] + + def as_volume_spec(self): + mode = 'ro' if self.read_only else 'rw' + return VolumeSpec(external=self.source, internal=self.target, mode=mode) + + def legacy_repr(self): + return self.as_volume_spec().repr() + + def repr(self): + res = {} + for field in self._fields: + if getattr(self, field, None): + res[field] = getattr(self, field) + if self.options: + res[self.type] = self.options + return res + + @property + def is_named_volume(self): + return self.type == 'volume' and self.source + + class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): @classmethod diff --git a/compose/service.py b/compose/service.py index b696fd664..07db3ac5f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -14,6 +14,7 @@ from docker.errors import APIError from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig +from docker.types import Mount from docker.utils import version_gte from docker.utils import version_lt from docker.utils.ports import build_port_bindings @@ -27,6 +28,7 @@ from .config import DOCKER_CONFIG_KEYS from .config import merge_environment from .config import merge_labels from .config.errors import DependencyError +from .config.types import MountSpec from .config.types import ServicePort from .config.types import VolumeSpec from .const import DEFAULT_TIMEOUT @@ -795,9 +797,13 @@ class Service(object): secret_volumes = self.get_secret_volumes() if secret_volumes: - override_options['binds'].extend(v.repr() for v in secret_volumes) - container_options['volumes'].update( - (v.internal, {}) for v in secret_volumes) + if version_lt(self.client.api_version, '1.30'): + override_options['binds'].extend(v.legacy_repr() for v in secret_volumes) + container_options['volumes'].update( + (v.target, {}) for v in secret_volumes + ) + else: + override_options['mounts'] = [build_mount(v) for v in secret_volumes] container_options['image'] = self.image_name @@ -891,6 +897,7 @@ class Service(object): device_read_iops=blkio_config.get('device_read_iops'), device_write_bps=blkio_config.get('device_write_bps'), device_write_iops=blkio_config.get('device_write_iops'), + mounts=options.get('mounts'), ) def get_secret_volumes(self): @@ -901,7 +908,7 @@ class Service(object): elif not os.path.isabs(target): target = '{}/{}'.format(const.SECRETS_PATH, target) - return VolumeSpec(secret['file'], target, 'ro') + return MountSpec('bind', secret['file'], target, read_only=True) return [build_spec(secret) for secret in self.secrets] @@ -1346,6 +1353,18 @@ def build_volume_from(volume_from_spec): return "{}:{}".format(volume_from_spec.source.id, volume_from_spec.mode) +def build_mount(mount_spec): + kwargs = {} + if mount_spec.options: + for option, sdk_name in mount_spec.options_map[mount_spec.type].items(): + if option in mount_spec.options: + kwargs[sdk_name] = mount_spec.options[option] + + return Mount( + type=mount_spec.type, target=mount_spec.target, source=mount_spec.source, + read_only=mount_spec.read_only, consistency=mount_spec.consistency, **kwargs + ) + # Labels diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 8e8f60203..87c86a731 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -1133,8 +1133,8 @@ class ServiceSecretTest(unittest.TestCase): ) volumes = service.get_secret_volumes() - assert volumes[0].external == secret1['file'] - assert volumes[0].internal == '{}/{}'.format(SECRETS_PATH, secret1['secret'].target) + assert volumes[0].source == secret1['file'] + assert volumes[0].target == '{}/{}'.format(SECRETS_PATH, secret1['secret'].target) def test_get_secret_volumes_abspath(self): secret1 = { @@ -1149,8 +1149,8 @@ class ServiceSecretTest(unittest.TestCase): ) volumes = service.get_secret_volumes() - assert volumes[0].external == secret1['file'] - assert volumes[0].internal == secret1['secret'].target + assert volumes[0].source == secret1['file'] + assert volumes[0].target == secret1['secret'].target def test_get_secret_volumes_no_target(self): secret1 = { @@ -1165,5 +1165,5 @@ class ServiceSecretTest(unittest.TestCase): ) volumes = service.get_secret_volumes() - assert volumes[0].external == secret1['file'] - assert volumes[0].internal == '{}/{}'.format(SECRETS_PATH, secret1['secret'].source) + assert volumes[0].source == secret1['file'] + assert volumes[0].target == '{}/{}'.format(SECRETS_PATH, secret1['secret'].source) From dba2abd523dc81d30a6e19c3817a93086ed8e301 Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Thu, 30 Nov 2017 10:21:27 -0600 Subject: [PATCH 39/48] Add config validation for service volumes, fixes #5352 Signed-off-by: Drew Romanyk --- compose/config/config_schema_v3.2.json | 1 + compose/config/config_schema_v3.3.json | 1 + compose/config/config_schema_v3.4.json | 1 + compose/config/config_schema_v3.5.json | 1 + tests/unit/config/config_test.py | 27 ++++++++++++++++++++++++++ 5 files changed, 31 insertions(+) diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index a74e2c66b..0baf6a1a9 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -245,6 +245,7 @@ { "type": "object", "required": ["type"], + "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, diff --git a/compose/config/config_schema_v3.3.json b/compose/config/config_schema_v3.3.json index 96dc1d7d0..efc0fdbd7 100644 --- a/compose/config/config_schema_v3.3.json +++ b/compose/config/config_schema_v3.3.json @@ -278,6 +278,7 @@ { "type": "object", "required": ["type"], + "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, diff --git a/compose/config/config_schema_v3.4.json b/compose/config/config_schema_v3.4.json index 8089c7e6d..576ecfd84 100644 --- a/compose/config/config_schema_v3.4.json +++ b/compose/config/config_schema_v3.4.json @@ -282,6 +282,7 @@ { "type": "object", "required": ["type"], + "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json index c3ac559ee..1e65b2087 100644 --- a/compose/config/config_schema_v3.5.json +++ b/compose/config/config_schema_v3.5.json @@ -282,6 +282,7 @@ { "type": "object", "required": ["type"], + "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 32ccf1cec..00ba6c2c6 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2631,6 +2631,33 @@ class ConfigTest(unittest.TestCase): ] assert service_sort(service_dicts) == service_sort(expected) + def test_service_volume_invalid_config(self): + config_details = build_config_details( + { + 'version': '3.2', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'args': None, + }, + 'volumes': [ + { + "type": "volume", + "source": "/data", + "garbage": { + "and": "error" + } + } + ] + }, + }, + } + ) + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + assert "services.web.volumes contains unsupported option: 'garbage'" in exc.exconly() + class NetworkModeTest(unittest.TestCase): From 4099c97758fa4333bfd3b70a82581d4a15a403d7 Mon Sep 17 00:00:00 2001 From: Drew Romanyk Date: Thu, 30 Nov 2017 10:59:25 -0600 Subject: [PATCH 40/48] Add ipam default driver, fixes #5248 Signed-off-by: Drew Romanyk --- compose/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/network.py b/compose/network.py index 2e0a7e6ec..ee5939c15 100644 --- a/compose/network.py +++ b/compose/network.py @@ -116,7 +116,7 @@ def create_ipam_config_from_dict(ipam_dict): return None return IPAMConfig( - driver=ipam_dict.get('driver'), + driver=ipam_dict.get('driver') or 'default', pool_configs=[ IPAMPool( subnet=config.get('subnet'), From 7765eed9db5d87c8676e2f8d2fd1dd69ea27e4cb Mon Sep 17 00:00:00 2001 From: Fumiaki MATSUSHIMA Date: Sun, 3 Dec 2017 01:07:17 +0900 Subject: [PATCH 41/48] Specify osx_image to fix CI Signed-off-by: Fumiaki Matsushima --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fbf269646..8fef7ed1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ matrix: services: - docker - os: osx + osx_image: xcode7.3 language: generic install: ./script/travis/install From 20a393d4f95adc5cca092b41138ff41e32627dc9 Mon Sep 17 00:00:00 2001 From: Samantha Miller Date: Fri, 24 Nov 2017 22:53:48 -0600 Subject: [PATCH 42/48] Adds support for a memory flag to docker-compose build. Signed-off-by: Samantha Miller --- compose/cli/main.py | 2 ++ compose/project.py | 5 +++-- compose/service.py | 5 ++++- contrib/completion/bash/docker-compose | 2 +- contrib/completion/zsh/_docker-compose | 1 + tests/acceptance/cli_test.py | 6 ++++++ tests/fixtures/build-memory/Dockerfile | 4 ++++ tests/fixtures/build-memory/docker-compose.yml | 6 ++++++ tests/unit/service_test.py | 2 ++ 9 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/build-memory/Dockerfile create mode 100644 tests/fixtures/build-memory/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 79f663096..f842f05c8 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -233,6 +233,7 @@ class TopLevelCommand(object): --force-rm Always remove intermediate containers. --no-cache Do not use cache when building the image. --pull Always attempt to pull a newer version of the image. + -m, --memory MEM Sets memory limit for the bulid container. --build-arg key=val Set build-time variables for one service. """ service_names = options['SERVICE'] @@ -249,6 +250,7 @@ class TopLevelCommand(object): no_cache=bool(options.get('--no-cache', False)), pull=bool(options.get('--pull', False)), force_rm=bool(options.get('--force-rm', False)), + memory=options.get('--memory'), build_args=build_args) def bundle(self, config_options, options): diff --git a/compose/project.py b/compose/project.py index 9cc726e42..411576386 100644 --- a/compose/project.py +++ b/compose/project.py @@ -357,10 +357,11 @@ class Project(object): ) return containers - def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, build_args=None): + def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None, + build_args=None): for service in self.get_services(service_names): if service.can_be_built(): - service.build(no_cache, pull, force_rm, build_args) + service.build(no_cache, pull, force_rm, memory, build_args) else: log.info('%s uses an image, skipping' % service.name) diff --git a/compose/service.py b/compose/service.py index 07db3ac5f..bfc2e5940 100644 --- a/compose/service.py +++ b/compose/service.py @@ -912,7 +912,7 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] - def build(self, no_cache=False, pull=False, force_rm=False, build_args_override=None): + def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None): log.info('Building %s' % self.name) build_opts = self.options.get('build', {}) @@ -943,6 +943,9 @@ class Service(object): target=build_opts.get('target', None), shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, extra_hosts=build_opts.get('extra_hosts', None), + container_limits={ + 'memory': parse_bytes(memory) if memory else None + }, ) try: diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index af0368177..87161d0ac 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -120,7 +120,7 @@ _docker_compose_build() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--build-arg --force-rm --help --no-cache --pull" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--build-arg --force-rm --help --memory --no-cache --pull" -- "$cur" ) ) ;; *) __docker_compose_services_from_build diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index f53f96334..c0a54cced 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -196,6 +196,7 @@ __docker-compose_subcommand() { $opts_help \ "*--build-arg=[Set build-time variables for one service.]:=: " \ '--force-rm[Always remove intermediate containers.]' \ + '--memory[Memory limit for the build container.]' \ '--no-cache[Do not use cache when building the image.]' \ '--pull[Always attempt to pull a newer version of the image.]' \ '*:services:__docker-compose_services_from_build' && ret=0 diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 0ea5f5a6f..21e716751 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -602,6 +602,12 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['build', '--no-cache'], None) assert 'shm_size: 96' in result.stdout + def test_build_memory_build_option(self): + pull_busybox(self.client) + self.base_dir = 'tests/fixtures/build-memory' + result = self.dispatch(['build', '--no-cache', '--memory', '96m', 'service'], None) + assert 'memory: 100663296' in result.stdout # 96 * 1024 * 1024 + def test_bundle_with_digests(self): self.base_dir = 'tests/fixtures/bundle-with-digests/' tmpdir = pytest.ensuretemp('cli_test_bundle') diff --git a/tests/fixtures/build-memory/Dockerfile b/tests/fixtures/build-memory/Dockerfile new file mode 100644 index 000000000..b27349b96 --- /dev/null +++ b/tests/fixtures/build-memory/Dockerfile @@ -0,0 +1,4 @@ +FROM busybox + +# Report the memory (through the size of the group memory) +RUN echo "memory:" $(cat /sys/fs/cgroup/memory/memory.limit_in_bytes) diff --git a/tests/fixtures/build-memory/docker-compose.yml b/tests/fixtures/build-memory/docker-compose.yml new file mode 100644 index 000000000..f98355851 --- /dev/null +++ b/tests/fixtures/build-memory/docker-compose.yml @@ -0,0 +1,6 @@ +version: '3.5' + +services: + service: + build: + context: . diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 87c86a731..16670cff5 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -499,6 +499,7 @@ class ServiceTest(unittest.TestCase): target=None, shmsize=None, extra_hosts=None, + container_limits={'memory': None}, ) def test_ensure_image_exists_no_build(self): @@ -541,6 +542,7 @@ class ServiceTest(unittest.TestCase): target=None, shmsize=None, extra_hosts=None, + container_limits={'memory': None}, ) def test_build_does_not_pull(self): From 34ea11fcb72dcf4d314f62c457d2cfab1f81ee6c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 1 Dec 2017 15:23:32 -0800 Subject: [PATCH 43/48] Allow port publish ranges Signed-off-by: Joffrey F --- compose/config/types.py | 18 +++++++++++++----- tests/unit/config/types_test.py | 26 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index 548f2c1cd..d3b3cfc53 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -319,11 +319,19 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext except ValueError: raise ConfigurationError('Invalid target port: {}'.format(target)) - try: - if published: - published = int(published) - except ValueError: - raise ConfigurationError('Invalid published port: {}'.format(published)) + if published: + if isinstance(published, six.string_types) and '-' in published: # "x-y:z" format + a, b = published.split('-', 1) + try: + int(a) + int(b) + except ValueError: + raise ConfigurationError('Invalid published port: {}'.format(published)) + else: + try: + published = int(published) + except ValueError: + raise ConfigurationError('Invalid published port: {}'.format(published)) return super(ServicePort, cls).__new__( cls, target, published, *args, **kwargs diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 3a43f727b..e7cc67b04 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -100,11 +100,37 @@ class TestServicePort(object): 'published': 25001 } in reprs + def test_parse_port_publish_range(self): + ports = ServicePort.parse('4440-4450:4000') + assert len(ports) == 1 + reprs = [p.repr() for p in ports] + assert { + 'target': 4000, + 'published': '4440-4450' + } in reprs + def test_parse_invalid_port(self): port_def = '4000p' with pytest.raises(ConfigurationError): ServicePort.parse(port_def) + def test_parse_invalid_publish_range(self): + port_def = '-4000:4000' + with pytest.raises(ConfigurationError): + ServicePort.parse(port_def) + + port_def = 'asdf:4000' + with pytest.raises(ConfigurationError): + ServicePort.parse(port_def) + + port_def = '1234-12f:4000' + with pytest.raises(ConfigurationError): + ServicePort.parse(port_def) + + port_def = '1234-1235-1239:4000' + with pytest.raises(ConfigurationError): + ServicePort.parse(port_def) + class TestVolumeSpec(object): From 58f2f10d49a8b46888173236277658af39971f36 Mon Sep 17 00:00:00 2001 From: Madeline Stager Date: Mon, 4 Dec 2017 20:01:00 -0600 Subject: [PATCH 44/48] Raise error if up used with both -d and --timeout Fix #5434 Signed-off-by: Madeline Stager --- compose/cli/main.py | 10 +++++++--- tests/acceptance/cli_test.py | 15 +++------------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index f842f05c8..222f7d013 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -898,8 +898,8 @@ class TopLevelCommand(object): Options: -d Detached mode: Run containers in the background, - print new container names. - Incompatible with --abort-on-container-exit. + print new container names. Incompatible with + --abort-on-container-exit and --timeout. --no-color Produce monochrome output. --no-deps Don't start linked services. --force-recreate Recreate containers even if their configuration @@ -913,7 +913,8 @@ class TopLevelCommand(object): --abort-on-container-exit Stops all containers if any container was stopped. Incompatible with -d. -t, --timeout TIMEOUT Use this timeout in seconds for container shutdown - when attached or when containers are already + when attached or when containers are already. + Incompatible with -d. running. (default: 10) --remove-orphans Remove containers for services not defined in the Compose file @@ -934,6 +935,9 @@ class TopLevelCommand(object): if detached and (cascade_stop or exit_value_from): raise UserError("--abort-on-container-exit and -d cannot be combined.") + if detached and timeout: + raise UserError("-d and --timeout cannot be combined.") + if no_start: for excluded in ['-d', '--abort-on-container-exit', '--exit-code-from']: if options.get(excluded): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 21e716751..251e39db6 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1325,18 +1325,9 @@ class CLITestCase(DockerClientTestCase): ['up', '-d', '--force-recreate', '--no-recreate'], returncode=1) - def test_up_with_timeout(self): - self.dispatch(['up', '-d', '-t', '1']) - service = self.project.get_service('simple') - another = self.project.get_service('another') - self.assertEqual(len(service.containers()), 1) - self.assertEqual(len(another.containers()), 1) - - # Ensure containers don't have stdin and stdout connected in -d mode - config = service.containers()[0].inspect()['Config'] - self.assertFalse(config['AttachStderr']) - self.assertFalse(config['AttachStdout']) - self.assertFalse(config['AttachStdin']) + def test_up_with_timeout_detached(self): + result = self.dispatch(['up', '-d', '-t', '1'], returncode=1) + assert "-d and --timeout cannot be combined." in result.stderr def test_up_handles_sigint(self): proc = start_process(self.base_dir, ['up', '-t', '2']) From 084818ce2b31e121268228e9b696ed0bab43bad2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 4 Dec 2017 22:47:33 -0800 Subject: [PATCH 45/48] Add support for mount syntax Signed-off-by: Joffrey F --- compose/config/config.py | 40 +++++++------ compose/config/config_schema_v2.3.json | 34 ++++++++++- compose/config/serialize.py | 6 ++ compose/config/types.py | 13 ++++ compose/service.py | 63 ++++++++++++++------ compose/utils.py | 2 +- compose/volume.py | 9 ++- tests/acceptance/cli_test.py | 22 ++++--- tests/helpers.py | 13 ++-- tests/integration/project_test.py | 21 +++++++ tests/integration/service_test.py | 82 ++++++++++++++++++++++++++ tests/integration/testcases.py | 6 +- tests/unit/config/config_test.py | 53 ++++++++++++++++- tests/unit/service_test.py | 4 +- 14 files changed, 309 insertions(+), 59 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 864bc7e90..9b4130536 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -35,6 +35,7 @@ from .interpolation import interpolate_environment_variables from .sort_services import get_container_name_from_network_mode from .sort_services import get_service_name_from_network_mode from .sort_services import sort_service_dicts +from .types import MountSpec from .types import parse_extra_hosts from .types import parse_restart_spec from .types import ServiceLink @@ -809,6 +810,20 @@ def process_healthcheck(service_dict): return service_dict +def finalize_service_volumes(service_dict, environment): + if 'volumes' in service_dict: + finalized_volumes = [] + normalize = environment.get_boolean('COMPOSE_CONVERT_WINDOWS_PATHS') + for v in service_dict['volumes']: + if isinstance(v, dict): + finalized_volumes.append(MountSpec.parse(v, normalize)) + else: + finalized_volumes.append(VolumeSpec.parse(v, normalize)) + service_dict['volumes'] = finalized_volumes + + return service_dict + + def finalize_service(service_config, service_names, version, environment): service_dict = dict(service_config.config) @@ -822,12 +837,7 @@ def finalize_service(service_config, service_names, version, environment): for vf in service_dict['volumes_from'] ] - if 'volumes' in service_dict: - service_dict['volumes'] = [ - VolumeSpec.parse( - v, environment.get_boolean('COMPOSE_CONVERT_WINDOWS_PATHS') - ) for v in service_dict['volumes'] - ] + service_dict = finalize_service_volumes(service_dict, environment) if 'net' in service_dict: network_mode = service_dict.pop('net') @@ -1143,19 +1153,13 @@ def resolve_volume_paths(working_dir, service_dict): def resolve_volume_path(working_dir, volume): - mount_params = None if isinstance(volume, dict): - container_path = volume.get('target') - host_path = volume.get('source') - mode = None - if host_path: - if volume.get('read_only'): - mode = 'ro' - if volume.get('volume', {}).get('nocopy'): - mode = 'nocopy' - mount_params = (host_path, mode) - else: - container_path, mount_params = split_path_mapping(volume) + if volume.get('source', '').startswith('.') and volume['type'] == 'mount': + volume['source'] = expand_path(working_dir, volume['source']) + return volume + + mount_params = None + container_path, mount_params = split_path_mapping(volume) if mount_params is not None: host_path, mode = mount_params diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index 6f923871b..d50df3e81 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -293,7 +293,39 @@ }, "user": {"type": "string"}, "userns_mode": {"type": "string"}, - "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "additionalProperties": false, + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + } + } + } + ], + "uniqueItems": true + } + }, "volume_driver": {"type": "string"}, "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "working_dir": {"type": "string"} diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 2b8c73f14..5e80e70e0 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -7,6 +7,7 @@ import yaml from compose.config import types from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 +from compose.const import COMPOSEFILE_V2_3 as V2_3 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_4 as V3_4 @@ -34,6 +35,7 @@ def serialize_string(dumper, data): return representer(data) +yaml.SafeDumper.add_representer(types.MountSpec, serialize_dict_type) yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type) @@ -141,5 +143,9 @@ def denormalize_service_dict(service_dict, version, image_digest=None): p.legacy_repr() if isinstance(p, types.ServicePort) else p for p in service_dict['ports'] ] + if 'volumes' in service_dict and (version < V2_3 or (version > V3_0 and version < V3_2)): + service_dict['volumes'] = [ + v.legacy_repr() if isinstance(v, types.MountSpec) else v for v in service_dict['volumes'] + ] return service_dict diff --git a/compose/config/types.py b/compose/config/types.py index d3b3cfc53..c134bd7ca 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -144,6 +144,15 @@ class MountSpec(object): } _fields = ['type', 'source', 'target', 'read_only', 'consistency'] + @classmethod + def parse(cls, mount_dict, normalize=False): + if mount_dict.get('source'): + mount_dict['source'] = os.path.normpath(mount_dict['source']) + if normalize: + mount_dict['source'] = normalize_path_for_engine(mount_dict['source']) + + return cls(**mount_dict) + def __init__(self, type, source=None, target=None, read_only=None, consistency=None, **kwargs): self.type = type self.source = source @@ -174,6 +183,10 @@ class MountSpec(object): def is_named_volume(self): return self.type == 'volume' and self.source + @property + def external(self): + return self.source + class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): diff --git a/compose/service.py b/compose/service.py index bfc2e5940..f51f0e5af 100644 --- a/compose/service.py +++ b/compose/service.py @@ -785,15 +785,23 @@ class Service(object): self.options.get('labels'), override_options.get('labels')) + container_volumes = [] + container_mounts = [] + if 'volumes' in container_options: + container_volumes = [ + v for v in container_options.get('volumes') if isinstance(v, VolumeSpec) + ] + container_mounts = [v for v in container_options.get('volumes') if isinstance(v, MountSpec)] + binds, affinity = merge_volume_bindings( - container_options.get('volumes') or [], - self.options.get('tmpfs') or [], - previous_container) + container_volumes, self.options.get('tmpfs') or [], previous_container, + container_mounts + ) override_options['binds'] = binds container_options['environment'].update(affinity) - container_options['volumes'] = dict( - (v.internal, {}) for v in container_options.get('volumes') or {}) + container_options['volumes'] = dict((v.internal, {}) for v in container_volumes or {}) + override_options['mounts'] = [build_mount(v) for v in container_mounts] or None secret_volumes = self.get_secret_volumes() if secret_volumes: @@ -803,7 +811,8 @@ class Service(object): (v.target, {}) for v in secret_volumes ) else: - override_options['mounts'] = [build_mount(v) for v in secret_volumes] + override_options['mounts'] = override_options.get('mounts') or [] + override_options['mounts'].extend([build_mount(v) for v in secret_volumes]) container_options['image'] = self.image_name @@ -1245,32 +1254,40 @@ def parse_repository_tag(repo_path): # Volumes -def merge_volume_bindings(volumes, tmpfs, previous_container): - """Return a list of volume bindings for a container. Container data volumes - are replaced by those from the previous container. +def merge_volume_bindings(volumes, tmpfs, previous_container, mounts): + """ + Return a list of volume bindings for a container. Container data volumes + are replaced by those from the previous container. + Anonymous mounts are updated in place. """ affinity = {} volume_bindings = dict( build_volume_binding(volume) for volume in volumes - if volume.external) + if volume.external + ) if previous_container: - old_volumes = get_container_data_volumes(previous_container, volumes, tmpfs) + old_volumes, old_mounts = get_container_data_volumes( + previous_container, volumes, tmpfs, mounts + ) warn_on_masked_volume(volumes, old_volumes, previous_container.service) volume_bindings.update( - build_volume_binding(volume) for volume in old_volumes) + build_volume_binding(volume) for volume in old_volumes + ) - if old_volumes: + if old_volumes or old_mounts: affinity = {'affinity:container': '=' + previous_container.id} return list(volume_bindings.values()), affinity -def get_container_data_volumes(container, volumes_option, tmpfs_option): - """Find the container data volumes that are in `volumes_option`, and return - a mapping of volume bindings for those volumes. +def get_container_data_volumes(container, volumes_option, tmpfs_option, mounts_option): + """ + Find the container data volumes that are in `volumes_option`, and return + a mapping of volume bindings for those volumes. + Anonymous volume mounts are updated in place instead. """ volumes = [] volumes_option = volumes_option or [] @@ -1309,7 +1326,19 @@ def get_container_data_volumes(container, volumes_option, tmpfs_option): volume = volume._replace(external=mount['Name']) volumes.append(volume) - return volumes + updated_mounts = False + for mount in mounts_option: + if mount.type != 'volume': + continue + + ctnr_mount = container_mounts.get(mount.target) + if not ctnr_mount.get('Name'): + continue + + mount.source = ctnr_mount['Name'] + updated_mounts = True + + return volumes, updated_mounts def warn_on_masked_volume(volumes_option, container_volumes, service): diff --git a/compose/utils.py b/compose/utils.py index 197ae6eb2..00b01df2e 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -101,7 +101,7 @@ def json_stream(stream): def json_hash(obj): - dump = json.dumps(obj, sort_keys=True, separators=(',', ':')) + dump = json.dumps(obj, sort_keys=True, separators=(',', ':'), default=lambda x: x.repr()) h = hashlib.sha256() h.update(dump.encode('utf8')) return h.hexdigest() diff --git a/compose/volume.py b/compose/volume.py index da8ba25ca..0b148620f 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -7,6 +7,7 @@ from docker.errors import NotFound from docker.utils import version_lt from .config import ConfigurationError +from .config.types import VolumeSpec from .const import LABEL_PROJECT from .const import LABEL_VOLUME @@ -145,5 +146,9 @@ class ProjectVolumes(object): if not volume_spec.is_named_volume: return volume_spec - volume = self.volumes[volume_spec.external] - return volume_spec._replace(external=volume.full_name) + if isinstance(volume_spec, VolumeSpec): + volume = self.volumes[volume_spec.external] + return volume_spec._replace(external=volume.full_name) + else: + volume_spec.source = self.volumes[volume_spec.source].full_name + return volume_spec diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 251e39db6..91e75abad 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -428,13 +428,21 @@ class CLITestCase(DockerClientTestCase): 'timeout': '1s', 'retries': 5, }, - 'volumes': [ - '/host/path:/container/path:ro', - 'foobar:/container/volumepath:rw', - '/anonymous', - 'foobar:/container/volumepath2:nocopy' - ], - + 'volumes': [{ + 'read_only': True, + 'source': '/host/path', + 'target': '/container/path', + 'type': 'bind' + }, { + 'source': 'foobar', 'target': '/container/volumepath', 'type': 'volume' + }, { + 'target': '/anonymous', 'type': 'volume' + }, { + 'source': 'foobar', + 'target': '/container/volumepath2', + 'type': 'volume', + 'volume': {'nocopy': True} + }], 'stop_grace_period': '20s', }, }, diff --git a/tests/helpers.py b/tests/helpers.py index a93de993f..f151f9cde 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -19,12 +19,8 @@ def build_config_details(contents, working_dir='working_dir', filename='filename ) -def create_host_file(client, filename): +def create_custom_host_file(client, filename, content): dirname = os.path.dirname(filename) - - with open(filename, 'r') as fh: - content = fh.read() - container = client.create_container( 'busybox:latest', ['sh', '-c', 'echo -n "{}" > {}'.format(content, filename)], @@ -48,3 +44,10 @@ def create_host_file(client, filename): return container_info['Node']['Name'] finally: client.remove_container(container, force=True) + + +def create_host_file(client, filename): + with open(filename, 'r') as fh: + content = fh.read() + + return create_custom_host_file(client, filename, content) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 953dd52be..6686d96cc 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -35,6 +35,7 @@ from tests.integration.testcases import is_cluster from tests.integration.testcases import no_cluster from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_2_only +from tests.integration.testcases import v2_3_only from tests.integration.testcases import v2_only from tests.integration.testcases import v3_only @@ -436,6 +437,26 @@ class ProjectTest(DockerClientTestCase): self.assertNotEqual(db_container.id, old_db_id) self.assertEqual(db_container.get('Volumes./etc'), db_volume_path) + @v2_3_only() + def test_recreate_preserves_mounts(self): + web = self.create_service('web') + db = self.create_service('db', volumes=[types.MountSpec(type='volume', target='/etc')]) + project = Project('composetest', [web, db], self.client) + project.start() + assert len(project.containers()) == 0 + + project.up(['db']) + assert len(project.containers()) == 1 + old_db_id = project.containers()[0].id + db_volume_path = project.containers()[0].get_mount('/etc')['Source'] + + project.up(strategy=ConvergenceStrategy.always) + assert len(project.containers()) == 2 + + db_container = [c for c in project.containers() if 'db' in c.name][0] + assert db_container.id != old_db_id + assert db_container.get_mount('/etc')['Source'] == db_volume_path + def test_project_up_with_no_recreate_running(self): web = self.create_service('web') db = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 00bacebf5..b9005b8e1 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -19,6 +19,7 @@ from .testcases import pull_busybox from .testcases import SWARM_SKIP_CONTAINERS_ALL from .testcases import SWARM_SKIP_CPU_SHARES from compose import __version__ +from compose.config.types import MountSpec from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM @@ -37,6 +38,7 @@ from compose.service import NetworkMode from compose.service import PidMode from compose.service import Service from compose.utils import parse_nanoseconds_int +from tests.helpers import create_custom_host_file from tests.integration.testcases import is_cluster from tests.integration.testcases import no_cluster from tests.integration.testcases import v2_1_only @@ -276,6 +278,54 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) + @v2_3_only() + def test_create_container_with_host_mount(self): + host_path = '/tmp/host-path' + container_path = '/container-path' + + create_custom_host_file(self.client, path.join(host_path, 'a.txt'), 'test') + + service = self.create_service( + 'db', + volumes=[ + MountSpec(type='bind', source=host_path, target=container_path, read_only=True) + ] + ) + container = service.create_container() + service.start_container(container) + mount = container.get_mount(container_path) + assert mount + assert path.basename(mount['Source']) == path.basename(host_path) + assert mount['RW'] is False + + @v2_3_only() + def test_create_container_with_tmpfs_mount(self): + container_path = '/container-tmpfs' + service = self.create_service( + 'db', + volumes=[MountSpec(type='tmpfs', target=container_path)] + ) + container = service.create_container() + service.start_container(container) + mount = container.get_mount(container_path) + assert mount + assert mount['Type'] == 'tmpfs' + + @v2_3_only() + def test_create_container_with_volume_mount(self): + container_path = '/container-volume' + volume_name = 'composetest_abcde' + self.client.create_volume(volume_name) + service = self.create_service( + 'db', + volumes=[MountSpec(type='volume', source=volume_name, target=container_path)] + ) + container = service.create_container() + service.start_container(container) + mount = container.get_mount(container_path) + assert mount + assert mount['Name'] == volume_name + def test_create_container_with_healthcheck_config(self): one_second = parse_nanoseconds_int('1s') healthcheck = { @@ -439,6 +489,38 @@ class ServiceTest(DockerClientTestCase): orig_container = new_container + @v2_3_only() + def test_execute_convergence_plan_recreate_twice_with_mount(self): + service = self.create_service( + 'db', + volumes=[MountSpec(target='/etc', type='volume')], + entrypoint=['top'], + command=['-d', '1'] + ) + + orig_container = service.create_container() + service.start_container(orig_container) + + orig_container.inspect() # reload volume data + volume_path = orig_container.get_mount('/etc')['Source'] + + # Do this twice to reproduce the bug + for _ in range(2): + new_container, = service.execute_convergence_plan( + ConvergencePlan('recreate', [orig_container]) + ) + + assert new_container.get_mount('/etc')['Source'] == volume_path + if not is_cluster(self.client): + assert ('affinity:container==%s' % orig_container.id in + new_container.get('Config.Env')) + else: + # In Swarm, the env marker is consumed and the container should be deployed + # on the same node. + assert orig_container.get('Node.Name') == new_container.get('Node.Name') + + orig_container = new_container + def test_execute_convergence_plan_when_containers_are_stopped(self): service = self.create_service( 'db', diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 8435f97dd..5505df1b4 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -20,7 +20,7 @@ from compose.const import COMPOSEFILE_V2_2 as V2_2 from compose.const import COMPOSEFILE_V2_3 as V2_3 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 -from compose.const import COMPOSEFILE_V3_3 as V3_3 +from compose.const import COMPOSEFILE_V3_5 as V3_5 from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service @@ -47,7 +47,7 @@ def get_links(container): def engine_max_version(): if 'DOCKER_VERSION' not in os.environ: - return V3_3 + return V3_5 version = os.environ['DOCKER_VERSION'].partition('-')[0] if version_lt(version, '1.10'): return V1 @@ -57,7 +57,7 @@ def engine_max_version(): return V2_1 if version_lt(version, '17.06'): return V3_2 - return V3_3 + return V3_5 def min_version_skip(version): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 00ba6c2c6..d519deb90 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1137,9 +1137,12 @@ class ConfigTest(unittest.TestCase): details = config.ConfigDetails('.', [base_file, override_file]) service_dicts = config.load(details).services svc_volumes = map(lambda v: v.repr(), service_dicts[0]['volumes']) - assert sorted(svc_volumes) == sorted( - ['/anonymous', '/c:/b:rw', 'vol:/x:ro'] - ) + for vol in svc_volumes: + assert vol in [ + '/anonymous', + '/c:/b:rw', + {'source': 'vol', 'target': '/x', 'type': 'volume', 'read_only': True} + ] @mock.patch.dict(os.environ) def test_volume_mode_override(self): @@ -1223,6 +1226,50 @@ class ConfigTest(unittest.TestCase): assert volume.external == 'data0028' assert volume.is_named_volume + def test_volumes_long_syntax(self): + base_file = config.ConfigFile( + 'base.yaml', { + 'version': '2.3', + 'services': { + 'web': { + 'image': 'busybox:latest', + 'volumes': [ + { + 'target': '/anonymous', 'type': 'volume' + }, { + 'source': '/abc', 'target': '/xyz', 'type': 'bind' + }, { + 'source': '\\\\.\\pipe\\abcd', 'target': '/named_pipe', 'type': 'npipe' + }, { + 'type': 'tmpfs', 'target': '/tmpfs' + } + ] + }, + }, + }, + ) + details = config.ConfigDetails('.', [base_file]) + config_data = config.load(details) + volumes = config_data.services[0].get('volumes') + anon_volume = [v for v in volumes if v.target == '/anonymous'][0] + tmpfs_mount = [v for v in volumes if v.type == 'tmpfs'][0] + host_mount = [v for v in volumes if v.type == 'bind'][0] + npipe_mount = [v for v in volumes if v.type == 'npipe'][0] + + assert anon_volume.type == 'volume' + assert not anon_volume.is_named_volume + + assert tmpfs_mount.target == '/tmpfs' + assert not tmpfs_mount.is_named_volume + + assert host_mount.source == os.path.normpath('/abc') + assert host_mount.target == '/xyz' + assert not host_mount.is_named_volume + + assert npipe_mount.source == '\\\\.\\pipe\\abcd' + assert npipe_mount.target == '/named_pipe' + assert not npipe_mount.is_named_volume + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 16670cff5..24ed60e94 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -939,7 +939,7 @@ class ServiceVolumesTest(unittest.TestCase): VolumeSpec.parse('imagedata:/mnt/image/data:rw'), ] - volumes = get_container_data_volumes(container, options, ['/dev/tmpfs']) + volumes, _ = get_container_data_volumes(container, options, ['/dev/tmpfs'], []) assert sorted(volumes) == sorted(expected) def test_merge_volume_bindings(self): @@ -975,7 +975,7 @@ class ServiceVolumesTest(unittest.TestCase): 'existingvolume:/existing/volume:rw', ] - binds, affinity = merge_volume_bindings(options, ['/dev/tmpfs'], previous_container) + binds, affinity = merge_volume_bindings(options, ['/dev/tmpfs'], previous_container, []) assert sorted(binds) == sorted(expected) assert affinity == {'affinity:container': '=cdefab'} From 99e9e32d7ebc2da54c4f1634560fbf344ce5b68d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 6 Dec 2017 16:48:14 -0800 Subject: [PATCH 46/48] Add support for custom names for networks, secrets, configs Finalize v3.5 schema Signed-off-by: Joffrey F --- compose/config/config.py | 5 +- compose/config/config_schema_v2.1.json | 3 +- compose/config/config_schema_v2.2.json | 3 +- compose/config/config_schema_v2.3.json | 3 +- compose/config/config_schema_v3.5.json | 57 ++++++++++++++----- compose/config/serialize.py | 10 +++- compose/config/types.py | 5 +- compose/network.py | 23 ++++---- compose/project.py | 2 +- docker-compose.spec | 5 ++ tests/acceptance/cli_test.py | 16 ++++++ .../networks/external-networks-v3-5.yml | 17 ++++++ tests/integration/project_test.py | 37 ++++++++++++ tests/unit/config/config_test.py | 54 ++++++++++++++---- 14 files changed, 196 insertions(+), 44 deletions(-) create mode 100644 tests/fixtures/networks/external-networks-v3-5.yml diff --git a/compose/config/config.py b/compose/config/config.py index 9b4130536..98719d6ba 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -410,12 +410,11 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): external = config.get('external') if external: - name_field = 'name' if entity_type == 'Volume' else 'external_name' validate_external(entity_type, name, config, config_file.version) if isinstance(external, dict): - config[name_field] = external.get('name') + config['name'] = external.get('name') elif not config.get('name'): - config[name_field] = name + config['name'] = name if 'driver_opts' in config: config['driver_opts'] = build_string_dict( diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 6b74f0ed6..15b78e5db 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -350,7 +350,8 @@ }, "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 21343b893..7a3eed0a9 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -357,7 +357,8 @@ }, "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index d50df3e81..7c0e54807 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -393,7 +393,8 @@ }, "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json index 1e65b2087..565da0193 100644 --- a/compose/config/config_schema_v3.5.json +++ b/compose/config/config_schema_v3.5.json @@ -155,6 +155,7 @@ "hostname": {"type": "string"}, "image": {"type": "string"}, "ipc": {"type": "string"}, + "isolation": {"type": "string"}, "labels": {"$ref": "#/definitions/list_or_dict"}, "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, @@ -282,7 +283,6 @@ { "type": "object", "required": ["type"], - "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, @@ -301,7 +301,8 @@ "nocopy": {"type": "boolean"} } } - } + }, + "additionalProperties": false } ], "uniqueItems": true @@ -318,7 +319,7 @@ "additionalProperties": false, "properties": { "disable": {"type": "boolean"}, - "interval": {"type": "string"}, + "interval": {"type": "string", "format": "duration"}, "retries": {"type": "number"}, "test": { "oneOf": [ @@ -326,7 +327,8 @@ {"type": "array", "items": {"type": "string"}} ] }, - "timeout": {"type": "string"} + "timeout": {"type": "string", "format": "duration"}, + "start_period": {"type": "string", "format": "duration"} } }, "deployment": { @@ -354,8 +356,23 @@ "resources": { "type": "object", "properties": { - "limits": {"$ref": "#/definitions/resource"}, - "reservations": {"$ref": "#/definitions/resource"} + "limits": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + "reservations": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"}, + "generic_resources": {"$ref": "#/definitions/generic_resources"} + }, + "additionalProperties": false + } }, "additionalProperties": false }, @@ -390,20 +407,30 @@ "additionalProperties": false }, - "resource": { - "id": "#/definitions/resource", - "type": "object", - "properties": { - "cpus": {"type": "string"}, - "memory": {"type": "string"} - }, - "additionalProperties": false + "generic_resources": { + "id": "#/definitions/generic_resources", + "type": "array", + "items": { + "type": "object", + "properties": { + "discrete_resource_spec": { + "type": "object", + "properties": { + "kind": {"type": "string"}, + "value": {"type": "number"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } }, "network": { "id": "#/definitions/network", "type": ["object", "null"], "properties": { + "name": {"type": "string"}, "driver": {"type": "string"}, "driver_opts": { "type": "object", @@ -470,6 +497,7 @@ "id": "#/definitions/secret", "type": "object", "properties": { + "name": {"type": "string"}, "file": {"type": "string"}, "external": { "type": ["boolean", "object"], @@ -486,6 +514,7 @@ "id": "#/definitions/config", "type": "object", "properties": { + "name": {"type": "string"}, "file": {"type": "string"}, "external": { "type": ["boolean", "object"], diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 5e80e70e0..3ab43fc59 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -11,6 +11,7 @@ from compose.const import COMPOSEFILE_V2_3 as V2_3 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_4 as V3_4 +from compose.const import COMPOSEFILE_V3_5 as V3_5 def serialize_config_type(dumper, data): @@ -69,7 +70,8 @@ def denormalize_config(config, image_digests=None): del conf['external_name'] if 'name' in conf: - if config.version < V2_1 or (config.version >= V3_0 and config.version < V3_4): + if config.version < V2_1 or ( + config.version >= V3_0 and config.version < v3_introduced_name_key(key)): del conf['name'] elif 'external' in conf: conf['external'] = True @@ -77,6 +79,12 @@ def denormalize_config(config, image_digests=None): return result +def v3_introduced_name_key(key): + if key == 'volumes': + return V3_4 + return V3_5 + + def serialize_config(config, image_digests=None): return yaml.safe_dump( denormalize_config(config, image_digests), diff --git a/compose/config/types.py b/compose/config/types.py index c134bd7ca..daf25f700 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -293,17 +293,18 @@ class ServiceLink(namedtuple('_ServiceLink', 'target alias')): return self.alias -class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode')): +class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode name')): @classmethod def parse(cls, spec): if isinstance(spec, six.string_types): - return cls(spec, None, None, None, None) + return cls(spec, None, None, None, None, None) return cls( spec.get('source'), spec.get('target'), spec.get('uid'), spec.get('gid'), spec.get('mode'), + spec.get('name') ) @property diff --git a/compose/network.py b/compose/network.py index ee5939c15..95e2bf60e 100644 --- a/compose/network.py +++ b/compose/network.py @@ -25,21 +25,22 @@ OPTS_EXCEPTIONS = [ class Network(object): def __init__(self, client, project, name, driver=None, driver_opts=None, - ipam=None, external_name=None, internal=False, enable_ipv6=False, - labels=None): + ipam=None, external=False, internal=False, enable_ipv6=False, + labels=None, custom_name=False): self.client = client self.project = project self.name = name self.driver = driver self.driver_opts = driver_opts self.ipam = create_ipam_config_from_dict(ipam) - self.external_name = external_name + self.external = external self.internal = internal self.enable_ipv6 = enable_ipv6 self.labels = labels + self.custom_name = custom_name def ensure(self): - if self.external_name: + if self.external: try: self.inspect() log.debug( @@ -51,7 +52,7 @@ class Network(object): 'Network {name} declared as external, but could' ' not be found. Please create the network manually' ' using `{command} {name}` and try again.'.format( - name=self.external_name, + name=self.full_name, command='docker network create' ) ) @@ -83,7 +84,7 @@ class Network(object): ) def remove(self): - if self.external_name: + if self.external: log.info("Network %s is external, skipping", self.full_name) return @@ -95,8 +96,8 @@ class Network(object): @property def full_name(self): - if self.external_name: - return self.external_name + if self.custom_name: + return self.name return '{0}_{1}'.format(self.project, self.name) @property @@ -203,14 +204,16 @@ def build_networks(name, config_data, client): network_config = config_data.networks or {} networks = { network_name: Network( - client=client, project=name, name=network_name, + client=client, project=name, + name=data.get('name', network_name), driver=data.get('driver'), driver_opts=data.get('driver_opts'), ipam=data.get('ipam'), - external_name=data.get('external_name'), + external=bool(data.get('external', False)), internal=data.get('internal'), enable_ipv6=data.get('enable_ipv6'), labels=data.get('labels'), + custom_name=data.get('name') is not None, ) for network_name, data in network_config.items() } diff --git a/compose/project.py b/compose/project.py index 411576386..11ee4a0b7 100644 --- a/compose/project.py +++ b/compose/project.py @@ -648,7 +648,7 @@ def get_secrets(service, service_secrets, secret_defs): "Service \"{service}\" uses an undefined secret \"{secret}\" " .format(service=service, secret=secret.source)) - if secret_def.get('external_name'): + if secret_def.get('external'): log.warn("Service \"{service}\" uses secret \"{secret}\" which is external. " "External secrets are not available to containers created by " "docker-compose.".format(service=service, secret=secret.source)) diff --git a/docker-compose.spec b/docker-compose.spec index 9c46421f0..83d7389f3 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -67,6 +67,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.4.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.5.json', + 'compose/config/config_schema_v3.5.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 91e75abad..3225eb49b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -350,6 +350,22 @@ class CLITestCase(DockerClientTestCase): } } + def test_config_external_network_v3_5(self): + self.base_dir = 'tests/fixtures/networks' + result = self.dispatch(['-f', 'external-networks-v3-5.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'networks' in json_result + assert json_result['networks'] == { + 'foo': { + 'external': True, + 'name': 'some_foo', + }, + 'bar': { + 'external': True, + 'name': 'some_bar', + }, + } + def test_config_v1(self): self.base_dir = 'tests/fixtures/v1-config' result = self.dispatch(['config']) diff --git a/tests/fixtures/networks/external-networks-v3-5.yml b/tests/fixtures/networks/external-networks-v3-5.yml new file mode 100644 index 000000000..9ac7b14b5 --- /dev/null +++ b/tests/fixtures/networks/external-networks-v3-5.yml @@ -0,0 +1,17 @@ +version: "3.5" + +services: + web: + image: busybox + command: top + networks: + - foo + - bar + +networks: + foo: + external: true + name: some_foo + bar: + external: + name: some_bar diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 6686d96cc..82e0adab3 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -953,6 +953,43 @@ class ProjectTest(DockerClientTestCase): assert 'LinkLocalIPs' in ipam_config assert ipam_config['LinkLocalIPs'] == ['169.254.8.8'] + @v2_1_only() + def test_up_with_custom_name_resources(self): + config_data = build_config( + version=V2_2, + services=[{ + 'name': 'web', + 'volumes': [VolumeSpec.parse('foo:/container-path')], + 'networks': {'foo': {}}, + 'image': 'busybox:latest' + }], + networks={ + 'foo': { + 'name': 'zztop', + 'labels': {'com.docker.compose.test_value': 'sharpdressedman'} + } + }, + volumes={ + 'foo': { + 'name': 'acdc', + 'labels': {'com.docker.compose.test_value': 'thefuror'} + } + } + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data + ) + + project.up(detached=True) + network = [n for n in self.client.networks() if n['Name'] == 'zztop'][0] + volume = [v for v in self.client.volumes()['Volumes'] if v['Name'] == 'acdc'][0] + + assert network['Labels']['com.docker.compose.test_value'] == 'sharpdressedman' + assert volume['Labels']['com.docker.compose.test_value'] == 'thefuror' + @v2_1_only() def test_up_with_isolation(self): self.require_api_version('1.24') diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d519deb90..7029fcb08 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -432,6 +432,40 @@ class ConfigTest(unittest.TestCase): 'label_key': 'label_val' } + def test_load_config_custom_resource_names(self): + base_file = config.ConfigFile( + 'base.yaml', { + 'version': '3.5', + 'volumes': { + 'abc': { + 'name': 'xyz' + } + }, + 'networks': { + 'abc': { + 'name': 'xyz' + } + }, + 'secrets': { + 'abc': { + 'name': 'xyz' + } + }, + 'configs': { + 'abc': { + 'name': 'xyz' + } + } + } + ) + details = config.ConfigDetails('.', [base_file]) + loaded_config = config.load(details) + + assert loaded_config.networks['abc'] == {'name': 'xyz'} + assert loaded_config.volumes['abc'] == {'name': 'xyz'} + assert loaded_config.secrets['abc']['name'] == 'xyz' + assert loaded_config.configs['abc']['name'] == 'xyz' + def test_load_config_volume_and_network_labels(self): base_file = config.ConfigFile( 'base.yaml', @@ -2539,8 +2573,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'secrets': [ - types.ServiceSecret('one', None, None, None, None), - types.ServiceSecret('source', 'target', '100', '200', 0o777), + types.ServiceSecret('one', None, None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -2586,8 +2620,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'secrets': [ - types.ServiceSecret('one', None, None, None, None), - types.ServiceSecret('source', 'target', '100', '200', 0o777), + types.ServiceSecret('one', None, None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -2624,8 +2658,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'configs': [ - types.ServiceConfig('one', None, None, None, None), - types.ServiceConfig('source', 'target', '100', '200', 0o777), + types.ServiceConfig('one', None, None, None, None, None), + types.ServiceConfig('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -2671,8 +2705,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'configs': [ - types.ServiceConfig('one', None, None, None, None), - types.ServiceConfig('source', 'target', '100', '200', 0o777), + types.ServiceConfig('one', None, None, None, None, None), + types.ServiceConfig('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -3131,7 +3165,7 @@ class InterpolationTest(unittest.TestCase): assert config_dict.secrets == { 'secretdata': { 'external': {'name': 'baz.bar'}, - 'external_name': 'baz.bar' + 'name': 'baz.bar' } } @@ -3149,7 +3183,7 @@ class InterpolationTest(unittest.TestCase): assert config_dict.configs == { 'configdata': { 'external': {'name': 'baz.bar'}, - 'external_name': 'baz.bar' + 'name': 'baz.bar' } } From 29c02ef598d1888fb389fa959ed2317afb40cc4f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 6 Dec 2017 17:39:38 -0800 Subject: [PATCH 47/48] Fix bad rebase Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 18 ------------------ tests/integration/testcases.py | 2 -- 2 files changed, 20 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 3225eb49b..c4905f909 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -536,24 +536,6 @@ class CLITestCase(DockerClientTestCase): assert self.dispatch(['pull', '--quiet']).stderr == '' assert self.dispatch(['pull', '--quiet']).stdout == '' - def test_pull_with_quiet(self): - assert self.dispatch(['pull', '--quiet']).stderr == '' - assert self.dispatch(['pull', '--quiet']).stdout == '' - - def test_pull_with_parallel_failure(self): - result = self.dispatch([ - '-f', 'ignore-pull-failures.yml', 'pull', '--parallel'], - returncode=1 - ) - - self.assertRegexpMatches(result.stderr, re.compile('^Pulling simple', re.MULTILINE)) - self.assertRegexpMatches(result.stderr, re.compile('^Pulling another', re.MULTILINE)) - self.assertRegexpMatches(result.stderr, - re.compile('^ERROR: for another .*does not exist.*', re.MULTILINE)) - self.assertRegexpMatches(result.stderr, - re.compile('''^(ERROR: )?(b')?.* nonexisting-image''', - re.MULTILINE)) - def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 5505df1b4..9427f3d0d 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -75,7 +75,6 @@ def v2_1_only(): return min_version_skip(V2_1) - def v2_2_only(): return min_version_skip(V2_2) @@ -84,7 +83,6 @@ def v2_3_only(): return min_version_skip(V2_3) - def v3_only(): return min_version_skip(V3_0) From e96dfbac2a7982b8703abd3774ab661516096931 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 6 Dec 2017 17:25:37 -0800 Subject: [PATCH 48/48] Bump 1.18.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 80 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0be7ea76..ba91a505b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,86 @@ Change log ========== +1.18.0 (2017-12-15) +------------------- + +### New features + +#### Compose file version 3.5 + +- Introduced version 3.5 of the `docker-compose.yml` specification. + This version requires to be used with Docker Engine 17.06.0 or above + +- Added support for the `shm_size` parameter in build configurations + +- Added support for the `isolation` parameter in service definitions + +- Added support for custom names for network, secret and config definitions + +#### Compose file version 2.3 + +- Added support for `extra_hosts` in build configuration + +- Added support for the + [long syntax](https://docs.docker.com/compose/compose-file/#long-syntax-3) + for volume entries, as previously introduced in the 3.2 format. + Note that using this syntax will create + [mounts](https://docs.docker.com/engine/admin/volumes/bind-mounts/) + instead of volumes. + +#### Compose file version 2.1 and up + +- Added support for the `oom_kill_disable` parameter in service definitions + (2.x only) + +- Added support for custom names for network, secret and config definitions + (2.x only) + + +#### All formats + +- Values interpolated from the environment will now be converted to the + proper type when used in non-string fields. + +- Added support for `--labels` in `docker-compose run` + +- Added support for `--timeout` in `docker-compose down` + +- Added support for `--memory` in `docker-compose build` + +- Setting `stop_grace_period` in service definitions now also sets the + container's `stop_timeout` + +### Bugfixes + +- Fixed an issue where Compose was still handling service hostname according + to legacy engine behavior, causing hostnames containing dots to be cut up + +- Fixed a bug where the `X-Y:Z` syntax for ports was considered invalid + by Compose + +- Fixed an issue with CLI logging causing duplicate messages and inelegant + output to occur + +- Fixed a bug where the valid `${VAR:-}` syntax would cause Compose to + error out + +- Fixed a bug where `env_file` entries using an UTF-8 BOM were being read + incorrectly + +- Fixed a bug where missing secret files would generate an empty directory + in their place + +- Added validation for the `test` field in healthchecks + +- Added validation for the `subnet` field in IPAM configurations + +- Added validation for `volumes` properties when using the long syntax in + service definitions + +- The CLI now explicit prevents using `-d` and `--timeout` together + in `docker-compose up` + 1.17.1 (2017-11-08) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index 7b954eb4f..2b363f3be 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.18.0dev' +__version__ = '1.18.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 58483196d..441c0d806 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.17.1" +VERSION="1.18.0-rc1" IMAGE="docker/compose:$VERSION"