From 790280ba7251da22ad0e3a7aa1ca4222376b963f Mon Sep 17 00:00:00 2001 From: Mihai Date: Thu, 5 May 2016 13:39:12 +0300 Subject: [PATCH 01/92] Ignore error output of `stty size` when stdin is not a terminal. Fixes #1876 Signed-off-by: Mihai Ciumeica --- compose/cli/formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/formatter.py b/compose/cli/formatter.py index d0ed0f87e..0fa51d06a 100644 --- a/compose/cli/formatter.py +++ b/compose/cli/formatter.py @@ -10,7 +10,7 @@ from compose.cli import colors def get_tty_width(): - tty_size = os.popen('stty size', 'r').read().split() + tty_size = os.popen('stty size 2> /dev/null', 'r').read().split() if len(tty_size) != 2: return 0 _, width = tty_size From bc5246366fcd3842716abe4f7afc9ef8822730d0 Mon Sep 17 00:00:00 2001 From: Daniil Guzanov Date: Mon, 30 May 2016 12:21:25 +0300 Subject: [PATCH 02/92] Add issue link for zero-downtime deploys Signed-off-by: Daniil Guzanov --- ROADMAP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ROADMAP.md b/ROADMAP.md index 287e54680..c2184e56a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -16,7 +16,7 @@ Some specific things we are considering: - It should roll back to a known good state if it fails. - It should allow a user to check the actions it is about to perform before running them. - It should be possible to partially modify the config file for different environments (dev/test/staging/prod), passing in e.g. custom ports, volume mount paths, or volume drivers. ([#1377](https://github.com/docker/compose/issues/1377)) -- Compose should recommend a technique for zero-downtime deploys. +- Compose should recommend a technique for zero-downtime deploys. ([#1786](https://github.com/docker/compose/issues/1786)) - It should be possible to continuously attempt to keep an application in the correct state, instead of just performing `up` a single time. ## Integration with Swarm From 11a2eab549f426904614cd93c34a03a6608235ee Mon Sep 17 00:00:00 2001 From: Milind Shakya Date: Thu, 2 Jun 2016 16:27:47 -0700 Subject: [PATCH 03/92] Replace assertEquals with assertEqual since the former is getting deprecated soon. Also, fix pep8 E309. Signed-off-by: Milind Shakya --- tests/acceptance/cli_test.py | 8 ++++---- tests/integration/service_test.py | 4 +++- tests/unit/cli_test.py | 14 +++++++------- tests/unit/config/config_test.py | 28 ++++++++++++++++++++-------- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index dfd75625c..568b8efc2 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -842,8 +842,8 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(self.project.containers()), 1) stdout, stderr = self.dispatch(['exec', '-T', 'console', 'ls', '-1d', '/']) - self.assertEquals(stdout, "/\n") - self.assertEquals(stderr, "") + self.assertEqual(stdout, "/\n") + self.assertEqual(stderr, "") def test_exec_custom_user(self): self.base_dir = 'tests/fixtures/links-composefile' @@ -851,8 +851,8 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(self.project.containers()), 1) stdout, stderr = self.dispatch(['exec', '-T', '--user=operator', 'console', 'whoami']) - self.assertEquals(stdout, "operator\n") - self.assertEquals(stderr, "") + self.assertEqual(stdout, "operator\n") + self.assertEqual(stderr, "") def test_run_service_without_links(self): self.base_dir = 'tests/fixtures/links-composefile' diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index df50d513a..38a8cc2b8 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -39,6 +39,7 @@ def create_and_start_container(service, **override_options): class ServiceTest(DockerClientTestCase): + def test_containers(self): foo = self.create_service('foo') bar = self.create_service('bar') @@ -940,7 +941,7 @@ class ServiceTest(DockerClientTestCase): with mock.patch.object(self.client, '_version', '1.20'): service = self.create_service('web') service_config = service._get_container_host_config({}) - self.assertEquals(service_config['NetworkMode'], 'default') + self.assertEqual(service_config['NetworkMode'], 'default') def test_labels(self): labels_dict = { @@ -1044,6 +1045,7 @@ def converge(service, strategy=ConvergenceStrategy.changed): class ConfigHashTest(DockerClientTestCase): + def test_no_config_hash_when_one_off(self): web = self.create_service('web') container = web.create_container(one_off=True) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 2c90b29b7..25afdbcc1 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -29,36 +29,36 @@ class CLITestCase(unittest.TestCase): test_dir = py._path.local.LocalPath('tests/fixtures/simple-composefile') with test_dir.as_cwd(): project_name = get_project_name('.') - self.assertEquals('simplecomposefile', project_name) + self.assertEqual('simplecomposefile', project_name) def test_project_name_with_explicit_base_dir(self): base_dir = 'tests/fixtures/simple-composefile' project_name = get_project_name(base_dir) - self.assertEquals('simplecomposefile', project_name) + self.assertEqual('simplecomposefile', project_name) def test_project_name_with_explicit_uppercase_base_dir(self): base_dir = 'tests/fixtures/UpperCaseDir' project_name = get_project_name(base_dir) - self.assertEquals('uppercasedir', project_name) + self.assertEqual('uppercasedir', project_name) def test_project_name_with_explicit_project_name(self): name = 'explicit-project-name' project_name = get_project_name(None, project_name=name) - self.assertEquals('explicitprojectname', project_name) + self.assertEqual('explicitprojectname', project_name) @mock.patch.dict(os.environ) def test_project_name_from_environment_new_var(self): name = 'namefromenv' os.environ['COMPOSE_PROJECT_NAME'] = name project_name = get_project_name(None) - self.assertEquals(project_name, name) + self.assertEqual(project_name, name) def test_project_name_with_empty_environment_var(self): base_dir = 'tests/fixtures/simple-composefile' with mock.patch.dict(os.environ): os.environ['COMPOSE_PROJECT_NAME'] = '' project_name = get_project_name(base_dir) - self.assertEquals('simplecomposefile', project_name) + self.assertEqual('simplecomposefile', project_name) @mock.patch.dict(os.environ) def test_project_name_with_environment_file(self): @@ -158,7 +158,7 @@ class CLITestCase(unittest.TestCase): '--workdir': None, }) - self.assertEquals( + self.assertEqual( mock_client.create_host_config.call_args[1]['restart_policy']['Name'], 'always' ) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 24ece4994..49b109fdf 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -48,6 +48,7 @@ def service_sort(services): class ConfigTest(unittest.TestCase): + def test_load(self): service_dicts = config.load( build_config_details( @@ -1384,6 +1385,7 @@ class ConfigTest(unittest.TestCase): class NetworkModeTest(unittest.TestCase): + def test_network_mode_standard(self): config_data = config.load(build_config_details({ 'version': '2', @@ -1595,6 +1597,7 @@ class PortsTest(unittest.TestCase): class InterpolationTest(unittest.TestCase): + @mock.patch.dict(os.environ) def test_config_file_with_environment_file(self): project_dir = 'tests/fixtures/default-env-file' @@ -1692,10 +1695,11 @@ class InterpolationTest(unittest.TestCase): None, ) ).services[0] - self.assertEquals(service_dict['environment']['POSTGRES_PASSWORD'], '') + self.assertEqual(service_dict['environment']['POSTGRES_PASSWORD'], '') class VolumeConfigTest(unittest.TestCase): + def test_no_binding(self): d = make_service_dict('foo', {'build': '.', 'volumes': ['/data']}, working_dir='.') self.assertEqual(d['volumes'], ['/data']) @@ -1840,6 +1844,7 @@ class MergeDevicesTest(unittest.TestCase, MergePathMappingTest): class BuildOrImageMergeTest(unittest.TestCase): + def test_merge_build_or_image_no_override(self): self.assertEqual( config.merge_service_dicts({'build': '.'}, {}, V1), @@ -1928,6 +1933,7 @@ class MergeNetworksTest(unittest.TestCase, MergeListsTest): class MergeStringsOrListsTest(unittest.TestCase): + def test_no_override(self): service_dict = config.merge_service_dicts( {'dns': '8.8.8.8'}, @@ -1958,6 +1964,7 @@ class MergeStringsOrListsTest(unittest.TestCase): class MergeLabelsTest(unittest.TestCase): + def test_empty(self): assert 'labels' not in config.merge_service_dicts({}, {}, DEFAULT_VERSION) @@ -1998,6 +2005,7 @@ class MergeLabelsTest(unittest.TestCase): class MemoryOptionsTest(unittest.TestCase): + def test_validation_fails_with_just_memswap_limit(self): """ When you set a 'memswap_limit' it is invalid config unless you also set @@ -2040,6 +2048,7 @@ class MemoryOptionsTest(unittest.TestCase): class EnvTest(unittest.TestCase): + def test_parse_environment_as_list(self): environment = [ 'NORMAL=F1', @@ -2185,6 +2194,7 @@ def load_from_filename(filename): class ExtendsTest(unittest.TestCase): + def test_extends(self): service_dicts = load_from_filename('tests/fixtures/extends/docker-compose.yml') @@ -2376,9 +2386,9 @@ class ExtendsTest(unittest.TestCase): ) ).services - self.assertEquals(len(service), 1) + self.assertEqual(len(service), 1) self.assertIsInstance(service[0], dict) - self.assertEquals(service[0]['command'], "/bin/true") + self.assertEqual(service[0]['command'], "/bin/true") def test_extended_service_with_invalid_config(self): with pytest.raises(ConfigurationError) as exc: @@ -2390,7 +2400,7 @@ class ExtendsTest(unittest.TestCase): def test_extended_service_with_valid_config(self): service = load_from_filename('tests/fixtures/extends/service-with-valid-composite-extends.yml') - self.assertEquals(service[0]['command'], "top") + self.assertEqual(service[0]['command'], "top") def test_extends_file_defaults_to_self(self): """ @@ -2622,7 +2632,7 @@ class ExtendsTest(unittest.TestCase): """) service = load_from_filename(str(tmpdir.join('docker-compose.yml'))) - self.assertEquals(service[0]['command'], "top") + self.assertEqual(service[0]['command'], "top") def test_extends_with_depends_on(self): tmpdir = py.test.ensuretemp('test_extends_with_defined_version') @@ -2666,6 +2676,7 @@ class ExpandPathTest(unittest.TestCase): class VolumePathTest(unittest.TestCase): + def test_split_path_mapping_with_windows_path(self): host_path = "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config" windows_volume_path = host_path + ":/opt/connect/config:ro" @@ -2685,6 +2696,7 @@ class VolumePathTest(unittest.TestCase): @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class BuildPathTest(unittest.TestCase): + def setUp(self): self.abs_context_path = os.path.join(os.getcwd(), 'tests/fixtures/build-ctx') @@ -2707,7 +2719,7 @@ class BuildPathTest(unittest.TestCase): {'build': relative_build_path}, working_dir='tests/fixtures/build-path' ) - self.assertEquals(service_dict['build'], self.abs_context_path) + self.assertEqual(service_dict['build'], self.abs_context_path) def test_absolute_path(self): service_dict = make_service_dict( @@ -2715,11 +2727,11 @@ class BuildPathTest(unittest.TestCase): {'build': self.abs_context_path}, working_dir='tests/fixtures/build-path' ) - self.assertEquals(service_dict['build'], self.abs_context_path) + self.assertEqual(service_dict['build'], self.abs_context_path) def test_from_file(self): service_dict = load_from_filename('tests/fixtures/build-path/docker-compose.yml') - self.assertEquals(service_dict, [{'name': 'foo', 'build': {'context': self.abs_context_path}}]) + self.assertEqual(service_dict, [{'name': 'foo', 'build': {'context': self.abs_context_path}}]) def test_valid_url_in_build_path(self): valid_urls = [ From 958758ff6d6e8fddc45c51a92f11ef8893f29a1d Mon Sep 17 00:00:00 2001 From: Nikola Kovacs Date: Fri, 22 Jul 2016 17:42:42 +0200 Subject: [PATCH 04/92] Remove anonymous volumes when using run --rm. Named volumes will not be removed. This is consistent with the behavior of docker run --rm. Fixes #2419, #3611 Signed-off-by: Nikola Kovacs --- compose/cli/main.py | 2 +- tests/acceptance/cli_test.py | 30 ++++++++++++++++++++++++ tests/fixtures/volume/docker-compose.yml | 11 +++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/volume/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 3f153d0d8..d92077934 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -947,7 +947,7 @@ def run_one_off_container(container_options, project, service, options): def remove_container(force=False): if options['--rm']: - project.client.remove_container(container.id, force=True) + project.client.remove_container(container.id, force=True, v=True) signals.set_signal_handler_to_shutdown() try: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 7641870b6..e9b114a0e 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -984,6 +984,36 @@ class CLITestCase(DockerClientTestCase): [u'/bin/true'], ) + def test_run_rm(self): + self.base_dir = 'tests/fixtures/volume' + proc = start_process(self.base_dir, ['run', '--rm', 'test']) + wait_on_condition(ContainerStateCondition( + self.project.client, + 'volume_test_run_1', + 'running')) + service = self.project.get_service('test') + containers = service.containers(one_off=OneOffFilter.only) + self.assertEqual(len(containers), 1) + mounts = containers[0].get('Mounts') + for mount in mounts: + if mount['Destination'] == '/container-path': + anonymousName = mount['Name'] + break + os.kill(proc.pid, signal.SIGINT) + wait_on_process(proc, 1) + + self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 0) + + volumes = self.client.volumes()['Volumes'] + assert volumes is not None + for volume in service.options.get('volumes'): + if volume.internal == '/container-named-path': + name = volume.external + break + volumeNames = [v['Name'] for v in volumes] + assert name in volumeNames + assert anonymousName not in volumeNames + def test_run_service_with_dockerfile_entrypoint(self): self.base_dir = 'tests/fixtures/entrypoint-dockerfile' self.dispatch(['run', 'test']) diff --git a/tests/fixtures/volume/docker-compose.yml b/tests/fixtures/volume/docker-compose.yml new file mode 100644 index 000000000..4335b0a09 --- /dev/null +++ b/tests/fixtures/volume/docker-compose.yml @@ -0,0 +1,11 @@ +version: '2' +services: + test: + image: busybox + command: top + volumes: + - /container-path + - testvolume:/container-named-path + +volumes: + testvolume: {} From 0839fb93c1338c97774a6e73976711113c731740 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Mon, 22 Aug 2016 14:39:02 -0700 Subject: [PATCH 05/92] Use posix shell instead of bash for run.sh There are no bash specific uses in the script so we can use posix shell. Signed-off-by: Natanael Copa --- script/run/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run/run.sh b/script/run/run.sh index 6205747af..49baa41c3 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # Run docker-compose in a container # From 086ae04b9e01c06e9614e58b5b9e402a8f5298c2 Mon Sep 17 00:00:00 2001 From: Nicolas Barbey Date: Mon, 17 Oct 2016 15:02:48 +0200 Subject: [PATCH 06/92] Fix TypeError : unorderable types: str() < int() While merging list items into a set, strings and ints are compared which is not possible. We cast everything to strings to avoid the issue. The issue was seen with python 3.5 while overriding configuration files with heterogenous port types (int in one file, string in another). Signed-off-by: Nicolas Barbey --- compose/config/config.py | 2 ++ tests/unit/config/config_test.py | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index be73e1dee..91f6ac9a0 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -778,6 +778,8 @@ def merge_service_dicts(base, override, version): def merge_unique_items_lists(base, override): + override = [str(o) for o in override] + base = [str(b) for b in base] return sorted(set().union(base, override)) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d9269ab43..3fcfec162 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1378,6 +1378,44 @@ class ConfigTest(unittest.TestCase): 'extends': {'service': 'foo'} } + def test_merge_service_dicts_heterogeneous(self): + base = { + 'volumes': ['.:/app'], + 'ports': ['5432'] + } + override = { + 'image': 'alpine:edge', + 'ports': [5432] + } + actual = config.merge_service_dicts_from_files( + base, + override, + DEFAULT_VERSION) + assert actual == { + 'image': 'alpine:edge', + 'volumes': ['.:/app'], + 'ports': ['5432'] + } + + def test_merge_service_dicts_heterogeneous_2(self): + base = { + 'volumes': ['.:/app'], + 'ports': [5432] + } + override = { + 'image': 'alpine:edge', + 'ports': ['5432'] + } + actual = config.merge_service_dicts_from_files( + base, + override, + DEFAULT_VERSION) + assert actual == { + 'image': 'alpine:edge', + 'volumes': ['.:/app'], + 'ports': ['5432'] + } + def test_merge_build_args(self): base = { 'build': { From b0b9a3703ddfb0ce357ca09233e5f02caccf03ef Mon Sep 17 00:00:00 2001 From: Bheesham Persaud Date: Sat, 3 Dec 2016 02:17:01 -0500 Subject: [PATCH 07/92] Add fish completion to contrib. Signed-off-by: Bheesham Persaud --- contrib/completion/fish/docker-compose.fish | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 contrib/completion/fish/docker-compose.fish diff --git a/contrib/completion/fish/docker-compose.fish b/contrib/completion/fish/docker-compose.fish new file mode 100644 index 000000000..69ecc5056 --- /dev/null +++ b/contrib/completion/fish/docker-compose.fish @@ -0,0 +1,24 @@ +# Tab completion for docker-compose (https://github.com/docker/compose). +# Version: 1.9.0 + +complete -e -c docker-compose + +for line in (docker-compose --help | \ + string match -r '^\s+\w+\s+[^\n]+' | \ + string trim) + set -l doc (string split -m 1 ' ' -- $line) + complete -c docker-compose -n '__fish_use_subcommand' -xa $doc[1] --description $doc[2] +end + +complete -c docker-compose -s f -l file -r -d 'Specify an alternate compose file' +complete -c docker-compose -s p -l project-name -x -d 'Specify an alternate project name' +complete -c docker-compose -l verbose -d 'Show more output' +complete -c docker-compose -s H -l host -x -d 'Daemon socket to connect to' +complete -c docker-compose -l tls -d 'Use TLS; implied by --tlsverify' +complete -c docker-compose -l tlscacert -r -d 'Trust certs signed only by this CA' +complete -c docker-compose -l tlscert -r -d 'Path to TLS certificate file' +complete -c docker-compose -l tlskey -r -d 'Path to TLS key file' +complete -c docker-compose -l tlsverify -d 'Use TLS and verify the remote' +complete -c docker-compose -l skip-hostname-check -d "Don't check the daemon's hostname against the name specified in the client certificate (for example if your docker host is an IP address)" +complete -c docker-compose -s h -l help -d 'Print usage' +complete -c docker-compose -s v -l version -d 'Print version and exit' From fbcc1510cca36c6e2e049b7692b11eccc8c33327 Mon Sep 17 00:00:00 2001 From: Danny Guo Date: Fri, 17 Jun 2016 18:41:40 -0400 Subject: [PATCH 08/92] Handle giving help a nonexistent command The CLI would show an unhandled exception when running: $ docker-compose help foobar Now, it lists the commands. Signed-off-by: Danny Guo --- .gitignore | 3 +++ compose/cli/main.py | 15 ++++++--------- tests/acceptance/cli_test.py | 6 ++++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 4b318e232..ef04ca15f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ /venv README.rst compose/GITSHA +*.swo +*.swp +.DS_Store diff --git a/compose/cli/main.py b/compose/cli/main.py index cf53f6aa4..43b9a374f 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -58,9 +58,8 @@ console_handler = logging.StreamHandler(sys.stderr) def main(): - command = dispatch() - try: + command = dispatch() command() except (KeyboardInterrupt, signals.ShutdownException): log.error("Aborting.") @@ -78,6 +77,10 @@ def main(): except NeedsBuildError as e: log.error("Service '%s' needs to be built, but --no-build was passed." % e.service.name) sys.exit(1) + except NoSuchCommand as e: + commands = "\n".join(parse_doc_section("commands:", getdoc(e.supercommand))) + log.error("No such command: %s\n\n%s", e.command, commands) + sys.exit(1) except (errors.ConnectionError, StreamParseError): sys.exit(1) @@ -88,13 +91,7 @@ def dispatch(): TopLevelCommand, {'options_first': True, 'version': get_version_info('compose')}) - try: - options, handler, command_options = dispatcher.parse(sys.argv[1:]) - except NoSuchCommand as e: - commands = "\n".join(parse_doc_section("commands:", getdoc(e.supercommand))) - log.error("No such command: %s\n\n%s", e.command, commands) - sys.exit(1) - + options, handler, command_options = dispatcher.parse(sys.argv[1:]) setup_console_handler(console_handler, options.get('--verbose')) return functools.partial(perform_command, options, handler, command_options) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index e9a416919..2dfda03f8 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -150,6 +150,12 @@ class CLITestCase(DockerClientTestCase): # Prevent tearDown from trying to create a project self.base_dir = None + def test_help_nonexistent(self): + self.base_dir = 'tests/fixtures/no-composefile' + result = self.dispatch(['help', 'foobar'], returncode=1) + assert 'No such command' in result.stderr + self.base_dir = None + def test_shorthand_host_opt(self): self.dispatch( ['-H={0}'.format(os.environ.get('DOCKER_HOST', 'unix://')), From f01ecda83cb442294e275c82985b235d5b176300 Mon Sep 17 00:00:00 2001 From: Allan de Queiroz Date: Wed, 4 Jan 2017 11:40:41 +0000 Subject: [PATCH 09/92] Fixing bash path problem, avoiding 'docker-compose-completion.sh: bash: bad interpreter: No such file or directory' error Signed-off-by: Allan de Queiroz --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 991f65729..9c1b5bf49 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -1,4 +1,4 @@ -#!bash +#!/bin/bash # # bash completion for docker-compose # From 22249add84831a02976f6c98020f05eb7418b287 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 6 Jun 2016 15:48:47 -0700 Subject: [PATCH 10/92] Use newer version of PyInstaller to fix prelinking issues Signed-off-by: Joffrey F --- requirements-build.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-build.txt b/requirements-build.txt index 3f1dbd75b..27f610ca9 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -pyinstaller==3.1.1 +pyinstaller==3.2.1 From 84774cacd210bb176c2daf73106c7dc849a6a0d7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 15:16:09 -0800 Subject: [PATCH 11/92] Upgrade python and pip versions in Dockerfile Add libbz2 dependency Signed-off-by: Joffrey F --- Dockerfile | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 63fac3eb3..a03e15106 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN set -ex; \ ca-certificates \ curl \ libsqlite3-dev \ + libbz2-dev \ ; \ rm -rf /var/lib/apt/lists/* @@ -20,40 +21,32 @@ RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.8.3 \ -o /usr/local/bin/docker && \ chmod +x /usr/local/bin/docker -# Build Python 2.7.9 from source +# Build Python 2.7.13 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz | tar -xz; \ - cd Python-2.7.9; \ + curl -L https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz | tar -xz; \ + cd Python-2.7.13; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-2.7.9 + rm -rf /Python-2.7.13 # Build python 3.4 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tgz | tar -xz; \ - cd Python-3.4.3; \ + curl -L https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz | tar -xz; \ + cd Python-3.4.6; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-3.4.3 + rm -rf /Python-3.4.6 # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib -# Install setuptools -RUN set -ex; \ - curl -L https://bootstrap.pypa.io/ez_setup.py | python - # Install pip RUN set -ex; \ - curl -L https://pypi.python.org/packages/source/p/pip/pip-8.1.1.tar.gz | tar -xz; \ - cd pip-8.1.1; \ - python setup.py install; \ - cd ..; \ - rm -rf pip-8.1.1 + curl -L https://bootstrap.pypa.io/get-pip.py | python # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen From a3a9d8944a413897ae1f0305d16d6d1071487ad6 Mon Sep 17 00:00:00 2001 From: Kevin Jing Qiu Date: Thu, 26 Jan 2017 14:23:12 -0500 Subject: [PATCH 12/92] Close the open file handle using context manager Signed-off-by: Kevin Jing Qiu --- compose/config/environment.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/config/environment.py b/compose/config/environment.py index 7b9269300..4ba228c8a 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import codecs +import contextlib import logging import os @@ -31,11 +32,12 @@ def env_vars_from_file(filename): elif not os.path.isfile(filename): raise ConfigurationError("%s is not a file." % (filename)) env = {} - for line in codecs.open(filename, 'r', 'utf-8'): - line = line.strip() - if line and not line.startswith('#'): - k, v = split_env(line) - env[k] = v + with contextlib.closing(codecs.open(filename, 'r', 'utf-8')) as fileobj: + for line in fileobj: + line = line.strip() + if line and not line.startswith('#'): + k, v = split_env(line) + env[k] = v return env From b392b6e12ed724ce39e0e65fd5582b70c47803af Mon Sep 17 00:00:00 2001 From: fate-grand-order Date: Tue, 7 Feb 2017 15:59:34 +0800 Subject: [PATCH 13/92] fix typo in CHANGELOG.md Signed-off-by: fate-grand-order --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c645ca25..e969b4537 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -525,7 +525,7 @@ Bug Fixes: if at least one container is using the network. - When printings logs during `up` or `logs`, flush the output buffer after - each line to prevent buffering issues from hideing logs. + each line to prevent buffering issues from hiding logs. - Recreate a container if one of its dependencies is being created. Previously a container was only recreated if it's dependencies already From f0835268296111cac54faa8701b7aba751c0a239 Mon Sep 17 00:00:00 2001 From: "Aaron.L.Xu" Date: Wed, 8 Feb 2017 18:50:14 +0800 Subject: [PATCH 14/92] referencing right segment of code Signed-off-by: Aaron.L.Xu --- compose/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/bundle.py b/compose/bundle.py index 854cc7995..505ce91fe 100644 --- a/compose/bundle.py +++ b/compose/bundle.py @@ -202,7 +202,7 @@ def convert_service_to_bundle(name, service_dict, image_digest): return container_config -# See https://github.com/docker/swarmkit/blob//agent/exec/container/container.go#L95 +# See https://github.com/docker/swarmkit/blob/agent/exec/container/container.go#L95 def set_command_and_args(config, entrypoint, command): if isinstance(entrypoint, six.string_types): entrypoint = split_command(entrypoint) From ac235a1f8571067e810fe0f9ba808e2c29bb9cc9 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Berthon Date: Wed, 8 Feb 2017 10:39:04 +0000 Subject: [PATCH 15/92] Add support to build docker-compose on ARM 32bit Added a new Dockerfile (Dockerfile.armhf) specific for ARM 32 bit. The Dockerfile was updated compare to default one: - Base image is armhf/debian instead of debian - Docker binary is downloaded with the correct arch (although it does not seems to be used) Signed-off-by: Jean-Christophe Berthon --- Dockerfile.armhf | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 Dockerfile.armhf diff --git a/Dockerfile.armhf b/Dockerfile.armhf new file mode 100644 index 000000000..9fd697155 --- /dev/null +++ b/Dockerfile.armhf @@ -0,0 +1,71 @@ +FROM armhf/debian:wheezy + +RUN set -ex; \ + apt-get update -qq; \ + apt-get install -y \ + locales \ + gcc \ + make \ + zlib1g \ + zlib1g-dev \ + libssl-dev \ + git \ + ca-certificates \ + curl \ + libsqlite3-dev \ + libbz2-dev \ + ; \ + rm -rf /var/lib/apt/lists/* + +RUN curl https://get.docker.com/builds/Linux/armel/docker-1.8.3 \ + -o /usr/local/bin/docker && \ + chmod +x /usr/local/bin/docker + +# Build Python 2.7.13 from source +RUN set -ex; \ + curl -L https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz | tar -xz; \ + cd Python-2.7.13; \ + ./configure --enable-shared; \ + make; \ + make install; \ + cd ..; \ + rm -rf /Python-2.7.13 + +# Build python 3.4 from source +RUN set -ex; \ + curl -L https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz | tar -xz; \ + cd Python-3.4.6; \ + ./configure --enable-shared; \ + make; \ + make install; \ + cd ..; \ + rm -rf /Python-3.4.6 + +# Make libpython findable +ENV LD_LIBRARY_PATH /usr/local/lib + +# Install pip +RUN set -ex; \ + curl -L https://bootstrap.pypa.io/get-pip.py | python + +# Python3 requires a valid locale +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 + +RUN useradd -d /home/user -m -s /bin/bash user +WORKDIR /code/ + +RUN pip install tox==2.1.1 + +ADD requirements.txt /code/ +ADD requirements-dev.txt /code/ +ADD .pre-commit-config.yaml /code/ +ADD setup.py /code/ +ADD tox.ini /code/ +ADD compose /code/compose/ +RUN tox --notest + +ADD . /code/ +RUN chown -R user /code/ + +ENTRYPOINT ["/code/.tox/py27/bin/docker-compose"] From 5193e56a653fa95d1b91b08ad08201aaebe34923 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Berthon Date: Wed, 8 Feb 2017 12:33:57 +0000 Subject: [PATCH 16/92] Add first attempt at supporting test suite on ARM Modify `script/test/default` so it supports a first attempt at testing on ARM. Call the script with: `$ DOCKERFILE=Dockerfile.armhf script/test/default` to use the Dockerfile.armhf instead of the default Dockerfile for building the image. However, running the script is not working fully. The problem is that `dockerswarm/dind` does not provide an ARM image and therefore cannot be executed. If that is fixed then we should be able to change the script in order to use the ARM image instead of the default x86_64 image for running further tests. Signed-off-by: Jean-Christophe Berthon --- script/test/default | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/script/test/default b/script/test/default index fa741a19d..aabb4e426 100755 --- a/script/test/default +++ b/script/test/default @@ -5,11 +5,15 @@ set -ex TAG="docker-compose:$(git rev-parse --short HEAD)" +# By default use the Dockerfile, but can be overriden to use an alternative file +# e.g DOCKERFILE=Dockerfile.armhf script/test/default +DOCKERFILE="${DOCKERFILE:-Dockerfile}" + rm -rf coverage-html # Create the host directory so it's owned by $USER mkdir -p coverage-html -docker build -t "$TAG" . +docker build -f ${DOCKERFILE} -t "$TAG" . GIT_VOLUME="--volume=$(pwd)/.git:/code/.git" . script/test/all From 979a0d53f7e989f13dc77865c7d1f4775f97319e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Feb 2017 15:26:34 -0800 Subject: [PATCH 17/92] Bump 1.11.0-rc1 Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/__init__.py b/compose/__init__.py index 384178364..ae8d759d4 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.11.0dev' +__version__ = '1.11.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 5872b081a..9de11d5fa 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.8.0" +VERSION="1.11.0-rc1" IMAGE="docker/compose:$VERSION" From 01d1895a350c445eab9deb37d1cd8b6fe3328b47 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Feb 2017 11:30:29 -0800 Subject: [PATCH 18/92] Bump 1.11.0 Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/__init__.py b/compose/__init__.py index ae8d759d4..d7468af25 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.11.0-rc1' +__version__ = '1.11.0' diff --git a/script/run/run.sh b/script/run/run.sh index 9de11d5fa..b45630f07 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.11.0-rc1" +VERSION="1.11.0" IMAGE="docker/compose:$VERSION" From 2cd6cb9a47b2d00cfad7d49a26641020f7f8a66a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 16:20:59 -0800 Subject: [PATCH 19/92] Bump 1.10.1 Signed-off-by: Joffrey F --- CHANGELOG.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6699f8807..d0681e8af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,53 @@ Change log ========== +1.11.0 (2017-02-08) +------------------- + +### New Features + +#### Compose file version 3.1 + +- Introduced version 3.1 of the `docker-compose.yml` specification. This + version requires Docker Engine 1.13.0 or above. It introduces support + for secrets. See the documentation for more information + +#### Compose file version 2.0 and up + +- Introduced the `docker-compose top` command that displays processes running + for the different services managed by Compose. + +### Bugfixes + +- Fixed a bug where extending a service defining a healthcheck dictionary + would cause `docker-compose` to error out. + +- Fixed an issue where the `pid` entry in a service definition was being + ignored when using multiple Compose files. + +1.10.1 (2017-02-01) +------------------ + +### Bugfixes + +- Fixed an issue where presence of older versions of the docker-py + package would cause unexpected crashes while running Compose + +- Fixed an issue where healthcheck dependencies would be lost when + using multiple compose files for a project + +- Fixed a few issues that made the output of the `config` command + invalid + +- Fixed an issue where adding volume labels to v3 Compose files would + result in an error + +- Fixed an issue on Windows where build context paths containing unicode + characters were being improperly encoded + +- Fixed a bug where Compose would occasionally crash while streaming logs + when containers would stop or restart + 1.10.0 (2017-01-18) ------------------- From fc7b74d7f900d6e94ebd46a05cdcadcfd1f7d407 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Feb 2017 13:47:08 -0800 Subject: [PATCH 20/92] Bump to next dev version Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/__init__.py b/compose/__init__.py index d7468af25..b2ca86f86 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.11.0' +__version__ = '1.12.0dev' diff --git a/script/run/run.sh b/script/run/run.sh index b45630f07..4e173894d 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.11.0" +VERSION="1.12.0dev" IMAGE="docker/compose:$VERSION" From 47e4442722373ce43f7878ee98fe9aceb1b9f177 Mon Sep 17 00:00:00 2001 From: kevinetc123 Date: Thu, 9 Feb 2017 19:10:26 +0800 Subject: [PATCH 21/92] fix typo in project.py Signed-off-by: kevinetc123 --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 0330ab80f..133071e7d 100644 --- a/compose/project.py +++ b/compose/project.py @@ -365,7 +365,7 @@ class Project(object): # TODO: get labels from the API v1.22 , see github issue 2618 try: - # this can fail if the conatiner has been removed + # this can fail if the container has been removed container = Container.from_id(self.client, event['id']) except APIError: continue From c092fa37de820e7d6dd20b30d2c4dec28f214dd3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 9 Feb 2017 10:27:06 -0500 Subject: [PATCH 22/92] Fix version 3.1 Signed-off-by: Daniel Nephin --- compose/config/config.py | 11 ++++------- docker-compose.spec | 5 +++++ tests/unit/config/config_test.py | 4 ++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index ae85674bd..09a717bea 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -186,11 +186,6 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): if version == '3': version = V3_0 - if version not in (V2_0, V2_1, V3_0): - raise ConfigurationError( - 'Version in "{}" is unsupported. {}' - .format(self.filename, VERSION_EXPLANATION)) - return version def get_service(self, name): @@ -479,7 +474,7 @@ def process_config_file(config_file, environment, service_name=None): 'service', environment) - if config_file.version in (V2_0, V2_1, V3_0): + if config_file.version in (V2_0, V2_1, V3_0, V3_1): processed_config = dict(config_file.config) processed_config['services'] = services processed_config['volumes'] = interpolate_config_section( @@ -495,7 +490,9 @@ def process_config_file(config_file, environment, service_name=None): elif config_file.version == V1: processed_config = services else: - raise Exception("Unsupported version: {}".format(repr(config_file.version))) + raise ConfigurationError( + 'Version in "{}" is unsupported. {}' + .format(config_file.filename, VERSION_EXPLANATION)) config_file = config_file._replace(config=processed_config) validate_against_config_schema(config_file) diff --git a/docker-compose.spec b/docker-compose.spec index ec5a2039c..ef0e2593e 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -37,6 +37,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.0.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.1.json', + 'compose/config/config_schema_v3.1.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 666b21f28..ef57bb57e 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -19,6 +19,7 @@ from compose.config.config import V1 from compose.config.config import V2_0 from compose.config.config import V2_1 from compose.config.config import V3_0 +from compose.config.config import V3_1 from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION @@ -168,6 +169,9 @@ class ConfigTest(unittest.TestCase): cfg = config.load(build_config_details({'version': version})) assert cfg.version == V3_0 + cfg = config.load(build_config_details({'version': '3.1'})) + assert cfg.version == V3_1 + def test_v1_file_version(self): cfg = config.load(build_config_details({'web': {'image': 'busybox'}})) assert cfg.version == V1 From dc5b3f3b3eb53ce747003089c8ae5ff21f4b1f70 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 10 Feb 2017 17:05:33 -0500 Subject: [PATCH 23/92] Fix secrets config. Signed-off-by: Daniel Nephin --- compose/config/config.py | 5 ++ compose/config/types.py | 8 +++ setup.py | 10 ++-- tests/unit/config/config_test.py | 86 ++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 6 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 09a717bea..4c9cf423b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -763,6 +763,11 @@ def finalize_service(service_config, service_names, version, environment): if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) + if 'secrets' in service_dict: + service_dict['secrets'] = [ + types.ServiceSecret.parse(s) for s in service_dict['secrets'] + ] + normalize_build(service_dict, service_config.working_dir, environment) service_dict['name'] = service_config.name diff --git a/compose/config/types.py b/compose/config/types.py index 17d5c8b37..f86c03199 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -253,3 +253,11 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): @property def merge_field(self): return self.source + + def repr(self): + return dict( + source=self.source, + target=self.target, + uid=self.uid, + gid=self.gid, + mode=self.mode) diff --git a/setup.py b/setup.py index 0b1d4e08f..eafbc356f 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import absolute_import +from __future__ import print_function from __future__ import unicode_literals import codecs -import logging import os import re import sys @@ -64,11 +64,9 @@ try: for key, value in extras_require.items(): if key.startswith(':') and pkg_resources.evaluate_marker(key[1:]): install_requires.extend(value) -except Exception: - logging.getLogger(__name__).exception( - 'Failed to compute platform dependencies. All dependencies will be ' - 'installed as a result.' - ) +except Exception as e: + print("Failed to compute platform dependencies: {}. ".format(e) + + "All dependencies will be installed as a result.", file=sys.stderr) for key, value in extras_require.items(): if key.startswith(':'): install_requires.extend(value) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ef57bb57e..d4d1ad2c4 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -13,6 +13,7 @@ import pytest from ...helpers import build_config_details from compose.config import config +from compose.config import types from compose.config.config import resolve_build_args from compose.config.config import resolve_environment from compose.config.config import V1 @@ -1849,6 +1850,91 @@ class ConfigTest(unittest.TestCase): config.load(config_details) assert 'has neither an image nor a build context' in exc.exconly() + def test_load_secrets(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'image': 'example/web', + 'secrets': [ + 'one', + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ], + }, + }, + 'secrets': { + 'one': {'file': 'secret.txt'}, + }, + }) + details = config.ConfigDetails('.', [base_file]) + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + }, + ] + assert service_sort(service_dicts) == service_sort(expected) + + def test_load_secrets_multi_file(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'image': 'example/web', + 'secrets': ['one'], + }, + }, + 'secrets': { + 'one': {'file': 'secret.txt'}, + }, + }) + override_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'secrets': [ + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ], + }, + }, + }) + details = config.ConfigDetails('.', [base_file, override_file]) + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + }, + ] + assert service_sort(service_dicts) == service_sort(expected) + class NetworkModeTest(unittest.TestCase): def test_network_mode_standard(self): From 252699c1d124ba365e4b9078f4a44f3acbcb08be Mon Sep 17 00:00:00 2001 From: Petr Karmashev Date: Sun, 12 Feb 2017 02:03:04 +0300 Subject: [PATCH 24/92] Compose file reference link fix in README.md Signed-off-by: Petr Karmashev --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5cf69b05c..35a10b908 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ A `docker-compose.yml` looks like this: image: redis For more information about the Compose file, see the -[Compose file reference](https://github.com/docker/docker.github.io/blob/master/compose/compose-file.md) +[Compose file reference](https://github.com/docker/docker.github.io/blob/master/compose/compose-file/compose-versioning.md) Compose has commands for managing the whole lifecycle of your application: From abce83ef25528fb36979208888ba0033c89f47a3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 13 Feb 2017 16:04:06 -0800 Subject: [PATCH 25/92] Fix `config` command output with service.secrets section Signed-off-by: Joffrey F --- compose/config/serialize.py | 3 ++ compose/config/types.py | 7 ++-- tests/unit/config/config_test.py | 57 ++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 3745de82d..46d283f08 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -102,4 +102,7 @@ def denormalize_service_dict(service_dict, version): service_dict['healthcheck']['timeout'] ) + if 'secrets' in service_dict: + service_dict['secrets'] = map(lambda s: s.repr(), service_dict['secrets']) + return service_dict diff --git a/compose/config/types.py b/compose/config/types.py index f86c03199..811e6c1fc 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -256,8 +256,5 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): def repr(self): return dict( - source=self.source, - target=self.target, - uid=self.uid, - gid=self.gid, - mode=self.mode) + [(k, v) for k, v in self._asdict().items() if v is not None] + ) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d4d1ad2c4..c26272d9e 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -54,6 +54,10 @@ def service_sort(services): return sorted(services, key=itemgetter('name')) +def secret_sort(secrets): + return sorted(secrets, key=itemgetter('source')) + + class ConfigTest(unittest.TestCase): def test_load(self): service_dicts = config.load( @@ -1771,6 +1775,38 @@ class ConfigTest(unittest.TestCase): 'labels': {'com.docker.compose.test': 'yes'} } + def test_merge_different_secrets(self): + base = { + 'image': 'busybox', + 'secrets': [ + {'source': 'src.txt'} + ] + } + override = {'secrets': ['other-src.txt']} + + actual = config.merge_service_dicts(base, override, V3_1) + assert secret_sort(actual['secrets']) == secret_sort([ + {'source': 'src.txt'}, + {'source': 'other-src.txt'} + ]) + + def test_merge_secrets_override(self): + base = { + 'image': 'busybox', + 'secrets': ['src.txt'], + } + override = { + 'secrets': [ + { + 'source': 'src.txt', + 'target': 'data.txt', + 'mode': 0o400 + } + ] + } + actual = config.merge_service_dicts(base, override, V3_1) + assert actual['secrets'] == override['secrets'] + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', @@ -3491,3 +3527,24 @@ class SerializeTest(unittest.TestCase): denormalized_service = denormalize_service_dict(processed_service, V2_1) assert denormalized_service['healthcheck']['interval'] == '100s' assert denormalized_service['healthcheck']['timeout'] == '30s' + + def test_denormalize_secrets(self): + service_dict = { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + } + denormalized_service = denormalize_service_dict(service_dict, V3_1) + assert secret_sort(denormalized_service['secrets']) == secret_sort([ + {'source': 'one'}, + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ]) From 66f4a795a2ded1a26b6cf8474edb423727dd585d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 15 Feb 2017 16:07:08 -0800 Subject: [PATCH 26/92] Don't import pip inside Compose Signed-off-by: Joffrey F --- compose/cli/__init__.py | 37 +++++++++++++++++++++++++++++++++++++ compose/cli/main.py | 24 ------------------------ 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index e69de29bb..c5db44558 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -0,0 +1,37 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import subprocess +import sys + +# Attempt to detect https://github.com/docker/compose/issues/4344 +try: + # We don't try importing pip because it messes with package imports + # on some Linux distros (Ubuntu, Fedora) + # https://github.com/docker/compose/issues/4425 + # https://github.com/docker/compose/issues/4481 + # https://github.com/pypa/pip/blob/master/pip/_vendor/__init__.py + s_cmd = subprocess.Popen( + ['pip', 'freeze'], stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) + packages = s_cmd.communicate()[0].splitlines() + dockerpy_installed = len( + list(filter(lambda p: p.startswith(b'docker-py=='), packages)) + ) > 0 + if dockerpy_installed: + from .colors import red + print( + red('ERROR:'), + "Dependency conflict: an older version of the 'docker-py' package " + "is polluting the namespace. " + "Run the following command to remedy the issue:\n" + "pip uninstall docker docker-py; pip install docker", + file=sys.stderr + ) + sys.exit(1) + +except OSError: + # pip command is not available, which indicates it's probably the binary + # distribution of Compose which is not affected + pass diff --git a/compose/cli/main.py b/compose/cli/main.py index e2ebce48e..51ba36a09 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -14,30 +14,6 @@ from distutils.spawn import find_executable from inspect import getdoc from operator import attrgetter - -# Attempt to detect https://github.com/docker/compose/issues/4344 -try: - # A regular import statement causes PyInstaller to freak out while - # trying to load pip. This way it is simply ignored. - pip = __import__('pip') - pip_packages = pip.get_installed_distributions() - if 'docker-py' in [pkg.project_name for pkg in pip_packages]: - from .colors import red - print( - red('ERROR:'), - "Dependency conflict: an older version of the 'docker-py' package " - "is polluting the namespace. " - "Run the following command to remedy the issue:\n" - "pip uninstall docker docker-py; pip install docker", - file=sys.stderr - ) - sys.exit(1) -except ImportError: - # pip is not available, which indicates it's probably the binary - # distribution of Compose which is not affected - pass - - from . import errors from . import signals from .. import __version__ From 27297fd1af64aae4e48fce254bace122014977fd Mon Sep 17 00:00:00 2001 From: "Aaron.L.Xu" Date: Thu, 16 Feb 2017 11:14:25 +0800 Subject: [PATCH 27/92] fix a typo in script/release/utils.sh Signed-off-by: Aaron.L.Xu --- script/release/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/utils.sh b/script/release/utils.sh index b4e5a2e6a..321c1fb7b 100644 --- a/script/release/utils.sh +++ b/script/release/utils.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Util functions for release scritps +# Util functions for release scripts # set -e From d20e3f334215a55cbebc27d8dde82108a58a0bae Mon Sep 17 00:00:00 2001 From: "Aaron.L.Xu" Date: Thu, 16 Feb 2017 15:25:04 +0800 Subject: [PATCH 28/92] function-name-modification for tests/* Signed-off-by: Aaron.L.Xu --- tests/acceptance/cli_test.py | 10 +++++----- tests/unit/cli_test.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 160e1913d..8366ca75e 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1234,7 +1234,7 @@ class CLITestCase(DockerClientTestCase): container = service.containers(stopped=True, one_off=OneOffFilter.only)[0] self.assertEqual(user, container.get('Config.User')) - def test_run_service_with_environement_overridden(self): + def test_run_service_with_environment_overridden(self): name = 'service' self.base_dir = 'tests/fixtures/environment-composefile' self.dispatch([ @@ -1246,9 +1246,9 @@ class CLITestCase(DockerClientTestCase): ]) service = self.project.get_service(name) container = service.containers(stopped=True, one_off=OneOffFilter.only)[0] - # env overriden + # env overridden self.assertEqual('notbar', container.environment['foo']) - # keep environement from yaml + # keep environment from yaml self.assertEqual('world', container.environment['hello']) # added option from command line self.assertEqual('beta', container.environment['alpha']) @@ -1293,7 +1293,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_range[0], "0.0.0.0:49153") self.assertEqual(port_range[1], "0.0.0.0:49154") - def test_run_service_with_explicitly_maped_ports(self): + def test_run_service_with_explicitly_mapped_ports(self): # create one off container self.base_dir = 'tests/fixtures/ports-composefile' self.dispatch(['run', '-d', '-p', '30000:3000', '--publish', '30001:3001', 'simple']) @@ -1310,7 +1310,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_short, "0.0.0.0:30000") self.assertEqual(port_full, "0.0.0.0:30001") - def test_run_service_with_explicitly_maped_ip_ports(self): + def test_run_service_with_explicitly_mapped_ip_ports(self): # create one off container self.base_dir = 'tests/fixtures/ports-composefile' self.dispatch([ diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index f9b60bff5..317650cb5 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -184,7 +184,7 @@ class CLITestCase(unittest.TestCase): mock_client.create_host_config.call_args[1].get('restart_policy') ) - def test_command_manula_and_service_ports_together(self): + def test_command_manual_and_service_ports_together(self): project = Project.from_config( name='composetest', client=None, From e0e862c0420f0fb6dea24b0a8dff198e5fc80cc1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 16 Feb 2017 17:05:09 -0800 Subject: [PATCH 29/92] Update docker SDK dependency Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3b06bff45..53b9294ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 colorama==0.3.7 -docker==2.0.2 +docker==2.1.0 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/setup.py b/setup.py index eafbc356f..13fe59b22 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.0.2, < 3.0', + 'docker >= 2.1.0, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 39b2c6636efc3573a73ffb966d49ac747dbef987 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Tue, 21 Feb 2017 17:54:20 +0100 Subject: [PATCH 30/92] Fix treatment of global arguments in bash completion Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 77d02b428..3cf37a707 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -18,7 +18,7 @@ __docker_compose_q() { - docker-compose 2>/dev/null $daemon_options "$@" + docker-compose 2>/dev/null "${daemon_options[@]}" "$@" } # Transforms a multiline list of strings into a single line string From c2553ac777b3442d82a6202af1a2838590b80374 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 21 Feb 2017 14:05:47 -0800 Subject: [PATCH 31/92] Update versions.py script to support new engine versioning schema Signed-off-by: Joffrey F --- script/test/versions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/test/versions.py b/script/test/versions.py index 0c3b8162d..97383ad99 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -44,7 +44,7 @@ class Version(namedtuple('_Version', 'major minor patch rc')): version = version.lstrip('v') version, _, rc = version.partition('-') major, minor, patch = version.split('.', 3) - return cls(int(major), int(minor), int(patch), rc) + return cls(major, minor, patch, rc) @property def major_minor(self): @@ -57,7 +57,7 @@ class Version(namedtuple('_Version', 'major minor patch rc')): """ # rc releases should appear before official releases rc = (0, self.rc) if self.rc else (1, ) - return (self.major, self.minor, self.patch) + rc + return (int(self.major), int(self.minor), int(self.patch)) + rc def __str__(self): rc = '-{}'.format(self.rc) if self.rc else '' From e4a87397af90a06ce9d72d3698bff93e6f26db27 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 21 Feb 2017 14:56:07 -0800 Subject: [PATCH 32/92] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35a10b908..d43bd8c4c 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Installation and documentation Contributing ------------ -[![Build Status](http://jenkins.dockerproject.org/buildStatus/icon?job=Compose%20Master)](http://jenkins.dockerproject.org/job/Compose%20Master/) +[![Build Status](https://jenkins.dockerproject.org/buildStatus/icon?job=docker/compose/master)](https://jenkins.dockerproject.org/job/docker/job/compose/job/master/) Want to help build Compose? Check out our [contributing documentation](https://github.com/docker/compose/blob/master/CONTRIBUTING.md). From 5b6191e6536abc4bbbae902c384bfa531315a897 Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Mon, 20 Feb 2017 13:18:52 -0700 Subject: [PATCH 33/92] Add cache_from to build opts Signed-off-by: Joey Payne --- compose/config/config_schema_v3.1.json | 3 ++- compose/service.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index b7037485f..77c2d35cb 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -71,7 +71,8 @@ "properties": { "context": {"type": "string"}, "dockerfile": {"type": "string"}, - "args": {"$ref": "#/definitions/list_or_dict"} + "args": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"type": "#/definitions/list_of_strings"} }, "additionalProperties": false } diff --git a/compose/service.py b/compose/service.py index 9f2fc68b4..023efa274 100644 --- a/compose/service.py +++ b/compose/service.py @@ -802,6 +802,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), buildargs=build_opts.get('args', None), + cache_from=build_opts.get('cache_from', None), ) try: From 33fcfca0409612df38caedb51126d83c51522c7d Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Mon, 20 Feb 2017 13:20:13 -0700 Subject: [PATCH 34/92] Add test for cache_from Signed-off-by: Joey Payne --- tests/integration/service_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 09758eee9..cb6e5d318 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -32,6 +32,7 @@ from compose.service import NetworkMode from compose.service import Service from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_only +from tests.integration.testcases import v3_only def create_and_start_container(service, **override_options): @@ -946,6 +947,20 @@ class ServiceTest(DockerClientTestCase): }.items(): self.assertEqual(env[k], v) + @v3_only() + def test_build_with_cachefrom(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + + service = self.create_service('cache_from', + build={'context': base_dir, + 'cache_from': ['build1']}) + service.build() + assert service.image() + @mock.patch.dict(os.environ) def test_resolve_env(self): os.environ['FILE_DEF'] = 'E1' From c64f7dde0486b6c2ceac5a3bc186bbfef3a3f572 Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Mon, 20 Feb 2017 13:39:32 -0700 Subject: [PATCH 35/92] Fix failing unit tests Signed-off-by: Joey Payne --- tests/unit/service_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 2d5b17619..0a66e4f3e 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -446,6 +446,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs=None, + cache_from=None, ) def test_ensure_image_exists_no_build(self): @@ -482,6 +483,7 @@ class ServiceTest(unittest.TestCase): nocache=False, rm=True, buildargs=None, + cache_from=None, ) def test_build_does_not_pull(self): From 3621787a74773cf749bb606e79108a42c2167151 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Feb 2017 15:21:03 -0800 Subject: [PATCH 36/92] Check for divergent containers when scaling up Signed-off-by: Joffrey F --- compose/service.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index 9f2fc68b4..fb51ef097 100644 --- a/compose/service.py +++ b/compose/service.py @@ -226,9 +226,20 @@ class Service(object): if num_running != len(all_containers): # we have some stopped containers, let's start them up again + stopped_containers = [ + c for c in all_containers if not c.is_running + ] + + # Remove containers that have diverged + divergent_containers = [ + c for c in stopped_containers if self._containers_have_diverged([c]) + ] stopped_containers = sorted( - (c for c in all_containers if not c.is_running), - key=attrgetter('number')) + set(stopped_containers) - set(divergent_containers), + key=attrgetter('number') + ) + for c in divergent_containers: + c.remove() num_stopped = len(stopped_containers) From 8b920494327852a70b6b61bfd9a8d29bdf9b63c9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Feb 2017 16:21:23 -0800 Subject: [PATCH 37/92] Detect the service that causes the invalid service name error Signed-off-by: Joffrey F --- compose/config/validation.py | 9 ++++++--- tests/unit/config/config_test.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 3f23f0a7a..d4d29565f 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -211,9 +211,12 @@ def handle_error_for_schema_with_id(error, path): if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties': return "Invalid service name '{}' - only {} characters are allowed".format( - # The service_name is the key to the json object - list(error.instance)[0], - VALID_NAME_CHARS) + # The service_name is one of the keys in the json object + [i for i in list(error.instance) if not i or any(filter( + lambda c: not re.match(VALID_NAME_CHARS, c), i + ))][0], + VALID_NAME_CHARS + ) if error.validator == 'additionalProperties': if schema_id == '#/definitions/service': diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c26272d9e..e5104f769 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -554,6 +554,20 @@ class ConfigTest(unittest.TestCase): excinfo.exconly() ) + def test_config_invalid_service_name_raise_validation_error(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details({ + 'version': '2', + 'services': { + 'test_app': {'build': '.'}, + 'mong\\o': {'image': 'mongo'}, + } + }) + ) + + assert 'Invalid service name \'mong\\o\'' in excinfo.exconly() + def test_load_with_multiple_files_v1(self): base_file = config.ConfigFile( 'base.yaml', From 2943d2e61d061f6a1b261ffbb18b1c31386a56ea Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 23 Feb 2017 14:41:08 +0100 Subject: [PATCH 38/92] Activate bash completion for Windows executable Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 3cf37a707..79f0fc313 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -571,4 +571,4 @@ _docker_compose() { return 0 } -complete -F _docker_compose docker-compose +complete -F _docker_compose docker-compose docker-compose.exe From 21529169adcec669904cbea5141d0f90452d8361 Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sat, 4 Feb 2017 13:26:45 +1300 Subject: [PATCH 39/92] Pull services in parallel Updates #1652 Signed-off-by: Evan Shaw --- compose/project.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 133071e7d..84accacb5 100644 --- a/compose/project.py +++ b/compose/project.py @@ -455,9 +455,16 @@ class Project(object): return plans def pull(self, service_names=None, ignore_pull_failures=False): - for service in self.get_services(service_names, include_deps=False): + def pull_service(service): service.pull(ignore_pull_failures) + services = self.get_services(service_names, include_deps=False) + parallel.parallel_execute( + services, + pull_service, + operator.attrgetter('name'), + 'Pulling') + def push(self, service_names=None, ignore_push_failures=False): for service in self.get_services(service_names, include_deps=False): service.push(ignore_push_failures) From 05aa8c72857dcc058b3aeb772a6864435a2071a5 Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Mon, 13 Feb 2017 16:15:49 +1300 Subject: [PATCH 40/92] Add optional limit to the number of parallel operations Signed-off-by: Evan Shaw --- compose/parallel.py | 41 +++++++++++++++++++++++++------------ compose/project.py | 3 ++- tests/unit/parallel_test.py | 2 +- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index e495410cf..94c479b15 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import logging import operator import sys +from threading import Semaphore from threading import Thread from docker.errors import APIError @@ -23,7 +24,7 @@ log = logging.getLogger(__name__) STOP = object() -def parallel_execute(objects, func, get_name, msg, get_deps=None): +def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): """Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. @@ -37,7 +38,7 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): for obj in objects: writer.initialize(get_name(obj)) - events = parallel_execute_iter(objects, func, get_deps) + events = parallel_execute_iter(objects, func, get_deps, limit) errors = {} results = [] @@ -94,7 +95,15 @@ class State(object): return set(self.objects) - self.started - self.finished - self.failed -def parallel_execute_iter(objects, func, get_deps): +class NoLimit(object): + def __enter__(self): + pass + + def __exit__(self, *ex): + pass + + +def parallel_execute_iter(objects, func, get_deps, limit): """ Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. @@ -113,11 +122,16 @@ def parallel_execute_iter(objects, func, get_deps): if get_deps is None: get_deps = _no_deps + if limit is None: + limiter = NoLimit() + else: + limiter = Semaphore(limit) + results = Queue() state = State(objects) while True: - feed_queue(objects, func, get_deps, results, state) + feed_queue(objects, func, get_deps, results, state, limiter) try: event = results.get(timeout=0.1) @@ -141,19 +155,20 @@ def parallel_execute_iter(objects, func, get_deps): yield event -def producer(obj, func, results): +def producer(obj, func, results, limiter): """ The entry point for a producer thread which runs func on a single object. Places a tuple on the results queue once func has either returned or raised. """ - try: - result = func(obj) - results.put((obj, result, None)) - except Exception as e: - results.put((obj, None, e)) + with limiter: + try: + result = func(obj) + results.put((obj, result, None)) + except Exception as e: + results.put((obj, None, e)) -def feed_queue(objects, func, get_deps, results, state): +def feed_queue(objects, func, get_deps, results, state, limiter): """ Starts producer threads for any objects which are ready to be processed (i.e. they have no dependencies which haven't been successfully processed). @@ -177,7 +192,7 @@ def feed_queue(objects, func, get_deps, results, state): ) for dep, ready_check in deps ): log.debug('Starting producer thread for {}'.format(obj)) - t = Thread(target=producer, args=(obj, func, results)) + t = Thread(target=producer, args=(obj, func, results, limiter)) t.daemon = True t.start() state.started.add(obj) @@ -199,7 +214,7 @@ class UpstreamError(Exception): class ParallelStreamWriter(object): """Write out messages for operations happening in parallel. - Each operation has it's own line, and ANSI code characters are used + Each operation has its own line, and ANSI code characters are used to jump to the correct line, and write over the line. """ diff --git a/compose/project.py b/compose/project.py index 84accacb5..8d34d8eae 100644 --- a/compose/project.py +++ b/compose/project.py @@ -463,7 +463,8 @@ class Project(object): services, pull_service, operator.attrgetter('name'), - 'Pulling') + 'Pulling', + limit=5) def push(self, service_names=None, ignore_push_failures=False): for service in self.get_services(service_names, include_deps=False): diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 2a50b7189..6b8045f14 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -82,7 +82,7 @@ def test_parallel_execute_with_upstream_errors(): events = [ (obj, result, type(exception)) for obj, result, exception - in parallel_execute_iter(objects, process, get_deps) + in parallel_execute_iter(objects, process, get_deps, None) ] assert (cache, None, type(None)) in events From f85da99ef3273794e855afda8678174419d3bf4f Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Mon, 13 Feb 2017 16:16:07 +1300 Subject: [PATCH 41/92] Silence service pull output when pulling in parallel Signed-off-by: Evan Shaw --- compose/project.py | 2 +- compose/service.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/project.py b/compose/project.py index 8d34d8eae..d02d8ece3 100644 --- a/compose/project.py +++ b/compose/project.py @@ -456,7 +456,7 @@ class Project(object): def pull(self, service_names=None, ignore_pull_failures=False): def pull_service(service): - service.pull(ignore_pull_failures) + service.pull(ignore_pull_failures, True) services = self.get_services(service_names, include_deps=False) parallel.parallel_execute( diff --git a/compose/service.py b/compose/service.py index 023efa274..0eadadfb9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -886,17 +886,19 @@ class Service(object): return any(has_host_port(binding) for binding in self.options.get('ports', [])) - def pull(self, ignore_pull_failures=False): + def pull(self, ignore_pull_failures=False, silent=False): if 'image' not in self.options: return repo, tag, separator = parse_repository_tag(self.options['image']) tag = tag or 'latest' - log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag)) + if not silent: + log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag)) try: output = self.client.pull(repo, tag=tag, stream=True) - return progress_stream.get_digest_from_pull( - stream_output(output, sys.stdout)) + if not silent: + return progress_stream.get_digest_from_pull( + stream_output(output, sys.stdout)) except (StreamOutputError, NotFound) as e: if not ignore_pull_failures: raise From c6a271e57c241ec99ebcf096ef23c8ef8e4e9137 Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Mon, 13 Feb 2017 16:22:55 +1300 Subject: [PATCH 42/92] Hide parallel pull behind --parallel flag Signed-off-by: Evan Shaw --- compose/cli/main.py | 4 +++- compose/project.py | 25 +++++++++++++++---------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 51ba36a09..423e214e9 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -602,10 +602,12 @@ class TopLevelCommand(object): Options: --ignore-pull-failures Pull what it can and ignores images with pull failures. + --parallel Pull multiple images in parallel. """ self.project.pull( service_names=options['SERVICE'], - ignore_pull_failures=options.get('--ignore-pull-failures') + ignore_pull_failures=options.get('--ignore-pull-failures'), + in_parallel=options.get('--parallel') ) def push(self, options): diff --git a/compose/project.py b/compose/project.py index d02d8ece3..c08fbc362 100644 --- a/compose/project.py +++ b/compose/project.py @@ -454,17 +454,22 @@ class Project(object): return plans - def pull(self, service_names=None, ignore_pull_failures=False): - def pull_service(service): - service.pull(ignore_pull_failures, True) - + def pull(self, service_names=None, ignore_pull_failures=False, in_parallel=False): services = self.get_services(service_names, include_deps=False) - parallel.parallel_execute( - services, - pull_service, - operator.attrgetter('name'), - 'Pulling', - limit=5) + + if in_parallel: + def pull_service(service): + service.pull(ignore_pull_failures, True) + + parallel.parallel_execute( + services, + pull_service, + operator.attrgetter('name'), + 'Pulling', + limit=5) + else: + for service in services: + service.pull(ignore_pull_failures) def push(self, service_names=None, ignore_push_failures=False): for service in self.get_services(service_names, include_deps=False): From b4b221f6a3d57f3af46d7be735bec624363a4465 Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Tue, 14 Feb 2017 10:52:57 +1300 Subject: [PATCH 43/92] Update to address code review feedback Signed-off-by: Evan Shaw --- compose/cli/main.py | 2 +- compose/project.py | 4 ++-- compose/service.py | 7 ++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 423e214e9..ae6b0ac69 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -607,7 +607,7 @@ class TopLevelCommand(object): self.project.pull( service_names=options['SERVICE'], ignore_pull_failures=options.get('--ignore-pull-failures'), - in_parallel=options.get('--parallel') + parallel_pull=options.get('--parallel') ) def push(self, options): diff --git a/compose/project.py b/compose/project.py index c08fbc362..5c21f3bf0 100644 --- a/compose/project.py +++ b/compose/project.py @@ -454,10 +454,10 @@ class Project(object): return plans - def pull(self, service_names=None, ignore_pull_failures=False, in_parallel=False): + def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False): services = self.get_services(service_names, include_deps=False) - if in_parallel: + if parallel_pull: def pull_service(service): service.pull(ignore_pull_failures, True) diff --git a/compose/service.py b/compose/service.py index 0eadadfb9..5305a151a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging +import os import re import sys from collections import namedtuple @@ -896,7 +897,11 @@ class Service(object): log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag)) try: output = self.client.pull(repo, tag=tag, stream=True) - if not silent: + if silent: + with open(os.devnull, 'w') as devnull: + return progress_stream.get_digest_from_pull( + stream_output(output, devnull)) + else: return progress_stream.get_digest_from_pull( stream_output(output, sys.stdout)) except (StreamOutputError, NotFound) as e: From e29e3f8da4b5f0e2f9a32206abd7efd2f1333928 Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sat, 25 Feb 2017 13:14:32 +1300 Subject: [PATCH 44/92] Test for parallel_execute with limit Signed-off-by: Evan Shaw --- tests/unit/parallel_test.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 6b8045f14..d10948eb0 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +from threading import Lock + import six from docker.errors import APIError @@ -40,6 +42,30 @@ def test_parallel_execute(): assert errors == {} +def test_parallel_execute_with_limit(): + limit = 1 + tasks = 20 + lock = Lock() + + def f(obj): + locked = lock.acquire(False) + # we should always get the lock because we're the only thread running + assert locked + lock.release() + return None + + results, errors = parallel_execute( + objects=list(range(tasks)), + func=f, + get_name=six.text_type, + msg="Testing", + limit=limit, + ) + + assert results == tasks*[None] + assert errors == {} + + def test_parallel_execute_with_deps(): log = [] From a507c7f72088fc4ccf48ea6f34482813c1fb9f81 Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Sun, 26 Feb 2017 19:19:38 +1300 Subject: [PATCH 45/92] Colorize statuses in parallel_execute output 'ok' displays in green 'error' displays in red Signed-off-by: Evan Shaw --- compose/parallel.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index e495410cf..cdeb07255 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -11,6 +11,8 @@ from six.moves import _thread as thread from six.moves.queue import Empty from six.moves.queue import Queue +from compose.cli.colors import green +from compose.cli.colors import red from compose.cli.signals import ShutdownException from compose.errors import HealthCheckFailed from compose.errors import NoHealthCheckConfigured @@ -45,16 +47,16 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): for obj, result, exception in events: if exception is None: - writer.write(get_name(obj), 'done') + writer.write(get_name(obj), green('done')) results.append(result) elif isinstance(exception, APIError): errors[get_name(obj)] = exception.explanation - writer.write(get_name(obj), 'error') + writer.write(get_name(obj), red('error')) elif isinstance(exception, (OperationFailedError, HealthCheckFailed, NoHealthCheckConfigured)): errors[get_name(obj)] = exception.msg - writer.write(get_name(obj), 'error') + writer.write(get_name(obj), red('error')) elif isinstance(exception, UpstreamError): - writer.write(get_name(obj), 'error') + writer.write(get_name(obj), red('error')) else: errors[get_name(obj)] = exception error_to_reraise = exception From a73190e1cc2c57ba2ac0361b376b2adc5c4dabeb Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Fri, 27 Jan 2017 08:56:02 -0800 Subject: [PATCH 46/92] Add support for returning the exit value of a specific container Current best practice for using docker-compose as a tool for continuous integration requires fragile shell pipelines to query the exit status of composed containers, e.g.: http://stackoverflow.com/questions/29568352/using-docker-compose-with-ci-how-to-deal-with-exit-codes-and-daemonized-linked http://blog.ministryofprogramming.com/docker-compose-and-exit-codes/ This PR adds a `--forward-exitval ` flag that allows `docker-compose up` to return the exit value of a specified container. The container may optionally have a number specified (foo_2) otherwise the first is defaulted to. Signed-off-by: Nathan J. Mehl --- compose/cli/main.py | 47 ++++++++++++++++++- contrib/completion/bash/docker-compose | 2 +- tests/acceptance/cli_test.py | 10 ++++ .../forward-exitval/docker-compose.yml | 6 +++ 4 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/forward-exitval/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 51ba36a09..bbd6952a0 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -854,6 +854,8 @@ class TopLevelCommand(object): running. (default: 10) --remove-orphans Remove containers for services not defined in the Compose file + --forward-exitval SERVICE Return the exit value of the selected service container. + Requires --abort-on-container-exit. """ start_deps = not options['--no-deps'] cascade_stop = options['--abort-on-container-exit'] @@ -861,10 +863,14 @@ class TopLevelCommand(object): timeout = timeout_from_opts(options) remove_orphans = options['--remove-orphans'] detached = options.get('-d') + forward_exitval = container_exitval_from_opts(options) if detached and cascade_stop: raise UserError("--abort-on-container-exit and -d cannot be combined.") + if forward_exitval and not cascade_stop: + raise UserError("--forward-exitval requires --abort-on-container-exit.") + with up_shutdown_context(self.project, service_names, timeout, detached): to_attach = self.project.up( service_names=service_names, @@ -878,9 +884,11 @@ class TopLevelCommand(object): if detached: return + all_containers = filter_containers_to_service_names(to_attach, service_names) + log_printer = log_printer_from_project( self.project, - filter_containers_to_service_names(to_attach, service_names), + all_containers, options['--no-color'], {'follow': True}, cascade_stop, @@ -891,6 +899,22 @@ class TopLevelCommand(object): if cascade_stop: print("Aborting on container exit...") self.project.stop(service_names=service_names, timeout=timeout) + if forward_exitval: + def is_us(container): + return container.name_without_project == forward_exitval + candidates = filter(is_us, all_containers) + if not candidates: + log.error('No containers matching the spec "%s" were run.', + forward_exitval) + sys.exit(2) + if len(candidates) > 1: + log.error('Multiple (%d) containers matching the spec "%s" ' + 'were found; cannot forward exit code because we ' + 'do not know which one to.', len(candidates), + forward_exitval) + sys.exit(2) + exit_code = candidates[0].inspect()['State']['ExitCode'] + sys.exit(exit_code) @classmethod def version(cls, options): @@ -923,6 +947,27 @@ def convergence_strategy_from_opts(options): return ConvergenceStrategy.changed +def container_exitval_from_opts(options): + """ Assemble a container name suitable for mapping into the + output of filter_containers_to_service_names. If the + container name ends in an underscore followed by a + positive integer, the user has deliberately specified + a container number and we believe her. Otherwise, append + `_1` to the name so as to return the exit value of the + first such named container. + """ + container_name = options.get('--forward-exitval') + if not container_name: + return None + segments = container_name.split('_') + if segments[-1].isdigit() and int(segments[-1]) > 0: + return '_'.join(segments) + else: + log.warn('"%s" does not specify a container number, ' + 'defaulting to "%s_1"', container_name, container_name) + return '_'.join(segments + ['1']) + + def timeout_from_opts(options): timeout = options.get('--timeout') return None if timeout is None else int(timeout) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 79f0fc313..979942f97 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -467,7 +467,7 @@ _docker_compose_up() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--abort-on-container-exit --build -d --force-recreate --help --no-build --no-color --no-deps --no-recreate --timeout -t --remove-orphans" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--forward-exitval --abort-on-container-exit --build -d --force-recreate --help --no-build --no-color --no-deps --no-recreate --timeout -t --remove-orphans" -- "$cur" ) ) ;; *) __docker_compose_services_all diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8366ca75e..6e03c448c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1927,3 +1927,13 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up', '-d']) result = self.dispatch(['top']) assert result.stdout.count("top") == 4 + + def test_forward_exitval(self): + self.base_dir = 'tests/fixtures/forward-exitval' + proc = start_process( + self.base_dir, + ['up', '--abort-on-container-exit', '--forward-exitval', 'another']) + + result = wait_on_process(proc, returncode=1) + + assert 'forwardexitval_another_1 exited with code 1' in result.stdout diff --git a/tests/fixtures/forward-exitval/docker-compose.yml b/tests/fixtures/forward-exitval/docker-compose.yml new file mode 100644 index 000000000..687e78b97 --- /dev/null +++ b/tests/fixtures/forward-exitval/docker-compose.yml @@ -0,0 +1,6 @@ +simple: + image: busybox:latest + command: sh -c "echo hello && tail -f /dev/null" +another: + image: busybox:latest + command: /bin/false From cffb76d4d9a731495195dec7acaab0b1b438f47c Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Mon, 27 Feb 2017 07:21:47 -0800 Subject: [PATCH 47/92] Address comments - set flag name to `--exit-code-from` (and rename some variable, function and test names to match) - force cascade_stop to true when exit-code-from flag is set - use lambda in filter statement - check that selected container name is in the project before running - remove fancy parsing of service name to container mappings: if there are multiple containers in a service, return the first nonzero exit value if any - flake8 changes Signed-off-by: Nathan J. Mehl --- compose/cli/main.py | 78 +++++++++---------- contrib/completion/bash/docker-compose | 2 +- tests/acceptance/cli_test.py | 6 +- .../docker-compose.yml | 0 4 files changed, 42 insertions(+), 44 deletions(-) rename tests/fixtures/{forward-exitval => exit-code-from}/docker-compose.yml (100%) diff --git a/compose/cli/main.py b/compose/cli/main.py index bbd6952a0..f1d343f3a 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -854,23 +854,20 @@ class TopLevelCommand(object): running. (default: 10) --remove-orphans Remove containers for services not defined in the Compose file - --forward-exitval SERVICE Return the exit value of the selected service container. + --exit-code-from SERVICE Return the exit code of the selected service container. Requires --abort-on-container-exit. """ start_deps = not options['--no-deps'] + exit_value_from = exitval_from_opts(options, self.project) cascade_stop = options['--abort-on-container-exit'] service_names = options['SERVICE'] timeout = timeout_from_opts(options) remove_orphans = options['--remove-orphans'] detached = options.get('-d') - forward_exitval = container_exitval_from_opts(options) if detached and cascade_stop: raise UserError("--abort-on-container-exit and -d cannot be combined.") - if forward_exitval and not cascade_stop: - raise UserError("--forward-exitval requires --abort-on-container-exit.") - with up_shutdown_context(self.project, service_names, timeout, detached): to_attach = self.project.up( service_names=service_names, @@ -884,11 +881,11 @@ class TopLevelCommand(object): if detached: return - all_containers = filter_containers_to_service_names(to_attach, service_names) + attached_containers = filter_containers_to_service_names(to_attach, service_names) log_printer = log_printer_from_project( self.project, - all_containers, + attached_containers, options['--no-color'], {'follow': True}, cascade_stop, @@ -899,22 +896,31 @@ class TopLevelCommand(object): if cascade_stop: print("Aborting on container exit...") self.project.stop(service_names=service_names, timeout=timeout) - if forward_exitval: - def is_us(container): - return container.name_without_project == forward_exitval - candidates = filter(is_us, all_containers) + if exit_value_from: + candidates = filter( + lambda c: c.service == exit_value_from, + attached_containers) if not candidates: log.error('No containers matching the spec "%s" were run.', - forward_exitval) + exit_value_from) sys.exit(2) if len(candidates) > 1: - log.error('Multiple (%d) containers matching the spec "%s" ' - 'were found; cannot forward exit code because we ' - 'do not know which one to.', len(candidates), - forward_exitval) - sys.exit(2) - exit_code = candidates[0].inspect()['State']['ExitCode'] - sys.exit(exit_code) + exit_values = [] + for candidate in candidates: + exit_val = candidate.inspect()['State']['ExitCode'] + if exit_val: + exit_values.append(exit_val) + if exit_values: + log.warn('Multiple (%d) containers matching the service name "%s" ' + 'were found and at least one exited nonzero; returning ' + 'the first non-zero exit code. See above for detailed ' + 'exit statuses.', len(candidates), exit_value_from) + sys.exit(exit_values[0]) + else: + exit_value = candidates[0].inspect()['State']['ExitCode'] + log.error('Returning exit value %d from container %s.', exit_value, + candidates[0].name_without_project) + sys.exit(exit_value) @classmethod def version(cls, options): @@ -947,32 +953,24 @@ def convergence_strategy_from_opts(options): return ConvergenceStrategy.changed -def container_exitval_from_opts(options): - """ Assemble a container name suitable for mapping into the - output of filter_containers_to_service_names. If the - container name ends in an underscore followed by a - positive integer, the user has deliberately specified - a container number and we believe her. Otherwise, append - `_1` to the name so as to return the exit value of the - first such named container. - """ - container_name = options.get('--forward-exitval') - if not container_name: - return None - segments = container_name.split('_') - if segments[-1].isdigit() and int(segments[-1]) > 0: - return '_'.join(segments) - else: - log.warn('"%s" does not specify a container number, ' - 'defaulting to "%s_1"', container_name, container_name) - return '_'.join(segments + ['1']) - - def timeout_from_opts(options): timeout = options.get('--timeout') return None if timeout is None else int(timeout) +def exitval_from_opts(options, project): + exit_value_from = options.get('--exit-code-from') + if exit_value_from: + if not options.get('--abort-on-container-exit'): + log.warn('using --exit-code-from implies --abort-on-container-exit') + options['--abort-on-container-exit'] = True + if exit_value_from not in [s.name for s in project.get_services()]: + log.error('No service named "%s" was found in your compose file.', + exit_value_from) + sys.exit(2) + return exit_value_from + + def image_type_from_opt(flag, value): if not value: return ImageType.none diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 979942f97..3acbb5806 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -467,7 +467,7 @@ _docker_compose_up() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--forward-exitval --abort-on-container-exit --build -d --force-recreate --help --no-build --no-color --no-deps --no-recreate --timeout -t --remove-orphans" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--exit-code-from --abort-on-container-exit --build -d --force-recreate --help --no-build --no-color --no-deps --no-recreate --timeout -t --remove-orphans" -- "$cur" ) ) ;; *) __docker_compose_services_all diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6e03c448c..42c4de384 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1929,11 +1929,11 @@ class CLITestCase(DockerClientTestCase): assert result.stdout.count("top") == 4 def test_forward_exitval(self): - self.base_dir = 'tests/fixtures/forward-exitval' + self.base_dir = 'tests/fixtures/exit-code-from' proc = start_process( self.base_dir, - ['up', '--abort-on-container-exit', '--forward-exitval', 'another']) + ['up', '--abort-on-container-exit', '--exit-code-from', 'another']) result = wait_on_process(proc, returncode=1) - assert 'forwardexitval_another_1 exited with code 1' in result.stdout + assert 'exitcodefrom_another_1 exited with code 1' in result.stdout diff --git a/tests/fixtures/forward-exitval/docker-compose.yml b/tests/fixtures/exit-code-from/docker-compose.yml similarity index 100% rename from tests/fixtures/forward-exitval/docker-compose.yml rename to tests/fixtures/exit-code-from/docker-compose.yml From 1f9fb2745673abd4e9562b9691f618484fed1714 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 18 Oct 2016 11:31:02 +0100 Subject: [PATCH 48/92] Allowing running containers to be rm'd by stop flag Signed-off-by: David McKay --- compose/cli/main.py | 10 ++++++++++ tests/acceptance/cli_test.py | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index 51ba36a09..49877eb70 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -635,6 +635,7 @@ class TopLevelCommand(object): Options: -f, --force Don't ask to confirm removal + -s, --stop Stop the containers, if required, before removing -v Remove any anonymous volumes attached to containers -a, --all Deprecated - no effect. """ @@ -645,6 +646,15 @@ class TopLevelCommand(object): ) one_off = OneOffFilter.include + if options.get('--stop'): + running_containers = self.project.containers( + service_names=options['SERVICE'], stopped=False, one_off=one_off + ) + self.project.stop( + service_names=running_containers, + one_off=one_off + ) + all_containers = self.project.containers( service_names=options['SERVICE'], stopped=True, one_off=one_off ) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8366ca75e..6426e8cbf 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1498,6 +1498,11 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(service.containers(stopped=True)), 1) self.dispatch(['rm', '-f'], None) self.assertEqual(len(service.containers(stopped=True)), 0) + service = self.project.get_service('simple') + service.create_container() + self.dispatch(['rm', '-fs'], None) + simple = self.project.get_service('simple') + self.assertEqual(len(simple.containers()), 0) def test_rm_all(self): service = self.project.get_service('simple') From 11788ef0ff0cbdb3e0ae264e3123c677b189dc54 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 27 Feb 2017 18:22:47 -0800 Subject: [PATCH 49/92] Add support for expanded port syntax in 3.1 format Signed-off-by: Joffrey F --- compose/config/config.py | 36 ++++++- compose/config/config_schema_v3.1.json | 17 +++- compose/config/serialize.py | 14 ++- compose/config/types.py | 59 ++++++++++++ compose/service.py | 25 ++++- tests/acceptance/cli_test.py | 13 +++ .../ports-composefile/expanded-notation.yml | 15 +++ tests/unit/config/config_test.py | 93 +++++++++++++++---- tests/unit/config/types_test.py | 44 +++++++++ tests/unit/service_test.py | 21 +++++ 10 files changed, 308 insertions(+), 29 deletions(-) create mode 100644 tests/fixtures/ports-composefile/expanded-notation.yml diff --git a/compose/config/config.py b/compose/config/config.py index 4c9cf423b..bb7d18a1c 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -35,6 +35,7 @@ from .sort_services import sort_service_dicts from .types import parse_extra_hosts from .types import parse_restart_spec from .types import ServiceLink +from .types import ServicePort from .types import VolumeFromSpec from .types import VolumeSpec from .validation import match_named_volumes @@ -683,10 +684,25 @@ def process_service(service_config): service_dict[field] = to_list(service_dict[field]) service_dict = process_healthcheck(service_dict, service_config.name) + service_dict = process_ports(service_dict) return service_dict +def process_ports(service_dict): + if 'ports' not in service_dict: + return service_dict + + ports = [] + for port_definition in service_dict['ports']: + if isinstance(port_definition, ServicePort): + ports.append(port_definition) + else: + ports.extend(ServicePort.parse(port_definition)) + service_dict['ports'] = ports + return service_dict + + def process_depends_on(service_dict): if 'depends_on' in service_dict and not isinstance(service_dict['depends_on'], dict): service_dict['depends_on'] = dict([ @@ -864,7 +880,7 @@ def merge_service_dicts(base, override, version): md.merge_field(field, merge_path_mappings) for field in [ - 'ports', 'cap_add', 'cap_drop', 'expose', 'external_links', + 'cap_add', 'cap_drop', 'expose', 'external_links', 'security_opt', 'volumes_from', ]: md.merge_field(field, merge_unique_items_lists, default=[]) @@ -873,6 +889,7 @@ def merge_service_dicts(base, override, version): md.merge_field(field, merge_list_or_string) md.merge_field('logging', merge_logging, default={}) + merge_ports(md, base, override) for field in set(ALLOWED_KEYS) - set(md): md.merge_scalar(field) @@ -889,6 +906,23 @@ def merge_unique_items_lists(base, override): return sorted(set().union(base, override)) +def merge_ports(md, base, override): + def parse_sequence_func(seq): + acc = [] + for item in seq: + acc.extend(ServicePort.parse(item)) + return to_mapping(acc, 'merge_field') + + field = 'ports' + + if not md.needs_merge(field): + return + + merged = parse_sequence_func(md.base.get(field, [])) + merged.update(parse_sequence_func(md.override.get(field, []))) + md[field] = [item for item in sorted(merged.values())] + + def merge_build(output, base, override): def to_dict(service): build_config = service.get('build', {}) diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index 77c2d35cb..219ccdd48 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -168,8 +168,21 @@ "ports": { "type": "array", "items": { - "type": ["string", "number"], - "format": "ports" + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "required": ["target"], + "additionalProperties": false + } + ] }, "uniqueItems": true }, diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 46d283f08..58581f7cc 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -7,6 +7,7 @@ import yaml from compose.config import types from compose.config.config import V1 from compose.config.config import V2_1 +from compose.config.config import V3_1 def serialize_config_type(dumper, data): @@ -14,8 +15,14 @@ def serialize_config_type(dumper, data): return representer(data.repr()) +def serialize_dict_type(dumper, data): + return dumper.represent_dict(data.repr()) + + yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) +yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type) +yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type) def denormalize_config(config): @@ -102,7 +109,10 @@ def denormalize_service_dict(service_dict, version): service_dict['healthcheck']['timeout'] ) - if 'secrets' in service_dict: - service_dict['secrets'] = map(lambda s: s.repr(), service_dict['secrets']) + if 'ports' in service_dict and version != V3_1: + service_dict['ports'] = map( + lambda p: p.legacy_repr() if isinstance(p, types.ServicePort) else p, + service_dict['ports'] + ) return service_dict diff --git a/compose/config/types.py b/compose/config/types.py index 811e6c1fc..aa1edcf0b 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -9,6 +9,7 @@ import re from collections import namedtuple import six +from docker.utils.ports import build_port_bindings from ..const import COMPOSEFILE_V1 as V1 from .errors import ConfigurationError @@ -258,3 +259,61 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): return dict( [(k, v) for k, v in self._asdict().items() if v is not None] ) + + +class ServicePort(namedtuple('_ServicePort', 'target published protocol mode external_ip')): + + @classmethod + def parse(cls, spec): + if not isinstance(spec, dict): + result = [] + for k, v in build_port_bindings([spec]).items(): + if '/' in k: + target, proto = k.split('/', 1) + else: + target, proto = (k, None) + for pub in v: + if pub is None: + result.append( + cls(target, None, proto, None, None) + ) + elif isinstance(pub, tuple): + result.append( + cls(target, pub[1], proto, None, pub[0]) + ) + else: + result.append( + cls(target, pub, proto, None, None) + ) + return result + + return [cls( + spec.get('target'), + spec.get('published'), + spec.get('protocol'), + spec.get('mode'), + None + )] + + @property + def merge_field(self): + return (self.target, self.published) + + def repr(self): + return dict( + [(k, v) for k, v in self._asdict().items() if v is not None] + ) + + def legacy_repr(self): + return normalize_port_dict(self.repr()) + + +def normalize_port_dict(port): + return '{external_ip}{has_ext_ip}{published}{is_pub}{target}/{protocol}'.format( + published=port.get('published', ''), + is_pub=(':' if port.get('published') else ''), + target=port.get('target'), + protocol=port.get('protocol', 'tcp'), + external_ip=port.get('external_ip', ''), + has_ext_ip=(':' if port.get('external_ip') else ''), + ) diff --git a/compose/service.py b/compose/service.py index 0f0dc57dd..ef593d84d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -21,6 +21,7 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config.types import ServicePort from .config.types import VolumeSpec from .const import DEFAULT_TIMEOUT from .const import IS_WINDOWS_PLATFORM @@ -693,7 +694,7 @@ class Service(object): if 'ports' in container_options or 'expose' in self.options: container_options['ports'] = build_container_ports( - container_options, + formatted_ports(container_options.get('ports', [])), self.options) container_options['environment'] = merge_environment( @@ -747,7 +748,9 @@ class Service(object): host_config = self.client.create_host_config( links=self._get_links(link_to_self=one_off), - port_bindings=build_port_bindings(options.get('ports') or []), + port_bindings=build_port_bindings( + formatted_ports(options.get('ports', [])) + ), binds=options.get('binds'), volumes_from=self._get_volumes_from(), privileged=options.get('privileged', False), @@ -875,7 +878,10 @@ class Service(object): def specifies_host_port(self): def has_host_port(binding): - _, external_bindings = split_port(binding) + if isinstance(binding, dict): + external_bindings = binding.get('published') + else: + _, external_bindings = split_port(binding) # there are no external bindings if external_bindings is None: @@ -1214,12 +1220,21 @@ def format_environment(environment): return '{key}={value}'.format(key=key, value=value) return [format_env(*item) for item in environment.items()] + # Ports +def formatted_ports(ports): + result = [] + for port in ports: + if isinstance(port, ServicePort): + result.append(port.legacy_repr()) + else: + result.append(port) + return result -def build_container_ports(container_options, options): +def build_container_ports(container_ports, options): ports = [] - all_ports = container_options.get('ports', []) + options.get('expose', []) + all_ports = container_ports + options.get('expose', []) for port_range in all_ports: internal_range, _ = split_port(port_range) for port in internal_range: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8366ca75e..c1605aa76 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1759,6 +1759,19 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(get_port(3001), "0.0.0.0:49152") self.assertEqual(get_port(3002), "0.0.0.0:49153") + def test_expanded_port(self): + self.base_dir = 'tests/fixtures/ports-composefile' + self.dispatch(['-f', 'expanded-notation.yml', 'up', '-d']) + container = self.project.get_service('simple').get_container() + + def get_port(number): + result = self.dispatch(['port', 'simple', str(number)]) + return result.stdout.rstrip() + + self.assertEqual(get_port(3000), container.get_local_port(3000)) + self.assertEqual(get_port(3001), "0.0.0.0:49152") + self.assertEqual(get_port(3002), "0.0.0.0:49153") + def test_port_with_scale(self): self.base_dir = 'tests/fixtures/ports-composefile-scale' self.dispatch(['scale', 'simple=2'], None) diff --git a/tests/fixtures/ports-composefile/expanded-notation.yml b/tests/fixtures/ports-composefile/expanded-notation.yml new file mode 100644 index 000000000..46d587363 --- /dev/null +++ b/tests/fixtures/ports-composefile/expanded-notation.yml @@ -0,0 +1,15 @@ +version: '3.1' +services: + simple: + image: busybox:latest + command: top + ports: + - target: 3000 + - target: 3001 + published: 49152 + - target: 3002 + published: 49153 + protocol: tcp + - target: 3003 + published: 49154 + protocol: udp diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e5104f769..4348129c8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -10,6 +10,7 @@ from operator import itemgetter import py import pytest +import yaml from ...helpers import build_config_details from compose.config import config @@ -25,6 +26,7 @@ from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION from compose.config.serialize import denormalize_service_dict +from compose.config.serialize import serialize_config from compose.config.serialize import serialize_ns_time_value from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM @@ -1737,6 +1739,30 @@ class ConfigTest(unittest.TestCase): } } + def test_merge_mixed_ports(self): + base = { + 'image': 'busybox:latest', + 'command': 'top', + 'ports': [ + { + 'target': '1245', + 'published': '1245', + 'protocol': 'tcp', + } + ] + } + + override = { + 'ports': ['1245:1245/udp'] + } + + actual = config.merge_service_dicts(base, override, V3_1) + assert actual == { + 'image': 'busybox:latest', + 'command': 'top', + 'ports': [types.ServicePort('1245', '1245', 'udp', None, None)] + } + def test_merge_depends_on_no_override(self): base = { 'image': 'busybox', @@ -2210,7 +2236,10 @@ class InterpolationTest(unittest.TestCase): self.assertEqual(service_dicts[0], { 'name': 'web', 'image': 'alpine:latest', - 'ports': ['5643', '9999'], + 'ports': [ + types.ServicePort.parse('5643')[0], + types.ServicePort.parse('9999')[0] + ], 'command': 'true' }) @@ -2233,7 +2262,7 @@ class InterpolationTest(unittest.TestCase): { 'name': 'web', 'image': 'busybox', - 'ports': ['80:8000'], + 'ports': types.ServicePort.parse('80:8000'), 'labels': {'mylabel': 'myvalue'}, 'hostname': 'host-', 'command': '${ESCAPED}', @@ -2515,13 +2544,37 @@ class MergePortsTest(unittest.TestCase, MergeListsTest): base_config = ['10:8000', '9000'] override_config = ['20:8000'] + def merged_config(self): + return self.convert(self.base_config) | self.convert(self.override_config) + + def convert(self, port_config): + return set(config.merge_service_dicts( + {self.config_name: port_config}, + {self.config_name: []}, + DEFAULT_VERSION + )[self.config_name]) + def test_duplicate_port_mappings(self): service_dict = config.merge_service_dicts( {self.config_name: self.base_config}, {self.config_name: self.base_config}, DEFAULT_VERSION ) - assert set(service_dict[self.config_name]) == set(self.base_config) + assert set(service_dict[self.config_name]) == self.convert(self.base_config) + + def test_no_override(self): + service_dict = config.merge_service_dicts( + {self.config_name: self.base_config}, + {}, + DEFAULT_VERSION) + assert set(service_dict[self.config_name]) == self.convert(self.base_config) + + def test_no_base(self): + service_dict = config.merge_service_dicts( + {}, + {self.config_name: self.base_config}, + DEFAULT_VERSION) + assert set(service_dict[self.config_name]) == self.convert(self.base_config) class MergeNetworksTest(unittest.TestCase, MergeListsTest): @@ -3542,23 +3595,25 @@ class SerializeTest(unittest.TestCase): assert denormalized_service['healthcheck']['interval'] == '100s' assert denormalized_service['healthcheck']['timeout'] == '30s' - def test_denormalize_secrets(self): + def test_serialize_secrets(self): service_dict = { - 'name': 'web', 'image': 'example/web', 'secrets': [ - types.ServiceSecret('one', None, None, None, None), - types.ServiceSecret('source', 'target', '100', '200', 0o777), - ], + {'source': 'one'}, + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + } + ] } - denormalized_service = denormalize_service_dict(service_dict, V3_1) - assert secret_sort(denormalized_service['secrets']) == secret_sort([ - {'source': 'one'}, - { - 'source': 'source', - 'target': 'target', - 'uid': '100', - 'gid': '200', - 'mode': 0o777, - }, - ]) + config_dict = config.load(build_config_details({ + 'version': '3.1', + 'services': {'web': service_dict} + })) + + serialized_config = yaml.load(serialize_config(config_dict)) + serialized_service = serialized_config['services']['web'] + assert secret_sort(serialized_service['secrets']) == secret_sort(service_dict['secrets']) diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 114273520..22d7aa88a 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -7,6 +7,7 @@ from compose.config.config import V1 from compose.config.config import V2_0 from compose.config.errors import ConfigurationError from compose.config.types import parse_extra_hosts +from compose.config.types import ServicePort from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec @@ -41,6 +42,49 @@ def test_parse_extra_hosts_dict(): } +class TestServicePort(object): + def test_parse_dict(self): + data = { + 'target': 8000, + 'published': 8000, + 'protocol': 'udp', + 'mode': 'global', + } + ports = ServicePort.parse(data) + assert len(ports) == 1 + assert ports[0].repr() == data + + def test_parse_simple_target_port(self): + ports = ServicePort.parse(8000) + assert len(ports) == 1 + assert ports[0].target == '8000' + + def test_parse_complete_port_definition(self): + port_def = '1.1.1.1:3000:3000/udp' + ports = ServicePort.parse(port_def) + assert len(ports) == 1 + assert ports[0].repr() == { + 'target': '3000', + 'published': '3000', + 'external_ip': '1.1.1.1', + 'protocol': 'udp', + } + assert ports[0].legacy_repr() == port_def + + def test_parse_port_range(self): + ports = ServicePort.parse('25000-25001:4000-4001') + assert len(ports) == 2 + reprs = [p.repr() for p in ports] + assert { + 'target': '4000', + 'published': '25000' + } in reprs + assert { + 'target': '4001', + 'published': '25001' + } in reprs + + class TestVolumeSpec(object): def test_parse_volume_spec_only_one_path(self): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0a66e4f3e..4d81623ba 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -7,6 +7,7 @@ from docker.errors import APIError from .. import mock from .. import unittest +from compose.config.types import ServicePort from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import LABEL_CONFIG_HASH @@ -19,6 +20,7 @@ from compose.service import build_ulimits from compose.service import build_volume_binding from compose.service import BuildAction from compose.service import ContainerNetworkMode +from compose.service import formatted_ports from compose.service import get_container_data_volumes from compose.service import ImageType from compose.service import merge_volume_bindings @@ -778,6 +780,25 @@ class NetTestCase(unittest.TestCase): self.assertEqual(network_mode.service_name, service_name) +class ServicePortsTest(unittest.TestCase): + def test_formatted_ports(self): + ports = [ + '3000', + '0.0.0.0:4025-4030:23000-23005', + ServicePort(6000, None, None, None, None), + ServicePort(8080, 8080, None, None, None), + ServicePort('20000', '20000', 'udp', 'ingress', None), + ServicePort(30000, '30000', 'tcp', None, '127.0.0.1'), + ] + formatted = formatted_ports(ports) + assert ports[0] in formatted + assert ports[1] in formatted + assert '6000/tcp' in formatted + assert '8080:8080/tcp' in formatted + assert '20000:20000/udp' in formatted + assert '127.0.0.1:30000:30000/tcp' in formatted + + def build_mount(destination, source, mode='rw'): return {'Source': source, 'Destination': destination, 'Mode': mode} From d67261f26ec6eb128657f99b8dbb82a828298822 Mon Sep 17 00:00:00 2001 From: Evan Shaw Date: Thu, 2 Mar 2017 10:03:31 +1300 Subject: [PATCH 50/92] Add --volumes flag to config command Closes #3609 Signed-off-by: Evan Shaw --- compose/cli/main.py | 5 +++++ tests/acceptance/cli_test.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index 51ba36a09..4c3695504 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -301,6 +301,7 @@ class TopLevelCommand(object): -q, --quiet Only validate the configuration, don't print anything. --services Print the service names, one per line. + --volumes Print the volume names, one per line. """ compose_config = get_config_from_options(self.project_dir, config_options) @@ -312,6 +313,10 @@ class TopLevelCommand(object): print('\n'.join(service['name'] for service in compose_config.services)) return + if options['--volumes']: + print('\n'.join(volume for volume in compose_config.volumes)) + return + print(serialize_config(compose_config)) def create(self, options): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8366ca75e..ee895b409 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -177,6 +177,11 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['config', '--services']) assert set(result.stdout.rstrip().split('\n')) == {'web', 'other'} + def test_config_list_volumes(self): + self.base_dir = 'tests/fixtures/v2-full' + result = self.dispatch(['config', '--volumes']) + assert set(result.stdout.rstrip().split('\n')) == {'data'} + def test_config_quiet_with_error(self): self.base_dir = None result = self.dispatch([ From 6c3641fdb7c757980dcd04b44d0a573536e149b8 Mon Sep 17 00:00:00 2001 From: kbroadwater Date: Fri, 10 Feb 2017 10:35:08 -0800 Subject: [PATCH 51/92] Returing 1 when a container exits with a non-zero exit code with --abort-on-container-exit is set. Signed-off-by: Kevin Broadwater Catching the first container to exit Signed-off-by: Kevin Broadwater Addressing feedback and fixing tests Signed-off-by: Kevin Broadwater Adding break and removing extra fixture files Signed-off-by: Kevin Broadwater Moving break Signed-off-by: Kevin Broadwater --- compose/cli/log_printer.py | 21 +++++++++++-------- compose/cli/main.py | 9 +++++++- tests/acceptance/cli_test.py | 14 ++++++++++--- .../docker-compose.yml | 6 ++++++ .../docker-compose.yml | 6 ++++++ tests/unit/cli/log_printer_test.py | 6 ++++-- 6 files changed, 47 insertions(+), 15 deletions(-) create mode 100644 tests/fixtures/abort-on-container-exit-0/docker-compose.yml create mode 100644 tests/fixtures/abort-on-container-exit-1/docker-compose.yml diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 299ddea46..043d3d068 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -87,6 +87,13 @@ class LogPrinter(object): for line in consume_queue(queue, self.cascade_stop): remove_stopped_threads(thread_map) + if self.cascade_stop: + matching_container = [cont.name for cont in self.containers if cont.name == line] + if line in matching_container: + # Returning the name of the container that started the + # the cascade_stop so we can return the correct exit code + return line + if not line: if not thread_map: # There are no running containers left to tail, so exit @@ -132,8 +139,8 @@ class QueueItem(namedtuple('_QueueItem', 'item is_stop exc')): return cls(None, None, exc) @classmethod - def stop(cls): - return cls(None, True, None) + def stop(cls, item=None): + return cls(item, True, None) def tail_container_logs(container, presenter, queue, log_args): @@ -145,10 +152,9 @@ def tail_container_logs(container, presenter, queue, log_args): except Exception as e: queue.put(QueueItem.exception(e)) return - if log_args.get('follow'): queue.put(QueueItem.new(presenter.color_func(wait_on_exit(container)))) - queue.put(QueueItem.stop()) + queue.put(QueueItem.stop(container.name)) def get_log_generator(container): @@ -228,10 +234,7 @@ def consume_queue(queue, cascade_stop): if item.exc: raise item.exc - if item.is_stop: - if cascade_stop: - raise StopIteration - else: - continue + if item.is_stop and not cascade_stop: + continue yield item.item diff --git a/compose/cli/main.py b/compose/cli/main.py index d0cf03cba..f5d2429f3 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -890,11 +890,18 @@ class TopLevelCommand(object): cascade_stop, event_stream=self.project.events(service_names=service_names)) print("Attaching to", list_containers(log_printer.containers)) - log_printer.run() + cascade_starter = log_printer.run() if cascade_stop: print("Aborting on container exit...") + exit_code = 0 + for e in self.project.containers(service_names=options['SERVICE'], stopped=True): + if (not e.is_running and cascade_starter == e.name): + if not e.exit_code == 0: + exit_code = e.exit_code + break self.project.stop(service_names=service_names, timeout=timeout) + sys.exit(exit_code) @classmethod def version(cls, options): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 417eadb5c..9379a6c31 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1085,10 +1085,18 @@ class CLITestCase(DockerClientTestCase): wait_on_condition(ContainerCountCondition(self.project, 0)) def test_up_handles_abort_on_container_exit(self): - start_process(self.base_dir, ['up', '--abort-on-container-exit']) - wait_on_condition(ContainerCountCondition(self.project, 2)) - self.project.stop(['simple']) + self.base_dir = 'tests/fixtures/abort-on-container-exit-0' + proc = start_process(self.base_dir, ['up', '--abort-on-container-exit']) wait_on_condition(ContainerCountCondition(self.project, 0)) + proc.wait() + self.assertEqual(proc.returncode, 0) + + def test_up_handles_abort_on_container_exit_code(self): + self.base_dir = 'tests/fixtures/abort-on-container-exit-1' + proc = start_process(self.base_dir, ['up', '--abort-on-container-exit']) + wait_on_condition(ContainerCountCondition(self.project, 0)) + proc.wait() + self.assertEqual(proc.returncode, 1) def test_exec_without_tty(self): self.base_dir = 'tests/fixtures/links-composefile' diff --git a/tests/fixtures/abort-on-container-exit-0/docker-compose.yml b/tests/fixtures/abort-on-container-exit-0/docker-compose.yml new file mode 100644 index 000000000..ce41697bc --- /dev/null +++ b/tests/fixtures/abort-on-container-exit-0/docker-compose.yml @@ -0,0 +1,6 @@ +simple: + image: busybox:latest + command: top +another: + image: busybox:latest + command: ls . diff --git a/tests/fixtures/abort-on-container-exit-1/docker-compose.yml b/tests/fixtures/abort-on-container-exit-1/docker-compose.yml new file mode 100644 index 000000000..7ec9b7e11 --- /dev/null +++ b/tests/fixtures/abort-on-container-exit-1/docker-compose.yml @@ -0,0 +1,6 @@ +simple: + image: busybox:latest + command: top +another: + image: busybox:latest + command: ls /thecakeisalie diff --git a/tests/unit/cli/log_printer_test.py b/tests/unit/cli/log_printer_test.py index b908eb68b..d0c4b56be 100644 --- a/tests/unit/cli/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -187,11 +187,13 @@ class TestConsumeQueue(object): assert next(generator) == 'b' def test_item_is_stop_with_cascade_stop(self): + """Return the name of the container that caused the cascade_stop""" queue = Queue() - for item in QueueItem.stop(), QueueItem.new('a'), QueueItem.new('b'): + for item in QueueItem.stop('foobar-1'), QueueItem.new('a'), QueueItem.new('b'): queue.put(item) - assert list(consume_queue(queue, True)) == [] + generator = consume_queue(queue, True) + assert next(generator) is 'foobar-1' def test_item_is_none_when_timeout_is_hit(self): queue = Queue() From 33f510b3409a55fd7a4e9050853e14cbd853d18e Mon Sep 17 00:00:00 2001 From: Jesus Tinoco Date: Sat, 11 Jun 2016 01:12:02 +0200 Subject: [PATCH 52/92] 3501 - Add a new command option (images) Signed-off-by: Jesus Rodriguez Tinoco --- compose/cli/main.py | 38 ++++++++++++++++++++++++++ contrib/completion/bash/docker-compose | 10 +++++++ tests/acceptance/cli_test.py | 27 ++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index a7aec945e..ba053e233 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -175,6 +175,7 @@ class TopLevelCommand(object): events Receive real time events from containers exec Execute a command in a running container help Get help on a command + images List images kill Kill containers logs View output from containers pause Pause services @@ -481,6 +482,43 @@ class TopLevelCommand(object): print(getdoc(subject)) + def images(self, options): + """ + List images. + Usage: images [options] [SERVICE...] + + Options: + -q Only display IDs + """ + containers = sorted( + self.project.containers(service_names=options['SERVICE'], stopped=True) + + self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only), + key=attrgetter('name')) + + if options['-q']: + for container in containers: + print(str.split(str(container.image), ':')[1]) + else: + headers = [ + 'Repository', + 'Tag', + 'Image Id', + 'Size' + ] + rows = [] + for container in containers: + image_config = container.image_config + repo_tags = str.split(str(image_config['RepoTags'][0]), ':') + image_id = str.split(str(container.image), ':')[1][0:12] + size = round(int(image_config['Size'])/float(1 << 20), 1) + rows.append([ + repo_tags[0], + repo_tags[1], + image_id, + size + ]) + print(Formatter().table(headers, rows)) + def kill(self, options): """ Force stop service containers. diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index f4b9342f3..fa099eac4 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -220,6 +220,16 @@ _docker_compose_help() { COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) } +_docker_compose_images() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help -q" -- "$cur" ) ) + ;; + *) + __docker_compose_services_all + ;; + esac +} _docker_compose_kill() { case "$prev" in diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index cc7bc5dfe..26baf3377 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1986,3 +1986,30 @@ class CLITestCase(DockerClientTestCase): result = wait_on_process(proc, returncode=1) assert 'exitcodefrom_another_1 exited with code 1' in result.stdout + + def test_images(self): + self.project.get_service('simple').create_container() + result = self.dispatch(['images']) + assert 'simplecomposefile_simple' in result.stdout + + def test_images_default_composefile(self): + self.base_dir = 'tests/fixtures/multiple-composefiles' + self.dispatch(['up', '-d']) + result = self.dispatch(['images']) + + self.assertIn('multiplecomposefiles_simple', result.stdout) + self.assertIn('multiplecomposefiles_another', result.stdout) + self.assertNotIn('multiplecomposefiles_yetanother', result.stdout) + + def test_images_alternate_composefile(self): + config_path = os.path.abspath( + 'tests/fixtures/multiple-composefiles/compose2.yml') + self._project = get_project(self.base_dir, [config_path]) + + self.base_dir = 'tests/fixtures/multiple-composefiles' + self.dispatch(['-f', 'compose2.yml', 'up', '-d']) + result = self.dispatch(['-f', 'compose2.yml', 'images']) + + self.assertNotIn('multiplecomposefiles_simple', result.stdout) + self.assertNotIn('multiplecomposefiles_another', result.stdout) + self.assertIn('multiplecomposefiles_yetanother', result.stdout) From 90beeaf21c15de1c0da55eb1abc22820450c3ab8 Mon Sep 17 00:00:00 2001 From: Jesus Tinoco Date: Sat, 11 Jun 2016 02:10:52 +0200 Subject: [PATCH 53/92] 3501 - Remove a test and fix other Signed-off-by: Jesus Rodriguez Tinoco --- tests/acceptance/cli_test.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 26baf3377..ef907d541 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1990,26 +1990,11 @@ class CLITestCase(DockerClientTestCase): def test_images(self): self.project.get_service('simple').create_container() result = self.dispatch(['images']) - assert 'simplecomposefile_simple' in result.stdout + assert 'busybox' in result.stdout def test_images_default_composefile(self): self.base_dir = 'tests/fixtures/multiple-composefiles' self.dispatch(['up', '-d']) result = self.dispatch(['images']) - self.assertIn('multiplecomposefiles_simple', result.stdout) - self.assertIn('multiplecomposefiles_another', result.stdout) - self.assertNotIn('multiplecomposefiles_yetanother', result.stdout) - - def test_images_alternate_composefile(self): - config_path = os.path.abspath( - 'tests/fixtures/multiple-composefiles/compose2.yml') - self._project = get_project(self.base_dir, [config_path]) - - self.base_dir = 'tests/fixtures/multiple-composefiles' - self.dispatch(['-f', 'compose2.yml', 'up', '-d']) - result = self.dispatch(['-f', 'compose2.yml', 'images']) - - self.assertNotIn('multiplecomposefiles_simple', result.stdout) - self.assertNotIn('multiplecomposefiles_another', result.stdout) - self.assertIn('multiplecomposefiles_yetanother', result.stdout) + self.assertIn('busybox', result.stdout) From 815a3af6d21b12a6b16fa52db85eec354d27626d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Mar 2017 14:59:36 -0800 Subject: [PATCH 54/92] Fix docstring for images command Signed-off-by: Joffrey F --- compose/cli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index ba053e233..894dd9abb 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -484,7 +484,7 @@ class TopLevelCommand(object): def images(self, options): """ - List images. + List images used by the created containers. Usage: images [options] [SERVICE...] Options: @@ -510,7 +510,7 @@ class TopLevelCommand(object): image_config = container.image_config repo_tags = str.split(str(image_config['RepoTags'][0]), ':') image_id = str.split(str(container.image), ':')[1][0:12] - size = round(int(image_config['Size'])/float(1 << 20), 1) + size = round(int(image_config['Size']) / float(1 << 20), 1) rows.append([ repo_tags[0], repo_tags[1], From 11329e779bb986709add9eabfa90f28920a322ec Mon Sep 17 00:00:00 2001 From: stefan Date: Thu, 30 Jun 2016 14:07:57 +0200 Subject: [PATCH 55/92] added failing test that ensures that named volume will not be printed with a mode suffix Signed-off-by: stefan --- tests/acceptance/cli_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 tests/acceptance/cli_test.py diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py old mode 100644 new mode 100755 index cc7bc5dfe..7cc5adb92 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -222,7 +222,7 @@ class CLITestCase(DockerClientTestCase): 'other': { 'image': 'busybox:latest', 'command': 'top', - 'volumes': ['/data:rw'], + 'volumes': ['/data'], }, }, } From 6a151aac046b18f4fe2ac46f79dc9c8ee83aa33a Mon Sep 17 00:00:00 2001 From: Yaroslav Molochko Date: Wed, 23 Nov 2016 12:12:57 -0800 Subject: [PATCH 56/92] introducing pids_limit, fix for #4178 Signed-off-by: Yaroslav Molochko --- compose/config/config.py | 1 + compose/config/config_schema_v2.1.json | 1 + compose/config/config_schema_v3.0.json | 1 + compose/service.py | 2 ++ tests/integration/service_test.py | 7 +++++++ 5 files changed, 12 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index 003b2e2f4..a106a758a 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -87,6 +87,7 @@ DOCKER_CONFIG_KEYS = [ 'secrets', 'security_opt', 'shm_size', + 'pids_limit', 'stdin_open', 'stop_signal', 'sysctls', diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index d1ffff89a..0f4455a7d 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -216,6 +216,7 @@ "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "shm_size": {"type": ["number", "string"]}, "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "pids_limit": {"type": ["number", "string"]}, "stdin_open": {"type": "boolean"}, "stop_grace_period": {"type": "string", "format": "duration"}, "stop_signal": {"type": "string"}, diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index fbcd8bb85..02a4e851d 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -168,6 +168,7 @@ "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "shm_size": {"type": ["number", "string"]}, "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "pids_limit": {"type": ["number", "string"]}, "stdin_open": {"type": "boolean"}, "stop_grace_period": {"type": "string", "format": "duration"}, "stop_signal": {"type": "string"}, diff --git a/compose/service.py b/compose/service.py index b42094e68..c924f9ece 100644 --- a/compose/service.py +++ b/compose/service.py @@ -66,6 +66,7 @@ DOCKER_START_KEYS = [ 'oom_score_adj', 'mem_swappiness', 'pid', + 'pids_limit', 'privileged', 'restart', 'security_opt', @@ -772,6 +773,7 @@ class Service(object): cpu_quota=options.get('cpu_quota'), shm_size=options.get('shm_size'), sysctls=options.get('sysctls'), + pids_limit=options.get('pids_limit'), tmpfs=options.get('tmpfs'), oom_score_adj=options.get('oom_score_adj'), mem_swappiness=options.get('mem_swappiness'), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 734da5dfa..5e3815011 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -115,6 +115,13 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(container.get('HostConfig.ShmSize'), 67108864) + def test_create_container_with_pids_limit(self): + self.require_api_version('1.23') + service = self.create_service('db', pids_limit=10) + container = service.create_container() + service.start_container(container) + self.assertEqual(container.get('HostConfig.PidsLimit'), 10) + def test_create_container_with_extra_hosts_list(self): extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) From 83388ec31ae0e81fec0c6904f74e04345f1fb069 Mon Sep 17 00:00:00 2001 From: Piotr Szymanski Date: Tue, 18 Oct 2016 19:05:32 +0200 Subject: [PATCH 57/92] enable -v flag for docker-compose run command Give user ability to attach volumes while running containers with docker-compose run command. Example is given in the test implementation, command is compatible with the one provided by docker engine. Signed-off-by: Piotr Szymanski --- compose/cli/main.py | 8 ++++- tests/acceptance/cli_test.py | 34 +++++++++++++++++++ .../docker-compose.yml | 2 ++ .../files/example.txt | 1 + tests/unit/cli_test.py | 3 ++ 5 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/simple-composefile-volume-ready/docker-compose.yml create mode 100644 tests/fixtures/simple-composefile-volume-ready/files/example.txt diff --git a/compose/cli/main.py b/compose/cli/main.py index a7aec945e..a16448293 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -24,6 +24,7 @@ from ..config import ConfigurationError from ..config import parse_environment from ..config.environment import Environment from ..config.serialize import serialize_config +from ..config.types import VolumeSpec from ..const import IS_WINDOWS_PLATFORM from ..errors import StreamParseError from ..progress_stream import StreamOutputError @@ -678,7 +679,7 @@ class TopLevelCommand(object): running. If you do not want to start linked services, use `docker-compose run --no-deps SERVICE COMMAND [ARGS...]`. - Usage: run [options] [-p PORT...] [-e KEY=VAL...] SERVICE [COMMAND] [ARGS...] + Usage: run [options] [-v VOLUME...] [-p PORT...] [-e KEY=VAL...] SERVICE [COMMAND] [ARGS...] Options: -d Detached mode: Run container in the background, print @@ -692,6 +693,7 @@ class TopLevelCommand(object): -p, --publish=[] Publish a container's port(s) to the host --service-ports Run command with the service's ports enabled and mapped to the host. + -v, --volume=[] Bind mount a volume (default []) -T Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY. -w, --workdir="" Working directory inside the container @@ -1035,6 +1037,10 @@ def build_container_options(options, detach, command): if options['--workdir']: container_options['working_dir'] = options['--workdir'] + if options['--volume']: + volumes = [VolumeSpec.parse(i) for i in options['--volume']] + container_options['volumes'] = volumes + return container_options diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index cc7bc5dfe..0ddada5a0 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import datetime import json import os +import os.path import signal import subprocess import time @@ -557,6 +558,39 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(old_ids, new_ids) + def test_run_one_off_with_volume(self): + self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' + volume_path = os.path.abspath(os.path.join(os.getcwd(), self.base_dir, 'files')) + cmd_result = self.dispatch([ + 'run', + '-v', '{}:/data'.format(volume_path), + 'simple', + 'cat', '/data/example.txt' + ]) + assert cmd_result.stdout.strip() == 'FILE_CONTENT' + + def test_run_one_off_with_multiple_volumes(self): + self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' + volume_path = os.path.abspath(os.path.join(os.getcwd(), self.base_dir, 'files')) + + cmd_result = self.dispatch([ + 'run', + '-v', '{}:/data'.format(volume_path), + '-v', '{}:/data1'.format(volume_path), + 'simple', + 'cat', '/data/example.txt' + ]) + assert cmd_result.stdout.strip() == 'FILE_CONTENT' + + cmd_result = self.dispatch([ + 'run', + '-v', '{}:/data'.format(volume_path), + '-v', '{}:/data1'.format(volume_path), + 'simple', + 'cat', '/data1/example.txt' + ]) + assert cmd_result.stdout.strip() == 'FILE_CONTENT' + def test_create_with_force_recreate_and_no_recreate(self): self.dispatch( ['create', '--force-recreate', '--no-recreate'], diff --git a/tests/fixtures/simple-composefile-volume-ready/docker-compose.yml b/tests/fixtures/simple-composefile-volume-ready/docker-compose.yml new file mode 100644 index 000000000..98a7d23b7 --- /dev/null +++ b/tests/fixtures/simple-composefile-volume-ready/docker-compose.yml @@ -0,0 +1,2 @@ +simple: + image: busybox:latest diff --git a/tests/fixtures/simple-composefile-volume-ready/files/example.txt b/tests/fixtures/simple-composefile-volume-ready/files/example.txt new file mode 100644 index 000000000..edb4d3390 --- /dev/null +++ b/tests/fixtures/simple-composefile-volume-ready/files/example.txt @@ -0,0 +1 @@ +FILE_CONTENT diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index fb0511f0a..f9ce240a3 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -119,6 +119,7 @@ class CLITestCase(unittest.TestCase): '--entrypoint': None, '--service-ports': None, '--publish': [], + '--volume': [], '--rm': None, '--name': None, '--workdir': None, @@ -153,6 +154,7 @@ class CLITestCase(unittest.TestCase): '--entrypoint': None, '--service-ports': None, '--publish': [], + '--volume': [], '--rm': None, '--name': None, '--workdir': None, @@ -175,6 +177,7 @@ class CLITestCase(unittest.TestCase): '--entrypoint': None, '--service-ports': None, '--publish': [], + '--volume': [], '--rm': True, '--name': None, '--workdir': None, From 44653f28126b731d9bc7390a0eeae66b7bc19a78 Mon Sep 17 00:00:00 2001 From: Marcos Lilljedahl Date: Fri, 19 Aug 2016 03:04:46 -0300 Subject: [PATCH 58/92] Improve Dockerfile.run This dockerfile generates a more lightweight image that works with the current official dynamically generated binaries Signed-off-by: Marcos Lilljedahl Change image script to use new Dockerfile.run image without building compose Signed-off-by: Marcos Lilljedahl Apply suggested fixes Signed-off-by: Marcos Lilljedahl --- Dockerfile.run | 20 ++++++++++---------- script/build/image | 4 +++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Dockerfile.run b/Dockerfile.run index de46e35e5..5d246e9e6 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,14 +1,14 @@ +FROM alpine:3.4 -FROM alpine:3.4 -ARG version -RUN apk -U add \ - python \ - py-pip +ENV GLIBC 2.23-r3 -COPY requirements.txt /code/requirements.txt -RUN pip install -r /code/requirements.txt +RUN apk update && apk add --no-cache openssl ca-certificates && \ + wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \ + wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ + apk add --no-cache glibc-$GLIBC.apk && rm glibc-$GLIBC.apk && \ + ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ + ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib -COPY dist/docker_compose-${version}-py2.py3-none-any.whl /code/ -RUN pip install --no-deps /code/docker_compose-${version}-py2.py3-none-any.whl +COPY dist/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose -ENTRYPOINT ["/usr/bin/docker-compose"] +ENTRYPOINT ["docker-compose"] diff --git a/script/build/image b/script/build/image index 3590ce14e..a3198c99f 100755 --- a/script/build/image +++ b/script/build/image @@ -8,8 +8,10 @@ if [ -z "$1" ]; then fi TAG=$1 + VERSION="$(python setup.py --version)" ./script/build/write-git-sha python setup.py sdist bdist_wheel -docker build --build-arg version=$VERSION -t docker/compose:$TAG -f Dockerfile.run . +./script/build/linux +docker build -t docker/compose:$TAG -f Dockerfile.run . From c7b8278e78197b87399977c8a512ef72631816a3 Mon Sep 17 00:00:00 2001 From: George Lester Date: Wed, 13 Jul 2016 02:05:08 -0700 Subject: [PATCH 59/92] Implemented dns_opt Signed-off-by: George Lester --- compose/config/config.py | 1 + compose/config/config_schema_v2.0.json | 7 +++++++ compose/service.py | 3 ++- tests/integration/service_test.py | 8 ++++++++ tests/unit/config/config_test.py | 20 +++++++++++++++++++- 5 files changed, 37 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 003b2e2f4..72d2e8e43 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -61,6 +61,7 @@ DOCKER_CONFIG_KEYS = [ 'devices', 'dns', 'dns_search', + 'dns_opt', 'domainname', 'entrypoint', 'env_file', diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 59c7b30c9..da0105aea 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -81,6 +81,13 @@ "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "dns": {"$ref": "#/definitions/string_or_list"}, "dns_search": {"$ref": "#/definitions/string_or_list"}, + "dns_opt": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, "domainname": {"type": "string"}, "entrypoint": { "oneOf": [ diff --git a/compose/service.py b/compose/service.py index b42094e68..0b6200dd5 100644 --- a/compose/service.py +++ b/compose/service.py @@ -54,6 +54,7 @@ DOCKER_START_KEYS = [ 'devices', 'dns', 'dns_search', + 'dns_opt', 'env_file', 'extra_hosts', 'group_add', @@ -755,7 +756,7 @@ class Service(object): network_mode=self.network_mode.mode, devices=options.get('devices'), dns=options.get('dns'), - dns_search=options.get('dns_search'), + dns_opt=options.get('dns_opt'), restart_policy=options.get('restart'), cap_add=options.get('cap_add'), cap_drop=options.get('cap_drop'), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 734da5dfa..f3dc346d1 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -890,6 +890,14 @@ class ServiceTest(DockerClientTestCase): self.assertTrue("root" in host_container_groupadd) self.assertTrue("1" in host_container_groupadd) + def test_dns_opt_value(self): + service = self.create_service('web', dns_opt=["use-vc", "no-tld-query"]) + container = create_and_start_container(service) + + dns_opt = container.get('HostConfig.DNSOptions') + self.assertTrue("use-vc" in dns_opt) + self.assertTrue("no-tld-query" in dns_opt) + def test_restart_on_failure_value(self): service = self.create_service('web', restart={ 'Name': 'on-failure', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index aaa7fbf8d..728206d57 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1411,7 +1411,6 @@ class ConfigTest(unittest.TestCase): ] def test_group_add_option(self): - actual = config.load(build_config_details({ 'version': '2', 'services': { @@ -1430,6 +1429,25 @@ class ConfigTest(unittest.TestCase): } ] + def test_dns_opt_option(self): + actual = config.load(build_config_details({ + 'version': '2', + 'services': { + 'web': { + 'image': 'alpine', + 'dns_opt': ["use-vc", "no-tld-query"] + } + } + })) + + assert actual.services == [ + { + 'name': 'web', + 'image': 'alpine', + 'dns_opt': ["use-vc", "no-tld-query"] + } + ] + def test_isolation_option(self): actual = config.load(build_config_details({ 'version': V2_1, From 7512dccaa85e1c891aca12c87fe0647676e34890 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Mar 2017 15:40:10 -0800 Subject: [PATCH 60/92] Add dns_opt to 2.1 schema Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 14 +++++++------- compose/config/config_schema_v2.1.json | 7 +++++++ compose/service.py | 1 + tests/integration/service_test.py | 10 +++++----- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index da0105aea..d20a0d89a 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -80,14 +80,14 @@ "depends_on": {"$ref": "#/definitions/list_of_strings"}, "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_opt": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, "dns_search": {"$ref": "#/definitions/string_or_list"}, - "dns_opt": { - "type": "array", - "items": { - "type": "string" - }, - "uniqueItems": true - }, "domainname": {"type": "string"}, "entrypoint": { "oneOf": [ diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index d1ffff89a..3b01ddf69 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -100,6 +100,13 @@ ] }, "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns_opt": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, "dns": {"$ref": "#/definitions/string_or_list"}, "dns_search": {"$ref": "#/definitions/string_or_list"}, "domainname": {"type": "string"}, diff --git a/compose/service.py b/compose/service.py index 0b6200dd5..353c96af4 100644 --- a/compose/service.py +++ b/compose/service.py @@ -757,6 +757,7 @@ class Service(object): devices=options.get('devices'), dns=options.get('dns'), dns_opt=options.get('dns_opt'), + dns_search=options.get('dns_search'), restart_policy=options.get('restart'), cap_add=options.get('cap_add'), cap_drop=options.get('cap_drop'), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index f3dc346d1..910f2c69c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -887,16 +887,16 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) host_container_groupadd = container.get('HostConfig.GroupAdd') - self.assertTrue("root" in host_container_groupadd) - self.assertTrue("1" in host_container_groupadd) + assert "root" in host_container_groupadd + assert "1" in host_container_groupadd def test_dns_opt_value(self): service = self.create_service('web', dns_opt=["use-vc", "no-tld-query"]) container = create_and_start_container(service) - dns_opt = container.get('HostConfig.DNSOptions') - self.assertTrue("use-vc" in dns_opt) - self.assertTrue("no-tld-query" in dns_opt) + dns_opt = container.get('HostConfig.DnsOptions') + assert 'use-vc' in dns_opt + assert 'no-tld-query' in dns_opt def test_restart_on_failure_value(self): service = self.create_service('web', restart={ From b6fb3f263421ebb5b8ec685665b17063b1fc2982 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Mar 2017 16:47:02 -0800 Subject: [PATCH 61/92] pids_limit not yet supported for swarm services - removing from v3 format Signed-off-by: Joffrey F --- compose/config/config_schema_v3.0.json | 1 - tests/integration/service_test.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 02a4e851d..fbcd8bb85 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -168,7 +168,6 @@ "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "shm_size": {"type": ["number", "string"]}, "sysctls": {"$ref": "#/definitions/list_or_dict"}, - "pids_limit": {"type": ["number", "string"]}, "stdin_open": {"type": "boolean"}, "stop_grace_period": {"type": "string", "format": "duration"}, "stop_signal": {"type": "string"}, diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 5e3815011..436d2db10 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -115,12 +115,13 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(container.get('HostConfig.ShmSize'), 67108864) + @pytest.mark.xfail(True, reason='Some kernels/configs do not support pids_limit') def test_create_container_with_pids_limit(self): self.require_api_version('1.23') service = self.create_service('db', pids_limit=10) container = service.create_container() service.start_container(container) - self.assertEqual(container.get('HostConfig.PidsLimit'), 10) + assert container.get('HostConfig.PidsLimit') == 10 def test_create_container_with_extra_hosts_list(self): extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] From 83728d2bcc3b56648ee53b177e1ad1ad1d586e17 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Mar 2017 15:27:04 -0800 Subject: [PATCH 62/92] Do not add mode in volume representation if it's not a host binding Signed-off-by: Joffrey F --- compose/config/types.py | 3 ++- tests/acceptance/cli_test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) mode change 100755 => 100644 tests/acceptance/cli_test.py diff --git a/compose/config/types.py b/compose/config/types.py index 811e6c1fc..f4d2c26d9 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -203,7 +203,8 @@ class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): def repr(self): external = self.external + ':' if self.external else '' - return '{ext}{v.internal}:{v.mode}'.format(ext=external, v=self) + mode = ':' + self.mode if self.external else '' + return '{ext}{v.internal}{mode}'.format(mode=mode, ext=external, v=self) @property def is_named_volume(self): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py old mode 100755 new mode 100644 index 7cc5adb92..115dc6439 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -299,7 +299,7 @@ class CLITestCase(DockerClientTestCase): }, 'volume': { 'image': 'busybox', - 'volumes': ['/data:rw'], + 'volumes': ['/data'], 'network_mode': 'bridge', }, 'app': { From 8f8678987b4496384452143b6eeec88d51b14510 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 6 Mar 2017 15:56:41 -0800 Subject: [PATCH 63/92] Improve readability of code and output for the images command Signed-off-by: Joffrey F --- compose/cli/main.py | 13 ++++++++----- compose/cli/utils.py | 13 +++++++++++++ tests/acceptance/cli_test.py | 5 ++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 894dd9abb..98fc4e451 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -47,6 +47,7 @@ from .formatter import Formatter from .log_printer import build_log_presenters from .log_printer import LogPrinter from .utils import get_version_info +from .utils import human_readable_file_size from .utils import yesno @@ -496,10 +497,11 @@ class TopLevelCommand(object): key=attrgetter('name')) if options['-q']: - for container in containers: - print(str.split(str(container.image), ':')[1]) + for image in set(c.image for c in containers): + print(image.split(':')[1]) else: headers = [ + 'Container', 'Repository', 'Tag', 'Image Id', @@ -508,10 +510,11 @@ class TopLevelCommand(object): rows = [] for container in containers: image_config = container.image_config - repo_tags = str.split(str(image_config['RepoTags'][0]), ':') - image_id = str.split(str(container.image), ':')[1][0:12] - size = round(int(image_config['Size']) / float(1 << 20), 1) + repo_tags = image_config['RepoTags'][0].split(':') + image_id = image_config['Id'].split(':')[1][:12] + size = human_readable_file_size(image_config['Size']) rows.append([ + container.name, repo_tags[0], repo_tags[1], image_id, diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 580bd1b07..4d4fc4c18 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals +import math import os import platform import ssl @@ -135,3 +136,15 @@ def unquote_path(s): if s[0] == '"' and s[-1] == '"': return s[1:-1] return s + + +def human_readable_file_size(size): + suffixes = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', ] + order = int(math.log(size, 2) / 10) if size else 0 + if order >= len(suffixes): + order = len(suffixes) - 1 + + return '{0:.3g} {1}'.format( + size / float(1 << (order * 10)), + suffixes[order] + ) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ef907d541..bec83ba1c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1991,10 +1991,13 @@ class CLITestCase(DockerClientTestCase): self.project.get_service('simple').create_container() result = self.dispatch(['images']) assert 'busybox' in result.stdout + assert 'simplecomposefile_simple_1' in result.stdout def test_images_default_composefile(self): self.base_dir = 'tests/fixtures/multiple-composefiles' self.dispatch(['up', '-d']) result = self.dispatch(['images']) - self.assertIn('busybox', result.stdout) + assert 'busybox' in result.stdout + assert 'multiplecomposefiles_another_1' in result.stdout + assert 'multiplecomposefiles_simple_1' in result.stdout From bf7c2bc0f885fbbcf33d6ed964bafaf41b28d3c4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Mar 2017 16:59:00 -0800 Subject: [PATCH 64/92] Use create_host_file in run -v tests to ensure file availability Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 31 +++++++++++++++++++------------ tests/helpers.py | 29 +++++++++++++++++++++++++++++ tests/integration/project_test.py | 28 +--------------------------- 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 0ddada5a0..4e3c070e8 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -19,6 +19,7 @@ import yaml from docker import errors from .. import mock +from ..helpers import create_host_file from compose.cli.command import get_project from compose.container import Container from compose.project import OneOffFilter @@ -561,35 +562,41 @@ class CLITestCase(DockerClientTestCase): def test_run_one_off_with_volume(self): self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' volume_path = os.path.abspath(os.path.join(os.getcwd(), self.base_dir, 'files')) - cmd_result = self.dispatch([ + create_host_file(self.client, os.path.join(volume_path, 'example.txt')) + + self.dispatch([ 'run', '-v', '{}:/data'.format(volume_path), 'simple', - 'cat', '/data/example.txt' - ]) - assert cmd_result.stdout.strip() == 'FILE_CONTENT' + 'test', '-f', '/data/example.txt' + ], returncode=0) + # FIXME: does not work with Python 3 + # assert cmd_result.stdout.strip() == 'FILE_CONTENT' def test_run_one_off_with_multiple_volumes(self): self.base_dir = 'tests/fixtures/simple-composefile-volume-ready' volume_path = os.path.abspath(os.path.join(os.getcwd(), self.base_dir, 'files')) + create_host_file(self.client, os.path.join(volume_path, 'example.txt')) - cmd_result = self.dispatch([ + self.dispatch([ 'run', '-v', '{}:/data'.format(volume_path), '-v', '{}:/data1'.format(volume_path), 'simple', - 'cat', '/data/example.txt' - ]) - assert cmd_result.stdout.strip() == 'FILE_CONTENT' + 'test', '-f', '/data/example.txt' + ], returncode=0) + # FIXME: does not work with Python 3 + # assert cmd_result.stdout.strip() == 'FILE_CONTENT' - cmd_result = self.dispatch([ + self.dispatch([ 'run', '-v', '{}:/data'.format(volume_path), '-v', '{}:/data1'.format(volume_path), 'simple', - 'cat', '/data1/example.txt' - ]) - assert cmd_result.stdout.strip() == 'FILE_CONTENT' + 'test', '-f' '/data1/example.txt' + ], returncode=0) + # FIXME: does not work with Python 3 + # assert cmd_result.stdout.strip() == 'FILE_CONTENT' def test_create_with_force_recreate_and_no_recreate(self): self.dispatch( diff --git a/tests/helpers.py b/tests/helpers.py index 4b422a6a0..59efd2557 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +import os + from compose.config.config import ConfigDetails from compose.config.config import ConfigFile from compose.config.config import load @@ -15,3 +17,30 @@ def build_config_details(contents, working_dir='working_dir', filename='filename working_dir, [ConfigFile(filename, contents)], ) + + +def create_host_file(client, filename): + dirname = os.path.dirname(filename) + + with open(filename, 'r') as fh: + content = fh.read() + + container = client.create_container( + 'busybox:latest', + ['sh', '-c', 'echo -n "{}" > {}'.format(content, filename)], + volumes={dirname: {}}, + host_config=client.create_host_config( + binds={dirname: {'bind': dirname, 'ro': False}}, + network_mode='none', + ), + ) + try: + client.start(container) + exitcode = client.wait(container) + + if exitcode != 0: + output = client.logs(container) + raise Exception( + "Container exited with code {}:\n{}".format(exitcode, output)) + finally: + client.remove_container(container, force=True) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 28762cd20..f0d21456b 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -10,6 +10,7 @@ from docker.errors import NotFound from .. import mock from ..helpers import build_config as load_config +from ..helpers import create_host_file from .testcases import DockerClientTestCase from compose.config import config from compose.config import ConfigurationError @@ -1517,30 +1518,3 @@ class ProjectTest(DockerClientTestCase): assert 'svc1' in svc2.get_dependency_names() with pytest.raises(NoHealthCheckConfigured): svc1.is_healthy() - - -def create_host_file(client, filename): - dirname = os.path.dirname(filename) - - with open(filename, 'r') as fh: - content = fh.read() - - container = client.create_container( - 'busybox:latest', - ['sh', '-c', 'echo -n "{}" > {}'.format(content, filename)], - volumes={dirname: {}}, - host_config=client.create_host_config( - binds={dirname: {'bind': dirname, 'ro': False}}, - network_mode='none', - ), - ) - try: - client.start(container) - exitcode = client.wait(container) - - if exitcode != 0: - output = client.logs(container) - raise Exception( - "Container exited with code {}:\n{}".format(exitcode, output)) - finally: - client.remove_container(container, force=True) From ec252350ae09d4028a960dcf50515e360b26cf72 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Mar 2017 16:03:35 -0800 Subject: [PATCH 65/92] Add mem_reservation option to service config in 2.0 and 2.1 formats Signed-off-by: Joffrey F --- compose/config/config.py | 3 ++- compose/config/config_schema_v2.0.json | 3 ++- compose/config/config_schema_v2.1.json | 3 ++- compose/service.py | 4 +++- tests/integration/service_test.py | 5 +++++ tests/unit/service_test.py | 14 ++++++++++++++ 6 files changed, 28 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 003b2e2f4..a6ea43cba 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -74,7 +74,8 @@ DOCKER_CONFIG_KEYS = [ 'labels', 'links', 'mac_address', - 'mem_limit', + 'mem_limit' + 'mem_reservation', 'memswap_limit', 'mem_swappiness', 'net', diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 59c7b30c9..3871dbf26 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -138,8 +138,9 @@ "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, - "memswap_limit": {"type": ["number", "string"]}, + "mem_reservation": {"type": ["string", "integer"]}, "mem_swappiness": {"type": "integer"}, + "memswap_limit": {"type": ["number", "string"]}, "network_mode": {"type": "string"}, "networks": { diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index d1ffff89a..05509ff1a 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -161,8 +161,9 @@ "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, - "memswap_limit": {"type": ["number", "string"]}, + "mem_reservation": {"type": ["string", "integer"]}, "mem_swappiness": {"type": "integer"}, + "memswap_limit": {"type": ["number", "string"]}, "network_mode": {"type": "string"}, "networks": { diff --git a/compose/service.py b/compose/service.py index b42094e68..3266f4f6c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -62,9 +62,10 @@ DOCKER_START_KEYS = [ 'log_driver', 'log_opt', 'mem_limit', + 'mem_reservation', 'memswap_limit', - 'oom_score_adj', 'mem_swappiness', + 'oom_score_adj', 'pid', 'privileged', 'restart', @@ -760,6 +761,7 @@ class Service(object): cap_add=options.get('cap_add'), cap_drop=options.get('cap_drop'), mem_limit=options.get('mem_limit'), + mem_reservation=options.get('mem_reservation'), memswap_limit=options.get('memswap_limit'), ulimits=build_ulimits(options.get('ulimits')), log_config=log_config, diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 734da5dfa..082bff93c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -872,6 +872,11 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.MemorySwappiness'), 11) + def test_mem_reservation(self): + service = self.create_service('web', mem_reservation='20m') + container = create_and_start_container(service) + assert container.get('HostConfig.MemoryReservation') == 20 * 1024 * 1024 + def test_restart_always_value(self): service = self.create_service('web', restart={'Name': 'always'}) container = create_and_start_container(service) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0a66e4f3e..6d2962fb9 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -168,6 +168,20 @@ class ServiceTest(unittest.TestCase): 2000000000 ) + def test_mem_reservation(self): + self.mock_client.create_host_config.return_value = {} + + service = Service( + name='foo', + image='foo', + hostname='name', + client=self.mock_client, + mem_reservation='512m' + ) + service._get_container_create_options({'some': 'overrides'}, 1) + assert self.mock_client.create_host_config.called is True + assert self.mock_client.create_host_config.call_args[1]['mem_reservation'] == '512m' + def test_cgroup_parent(self): self.mock_client.create_host_config.return_value = {} From 449dcc9d7b10a05aa8a682c6d2f5d1a686eb8843 Mon Sep 17 00:00:00 2001 From: Dat Tran Date: Thu, 11 Aug 2016 07:54:28 -0700 Subject: [PATCH 66/92] support --build-arg for build command Signed-off-by: Dat Tran --- compose/cli/main.py | 20 ++++++++++++++------ compose/config/__init__.py | 1 + compose/config/config.py | 6 ++++++ compose/project.py | 4 ++-- compose/service.py | 12 +++++++++--- tests/unit/config/config_test.py | 19 +++++++++++++++++++ tests/unit/service_test.py | 19 +++++++++++++++++++ 7 files changed, 70 insertions(+), 11 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 810b13f54..78e3d84b5 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -209,18 +209,26 @@ class TopLevelCommand(object): e.g. `composetest_db`. If you change a service's `Dockerfile` or the contents of its build directory, you can run `docker-compose build` to rebuild it. - Usage: build [options] [SERVICE...] + Usage: build [options] [--build-arg key=val...] [SERVICE...] Options: - --force-rm Always remove intermediate containers. - --no-cache Do not use cache when building the image. - --pull Always attempt to pull a newer version of the image. + --force-rm Always remove intermediate containers. + --no-cache Do not use cache when building the image. + --pull Always attempt to pull a newer version of the image. + --build-arg key=val Set build-time variables for one service. """ + service_names = options['SERVICE'] + build_args = options.get('--build-arg', None) + + if not service_names and build_args: + raise UserError("Need service name for --build-arg option") + self.project.build( - service_names=options['SERVICE'], + service_names=service_names, no_cache=bool(options.get('--no-cache', False)), pull=bool(options.get('--pull', False)), - force_rm=bool(options.get('--force-rm', False))) + force_rm=bool(options.get('--force-rm', False)), + build_args=build_args) def bundle(self, config_options, options): """ diff --git a/compose/config/__init__.py b/compose/config/__init__.py index 7cf71eb98..b6e5e8d38 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -7,5 +7,6 @@ from .config import ConfigurationError from .config import DOCKER_CONFIG_KEYS from .config import find from .config import load +from .config import merge_build_args from .config import merge_environment from .config import parse_environment diff --git a/compose/config/config.py b/compose/config/config.py index dbf64bae2..718d3bf02 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -602,6 +602,12 @@ def resolve_environment(service_dict, environment=None): return dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(env)) +def merge_build_args(base, override, environment): + override_args = parse_build_arguments(override) + override_dict = dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(override_args)) + base.update(override_dict) + + def resolve_build_args(build, environment): args = parse_build_arguments(build.get('args')) return dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(args)) diff --git a/compose/project.py b/compose/project.py index 5c21f3bf0..a75d71efc 100644 --- a/compose/project.py +++ b/compose/project.py @@ -307,10 +307,10 @@ class Project(object): 'Restarting') return containers - def build(self, service_names=None, no_cache=False, pull=False, force_rm=False): + def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, build_args=None): for service in self.get_services(service_names): if service.can_be_built(): - service.build(no_cache, pull, force_rm) + service.build(no_cache, pull, force_rm, build_args) else: log.info('%s uses an image, skipping' % service.name) diff --git a/compose/service.py b/compose/service.py index 712d5ac15..a889dd58c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -21,6 +21,7 @@ from . import __version__ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS +from .config import merge_build_args from .config import merge_environment from .config.types import ServicePort from .config.types import VolumeSpec @@ -803,13 +804,18 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] - def build(self, no_cache=False, pull=False, force_rm=False): + def build(self, no_cache=False, pull=False, force_rm=False, build_args=None): log.info('Building %s' % self.name) build_opts = self.options.get('build', {}) - path = build_opts.get('context') + + self_args_opts = build_opts.get('args', None) + if self_args_opts and build_args: + merge_build_args(self_args_opts, build_args, self.options.get('environment')) + # python2 os.stat() doesn't support unicode on some UNIX, so we # encode it to a bytestring to be safe + path = build_opts.get('context') if not six.PY3 and not IS_WINDOWS_PLATFORM: path = path.encode('utf8') @@ -822,8 +828,8 @@ class Service(object): pull=pull, nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), - buildargs=build_opts.get('args', None), cache_from=build_opts.get('cache_from', None), + buildargs=self_args_opts ) try: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d016ae4e2..3e3bd2bbf 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -15,6 +15,7 @@ import yaml from ...helpers import build_config_details from compose.config import config from compose.config import types +from compose.config.config import merge_build_args from compose.config.config import resolve_build_args from compose.config.config import resolve_environment from compose.config.config import V1 @@ -2881,6 +2882,24 @@ class EnvTest(unittest.TestCase): {'arg1': 'value1', 'empty_arg': '', 'env_arg': 'value2', 'no_env': None}, ) + @mock.patch.dict(os.environ) + def test_merge_build_args(self): + os.environ['env_arg'] = 'value2' + + base = { + 'arg1': 'arg1_value', + 'arg2': 'arg2_value' + } + override = { + 'arg1': 'arg1_new_value', + 'arg2': 'arg2_value' + } + self.assertEqual(base['arg1'], 'arg1_value') + + merge_build_args(base, override, os.environ) + + self.assertEqual(base['arg1'], 'arg1_new_value') + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @mock.patch.dict(os.environ) def test_resolve_path(self): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 5dc5265e6..ca7410414 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -513,6 +513,25 @@ class ServiceTest(unittest.TestCase): self.assertEqual(self.mock_client.build.call_count, 1) self.assertFalse(self.mock_client.build.call_args[1]['pull']) + def test_build_with_override_build_args(self): + self.mock_client.build.return_value = [ + b'{"stream": "Successfully built 12345"}', + ] + + build_args = [ + 'arg1=arg1_new_value', + 'arg2=arg2_value' + ] + service = Service('foo', client=self.mock_client, + build={'context': '.', 'args': {'arg1': 'arg1', 'arg2': 'arg2'}}) + service.build(build_args=build_args) + + called_build_args = self.mock_client.build.call_args[1]['buildargs'] + + for arg in called_build_args: + if "arg1=" in arg: + self.assertEquals(arg, 'arg1=arg1_new_value') + def test_config_dict(self): self.mock_client.inspect_image.return_value = {'Id': 'abcd'} service = Service( From d5a2d37d059cd1de5865fc5eaa05461d94aadfff Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 7 Mar 2017 17:47:41 -0800 Subject: [PATCH 67/92] Properly resolve build args against host environment values Signed-off-by: Joffrey F --- compose/cli/main.py | 4 +++ compose/config/__init__.py | 2 +- compose/config/config.py | 12 ++----- compose/service.py | 11 +++--- tests/integration/service_test.py | 20 ++++++++++- tests/unit/config/config_test.py | 58 ++----------------------------- tests/unit/service_test.py | 18 +++++----- 7 files changed, 43 insertions(+), 82 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 78e3d84b5..4d31e25e8 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -22,6 +22,7 @@ from ..bundle import MissingDigests from ..bundle import serialize_bundle from ..config import ConfigurationError from ..config import parse_environment +from ..config import resolve_build_args from ..config.environment import Environment from ..config.serialize import serialize_config from ..config.types import VolumeSpec @@ -219,6 +220,9 @@ class TopLevelCommand(object): """ service_names = options['SERVICE'] build_args = options.get('--build-arg', None) + if build_args: + environment = Environment.from_env_file(self.project_dir) + build_args = resolve_build_args(build_args, environment) if not service_names and build_args: raise UserError("Need service name for --build-arg option") diff --git a/compose/config/__init__.py b/compose/config/__init__.py index b6e5e8d38..b629edf66 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -7,6 +7,6 @@ from .config import ConfigurationError from .config import DOCKER_CONFIG_KEYS from .config import find from .config import load -from .config import merge_build_args from .config import merge_environment from .config import parse_environment +from .config import resolve_build_args diff --git a/compose/config/config.py b/compose/config/config.py index 718d3bf02..c85ffdabb 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -602,14 +602,8 @@ def resolve_environment(service_dict, environment=None): return dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(env)) -def merge_build_args(base, override, environment): - override_args = parse_build_arguments(override) - override_dict = dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(override_args)) - base.update(override_dict) - - -def resolve_build_args(build, environment): - args = parse_build_arguments(build.get('args')) +def resolve_build_args(buildargs, environment): + args = parse_build_arguments(buildargs) return dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(args)) @@ -1057,7 +1051,7 @@ def normalize_build(service_dict, working_dir, environment): build.update(service_dict['build']) if 'args' in build: build['args'] = build_string_dict( - resolve_build_args(build, environment) + resolve_build_args(build.get('args'), environment) ) service_dict['build'] = build diff --git a/compose/service.py b/compose/service.py index a889dd58c..29ee70476 100644 --- a/compose/service.py +++ b/compose/service.py @@ -21,7 +21,6 @@ from . import __version__ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS -from .config import merge_build_args from .config import merge_environment from .config.types import ServicePort from .config.types import VolumeSpec @@ -804,14 +803,14 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] - def build(self, no_cache=False, pull=False, force_rm=False, build_args=None): + def build(self, no_cache=False, pull=False, force_rm=False, build_args_override=None): log.info('Building %s' % self.name) build_opts = self.options.get('build', {}) - self_args_opts = build_opts.get('args', None) - if self_args_opts and build_args: - merge_build_args(self_args_opts, build_args, self.options.get('environment')) + build_args = build_opts.get('args', {}).copy() + if build_args_override: + build_args.update(build_args_override) # python2 os.stat() doesn't support unicode on some UNIX, so we # encode it to a bytestring to be safe @@ -829,7 +828,7 @@ class Service(object): nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), cache_from=build_opts.get('cache_from', None), - buildargs=self_args_opts + buildargs=build_args ) try: diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index ddc2f3bae..12ec8a993 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -597,12 +597,30 @@ class ServiceTest(DockerClientTestCase): with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: f.write("FROM busybox\n") f.write("ARG build_version\n") + f.write("RUN echo ${build_version}\n") service = self.create_service('buildwithargs', build={'context': text_type(base_dir), 'args': {"build_version": "1"}}) service.build() assert service.image() + assert "build_version=1" in service.image()['ContainerConfig']['Cmd'] + + def test_build_with_build_args_override(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + f.write("ARG build_version\n") + f.write("RUN echo ${build_version}\n") + + service = self.create_service('buildwithargs', + build={'context': text_type(base_dir), + 'args': {"build_version": "1"}}) + service.build(build_args_override={'build_version': '2'}) + assert service.image() + assert "build_version=2" in service.image()['ContainerConfig']['Cmd'] def test_start_container_stays_unprivileged(self): service = self.create_service('web') @@ -1057,7 +1075,7 @@ class ServiceTest(DockerClientTestCase): one_off_container = service.create_container(one_off=True) self.assertNotEqual(one_off_container.name, 'my-web-container') - @pytest.mark.skipif(True, reason="Broken on 1.11.0rc1") + @pytest.mark.skipif(True, reason="Broken on 1.11.0 - 17.03.0") def test_log_drive_invalid(self): service = self.create_service('web', logging={'driver': 'xxx'}) expected_error_msg = "logger: no log driver named 'xxx' is registered" diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 3e3bd2bbf..d7d342afa 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -15,7 +15,6 @@ import yaml from ...helpers import build_config_details from compose.config import config from compose.config import types -from compose.config.config import merge_build_args from compose.config.config import resolve_build_args from compose.config.config import resolve_environment from compose.config.config import V1 @@ -1523,7 +1522,7 @@ class ConfigTest(unittest.TestCase): assert actual == { 'image': 'alpine:edge', 'volumes': ['.:/app'], - 'ports': ['5432'] + 'ports': types.ServicePort.parse('5432') } def test_merge_service_dicts_heterogeneous_2(self): @@ -1542,40 +1541,7 @@ class ConfigTest(unittest.TestCase): assert actual == { 'image': 'alpine:edge', 'volumes': ['.:/app'], - 'ports': ['5432'] - } - - def test_merge_build_args(self): - base = { - 'build': { - 'context': '.', - 'args': { - 'ONE': '1', - 'TWO': '2', - }, - } - } - override = { - 'build': { - 'args': { - 'TWO': 'dos', - 'THREE': '3', - }, - } - } - actual = config.merge_service_dicts( - base, - override, - DEFAULT_VERSION) - assert actual == { - 'build': { - 'context': '.', - 'args': { - 'ONE': '1', - 'TWO': 'dos', - 'THREE': '3', - }, - } + 'ports': types.ServicePort.parse('5432') } def test_merge_logging_v1(self): @@ -2878,28 +2844,10 @@ class EnvTest(unittest.TestCase): } } self.assertEqual( - resolve_build_args(build, Environment.from_env_file(build['context'])), + resolve_build_args(build['args'], Environment.from_env_file(build['context'])), {'arg1': 'value1', 'empty_arg': '', 'env_arg': 'value2', 'no_env': None}, ) - @mock.patch.dict(os.environ) - def test_merge_build_args(self): - os.environ['env_arg'] = 'value2' - - base = { - 'arg1': 'arg1_value', - 'arg2': 'arg2_value' - } - override = { - 'arg1': 'arg1_new_value', - 'arg2': 'arg2_value' - } - self.assertEqual(base['arg1'], 'arg1_value') - - merge_build_args(base, override, os.environ) - - self.assertEqual(base['arg1'], 'arg1_new_value') - @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @mock.patch.dict(os.environ) def test_resolve_path(self): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index ca7410414..b3c8c4d7d 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -461,7 +461,7 @@ class ServiceTest(unittest.TestCase): forcerm=False, nocache=False, rm=True, - buildargs=None, + buildargs={}, cache_from=None, ) @@ -498,7 +498,7 @@ class ServiceTest(unittest.TestCase): forcerm=False, nocache=False, rm=True, - buildargs=None, + buildargs={}, cache_from=None, ) @@ -518,19 +518,17 @@ class ServiceTest(unittest.TestCase): b'{"stream": "Successfully built 12345"}', ] - build_args = [ - 'arg1=arg1_new_value', - 'arg2=arg2_value' - ] + build_args = { + 'arg1': 'arg1_new_value', + } service = Service('foo', client=self.mock_client, build={'context': '.', 'args': {'arg1': 'arg1', 'arg2': 'arg2'}}) - service.build(build_args=build_args) + service.build(build_args_override=build_args) called_build_args = self.mock_client.build.call_args[1]['buildargs'] - for arg in called_build_args: - if "arg1=" in arg: - self.assertEquals(arg, 'arg1=arg1_new_value') + assert called_build_args['arg1'] == build_args['arg1'] + assert called_build_args['arg2'] == 'arg2' def test_config_dict(self): self.mock_client.inspect_image.return_value = {'Id': 'abcd'} From 0652530ee94c5c7b15fb2d340c5bf8d9a5564300 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Wed, 8 Mar 2017 12:49:31 -0800 Subject: [PATCH 68/92] Add bash completion for `run --volume` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index fa099eac4..59eb53499 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -375,14 +375,14 @@ _docker_compose_run() { __docker_compose_nospace return ;; - --entrypoint|--name|--user|-u|--workdir|-w) + --entrypoint|--name|--user|-u|--volume|-v|--workdir|-w) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --name --no-deps --publish -p --rm --service-ports -T --user -u --workdir -w" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --name --no-deps --publish -p --rm --service-ports -T --user -u --volume -v --workdir -w" -- "$cur" ) ) ;; *) __docker_compose_services_all From 38087a288889bae4b8ce13b15e7e7b0ca392c071 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Wed, 8 Mar 2017 13:07:25 -0800 Subject: [PATCH 69/92] Fix bash completion for `docker-compose images` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index fa099eac4..49a76d001 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -221,14 +221,14 @@ _docker_compose_help() { } _docker_compose_images() { - case "$cur" in - -*) - COMPREPLY=( $( compgen -W "--help -q" -- "$cur" ) ) - ;; - *) - __docker_compose_services_all - ;; - esac + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help -q" -- "$cur" ) ) + ;; + *) + __docker_compose_services_all + ;; + esac } _docker_compose_kill() { @@ -508,6 +508,7 @@ _docker_compose() { events exec help + images kill logs pause From 1899eac2ba428f2dfc0329823489549cb6159674 Mon Sep 17 00:00:00 2001 From: Henrik Holst Date: Sat, 17 Dec 2016 17:39:58 +0100 Subject: [PATCH 70/92] Fixes https://github.com/docker/compose/issues/4099 Signed-off-by: Henrik Holst --- compose/cli/command.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 020354283..26ff2cae5 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -54,7 +54,8 @@ def get_config_path_from_options(base_dir, options, environment): config_files = environment.get('COMPOSE_FILE') if config_files: - return config_files.split(os.pathsep) + pathsep = environment.get('COMPOSE_FILE_SEPARATOR', os.pathsep) + return config_files.split(pathsep) return None From ac12ab95c4c99a60811f4c34e49efcceb9a13c8e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Mar 2017 14:30:15 -0800 Subject: [PATCH 71/92] Rename COMPOSE_FILE_SEPARATOR -> COMPOSE_PATH_SEPARATOR Add unit test Signed-off-by: Joffrey F --- compose/cli/command.py | 2 +- tests/unit/cli/command_test.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 26ff2cae5..c74e585de 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -54,7 +54,7 @@ def get_config_path_from_options(base_dir, options, environment): config_files = environment.get('COMPOSE_FILE') if config_files: - pathsep = environment.get('COMPOSE_FILE_SEPARATOR', os.pathsep) + pathsep = environment.get('COMPOSE_PATH_SEPARATOR', os.pathsep) return config_files.split(pathsep) return None diff --git a/tests/unit/cli/command_test.py b/tests/unit/cli/command_test.py index 50fc84e17..3655c432e 100644 --- a/tests/unit/cli/command_test.py +++ b/tests/unit/cli/command_test.py @@ -45,6 +45,15 @@ class TestGetConfigPathFromOptions(object): '.', {}, environment ) == ['one.yml', 'two.yml'] + def test_multiple_path_from_env_custom_separator(self): + with mock.patch.dict(os.environ): + os.environ['COMPOSE_PATH_SEPARATOR'] = '^' + os.environ['COMPOSE_FILE'] = 'c:\\one.yml^.\\semi;colon.yml' + environment = Environment.from_env_file('.') + assert get_config_path_from_options( + '.', {}, environment + ) == ['c:\\one.yml', '.\\semi;colon.yml'] + def test_no_path(self): environment = Environment.from_env_file('.') assert not get_config_path_from_options('.', {}, environment) From 707210ae95e2e3961e2eb41312d3c06ca56d15a8 Mon Sep 17 00:00:00 2001 From: Dimitar Bonev Date: Thu, 7 Jan 2016 22:15:02 +0200 Subject: [PATCH 72/92] Ability to change working directory via a CLI flag Signed-off-by: Dimitar Bonev --- compose/cli/command.py | 7 ++++--- compose/cli/main.py | 2 ++ compose/config/config.py | 6 +++--- tests/acceptance/cli_test.py | 20 ++++++++++++++++++- .../docker-compose.yml | 2 ++ tests/unit/config/config_test.py | 10 ++++++++-- 6 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/build-path-override-dir/docker-compose.yml diff --git a/compose/cli/command.py b/compose/cli/command.py index 020354283..f116a5278 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -33,7 +33,8 @@ def project_from_options(project_dir, options): verbose=options.get('--verbose'), host=host, tls_config=tls_config_from_options(options), - environment=environment + environment=environment, + override_dir=options.get('--project-directory'), ) @@ -93,10 +94,10 @@ def get_client(environment, verbose=False, version=None, tls_config=None, host=N def get_project(project_dir, config_path=None, project_name=None, verbose=False, - host=None, tls_config=None, environment=None): + host=None, tls_config=None, environment=None, override_dir=None): if not environment: environment = Environment.from_env_file(project_dir) - config_details = config.find(project_dir, config_path, environment) + config_details = config.find(project_dir, config_path, environment, override_dir) project_name = get_project_name( config_details.working_dir, project_name, environment ) diff --git a/compose/cli/main.py b/compose/cli/main.py index 4d31e25e8..84786abc7 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -168,6 +168,8 @@ class TopLevelCommand(object): --skip-hostname-check Don't check the daemon's hostname against the name specified in the client certificate (for example if your docker host is an IP address) + --project-directory PATH Specify an alternate working directory + (default: the path of the compose file) Commands: build Build or rebuild services diff --git a/compose/config/config.py b/compose/config/config.py index c85ffdabb..5d74fc76f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -235,10 +235,10 @@ class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name conf config) -def find(base_dir, filenames, environment): +def find(base_dir, filenames, environment, override_dir='.'): if filenames == ['-']: return ConfigDetails( - os.getcwd(), + os.path.abspath(override_dir), [ConfigFile(None, yaml.safe_load(sys.stdin))], environment ) @@ -250,7 +250,7 @@ def find(base_dir, filenames, environment): log.debug("Using configuration files: {}".format(",".join(filenames))) return ConfigDetails( - os.path.dirname(filenames[0]), + override_dir or os.path.dirname(filenames[0]), [ConfigFile.from_filename(f) for f in filenames], environment ) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 31853a11b..6a498e250 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -107,6 +107,7 @@ class CLITestCase(DockerClientTestCase): def setUp(self): super(CLITestCase, self).setUp() self.base_dir = 'tests/fixtures/simple-composefile' + self.override_dir = None def tearDown(self): if self.base_dir: @@ -129,7 +130,7 @@ class CLITestCase(DockerClientTestCase): def project(self): # Hack: allow project to be overridden if not hasattr(self, '_project'): - self._project = get_project(self.base_dir) + self._project = get_project(self.base_dir, override_dir=self.override_dir) return self._project def dispatch(self, options, project_options=None, returncode=0): @@ -518,6 +519,23 @@ class CLITestCase(DockerClientTestCase): }, } + def test_build_override_dir(self): + self.base_dir = 'tests/fixtures/build-path-override-dir' + self.override_dir = os.path.abspath('tests/fixtures') + result = self.dispatch([ + '--project-directory', self.override_dir, + 'build']) + + assert 'Successfully built' in result.stdout + + def test_build_override_dir_invalid_path(self): + config_path = os.path.abspath('tests/fixtures/build-path-override-dir/docker-compose.yml') + result = self.dispatch([ + '-f', config_path, + 'build'], returncode=1) + + assert 'does not exist, is not accessible, or is not a valid URL' in result.stderr + def test_create(self): self.dispatch(['create']) service = self.project.get_service('simple') diff --git a/tests/fixtures/build-path-override-dir/docker-compose.yml b/tests/fixtures/build-path-override-dir/docker-compose.yml new file mode 100644 index 000000000..15dbb3e68 --- /dev/null +++ b/tests/fixtures/build-path-override-dir/docker-compose.yml @@ -0,0 +1,2 @@ +foo: + build: ./build-ctx/ diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d7d342afa..1b98a5ece 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2875,9 +2875,9 @@ class EnvTest(unittest.TestCase): set([VolumeSpec.parse('/opt/tmp:/opt/host/tmp')])) -def load_from_filename(filename): +def load_from_filename(filename, override_dir=None): return config.load( - config.find('.', [filename], Environment.from_env_file('.')) + config.find('.', [filename], Environment.from_env_file('.'), override_dir=override_dir) ).services @@ -3443,6 +3443,12 @@ class BuildPathTest(unittest.TestCase): service_dict = load_from_filename('tests/fixtures/build-path/docker-compose.yml') self.assertEqual(service_dict, [{'name': 'foo', 'build': {'context': self.abs_context_path}}]) + def test_from_file_override_dir(self): + override_dir = os.path.join(os.getcwd(), 'tests/fixtures/') + service_dict = load_from_filename( + 'tests/fixtures/build-path-override-dir/docker-compose.yml', override_dir=override_dir) + self.assertEquals(service_dict, [{'name': 'foo', 'build': {'context': self.abs_context_path}}]) + def test_valid_url_in_build_path(self): valid_urls = [ 'git://github.com/docker/docker', From 5ea916733495a3e09726b96c4716a87da60f8bb2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Mar 2017 18:29:09 -0800 Subject: [PATCH 73/92] Prevent service to create a container if it is referencing itself in an external link Signed-off-by: Joffrey F --- compose/service.py | 13 ++++++++++++- tests/unit/service_test.py | 9 +++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 29ee70476..b9f77beb9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -22,6 +22,7 @@ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config.errors import DependencyError from .config.types import ServicePort from .config.types import VolumeSpec from .const import DEFAULT_TIMEOUT @@ -872,7 +873,17 @@ class Service(object): if self.custom_container_name and not one_off: return self.custom_container_name - return build_container_name(self.project, self.name, number, one_off) + container_name = build_container_name( + self.project, self.name, number, one_off, + ) + ext_links_origins = [l.split(':')[0] for l in self.options.get('external_links', [])] + if container_name in ext_links_origins: + raise DependencyError( + 'Service {0} has a self-referential external link: {1}'.format( + self.name, container_name + ) + ) + return container_name def remove_image(self, image_type): if not image_type or image_type == ImageType.none: diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index b3c8c4d7d..f3f3a2a83 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -7,6 +7,7 @@ from docker.errors import APIError from .. import mock from .. import unittest +from compose.config.errors import DependencyError from compose.config.types import ServicePort from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec @@ -170,6 +171,14 @@ class ServiceTest(unittest.TestCase): 2000000000 ) + def test_self_reference_external_link(self): + service = Service( + name='foo', + external_links=['default_foo_1'] + ) + with self.assertRaises(DependencyError): + service.get_container_name(1) + def test_mem_reservation(self): self.mock_client.create_host_config.return_value = {} From ba0468395b1418ab434f1c0d82e840131ca479a8 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 9 Mar 2017 10:21:53 +0100 Subject: [PATCH 74/92] Add bash completion for `docker-compose rm --stop` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 9012ec86e..2591f500b 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -36,6 +36,18 @@ __docker_compose_to_extglob() { echo "@($extglob)" } +# Determines whether the option passed as the first argument exist on +# the commandline. The option may be a pattern, e.g. `--force|-f`. +__docker_compose_has_option() { + local pattern="$1" + for (( i=2; i < $cword; ++i)); do + if [[ ${words[$i]} =~ ^($pattern)$ ]] ; then + return 0 + fi + done + return 1 +} + # suppress trailing whitespace __docker_compose_nospace() { # compopt is not available in ancient bash versions @@ -359,10 +371,14 @@ _docker_compose_restart() { _docker_compose_rm() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--force -f --help -v" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--force -f --help --stop -s -v" -- "$cur" ) ) ;; *) - __docker_compose_services_stopped + if __docker_compose_has_option "--stop|-s" ; then + __docker_compose_services_all + else + __docker_compose_services_stopped + fi ;; esac } From c3bcd59aeb909d98637476a2a968c8daa66ad788 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Mar 2017 18:51:07 -0800 Subject: [PATCH 75/92] Avoid encoding crash in log_api_error Signed-off-by: Joffrey F --- compose/cli/errors.py | 11 ++++++++--- tests/unit/cli/errors_test.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 5b9770955..23e065c99 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -7,6 +7,7 @@ import socket from distutils.spawn import find_executable from textwrap import dedent +import six from docker.errors import APIError from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import ReadTimeout @@ -68,14 +69,18 @@ def log_timeout_error(timeout): def log_api_error(e, client_version): - if b'client is newer than server' not in e.explanation: - log.error(e.explanation) + explanation = e.explanation + if isinstance(explanation, six.binary_type): + explanation = explanation.decode('utf-8') + + if 'client is newer than server' not in explanation: + log.error(explanation) return version = API_VERSION_TO_ENGINE_VERSION.get(client_version) if not version: # They've set a custom API version - log.error(e.explanation) + log.error(explanation) return log.error( diff --git a/tests/unit/cli/errors_test.py b/tests/unit/cli/errors_test.py index a7b57562f..7406a8880 100644 --- a/tests/unit/cli/errors_test.py +++ b/tests/unit/cli/errors_test.py @@ -42,10 +42,26 @@ class TestHandleConnectionErrors(object): _, args, _ = mock_logging.error.mock_calls[0] assert "Docker Engine of version 1.10.0 or greater" in args[0] + def test_api_error_version_mismatch_unicode_explanation(self, mock_logging): + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise APIError(None, None, u"client is newer than server") + + _, args, _ = mock_logging.error.mock_calls[0] + assert "Docker Engine of version 1.10.0 or greater" in args[0] + def test_api_error_version_other(self, mock_logging): msg = b"Something broke!" with pytest.raises(errors.ConnectionError): with handle_connection_errors(mock.Mock(api_version='1.22')): raise APIError(None, None, msg) + mock_logging.error.assert_called_once_with(msg.decode('utf-8')) + + def test_api_error_version_other_unicode_explanation(self, mock_logging): + msg = u"Something broke!" + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise APIError(None, None, msg) + mock_logging.error.assert_called_once_with(msg) From 23b873c2ceaf02ec0b57a299d76f58752ae910ea Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 10 Mar 2017 14:32:55 -0800 Subject: [PATCH 76/92] Add "secrets" section to docker-compose config output when applicable Signed-off-by: Joffrey F --- compose/config/errors.py | 2 +- compose/config/serialize.py | 24 +++++++++--------------- tests/unit/config/config_test.py | 8 +++++++- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/compose/config/errors.py b/compose/config/errors.py index 16ed01b86..0f78d4a94 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals VERSION_EXPLANATION = ( 'You might be seeing this error because you\'re using the wrong Compose file version. ' - 'Either specify a supported version ("2.0", "2.1", "3.0") and place your ' + 'Either specify a supported version ("2.0", "2.1", "3.0", "3.1") and place your ' 'service definitions under the `services` key, or omit the `version` key ' 'and place your service definitions at the root of the file to use ' 'version 1.\nFor more on the Compose file format versions, see ' diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 58581f7cc..6e2ad5906 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -26,34 +26,28 @@ yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type) def denormalize_config(config): + result = {'version': V2_1 if config.version == V1 else config.version} denormalized_services = [ denormalize_service_dict(service_dict, config.version) for service_dict in config.services ] - services = { + result['services'] = { service_dict.pop('name'): service_dict for service_dict in denormalized_services } - networks = config.networks.copy() - for net_name, net_conf in networks.items(): + result['networks'] = config.networks.copy() + for net_name, net_conf in result['networks'].items(): if 'external_name' in net_conf: del net_conf['external_name'] - volumes = config.volumes.copy() - for vol_name, vol_conf in volumes.items(): + result['volumes'] = config.volumes.copy() + for vol_name, vol_conf in result['volumes'].items(): if 'external_name' in vol_conf: del vol_conf['external_name'] - version = config.version - if version == V1: - version = V2_1 - - return { - 'version': version, - 'services': services, - 'networks': networks, - 'volumes': volumes, - } + if config.version in (V3_1,): + result['secrets'] = config.secrets + return result def serialize_config(config): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1b98a5ece..fe896d8b5 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3650,11 +3650,17 @@ class SerializeTest(unittest.TestCase): } ] } + secrets_dict = { + 'one': {'file': '/one.txt'}, + 'source': {'file': '/source.pem'} + } config_dict = config.load(build_config_details({ 'version': '3.1', - 'services': {'web': service_dict} + 'services': {'web': service_dict}, + 'secrets': secrets_dict })) serialized_config = yaml.load(serialize_config(config_dict)) serialized_service = serialized_config['services']['web'] assert secret_sort(serialized_service['secrets']) == secret_sort(service_dict['secrets']) + assert serialized_config['secrets'] == secrets_dict From a2e32b8166e0259fbb1773845cef0d3a54f55df6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 13 Mar 2017 11:23:17 -0700 Subject: [PATCH 77/92] Add missing comma in DOCKER_CONFIG_KEYS Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 5d74fc76f..4655fbdfc 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -76,7 +76,7 @@ DOCKER_CONFIG_KEYS = [ 'labels', 'links', 'mac_address', - 'mem_limit' + 'mem_limit', 'mem_reservation', 'memswap_limit', 'mem_swappiness', From 963b672cbd6445f047f719ff22f7d6d69ee19adf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 10 Mar 2017 16:06:35 -0800 Subject: [PATCH 78/92] Ensure network config matches remote for all properties Signed-off-by: Joffrey F --- compose/network.py | 58 +++++++++++++++++++++++++----- tests/unit/network_test.py | 74 +++++++++++++++++++++++++++++++++++--- 2 files changed, 119 insertions(+), 13 deletions(-) diff --git a/compose/network.py b/compose/network.py index d98f68d2f..053fdacd8 100644 --- a/compose/network.py +++ b/compose/network.py @@ -126,22 +126,64 @@ def create_ipam_config_from_dict(ipam_dict): ) +class NetworkConfigChangedError(ConfigurationError): + def __init__(self, net_name, property_name): + super(NetworkConfigChangedError, self).__init__( + 'Network "{}" needs to be recreated - {} has changed'.format( + net_name, property_name + ) + ) + + +def check_remote_ipam_config(remote, local): + remote_ipam = remote.get('IPAM') + ipam_dict = create_ipam_config_from_dict(local.ipam) + if local.ipam.get('driver') and local.ipam.get('driver') != remote_ipam.get('Driver'): + raise NetworkConfigChangedError(local.full_name, 'IPAM driver') + if len(ipam_dict['Config']) != 0: + if len(ipam_dict['Config']) != len(remote_ipam['Config']): + raise NetworkConfigChangedError(local.full_name, 'IPAM configs') + remote_configs = sorted(remote_ipam['Config'], key='Subnet') + local_configs = sorted(ipam_dict['Config'], key='Subnet') + while local_configs: + lc = local_configs.pop() + rc = remote_configs.pop() + if lc.get('Subnet') != rc.get('Subnet'): + raise NetworkConfigChangedError(local.full_name, 'IPAM config subnet') + if lc.get('Gateway') is not None and lc.get('Gateway') != rc.get('Gateway'): + raise NetworkConfigChangedError(local.full_name, 'IPAM config gateway') + if lc.get('IPRange') != rc.get('IPRange'): + raise NetworkConfigChangedError(local.full_name, 'IPAM config ip_range') + if sorted(lc.get('AuxiliaryAddresses')) != sorted(rc.get('AuxiliaryAddresses')): + raise NetworkConfigChangedError(local.full_name, 'IPAM config aux_addresses') + + def check_remote_network_config(remote, local): if local.driver and remote.get('Driver') != local.driver: - raise ConfigurationError( - 'Network "{}" needs to be recreated - driver has changed' - .format(local.full_name) - ) + raise NetworkConfigChangedError(local.full_name, 'driver') local_opts = local.driver_opts or {} remote_opts = remote.get('Options') or {} for k in set.union(set(remote_opts.keys()), set(local_opts.keys())): if k in OPTS_EXCEPTIONS: continue if remote_opts.get(k) != local_opts.get(k): - raise ConfigurationError( - 'Network "{}" needs to be recreated - options have changed' - .format(local.full_name) - ) + raise NetworkConfigChangedError(local.full_name, 'option "{}"'.format(k)) + + if local.ipam is not None: + check_remote_ipam_config(remote, local) + + if local.internal is not None and local.internal != remote.get('Internal', False): + raise NetworkConfigChangedError(local.full_name, 'internal') + if local.enable_ipv6 is not None and local.enable_ipv6 != remote.get('EnableIPv6', False): + raise NetworkConfigChangedError(local.full_name, 'enable_ipv6') + + local_labels = local.labels or {} + remote_labels = remote.get('Labels', {}) + for k in set.union(set(remote_labels.keys()), set(local_labels.keys())): + if k.startswith('com.docker.compose.'): # We are only interested in user-specified labels + continue + if remote_labels.get(k) != local_labels.get(k): + raise NetworkConfigChangedError(local.full_name, 'label "{}"'.format(k)) def build_networks(name, config_data, client): diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index 12d06f415..a325f1948 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -4,20 +4,62 @@ from __future__ import unicode_literals import pytest from .. import unittest -from compose.config import ConfigurationError from compose.network import check_remote_network_config from compose.network import Network +from compose.network import NetworkConfigChangedError class NetworkTest(unittest.TestCase): def test_check_remote_network_config_success(self): options = {'com.docker.network.driver.foo': 'bar'} + ipam_config = { + 'driver': 'default', + 'config': [ + {'subnet': '172.0.0.1/16', }, + { + 'subnet': '156.0.0.1/25', + 'gateway': '156.0.0.1', + 'aux_addresses': ['11.0.0.1', '24.25.26.27'], + 'ip_range': '156.0.0.1-254' + } + ] + } + labels = { + 'com.project.tests.istest': 'true', + 'com.project.sound.track': 'way out of here', + } + remote_labels = labels.copy() + remote_labels.update({ + 'com.docker.compose.project': 'compose_test', + 'com.docker.compose.network': 'net1', + }) net = Network( None, 'compose_test', 'net1', 'bridge', - options + options, enable_ipv6=True, ipam=ipam_config, + labels=labels ) check_remote_network_config( - {'Driver': 'bridge', 'Options': options}, net + { + 'Driver': 'bridge', + 'Options': options, + 'EnableIPv6': True, + 'Internal': False, + 'Attachable': True, + 'IPAM': { + 'Driver': 'default', + 'Config': [{ + 'Subnet': '156.0.0.1/25', + 'Gateway': '156.0.0.1', + 'AuxiliaryAddresses': ['24.25.26.27', '11.0.0.1'], + 'IPRange': '156.0.0.1-254' + }, { + 'Subnet': '172.0.0.1/16', + 'Gateway': '172.0.0.1' + }], + }, + 'Labels': remote_labels + }, + net ) def test_check_remote_network_config_whitelist(self): @@ -36,20 +78,42 @@ class NetworkTest(unittest.TestCase): def test_check_remote_network_config_driver_mismatch(self): net = Network(None, 'compose_test', 'net1', 'overlay') - with pytest.raises(ConfigurationError): + with pytest.raises(NetworkConfigChangedError) as e: check_remote_network_config( {'Driver': 'bridge', 'Options': {}}, net ) + assert 'driver has changed' in str(e.value) + def test_check_remote_network_config_options_mismatch(self): net = Network(None, 'compose_test', 'net1', 'overlay') - with pytest.raises(ConfigurationError): + with pytest.raises(NetworkConfigChangedError) as e: check_remote_network_config({'Driver': 'overlay', 'Options': { 'com.docker.network.driver.foo': 'baz' }}, net) + assert 'option "com.docker.network.driver.foo" has changed' in str(e.value) + def test_check_remote_network_config_null_remote(self): net = Network(None, 'compose_test', 'net1', 'overlay') check_remote_network_config( {'Driver': 'overlay', 'Options': None}, net ) + + def test_check_remote_network_labels_mismatch(self): + net = Network(None, 'compose_test', 'net1', 'overlay', labels={ + 'com.project.touhou.character': 'sakuya.izayoi' + }) + remote = { + 'Driver': 'overlay', + 'Options': None, + 'Labels': { + 'com.docker.compose.network': 'net1', + 'com.docker.compose.project': 'compose_test', + 'com.project.touhou.character': 'marisa.kirisame', + } + } + with pytest.raises(NetworkConfigChangedError) as e: + check_remote_network_config(remote, net) + + assert 'label "com.project.touhou.character" has changed' in str(e.value) From a6db78e5d4c3e4233d78508d6b851cd1bd80638a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 10 Mar 2017 14:50:26 -0800 Subject: [PATCH 79/92] Enable variable substitution in config.secrets Signed-off-by: Joffrey F --- compose/config/config.py | 9 +++++ tests/unit/config/config_test.py | 52 +++++++++++++++++-------- tests/unit/config/interpolation_test.py | 27 ++++++++++++- 3 files changed, 70 insertions(+), 18 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 5d74fc76f..413f1d319 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -218,6 +218,8 @@ class Config(namedtuple('_Config', 'version services volumes networks secrets')) :type volumes: :class:`dict` :param networks: Dictionary mapping network names to description dictionaries :type networks: :class:`dict` + :param secrets: Dictionary mapping secret names to description dictionaries + :type secrets: :class:`dict` """ @@ -491,6 +493,13 @@ def process_config_file(config_file, environment, service_name=None): config_file.get_networks(), 'network', environment) + if config_file.version in (V3_1,): + processed_config['secrets'] = interpolate_config_section( + config_file, + config_file.get_secrets(), + 'secrets', + environment + ) elif config_file.version == V1: processed_config = services else: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index fe896d8b5..93bae9726 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1821,6 +1821,23 @@ class ConfigTest(unittest.TestCase): } } + def test_empty_environment_key_allowed(self): + service_dict = config.load( + build_config_details( + { + 'web': { + 'build': '.', + 'environment': { + 'POSTGRES_PASSWORD': '' + }, + }, + }, + '.', + None, + ) + ).services[0] + self.assertEqual(service_dict['environment']['POSTGRES_PASSWORD'], '') + def test_merge_pid(self): # Regression: https://github.com/docker/compose/issues/4184 base = { @@ -2335,22 +2352,23 @@ class InterpolationTest(unittest.TestCase): self.assertIn('in service "web"', cm.exception.msg) self.assertIn('"${"', cm.exception.msg) - def test_empty_environment_key_allowed(self): - service_dict = config.load( - build_config_details( - { - 'web': { - 'build': '.', - 'environment': { - 'POSTGRES_PASSWORD': '' - }, - }, - }, - '.', - None, - ) - ).services[0] - self.assertEqual(service_dict['environment']['POSTGRES_PASSWORD'], '') + @mock.patch.dict(os.environ) + def test_interpolation_secrets_section(self): + os.environ['FOO'] = 'baz.bar' + config_dict = config.load(build_config_details({ + 'version': '3.1', + 'secrets': { + 'secretdata': { + 'external': {'name': '$FOO'} + } + } + })) + assert config_dict.secrets == { + 'secretdata': { + 'external': {'name': 'baz.bar'}, + 'external_name': 'baz.bar' + } + } class VolumeConfigTest(unittest.TestCase): @@ -3663,4 +3681,4 @@ class SerializeTest(unittest.TestCase): serialized_config = yaml.load(serialize_config(config_dict)) serialized_service = serialized_config['services']['web'] assert secret_sort(serialized_service['secrets']) == secret_sort(service_dict['secrets']) - assert serialized_config['secrets'] == secrets_dict + assert 'secrets' in serialized_config diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index fd40153d2..256c74d9b 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -75,7 +75,32 @@ def test_interpolate_environment_variables_in_volumes(mock_env): }, 'other': {}, } - value = interpolate_environment_variables("2.0", volumes, 'volume', mock_env) + value = interpolate_environment_variables("2.0", volumes, 'volume', mock_env) + assert value == expected + + +def test_interpolate_environment_variables_in_secrets(mock_env): + secrets = { + 'secretservice': { + 'file': '$FOO', + 'labels': { + 'max': 2, + 'user': '${USER}' + } + }, + 'other': None, + } + expected = { + 'secretservice': { + 'file': 'bar', + 'labels': { + 'max': 2, + 'user': 'jenny' + } + }, + 'other': {}, + } + value = interpolate_environment_variables("3.1", secrets, 'volume', mock_env) assert value == expected From 56357d6117f3fed4bff373b000b882221aa4a5eb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 14 Mar 2017 14:21:19 -0700 Subject: [PATCH 80/92] Do not raise a broken pipe error when receiving SIGPIPE from grep or head Signed-off-by: Joffrey F --- compose/cli/main.py | 1 + compose/cli/signals.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index 84786abc7..63a0036b4 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -61,6 +61,7 @@ console_handler = logging.StreamHandler(sys.stderr) def main(): + signals.ignore_sigpipe() try: command = dispatch() command() diff --git a/compose/cli/signals.py b/compose/cli/signals.py index 68a0598e1..9b360c44e 100644 --- a/compose/cli/signals.py +++ b/compose/cli/signals.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import signal +from ..const import IS_WINDOWS_PLATFORM + class ShutdownException(Exception): pass @@ -19,3 +21,10 @@ def set_signal_handler(handler): def set_signal_handler_to_shutdown(): set_signal_handler(shutdown) + + +def ignore_sigpipe(): + # Restore default behavior for SIGPIPE instead of raising + # an exception when encountered. + if not IS_WINDOWS_PLATFORM: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) From 0ba1f61e9b971f71f3964104d6064066c433fb6a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 13 Mar 2017 15:09:59 -0400 Subject: [PATCH 81/92] Add schema for v3.2, and revert some changes made to v3.1 after it was released. Signed-off-by: Daniel Nephin --- compose/config/config_schema_v3.1.json | 20 +- compose/config/config_schema_v3.2.json | 473 +++++++++++++++++++++++++ 2 files changed, 476 insertions(+), 17 deletions(-) create mode 100644 compose/config/config_schema_v3.2.json diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index 219ccdd48..b7037485f 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -71,8 +71,7 @@ "properties": { "context": {"type": "string"}, "dockerfile": {"type": "string"}, - "args": {"$ref": "#/definitions/list_or_dict"}, - "cache_from": {"type": "#/definitions/list_of_strings"} + "args": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false } @@ -168,21 +167,8 @@ "ports": { "type": "array", "items": { - "oneOf": [ - {"type": "number", "format": "ports"}, - {"type": "string", "format": "ports"}, - { - "type": "object", - "properties": { - "mode": {"type": "string"}, - "target": {"type": "integer"}, - "published": {"type": "integer"}, - "protocol": {"type": "string"} - }, - "required": ["target"], - "additionalProperties": false - } - ] + "type": ["string", "number"], + "format": "ports" }, "uniqueItems": true }, diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json new file mode 100644 index 000000000..e47c879a4 --- /dev/null +++ b/compose/config/config_schema_v3.2.json @@ -0,0 +1,473 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.1.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secret" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "additionalProperties": false + } + ] + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + } + } + } + ], + "uniqueItems": true + } + }, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "endpoint_mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + } + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "attachable": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} From 2acf286ed611651fda1a87360d04de2ddca6e649 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 13 Mar 2017 15:57:13 -0400 Subject: [PATCH 82/92] Support V3.2 Signed-off-by: Daniel Nephin --- compose/config/config.py | 37 +++++++------------ compose/config/serialize.py | 8 ++-- compose/config/validation.py | 18 ++++++--- compose/const.py | 4 ++ .../ports-composefile/expanded-notation.yml | 2 +- tests/integration/project_test.py | 6 +-- tests/integration/testcases.py | 8 ++-- tests/unit/config/config_test.py | 10 ++--- tests/unit/config/types_test.py | 4 +- 9 files changed, 49 insertions(+), 48 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 3d7cbe8d6..5a0a3c847 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -13,11 +13,8 @@ import yaml from cached_property import cached_property from . import types +from .. import const from ..const import COMPOSEFILE_V1 as V1 -from ..const import COMPOSEFILE_V2_0 as V2_0 -from ..const import COMPOSEFILE_V2_1 as V2_1 -from ..const import COMPOSEFILE_V3_0 as V3_0 -from ..const import COMPOSEFILE_V3_1 as V3_1 from ..utils import build_string_dict from ..utils import parse_nanoseconds_int from ..utils import splitdrive @@ -185,10 +182,10 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): .format(self.filename, VERSION_EXPLANATION)) if version == '2': - version = V2_0 + version = const.COMPOSEFILE_V2_0 if version == '3': - version = V3_0 + version = const.COMPOSEFILE_V3_0 return version @@ -205,7 +202,7 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): return {} if self.version == V1 else self.config.get('networks', {}) def get_secrets(self): - return {} if self.version < V3_1 else self.config.get('secrets', {}) + return {} if self.version < const.COMPOSEFILE_V3_1 else self.config.get('secrets', {}) class Config(namedtuple('_Config', 'version services volumes networks secrets')): @@ -427,7 +424,7 @@ def load_services(config_details, config_file): service_dict = process_service(resolver.run()) service_config = service_config._replace(config=service_dict) - validate_service(service_config, service_names, config_file.version) + validate_service(service_config, service_names, config_file) service_dict = finalize_service( service_config, service_names, @@ -480,7 +477,7 @@ def process_config_file(config_file, environment, service_name=None): 'service', environment) - if config_file.version in (V2_0, V2_1, V3_0, V3_1): + if config_file.version != V1: processed_config = dict(config_file.config) processed_config['services'] = services processed_config['volumes'] = interpolate_config_section( @@ -493,19 +490,13 @@ def process_config_file(config_file, environment, service_name=None): config_file.get_networks(), 'network', environment) - if config_file.version in (V3_1,): - processed_config['secrets'] = interpolate_config_section( - config_file, - config_file.get_secrets(), - 'secrets', - environment - ) - elif config_file.version == V1: - processed_config = services + processed_config['secrets'] = interpolate_config_section( + config_file, + config_file.get_secrets(), + 'secrets', + environment) else: - raise ConfigurationError( - 'Version in "{}" is unsupported. {}' - .format(config_file.filename, VERSION_EXPLANATION)) + processed_config = services config_file = config_file._replace(config=processed_config) validate_against_config_schema(config_file) @@ -642,9 +633,9 @@ def validate_extended_service_dict(service_dict, filename, service): "%s services with 'depends_on' cannot be extended" % error_prefix) -def validate_service(service_config, service_names, version): +def validate_service(service_config, service_names, config_file): service_dict, service_name = service_config.config, service_config.name - validate_service_constraints(service_dict, service_name, version) + validate_service_constraints(service_dict, service_name, config_file) validate_paths(service_dict) validate_ulimits(service_config) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 6e2ad5906..46e1d9f44 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -4,10 +4,10 @@ from __future__ import unicode_literals import six import yaml +from compose import const from compose.config import types -from compose.config.config import V1 -from compose.config.config import V2_1 -from compose.config.config import V3_1 +from compose.const import COMPOSEFILE_V1 as V1 +from compose.const import COMPOSEFILE_V2_1 as V2_1 def serialize_config_type(dumper, data): @@ -103,7 +103,7 @@ def denormalize_service_dict(service_dict, version): service_dict['healthcheck']['timeout'] ) - if 'ports' in service_dict and version != V3_1: + if 'ports' in service_dict and version < const.COMPOSEFILE_V3_2: service_dict['ports'] = map( lambda p: p.legacy_repr() if isinstance(p, types.ServicePort) else p, service_dict['ports'] diff --git a/compose/config/validation.py b/compose/config/validation.py index d4d29565f..1df6dd6b7 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -365,7 +365,7 @@ def process_config_schema_errors(error): def validate_against_config_schema(config_file): - schema = load_jsonschema(config_file.version) + schema = load_jsonschema(config_file) format_checker = FormatChecker(["ports", "expose"]) validator = Draft4Validator( schema, @@ -377,11 +377,12 @@ def validate_against_config_schema(config_file): config_file.filename) -def validate_service_constraints(config, service_name, version): +def validate_service_constraints(config, service_name, config_file): def handler(errors): - return process_service_constraint_errors(errors, service_name, version) + return process_service_constraint_errors( + errors, service_name, config_file.version) - schema = load_jsonschema(version) + schema = load_jsonschema(config_file) validator = Draft4Validator(schema['definitions']['constraints']['service']) handle_errors(validator.iter_errors(config), handler, None) @@ -390,10 +391,15 @@ def get_schema_path(): return os.path.dirname(os.path.abspath(__file__)) -def load_jsonschema(version): +def load_jsonschema(config_file): filename = os.path.join( get_schema_path(), - "config_schema_v{0}.json".format(version)) + "config_schema_v{0}.json".format(config_file.version)) + + if not os.path.exists(filename): + raise ConfigurationError( + 'Version in "{}" is unsupported. {}' + .format(config_file.filename, VERSION_EXPLANATION)) with open(filename, "r") as fh: return json.load(fh) diff --git a/compose/const.py b/compose/const.py index e694dbdae..8de693445 100644 --- a/compose/const.py +++ b/compose/const.py @@ -21,8 +21,10 @@ SECRETS_PATH = '/run/secrets' COMPOSEFILE_V1 = '1' COMPOSEFILE_V2_0 = '2.0' COMPOSEFILE_V2_1 = '2.1' + COMPOSEFILE_V3_0 = '3.0' COMPOSEFILE_V3_1 = '3.1' +COMPOSEFILE_V3_2 = '3.2' API_VERSIONS = { COMPOSEFILE_V1: '1.21', @@ -30,6 +32,7 @@ API_VERSIONS = { COMPOSEFILE_V2_1: '1.24', COMPOSEFILE_V3_0: '1.25', COMPOSEFILE_V3_1: '1.25', + COMPOSEFILE_V3_2: '1.25', } API_VERSION_TO_ENGINE_VERSION = { @@ -38,4 +41,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0', API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0', + API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0', } diff --git a/tests/fixtures/ports-composefile/expanded-notation.yml b/tests/fixtures/ports-composefile/expanded-notation.yml index 46d587363..6fbe59176 100644 --- a/tests/fixtures/ports-composefile/expanded-notation.yml +++ b/tests/fixtures/ports-composefile/expanded-notation.yml @@ -1,4 +1,4 @@ -version: '3.1' +version: '3.2' services: simple: image: busybox:latest diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index f0d21456b..e8dbe8fbf 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -15,11 +15,11 @@ from .testcases import DockerClientTestCase from compose.config import config from compose.config import ConfigurationError from compose.config import types -from compose.config.config import V2_0 -from compose.config.config import V2_1 -from compose.config.config import V3_1 from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec +from compose.const import COMPOSEFILE_V2_0 as V2_0 +from compose.const import COMPOSEFILE_V2_1 as V2_1 +from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.container import Container diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index efc1551b4..38fdcc660 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -10,12 +10,12 @@ from pytest import skip from .. import unittest from compose.cli.docker_client import docker_client from compose.config.config import resolve_environment -from compose.config.config import V1 -from compose.config.config import V2_0 -from compose.config.config import V2_1 -from compose.config.config import V3_0 from compose.config.environment import Environment from compose.const import API_VERSIONS +from compose.const import COMPOSEFILE_V1 as V1 +from compose.const import COMPOSEFILE_V2_0 as V2_0 +from compose.const import COMPOSEFILE_V2_0 as V2_1 +from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 93bae9726..c86485d7b 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -17,11 +17,6 @@ from compose.config import config from compose.config import types from compose.config.config import resolve_build_args from compose.config.config import resolve_environment -from compose.config.config import V1 -from compose.config.config import V2_0 -from compose.config.config import V2_1 -from compose.config.config import V3_0 -from compose.config.config import V3_1 from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION @@ -29,6 +24,11 @@ from compose.config.serialize import denormalize_service_dict from compose.config.serialize import serialize_config from compose.config.serialize import serialize_ns_time_value from compose.config.types import VolumeSpec +from compose.const import COMPOSEFILE_V1 as V1 +from compose.const import COMPOSEFILE_V2_0 as V2_0 +from compose.const import COMPOSEFILE_V2_1 as V2_1 +from compose.const import COMPOSEFILE_V3_0 as V3_0 +from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import IS_WINDOWS_PLATFORM from compose.utils import nanoseconds_from_time_seconds from tests import mock diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 22d7aa88a..66588d629 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -3,13 +3,13 @@ from __future__ import unicode_literals import pytest -from compose.config.config import V1 -from compose.config.config import V2_0 from compose.config.errors import ConfigurationError from compose.config.types import parse_extra_hosts from compose.config.types import ServicePort from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec +from compose.const import COMPOSEFILE_V1 as V1 +from compose.const import COMPOSEFILE_V2_0 as V2_0 def test_parse_extra_hosts_list(): From 7b1900951199628f879acce219a26619827a0b35 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 15 Mar 2017 17:11:53 -0700 Subject: [PATCH 83/92] Fix a handful of issues with 3.2 schema Signed-off-by: Joffrey F --- compose/config/config.py | 11 ++++++----- compose/config/config_schema_v3.2.json | 5 ++--- compose/config/serialize.py | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 5a0a3c847..225919415 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -490,11 +490,12 @@ def process_config_file(config_file, environment, service_name=None): config_file.get_networks(), 'network', environment) - processed_config['secrets'] = interpolate_config_section( - config_file, - config_file.get_secrets(), - 'secrets', - environment) + if config_file.version in (const.COMPOSEFILE_V3_1, const.COMPOSEFILE_V3_2): + processed_config['secrets'] = interpolate_config_section( + config_file, + config_file.get_secrets(), + 'secrets', + environment) else: processed_config = services diff --git a/compose/config/config_schema_v3.2.json b/compose/config/config_schema_v3.2.json index e47c879a4..ea702fcd5 100644 --- a/compose/config/config_schema_v3.2.json +++ b/compose/config/config_schema_v3.2.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "id": "config_schema_v3.1.json", + "id": "config_schema_v3.2.json", "type": "object", "required": ["version"], @@ -169,8 +169,7 @@ "type": "array", "items": { "oneOf": [ - {"type": "number", "format": "ports"}, - {"type": "string", "format": "ports"}, + {"type": ["string", "number"], "format": "ports"}, { "type": "object", "properties": { diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 46e1d9f44..1de1f14fb 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -4,10 +4,11 @@ from __future__ import unicode_literals import six import yaml -from compose import const from compose.config import types from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_1 as V2_1 +from compose.const import COMPOSEFILE_V3_1 as V3_1 +from compose.const import COMPOSEFILE_V3_1 as V3_2 def serialize_config_type(dumper, data): @@ -45,7 +46,7 @@ def denormalize_config(config): if 'external_name' in vol_conf: del vol_conf['external_name'] - if config.version in (V3_1,): + if config.version in (V3_1, V3_2): result['secrets'] = config.secrets return result @@ -103,7 +104,7 @@ def denormalize_service_dict(service_dict, version): service_dict['healthcheck']['timeout'] ) - if 'ports' in service_dict and version < const.COMPOSEFILE_V3_2: + if 'ports' in service_dict and version not in (V3_2,): service_dict['ports'] = map( lambda p: p.legacy_repr() if isinstance(p, types.ServicePort) else p, service_dict['ports'] From 583c673c8a1cdf5473c93d28f95433fffec55daf Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 16 Mar 2017 11:29:10 +0100 Subject: [PATCH 84/92] Add bash completion for `build --build-arg` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 2591f500b..d39a6da25 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -110,9 +110,17 @@ __docker_compose_services_stopped() { _docker_compose_build() { + case "$prev" in + --build-arg) + COMPREPLY=( $( compgen -e -- "$cur" ) ) + __docker_compose_nospace + return + ;; + esac + case "$cur" in -*) - COMPREPLY=( $( compgen -W "--force-rm --help --no-cache --pull" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--build-arg --force-rm --help --no-cache --pull" -- "$cur" ) ) ;; *) __docker_compose_services_from_build From de2dd5b3d380cbbe50a2c040e55b94cd4fe2c259 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 16 Mar 2017 15:47:01 +0100 Subject: [PATCH 85/92] Add bash completion for `--project-directory` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 2591f500b..e6e5bd6cd 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -160,6 +160,10 @@ _docker_compose_docker_compose() { _filedir "y?(a)ml" return ;; + --project-directory) + _filedir -d + return + ;; $(__docker_compose_to_extglob "$daemon_options_with_args") ) return ;; @@ -554,6 +558,7 @@ _docker_compose() { local daemon_options_with_args=" --file -f --host -H + --project-directory --project-name -p --tlscacert --tlscert From 6a773a018e1ca917d8c05abe9b6d5fceda845e6e Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 16 Mar 2017 16:01:43 +0100 Subject: [PATCH 86/92] Rename variables in bash completion The old names were supposed to designate variables that get propagated to the Docker daemon. This is not true for all contained options and therefore confused me. The new names focus on the placement after the top level command. Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index e6e5bd6cd..89485e24a 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -18,7 +18,7 @@ __docker_compose_q() { - docker-compose 2>/dev/null "${daemon_options[@]}" "$@" + docker-compose 2>/dev/null "${top_level_options[@]}" "$@" } # Transforms a multiline list of strings into a single line string @@ -164,14 +164,14 @@ _docker_compose_docker_compose() { _filedir -d return ;; - $(__docker_compose_to_extglob "$daemon_options_with_args") ) + $(__docker_compose_to_extglob "$top_level_options_with_args") ) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "$daemon_boolean_options $daemon_options_with_args --help -h --verbose --version -v" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "$top_level_boolean_options $top_level_options_with_args --help -h --verbose --version -v" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -550,12 +550,12 @@ _docker_compose() { # options for the docker daemon that have to be passed to secondary calls to # docker-compose executed by this script - local daemon_boolean_options=" + local top_level_boolean_options=" --skip-hostname-check --tls --tlsverify " - local daemon_options_with_args=" + local top_level_options_with_args=" --file -f --host -H --project-directory @@ -572,19 +572,19 @@ _docker_compose() { # search subcommand and invoke its handler. # special treatment of some top-level options local command='docker_compose' - local daemon_options=() + local top_level_options=() local counter=1 while [ $counter -lt $cword ]; do case "${words[$counter]}" in - $(__docker_compose_to_extglob "$daemon_boolean_options") ) + $(__docker_compose_to_extglob "$top_level_boolean_options") ) local opt=${words[counter]} - daemon_options+=($opt) + top_level_options+=($opt) ;; - $(__docker_compose_to_extglob "$daemon_options_with_args") ) + $(__docker_compose_to_extglob "$top_level_options_with_args") ) local opt=${words[counter]} local arg=${words[++counter]} - daemon_options+=($opt $arg) + top_level_options+=($opt $arg) ;; -*) ;; From 1a7e01c39abcf90ad70dbf0d20e9206e1bc151b1 Mon Sep 17 00:00:00 2001 From: King Chung Huang Date: Sat, 14 Jan 2017 10:54:29 -0700 Subject: [PATCH 87/92] Add image digest arguments to config serialization Add arguments for image digests in the config.serialize module to optionally pin images to digests, like bundles. Signed-off-by: King Chung Huang --- compose/config/serialize.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 1de1f14fb..5b36124d0 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -26,10 +26,13 @@ yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type) yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type) -def denormalize_config(config): +def denormalize_config(config, image_digests=None): result = {'version': V2_1 if config.version == V1 else config.version} denormalized_services = [ - denormalize_service_dict(service_dict, config.version) + denormalize_service_dict( + service_dict, + config.version, + image_digests[service_dict['name']] if image_digests else None) for service_dict in config.services ] result['services'] = { @@ -51,9 +54,9 @@ def denormalize_config(config): return result -def serialize_config(config): +def serialize_config(config, image_digests=None): return yaml.safe_dump( - denormalize_config(config), + denormalize_config(config, image_digests), default_flow_style=False, indent=2, width=80) @@ -78,9 +81,12 @@ def serialize_ns_time_value(value): return '{0}{1}'.format(*result) -def denormalize_service_dict(service_dict, version): +def denormalize_service_dict(service_dict, version, image_digest=None): service_dict = service_dict.copy() + if image_digest: + service_dict['image'] = image_digest + if 'restart' in service_dict: service_dict['restart'] = types.serialize_restart_spec( service_dict['restart'] From 1da3ac4715589cc59a0811becab04e363b1de9d3 Mon Sep 17 00:00:00 2001 From: King Chung Huang Date: Sat, 14 Jan 2017 10:56:45 -0700 Subject: [PATCH 88/92] Add --resolve-image-digests argument to config command Add a --resolve-image-digests argument to the config command that pins images to a specific image digest, just like the bundle command. This can be used to pin images in compose files being used to deploy stacks in Docker 1.13. Signed-off-by: King Chung Huang --- compose/cli/main.py | 53 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 63a0036b4..2d763a922 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -313,13 +313,56 @@ class TopLevelCommand(object): Usage: config [options] Options: - -q, --quiet Only validate the configuration, don't print - anything. - --services Print the service names, one per line. - --volumes Print the volume names, one per line. + --resolve-image-digests Pin image tags to digests. + -q, --quiet Only validate the configuration, don't print + anything. + --services Print the service names, one per line. + --volumes Print the volume names, one per line. """ + compose_config = get_config_from_options(self.project_dir, config_options) + image_digests = None + + if options['--resolve-image-digests']: + self.project = project_from_options('.', config_options) + + with errors.handle_connection_errors(self.project.client): + try: + image_digests = get_image_digests( + self.project, + allow_push=False + ) + except MissingDigests as e: + def list_images(images): + return "\n".join(" {}".format(name) for name in sorted(images)) + + paras = ["Some images are missing digests."] + + if e.needs_push: + command_hint = ( + "Use `docker-compose push {}` to push them. " + .format(" ".join(sorted(e.needs_push))) + ) + paras += [ + "The following images can be pushed:", + list_images(e.needs_push), + command_hint, + ] + + if e.needs_pull: + command_hint = ( + "Use `docker-compose pull {}` to pull them. " + .format(" ".join(sorted(e.needs_pull))) + ) + + paras += [ + "The following images need to be pulled:", + list_images(e.needs_pull), + command_hint, + ] + + raise UserError("\n\n".join(paras)) if options['--quiet']: return @@ -332,7 +375,7 @@ class TopLevelCommand(object): print('\n'.join(volume for volume in compose_config.volumes)) return - print(serialize_config(compose_config)) + print(serialize_config(compose_config, image_digests)) def create(self, options): """ From 0464476f0857e527b19b68f5046ab742ebe3b138 Mon Sep 17 00:00:00 2001 From: King Chung Huang Date: Sat, 14 Jan 2017 11:24:05 -0700 Subject: [PATCH 89/92] Add unit test for image digests in config Add two unit tests to validate that the denormalize_service_dict function still works without passing a third argument for image_digest, and correctly uses an image digest if one is provided. Signed-off-by: King Chung Huang --- tests/unit/config/config_test.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c86485d7b..49da2b473 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3654,6 +3654,25 @@ class SerializeTest(unittest.TestCase): assert denormalized_service['healthcheck']['interval'] == '100s' assert denormalized_service['healthcheck']['timeout'] == '30s' + def test_denormalize_image_has_digest(self): + service_dict = { + 'image': 'busybox' + } + image_digest = 'busybox@sha256:abcde' + + assert denormalize_service_dict(service_dict, V3_0, image_digest) == { + 'image': 'busybox@sha256:abcde' + } + + def test_denormalize_image_no_digest(self): + service_dict = { + 'image': 'busybox' + } + + assert denormalize_service_dict(service_dict, V3_0) == { + 'image': 'busybox' + } + def test_serialize_secrets(self): service_dict = { 'image': 'example/web', From 962ba5b9379c46e54d3f8a31d3e61930b90ccbfa Mon Sep 17 00:00:00 2001 From: King Chung Huang Date: Sun, 12 Feb 2017 11:18:11 -0700 Subject: [PATCH 90/92] Extract image tag to digest code into a function Extract the code in bundle() and config() that translates image tags into digests into a function named image_digests_for_project(). Signed-off-by: King Chung Huang --- compose/cli/main.py | 115 ++++++++++++++++---------------------------- 1 file changed, 41 insertions(+), 74 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 2d763a922..84cae9f53 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -263,43 +263,7 @@ class TopLevelCommand(object): if not output: output = "{}.dab".format(self.project.name) - with errors.handle_connection_errors(self.project.client): - try: - image_digests = get_image_digests( - self.project, - allow_push=options['--push-images'], - ) - except MissingDigests as e: - def list_images(images): - return "\n".join(" {}".format(name) for name in sorted(images)) - - paras = ["Some images are missing digests."] - - if e.needs_push: - command_hint = ( - "Use `docker-compose push {}` to push them. " - "You can do this automatically with `docker-compose bundle --push-images`." - .format(" ".join(sorted(e.needs_push))) - ) - paras += [ - "The following images can be pushed:", - list_images(e.needs_push), - command_hint, - ] - - if e.needs_pull: - command_hint = ( - "Use `docker-compose pull {}` to pull them. " - .format(" ".join(sorted(e.needs_pull))) - ) - - paras += [ - "The following images need to be pulled:", - list_images(e.needs_pull), - command_hint, - ] - - raise UserError("\n\n".join(paras)) + image_digests = image_digests_for_project(self.project, options['--push-images']) with open(output, 'w') as f: f.write(serialize_bundle(compose_config, image_digests)) @@ -326,43 +290,7 @@ class TopLevelCommand(object): if options['--resolve-image-digests']: self.project = project_from_options('.', config_options) - - with errors.handle_connection_errors(self.project.client): - try: - image_digests = get_image_digests( - self.project, - allow_push=False - ) - except MissingDigests as e: - def list_images(images): - return "\n".join(" {}".format(name) for name in sorted(images)) - - paras = ["Some images are missing digests."] - - if e.needs_push: - command_hint = ( - "Use `docker-compose push {}` to push them. " - .format(" ".join(sorted(e.needs_push))) - ) - paras += [ - "The following images can be pushed:", - list_images(e.needs_push), - command_hint, - ] - - if e.needs_pull: - command_hint = ( - "Use `docker-compose pull {}` to pull them. " - .format(" ".join(sorted(e.needs_pull))) - ) - - paras += [ - "The following images need to be pulled:", - list_images(e.needs_pull), - command_hint, - ] - - raise UserError("\n\n".join(paras)) + image_digests = image_digests_for_project(self.project) if options['--quiet']: return @@ -1077,6 +1005,45 @@ def timeout_from_opts(options): return None if timeout is None else int(timeout) +def image_digests_for_project(project, allow_push=False): + with errors.handle_connection_errors(project.client): + try: + return get_image_digests( + project, + allow_push=allow_push + ) + except MissingDigests as e: + def list_images(images): + return "\n".join(" {}".format(name) for name in sorted(images)) + + paras = ["Some images are missing digests."] + + if e.needs_push: + command_hint = ( + "Use `docker-compose push {}` to push them. " + .format(" ".join(sorted(e.needs_push))) + ) + paras += [ + "The following images can be pushed:", + list_images(e.needs_push), + command_hint, + ] + + if e.needs_pull: + command_hint = ( + "Use `docker-compose pull {}` to pull them. " + .format(" ".join(sorted(e.needs_pull))) + ) + + paras += [ + "The following images need to be pulled:", + list_images(e.needs_pull), + command_hint, + ] + + raise UserError("\n\n".join(paras)) + + def exitval_from_opts(options, project): exit_value_from = options.get('--exit-code-from') if exit_value_from: From 442a5f6eeb8ef2dc21fb0f580206f47beaa5ee76 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 17 Mar 2017 18:22:44 -0700 Subject: [PATCH 91/92] Add support for expanded mount/volume notation Signed-off-by: Joffrey F --- compose/config/config.py | 14 ++++- tests/acceptance/cli_test.py | 7 ++- tests/fixtures/v3-full/docker-compose.yml | 13 ++++- tests/unit/config/config_test.py | 62 +++++++++++++++++++++++ 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 225919415..8cbaae272 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1030,7 +1030,13 @@ def resolve_volume_paths(working_dir, service_dict): def resolve_volume_path(working_dir, volume): - container_path, host_path = split_path_mapping(volume) + if isinstance(volume, dict): + host_path = volume.get('source') + container_path = volume.get('target') + if host_path and volume.get('read_only'): + container_path += ':ro' + else: + container_path, host_path = split_path_mapping(volume) if host_path is not None: if host_path.startswith('.'): @@ -1112,6 +1118,8 @@ def split_path_mapping(volume_path): path. Using splitdrive so windows absolute paths won't cause issues with splitting on ':'. """ + if isinstance(volume_path, dict): + return (volume_path.get('target'), volume_path) drive, volume_config = splitdrive(volume_path) if ':' in volume_config: @@ -1123,7 +1131,9 @@ def split_path_mapping(volume_path): def join_path_mapping(pair): (container, host) = pair - if host is None: + if isinstance(host, dict): + return host + elif host is None: return container else: return ":".join((host, container)) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6a498e250..14e6f7336 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -321,7 +321,7 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['config']) assert yaml.load(result.stdout) == { - 'version': '3.0', + 'version': '3.2', 'networks': {}, 'volumes': { 'foobar': { @@ -371,6 +371,11 @@ class CLITestCase(DockerClientTestCase): 'timeout': '1s', 'retries': 5, }, + 'volumes': [ + '/host/path:/container/path:ro', + 'foobar:/container/volumepath:rw', + '/anonymous' + ], 'stop_grace_period': '20s', }, diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml index a1661ab93..27f3c6e04 100644 --- a/tests/fixtures/v3-full/docker-compose.yml +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3" +version: "3.2" services: web: image: busybox @@ -34,6 +34,17 @@ services: timeout: 1s retries: 5 + volumes: + - source: /host/path + target: /container/path + type: bind + read_only: true + - source: foobar + type: volume + target: /container/volumepath + - type: volume + target: /anonymous + stop_grace_period: 20s volumes: foobar: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 49da2b473..195efe3b9 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -29,6 +29,7 @@ from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_1 as V2_1 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_1 as V3_1 +from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import IS_WINDOWS_PLATFORM from compose.utils import nanoseconds_from_time_seconds from tests import mock @@ -964,6 +965,44 @@ class ConfigTest(unittest.TestCase): ] assert service_sort(service_dicts) == service_sort(expected) + @mock.patch.dict(os.environ) + def test_load_with_multiple_files_v3_2(self): + os.environ['COMPOSE_CONVERT_WINDOWS_PATHS'] = 'true' + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.2', + 'services': { + 'web': { + 'image': 'example/web', + 'volumes': [ + {'source': '/a', 'target': '/b', 'type': 'bind'}, + {'source': 'vol', 'target': '/x', 'type': 'volume', 'read_only': True} + ] + } + }, + 'volumes': {'vol': {}} + } + ) + + override_file = config.ConfigFile( + 'override.yaml', + { + 'version': '3.2', + 'services': { + 'web': { + 'volumes': ['/c:/b', '/anonymous'] + } + } + } + ) + details = config.ConfigDetails('.', [base_file, override_file]) + service_dicts = config.load(details).services + svc_volumes = map(lambda v: v.repr(), service_dicts[0]['volumes']) + assert sorted(svc_volumes) == sorted( + ['/anonymous', '/c:/b:rw', 'vol:/x:ro'] + ) + def test_undeclared_volume_v2(self): base_file = config.ConfigFile( 'base.yaml', @@ -1544,6 +1583,29 @@ class ConfigTest(unittest.TestCase): 'ports': types.ServicePort.parse('5432') } + def test_merge_service_dicts_heterogeneous_volumes(self): + base = { + 'volumes': ['/a:/b', '/x:/z'], + } + + override = { + 'image': 'alpine:edge', + 'volumes': [ + {'source': '/e', 'target': '/b', 'type': 'bind'}, + {'source': '/c', 'target': '/d', 'type': 'bind'} + ] + } + + actual = config.merge_service_dicts_from_files( + base, override, V3_2 + ) + + assert actual['volumes'] == [ + {'source': '/e', 'target': '/b', 'type': 'bind'}, + {'source': '/c', 'target': '/d', 'type': 'bind'}, + '/x:/z' + ] + def test_merge_logging_v1(self): base = { 'image': 'alpine:edge', From 5e0a7939458ee9c1533724b15b27e82b86864418 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Sat, 18 Mar 2017 16:41:48 -0700 Subject: [PATCH 92/92] Bump 1.12.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 120 ++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 122 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c07e9ca2..581d45bbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,126 @@ Change log ========== +1.12.0 (2017-03-21) +------------------- + +### New features + +#### Compose file version 3.2 + +- Introduced version 3.2 of the `docker-compose.yml` specification. + +- Added support for `cache_from` in the `build` section of services + +- Added support for the new expanded ports syntax in service definitions + +- Added support for the new expanded volumes syntax in service definitions + +#### Compose file version 2.1 + +- Added support for `pids_limit` in service definitions + +#### Compose file version 2.0 and up + +- Added `--volumes` option to `docker-compose config` that lists named + volumes declared for that project + +- Added support for `mem_reservation` in service definitions (2.x only) + +- Added support for `dns_opt` in service definitions (2.x only) + +#### All formats + +- Added a new `docker-compose images` command that lists images used by + the current project's containers + +- Added a `--stop` (shorthand `-s`) option to `docker-compose rm` that stops + the running containers before removing them + +- Added a `--resolve-image-digests` option to `docker-compose config` that + pins the image version for each service to a permanent digest + +- Added a `--exit-code-from SERVICE` option to `docker-compose up`. When + used, `docker-compose` will exit on any container's exit with the code + corresponding to the specified service's exit code + +- Added a `--parallel` option to `docker-compose pull` that enables images + for multiple services to be pulled simultaneously + +- Added a `--build-arg` option to `docker-compose build` + +- Added a `--volume ` (shorthand `-v`) option to + `docker-compose run` to declare runtime volumes to be mounted + +- Added a `--project-directory PATH` option to `docker-compose` that will + affect path resolution for the project + +- When using `--abort-on-container-exit` in `docker-compose up`, the exit + code for the container that caused the abort will be the exit code of + the `docker-compose up` command + +- Users can now configure which path separator character they want to use + to separate the `COMPOSE_FILE` environment value using the + `COMPOSE_PATH_SEPARATOR` environment variable + +- Added support for port range to single port in port mappings + (e.g. `8000-8010:80`) + +### Bugfixes + +- `docker-compose run --rm` now removes anonymous volumes after execution, + matching the behavior of `docker run --rm`. + +- Fixed a bug where override files containing port lists would cause a + TypeError to be raised + +- Fixed a bug where scaling services up or down would sometimes re-use + obsolete containers + +- Fixed a bug where the output of `docker-compose config` would be invalid + if the project declared anonymous volumes + +- Variable interpolation now properly occurs in the `secrets` section of + the Compose file + +- The `secrets` section now properly appears in the output of + `docker-compose config` + +- Fixed a bug where changes to some networks properties would not be + detected against previously created networks + +- Fixed a bug where `docker-compose` would crash when trying to write into + a closed pipe + +1.11.2 (2017-02-17) +------------------- + +### Bugfixes + +- Fixed a bug that was preventing secrets configuration from being + loaded properly + +- Fixed a bug where the `docker-compose config` command would fail + if the config file contained secrets definitions + +- Fixed an issue where Compose on some linux distributions would + pick up and load an outdated version of the requests library + +- Fixed an issue where socket-type files inside a build folder + would cause `docker-compose` to crash when trying to build that + service + +- Fixed an issue where recursive wildcard patterns `**` were not being + recognized in `.dockerignore` files. + +1.11.1 (2017-02-09) +------------------- + +### Bugfixes + +- Fixed a bug where the 3.1 file format was not being recognized as valid + by the Compose parser + 1.11.0 (2017-02-08) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index b2ca86f86..502e9cc4f 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.12.0dev' +__version__ = '1.12.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 31c5d3151..192b31219 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.12.0dev" +VERSION="1.12.0-rc1" IMAGE="docker/compose:$VERSION"