From 47584a37c9f0a4c69e3e5dbea68e57a8563e9a40 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jun 2018 11:49:35 -0700 Subject: [PATCH 01/46] Fix bintray API client Signed-off-by: Joffrey F --- script/release/release/bintray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/release/release/bintray.py b/script/release/release/bintray.py index 554611a40..b291a34ac 100644 --- a/script/release/release/bintray.py +++ b/script/release/release/bintray.py @@ -15,7 +15,7 @@ class BintrayAPI(requests.Session): self.base_url = 'https://api.bintray.com/' def create_repository(self, subject, repo_name, repo_type='generic'): - url = '{base}/repos/{subject}/{repo_name}'.format( + url = '{base}repos/{subject}/{repo_name}'.format( base=self.base_url, subject=subject, repo_name=repo_name, ) data = { @@ -30,7 +30,7 @@ class BintrayAPI(requests.Session): return result def delete_repository(self, subject, repo_name): - url = '{base}/repos/{subject}/{repo_name}'.format( + url = '{base}repos/{subject}/{repo_name}'.format( base=self.base_url, subject=subject, repo_name=repo_name, ) return self.delete(url) From 73663e46b9761e3d90917bfdf49b6f340c4bc4ed Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jun 2018 13:47:44 -0700 Subject: [PATCH 02/46] 3.7 --> API v1.38 Signed-off-by: Joffrey F --- compose/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/const.py b/compose/const.py index 374a09711..ffb68db01 100644 --- a/compose/const.py +++ b/compose/const.py @@ -52,7 +52,7 @@ API_VERSIONS = { COMPOSEFILE_V3_4: '1.30', COMPOSEFILE_V3_5: '1.30', COMPOSEFILE_V3_6: '1.36', - COMPOSEFILE_V3_7: '1.36', + COMPOSEFILE_V3_7: '1.38', } API_VERSION_TO_ENGINE_VERSION = { From a82986943b0e873b3b97bac2b99e2a46a8bf968e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jun 2018 13:27:42 -0700 Subject: [PATCH 03/46] Fix release script Signed-off-by: Joffrey F --- script/release/release.py | 7 +++++-- script/release/release/bintray.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index d0545a7e6..476adc4c3 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -58,8 +58,11 @@ def create_bump_commit(repository, release_branch, bintray_user, bintray_org): repository.push_branch_to_remote(release_branch) bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], bintray_user) - print('Creating data repository {} on bintray'.format(release_branch.name)) - bintray_api.create_repository(bintray_org, release_branch.name, 'generic') + if not bintray_api.repository_exists(bintray_org, release_branch.name): + print('Creating data repository {} on bintray'.format(release_branch.name)) + bintray_api.create_repository(bintray_org, release_branch.name, 'generic') + else: + print('Bintray repository {} already exists. Skipping'.format(release_branch.name)) def monitor_pr_status(pr_data): diff --git a/script/release/release/bintray.py b/script/release/release/bintray.py index b291a34ac..16bb4d514 100644 --- a/script/release/release/bintray.py +++ b/script/release/release/bintray.py @@ -29,6 +29,16 @@ class BintrayAPI(requests.Session): result.raise_for_status() return result + def repository_exists(self, subject, repo_name): + url = '{base}/repos/{subject}/{repo_name}'.format( + base=self.base_url, subject=subject, repo_name=repo_name, + ) + result = self.get(url) + if result.status_code == 404: + return False + result.raise_for_status() + return True + def delete_repository(self, subject, repo_name): url = '{base}repos/{subject}/{repo_name}'.format( base=self.base_url, subject=subject, repo_name=repo_name, From b00db08aa9e54f57c56b8090142dec6aed314e27 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 22 Jun 2018 15:56:53 -0700 Subject: [PATCH 04/46] Prevent attempts to create image names starting with - or _ Signed-off-by: Joffrey F --- compose/service.py | 4 +++- tests/integration/service_test.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 48cbc1702..e77780fd8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -363,7 +363,9 @@ class Service(object): @property def image_name(self): - return self.options.get('image', '{s.project}_{s.name}'.format(s=self)) + return self.options.get('image', '{project}_{s.name}'.format( + s=self, project=self.project.lstrip('_-') + )) @property def platform(self): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index d8f4d094a..88123152c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -1137,6 +1137,21 @@ class ServiceTest(DockerClientTestCase): service.build() assert service.image() + def test_build_with_illegal_leading_chars(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write('FROM busybox\nRUN echo "Embodiment of Scarlet Devil"\n') + service = Service( + 'build_leading_slug', client=self.client, + project='___-composetest', build={ + 'context': text_type(base_dir) + } + ) + assert service.image_name == 'composetest_build_leading_slug' + service.build() + assert service.image() + def test_start_container_stays_unprivileged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() From e8713d7cef948736351840b1ff1d494278bb4a74 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 29 Jun 2018 13:05:20 -0700 Subject: [PATCH 05/46] Docker SDK -> 3.4.1 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 05b38526a..a3d6e02d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.4.0 +docker==3.4.1 docker-pycreds==0.3.0 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index fc024078e..e0a26b0ec 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.4.0, < 4.0', + 'docker >= 3.4.1, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 28085ebee2bc1fc995cbe8614a1316357dcb8ce1 Mon Sep 17 00:00:00 2001 From: Nicholas Higgins Date: Mon, 9 Jul 2018 08:38:43 +1000 Subject: [PATCH 06/46] Attach logger to containers after crashing. Fixes #6060 Signed-off-by: Nicholas Higgins --- compose/cli/log_printer.py | 10 ++++++++++ tests/acceptance/cli_test.py | 16 ++++++++++++++++ .../logs-restart-composefile/docker-compose.yml | 7 +++++++ 3 files changed, 33 insertions(+) create mode 100644 tests/fixtures/logs-restart-composefile/docker-compose.yml diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 60bba8da6..bd6723ef2 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -210,10 +210,15 @@ def start_producer_thread(thread_args): def watch_events(thread_map, event_stream, presenters, thread_args): + crashed_containers = set() for event in event_stream: if event['action'] == 'stop': thread_map.pop(event['id'], None) + if event['action'] == 'die': + thread_map.pop(event['id'], None) + crashed_containers.add(event['id']) + if event['action'] != 'start': continue @@ -223,6 +228,11 @@ def watch_events(thread_map, event_stream, presenters, thread_args): # Container was stopped and started, we need a new thread thread_map.pop(event['id'], None) + # Container crashed so we should reattach to it + if event['id'] in crashed_containers: + event['container'].attach_log_stream() + crashed_containers.remove(event['id']) + thread_map[event['id']] = build_thread( event['container'], next(presenters), diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 43e8fa822..6da01ab1b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2253,6 +2253,22 @@ class CLITestCase(DockerClientTestCase): assert 'logs-composefile_another_1 exited with code 0' in result.stdout assert 'logs-composefile_simple_1 exited with code 137' in result.stdout + def test_logs_follow_logs_from_restarted_containers(self): + self.base_dir = 'tests/fixtures/logs-restart-composefile' + proc = start_process(self.base_dir, ['up']) + + wait_on_condition(ContainerStateCondition( + self.project.client, + 'logs-restart-composefile_another_1', + 'exited')) + + self.dispatch(['kill', 'simple']) + + result = wait_on_process(proc) + + assert result.stdout.count('logs-restart-composefile_another_1 exited with code 1') == 3 + assert result.stdout.count('world') == 3 + def test_logs_default(self): self.base_dir = 'tests/fixtures/logs-composefile' self.dispatch(['up', '-d']) diff --git a/tests/fixtures/logs-restart-composefile/docker-compose.yml b/tests/fixtures/logs-restart-composefile/docker-compose.yml new file mode 100644 index 000000000..c662a1e71 --- /dev/null +++ b/tests/fixtures/logs-restart-composefile/docker-compose.yml @@ -0,0 +1,7 @@ +simple: + image: busybox:latest + command: sh -c "echo hello && tail -f /dev/null" +another: + image: busybox:latest + command: sh -c "sleep 0.5 && echo world && /bin/false" + restart: "on-failure:2" From 9c2ffe6384e56f26dcdb7014def873a8235a979e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 Jul 2018 15:28:32 -0400 Subject: [PATCH 07/46] Avoid overriding external = False in serializer Signed-off-by: Joffrey F --- compose/config/serialize.py | 2 +- tests/unit/config/config_test.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index ccddbf532..8cb8a2808 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -78,7 +78,7 @@ def denormalize_config(config, image_digests=None): config.version >= V3_0 and config.version < v3_introduced_name_key(key)): del conf['name'] elif 'external' in conf: - conf['external'] = True + conf['external'] = bool(conf['external']) if 'attachable' in conf and config.version < V3_2: # For compatibility mode, this option is invalid in v2 diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 085a2d010..08b92a573 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -5096,3 +5096,19 @@ class SerializeTest(unittest.TestCase): serialized_config = yaml.load(serialize_config(config_dict)) serialized_service = serialized_config['services']['web'] assert serialized_service['command'] == 'echo 十六夜　咲夜' + + def test_serialize_external_false(self): + cfg = { + 'version': '3.4', + 'volumes': { + 'test': { + 'name': 'test-false', + 'external': False + } + } + } + + config_dict = config.load(build_config_details(cfg)) + serialized_config = yaml.load(serialize_config(config_dict)) + serialized_volume = serialized_config['volumes']['test'] + assert serialized_volume['external'] is False From e9aaece40d31b8cacc1f80e5506a6f65b2bbbb0b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 Jul 2018 15:46:56 -0400 Subject: [PATCH 08/46] s/release.py/release.sh/ Signed-off-by: Joffrey F --- script/release/release.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 476adc4c3..9be06b211 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -128,7 +128,7 @@ def print_final_instructions(args): "You're almost done! Please verify that everything is in order and " "you are ready to make the release public, then run the following " "command:\n{exe} -b {user} finalize {version}".format( - exe=sys.argv[0], user=args.bintray_user, version=args.release + exe='./script/release/release.sh', user=args.bintray_user, version=args.release ) ) @@ -263,13 +263,13 @@ ACTIONS = [ EPILOG = '''Example uses: * Start a new feature release (includes all changes currently in master) - release.py -b user start 1.23.0 + release.sh -b user start 1.23.0 * Start a new patch release - release.py -b user --patch 1.21.0 start 1.21.1 + release.sh -b user --patch 1.21.0 start 1.21.1 * Cancel / rollback an existing release draft - release.py -b user cancel 1.23.0 + release.sh -b user cancel 1.23.0 * Restart a previously aborted patch release - release.py -b user -p 1.21.0 resume 1.21.1 + release.sh -b user -p 1.21.0 resume 1.21.1 ''' From 8a7ee5a7d5844c60068f3436f2f327e61690551c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 Jul 2018 16:19:17 -0400 Subject: [PATCH 09/46] Add distclean to remove old build files Signed-off-by: Joffrey F --- script/release/release.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/script/release/release.py b/script/release/release.py index 9be06b211..b47c38875 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import argparse import os +import shutil import sys import time from distutils.core import run_setup @@ -133,8 +134,37 @@ def print_final_instructions(args): ) +def distclean(): + print('Running distclean...') + dirs = [ + os.path.join(REPO_ROOT, 'build'), os.path.join(REPO_ROOT, 'dist'), + os.path.join(REPO_ROOT, 'docker-compose.egg-info') + ] + files = [] + for base, dirnames, fnames in os.walk(REPO_ROOT): + for fname in fnames: + path = os.path.normpath(os.path.join(base, fname)) + if fname.endswith('.pyc'): + files.append(path) + elif fname.startswith('.coverage.'): + files.append(path) + for dirname in dirnames: + path = os.path.normpath(os.path.join(base, dirname)) + if dirname == '__pycache__': + dirs.append(path) + elif dirname == '.coverage-binfiles': + dirs.append(path) + + for file in files: + os.unlink(file) + + for folder in dirs: + shutil.rmtree(folder, ignore_errors=True) + + def resume(args): try: + distclean() repository = Repository(REPO_ROOT, args.repo) br_name = branch_name(args.release) if not repository.branch_exists(br_name): @@ -186,6 +216,7 @@ def cancel(args): bintray_api = BintrayAPI(os.environ['BINTRAY_TOKEN'], args.bintray_user) print('Removing Bintray data repository for {}'.format(args.release)) bintray_api.delete_repository(args.bintray_org, branch_name(args.release)) + distclean() except ScriptError as e: print(e) return 1 @@ -194,6 +225,7 @@ def cancel(args): def start(args): + distclean() try: repository = Repository(REPO_ROOT, args.repo) create_initial_branch(repository, args) @@ -216,6 +248,7 @@ def start(args): def finalize(args): + distclean() try: repository = Repository(REPO_ROOT, args.repo) img_manager = ImageManager(args.release) From 0b5f68098c9a9c5e24f1fbc02d08207f33f4c02f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 Jul 2018 16:25:06 -0400 Subject: [PATCH 10/46] Avoid unrelated file uploads with twine Signed-off-by: Joffrey F --- script/release/release.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/script/release/release.py b/script/release/release.py index b47c38875..736b88491 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -277,7 +277,10 @@ def finalize(args): if not merge_status.merged: raise ScriptError('Unable to merge PR #{}: {}'.format(pr_data.number, merge_status.message)) print('Uploading to PyPi') - twine_upload(['dist/*']) + twine_upload([ + 'dist/docker_compose-{}*.whl'.format(args.release), + 'dist/docker-compose-{}*.tar.gz'.format(args.release) + ]) img_manager.push_images() repository.publish_release(gh_release) except ScriptError as e: From d7f5220292a9cf79cd7b214b9fe4d88a5ad8236f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 Jul 2018 16:51:01 -0400 Subject: [PATCH 11/46] Improve finalize robustness and allow resume using special --finalize-resume flag Signed-off-by: Joffrey F --- script/release/release.py | 37 ++++++++++++++++++++++++++------ script/release/release/images.py | 4 ++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 736b88491..c6dc146a7 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -28,6 +28,7 @@ from release.utils import ScriptError from release.utils import update_init_py_version from release.utils import update_run_sh_version from release.utils import yesno +from requests.exceptions import HTTPError from twine.commands.upload import main as twine_upload @@ -162,6 +163,24 @@ def distclean(): shutil.rmtree(folder, ignore_errors=True) +def pypi_upload(args): + print('Uploading to PyPi') + try: + twine_upload([ + 'dist/docker_compose-{}*.whl'.format(args.release), + 'dist/docker-compose-{}*.tar.gz'.format(args.release) + ]) + except HTTPError as e: + if e.response.status_code == 400 and 'File already exists' in e.message: + if not args.finalize_resume: + raise ScriptError( + 'Package already uploaded on PyPi.' + ) + print('Skipping PyPi upload - package already uploaded') + else: + raise ScriptError('Unexpected HTTP error uploading package to PyPi: {}'.format(e)) + + def resume(args): try: distclean() @@ -274,13 +293,13 @@ def finalize(args): run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['sdist', 'bdist_wheel']) merge_status = pr_data.merge() - if not merge_status.merged: - raise ScriptError('Unable to merge PR #{}: {}'.format(pr_data.number, merge_status.message)) - print('Uploading to PyPi') - twine_upload([ - 'dist/docker_compose-{}*.whl'.format(args.release), - 'dist/docker-compose-{}*.tar.gz'.format(args.release) - ]) + if not merge_status.merged and not args.finalize_resume: + raise ScriptError( + 'Unable to merge PR #{}: {}'.format(pr_data.number, merge_status.message) + ) + + pypi_upload(args) + img_manager.push_images() repository.publish_release(gh_release) except ScriptError as e: @@ -355,6 +374,10 @@ def main(): '--skip-ci-checks', dest='skip_ci', action='store_true', help='If set, the program will not wait for CI jobs to complete' ) + parser.add_argument( + '--finalize-resume', dest='finalize_resume', action='store_true', + help='If set, finalize will continue through steps that have already been completed.' + ) args = parser.parse_args() if args.action == 'start': diff --git a/script/release/release/images.py b/script/release/release/images.py index 24672f2ba..b8f7ed3d6 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -81,3 +81,7 @@ class ImageManager(object): for chunk in logstream: if 'status' in chunk: print(chunk['status']) + if 'error' in chunk: + raise ScriptError( + 'Error pushing {name}: {err}'.format(name=name, err=chunk['error']) + ) From e6d18b188143bb8581cf935888cafd7cca59a463 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 10 Jul 2018 15:28:55 -0400 Subject: [PATCH 12/46] Fix --exit-code-from to reflect exit code after termination by Compose Signed-off-by: Joffrey F --- compose/cli/main.py | 53 +++++++++++++++++++----------------- tests/acceptance/cli_test.py | 9 ++++++ 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index a97205836..fa6401412 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1085,6 +1085,9 @@ class TopLevelCommand(object): ) self.project.stop(service_names=service_names, timeout=timeout) + if exit_value_from: + exit_code = compute_service_exit_code(exit_value_from, attached_containers) + sys.exit(exit_code) @classmethod @@ -1103,33 +1106,33 @@ class TopLevelCommand(object): print(get_version_info('full')) +def compute_service_exit_code(exit_value_from, attached_containers): + candidates = list(filter( + lambda c: c.service == exit_value_from, + attached_containers)) + if not candidates: + log.error( + 'No containers matching the spec "{0}" ' + 'were run.'.format(exit_value_from) + ) + return 2 + if len(candidates) > 1: + exit_values = filter( + lambda e: e != 0, + [c.inspect()['State']['ExitCode'] for c in candidates] + ) + + return exit_values[0] + return candidates[0].inspect()['State']['ExitCode'] + + def compute_exit_code(exit_value_from, attached_containers, cascade_starter, all_containers): exit_code = 0 - if exit_value_from: - candidates = list(filter( - lambda c: c.service == exit_value_from, - attached_containers)) - if not candidates: - log.error( - 'No containers matching the spec "{0}" ' - 'were run.'.format(exit_value_from) - ) - exit_code = 2 - elif len(candidates) > 1: - exit_values = filter( - lambda e: e != 0, - [c.inspect()['State']['ExitCode'] for c in candidates] - ) - - exit_code = exit_values[0] - else: - exit_code = candidates[0].inspect()['State']['ExitCode'] - else: - for e in all_containers: - if (not e.is_running and cascade_starter == e.name): - if not e.exit_code == 0: - exit_code = e.exit_code - break + for e in all_containers: + if (not e.is_running and cascade_starter == e.name): + if not e.exit_code == 0: + exit_code = e.exit_code + break return exit_code diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 43e8fa822..471b1831a 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2620,6 +2620,15 @@ class CLITestCase(DockerClientTestCase): assert 'exit-code-from_another_1 exited with code 1' in result.stdout + def test_exit_code_from_signal_stop(self): + self.base_dir = 'tests/fixtures/exit-code-from' + proc = start_process( + self.base_dir, + ['up', '--abort-on-container-exit', '--exit-code-from', 'simple'] + ) + result = wait_on_process(proc, returncode=137) # SIGKILL + assert 'exit-code-from_another_1 exited with code 1' in result.stdout + def test_images(self): self.project.get_service('simple').create_container() result = self.dispatch(['images']) From bb00352c34249f16f6c49c562461bb43dabcddfc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Jul 2018 11:09:25 -0700 Subject: [PATCH 13/46] Fix up_with_networks test Signed-off-by: Joffrey F --- tests/fixtures/networks/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/networks/docker-compose.yml b/tests/fixtures/networks/docker-compose.yml index c11fa6821..275376aef 100644 --- a/tests/fixtures/networks/docker-compose.yml +++ b/tests/fixtures/networks/docker-compose.yml @@ -2,17 +2,17 @@ version: "2" services: web: - image: busybox + image: alpine:3.7 command: top networks: ["front"] app: - image: busybox + image: alpine:3.7 command: top networks: ["front", "back"] links: - "db:database" db: - image: busybox + image: alpine:3.7 command: top networks: ["back"] From 6cb17b90ef60bce2b984c569e02fa89d277279b5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Jul 2018 11:11:34 -0700 Subject: [PATCH 14/46] 1.23.0dev Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- script/release/release.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/compose/__init__.py b/compose/__init__.py index 10ae3675f..3433b63cc 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.22.0' +__version__ = '1.23.0dev' diff --git a/script/release/release.sh b/script/release/release.sh index eddc315b7..201182657 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -17,7 +17,6 @@ fi docker run -e GITHUB_TOKEN=$GITHUB_TOKEN -e BINTRAY_TOKEN=$BINTRAY_TOKEN -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK -it \ --mount type=bind,source=$(pwd),target=/src \ - --mount type=bind,source=$(pwd)/.git,target=/src/.git \ --mount type=bind,source=$HOME/.docker,target=/root/.docker \ --mount type=bind,source=$HOME/.gitconfig,target=/root/.gitconfig \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ From 88d88d1998ea94c465cd370c4e8e06b64d34e707 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 18 Jul 2018 22:22:45 -0400 Subject: [PATCH 15/46] support newer minor version of requests Signed-off-by: Ofek Lev --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a3d6e02d6..96a98417e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==220; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.6.7 PyYAML==3.12 -requests==2.18.4 +requests==2.19.1 six==1.10.0 texttable==0.9.1 urllib3==1.21.1 diff --git a/setup.py b/setup.py index e0a26b0ec..213f66625 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires = [ 'cached-property >= 1.2.0, < 2', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', - 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', + 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.20', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', 'docker >= 3.4.1, < 4.0', From 450efd557af180f2d5e8bd275a4c30afccc70ab2 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Tue, 10 Jul 2018 13:52:57 +0200 Subject: [PATCH 16/46] macOS: Rework build scripts Allows us to build for older versions of macOS by downloading an older SDK and building OpenSSL and Python against it. Signed-off-by: Christopher Crone --- .circleci/config.yml | 43 ++----------- script/build/osx | 4 +- script/setup/osx | 123 +++++++++++++++++++++++++++--------- script/setup/osx_helpers.sh | 41 ++++++++++++ 4 files changed, 142 insertions(+), 69 deletions(-) create mode 100644 script/setup/osx_helpers.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index d422fdcc5..e3e798f51 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: test: macos: - xcode: "8.3.3" + xcode: "9.4.1" steps: - checkout - run: @@ -17,7 +17,7 @@ jobs: build-osx-binary: macos: - xcode: "8.3.3" + xcode: "9.4.1" steps: - checkout - run: @@ -25,18 +25,17 @@ jobs: command: sudo pip install --upgrade pip virtualenv - run: name: setup script - command: ./script/setup/osx + command: DEPLOYMENT_TARGET=10.11 ./script/setup/osx - run: name: build script command: ./script/build/osx - store_artifacts: path: dist/docker-compose-Darwin-x86_64 destination: docker-compose-Darwin-x86_64 - # - deploy: - # name: Deploy binary to bintray - # command: | - # OS_NAME=Darwin PKG_NAME=osx ./script/circle/bintray-deploy.sh - + - deploy: + name: Deploy binary to bintray + command: | + OS_NAME=Darwin PKG_NAME=osx ./script/circle/bintray-deploy.sh build-linux-binary: machine: @@ -54,28 +53,6 @@ jobs: command: | OS_NAME=Linux PKG_NAME=linux ./script/circle/bintray-deploy.sh - trigger-osx-binary-deploy: - # We use a separate repo to build OSX binaries meant for distribution - # with support for OSSX 10.11 (xcode 7). This job triggers a build on - # that repo. - docker: - - image: alpine:3.6 - - steps: - - run: - name: install curl - command: apk update && apk add curl - - - run: - name: API trigger - command: | - curl -X POST -H "Content-Type: application/json" -d "{\ - \"build_parameters\": {\ - \"COMPOSE_BRANCH\": \"${CIRCLE_BRANCH}\"\ - }\ - }" https://circleci.com/api/v1.1/project/github/docker/compose-osx-release?circle-token=${OSX_RELEASE_TOKEN} \ - > /dev/null - workflows: version: 2 @@ -84,9 +61,3 @@ workflows: - test - build-linux-binary - build-osx-binary - - trigger-osx-binary-deploy: - filters: - branches: - only: - - master - - /bump-.*/ diff --git a/script/build/osx b/script/build/osx index 0c4b062bb..c62b27024 100755 --- a/script/build/osx +++ b/script/build/osx @@ -1,11 +1,11 @@ #!/bin/bash set -ex -PATH="/usr/local/bin:$PATH" +TOOLCHAIN_PATH="$(realpath $(dirname $0)/../../build/toolchain)" rm -rf venv -virtualenv -p /usr/local/bin/python3 venv +virtualenv -p ${TOOLCHAIN_PATH}/bin/python3 venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-build.txt venv/bin/pip install --no-deps . diff --git a/script/setup/osx b/script/setup/osx index 972e79efb..08491b6e5 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -1,43 +1,104 @@ -#!/bin/bash +#!/usr/bin/env bash set -ex -python_version() { - python -V 2>&1 -} +. $(dirname $0)/osx_helpers.sh -python3_version() { - python3 -V 2>&1 -} +DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET:-"$(macos_version)"} +SDK_FETCH= +if ! [ ${DEPLOYMENT_TARGET} == "$(macos_version)" ]; then + SDK_FETCH=1 + # SDK URL from https://github.com/docker/golang-cross/blob/master/osx-cross.sh + SDK_URL=https://s3.dockerproject.org/darwin/v2/MacOSX${DEPLOYMENT_TARGET}.sdk.tar.xz + SDK_SHA1=dd228a335194e3392f1904ce49aff1b1da26ca62 +fi -openssl_version() { - python -c "import ssl; print ssl.OPENSSL_VERSION" -} +OPENSSL_VERSION=1.1.0h +OPENSSL_URL=https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz +OPENSSL_SHA1=0fc39f6aa91b6e7f4d05018f7c5e991e1d2491fd -desired_python3_version="3.6.4" -desired_python3_brew_version="3.6.4_2" -python3_formula="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e69a9a592232fa5a82741f6acecffc2f1d198d/Formula/python3.rb" +PYTHON_VERSION=3.6.6 +PYTHON_URL=https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz +PYTHON_SHA1=ae1fc9ddd29ad8c1d5f7b0d799ff0787efeb9652 -PATH="/usr/local/bin:$PATH" - -if !(which brew); then +# +# Install prerequisites. +# +if ! [ -x "$(command -v brew)" ]; then ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi - -brew update > /dev/null - -if !(python3_version | grep "$desired_python3_version"); then - if brew list | grep python3; then - brew unlink python3 - fi - - brew install "$python3_formula" - brew switch python3 "$desired_python3_brew_version" +if ! [ -x "$(command -v grealpath)" ]; then + brew update > /dev/null + brew install coreutils fi - -echo "*** Using $(python3_version) ; $(python_version)" -echo "*** Using $(openssl_version)" - -if !(which virtualenv); then +if ! [ -x "$(command -v python3)" ]; then + brew update > /dev/null + brew install python3 +fi +if ! [ -x "$(command -v virtualenv)" ]; then pip install virtualenv fi + +# +# Create toolchain directory. +# +BUILD_PATH="$(grealpath $(dirname $0)/../../build)" +mkdir -p ${BUILD_PATH} +TOOLCHAIN_PATH="${BUILD_PATH}/toolchain" +mkdir -p ${TOOLCHAIN_PATH} + +# +# Set macOS SDK. +# +if [ ${SDK_FETCH} ]; then + SDK_PATH=${TOOLCHAIN_PATH}/MacOSX${DEPLOYMENT_TARGET}.sdk + fetch_tarball ${SDK_URL} ${SDK_PATH} ${SDK_SHA1} +else + SDK_PATH="$(xcode-select --print-path)/Platforms/MacOSX.platform/Developer/SDKs/MacOSX${DEPLOYMENT_TARGET}.sdk" +fi + +# +# Build OpenSSL. +# +OPENSSL_SRC_PATH=${TOOLCHAIN_PATH}/openssl-${OPENSSL_VERSION} +if ! [ -f ${TOOLCHAIN_PATH}/bin/openssl ]; then + rm -rf ${OPENSSL_SRC_PATH} + fetch_tarball ${OPENSSL_URL} ${OPENSSL_SRC_PATH} ${OPENSSL_SHA1} + ( + cd ${OPENSSL_SRC_PATH} + export MACOSX_DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET} + export SDKROOT=${SDK_PATH} + ./Configure darwin64-x86_64-cc --prefix=${TOOLCHAIN_PATH} + make install_sw install_dev + ) +fi + +# +# Build Python. +# +PYTHON_SRC_PATH=${TOOLCHAIN_PATH}/Python-${PYTHON_VERSION} +if ! [ -f ${TOOLCHAIN_PATH}/bin/python3 ]; then + rm -rf ${PYTHON_SRC_PATH} + fetch_tarball ${PYTHON_URL} ${PYTHON_SRC_PATH} ${PYTHON_SHA1} + ( + cd ${PYTHON_SRC_PATH} + ./configure --prefix=${TOOLCHAIN_PATH} \ + --enable-ipv6 --without-ensurepip --with-dtrace --without-gcc \ + --datarootdir=${TOOLCHAIN_PATH}/share \ + --datadir=${TOOLCHAIN_PATH}/share \ + --enable-framework=${TOOLCHAIN_PATH}/Frameworks \ + MACOSX_DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET} \ + CFLAGS="-isysroot ${SDK_PATH} -I${TOOLCHAIN_PATH}/include" \ + CPPFLAGS="-I${SDK_PATH}/usr/include -I${TOOLCHAIN_PATH}include" \ + LDFLAGS="-isysroot ${SDK_PATH} -L ${TOOLCHAIN_PATH}/lib" + make -j 4 + make install PYTHONAPPSDIR=${TOOLCHAIN_PATH} + make frameworkinstallextras PYTHONAPPSDIR=${TOOLCHAIN_PATH}/share + ) +fi + +echo "" +echo "*** Targeting macOS: ${DEPLOYMENT_TARGET}" +echo "*** Using SDK ${SDK_PATH}" +echo "*** Using $(python3_version ${TOOLCHAIN_PATH})" +echo "*** Using $(openssl_version ${TOOLCHAIN_PATH})" diff --git a/script/setup/osx_helpers.sh b/script/setup/osx_helpers.sh new file mode 100644 index 000000000..d60a30b6d --- /dev/null +++ b/script/setup/osx_helpers.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Check file's ($1) SHA1 ($2). +check_sha1() { + echo -n "$2 *$1" | shasum -c - +} + +# Download URL ($1) to path ($2). +download() { + curl -L $1 -o $2 +} + +# Extract tarball ($1) in folder ($2). +extract() { + tar xf $1 -C $2 +} + +# Download URL ($1), check SHA1 ($3), and extract utility ($2). +fetch_tarball() { + url=$1 + tarball=$2.tarball + sha1=$3 + download $url $tarball + check_sha1 $tarball $sha1 + extract $tarball $(dirname $tarball) +} + +# Version of Python at toolchain path ($1). +python3_version() { + $1/bin/python3 -V 2>&1 +} + +# Version of OpenSSL used by toolchain ($1) Python. +openssl_version() { + $1/bin/python3 -c "import ssl; print(ssl.OPENSSL_VERSION)" +} + +# System macOS version. +macos_version() { + sw_vers -productVersion | cut -f1,2 -d'.' +} From 7f9c042300e023f63ca76449b893ade64355c290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arkadiusz=20Dzi=C4=99giel?= Date: Tue, 24 Jul 2018 12:21:37 +0200 Subject: [PATCH 17/46] Fixes pipe handling in container mode. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #4599, #4460 - adds a way to provide options from env in both cases (tty & non tty) - allocates TTY only if both stdin & stdout are TTYs - enables interactive mode if stdin is not TTY Signed-off-by: Arkadiusz Dzięgiel --- script/run/run.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/script/run/run.sh b/script/run/run.sh index 52ff9513f..fe253875e 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -47,10 +47,11 @@ if [ -n "$HOME" ]; then fi # Only allocate tty if we detect one -if [ -t 1 ]; then - DOCKER_RUN_OPTIONS="-t" -fi if [ -t 0 ]; then + if [ -t 1 ]; then + DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t" + fi +else DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i" fi From c956785cdc242f081043f3879047c428f0bd893a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Jul 2018 15:37:15 -0700 Subject: [PATCH 18/46] Add progress messages to parallel pull Signed-off-by: Joffrey F --- compose/parallel.py | 7 ++++++ compose/progress_stream.py | 5 +--- compose/project.py | 25 +++++++++++++++++-- compose/service.py | 39 ++++++++++++++++++------------ tests/integration/testcases.py | 4 ++- tests/unit/progress_stream_test.py | 12 ++++----- 6 files changed, 63 insertions(+), 29 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index a2eb160e5..34a498ca7 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -313,6 +313,13 @@ class ParallelStreamWriter(object): self._write_ansi(msg, obj_index, color_func(status)) +def get_stream_writer(): + instance = ParallelStreamWriter.instance + if instance is None: + raise RuntimeError('ParallelStreamWriter has not yet been instantiated') + return instance + + def parallel_operation(containers, operation, options, message): parallel_execute( containers, diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 5e709770a..4cd311432 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -19,12 +19,11 @@ def write_to_stream(s, stream): def stream_output(output, stream): is_terminal = hasattr(stream, 'isatty') and stream.isatty() stream = utils.get_output_stream(stream) - all_events = [] lines = {} diff = 0 for event in utils.json_stream(output): - all_events.append(event) + yield event is_progress_event = 'progress' in event or 'progressDetail' in event if not is_progress_event: @@ -57,8 +56,6 @@ def stream_output(output, stream): stream.flush() - return all_events - def print_output_event(event, stream, is_terminal): if 'errorDetail' in event: diff --git a/compose/project.py b/compose/project.py index 005b7e240..391bbd038 100644 --- a/compose/project.py +++ b/compose/project.py @@ -548,16 +548,37 @@ class Project(object): def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False, include_deps=False): services = self.get_services(service_names, include_deps) + msg = not silent and 'Pulling' or None if parallel_pull: def pull_service(service): - service.pull(ignore_pull_failures, True) + strm = service.pull(ignore_pull_failures, True, stream=True) + writer = parallel.get_stream_writer() + + def trunc(s): + if len(s) > 35: + return s[:33] + '...' + return s + + for event in strm: + if 'status' not in event: + continue + status = event['status'].lower() + if 'progressDetail' in event: + detail = event['progressDetail'] + if 'current' in detail and 'total' in detail: + percentage = float(detail['current']) / float(detail['total']) + status = '{} ({:.1%})'.format(status, percentage) + + writer.write( + msg, service.name, trunc(status), lambda s: s + ) _, errors = parallel.parallel_execute( services, pull_service, operator.attrgetter('name'), - not silent and 'Pulling' or None, + msg, limit=5, ) if len(errors): diff --git a/compose/service.py b/compose/service.py index e77780fd8..4b545ab02 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1068,7 +1068,7 @@ class Service(object): ) try: - all_events = stream_output(build_output, sys.stdout) + all_events = list(stream_output(build_output, sys.stdout)) except StreamOutputError as e: raise BuildError(self, six.text_type(e)) @@ -1162,7 +1162,23 @@ class Service(object): return any(has_host_port(binding) for binding in self.options.get('ports', [])) - def pull(self, ignore_pull_failures=False, silent=False): + def _do_pull(self, repo, pull_kwargs, silent, ignore_pull_failures): + try: + output = self.client.pull(repo, **pull_kwargs) + if silent: + with open(os.devnull, 'w') as devnull: + for event in stream_output(output, devnull): + yield event + else: + for event in stream_output(output, sys.stdout): + yield event + except (StreamOutputError, NotFound) as e: + if not ignore_pull_failures: + raise + else: + log.error(six.text_type(e)) + + def pull(self, ignore_pull_failures=False, silent=False, stream=False): if 'image' not in self.options: return @@ -1179,20 +1195,11 @@ class Service(object): raise OperationFailedError( 'Impossible to perform platform-targeted pulls for API version < 1.35' ) - try: - output = self.client.pull(repo, **kwargs) - if silent: - with open(os.devnull, 'w') as devnull: - return progress_stream.get_digest_from_pull( - stream_output(output, devnull)) - else: - return progress_stream.get_digest_from_pull( - stream_output(output, sys.stdout)) - except (StreamOutputError, NotFound) as e: - if not ignore_pull_failures: - raise - else: - log.error(six.text_type(e)) + + event_stream = self._do_pull(repo, kwargs, silent, ignore_pull_failures) + if stream: + return event_stream + return progress_stream.get_digest_from_pull(event_stream) def push(self, ignore_push_failures=False): if 'image' not in self.options or 'build' not in self.options: diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 4440d771e..cfdf22f7e 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -139,7 +139,9 @@ class DockerClientTestCase(unittest.TestCase): def check_build(self, *args, **kwargs): kwargs.setdefault('rm', True) build_output = self.client.build(*args, **kwargs) - stream_output(build_output, open('/dev/null', 'w')) + with open(os.devnull, 'w') as devnull: + for event in stream_output(build_output, devnull): + pass def require_api_version(self, minimum): api_version = self.client.version()['ApiVersion'] diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index f4a0ab063..d29227458 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -21,7 +21,7 @@ class ProgressStreamTestCase(unittest.TestCase): b'31019763, "start": 1413653874, "total": 62763875}, ' b'"progress": "..."}', ] - events = progress_stream.stream_output(output, StringIO()) + events = list(progress_stream.stream_output(output, StringIO())) assert len(events) == 1 def test_stream_output_div_zero(self): @@ -30,7 +30,7 @@ class ProgressStreamTestCase(unittest.TestCase): b'0, "start": 1413653874, "total": 0}, ' b'"progress": "..."}', ] - events = progress_stream.stream_output(output, StringIO()) + events = list(progress_stream.stream_output(output, StringIO())) assert len(events) == 1 def test_stream_output_null_total(self): @@ -39,7 +39,7 @@ class ProgressStreamTestCase(unittest.TestCase): b'0, "start": 1413653874, "total": null}, ' b'"progress": "..."}', ] - events = progress_stream.stream_output(output, StringIO()) + events = list(progress_stream.stream_output(output, StringIO())) assert len(events) == 1 def test_stream_output_progress_event_tty(self): @@ -52,7 +52,7 @@ class ProgressStreamTestCase(unittest.TestCase): return True output = TTYStringIO() - events = progress_stream.stream_output(events, output) + events = list(progress_stream.stream_output(events, output)) assert len(output.getvalue()) > 0 def test_stream_output_progress_event_no_tty(self): @@ -61,7 +61,7 @@ class ProgressStreamTestCase(unittest.TestCase): ] output = StringIO() - events = progress_stream.stream_output(events, output) + events = list(progress_stream.stream_output(events, output)) assert len(output.getvalue()) == 0 def test_stream_output_no_progress_event_no_tty(self): @@ -70,7 +70,7 @@ class ProgressStreamTestCase(unittest.TestCase): ] output = StringIO() - events = progress_stream.stream_output(events, output) + events = list(progress_stream.stream_output(events, output)) assert len(output.getvalue()) > 0 def test_mismatched_encoding_stream_write(self): From 89f2bfe4f392b1bb74ab85af01dc95d2d6b47202 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Tue, 31 Jul 2018 11:54:32 -0400 Subject: [PATCH 19/46] add --parallel option to build Signed-off-by: Gil Raphaelli --- compose/cli/main.py | 2 ++ compose/project.py | 27 +++++++++++++++++-- tests/acceptance/cli_test.py | 7 +++++ .../build-multiple-composefile/a/Dockerfile | 4 +++ .../build-multiple-composefile/b/Dockerfile | 4 +++ .../docker-compose.yml | 8 ++++++ 6 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/build-multiple-composefile/a/Dockerfile create mode 100644 tests/fixtures/build-multiple-composefile/b/Dockerfile create mode 100644 tests/fixtures/build-multiple-composefile/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index fa6401412..d224093cb 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -260,6 +260,7 @@ class TopLevelCommand(object): --pull Always attempt to pull a newer version of the image. -m, --memory MEM Sets memory limit for the build container. --build-arg key=val Set build-time variables for services. + --parallel Build images in parallel. """ service_names = options['SERVICE'] build_args = options.get('--build-arg', None) @@ -280,6 +281,7 @@ class TopLevelCommand(object): memory=options.get('--memory'), build_args=build_args, gzip=options.get('--compress', False), + parallel_build=options.get('--parallel', False), ) def bundle(self, options): diff --git a/compose/project.py b/compose/project.py index 005b7e240..a7ef327e6 100644 --- a/compose/project.py +++ b/compose/project.py @@ -372,13 +372,36 @@ class Project(object): return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None, - build_args=None, gzip=False): + build_args=None, gzip=False, parallel_build=False): + + services = [] for service in self.get_services(service_names): if service.can_be_built(): - service.build(no_cache, pull, force_rm, memory, build_args, gzip) + services.append(service) else: log.info('%s uses an image, skipping' % service.name) + def build_service(service): + service.build(no_cache, pull, force_rm, memory, build_args, gzip) + + if parallel_build: + _, errors = parallel.parallel_execute( + services, + build_service, + operator.attrgetter('name'), + 'Building', + limit=5, + ) + if len(errors): + combined_errors = '\n'.join([ + e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values() + ]) + raise ProjectError(combined_errors) + + else: + for service in services: + build_service(service) + def create( self, service_names=None, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6c6cf32f5..2361a1fbf 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -773,6 +773,13 @@ class CLITestCase(DockerClientTestCase): assert 'does not exist, is not accessible, or is not a valid URL' in result.stderr + def test_build_parallel(self): + self.base_dir = 'tests/fixtures/build-multiple-composefile' + result = self.dispatch(['build', '--parallel']) + assert 'Successfully tagged build-multiple-composefile_a:latest' in result.stdout + assert 'Successfully tagged build-multiple-composefile_b:latest' in result.stdout + assert 'Successfully built' in result.stdout + def test_create(self): self.dispatch(['create']) service = self.project.get_service('simple') diff --git a/tests/fixtures/build-multiple-composefile/a/Dockerfile b/tests/fixtures/build-multiple-composefile/a/Dockerfile new file mode 100644 index 000000000..2ba45ce55 --- /dev/null +++ b/tests/fixtures/build-multiple-composefile/a/Dockerfile @@ -0,0 +1,4 @@ + +FROM busybox:latest +RUN echo a +CMD top diff --git a/tests/fixtures/build-multiple-composefile/b/Dockerfile b/tests/fixtures/build-multiple-composefile/b/Dockerfile new file mode 100644 index 000000000..e282e8bbf --- /dev/null +++ b/tests/fixtures/build-multiple-composefile/b/Dockerfile @@ -0,0 +1,4 @@ + +FROM busybox:latest +RUN echo b +CMD top diff --git a/tests/fixtures/build-multiple-composefile/docker-compose.yml b/tests/fixtures/build-multiple-composefile/docker-compose.yml new file mode 100644 index 000000000..efa70d7e0 --- /dev/null +++ b/tests/fixtures/build-multiple-composefile/docker-compose.yml @@ -0,0 +1,8 @@ + +version: "2" + +services: + a: + build: ./a + b: + build: ./b From 541fb6525991ae7110bf512a3313c0e13ee7a42d Mon Sep 17 00:00:00 2001 From: Fender William Date: Tue, 24 Apr 2018 07:41:55 +0200 Subject: [PATCH 20/46] Add --hash opt for config command Signed-off-by: Fender William --- .gitignore | 1 + compose/cli/main.py | 17 ++++++++++++++++- contrib/completion/bash/docker-compose | 2 +- contrib/completion/zsh/_docker-compose | 3 ++- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 11266c2e3..18afd643d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ compose/GITSHA *.swp .DS_Store .cache +.idea diff --git a/compose/cli/main.py b/compose/cli/main.py index d224093cb..231767424 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -328,7 +328,8 @@ class TopLevelCommand(object): anything. --services Print the service names, one per line. --volumes Print the volume names, one per line. - + --hash="all" Print the service config hash, one per line. + Set "service1,service2" for a list of specified services. """ compose_config = get_config_from_options(self.project_dir, self.toplevel_options) @@ -350,6 +351,20 @@ class TopLevelCommand(object): print('\n'.join(volume for volume in compose_config.volumes)) return + if options['--hash'] is not None: + self.project = project_from_options('.', self.toplevel_options) + if options['--hash'] == "all": + for service in self.project.services: + print('{} {}'.format(service.name, service.config_hash)) + else: + for service_name in options['--hash'].split(','): + try: + print('{} {}'.format(service_name, + self.project.get_service(service_name).config_hash)) + except NoSuchService as s: + print('{}'.format(s)) + return + print(serialize_config(compose_config, image_digests)) def create(self, options): diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index b90af45d1..f4c42362c 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -136,7 +136,7 @@ _docker_compose_bundle() { _docker_compose_config() { - COMPREPLY=( $( compgen -W "--help --quiet -q --resolve-image-digests --services --volumes" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --quiet -q --resolve-image-digests --services --volumes --hash" -- "$cur" ) ) } diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index aba367706..676aa117b 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -213,7 +213,8 @@ __docker-compose_subcommand() { '(--quiet -q)'{--quiet,-q}"[Only validate the configuration, don't print anything.]" \ '--resolve-image-digests[Pin image tags to digests.]' \ '--services[Print the service names, one per line.]' \ - '--volumes[Print the volume names, one per line.]' && ret=0 + '--volumes[Print the volume names, one per line.]' \ + '--hash[Print the service config hash, one per line. Set "service1,service2" for a list of specified services.]' \ && ret=0 ;; (create) _arguments \ From 707e21183f5117f4f2b5005615a7c483958b5b3d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 7 Aug 2018 16:45:34 -0700 Subject: [PATCH 21/46] Fix config hash consistency with unprioritized networks Signed-off-by: Joffrey F --- compose/network.py | 13 +++++++++---- tests/unit/service_test.py | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/compose/network.py b/compose/network.py index 9751f2037..2491a5989 100644 --- a/compose/network.py +++ b/compose/network.py @@ -323,7 +323,12 @@ def get_networks(service_dict, network_definitions): 'Service "{}" uses an undefined network "{}"' .format(service_dict['name'], name)) - return OrderedDict(sorted( - networks.items(), - key=lambda t: t[1].get('priority') or 0, reverse=True - )) + if any([v.get('priority') for v in networks.values()]): + return OrderedDict(sorted( + networks.items(), + key=lambda t: t[1].get('priority') or 0, reverse=True + )) + else: + # Ensure Compose will pick a consistent primary network if no + # priority is set + return OrderedDict(sorted(networks.items(), key=lambda t: t[0])) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index f5a35d814..791019a4e 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -701,9 +701,11 @@ class ServiceTest(unittest.TestCase): image='example.com/foo', client=self.mock_client, network_mode=NetworkMode('bridge'), - networks={'bridge': {}}, + networks={'bridge': {}, 'net2': {}}, links=[(Service('one', client=self.mock_client), 'one')], - volumes_from=[VolumeFromSpec(Service('two', client=self.mock_client), 'rw', 'service')] + volumes_from=[VolumeFromSpec(Service('two', client=self.mock_client), 'rw', 'service')], + volumes=[VolumeSpec('/ext', '/int', 'ro')], + build={'context': 'some/random/path'}, ) config_hash = service.config_hash From 861031b9b7ed83866a73b48dc1c17119cd0a708e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 7 Aug 2018 16:47:34 -0700 Subject: [PATCH 22/46] Reduce config --hash code complexity and add test Signed-off-by: Joffrey F --- compose/cli/main.py | 20 ++++++++------------ tests/acceptance/cli_test.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 231767424..4c18d19f7 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -328,8 +328,9 @@ class TopLevelCommand(object): anything. --services Print the service names, one per line. --volumes Print the volume names, one per line. - --hash="all" Print the service config hash, one per line. - Set "service1,service2" for a list of specified services. + --hash="*" Print the service config hash, one per line. + Set "service1,service2" for a list of specified services + or use the wildcard symbol to display all services """ compose_config = get_config_from_options(self.project_dir, self.toplevel_options) @@ -352,17 +353,12 @@ class TopLevelCommand(object): return if options['--hash'] is not None: + h = options['--hash'] self.project = project_from_options('.', self.toplevel_options) - if options['--hash'] == "all": - for service in self.project.services: - print('{} {}'.format(service.name, service.config_hash)) - else: - for service_name in options['--hash'].split(','): - try: - print('{} {}'.format(service_name, - self.project.get_service(service_name).config_hash)) - except NoSuchService as s: - print('{}'.format(s)) + services = [svc for svc in options['--hash'].split(',')] if h != '*' else None + + for service in self.project.get_services(services): + print('{} {}'.format(service.name, service.config_hash)) return print(serialize_config(compose_config, image_digests)) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 2361a1fbf..815b92c8d 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -222,6 +222,17 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/v2-full' assert self.dispatch(['config', '--quiet']).stdout == '' + def test_config_with_hash_option(self): + self.base_dir = 'tests/fixtures/v2-full' + self.project.build() + result = self.dispatch(['config', '--hash=*']) + for service in self.project.get_services(): + assert '{} {}\n'.format(service.name, service.config_hash) in result.stdout + + svc = self.project.get_service('other') + result = self.dispatch(['config', '--hash=other']) + assert result.stdout == '{} {}\n'.format(svc.name, svc.config_hash) + def test_config_default(self): self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config']) From ee878aee4cddad7d652f4908ff1b1ddd5474fbbd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 7 Aug 2018 17:32:31 -0700 Subject: [PATCH 23/46] Handle missing (not built) service image in config --hash Signed-off-by: Joffrey F --- compose/cli/main.py | 6 +++--- compose/service.py | 8 +++++++- tests/acceptance/cli_test.py | 1 - 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 4c18d19f7..07447d671 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -356,9 +356,9 @@ class TopLevelCommand(object): h = options['--hash'] self.project = project_from_options('.', self.toplevel_options) services = [svc for svc in options['--hash'].split(',')] if h != '*' else None - - for service in self.project.get_services(services): - print('{} {}'.format(service.name, service.config_hash)) + with errors.handle_connection_errors(self.project.client): + for service in self.project.get_services(services): + print('{} {}'.format(service.name, service.config_hash)) return print(serialize_config(compose_config, image_digests)) diff --git a/compose/service.py b/compose/service.py index e77780fd8..a31f75a3d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -656,9 +656,15 @@ class Service(object): return json_hash(self.config_dict()) def config_dict(self): + def image_id(): + try: + return self.image()['Id'] + except NoSuchImageError: + return None + return { 'options': self.options, - 'image_id': self.image()['Id'], + 'image_id': image_id(), 'links': self.get_link_names(), 'net': self.network_mode.id, 'networks': self.networks, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 815b92c8d..f9d2821b0 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -224,7 +224,6 @@ class CLITestCase(DockerClientTestCase): def test_config_with_hash_option(self): self.base_dir = 'tests/fixtures/v2-full' - self.project.build() result = self.dispatch(['config', '--hash=*']) for service in self.project.get_services(): assert '{} {}\n'.format(service.name, service.config_hash) in result.stdout From 5ad50dc0b3f978e4911dd24279c4d99d3bc3c51e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Aug 2018 17:51:39 -0700 Subject: [PATCH 24/46] Bump Python SDK -> 3.5.0 Add support for Python 3.7 Signed-off-by: Joffrey F --- .circleci/config.yml | 2 +- Jenkinsfile | 3 ++- appveyor.yml | 2 +- requirements-dev.txt | 2 +- requirements.txt | 6 +++--- script/build/windows.ps1 | 2 +- setup.py | 3 ++- tox.ini | 2 +- 8 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e3e798f51..f4e90d6de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ jobs: command: sudo pip install --upgrade tox==2.1.1 - run: name: unit tests - command: tox -e py27,py36 -- tests/unit + command: tox -e py27,py36,py37 -- tests/unit build-osx-binary: macos: diff --git a/Jenkinsfile b/Jenkinsfile index 44cd7c3c2..04f5cfbda 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -74,10 +74,11 @@ buildImage() def testMatrix = [failFast: true] def docker_versions = get_versions(2) -for (int i = 0 ;i < docker_versions.length ; i++) { +for (int i = 0; i < docker_versions.length; i++) { def dockerVersion = docker_versions[i] testMatrix["${dockerVersion}_py27"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py27"]) testMatrix["${dockerVersion}_py36"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py36"]) + testMatrix["${dockerVersion}_py37"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py37"]) } parallel(testMatrix) diff --git a/appveyor.yml b/appveyor.yml index f027a1180..da80d01d9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,7 +10,7 @@ install: build: false test_script: - - "tox -e py27,py36 -- tests/unit" + - "tox -e py27,py36,py37 -- tests/unit" - ps: ".\\script\\build\\windows.ps1" artifacts: diff --git a/requirements-dev.txt b/requirements-dev.txt index 32c5c23a1..4d74f6d15 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ coverage==4.4.2 flake8==3.5.0 mock>=1.0.1 -pytest==2.9.2 +pytest==3.6.3 pytest-cov==2.5.1 diff --git a/requirements.txt b/requirements.txt index 96a98417e..41d21172e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.4.1 +docker==3.5.0 docker-pycreds==0.3.0 dockerpty==0.4.1 docopt==0.6.2 @@ -13,11 +13,11 @@ idna==2.5 ipaddress==1.0.18 jsonschema==2.6.0 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' -pypiwin32==220; sys_platform == 'win32' and python_version >= '3.6' +pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.6.7 PyYAML==3.12 requests==2.19.1 six==1.10.0 texttable==0.9.1 -urllib3==1.21.1 +urllib3==1.21.1; python_version == '3.3' websocket-client==0.32.0 diff --git a/script/build/windows.ps1 b/script/build/windows.ps1 index 1de9bbfa4..41dc51e31 100644 --- a/script/build/windows.ps1 +++ b/script/build/windows.ps1 @@ -44,7 +44,7 @@ virtualenv .\venv # pip and pyinstaller generate lots of warnings, so we need to ignore them $ErrorActionPreference = "Continue" -.\venv\Scripts\pip install pypiwin32==220 +.\venv\Scripts\pip install pypiwin32==223 .\venv\Scripts\pip install -r requirements.txt .\venv\Scripts\pip install --no-deps . .\venv\Scripts\pip install -r requirements-build.txt diff --git a/setup.py b/setup.py index 213f66625..2819810c2 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.20', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.4.1, < 4.0', + 'docker >= 3.5.0, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', @@ -100,5 +100,6 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ], ) diff --git a/tox.ini b/tox.ini index 33347df20..08efd4e68 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py36,pre-commit +envlist = py27,py36,py37,pre-commit [testenv] usedevelop=True From eb63e9f3c76a41e21ac27608b3cd0576dc19d49a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 10 Aug 2018 17:02:56 -0700 Subject: [PATCH 25/46] Fix --project-directory handling to apply to .env files as well Signed-off-by: Joffrey F --- compose/cli/main.py | 7 +++-- tests/acceptance/cli_test.py | 30 +++++++++++++++++++ tests/fixtures/default-env-file/alt/.env | 4 +++ .../default-env-file/docker-compose.yml | 4 ++- 4 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/default-env-file/alt/.env diff --git a/compose/cli/main.py b/compose/cli/main.py index 07447d671..c3e8f2789 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -238,11 +238,14 @@ class TopLevelCommand(object): version Show the Docker-Compose version information """ - def __init__(self, project, project_dir='.', options=None): + def __init__(self, project, options=None): self.project = project - self.project_dir = '.' self.toplevel_options = options or {} + @property + def project_dir(self): + return self.toplevel_options.get('--project-directory') or '.' + def build(self, options): """ Build or rebuild services. diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index f9d2821b0..66e7d4c38 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -303,6 +303,36 @@ class CLITestCase(DockerClientTestCase): } } + def test_config_with_dot_env(self): + self.base_dir = 'tests/fixtures/default-env-file' + result = self.dispatch(['config']) + json_result = yaml.load(result.stdout) + assert json_result == { + 'services': { + 'web': { + 'command': 'true', + 'image': 'alpine:latest', + 'ports': ['5643/tcp', '9999/tcp'] + } + }, + 'version': '2.4' + } + + def test_config_with_dot_env_and_override_dir(self): + self.base_dir = 'tests/fixtures/default-env-file' + result = self.dispatch(['--project-directory', 'alt/', 'config']) + json_result = yaml.load(result.stdout) + assert json_result == { + 'services': { + 'web': { + 'command': 'echo uwu', + 'image': 'alpine:3.4', + 'ports': ['3341/tcp', '4449/tcp'] + } + }, + 'version': '2.4' + } + def test_config_external_volume_v2(self): self.base_dir = 'tests/fixtures/volumes' result = self.dispatch(['-f', 'external-volumes-v2.yml', 'config']) diff --git a/tests/fixtures/default-env-file/alt/.env b/tests/fixtures/default-env-file/alt/.env new file mode 100644 index 000000000..163668d22 --- /dev/null +++ b/tests/fixtures/default-env-file/alt/.env @@ -0,0 +1,4 @@ +IMAGE=alpine:3.4 +COMMAND=echo uwu +PORT1=3341 +PORT2=4449 diff --git a/tests/fixtures/default-env-file/docker-compose.yml b/tests/fixtures/default-env-file/docker-compose.yml index aa8e4409e..793635861 100644 --- a/tests/fixtures/default-env-file/docker-compose.yml +++ b/tests/fixtures/default-env-file/docker-compose.yml @@ -1,4 +1,6 @@ -web: +version: '2.4' +services: + web: image: ${IMAGE} command: ${COMMAND} ports: From 3a93e8576247a5d6828cd98afd7816590bc9f072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Fri, 17 Aug 2018 14:08:41 -0300 Subject: [PATCH 26/46] Fix broken url MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per https://github.com/sgerrand/alpine-pkg-glibc#please-note. Signed-off-by: David Rodríguez --- Dockerfile.run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.run b/Dockerfile.run index c403ac230..e9ba19fd4 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -4,7 +4,7 @@ ENV GLIBC 2.27-r0 ENV DOCKERBINS_SHA 1270dce1bd7e1838d62ae21d2505d87f16efc1d9074645571daaefdfd0c14054 RUN apk update && apk add --no-cache openssl ca-certificates curl libgcc && \ - curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \ + curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \ curl -fsSL -o glibc-$GLIBC.apk https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ apk add --no-cache glibc-$GLIBC.apk && \ ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ From a541d88d570617ea3ef2694798170386d03d7ad4 Mon Sep 17 00:00:00 2001 From: Josenivaldo Benito Jr Date: Wed, 5 Sep 2018 11:52:50 -0300 Subject: [PATCH 27/46] [armhf] Make Dockerfile.armhf compatible with main Dockerfile now uses python:3.6 image while Dockerfile.armhf uses debian. Python image is officially supported in ARM archtecture hence, the now both dockerfiles differs only on dockerbins.tgz file version. May we use environmental variables to select dockerbins.tgz? Signed-off-by: Josenivaldo Benito Jr --- Dockerfile.armhf | 46 ++++++---------------------------------------- 1 file changed, 6 insertions(+), 40 deletions(-) diff --git a/Dockerfile.armhf b/Dockerfile.armhf index ce4ab7c13..ee2ce8941 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -1,55 +1,21 @@ -FROM armhf/debian:wheezy +FROM python:3.6 RUN set -ex; \ apt-get update -qq; \ apt-get install -y \ locales \ - gcc \ - make \ - zlib1g \ - zlib1g-dev \ - libssl-dev \ - git \ - ca-certificates \ curl \ - libsqlite3-dev \ - libbz2-dev \ - ; \ - rm -rf /var/lib/apt/lists/* + python-dev \ + git RUN curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/armhf/docker-17.12.0-ce.tgz" && \ + SHA256=f8de6378dad825b9fd5c3c2f949e791d22f918623c27a72c84fd6975a0e5d0a2; \ + echo "${SHA256} dockerbins.tgz" | sha256sum -c - && \ tar xvf dockerbins.tgz docker/docker --strip-components 1 && \ mv docker /usr/local/bin/docker && \ chmod +x /usr/local/bin/docker && \ rm dockerbins.tgz -# Build Python 2.7.13 from source -RUN set -ex; \ - curl -L https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz | tar -xz; \ - cd Python-2.7.13; \ - ./configure --enable-shared; \ - make; \ - make install; \ - cd ..; \ - rm -rf /Python-2.7.13 - -# Build python 3.6 from source -RUN set -ex; \ - curl -L https://www.python.org/ftp/python/3.6.4/Python-3.6.4.tgz | tar -xz; \ - cd Python-3.6.4; \ - ./configure --enable-shared; \ - make; \ - make install; \ - cd ..; \ - rm -rf /Python-3.6.4 - -# Make libpython findable -ENV LD_LIBRARY_PATH /usr/local/lib - -# Install pip -RUN set -ex; \ - curl -L https://bootstrap.pypa.io/get-pip.py | python - # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen ENV LANG en_US.UTF-8 @@ -70,4 +36,4 @@ RUN tox --notest ADD . /code/ RUN chown -R user /code/ -ENTRYPOINT ["/code/.tox/py27/bin/docker-compose"] +ENTRYPOINT ["/code/.tox/py36/bin/docker-compose"] From 5713215e847d4551b329234ff173a3e3299b28d5 Mon Sep 17 00:00:00 2001 From: ruicao Date: Fri, 7 Sep 2018 16:08:19 +0800 Subject: [PATCH 28/46] Typo fix: overriden -> overridden Signed-off-by: ruicao --- script/test/default | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test/default b/script/test/default index aabb4e426..cbb6a67cb 100755 --- a/script/test/default +++ b/script/test/default @@ -5,7 +5,7 @@ set -ex TAG="docker-compose:$(git rev-parse --short HEAD)" -# By default use the Dockerfile, but can be overriden to use an alternative file +# By default use the Dockerfile, but can be overridden to use an alternative file # e.g DOCKERFILE=Dockerfile.armhf script/test/default DOCKERFILE="${DOCKERFILE:-Dockerfile}" From b66782b412b794b0a8fbd276485f47ac924f80f6 Mon Sep 17 00:00:00 2001 From: Xiaoxi He Date: Fri, 7 Sep 2018 14:48:27 +0800 Subject: [PATCH 29/46] Fix typos in CHANGELOG Signed-off-by: Xiaoxi He --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b791c1e08..d22c16454 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,7 +60,7 @@ Change log ### Bugfixes -- Fixed a bug where the ip_range attirbute in IPAM configs was prevented +- Fixed a bug where the ip_range attribute in IPAM configs was prevented from passing validation 1.21.1 (2018-04-27) @@ -285,7 +285,7 @@ Change log preventing Compose from recovering volume data from previous containers for anonymous volumes -- Added limit for number of simulatenous parallel operations, which should +- Added limit for number of simultaneous parallel operations, which should prevent accidental resource exhaustion of the server. Default is 64 and can be configured using the `COMPOSE_PARALLEL_LIMIT` environment variable @@ -583,7 +583,7 @@ Change log ### Bugfixes - Volumes specified through the `--volume` flag of `docker-compose run` now - complement volumes declared in the service's defintion instead of replacing + complement volumes declared in the service's definition instead of replacing them - Fixed a bug where using multiple Compose files would unset the scale value From 373c83ccd7031993d2229258e81dc324914ac685 Mon Sep 17 00:00:00 2001 From: rongzhang Date: Fri, 7 Sep 2018 15:32:09 +0800 Subject: [PATCH 30/46] Fix some typo Signed-off-by: rongzhang --- compose/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index c3e8f2789..e0acf0711 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1109,7 +1109,7 @@ class TopLevelCommand(object): @classmethod def version(cls, options): """ - Show version informations + Show version information Usage: version [--short] From d491a81cecb191adf447f7874fec9b999cf6f663 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 31 Aug 2018 16:58:30 -0700 Subject: [PATCH 31/46] Skip testing TPs/betas for now Signed-off-by: Joffrey F --- script/test/versions.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/script/test/versions.py b/script/test/versions.py index f699f2681..0dd27538f 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -37,22 +37,22 @@ import requests GITHUB_API = 'https://api.github.com/repos' -class Version(namedtuple('_Version', 'major minor patch rc edition')): +class Version(namedtuple('_Version', 'major minor patch stage edition')): @classmethod def parse(cls, version): edition = None version = version.lstrip('v') - version, _, rc = version.partition('-') - if rc: - if 'rc' not in rc: - edition = rc - rc = None - elif '-' in rc: - edition, rc = rc.split('-') + version, _, stage = version.partition('-') + if stage: + if not any(marker in stage for marker in ['rc', 'tp', 'beta']): + edition = stage + stage = None + elif '-' in stage: + edition, stage = stage.split('-') major, minor, patch = version.split('.', 3) - return cls(major, minor, patch, rc, edition) + return cls(major, minor, patch, stage, edition) @property def major_minor(self): @@ -64,13 +64,13 @@ class Version(namedtuple('_Version', 'major minor patch rc edition')): correctly with the default comparator. """ # rc releases should appear before official releases - rc = (0, self.rc) if self.rc else (1, ) - return (int(self.major), int(self.minor), int(self.patch)) + rc + stage = (0, self.stage) if self.stage else (1, ) + return (int(self.major), int(self.minor), int(self.patch)) + stage def __str__(self): - rc = '-{}'.format(self.rc) if self.rc else '' + stage = '-{}'.format(self.stage) if self.stage else '' edition = '-{}'.format(self.edition) if self.edition else '' - return '.'.join(map(str, self[:3])) + edition + rc + return '.'.join(map(str, self[:3])) + edition + stage BLACKLIST = [ # List of versions known to be broken and should not be used @@ -113,9 +113,9 @@ def get_latest_versions(versions, num=1): def get_default(versions): - """Return a :class:`Version` for the latest non-rc version.""" + """Return a :class:`Version` for the latest GA version.""" for version in versions: - if not version.rc: + if not version.stage: return version @@ -123,8 +123,12 @@ def get_versions(tags): for tag in tags: try: v = Version.parse(tag['name']) - if v not in BLACKLIST: - yield v + if v in BLACKLIST: + continue + # FIXME: Temporary. Remove once these versions are built on dockerswarm/dind + if v.stage and 'rc' not in v.stage: + continue + yield v except ValueError: print("Skipping invalid tag: {name}".format(**tag), file=sys.stderr) From 4e2de3c1ff4c30310e40f8fa4695cc0f518ce436 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 13 Aug 2018 19:29:29 -0700 Subject: [PATCH 32/46] Replace sequential container indexes with randomly generated IDs Signed-off-by: Joffrey F --- compose/cli/main.py | 14 +- compose/container.py | 9 +- compose/service.py | 38 ++--- compose/utils.py | 19 +++ tests/acceptance/cli_test.py | 138 ++++++++++-------- .../logs-tail-composefile/docker-compose.yml | 2 +- tests/integration/project_test.py | 11 +- tests/integration/service_test.py | 77 +++++----- tests/integration/state_test.py | 35 +++-- tests/unit/container_test.py | 6 +- tests/unit/service_test.py | 55 +++---- 11 files changed, 230 insertions(+), 174 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index e0acf0711..2cee9e033 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -474,15 +474,16 @@ class TopLevelCommand(object): -u, --user USER Run the command as this user. -T Disable pseudo-tty allocation. By default `docker-compose exec` allocates a TTY. - --index=index index of the container if there are multiple - instances of a service [default: 1] + --index=index "index" of the container if there are multiple + instances of a service. If missing, Compose will pick an + arbitrary container. -e, --env KEY=VAL Set environment variables (can be used multiple times, not supported in API < 1.25) -w, --workdir DIR Path to workdir directory for this command. """ environment = Environment.from_env_file(self.project_dir) use_cli = not environment.get_boolean('COMPOSE_INTERACTIVE_NO_CLI') - index = int(options.get('--index')) + index = options.get('--index') service = self.project.get_service(options['SERVICE']) detach = options.get('--detach') @@ -659,10 +660,11 @@ class TopLevelCommand(object): Options: --protocol=proto tcp or udp [default: tcp] - --index=index index of the container if there are multiple - instances of a service [default: 1] + --index=index "index" of the container if there are multiple + instances of a service. If missing, Compose will pick an + arbitrary container. """ - index = int(options.get('--index')) + index = options.get('--index') service = self.project.get_service(options['SERVICE']) try: container = service.get_container(number=index) diff --git a/compose/container.py b/compose/container.py index 8dac8cacd..9b5bbba04 100644 --- a/compose/container.py +++ b/compose/container.py @@ -10,6 +10,7 @@ from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .const import LABEL_VERSION +from .utils import truncate_id from .version import ComposeVersion @@ -80,7 +81,7 @@ class Container(object): @property def name_without_project(self): if self.name.startswith('{0}_{1}'.format(self.project, self.service)): - return '{0}_{1}'.format(self.service, self.number) + return '{0}_{1}'.format(self.service, self.short_number) else: return self.name @@ -90,7 +91,11 @@ class Container(object): if not number: raise ValueError("Container {0} does not have a {1} label".format( self.short_id, LABEL_CONTAINER_NUMBER)) - return int(number) + return number + + @property + def short_number(self): + return truncate_id(self.number) @property def ports(self): diff --git a/compose/service.py b/compose/service.py index 33bb3fe34..5989217d7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1,7 +1,6 @@ from __future__ import absolute_import from __future__ import unicode_literals -import itertools import logging import os import re @@ -49,9 +48,11 @@ from .errors import OperationFailedError from .parallel import parallel_execute from .progress_stream import stream_output from .progress_stream import StreamOutputError +from .utils import generate_random_id from .utils import json_hash from .utils import parse_bytes from .utils import parse_seconds_float +from .utils import truncate_id log = logging.getLogger(__name__) @@ -215,13 +216,17 @@ class Service(object): ) ) - def get_container(self, number=1): + def get_container(self, number=None): """Return a :class:`compose.container.Container` for this service. The container must be active, and match `number`. """ - - for container in self.containers(labels=['{0}={1}'.format(LABEL_CONTAINER_NUMBER, number)]): - return container + if number is not None and len(number) == 64: + for container in self.containers(labels=['{0}={1}'.format(LABEL_CONTAINER_NUMBER, number)]): + return container + else: + for container in self.containers(): + if number is None or container.number.startswith(number): + return container raise ValueError("No container found for %s_%s" % (self.name, number)) @@ -426,7 +431,6 @@ class Service(object): return has_diverged def _execute_convergence_create(self, scale, detached, start, project_services=None): - i = self._next_container_number() def create_and_start(service, n): container = service.create_container(number=n, quiet=True) @@ -437,7 +441,9 @@ class Service(object): return container containers, errors = parallel_execute( - [ServiceName(self.project, self.name, index) for index in range(i, i + scale)], + [ServiceName(self.project, self.name, number) for number in [ + self._next_container_number() for _ in range(scale) + ]], lambda service_name: create_and_start(self, service_name.number), lambda service_name: self.get_container_name(service_name.service, service_name.number), "Creating" @@ -568,7 +574,7 @@ class Service(object): container.rename_to_tmp_name() new_container = self.create_container( previous_container=container if not renew_anonymous_volumes else None, - number=container.labels.get(LABEL_CONTAINER_NUMBER), + number=container.number, quiet=True, ) if attach_logs: @@ -723,20 +729,8 @@ class Service(object): def get_volumes_from_names(self): return [s.source.name for s in self.volumes_from if isinstance(s.source, Service)] - # TODO: this would benefit from github.com/docker/docker/pull/14699 - # to remove the need to inspect every container def _next_container_number(self, one_off=False): - containers = itertools.chain( - self._fetch_containers( - all=True, - filters={'label': self.labels(one_off=one_off)} - ), self._fetch_containers( - all=True, - filters={'label': self.labels(one_off=one_off, legacy=True)} - ) - ) - numbers = [c.number for c in containers] - return 1 if not numbers else max(numbers) + 1 + return generate_random_id() def _fetch_containers(self, **fetch_options): # Account for containers that might have been removed since we fetched @@ -1377,7 +1371,7 @@ def build_container_name(project, service, number, one_off=False): bits = [project.lstrip('-_'), service] if one_off: bits.append('run') - return '_'.join(bits + [str(number)]) + return '_'.join(bits + [truncate_id(number)]) # Images diff --git a/compose/utils.py b/compose/utils.py index 956673b4b..8f0b3e549 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -7,6 +7,7 @@ import json import json.decoder import logging import ntpath +import random import six from docker.errors import DockerException @@ -151,3 +152,21 @@ def unquote_path(s): if s[0] == '"' and s[-1] == '"': return s[1:-1] return s + + +def generate_random_id(): + while True: + val = hex(random.getrandbits(32 * 8))[2:-1] + try: + int(truncate_id(val)) + continue + except ValueError: + return val + + +def truncate_id(value): + if ':' in value: + value = value[value.index(':') + 1:] + if len(value) > 12: + return value[:12] + return value diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 66e7d4c38..a41250d3d 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -99,7 +99,14 @@ class ContainerStateCondition(object): def __call__(self): try: - container = self.client.inspect_container(self.name) + if self.name.endswith('*'): + ctnrs = self.client.containers(all=True, filters={'name': self.name[:-1]}) + if len(ctnrs) > 0: + container = self.client.inspect_container(ctnrs[0]['Id']) + else: + return False + else: + container = self.client.inspect_container(self.name) return container['State']['Status'] == self.status except errors.APIError: return False @@ -540,16 +547,16 @@ class CLITestCase(DockerClientTestCase): def test_ps(self): self.project.get_service('simple').create_container() result = self.dispatch(['ps']) - assert 'simple-composefile_simple_1' in result.stdout + assert 'simple-composefile_simple_' in result.stdout def test_ps_default_composefile(self): self.base_dir = 'tests/fixtures/multiple-composefiles' self.dispatch(['up', '-d']) result = self.dispatch(['ps']) - assert 'multiple-composefiles_simple_1' in result.stdout - assert 'multiple-composefiles_another_1' in result.stdout - assert 'multiple-composefiles_yetanother_1' not in result.stdout + assert 'multiple-composefiles_simple_' in result.stdout + assert 'multiple-composefiles_another_' in result.stdout + assert 'multiple-composefiles_yetanother_' not in result.stdout def test_ps_alternate_composefile(self): config_path = os.path.abspath( @@ -560,9 +567,9 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['-f', 'compose2.yml', 'up', '-d']) result = self.dispatch(['-f', 'compose2.yml', 'ps']) - assert 'multiple-composefiles_simple_1' not in result.stdout - assert 'multiple-composefiles_another_1' not in result.stdout - assert 'multiple-composefiles_yetanother_1' in result.stdout + assert 'multiple-composefiles_simple_' not in result.stdout + assert 'multiple-composefiles_another_' not in result.stdout + assert 'multiple-composefiles_yetanother_' in result.stdout def test_ps_services_filter_option(self): self.base_dir = 'tests/fixtures/ps-services-filter' @@ -956,13 +963,13 @@ class CLITestCase(DockerClientTestCase): assert len(self.project.containers(one_off=OneOffFilter.only, stopped=True)) == 2 result = self.dispatch(['down', '--rmi=local', '--volumes']) - assert 'Stopping v2-full_web_1' in result.stderr - assert 'Stopping v2-full_other_1' in result.stderr - assert 'Stopping v2-full_web_run_2' in result.stderr - assert 'Removing v2-full_web_1' in result.stderr - assert 'Removing v2-full_other_1' in result.stderr - assert 'Removing v2-full_web_run_1' in result.stderr - assert 'Removing v2-full_web_run_2' in result.stderr + assert 'Stopping v2-full_web_' in result.stderr + assert 'Stopping v2-full_other_' in result.stderr + assert 'Stopping v2-full_web_run_' in result.stderr + assert 'Removing v2-full_web_' in result.stderr + assert 'Removing v2-full_other_' in result.stderr + assert 'Removing v2-full_web_run_' in result.stderr + assert 'Removing v2-full_web_run_' in result.stderr assert 'Removing volume v2-full_data' in result.stderr assert 'Removing image v2-full_web' in result.stderr assert 'Removing image busybox' not in result.stderr @@ -1019,11 +1026,13 @@ class CLITestCase(DockerClientTestCase): def test_up_attached(self): self.base_dir = 'tests/fixtures/echo-services' result = self.dispatch(['up', '--no-color']) + simple_num = self.project.get_service('simple').containers(stopped=True)[0].short_number + another_num = self.project.get_service('another').containers(stopped=True)[0].short_number - assert 'simple_1 | simple' in result.stdout - assert 'another_1 | another' in result.stdout - assert 'simple_1 exited with code 0' in result.stdout - assert 'another_1 exited with code 0' in result.stdout + assert 'simple_{} | simple'.format(simple_num) in result.stdout + assert 'another_{} | another'.format(another_num) in result.stdout + assert 'simple_{} exited with code 0'.format(simple_num) in result.stdout + assert 'another_{} exited with code 0'.format(another_num) in result.stdout @v2_only() def test_up(self): @@ -1727,11 +1736,12 @@ class CLITestCase(DockerClientTestCase): def test_run_rm(self): self.base_dir = 'tests/fixtures/volume' proc = start_process(self.base_dir, ['run', '--rm', 'test']) + service = self.project.get_service('test') wait_on_condition(ContainerStateCondition( self.project.client, - 'volume_test_run_1', - 'running')) - service = self.project.get_service('test') + 'volume_test_run_*', + 'running') + ) containers = service.containers(one_off=OneOffFilter.only) assert len(containers) == 1 mounts = containers[0].get('Mounts') @@ -2054,39 +2064,39 @@ class CLITestCase(DockerClientTestCase): proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) wait_on_condition(ContainerStateCondition( self.project.client, - 'simple-composefile_simple_run_1', + 'simple-composefile_simple_run_*', 'running')) os.kill(proc.pid, signal.SIGINT) wait_on_condition(ContainerStateCondition( self.project.client, - 'simple-composefile_simple_run_1', + 'simple-composefile_simple_run_*', 'exited')) def test_run_handles_sigterm(self): proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) wait_on_condition(ContainerStateCondition( self.project.client, - 'simple-composefile_simple_run_1', + 'simple-composefile_simple_run_*', 'running')) os.kill(proc.pid, signal.SIGTERM) wait_on_condition(ContainerStateCondition( self.project.client, - 'simple-composefile_simple_run_1', + 'simple-composefile_simple_run_*', 'exited')) def test_run_handles_sighup(self): proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) wait_on_condition(ContainerStateCondition( self.project.client, - 'simple-composefile_simple_run_1', + 'simple-composefile_simple_run_*', 'running')) os.kill(proc.pid, signal.SIGHUP) wait_on_condition(ContainerStateCondition( self.project.client, - 'simple-composefile_simple_run_1', + 'simple-composefile_simple_run_*', 'exited')) @mock.patch.dict(os.environ) @@ -2286,34 +2296,44 @@ class CLITestCase(DockerClientTestCase): proc = start_process(self.base_dir, ['logs', '-f']) self.dispatch(['up', '-d', 'another']) - wait_on_condition(ContainerStateCondition( - self.project.client, - 'logs-composefile_another_1', - 'exited')) + another_num = self.project.get_service('another').get_container().short_number + wait_on_condition( + ContainerStateCondition( + self.project.client, + 'logs-composefile_another_{}'.format(another_num), + 'exited' + ) + ) + simple_num = self.project.get_service('simple').get_container().short_number self.dispatch(['kill', 'simple']) result = wait_on_process(proc) assert 'hello' in result.stdout assert 'test' in result.stdout - assert 'logs-composefile_another_1 exited with code 0' in result.stdout - assert 'logs-composefile_simple_1 exited with code 137' in result.stdout + assert 'logs-composefile_another_{} exited with code 0'.format(another_num) in result.stdout + assert 'logs-composefile_simple_{} exited with code 137'.format(simple_num) in result.stdout def test_logs_follow_logs_from_restarted_containers(self): self.base_dir = 'tests/fixtures/logs-restart-composefile' proc = start_process(self.base_dir, ['up']) - wait_on_condition(ContainerStateCondition( - self.project.client, - 'logs-restart-composefile_another_1', - 'exited')) - + wait_on_condition( + ContainerStateCondition( + self.project.client, + 'logs-restart-composefile_another_*', + 'exited' + ) + ) self.dispatch(['kill', 'simple']) result = wait_on_process(proc) - assert result.stdout.count('logs-restart-composefile_another_1 exited with code 1') == 3 + assert len(re.findall( + r'logs-restart-composefile_another_[a-f0-9]{12} exited with code 1', + result.stdout + )) == 3 assert result.stdout.count('world') == 3 def test_logs_default(self): @@ -2346,10 +2366,10 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up']) result = self.dispatch(['logs', '--tail', '2']) - assert 'c\n' in result.stdout - assert 'd\n' in result.stdout - assert 'a\n' not in result.stdout - assert 'b\n' not in result.stdout + assert 'y\n' in result.stdout + assert 'z\n' in result.stdout + assert 'w\n' not in result.stdout + assert 'x\n' not in result.stdout def test_kill(self): self.dispatch(['up', '-d'], None) @@ -2523,9 +2543,9 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['port', '--index=' + str(index), 'simple', str(number)]) return result.stdout.rstrip() - assert get_port(3000) == containers[0].get_local_port(3000) - assert get_port(3000, index=1) == containers[0].get_local_port(3000) - assert get_port(3000, index=2) == containers[1].get_local_port(3000) + assert get_port(3000) in (containers[0].get_local_port(3000), containers[1].get_local_port(3000)) + assert get_port(3000, index=containers[0].number) == containers[0].get_local_port(3000) + assert get_port(3000, index=containers[1].number) == containers[1].get_local_port(3000) assert get_port(3002) == "" def test_events_json(self): @@ -2561,7 +2581,7 @@ class CLITestCase(DockerClientTestCase): container, = self.project.containers() expected_template = ' container {} {}' - expected_meta_info = ['image=busybox:latest', 'name=simple-composefile_simple_1'] + expected_meta_info = ['image=busybox:latest', 'name=simple-composefile_simple_'] assert expected_template.format('create', container.id) in lines[0] assert expected_template.format('start', container.id) in lines[1] @@ -2643,8 +2663,11 @@ class CLITestCase(DockerClientTestCase): assert len(containers) == 2 web = containers[1] + db_num = containers[0].short_number - assert set(get_links(web)) == set(['db', 'mydb_1', 'extends_mydb_1']) + assert set(get_links(web)) == set( + ['db', 'mydb_{}'.format(db_num), 'extends_mydb_{}'.format(db_num)] + ) expected_env = set([ "FOO=1", @@ -2677,11 +2700,11 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/exit-code-from' proc = start_process( self.base_dir, - ['up', '--abort-on-container-exit', '--exit-code-from', 'another']) + ['up', '--abort-on-container-exit', '--exit-code-from', 'another'] + ) result = wait_on_process(proc, returncode=1) - - assert 'exit-code-from_another_1 exited with code 1' in result.stdout + assert re.findall(r'exit-code-from_another_[a-f0-9]{12} exited with code 1', result.stdout) def test_exit_code_from_signal_stop(self): self.base_dir = 'tests/fixtures/exit-code-from' @@ -2690,13 +2713,14 @@ class CLITestCase(DockerClientTestCase): ['up', '--abort-on-container-exit', '--exit-code-from', 'simple'] ) result = wait_on_process(proc, returncode=137) # SIGKILL - assert 'exit-code-from_another_1 exited with code 1' in result.stdout + num = self.project.get_service('another').containers(stopped=True)[0].short_number + assert 'exit-code-from_another_{} exited with code 1'.format(num) in result.stdout def test_images(self): self.project.get_service('simple').create_container() result = self.dispatch(['images']) assert 'busybox' in result.stdout - assert 'simple-composefile_simple_1' in result.stdout + assert 'simple-composefile_simple_' in result.stdout def test_images_default_composefile(self): self.base_dir = 'tests/fixtures/multiple-composefiles' @@ -2704,8 +2728,8 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['images']) assert 'busybox' in result.stdout - assert 'multiple-composefiles_another_1' in result.stdout - assert 'multiple-composefiles_simple_1' in result.stdout + assert 'multiple-composefiles_another_' in result.stdout + assert 'multiple-composefiles_simple_' in result.stdout @mock.patch.dict(os.environ) def test_images_tagless_image(self): @@ -2725,7 +2749,7 @@ class CLITestCase(DockerClientTestCase): self.project.get_service('foo').create_container() result = self.dispatch(['images']) assert '' in result.stdout - assert 'tagless-image_foo_1' in result.stdout + assert 'tagless-image_foo_' in result.stdout def test_up_with_override_yaml(self): self.base_dir = 'tests/fixtures/override-yaml-files' diff --git a/tests/fixtures/logs-tail-composefile/docker-compose.yml b/tests/fixtures/logs-tail-composefile/docker-compose.yml index 80d8feaec..b70d0cc63 100644 --- a/tests/fixtures/logs-tail-composefile/docker-compose.yml +++ b/tests/fixtures/logs-tail-composefile/docker-compose.yml @@ -1,3 +1,3 @@ simple: image: busybox:latest - command: sh -c "echo a && echo b && echo c && echo d" + command: sh -c "echo w && echo x && echo y && echo z" diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 8813e84ce..858a8dfd7 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -90,7 +90,8 @@ class ProjectTest(DockerClientTestCase): project.up() containers = project.containers(['web']) - assert [c.name for c in containers] == ['composetest_web_1'] + assert len(containers) == 1 + assert containers[0].name.startswith('composetest_web_') def test_containers_with_extra_service(self): web = self.create_service('web') @@ -464,14 +465,14 @@ class ProjectTest(DockerClientTestCase): project.up(['db']) assert len(project.containers()) == 1 - old_db_id = project.containers()[0].id container, = project.containers() + old_db_id = container.id db_volume_path = container.get_mount('/var/db')['Source'] project.up(strategy=ConvergenceStrategy.never) assert len(project.containers()) == 2 - db_container = [c for c in project.containers() if 'db' in c.name][0] + db_container = [c for c in project.containers() if c.name == container.name][0] assert db_container.id == old_db_id assert db_container.get_mount('/var/db')['Source'] == db_volume_path @@ -1944,7 +1945,7 @@ class ProjectTest(DockerClientTestCase): containers = project.containers(stopped=True) assert len(containers) == 1 - assert containers[0].name == 'underscoretest_svc1_1' + assert containers[0].name.startswith('underscoretest_svc1_') assert containers[0].project == '_underscoretest' full_vol_name = 'underscoretest_foo' @@ -1965,7 +1966,7 @@ class ProjectTest(DockerClientTestCase): containers = project2.containers(stopped=True) assert len(containers) == 1 - assert containers[0].name == 'dashtest_svc1_1' + assert containers[0].name.startswith('dashtest_svc1_') assert containers[0].project == '-dashtest' full_vol_name = 'dashtest_foo' diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 88123152c..d7422e2f6 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -67,7 +67,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(foo) assert len(foo.containers()) == 1 - assert foo.containers()[0].name == 'composetest_foo_1' + assert foo.containers()[0].name.startswith('composetest_foo_') assert len(bar.containers()) == 0 create_and_start_container(bar) @@ -77,8 +77,8 @@ class ServiceTest(DockerClientTestCase): assert len(bar.containers()) == 2 names = [c.name for c in bar.containers()] - assert 'composetest_bar_1' in names - assert 'composetest_bar_2' in names + assert len(names) == 2 + assert all(name.startswith('composetest_bar_') for name in names) def test_containers_one_off(self): db = self.create_service('db') @@ -89,18 +89,18 @@ class ServiceTest(DockerClientTestCase): def test_project_is_added_to_container_name(self): service = self.create_service('web') create_and_start_container(service) - assert service.containers()[0].name == 'composetest_web_1' + assert service.containers()[0].name.startswith('composetest_web_') def test_create_container_with_one_off(self): db = self.create_service('db') container = db.create_container(one_off=True) - assert container.name == 'composetest_db_run_1' + assert container.name.startswith('composetest_db_run_') def test_create_container_with_one_off_when_existing_container_is_running(self): db = self.create_service('db') db.start() container = db.create_container(one_off=True) - assert container.name == 'composetest_db_run_1' + assert container.name.startswith('composetest_db_run_') def test_create_container_with_unspecified_volume(self): service = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) @@ -489,7 +489,7 @@ class ServiceTest(DockerClientTestCase): assert old_container.get('Config.Entrypoint') == ['top'] assert old_container.get('Config.Cmd') == ['-d', '1'] assert 'FOO=1' in old_container.get('Config.Env') - assert old_container.name == 'composetest_db_1' + assert old_container.name.startswith('composetest_db_') service.start_container(old_container) old_container.inspect() # reload volume data volume_path = old_container.get_mount('/etc')['Source'] @@ -503,7 +503,7 @@ class ServiceTest(DockerClientTestCase): assert new_container.get('Config.Entrypoint') == ['top'] assert new_container.get('Config.Cmd') == ['-d', '1'] assert 'FOO=2' in new_container.get('Config.Env') - assert new_container.name == 'composetest_db_1' + assert new_container.name.startswith('composetest_db_') assert new_container.get_mount('/etc')['Source'] == volume_path if not is_cluster(self.client): assert ( @@ -836,13 +836,13 @@ class ServiceTest(DockerClientTestCase): db = self.create_service('db') web = self.create_service('web', links=[(db, None)]) - create_and_start_container(db) - create_and_start_container(db) + db1 = create_and_start_container(db) + db2 = create_and_start_container(db) create_and_start_container(web) assert set(get_links(web.containers()[0])) == set([ - 'composetest_db_1', 'db_1', - 'composetest_db_2', 'db_2', + db1.name, db1.name_without_project, + db2.name, db2.name_without_project, 'db' ]) @@ -851,30 +851,33 @@ class ServiceTest(DockerClientTestCase): db = self.create_service('db') web = self.create_service('web', links=[(db, 'custom_link_name')]) - create_and_start_container(db) - create_and_start_container(db) + db1 = create_and_start_container(db) + db2 = create_and_start_container(db) create_and_start_container(web) assert set(get_links(web.containers()[0])) == set([ - 'composetest_db_1', 'db_1', - 'composetest_db_2', 'db_2', + db1.name, db1.name_without_project, + db2.name, db2.name_without_project, 'custom_link_name' ]) @no_cluster('No legacy links support in Swarm') def test_start_container_with_external_links(self): db = self.create_service('db') - web = self.create_service('web', external_links=['composetest_db_1', - 'composetest_db_2', - 'composetest_db_3:db_3']) + db_ctnrs = [create_and_start_container(db) for _ in range(3)] + web = self.create_service( + 'web', external_links=[ + 'composetest_db_{}'.format(db_ctnrs[0].short_number), + 'composetest_db_{}'.format(db_ctnrs[1].short_number), + 'composetest_db_{}:db_3'.format(db_ctnrs[2].short_number) + ] + ) - for _ in range(3): - create_and_start_container(db) create_and_start_container(web) assert set(get_links(web.containers()[0])) == set([ - 'composetest_db_1', - 'composetest_db_2', + 'composetest_db_{}'.format(db_ctnrs[0].short_number), + 'composetest_db_{}'.format(db_ctnrs[1].short_number), 'db_3' ]) @@ -892,14 +895,14 @@ class ServiceTest(DockerClientTestCase): def test_start_one_off_container_creates_links_to_its_own_service(self): db = self.create_service('db') - create_and_start_container(db) - create_and_start_container(db) + db1 = create_and_start_container(db) + db2 = create_and_start_container(db) c = create_and_start_container(db, one_off=OneOffFilter.only) assert set(get_links(c)) == set([ - 'composetest_db_1', 'db_1', - 'composetest_db_2', 'db_2', + db1.name, db1.name_without_project, + db2.name, db2.name_without_project, 'db' ]) @@ -1249,10 +1252,9 @@ class ServiceTest(DockerClientTestCase): test that those containers are restarted and not removed/recreated. """ service = self.create_service('web') - next_number = service._next_container_number() - valid_numbers = [next_number, next_number + 1] - service.create_container(number=next_number) - service.create_container(number=next_number + 1) + valid_numbers = [service._next_container_number(), service._next_container_number()] + service.create_container(number=valid_numbers[0]) + service.create_container(number=valid_numbers[1]) ParallelStreamWriter.instance = None with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: @@ -1310,10 +1312,8 @@ class ServiceTest(DockerClientTestCase): assert len(service.containers()) == 1 assert service.containers()[0].is_running - assert ( - "ERROR: for composetest_web_2 Cannot create container for service" - " web: Boom" in mock_stderr.getvalue() - ) + assert "ERROR: for composetest_web_" in mock_stderr.getvalue() + assert "Cannot create container for service web: Boom" in mock_stderr.getvalue() def test_scale_with_unexpected_exception(self): """Test that when scaling if the API returns an error, that is not of type @@ -1580,7 +1580,6 @@ class ServiceTest(DockerClientTestCase): } compose_labels = { - LABEL_CONTAINER_NUMBER: '1', LABEL_ONE_OFF: 'False', LABEL_PROJECT: 'composetest', LABEL_SERVICE: 'web', @@ -1589,9 +1588,11 @@ class ServiceTest(DockerClientTestCase): expected = dict(labels_dict, **compose_labels) service = self.create_service('web', labels=labels_dict) - labels = create_and_start_container(service).labels.items() + ctnr = create_and_start_container(service) + labels = ctnr.labels.items() for pair in expected.items(): assert pair in labels + assert ctnr.labels[LABEL_CONTAINER_NUMBER] == ctnr.number def test_empty_labels(self): labels_dict = {'foo': '', 'bar': ''} @@ -1655,7 +1656,7 @@ class ServiceTest(DockerClientTestCase): def test_duplicate_containers(self): service = self.create_service('web') - options = service._get_container_create_options({}, 1) + options = service._get_container_create_options({}, service._next_container_number()) original = Container.create(service.client, **options) assert set(service.containers(stopped=True)) == set([original]) diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 5992a02a4..7652c06c8 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -55,8 +55,8 @@ class BasicProjectTest(ProjectTestCase): def test_partial_change(self): old_containers = self.run_up(self.cfg) - old_db = [c for c in old_containers if c.name_without_project == 'db_1'][0] - old_web = [c for c in old_containers if c.name_without_project == 'web_1'][0] + old_db = [c for c in old_containers if c.name_without_project.startswith('db_')][0] + old_web = [c for c in old_containers if c.name_without_project.startswith('web_')][0] self.cfg['web']['command'] = '/bin/true' @@ -71,7 +71,7 @@ class BasicProjectTest(ProjectTestCase): created = list(new_containers - old_containers) assert len(created) == 1 - assert created[0].name_without_project == 'web_1' + assert created[0].name_without_project == old_web.name_without_project assert created[0].get('Config.Cmd') == ['/bin/true'] def test_all_change(self): @@ -114,7 +114,7 @@ class ProjectWithDependenciesTest(ProjectTestCase): def test_up(self): containers = self.run_up(self.cfg) - assert set(c.name_without_project for c in containers) == set(['db_1', 'web_1', 'nginx_1']) + assert set(c.service for c in containers) == set(['db', 'web', 'nginx']) def test_change_leaf(self): old_containers = self.run_up(self.cfg) @@ -122,7 +122,7 @@ class ProjectWithDependenciesTest(ProjectTestCase): self.cfg['nginx']['environment'] = {'NEW_VAR': '1'} new_containers = self.run_up(self.cfg) - assert set(c.name_without_project for c in new_containers - old_containers) == set(['nginx_1']) + assert set(c.service for c in new_containers - old_containers) == set(['nginx']) def test_change_middle(self): old_containers = self.run_up(self.cfg) @@ -130,7 +130,7 @@ class ProjectWithDependenciesTest(ProjectTestCase): self.cfg['web']['environment'] = {'NEW_VAR': '1'} new_containers = self.run_up(self.cfg) - assert set(c.name_without_project for c in new_containers - old_containers) == set(['web_1']) + assert set(c.service for c in new_containers - old_containers) == set(['web']) def test_change_middle_always_recreate_deps(self): old_containers = self.run_up(self.cfg, always_recreate_deps=True) @@ -138,8 +138,7 @@ class ProjectWithDependenciesTest(ProjectTestCase): self.cfg['web']['environment'] = {'NEW_VAR': '1'} new_containers = self.run_up(self.cfg, always_recreate_deps=True) - assert set(c.name_without_project - for c in new_containers - old_containers) == {'web_1', 'nginx_1'} + assert set(c.service for c in new_containers - old_containers) == {'web', 'nginx'} def test_change_root(self): old_containers = self.run_up(self.cfg) @@ -147,7 +146,7 @@ class ProjectWithDependenciesTest(ProjectTestCase): self.cfg['db']['environment'] = {'NEW_VAR': '1'} new_containers = self.run_up(self.cfg) - assert set(c.name_without_project for c in new_containers - old_containers) == set(['db_1']) + assert set(c.service for c in new_containers - old_containers) == set(['db']) def test_change_root_always_recreate_deps(self): old_containers = self.run_up(self.cfg, always_recreate_deps=True) @@ -155,8 +154,9 @@ class ProjectWithDependenciesTest(ProjectTestCase): self.cfg['db']['environment'] = {'NEW_VAR': '1'} new_containers = self.run_up(self.cfg, always_recreate_deps=True) - assert set(c.name_without_project - for c in new_containers - old_containers) == {'db_1', 'web_1', 'nginx_1'} + assert set(c.service for c in new_containers - old_containers) == { + 'db', 'web', 'nginx' + } def test_change_root_no_recreate(self): old_containers = self.run_up(self.cfg) @@ -195,9 +195,18 @@ class ProjectWithDependenciesTest(ProjectTestCase): web, = [c for c in containers if c.service == 'web'] nginx, = [c for c in containers if c.service == 'nginx'] + db, = [c for c in containers if c.service == 'db'] - assert set(get_links(web)) == {'composetest_db_1', 'db', 'db_1'} - assert set(get_links(nginx)) == {'composetest_web_1', 'web', 'web_1'} + assert set(get_links(web)) == { + 'composetest_db_{}'.format(db.short_number), + 'db', + 'db_{}'.format(db.short_number) + } + assert set(get_links(nginx)) == { + 'composetest_web_{}'.format(web.short_number), + 'web', + 'web_{}'.format(web.short_number) + } class ServiceStateTest(DockerClientTestCase): diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index d64263c1f..4f2f08302 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -30,7 +30,7 @@ class ContainerTest(unittest.TestCase): "Labels": { "com.docker.compose.project": "composetest", "com.docker.compose.service": "web", - "com.docker.compose.container-number": 7, + "com.docker.compose.container-number": "092cd63296fdc446ad432d3905dd1fcbe12a2ba6b52", }, } } @@ -77,7 +77,7 @@ class ContainerTest(unittest.TestCase): def test_number(self): container = Container(None, self.container_dict, has_been_inspected=True) - assert container.number == 7 + assert container.number == "092cd63296fdc446ad432d3905dd1fcbe12a2ba6b52" def test_name(self): container = Container.from_ps(None, @@ -88,7 +88,7 @@ class ContainerTest(unittest.TestCase): def test_name_without_project(self): self.container_dict['Name'] = "/composetest_web_7" container = Container(None, self.container_dict, has_been_inspected=True) - assert container.name_without_project == "web_7" + assert container.name_without_project == "web_092cd63296fd" def test_name_without_project_custom_container_name(self): self.container_dict['Name'] = "/custom_name_of_container" diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 791019a4e..ac234e624 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -41,6 +41,7 @@ from compose.service import parse_repository_tag from compose.service import Service from compose.service import ServiceNetworkMode from compose.service import warn_on_masked_volume +from compose.utils import generate_random_id as generate_id class ServiceTest(unittest.TestCase): @@ -81,8 +82,7 @@ class ServiceTest(unittest.TestCase): service = Service('db', self.mock_client, 'myproject', image='foo') assert [c.id for c in service.containers()] == ['1'] - assert service._next_container_number() == 2 - assert service.get_container(1).id == '1' + assert service.get_container().id == '1' def test_get_volumes_from_container(self): container_id = 'aabbccddee' @@ -164,7 +164,7 @@ class ServiceTest(unittest.TestCase): client=self.mock_client, mem_limit=1000000000, memswap_limit=2000000000) - service._get_container_create_options({'some': 'overrides'}, 1) + service._get_container_create_options({'some': 'overrides'}, generate_id()) assert self.mock_client.create_host_config.called assert self.mock_client.create_host_config.call_args[1]['mem_limit'] == 1000000000 @@ -173,10 +173,10 @@ class ServiceTest(unittest.TestCase): def test_self_reference_external_link(self): service = Service( name='foo', - external_links=['default_foo_1'] + external_links=['default_foo_bdfa3ed91e2c'] ) with pytest.raises(DependencyError): - service.get_container_name('foo', 1) + service.get_container_name('foo', 'bdfa3ed91e2c') def test_mem_reservation(self): self.mock_client.create_host_config.return_value = {} @@ -188,7 +188,7 @@ class ServiceTest(unittest.TestCase): client=self.mock_client, mem_reservation='512m' ) - service._get_container_create_options({'some': 'overrides'}, 1) + service._get_container_create_options({'some': 'overrides'}, generate_id()) assert self.mock_client.create_host_config.called is True assert self.mock_client.create_host_config.call_args[1]['mem_reservation'] == '512m' @@ -201,7 +201,7 @@ class ServiceTest(unittest.TestCase): hostname='name', client=self.mock_client, cgroup_parent='test') - service._get_container_create_options({'some': 'overrides'}, 1) + service._get_container_create_options({'some': 'overrides'}, generate_id()) assert self.mock_client.create_host_config.called assert self.mock_client.create_host_config.call_args[1]['cgroup_parent'] == 'test' @@ -218,7 +218,7 @@ class ServiceTest(unittest.TestCase): client=self.mock_client, log_driver='syslog', logging=logging) - service._get_container_create_options({'some': 'overrides'}, 1) + service._get_container_create_options({'some': 'overrides'}, generate_id()) assert self.mock_client.create_host_config.called assert self.mock_client.create_host_config.call_args[1]['log_config'] == { @@ -233,7 +233,7 @@ class ServiceTest(unittest.TestCase): image='foo', client=self.mock_client, stop_grace_period="1m35s") - opts = service._get_container_create_options({'image': 'foo'}, 1) + opts = service._get_container_create_options({'image': 'foo'}, generate_id()) assert opts['stop_timeout'] == 95 def test_split_domainname_none(self): @@ -242,7 +242,7 @@ class ServiceTest(unittest.TestCase): image='foo', hostname='name.domain.tld', client=self.mock_client) - opts = service._get_container_create_options({'image': 'foo'}, 1) + opts = service._get_container_create_options({'image': 'foo'}, generate_id()) assert opts['hostname'] == 'name.domain.tld', 'hostname' assert not ('domainname' in opts), 'domainname' @@ -253,7 +253,7 @@ class ServiceTest(unittest.TestCase): hostname='name.domain.tld', image='foo', client=self.mock_client) - opts = service._get_container_create_options({'image': 'foo'}, 1) + opts = service._get_container_create_options({'image': 'foo'}, generate_id()) assert opts['hostname'] == 'name', 'hostname' assert opts['domainname'] == 'domain.tld', 'domainname' @@ -265,7 +265,7 @@ class ServiceTest(unittest.TestCase): image='foo', domainname='domain.tld', client=self.mock_client) - opts = service._get_container_create_options({'image': 'foo'}, 1) + opts = service._get_container_create_options({'image': 'foo'}, generate_id()) assert opts['hostname'] == 'name', 'hostname' assert opts['domainname'] == 'domain.tld', 'domainname' @@ -277,7 +277,7 @@ class ServiceTest(unittest.TestCase): domainname='domain.tld', image='foo', client=self.mock_client) - opts = service._get_container_create_options({'image': 'foo'}, 1) + opts = service._get_container_create_options({'image': 'foo'}, generate_id()) assert opts['hostname'] == 'name.sub', 'hostname' assert opts['domainname'] == 'domain.tld', 'domainname' @@ -288,7 +288,7 @@ class ServiceTest(unittest.TestCase): use_networking=False, client=self.mock_client, ) - opts = service._get_container_create_options({'image': 'foo'}, 1) + opts = service._get_container_create_options({'image': 'foo'}, generate_id()) assert opts.get('hostname') is None def test_get_container_create_options_with_name_option(self): @@ -321,9 +321,8 @@ class ServiceTest(unittest.TestCase): prev_container.get.return_value = None opts = service._get_container_create_options( - {}, - 1, - previous_container=prev_container) + {}, generate_id(), previous_container=prev_container + ) assert service.options['labels'] == labels assert service.options['environment'] == environment @@ -358,7 +357,7 @@ class ServiceTest(unittest.TestCase): opts = service._get_container_create_options( {}, - 1, + generate_id(), previous_container=prev_container) assert opts['environment'] == ['affinity:container==ababab'] @@ -373,7 +372,7 @@ class ServiceTest(unittest.TestCase): opts = service._get_container_create_options( {}, - 1, + generate_id(), previous_container=prev_container) assert opts['environment'] == [] @@ -386,11 +385,11 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.Container', autospec=True) def test_get_container(self, mock_container_class): - container_dict = dict(Name='default_foo_2') + container_dict = dict(Name='default_foo_bdfa3ed91e2c') self.mock_client.containers.return_value = [container_dict] service = Service('foo', image='foo', client=self.mock_client) - container = service.get_container(number=2) + container = service.get_container(number="bdfa3ed91e2c") assert container == mock_container_class.from_ps.return_value mock_container_class.from_ps.assert_called_once_with( self.mock_client, container_dict) @@ -463,6 +462,7 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.Container', autospec=True) def test_recreate_container(self, _): mock_container = mock.create_autospec(Container) + mock_container.number = generate_id() service = Service('foo', client=self.mock_client, image='someimage') service.image = lambda: {'Id': 'abc123'} new_container = service.recreate_container(mock_container) @@ -476,6 +476,7 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.Container', autospec=True) def test_recreate_container_with_timeout(self, _): mock_container = mock.create_autospec(Container) + mock_container.number = generate_id() self.mock_client.inspect_image.return_value = {'Id': 'abc123'} service = Service('foo', client=self.mock_client, image='someimage') service.recreate_container(mock_container, timeout=1) @@ -711,9 +712,9 @@ class ServiceTest(unittest.TestCase): for api_version in set(API_VERSIONS.values()): self.mock_client.api_version = api_version - assert service._get_container_create_options({}, 1)['labels'][LABEL_CONFIG_HASH] == ( - config_hash - ) + assert service._get_container_create_options( + {}, generate_id() + )['labels'][LABEL_CONFIG_HASH] == config_hash def test_remove_image_none(self): web = Service('web', image='example', client=self.mock_client) @@ -971,7 +972,7 @@ class ServiceTest(unittest.TestCase): service = Service('foo', client=self.mock_client, environment=environment) - create_opts = service._get_container_create_options(override_options, 1) + create_opts = service._get_container_create_options(override_options, generate_id()) assert set(create_opts['environment']) == set(format_environment({ 'HTTP_PROXY': default_proxy_config['httpProxy'], 'http_proxy': default_proxy_config['httpProxy'], @@ -1296,7 +1297,7 @@ class ServiceVolumesTest(unittest.TestCase): service._get_container_create_options( override_options={}, - number=1, + number=generate_id(), ) assert set(self.mock_client.create_host_config.call_args[1]['binds']) == set([ @@ -1339,7 +1340,7 @@ class ServiceVolumesTest(unittest.TestCase): service._get_container_create_options( override_options={}, - number=1, + number=generate_id(), previous_container=Container(self.mock_client, {'Id': '123123123'}), ) From 5916639383d334145f0566502f80d4152528a158 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 31 Aug 2018 16:18:19 -0700 Subject: [PATCH 33/46] Preserve container numbers, add slug to prevent name collisions Signed-off-by: Joffrey F --- compose/cli/main.py | 14 ++--- compose/const.py | 1 + compose/container.py | 13 +++-- compose/project.py | 22 ------- compose/service.py | 96 ++++++++++++++++++------------- script/test/versions.py | 1 - tests/acceptance/cli_test.py | 70 +++++++++++----------- tests/integration/service_test.py | 14 +++-- tests/integration/state_test.py | 8 +-- tests/unit/container_test.py | 7 ++- tests/unit/service_test.py | 59 ++++++++++--------- 11 files changed, 157 insertions(+), 148 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 2cee9e033..e0acf0711 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -474,16 +474,15 @@ class TopLevelCommand(object): -u, --user USER Run the command as this user. -T Disable pseudo-tty allocation. By default `docker-compose exec` allocates a TTY. - --index=index "index" of the container if there are multiple - instances of a service. If missing, Compose will pick an - arbitrary container. + --index=index index of the container if there are multiple + instances of a service [default: 1] -e, --env KEY=VAL Set environment variables (can be used multiple times, not supported in API < 1.25) -w, --workdir DIR Path to workdir directory for this command. """ environment = Environment.from_env_file(self.project_dir) use_cli = not environment.get_boolean('COMPOSE_INTERACTIVE_NO_CLI') - index = options.get('--index') + index = int(options.get('--index')) service = self.project.get_service(options['SERVICE']) detach = options.get('--detach') @@ -660,11 +659,10 @@ class TopLevelCommand(object): Options: --protocol=proto tcp or udp [default: tcp] - --index=index "index" of the container if there are multiple - instances of a service. If missing, Compose will pick an - arbitrary container. + --index=index index of the container if there are multiple + instances of a service [default: 1] """ - index = options.get('--index') + index = int(options.get('--index')) service = self.project.get_service(options['SERVICE']) try: container = service.get_container(number=index) diff --git a/compose/const.py b/compose/const.py index ffb68db01..f4b9489e1 100644 --- a/compose/const.py +++ b/compose/const.py @@ -15,6 +15,7 @@ LABEL_PROJECT = 'com.docker.compose.project' LABEL_SERVICE = 'com.docker.compose.service' LABEL_NETWORK = 'com.docker.compose.network' LABEL_VERSION = 'com.docker.compose.version' +LABEL_SLUG = 'com.docker.compose.slug' LABEL_VOLUME = 'com.docker.compose.volume' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' NANOCPUS_SCALE = 1000000000 diff --git a/compose/container.py b/compose/container.py index 9b5bbba04..3ee45c8f3 100644 --- a/compose/container.py +++ b/compose/container.py @@ -9,6 +9,7 @@ from docker.errors import ImageNotFound from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_PROJECT from .const import LABEL_SERVICE +from .const import LABEL_SLUG from .const import LABEL_VERSION from .utils import truncate_id from .version import ComposeVersion @@ -81,7 +82,7 @@ class Container(object): @property def name_without_project(self): if self.name.startswith('{0}_{1}'.format(self.project, self.service)): - return '{0}_{1}'.format(self.service, self.short_number) + return '{0}_{1}{2}'.format(self.service, self.number, '_' + self.slug if self.slug else '') else: return self.name @@ -91,11 +92,15 @@ class Container(object): if not number: raise ValueError("Container {0} does not have a {1} label".format( self.short_id, LABEL_CONTAINER_NUMBER)) - return number + return int(number) @property - def short_number(self): - return truncate_id(self.number) + def slug(self): + return truncate_id(self.full_slug) + + @property + def full_slug(self): + return self.labels.get(LABEL_SLUG) @property def ports(self): diff --git a/compose/project.py b/compose/project.py index 22ef8be44..4340577c9 100644 --- a/compose/project.py +++ b/compose/project.py @@ -31,7 +31,6 @@ from .service import ConvergenceStrategy from .service import NetworkMode from .service import PidMode from .service import Service -from .service import ServiceName from .service import ServiceNetworkMode from .service import ServicePidMode from .utils import microseconds_from_time_nano @@ -198,25 +197,6 @@ class Project(object): service.remove_duplicate_containers() return services - def get_scaled_services(self, services, scale_override): - """ - Returns a list of this project's services as scaled ServiceName objects. - - services: a list of Service objects - scale_override: a dict with the scale to apply to each service (k: service_name, v: scale) - """ - service_names = [] - for service in services: - if service.name in scale_override: - scale = scale_override[service.name] - else: - scale = service.scale_num - - for i in range(1, scale + 1): - service_names.append(ServiceName(self.name, service.name, i)) - - return service_names - def get_links(self, service_dict): links = [] if 'links' in service_dict: @@ -494,7 +474,6 @@ class Project(object): svc.ensure_image_exists(do_build=do_build, silent=silent) plans = self._get_convergence_plans( services, strategy, always_recreate_deps=always_recreate_deps) - scaled_services = self.get_scaled_services(services, scale_override) def do(service): @@ -505,7 +484,6 @@ class Project(object): scale_override=scale_override.get(service.name), rescale=rescale, start=start, - project_services=scaled_services, reset_container_image=reset_container_image, renew_anonymous_volumes=renew_anonymous_volumes, ) diff --git a/compose/service.py b/compose/service.py index 5989217d7..199be8f1f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import itertools import logging import os import re @@ -39,6 +40,7 @@ from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT from .const import LABEL_SERVICE +from .const import LABEL_SLUG from .const import LABEL_VERSION from .const import NANOCPUS_SCALE from .container import Container @@ -123,7 +125,7 @@ class NoSuchImageError(Exception): pass -ServiceName = namedtuple('ServiceName', 'project service number') +ServiceName = namedtuple('ServiceName', 'project service number slug') ConvergencePlan = namedtuple('ConvergencePlan', 'action containers') @@ -216,17 +218,12 @@ class Service(object): ) ) - def get_container(self, number=None): + def get_container(self, number=1): """Return a :class:`compose.container.Container` for this service. The container must be active, and match `number`. """ - if number is not None and len(number) == 64: - for container in self.containers(labels=['{0}={1}'.format(LABEL_CONTAINER_NUMBER, number)]): - return container - else: - for container in self.containers(): - if number is None or container.number.startswith(number): - return container + for container in self.containers(labels=['{0}={1}'.format(LABEL_CONTAINER_NUMBER, number)]): + return container raise ValueError("No container found for %s_%s" % (self.name, number)) @@ -430,28 +427,33 @@ class Service(object): return has_diverged - def _execute_convergence_create(self, scale, detached, start, project_services=None): + def _execute_convergence_create(self, scale, detached, start): - def create_and_start(service, n): - container = service.create_container(number=n, quiet=True) - if not detached: - container.attach_log_stream() - if start: - self.start_container(container) - return container + i = self._next_container_number() - containers, errors = parallel_execute( - [ServiceName(self.project, self.name, number) for number in [ - self._next_container_number() for _ in range(scale) - ]], - lambda service_name: create_and_start(self, service_name.number), - lambda service_name: self.get_container_name(service_name.service, service_name.number), - "Creating" - ) - for error in errors.values(): - raise OperationFailedError(error) + def create_and_start(service, n): + container = service.create_container(number=n, quiet=True) + if not detached: + container.attach_log_stream() + if start: + self.start_container(container) + return container - return containers + containers, errors = parallel_execute( + [ + ServiceName(self.project, self.name, index, generate_random_id()) + for index in range(i, i + scale) + ], + lambda service_name: create_and_start(self, service_name.number), + lambda service_name: self.get_container_name( + service_name.service, service_name.number, service_name.slug + ), + "Creating" + ) + for error in errors.values(): + raise OperationFailedError(error) + + return containers def _execute_convergence_recreate(self, containers, scale, timeout, detached, start, renew_anonymous_volumes): @@ -514,8 +516,8 @@ class Service(object): def execute_convergence_plan(self, plan, timeout=None, detached=False, start=True, scale_override=None, - rescale=True, project_services=None, - reset_container_image=False, renew_anonymous_volumes=False): + rescale=True, reset_container_image=False, + renew_anonymous_volumes=False): (action, containers) = plan scale = scale_override if scale_override is not None else self.scale_num containers = sorted(containers, key=attrgetter('number')) @@ -524,7 +526,7 @@ class Service(object): if action == 'create': return self._execute_convergence_create( - scale, detached, start, project_services + scale, detached, start ) # The create action needs always needs an initial scale, but otherwise, @@ -730,7 +732,17 @@ class Service(object): return [s.source.name for s in self.volumes_from if isinstance(s.source, Service)] def _next_container_number(self, one_off=False): - return generate_random_id() + containers = itertools.chain( + self._fetch_containers( + all=True, + filters={'label': self.labels(one_off=one_off)} + ), self._fetch_containers( + all=True, + filters={'label': self.labels(one_off=one_off, legacy=True)} + ) + ) + numbers = [c.number for c in containers] + return 1 if not numbers else max(numbers) + 1 def _fetch_containers(self, **fetch_options): # Account for containers that might have been removed since we fetched @@ -807,6 +819,7 @@ class Service(object): one_off=False, previous_container=None): add_config_hash = (not one_off and not override_options) + slug = generate_random_id() if previous_container is None else previous_container.full_slug container_options = dict( (k, self.options[k]) @@ -815,7 +828,7 @@ class Service(object): container_options.update(override_options) if not container_options.get('name'): - container_options['name'] = self.get_container_name(self.name, number, one_off) + container_options['name'] = self.get_container_name(self.name, number, slug, one_off) container_options.setdefault('detach', True) @@ -867,7 +880,9 @@ class Service(object): container_options.get('labels', {}), self.labels(one_off=one_off), number, - self.config_hash if add_config_hash else None) + self.config_hash if add_config_hash else None, + slug + ) # Delete options which are only used in HostConfig for key in HOST_CONFIG_KEYS: @@ -1105,12 +1120,12 @@ class Service(object): def custom_container_name(self): return self.options.get('container_name') - def get_container_name(self, service_name, number, one_off=False): + def get_container_name(self, service_name, number, slug, one_off=False): if self.custom_container_name and not one_off: return self.custom_container_name container_name = build_container_name( - self.project, service_name, number, one_off, + self.project, service_name, number, slug, one_off, ) ext_links_origins = [l.split(':')[0] for l in self.options.get('external_links', [])] if container_name in ext_links_origins: @@ -1367,11 +1382,13 @@ class ServiceNetworkMode(object): # Names -def build_container_name(project, service, number, one_off=False): +def build_container_name(project, service, number, slug, one_off=False): bits = [project.lstrip('-_'), service] if one_off: bits.append('run') - return '_'.join(bits + [truncate_id(number)]) + return '_'.join( + bits + ([str(number), truncate_id(slug)] if slug else [str(number)]) + ) # Images @@ -1552,10 +1569,11 @@ def build_mount(mount_spec): # Labels -def build_container_labels(label_options, service_labels, number, config_hash): +def build_container_labels(label_options, service_labels, number, config_hash, slug): labels = dict(label_options or {}) labels.update(label.split('=', 1) for label in service_labels) labels[LABEL_CONTAINER_NUMBER] = str(number) + labels[LABEL_SLUG] = slug labels[LABEL_VERSION] = __version__ if config_hash: diff --git a/script/test/versions.py b/script/test/versions.py index 0dd27538f..6d273a9e6 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -50,7 +50,6 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')): stage = None elif '-' in stage: edition, stage = stage.split('-') - major, minor, patch = version.split('.', 3) return cls(major, minor, patch, stage, edition) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a41250d3d..015180bc7 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -547,16 +547,16 @@ class CLITestCase(DockerClientTestCase): def test_ps(self): self.project.get_service('simple').create_container() result = self.dispatch(['ps']) - assert 'simple-composefile_simple_' in result.stdout + assert 'simple-composefile_simple_1' in result.stdout def test_ps_default_composefile(self): self.base_dir = 'tests/fixtures/multiple-composefiles' self.dispatch(['up', '-d']) result = self.dispatch(['ps']) - assert 'multiple-composefiles_simple_' in result.stdout - assert 'multiple-composefiles_another_' in result.stdout - assert 'multiple-composefiles_yetanother_' not in result.stdout + assert 'multiple-composefiles_simple_1' in result.stdout + assert 'multiple-composefiles_another_1' in result.stdout + assert 'multiple-composefiles_yetanother_1' not in result.stdout def test_ps_alternate_composefile(self): config_path = os.path.abspath( @@ -567,9 +567,9 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['-f', 'compose2.yml', 'up', '-d']) result = self.dispatch(['-f', 'compose2.yml', 'ps']) - assert 'multiple-composefiles_simple_' not in result.stdout - assert 'multiple-composefiles_another_' not in result.stdout - assert 'multiple-composefiles_yetanother_' in result.stdout + assert 'multiple-composefiles_simple_1' not in result.stdout + assert 'multiple-composefiles_another_1' not in result.stdout + assert 'multiple-composefiles_yetanother_1' in result.stdout def test_ps_services_filter_option(self): self.base_dir = 'tests/fixtures/ps-services-filter' @@ -963,13 +963,13 @@ class CLITestCase(DockerClientTestCase): assert len(self.project.containers(one_off=OneOffFilter.only, stopped=True)) == 2 result = self.dispatch(['down', '--rmi=local', '--volumes']) - assert 'Stopping v2-full_web_' in result.stderr - assert 'Stopping v2-full_other_' in result.stderr - assert 'Stopping v2-full_web_run_' in result.stderr - assert 'Removing v2-full_web_' in result.stderr - assert 'Removing v2-full_other_' in result.stderr - assert 'Removing v2-full_web_run_' in result.stderr - assert 'Removing v2-full_web_run_' in result.stderr + assert 'Stopping v2-full_web_1' in result.stderr + assert 'Stopping v2-full_other_1' in result.stderr + assert 'Stopping v2-full_web_run_2' in result.stderr + assert 'Removing v2-full_web_1' in result.stderr + assert 'Removing v2-full_other_1' in result.stderr + assert 'Removing v2-full_web_run_1' in result.stderr + assert 'Removing v2-full_web_run_2' in result.stderr assert 'Removing volume v2-full_data' in result.stderr assert 'Removing image v2-full_web' in result.stderr assert 'Removing image busybox' not in result.stderr @@ -1026,13 +1026,15 @@ class CLITestCase(DockerClientTestCase): def test_up_attached(self): self.base_dir = 'tests/fixtures/echo-services' result = self.dispatch(['up', '--no-color']) - simple_num = self.project.get_service('simple').containers(stopped=True)[0].short_number - another_num = self.project.get_service('another').containers(stopped=True)[0].short_number + simple_name = self.project.get_service('simple').containers(stopped=True)[0].name_without_project + another_name = self.project.get_service('another').containers( + stopped=True + )[0].name_without_project - assert 'simple_{} | simple'.format(simple_num) in result.stdout - assert 'another_{} | another'.format(another_num) in result.stdout - assert 'simple_{} exited with code 0'.format(simple_num) in result.stdout - assert 'another_{} exited with code 0'.format(another_num) in result.stdout + assert '{} | simple'.format(simple_name) in result.stdout + assert '{} | another'.format(another_name) in result.stdout + assert '{} exited with code 0'.format(simple_name) in result.stdout + assert '{} exited with code 0'.format(another_name) in result.stdout @v2_only() def test_up(self): @@ -2296,24 +2298,24 @@ class CLITestCase(DockerClientTestCase): proc = start_process(self.base_dir, ['logs', '-f']) self.dispatch(['up', '-d', 'another']) - another_num = self.project.get_service('another').get_container().short_number + another_name = self.project.get_service('another').get_container().name_without_project wait_on_condition( ContainerStateCondition( self.project.client, - 'logs-composefile_another_{}'.format(another_num), + 'logs-composefile_another_*', 'exited' ) ) - simple_num = self.project.get_service('simple').get_container().short_number + simple_name = self.project.get_service('simple').get_container().name_without_project self.dispatch(['kill', 'simple']) result = wait_on_process(proc) assert 'hello' in result.stdout assert 'test' in result.stdout - assert 'logs-composefile_another_{} exited with code 0'.format(another_num) in result.stdout - assert 'logs-composefile_simple_{} exited with code 137'.format(simple_num) in result.stdout + assert '{} exited with code 0'.format(another_name) in result.stdout + assert '{} exited with code 137'.format(simple_name) in result.stdout def test_logs_follow_logs_from_restarted_containers(self): self.base_dir = 'tests/fixtures/logs-restart-composefile' @@ -2331,7 +2333,7 @@ class CLITestCase(DockerClientTestCase): result = wait_on_process(proc) assert len(re.findall( - r'logs-restart-composefile_another_[a-f0-9]{12} exited with code 1', + r'logs-restart-composefile_another_1_[a-f0-9]{12} exited with code 1', result.stdout )) == 3 assert result.stdout.count('world') == 3 @@ -2663,10 +2665,10 @@ class CLITestCase(DockerClientTestCase): assert len(containers) == 2 web = containers[1] - db_num = containers[0].short_number + db_name = containers[0].name_without_project assert set(get_links(web)) == set( - ['db', 'mydb_{}'.format(db_num), 'extends_mydb_{}'.format(db_num)] + ['db', db_name, 'extends_{}'.format(db_name)] ) expected_env = set([ @@ -2704,7 +2706,7 @@ class CLITestCase(DockerClientTestCase): ) result = wait_on_process(proc, returncode=1) - assert re.findall(r'exit-code-from_another_[a-f0-9]{12} exited with code 1', result.stdout) + assert re.findall(r'exit-code-from_another_1_[a-f0-9]{12} exited with code 1', result.stdout) def test_exit_code_from_signal_stop(self): self.base_dir = 'tests/fixtures/exit-code-from' @@ -2713,8 +2715,8 @@ class CLITestCase(DockerClientTestCase): ['up', '--abort-on-container-exit', '--exit-code-from', 'simple'] ) result = wait_on_process(proc, returncode=137) # SIGKILL - num = self.project.get_service('another').containers(stopped=True)[0].short_number - assert 'exit-code-from_another_{} exited with code 1'.format(num) in result.stdout + name = self.project.get_service('another').containers(stopped=True)[0].name_without_project + assert '{} exited with code 1'.format(name) in result.stdout def test_images(self): self.project.get_service('simple').create_container() @@ -2728,8 +2730,8 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['images']) assert 'busybox' in result.stdout - assert 'multiple-composefiles_another_' in result.stdout - assert 'multiple-composefiles_simple_' in result.stdout + assert 'multiple-composefiles_another_1' in result.stdout + assert 'multiple-composefiles_simple_1' in result.stdout @mock.patch.dict(os.environ) def test_images_tagless_image(self): @@ -2749,7 +2751,7 @@ class CLITestCase(DockerClientTestCase): self.project.get_service('foo').create_container() result = self.dispatch(['images']) assert '' in result.stdout - assert 'tagless-image_foo_' in result.stdout + assert 'tagless-image_foo_1' in result.stdout def test_up_with_override_yaml(self): self.base_dir = 'tests/fixtures/override-yaml-files' diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index d7422e2f6..db40409f8 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -32,6 +32,7 @@ from compose.const import LABEL_CONTAINER_NUMBER from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE +from compose.const import LABEL_SLUG from compose.const import LABEL_VERSION from compose.container import Container from compose.errors import OperationFailedError @@ -867,17 +868,17 @@ class ServiceTest(DockerClientTestCase): db_ctnrs = [create_and_start_container(db) for _ in range(3)] web = self.create_service( 'web', external_links=[ - 'composetest_db_{}'.format(db_ctnrs[0].short_number), - 'composetest_db_{}'.format(db_ctnrs[1].short_number), - 'composetest_db_{}:db_3'.format(db_ctnrs[2].short_number) + db_ctnrs[0].name, + db_ctnrs[1].name, + '{}:db_3'.format(db_ctnrs[2].name) ] ) create_and_start_container(web) assert set(get_links(web.containers()[0])) == set([ - 'composetest_db_{}'.format(db_ctnrs[0].short_number), - 'composetest_db_{}'.format(db_ctnrs[1].short_number), + db_ctnrs[0].name, + db_ctnrs[1].name, 'db_3' ]) @@ -1584,6 +1585,7 @@ class ServiceTest(DockerClientTestCase): LABEL_PROJECT: 'composetest', LABEL_SERVICE: 'web', LABEL_VERSION: __version__, + LABEL_CONTAINER_NUMBER: '1' } expected = dict(labels_dict, **compose_labels) @@ -1592,7 +1594,7 @@ class ServiceTest(DockerClientTestCase): labels = ctnr.labels.items() for pair in expected.items(): assert pair in labels - assert ctnr.labels[LABEL_CONTAINER_NUMBER] == ctnr.number + assert ctnr.labels[LABEL_SLUG] == ctnr.full_slug def test_empty_labels(self): labels_dict = {'foo': '', 'bar': ''} diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 7652c06c8..a41986f46 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -198,14 +198,14 @@ class ProjectWithDependenciesTest(ProjectTestCase): db, = [c for c in containers if c.service == 'db'] assert set(get_links(web)) == { - 'composetest_db_{}'.format(db.short_number), + 'composetest_db_{}_{}'.format(db.number, db.slug), 'db', - 'db_{}'.format(db.short_number) + 'db_{}_{}'.format(db.number, db.slug) } assert set(get_links(nginx)) == { - 'composetest_web_{}'.format(web.short_number), + 'composetest_web_{}_{}'.format(web.number, web.slug), 'web', - 'web_{}'.format(web.short_number) + 'web_{}_{}'.format(web.number, web.slug) } diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 4f2f08302..64c9cc344 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -30,7 +30,8 @@ class ContainerTest(unittest.TestCase): "Labels": { "com.docker.compose.project": "composetest", "com.docker.compose.service": "web", - "com.docker.compose.container-number": "092cd63296fdc446ad432d3905dd1fcbe12a2ba6b52", + "com.docker.compose.container-number": "7", + "com.docker.compose.slug": "092cd63296fdc446ad432d3905dd1fcbe12a2ba6b52" }, } } @@ -77,7 +78,7 @@ class ContainerTest(unittest.TestCase): def test_number(self): container = Container(None, self.container_dict, has_been_inspected=True) - assert container.number == "092cd63296fdc446ad432d3905dd1fcbe12a2ba6b52" + assert container.number == 7 def test_name(self): container = Container.from_ps(None, @@ -88,7 +89,7 @@ class ContainerTest(unittest.TestCase): def test_name_without_project(self): self.container_dict['Name'] = "/composetest_web_7" container = Container(None, self.container_dict, has_been_inspected=True) - assert container.name_without_project == "web_092cd63296fd" + assert container.name_without_project == "web_7_092cd63296fd" def test_name_without_project_custom_container_name(self): self.container_dict['Name'] = "/custom_name_of_container" diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index ac234e624..d5dbcbea6 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -41,7 +41,6 @@ from compose.service import parse_repository_tag from compose.service import Service from compose.service import ServiceNetworkMode from compose.service import warn_on_masked_volume -from compose.utils import generate_random_id as generate_id class ServiceTest(unittest.TestCase): @@ -82,7 +81,8 @@ class ServiceTest(unittest.TestCase): service = Service('db', self.mock_client, 'myproject', image='foo') assert [c.id for c in service.containers()] == ['1'] - assert service.get_container().id == '1' + assert service._next_container_number() == 2 + assert service.get_container(1).id == '1' def test_get_volumes_from_container(self): container_id = 'aabbccddee' @@ -164,7 +164,7 @@ class ServiceTest(unittest.TestCase): client=self.mock_client, mem_limit=1000000000, memswap_limit=2000000000) - service._get_container_create_options({'some': 'overrides'}, generate_id()) + service._get_container_create_options({'some': 'overrides'}, 1) assert self.mock_client.create_host_config.called assert self.mock_client.create_host_config.call_args[1]['mem_limit'] == 1000000000 @@ -173,10 +173,10 @@ class ServiceTest(unittest.TestCase): def test_self_reference_external_link(self): service = Service( name='foo', - external_links=['default_foo_bdfa3ed91e2c'] + external_links=['default_foo_1_bdfa3ed91e2c'] ) with pytest.raises(DependencyError): - service.get_container_name('foo', 'bdfa3ed91e2c') + service.get_container_name('foo', 1, 'bdfa3ed91e2c') def test_mem_reservation(self): self.mock_client.create_host_config.return_value = {} @@ -188,7 +188,7 @@ class ServiceTest(unittest.TestCase): client=self.mock_client, mem_reservation='512m' ) - service._get_container_create_options({'some': 'overrides'}, generate_id()) + service._get_container_create_options({'some': 'overrides'}, 1) assert self.mock_client.create_host_config.called is True assert self.mock_client.create_host_config.call_args[1]['mem_reservation'] == '512m' @@ -201,7 +201,7 @@ class ServiceTest(unittest.TestCase): hostname='name', client=self.mock_client, cgroup_parent='test') - service._get_container_create_options({'some': 'overrides'}, generate_id()) + service._get_container_create_options({'some': 'overrides'}, 1) assert self.mock_client.create_host_config.called assert self.mock_client.create_host_config.call_args[1]['cgroup_parent'] == 'test' @@ -218,7 +218,7 @@ class ServiceTest(unittest.TestCase): client=self.mock_client, log_driver='syslog', logging=logging) - service._get_container_create_options({'some': 'overrides'}, generate_id()) + service._get_container_create_options({'some': 'overrides'}, 1) assert self.mock_client.create_host_config.called assert self.mock_client.create_host_config.call_args[1]['log_config'] == { @@ -233,7 +233,7 @@ class ServiceTest(unittest.TestCase): image='foo', client=self.mock_client, stop_grace_period="1m35s") - opts = service._get_container_create_options({'image': 'foo'}, generate_id()) + opts = service._get_container_create_options({'image': 'foo'}, 1) assert opts['stop_timeout'] == 95 def test_split_domainname_none(self): @@ -242,7 +242,7 @@ class ServiceTest(unittest.TestCase): image='foo', hostname='name.domain.tld', client=self.mock_client) - opts = service._get_container_create_options({'image': 'foo'}, generate_id()) + opts = service._get_container_create_options({'image': 'foo'}, 1) assert opts['hostname'] == 'name.domain.tld', 'hostname' assert not ('domainname' in opts), 'domainname' @@ -253,7 +253,7 @@ class ServiceTest(unittest.TestCase): hostname='name.domain.tld', image='foo', client=self.mock_client) - opts = service._get_container_create_options({'image': 'foo'}, generate_id()) + opts = service._get_container_create_options({'image': 'foo'}, 1) assert opts['hostname'] == 'name', 'hostname' assert opts['domainname'] == 'domain.tld', 'domainname' @@ -265,7 +265,7 @@ class ServiceTest(unittest.TestCase): image='foo', domainname='domain.tld', client=self.mock_client) - opts = service._get_container_create_options({'image': 'foo'}, generate_id()) + opts = service._get_container_create_options({'image': 'foo'}, 1) assert opts['hostname'] == 'name', 'hostname' assert opts['domainname'] == 'domain.tld', 'domainname' @@ -277,7 +277,7 @@ class ServiceTest(unittest.TestCase): domainname='domain.tld', image='foo', client=self.mock_client) - opts = service._get_container_create_options({'image': 'foo'}, generate_id()) + opts = service._get_container_create_options({'image': 'foo'}, 1) assert opts['hostname'] == 'name.sub', 'hostname' assert opts['domainname'] == 'domain.tld', 'domainname' @@ -288,7 +288,7 @@ class ServiceTest(unittest.TestCase): use_networking=False, client=self.mock_client, ) - opts = service._get_container_create_options({'image': 'foo'}, generate_id()) + opts = service._get_container_create_options({'image': 'foo'}, 1) assert opts.get('hostname') is None def test_get_container_create_options_with_name_option(self): @@ -317,11 +317,13 @@ class ServiceTest(unittest.TestCase): self.mock_client.inspect_image.return_value = {'Id': 'abcd'} prev_container = mock.Mock( id='ababab', - image_config={'ContainerConfig': {}}) + image_config={'ContainerConfig': {}} + ) + prev_container.full_slug = 'abcdefff1234' prev_container.get.return_value = None opts = service._get_container_create_options( - {}, generate_id(), previous_container=prev_container + {}, 1, previous_container=prev_container ) assert service.options['labels'] == labels @@ -354,11 +356,13 @@ class ServiceTest(unittest.TestCase): }.get(key, None) prev_container.get.side_effect = container_get + prev_container.full_slug = 'abcdefff1234' opts = service._get_container_create_options( {}, - generate_id(), - previous_container=prev_container) + 1, + previous_container=prev_container + ) assert opts['environment'] == ['affinity:container==ababab'] @@ -369,10 +373,11 @@ class ServiceTest(unittest.TestCase): id='ababab', image_config={'ContainerConfig': {}}) prev_container.get.return_value = None + prev_container.full_slug = 'abcdefff1234' opts = service._get_container_create_options( {}, - generate_id(), + 1, previous_container=prev_container) assert opts['environment'] == [] @@ -385,11 +390,11 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.Container', autospec=True) def test_get_container(self, mock_container_class): - container_dict = dict(Name='default_foo_bdfa3ed91e2c') + container_dict = dict(Name='default_foo_2_bdfa3ed91e2c') self.mock_client.containers.return_value = [container_dict] service = Service('foo', image='foo', client=self.mock_client) - container = service.get_container(number="bdfa3ed91e2c") + container = service.get_container(number=2) assert container == mock_container_class.from_ps.return_value mock_container_class.from_ps.assert_called_once_with( self.mock_client, container_dict) @@ -462,7 +467,7 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.Container', autospec=True) def test_recreate_container(self, _): mock_container = mock.create_autospec(Container) - mock_container.number = generate_id() + mock_container.full_slug = 'abcdefff1234' service = Service('foo', client=self.mock_client, image='someimage') service.image = lambda: {'Id': 'abc123'} new_container = service.recreate_container(mock_container) @@ -476,7 +481,7 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.Container', autospec=True) def test_recreate_container_with_timeout(self, _): mock_container = mock.create_autospec(Container) - mock_container.number = generate_id() + mock_container.full_slug = 'abcdefff1234' self.mock_client.inspect_image.return_value = {'Id': 'abc123'} service = Service('foo', client=self.mock_client, image='someimage') service.recreate_container(mock_container, timeout=1) @@ -713,7 +718,7 @@ class ServiceTest(unittest.TestCase): for api_version in set(API_VERSIONS.values()): self.mock_client.api_version = api_version assert service._get_container_create_options( - {}, generate_id() + {}, 1 )['labels'][LABEL_CONFIG_HASH] == config_hash def test_remove_image_none(self): @@ -972,7 +977,7 @@ class ServiceTest(unittest.TestCase): service = Service('foo', client=self.mock_client, environment=environment) - create_opts = service._get_container_create_options(override_options, generate_id()) + create_opts = service._get_container_create_options(override_options, 1) assert set(create_opts['environment']) == set(format_environment({ 'HTTP_PROXY': default_proxy_config['httpProxy'], 'http_proxy': default_proxy_config['httpProxy'], @@ -1297,7 +1302,7 @@ class ServiceVolumesTest(unittest.TestCase): service._get_container_create_options( override_options={}, - number=generate_id(), + number=1, ) assert set(self.mock_client.create_host_config.call_args[1]['binds']) == set([ @@ -1340,7 +1345,7 @@ class ServiceVolumesTest(unittest.TestCase): service._get_container_create_options( override_options={}, - number=generate_id(), + number=1, previous_container=Container(self.mock_client, {'Id': '123123123'}), ) From 265d9dae4b92d590e8809d1c98d5d648e700114e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 12 Sep 2018 16:17:30 -0700 Subject: [PATCH 34/46] Update zsh completion with new options, and ensure service names are properly retrieved Signed-off-by: Joffrey F --- contrib/completion/zsh/_docker-compose | 142 +++++++------------------ 1 file changed, 37 insertions(+), 105 deletions(-) mode change 100644 => 100755 contrib/completion/zsh/_docker-compose diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose old mode 100644 new mode 100755 index 676aa117b..eb6199831 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -23,7 +23,7 @@ __docker-compose_all_services_in_compose_file() { local already_selected local -a services already_selected=$(echo $words | tr " " "|") - __docker-compose_q config --services \ + __docker-compose_q ps --services "$@" \ | grep -Ev "^(${already_selected})$" } @@ -31,125 +31,42 @@ __docker-compose_all_services_in_compose_file() { __docker-compose_services_all() { [[ $PREFIX = -* ]] && return 1 integer ret=1 - services=$(__docker-compose_all_services_in_compose_file) + services=$(__docker-compose_all_services_in_compose_file "$@") _alternative "args:services:($services)" && ret=0 return ret } -# All services that have an entry with the given key in their docker-compose.yml section -__docker-compose_services_with_key() { - local already_selected - local -a buildable - already_selected=$(echo $words | tr " " "|") - # flatten sections to one line, then filter lines containing the key and return section name. - __docker-compose_q config \ - | sed -n -e '/^services:/,/^[^ ]/p' \ - | sed -n 's/^ //p' \ - | awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' \ - | grep " \+$1:" \ - | cut -d: -f1 \ - | grep -Ev "^(${already_selected})$" -} - # All services that are defined by a Dockerfile reference __docker-compose_services_from_build() { [[ $PREFIX = -* ]] && return 1 - integer ret=1 - buildable=$(__docker-compose_services_with_key build) - _alternative "args:buildable services:($buildable)" && ret=0 - - return ret + __docker-compose_services_all --filter source=build } # All services that are defined by an image __docker-compose_services_from_image() { [[ $PREFIX = -* ]] && return 1 - integer ret=1 - pullable=$(__docker-compose_services_with_key image) - _alternative "args:pullable services:($pullable)" && ret=0 - - return ret -} - -__docker-compose_get_services() { - [[ $PREFIX = -* ]] && return 1 - integer ret=1 - local kind - declare -a running paused stopped lines args services - - docker_status=$(docker ps > /dev/null 2>&1) - if [ $? -ne 0 ]; then - _message "Error! Docker is not running." - return 1 - fi - - kind=$1 - shift - [[ $kind =~ (stopped|all) ]] && args=($args -a) - - lines=(${(f)"$(_call_program commands docker $docker_options ps --format 'table' $args)"}) - services=(${(f)"$(_call_program commands docker-compose 2>/dev/null $compose_options ps -q)"}) - - # Parse header line to find columns - local i=1 j=1 k header=${lines[1]} - declare -A begin end - while (( j < ${#header} - 1 )); do - i=$(( j + ${${header[$j,-1]}[(i)[^ ]]} - 1 )) - j=$(( i + ${${header[$i,-1]}[(i) ]} - 1 )) - k=$(( j + ${${header[$j,-1]}[(i)[^ ]]} - 2 )) - begin[${header[$i,$((j-1))]}]=$i - end[${header[$i,$((j-1))]}]=$k - done - lines=(${lines[2,-1]}) - - # Container ID - local line s name - local -a names - for line in $lines; do - if [[ ${services[@]} == *"${line[${begin[CONTAINER ID]},${end[CONTAINER ID]}]%% ##}"* ]]; then - names=(${(ps:,:)${${line[${begin[NAMES]},-1]}%% *}}) - for name in $names; do - s="${${name%_*}#*_}:${(l:15:: :::)${${line[${begin[CREATED]},${end[CREATED]}]/ ago/}%% ##}}" - s="$s, ${line[${begin[CONTAINER ID]},${end[CONTAINER ID]}]%% ##}" - s="$s, ${${${line[${begin[IMAGE]},${end[IMAGE]}]}/:/\\:}%% ##}" - if [[ ${line[${begin[STATUS]},${end[STATUS]}]} = Exit* ]]; then - stopped=($stopped $s) - else - if [[ ${line[${begin[STATUS]},${end[STATUS]}]} = *\(Paused\)* ]]; then - paused=($paused $s) - fi - running=($running $s) - fi - done - fi - done - - [[ $kind =~ (running|all) ]] && _describe -t services-running "running services" running "$@" && ret=0 - [[ $kind =~ (paused|all) ]] && _describe -t services-paused "paused services" paused "$@" && ret=0 - [[ $kind =~ (stopped|all) ]] && _describe -t services-stopped "stopped services" stopped "$@" && ret=0 - - return ret + __docker-compose_services_all --filter source=image } __docker-compose_pausedservices() { [[ $PREFIX = -* ]] && return 1 - __docker-compose_get_services paused "$@" + __docker-compose_services_all --filter status=paused } __docker-compose_stoppedservices() { [[ $PREFIX = -* ]] && return 1 - __docker-compose_get_services stopped "$@" + __docker-compose_services_all --filter status=stopped } __docker-compose_runningservices() { [[ $PREFIX = -* ]] && return 1 - __docker-compose_get_services running "$@" + __docker-compose_services_all --filter status=running } __docker-compose_services() { [[ $PREFIX = -* ]] && return 1 - __docker-compose_get_services all "$@" + __docker-compose_services_all } __docker-compose_caching_policy() { @@ -196,9 +113,10 @@ __docker-compose_subcommand() { $opts_help \ "*--build-arg=[Set build-time variables for one service.]:=: " \ '--force-rm[Always remove intermediate containers.]' \ - '--memory[Memory limit for the build container.]' \ + '(--memory -m)'{--memory,-m}'[Memory limit for the build container.]' \ '--no-cache[Do not use cache when building the image.]' \ '--pull[Always attempt to pull a newer version of the image.]' \ + '--compress[Compress the build context using gzip.]' \ '*:services:__docker-compose_services_from_build' && ret=0 ;; (bundle) @@ -223,11 +141,12 @@ __docker-compose_subcommand() { $opts_no_recreate \ $opts_no_build \ "(--no-build)--build[Build images before creating containers.]" \ - '*:services:__docker-compose_services_all' && ret=0 + '*:services:__docker-compose_services' && ret=0 ;; (down) _arguments \ $opts_help \ + $opts_timeout \ "--rmi[Remove images. Type must be one of: 'all': Remove all images used by any service. 'local': Remove only images that don't have a custom tag set by the \`image\` field.]:type:(all local)" \ '(-v --volumes)'{-v,--volumes}"[Remove named volumes declared in the \`volumes\` section of the Compose file and anonymous volumes attached to containers.]" \ $opts_remove_orphans && ret=0 @@ -236,16 +155,18 @@ __docker-compose_subcommand() { _arguments \ $opts_help \ '--json[Output events as a stream of json objects]' \ - '*:services:__docker-compose_services_all' && ret=0 + '*:services:__docker-compose_services' && ret=0 ;; (exec) _arguments \ $opts_help \ '-d[Detached mode: Run command in the background.]' \ '--privileged[Give extended privileges to the process.]' \ - '(-u --user)'{-u,--user=}'[Run the command as this user.]:username:_users' \ + '(-u --user)'{-u,--user=}'[Run the command as this user.]:username:_users' \ '-T[Disable pseudo-tty allocation. By default `docker-compose exec` allocates a TTY.]' \ '--index=[Index of the container if there are multiple instances of a service \[default: 1\]]:index: ' \ + '*'{-e,--env}'[KEY=VAL Set an environment variable (can be used multiple times)]:environment variable KEY=VAL: ' \ + '(-w --workdir)'{-w,--workdir=}'[Working directory inside the container]:workdir: ' \ '(-):running services:__docker-compose_runningservices' \ '(-):command: _command_names -e' \ '*::arguments: _normal' && ret=0 @@ -253,12 +174,12 @@ __docker-compose_subcommand() { (help) _arguments ':subcommand:__docker-compose_commands' && ret=0 ;; - (images) - _arguments \ - $opts_help \ - '-q[Only display IDs]' \ - '*:services:__docker-compose_services_all' && ret=0 - ;; + (images) + _arguments \ + $opts_help \ + '-q[Only display IDs]' \ + '*:services:__docker-compose_services' && ret=0 + ;; (kill) _arguments \ $opts_help \ @@ -272,7 +193,7 @@ __docker-compose_subcommand() { $opts_no_color \ '--tail=[Number of lines to show from the end of the logs for each container.]:number of lines: ' \ '(-t --timestamps)'{-t,--timestamps}'[Show timestamps]' \ - '*:services:__docker-compose_services_all' && ret=0 + '*:services:__docker-compose_services' && ret=0 ;; (pause) _arguments \ @@ -291,12 +212,16 @@ __docker-compose_subcommand() { _arguments \ $opts_help \ '-q[Only display IDs]' \ - '*:services:__docker-compose_services_all' && ret=0 + '--filter KEY=VAL[Filter services by a property]:=:' \ + '*:services:__docker-compose_services' && ret=0 ;; (pull) _arguments \ $opts_help \ '--ignore-pull-failures[Pull what it can and ignores images with pull failures.]' \ + '--no-parallel[Disable parallel pulling]' \ + '(-q --quiet)'{-q,--quiet}'[Pull without printing progress information]' \ + '--include-deps[Also pull services declared as dependencies]' \ '*:services:__docker-compose_services_from_image' && ret=0 ;; (push) @@ -318,6 +243,7 @@ __docker-compose_subcommand() { $opts_no_deps \ '-d[Detached mode: Run container in the background, print new container name.]' \ '*-e[KEY=VAL Set an environment variable (can be used multiple times)]:environment variable KEY=VAL: ' \ + '*'{-l,--label}'[KEY=VAL Add or override a label (can be used multiple times)]:label KEY=VAL: ' \ '--entrypoint[Overwrite the entrypoint of the image.]:entry point: ' \ '--name=[Assign a name to the container]:name: ' \ '(-p --publish)'{-p,--publish=}"[Publish a container's port(s) to the host]" \ @@ -327,6 +253,7 @@ __docker-compose_subcommand() { '(-u --user)'{-u,--user=}'[Run as specified username or uid]:username or uid:_users' \ '(-v --volume)*'{-v,--volume=}'[Bind mount a volume]:volume: ' \ '(-w --workdir)'{-w,--workdir=}'[Working directory inside the container]:workdir: ' \ + "--use-aliases[Use the services network aliases in the network(s) the container connects to]" \ '(-):services:__docker-compose_services' \ '(-):command: _command_names -e' \ '*::arguments: _normal' && ret=0 @@ -370,8 +297,10 @@ __docker-compose_subcommand() { "(--no-build)--build[Build images before starting containers.]" \ "(-d)--abort-on-container-exit[Stops all containers if any container was stopped. Incompatible with -d.]" \ '(-t --timeout)'{-t,--timeout}"[Use this timeout in seconds for container shutdown when attached or when containers are already running. (default: 10)]:seconds: " \ + '--scale[SERVICE=NUM Scale SERVICE to NUM instances. Overrides the `scale` setting in the Compose file if present.]:service scale SERVICE=NUM: ' \ + '--exit-code-from=[Return the exit code of the selected service container. Implies --abort-on-container-exit]:service:__docker-compose_services' \ $opts_remove_orphans \ - '*:services:__docker-compose_services_all' && ret=0 + '*:services:__docker-compose_services' && ret=0 ;; (version) _arguments \ @@ -410,8 +339,11 @@ _docker-compose() { '(- :)'{-h,--help}'[Get help]' \ '*'{-f,--file}"[${file_description}]:file:_files -g '*.yml'" \ '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ - '--verbose[Show more output]' \ + "--compatibility[If set, Compose will attempt to convert deploy keys in v3 files to their non-Swarm equivalent]" \ '(- :)'{-v,--version}'[Print version and exit]' \ + '--verbose[Show more output]' \ + '--log-level=[Set log level]:level:(DEBUG INFO WARNING ERROR CRITICAL)' \ + '--no-ansi[Do not print ANSI control characters]' \ '(-H --host)'{-H,--host}'[Daemon socket to connect to]:host:' \ '--tls[Use TLS; implied by --tlsverify]' \ '--tlscacert=[Trust certs signed only by this CA]:ca path:' \ From a7c05f41f1f213daff93a253412aa30d2c3769c0 Mon Sep 17 00:00:00 2001 From: Maxwell Bloch Date: Wed, 12 Sep 2018 19:11:10 -0400 Subject: [PATCH 35/46] Handle userns security - Adds `--userns=host` when `userns-remap` is set Signed-off-by: Maxwell Bloch --- script/run/run.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/run/run.sh b/script/run/run.sh index fe253875e..6b004606c 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -55,4 +55,9 @@ else DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i" fi +# Handle userns security +if [ ! -z "$(docker info 2>/dev/null | grep userns)" ]; then + DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS --userns=host" +fi + exec docker run --rm $DOCKER_RUN_OPTIONS $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w "$(pwd)" $IMAGE "$@" From 9f9122cd9543931760ae6fe4546d7b2e0c8b82cc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 19 Sep 2018 11:18:09 -0700 Subject: [PATCH 36/46] Don't convert slashes for UNIX paths on Windows hosts Signed-off-by: Joffrey F --- compose/config/types.py | 19 ++++++++++++++++--- tests/unit/config/config_test.py | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index ff9875218..838fb9f58 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -136,6 +136,20 @@ def normalize_path_for_engine(path): return path.replace('\\', '/') +def normpath(path, win_host=False): + """ Custom path normalizer that handles Compose-specific edge cases like + UNIX paths on Windows hosts and vice-versa. """ + + sysnorm = ntpath.normpath if win_host else os.path.normpath + # If a path looks like a UNIX absolute path on Windows, it probably is; + # we'll need to revert the backslashes to forward slashes after normalization + flip_slashes = path.startswith('/') and IS_WINDOWS_PLATFORM + path = sysnorm(path) + if flip_slashes: + path = path.replace('\\', '/') + return path + + class MountSpec(object): options_map = { 'volume': { @@ -152,12 +166,11 @@ class MountSpec(object): @classmethod def parse(cls, mount_dict, normalize=False, win_host=False): - normpath = ntpath.normpath if win_host else os.path.normpath if mount_dict.get('source'): if mount_dict['type'] == 'tmpfs': raise ConfigurationError('tmpfs mounts can not specify a source') - mount_dict['source'] = normpath(mount_dict['source']) + mount_dict['source'] = normpath(mount_dict['source'], win_host) if normalize: mount_dict['source'] = normalize_path_for_engine(mount_dict['source']) @@ -247,7 +260,7 @@ class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): else: external = parts[0] parts = separate_next_section(parts[1]) - external = ntpath.normpath(external) + external = normpath(external, True) internal = parts[0] if len(parts) > 1: if ':' in parts[1]: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 08b92a573..1d42c10d5 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1291,7 +1291,7 @@ class ConfigTest(unittest.TestCase): assert tmpfs_mount.target == '/tmpfs' assert not tmpfs_mount.is_named_volume - assert host_mount.source == os.path.normpath('/abc') + assert host_mount.source == '/abc' assert host_mount.target == '/xyz' assert not host_mount.is_named_volume From 96a49a02534d5bd58f8adcbd4148076d463afea9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 19 Sep 2018 15:35:22 -0700 Subject: [PATCH 37/46] Force consistent behavior around long paths on Windows builds Signed-off-by: Joffrey F --- compose/const.py | 1 + compose/service.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/compose/const.py b/compose/const.py index f4b9489e1..0e66a297a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -22,6 +22,7 @@ NANOCPUS_SCALE = 1000000000 PARALLEL_LIMIT = 64 SECRETS_PATH = '/run/secrets' +WINDOWS_LONGPATH_PREFIX = '\\\\?\\' COMPOSEFILE_V1 = ComposeVersion('1') COMPOSEFILE_V2_0 = ComposeVersion('2.0') diff --git a/compose/service.py b/compose/service.py index 199be8f1f..aca24ce17 100644 --- a/compose/service.py +++ b/compose/service.py @@ -43,6 +43,7 @@ from .const import LABEL_SERVICE from .const import LABEL_SLUG from .const import LABEL_VERSION from .const import NANOCPUS_SCALE +from .const import WINDOWS_LONGPATH_PREFIX from .container import Container from .errors import HealthCheckFailed from .errors import NoHealthCheckConfigured @@ -1048,12 +1049,7 @@ class Service(object): for k, v in self._parse_proxy_config().items(): build_args.setdefault(k, v) - # python2 os.stat() doesn't support unicode on some UNIX, so we - # encode it to a bytestring to be safe - path = build_opts.get('context') - if not six.PY3 and not IS_WINDOWS_PLATFORM: - path = path.encode('utf8') - + path = rewrite_build_path(build_opts.get('context')) if self.platform and version_lt(self.client.api_version, '1.35'): raise OperationFailedError( 'Impossible to perform platform-targeted builds for API version < 1.35' @@ -1662,3 +1658,15 @@ def convert_blkio_config(blkio_config): arr.append(dict([(k.capitalize(), v) for k, v in item.items()])) result[field] = arr return result + + +def rewrite_build_path(path): + # python2 os.stat() doesn't support unicode on some UNIX, so we + # encode it to a bytestring to be safe + if not six.PY3 and not IS_WINDOWS_PLATFORM: + path = path.encode('utf8') + + if IS_WINDOWS_PLATFORM and not path.startswith(WINDOWS_LONGPATH_PREFIX): + path = WINDOWS_LONGPATH_PREFIX + os.path.normpath(path) + + return path From a2ec572fdf45655d5805df0abb0a84e76e6f8bf3 Mon Sep 17 00:00:00 2001 From: Boris HUISGEN Date: Sun, 6 May 2018 12:26:37 +0200 Subject: [PATCH 38/46] Use same tag as service definition Signed-off-by: Boris HUISGEN --- compose/cli/main.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index e0acf0711..69d6ca4dd 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -579,10 +579,18 @@ class TopLevelCommand(object): rows = [] for container in containers: image_config = container.image_config - repo_tags = ( - image_config['RepoTags'][0].rsplit(':', 1) if image_config['RepoTags'] - else ('', '') - ) + service = self.project.get_service(container.service) + if service.image_name in image_config['RepoTags']: + index = image_config['RepoTags'].index(service.image_name) + repo_tags = ( + image_config['RepoTags'][index].rsplit(':', 1) if image_config['RepoTags'] + else ('', '') + ) + else: + repo_tags = ( + image_config['RepoTags'][0].rsplit(':', 1) if image_config['RepoTags'] + else ('', '') + ) image_id = image_config['Id'].split(':')[1][:12] size = human_readable_file_size(image_config['Size']) rows.append([ From 1b668973a2f3c28370a77d8af51cd2dc082dda0a Mon Sep 17 00:00:00 2001 From: Boris HUISGEN Date: Thu, 5 Jul 2018 23:33:13 +0200 Subject: [PATCH 39/46] Add acceptance test Signed-off-by: Boris HUISGEN --- tests/acceptance/cli_test.py | 13 +++++++++++++ tests/fixtures/images-service-tag/dev/Dockerfile | 2 ++ .../images-service-tag/dev/docker-compose.yml | 6 ++++++ .../fixtures/images-service-tag/docker-compose.yml | 5 +++++ tests/fixtures/images-service-tag/prod/Dockerfile | 2 ++ .../images-service-tag/prod/docker-compose.yml | 6 ++++++ 6 files changed, 34 insertions(+) create mode 100644 tests/fixtures/images-service-tag/dev/Dockerfile create mode 100644 tests/fixtures/images-service-tag/dev/docker-compose.yml create mode 100644 tests/fixtures/images-service-tag/docker-compose.yml create mode 100644 tests/fixtures/images-service-tag/prod/Dockerfile create mode 100644 tests/fixtures/images-service-tag/prod/docker-compose.yml diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 015180bc7..ef7e88990 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2770,3 +2770,16 @@ class CLITestCase(DockerClientTestCase): with pytest.raises(DuplicateOverrideFileFound): get_project(self.base_dir, []) self.base_dir = None + + def test_images_use_service_tag(self): + pull_busybox(self.client) + self.base_dir = 'tests/fixtures/images-service-tag/dev' + self.dispatch(['build']) + self.base_dir = 'tests/fixtures/images-service-tag/prod' + self.dispatch(['build']) + self.base_dir = 'tests/fixtures/images-service-tag' + self.dispatch(['up', '-d']) + result = self.dispatch(['images']) + self.dispatch(['down']) + + assert 'dev' in result.stdout diff --git a/tests/fixtures/images-service-tag/dev/Dockerfile b/tests/fixtures/images-service-tag/dev/Dockerfile new file mode 100644 index 000000000..570e11ae3 --- /dev/null +++ b/tests/fixtures/images-service-tag/dev/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:latest +RUN touch /same-image diff --git a/tests/fixtures/images-service-tag/dev/docker-compose.yml b/tests/fixtures/images-service-tag/dev/docker-compose.yml new file mode 100644 index 000000000..cb01f6f35 --- /dev/null +++ b/tests/fixtures/images-service-tag/dev/docker-compose.yml @@ -0,0 +1,6 @@ +version: "2.2" + +services: + test: + image: busybox:dev + build: . diff --git a/tests/fixtures/images-service-tag/docker-compose.yml b/tests/fixtures/images-service-tag/docker-compose.yml new file mode 100644 index 000000000..824b9416b --- /dev/null +++ b/tests/fixtures/images-service-tag/docker-compose.yml @@ -0,0 +1,5 @@ +version: "2.2" + +services: + test: + image: busybox:dev diff --git a/tests/fixtures/images-service-tag/prod/Dockerfile b/tests/fixtures/images-service-tag/prod/Dockerfile new file mode 100644 index 000000000..570e11ae3 --- /dev/null +++ b/tests/fixtures/images-service-tag/prod/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:latest +RUN touch /same-image diff --git a/tests/fixtures/images-service-tag/prod/docker-compose.yml b/tests/fixtures/images-service-tag/prod/docker-compose.yml new file mode 100644 index 000000000..cb01f6f35 --- /dev/null +++ b/tests/fixtures/images-service-tag/prod/docker-compose.yml @@ -0,0 +1,6 @@ +version: "2.2" + +services: + test: + image: busybox:dev + build: . From 7d0fb7d3f33a81e39d08871d71f38cc68b51f39f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 20 Sep 2018 15:34:14 -0700 Subject: [PATCH 40/46] Rewrite images command method to decrease complexity Also ensure we properly detect matching image names when tag is omitted Signed-off-by: Joffrey F --- compose/cli/main.py | 70 ++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 69d6ca4dd..f2e76c1ad 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -568,39 +568,43 @@ class TopLevelCommand(object): if options['--quiet']: for image in set(c.image for c in containers): print(image.split(':')[1]) - else: - headers = [ - 'Container', - 'Repository', - 'Tag', - 'Image Id', - 'Size' - ] - rows = [] - for container in containers: - image_config = container.image_config - service = self.project.get_service(container.service) - if service.image_name in image_config['RepoTags']: - index = image_config['RepoTags'].index(service.image_name) - repo_tags = ( - image_config['RepoTags'][index].rsplit(':', 1) if image_config['RepoTags'] - else ('', '') - ) - else: - repo_tags = ( - image_config['RepoTags'][0].rsplit(':', 1) if image_config['RepoTags'] - else ('', '') - ) - image_id = image_config['Id'].split(':')[1][:12] - size = human_readable_file_size(image_config['Size']) - rows.append([ - container.name, - repo_tags[0], - repo_tags[1], - image_id, - size - ]) - print(Formatter().table(headers, rows)) + return + + def add_default_tag(img_name): + if ':' not in img_name.split('/')[-1]: + return '{}:latest'.format(img_name) + return img_name + + headers = [ + 'Container', + 'Repository', + 'Tag', + 'Image Id', + 'Size' + ] + rows = [] + for container in containers: + image_config = container.image_config + service = self.project.get_service(container.service) + index = 0 + img_name = add_default_tag(service.image_name) + if img_name in image_config['RepoTags']: + index = image_config['RepoTags'].index(img_name) + repo_tags = ( + image_config['RepoTags'][index].rsplit(':', 1) if image_config['RepoTags'] + else ('', '') + ) + + image_id = image_config['Id'].split(':')[1][:12] + size = human_readable_file_size(image_config['Size']) + rows.append([ + container.name, + repo_tags[0], + repo_tags[1], + image_id, + size + ]) + print(Formatter().table(headers, rows)) def kill(self, options): """ From 834acca49712735ec9e64b39991d8d66989bf1d6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 20 Sep 2018 15:37:42 -0700 Subject: [PATCH 41/46] Update acceptance test for image matching Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 11 ++++------- tests/fixtures/images-service-tag/Dockerfile | 2 ++ tests/fixtures/images-service-tag/dev/Dockerfile | 2 -- .../images-service-tag/dev/docker-compose.yml | 6 ------ .../fixtures/images-service-tag/docker-compose.yml | 13 +++++++++---- tests/fixtures/images-service-tag/prod/Dockerfile | 2 -- .../images-service-tag/prod/docker-compose.yml | 6 ------ 7 files changed, 15 insertions(+), 27 deletions(-) create mode 100644 tests/fixtures/images-service-tag/Dockerfile delete mode 100644 tests/fixtures/images-service-tag/dev/Dockerfile delete mode 100644 tests/fixtures/images-service-tag/dev/docker-compose.yml delete mode 100644 tests/fixtures/images-service-tag/prod/Dockerfile delete mode 100644 tests/fixtures/images-service-tag/prod/docker-compose.yml diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ef7e88990..3d063d853 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2773,13 +2773,10 @@ class CLITestCase(DockerClientTestCase): def test_images_use_service_tag(self): pull_busybox(self.client) - self.base_dir = 'tests/fixtures/images-service-tag/dev' - self.dispatch(['build']) - self.base_dir = 'tests/fixtures/images-service-tag/prod' - self.dispatch(['build']) self.base_dir = 'tests/fixtures/images-service-tag' - self.dispatch(['up', '-d']) + self.dispatch(['up', '-d', '--build']) result = self.dispatch(['images']) - self.dispatch(['down']) - assert 'dev' in result.stdout + assert re.search(r'foo1.+test[ \t]+dev', result.stdout) is not None + assert re.search(r'foo2.+test[ \t]+prod', result.stdout) is not None + assert re.search(r'foo3.+_foo3[ \t]+latest', result.stdout) is not None diff --git a/tests/fixtures/images-service-tag/Dockerfile b/tests/fixtures/images-service-tag/Dockerfile new file mode 100644 index 000000000..145e0202f --- /dev/null +++ b/tests/fixtures/images-service-tag/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:latest +RUN touch /foo diff --git a/tests/fixtures/images-service-tag/dev/Dockerfile b/tests/fixtures/images-service-tag/dev/Dockerfile deleted file mode 100644 index 570e11ae3..000000000 --- a/tests/fixtures/images-service-tag/dev/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM busybox:latest -RUN touch /same-image diff --git a/tests/fixtures/images-service-tag/dev/docker-compose.yml b/tests/fixtures/images-service-tag/dev/docker-compose.yml deleted file mode 100644 index cb01f6f35..000000000 --- a/tests/fixtures/images-service-tag/dev/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "2.2" - -services: - test: - image: busybox:dev - build: . diff --git a/tests/fixtures/images-service-tag/docker-compose.yml b/tests/fixtures/images-service-tag/docker-compose.yml index 824b9416b..aff3cf285 100644 --- a/tests/fixtures/images-service-tag/docker-compose.yml +++ b/tests/fixtures/images-service-tag/docker-compose.yml @@ -1,5 +1,10 @@ -version: "2.2" - +version: "2.4" services: - test: - image: busybox:dev + foo1: + build: . + image: test:dev + foo2: + build: . + image: test:prod + foo3: + build: . diff --git a/tests/fixtures/images-service-tag/prod/Dockerfile b/tests/fixtures/images-service-tag/prod/Dockerfile deleted file mode 100644 index 570e11ae3..000000000 --- a/tests/fixtures/images-service-tag/prod/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM busybox:latest -RUN touch /same-image diff --git a/tests/fixtures/images-service-tag/prod/docker-compose.yml b/tests/fixtures/images-service-tag/prod/docker-compose.yml deleted file mode 100644 index cb01f6f35..000000000 --- a/tests/fixtures/images-service-tag/prod/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "2.2" - -services: - test: - image: busybox:dev - build: . From 936e6971f91438394b7fe051ac5ab56797d1feaf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Sep 2018 23:46:38 +0000 Subject: [PATCH 42/46] "Bump 1.23.0-rc1" Signed-off-by: Joffrey F --- CHANGELOG.md | 62 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d22c16454..e9226db23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,68 @@ Change log ========== +1.23.0 (2018-10-10) +------------------- + +### Features + +### Important note + +The default naming scheme for containers created by Compose in this version +has changed from `__` to +`___`, where `` is a randomly-generated +hexadecimal string. Please make sure to update scripts relying on the old +naming scheme accordingly before upgrading. + +### All versions + +- Logs for containers restarting after a crash will now appear in the output + of the `up` and `logs` commands. + +- Added `--hash` option to the `docker-compose config` command, allowing users + to print a hash string for each service's configuration to facilitate rolling + updates. + +- Output for the `pull` command now reports status / progress even when pulling + multiple images in parallel. + +- For images with multiple names, Compose will now attempt to match the one + present in the service configuration in the output of the `images` command. + +### Bugfixes + +- Parallel `run` commands for the same service will no longer fail due to name + collisions. + +- Fixed an issue where paths longer than 260 characters on Windows clients would + cause `docker-compose build` to fail. + +- Fixed a bug where attempting to mount `/var/run/docker.sock` with + Docker Desktop for Windows would result in failure. + +- The `--project-directory` option is now used by Compose to determine where to + look for the `.env` file. + +- `docker-compose build` no longer fails when attempting to pull an image with + credentials provided by the gcloud credential helper. + +- Fixed the `--exit-code-from` option in `docker-compose up` to always report + the actual exit code even when the watched container isn't the cause of the + exit. + +- Fixed a bug that caused hash configuration with multiple networks to be + inconsistent, causing some services to be unnecessarily restarted. + +- Fixed a pipe handling issue when using the containerized version of Compose. + +- Fixed a bug causing `external: false` entries in the Compose file to be + printed as `external: true` in the output of `docker-compose config` + +### Miscellaneous + +- The `zsh` completion script has been updated with new options, and no + longer suggests container names where service names are expected. + 1.22.0 (2018-07-17) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 3433b63cc..f0e3f3274 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.23.0dev' +__version__ = '1.23.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 6b004606c..fa2248609 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.22.0" +VERSION="1.23.0-rc1" IMAGE="docker/compose:$VERSION" From ec4ea8d2f14a72a31b6d14d54b274159c39095a6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 25 Sep 2018 00:46:52 +0000 Subject: [PATCH 43/46] "Bump 1.23.0-rc1" Signed-off-by: Joffrey F --- CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9226db23..3f2128090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,6 @@ Change log 1.23.0 (2018-10-10) ------------------- -### Features - ### Important note The default naming scheme for containers created by Compose in this version @@ -14,7 +12,7 @@ has changed from `__` to hexadecimal string. Please make sure to update scripts relying on the old naming scheme accordingly before upgrading. -### All versions +### Features - Logs for containers restarting after a crash will now appear in the output of the `up` and `logs` commands. From 47d740b800addf285bcbf96a4ce35918724a4ba6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Sep 2018 20:05:40 -0700 Subject: [PATCH 44/46] Fix some release script issues Signed-off-by: Joffrey F --- script/release/release.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index c6dc146a7..749ea49d3 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -77,19 +77,24 @@ def monitor_pr_status(pr_data): 'pending': 0, 'success': 0, 'failure': 0, + 'error': 0, } for detail in status.statuses: if detail.context == 'dco-signed': # dco-signed check breaks on merge remote-tracking ; ignore it continue - summary[detail.state] += 1 - print('{pending} pending, {success} successes, {failure} failures'.format(**summary)) - if summary['pending'] == 0 and summary['failure'] == 0 and summary['success'] > 0: + if detail.state in summary: + summary[detail.state] += 1 + print( + '{pending} pending, {success} successes, {failure} failures, ' + '{error} errors'.format(**summary) + ) + if summary['failure'] > 0 or summary['error'] > 0: + raise ScriptError('CI failures detected!') + elif summary['pending'] == 0 and summary['success'] > 0: # This check assumes at least 1 non-DCO CI check to avoid race conditions. # If testing on a repo without CI, use --skip-ci-check to avoid looping eternally return True - elif summary['failure'] > 0: - raise ScriptError('CI failures detected!') time.sleep(30) elif status.state == 'success': print('{} successes: all clear!'.format(status.total_count)) @@ -97,12 +102,14 @@ def monitor_pr_status(pr_data): def check_pr_mergeable(pr_data): - if not pr_data.mergeable: + if pr_data.mergeable is False: + # mergeable can also be null, in which case the warning would be a false positive. print( 'WARNING!! PR #{} can not currently be merged. You will need to ' 'resolve the conflicts manually before finalizing the release.'.format(pr_data.number) ) - return pr_data.mergeable + + return pr_data.mergeable is True def create_release_draft(repository, version, pr_data, files): From c327a498b03dfcc4137d3e05c904ab620eb12b90 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 25 Sep 2018 09:13:12 -0700 Subject: [PATCH 45/46] Don't rely on container names containing the db string to identify them Signed-off-by: Joffrey F --- tests/integration/project_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 858a8dfd7..63939676e 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -432,7 +432,7 @@ class ProjectTest(DockerClientTestCase): project.up(strategy=ConvergenceStrategy.always) assert len(project.containers()) == 2 - db_container = [c for c in project.containers() if 'db' in c.name][0] + db_container = [c for c in project.containers() if c.service == 'db'][0] assert db_container.id != old_db_id assert db_container.get('Volumes./etc') == db_volume_path @@ -452,7 +452,7 @@ class ProjectTest(DockerClientTestCase): project.up(strategy=ConvergenceStrategy.always) assert len(project.containers()) == 2 - db_container = [c for c in project.containers() if 'db' in c.name][0] + db_container = [c for c in project.containers() if c.service == 'db'][0] assert db_container.id != old_db_id assert db_container.get_mount('/etc')['Source'] == db_volume_path @@ -499,7 +499,7 @@ class ProjectTest(DockerClientTestCase): assert len(new_containers) == 2 assert [c.is_running for c in new_containers] == [True, True] - db_container = [c for c in new_containers if 'db' in c.name][0] + db_container = [c for c in new_containers if c.service == 'db'][0] assert db_container.id == old_db_id assert db_container.get_mount('/var/db')['Source'] == db_volume_path From 320e4819d873a857630f41064f6bb6f76ee0ff31 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 26 Sep 2018 13:44:42 -0700 Subject: [PATCH 46/46] Avoid cred helpers errors in release script Signed-off-by: Joffrey F --- script/release/README.md | 6 ++++++ script/release/release.sh | 14 ++++++++++++-- script/release/release/images.py | 8 ++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/script/release/README.md b/script/release/README.md index c5136c764..65883f5d3 100644 --- a/script/release/README.md +++ b/script/release/README.md @@ -20,6 +20,12 @@ following repositories: - docker/compose - docker/compose-tests +### A local Python environment + +While most of the release script is running inside a Docker container, +fetching local Docker credentials depends on the `docker` Python package +being available locally. + ### A Github account and Github API token Your Github account needs to have write access on the `docker/compose` repo. diff --git a/script/release/release.sh b/script/release/release.sh index 201182657..ee75b13a6 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -15,9 +15,19 @@ if test -z $BINTRAY_TOKEN; then exit 1 fi -docker run -e GITHUB_TOKEN=$GITHUB_TOKEN -e BINTRAY_TOKEN=$BINTRAY_TOKEN -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK -it \ +if test -z $(python -c "import docker; print(docker.version)" 2>/dev/null); then + echo "This script requires the 'docker' Python package to be installed locally" + exit 1 +fi + +hub_credentials=$(python -c "from docker import auth; cfg = auth.load_config(); print(auth.encode_header(auth.resolve_authconfig(cfg, 'docker.io')).decode('ascii'))") + +docker run -it \ + -e GITHUB_TOKEN=$GITHUB_TOKEN \ + -e BINTRAY_TOKEN=$BINTRAY_TOKEN \ + -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK \ + -e HUB_CREDENTIALS=$hub_credentials \ --mount type=bind,source=$(pwd),target=/src \ - --mount type=bind,source=$HOME/.docker,target=/root/.docker \ --mount type=bind,source=$HOME/.gitconfig,target=/root/.gitconfig \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ --mount type=bind,source=$HOME/.ssh,target=/root/.ssh \ diff --git a/script/release/release/images.py b/script/release/release/images.py index b8f7ed3d6..e247f596d 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -2,6 +2,8 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import base64 +import json import os import shutil @@ -15,6 +17,12 @@ class ImageManager(object): def __init__(self, version): self.docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) self.version = version + if 'HUB_CREDENTIALS' in os.environ: + print('HUB_CREDENTIALS found in environment, issuing login') + credentials = json.loads(base64.urlsafe_b64decode(os.environ['HUB_CREDENTIALS'])) + self.docker_client.login( + username=credentials['Username'], password=credentials['Password'] + ) def build_images(self, repository, files): print("Building release images...")