diff --git a/.circleci/config.yml b/.circleci/config.yml index 906b1c0dc..36dd8d57e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,6 +32,9 @@ jobs: - store_artifacts: path: dist/docker-compose-Darwin-x86_64 destination: docker-compose-Darwin-x86_64 + - store_artifacts: + path: dist/docker-compose-Darwin-x86_64.tgz + destination: docker-compose-Darwin-x86_64.tgz - deploy: name: Deploy binary to bintray command: | diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..13fb9bac0 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# GitHub code owners +# See https://help.github.com/articles/about-codeowners/ +# +# KEEP THIS FILE SORTED. Order is important. Last match takes precedence. + +* @ndeloof @rumpl @ulyssessouza diff --git a/CHANGELOG.md b/CHANGELOG.md index d1a6fae32..ea0ddf802 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ Change log ========== +1.25.1-rc1 (2019-11-29) +----------------------- + +### Bugfixes + +- Discard label `com.docker.compose.filepaths` having `None` as value. Typically, when coming from stdin + +- Add OSX binary as a directory to solve slow start up time caused by MacOS Catalina binary scan + +- Pass in HOME env-var in container mode (running with `script/run/run.sh`) + +- Revert behavior of "only pull images that we can't build" and replace by a warning informing the image we can't pull and must be built + 1.25.0 (2019-11-18) ------------------- diff --git a/MAINTAINERS b/MAINTAINERS index 5d4bd6a63..27bde5bb3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11,6 +11,7 @@ [Org] [Org."Core maintainers"] people = [ + "ndeloof", "rumpl", "ulyssessouza", ] @@ -77,6 +78,11 @@ Email = "mazz@houseofmnowster.com" GitHub = "mnowster" + [people.ndeloof] + Name = "Nicolas De Loof" + Email = "nicolas.deloof@gmail.com" + GitHub = "ndeloof" + [people.rumpl] Name = "Djordje Lukic" Email = "djordje.lukic@docker.com" diff --git a/README.md b/README.md index fd643f174..f1ae7e1ed 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Docker Compose ============== ![Docker Compose](logo.png?raw=true "Docker Compose Logo") +## :exclamation: The docker-compose project announces that as Python 2 reaches it's EOL, versions 1.25.x will be the last to support it. For more information, please refer to this [issue](https://github.com/docker/compose/issues/6890). + Compose is a tool for defining and running multi-container Docker applications. With Compose, you use a Compose file to configure your application's services. Then, using a single command, you create and start all the services diff --git a/compose/__init__.py b/compose/__init__.py index d35e818c7..cda820978 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.25.0' +__version__ = '1.25.1-rc1' diff --git a/compose/cli/command.py b/compose/cli/command.py index c3a10a043..1fa8a17a2 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -159,15 +159,25 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, def execution_context_labels(config_details, environment_file): extra_labels = [ - '{0}={1}'.format(LABEL_WORKING_DIR, os.path.abspath(config_details.working_dir)), - '{0}={1}'.format(LABEL_CONFIG_FILES, config_files_label(config_details)), + '{0}={1}'.format(LABEL_WORKING_DIR, os.path.abspath(config_details.working_dir)) ] + + if not use_config_from_stdin(config_details): + extra_labels.append('{0}={1}'.format(LABEL_CONFIG_FILES, config_files_label(config_details))) + if environment_file is not None: extra_labels.append('{0}={1}'.format(LABEL_ENVIRONMENT_FILE, os.path.normpath(environment_file))) return extra_labels +def use_config_from_stdin(config_details): + for c in config_details.config_files: + if not c.filename: + return True + return False + + def config_files_label(config_details): return ",".join( map(str, (os.path.normpath(c.filename) for c in config_details.config_files))) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index c4281cb4c..522ddf75d 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -114,3 +114,13 @@ def get_digest_from_push(events): if digest: return digest return None + + +def read_status(event): + status = event['status'].lower() + if 'progressDetail' in event: + detail = event['progressDetail'] + if 'current' in detail and 'total' in detail: + percentage = float(detail['current']) / float(detail['total']) + status = '{} ({:.1%})'.format(status, percentage) + return status diff --git a/compose/project.py b/compose/project.py index 094ce4d7a..d7405defd 100644 --- a/compose/project.py +++ b/compose/project.py @@ -11,6 +11,8 @@ from os import path import enum import six from docker.errors import APIError +from docker.errors import ImageNotFound +from docker.errors import NotFound from docker.utils import version_lt from . import parallel @@ -25,6 +27,7 @@ from .container import Container from .network import build_networks from .network import get_networks from .network import ProjectNetworks +from .progress_stream import read_status from .service import BuildAction from .service import ContainerNetworkMode from .service import ContainerPidMode @@ -619,49 +622,68 @@ class Project(object): def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False, include_deps=False): services = self.get_services(service_names, include_deps) - images_to_build = {service.image_name for service in services if service.can_be_built()} - services_to_pull = [service for service in services if service.image_name not in images_to_build] - - msg = not silent and 'Pulling' or None if parallel_pull: - def pull_service(service): - strm = service.pull(ignore_pull_failures, True, stream=True) - if strm is None: # Attempting to pull service with no `image` key is a no-op - return + self.parallel_pull(services, silent=silent) + else: + must_build = [] + for service in services: + try: + service.pull(ignore_pull_failures, silent=silent) + except (ImageNotFound, NotFound): + if service.can_be_built(): + must_build.append(service.name) + else: + raise + + if len(must_build): + log.warning('Some service image(s) must be built from source by running:\n' + ' docker-compose build {}' + .format(' '.join(must_build))) + + def parallel_pull(self, services, ignore_pull_failures=False, silent=False): + msg = 'Pulling' if not silent else None + must_build = [] + + def pull_service(service): + strm = service.pull(ignore_pull_failures, True, stream=True) + + if strm is None: # Attempting to pull service with no `image` key is a no-op + return + + try: writer = parallel.get_stream_writer() - for event in strm: if 'status' not in event: continue - status = event['status'].lower() - if 'progressDetail' in event: - detail = event['progressDetail'] - if 'current' in detail and 'total' in detail: - percentage = float(detail['current']) / float(detail['total']) - status = '{} ({:.1%})'.format(status, percentage) - + status = read_status(event) writer.write( msg, service.name, truncate_string(status), lambda s: s ) + except (ImageNotFound, NotFound): + if service.can_be_built(): + must_build.append(service.name) + else: + raise - _, errors = parallel.parallel_execute( - services_to_pull, - pull_service, - operator.attrgetter('name'), - msg, - limit=5, - ) - if len(errors): - combined_errors = '\n'.join([ - e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values() - ]) - raise ProjectError(combined_errors) + _, errors = parallel.parallel_execute( + services, + pull_service, + operator.attrgetter('name'), + msg, + limit=5, + ) - else: - for service in services_to_pull: - service.pull(ignore_pull_failures, silent=silent) + if len(must_build): + log.warning('Some service image(s) must be built from source by running:\n' + ' docker-compose build {}' + .format(' '.join(must_build))) + if len(errors): + combined_errors = '\n'.join([ + e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values() + ]) + raise ProjectError(combined_errors) def push(self, service_names=None, ignore_push_failures=False): unique_images = set() diff --git a/docker-compose_darwin.spec b/docker-compose_darwin.spec new file mode 100644 index 000000000..344c070d5 --- /dev/null +++ b/docker-compose_darwin.spec @@ -0,0 +1,108 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + +a = Analysis(['bin/docker-compose'], + pathex=['.'], + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + cipher=block_cipher) + +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) + +exe = EXE(pyz, + a.scripts, + exclude_binaries=True, + name='docker-compose', + debug=False, + strip=False, + upx=True, + console=True, + bootloader_ignore_signals=True) +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + [ + ( + 'compose/config/config_schema_v1.json', + 'compose/config/config_schema_v1.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v2.0.json', + 'compose/config/config_schema_v2.0.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v2.1.json', + 'compose/config/config_schema_v2.1.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v2.2.json', + 'compose/config/config_schema_v2.2.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v2.3.json', + 'compose/config/config_schema_v2.3.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v2.4.json', + 'compose/config/config_schema_v2.4.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.0.json', + 'compose/config/config_schema_v3.0.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.1.json', + 'compose/config/config_schema_v3.1.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.2.json', + 'compose/config/config_schema_v3.2.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.3.json', + 'compose/config/config_schema_v3.3.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.4.json', + 'compose/config/config_schema_v3.4.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.5.json', + 'compose/config/config_schema_v3.5.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.6.json', + 'compose/config/config_schema_v3.6.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.7.json', + 'compose/config/config_schema_v3.7.json', + 'DATA' + ), + ( + 'compose/GITSHA', + 'compose/GITSHA', + 'DATA' + ) + ], + strip=False, + upx=True, + upx_exclude=[], + name='docker-compose-Darwin-x86_64') diff --git a/script/build/osx b/script/build/osx index 529914586..66868756b 100755 --- a/script/build/osx +++ b/script/build/osx @@ -11,6 +11,14 @@ venv/bin/pip install -r requirements-build.txt venv/bin/pip install --no-deps . DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA + +# Build as a folder for macOS Catalina. +venv/bin/pyinstaller docker-compose_darwin.spec +dist/docker-compose-Darwin-x86_64/docker-compose version +(cd dist/docker-compose-Darwin-x86_64/ && tar zcvf ../docker-compose-Darwin-x86_64.tgz .) +rm -rf dist/docker-compose-Darwin-x86_64 + +# Build static binary for legacy. venv/bin/pyinstaller docker-compose.spec mv dist/docker-compose dist/docker-compose-Darwin-x86_64 dist/docker-compose-Darwin-x86_64 version diff --git a/script/circle/bintray-deploy.sh b/script/circle/bintray-deploy.sh index d508da365..a7cce726e 100755 --- a/script/circle/bintray-deploy.sh +++ b/script/circle/bintray-deploy.sh @@ -25,3 +25,11 @@ curl -f -T dist/docker-compose-${OS_NAME}-x86_64 -u$BINTRAY_USERNAME:$BINTRAY_AP -H "X-Bintray-Package: ${PKG_NAME}" -H "X-Bintray-Version: $CIRCLE_BRANCH" \ -H "X-Bintray-Override: 1" -H "X-Bintray-Publish: 1" -X PUT \ https://api.bintray.com/content/docker-compose/${CIRCLE_BRANCH}/docker-compose-${OS_NAME}-x86_64 || exit 1 + +# Upload folder format of docker-compose for macOS in addition to binary. +if [ "${OS_NAME}" == "Darwin" ]; then + curl -f -T dist/docker-compose-${OS_NAME}-x86_64.tgz -u$BINTRAY_USERNAME:$BINTRAY_API_KEY \ + -H "X-Bintray-Package: ${PKG_NAME}" -H "X-Bintray-Version: $CIRCLE_BRANCH" \ + -H "X-Bintray-Override: 1" -H "X-Bintray-Publish: 1" -X PUT \ + https://api.bintray.com/content/docker-compose/${CIRCLE_BRANCH}/docker-compose-${OS_NAME}-x86_64.tgz || exit 1 +fi diff --git a/script/release/release/downloader.py b/script/release/release/downloader.py index d92ae78b5..0e9b80130 100644 --- a/script/release/release/downloader.py +++ b/script/release/release/downloader.py @@ -55,6 +55,7 @@ class BinaryDownloader(requests.Session): def download_all(self, version): files = { + 'docker-compose-Darwin-x86_64.tgz': None, 'docker-compose-Darwin-x86_64': None, 'docker-compose-Linux-x86_64': None, 'docker-compose-Windows-x86_64.exe': None, diff --git a/script/run/run.sh b/script/run/run.sh index ffeec59a2..744766a06 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.25.0" +VERSION="1.25.1-rc1" IMAGE="docker/compose:$VERSION" @@ -36,18 +36,18 @@ if [ "$(pwd)" != '/' ]; then fi if [ -n "$COMPOSE_FILE" ]; then COMPOSE_OPTIONS="$COMPOSE_OPTIONS -e COMPOSE_FILE=$COMPOSE_FILE" - compose_dir=$(realpath $(dirname $COMPOSE_FILE)) + compose_dir=$(realpath "$(dirname "$COMPOSE_FILE")") fi # TODO: also check --file argument if [ -n "$compose_dir" ]; then VOLUMES="$VOLUMES -v $compose_dir:$compose_dir" fi if [ -n "$HOME" ]; then - VOLUMES="$VOLUMES -v $HOME:$HOME -v $HOME:/root" # mount $HOME in /root to share docker.config + VOLUMES="$VOLUMES -v $HOME:$HOME -e HOME" # Pass in HOME to share docker.config and allow ~/-relative paths to work. fi # Only allocate tty if we detect one -if [ -t 0 -a -t 1 ]; then +if [ -t 0 ] && [ -t 1 ]; then DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t" fi @@ -56,8 +56,9 @@ DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i" # Handle userns security -if [ ! -z "$(docker info 2>/dev/null | grep userns)" ]; then +if docker info --format '{{json .SecurityOptions}}' 2>/dev/null | grep -q 'name=userns'; then DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS --userns=host" fi +# shellcheck disable=SC2086 exec docker run --rm $DOCKER_RUN_OPTIONS $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w "$(pwd)" $IMAGE "$@" diff --git a/script/setup/osx b/script/setup/osx index 69280f8a2..08420fb23 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -36,7 +36,7 @@ if ! [ -x "$(command -v python3)" ]; then brew install python3 fi if ! [ -x "$(command -v virtualenv)" ]; then - pip install virtualenv==16.2.0 + pip3 install virtualenv==16.2.0 fi # diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a03d56567..b729e7d76 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -48,6 +48,7 @@ BUILD_PULL_TEXT = 'Status: Image is up to date for busybox:1.27.2' def start_process(base_dir, options): proc = subprocess.Popen( ['docker-compose'] + options, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=base_dir) @@ -55,8 +56,8 @@ def start_process(base_dir, options): return proc -def wait_on_process(proc, returncode=0): - stdout, stderr = proc.communicate() +def wait_on_process(proc, returncode=0, stdin=None): + stdout, stderr = proc.communicate(input=stdin) if proc.returncode != returncode: print("Stderr: {}".format(stderr)) print("Stdout: {}".format(stdout)) @@ -64,10 +65,10 @@ def wait_on_process(proc, returncode=0): return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) -def dispatch(base_dir, options, project_options=None, returncode=0): +def dispatch(base_dir, options, project_options=None, returncode=0, stdin=None): project_options = project_options or [] proc = start_process(base_dir, project_options + options) - return wait_on_process(proc, returncode=returncode) + return wait_on_process(proc, returncode=returncode, stdin=stdin) def wait_on_condition(condition, delay=0.1, timeout=40): @@ -156,8 +157,8 @@ class CLITestCase(DockerClientTestCase): self._project = get_project(self.base_dir, override_dir=self.override_dir) return self._project - def dispatch(self, options, project_options=None, returncode=0): - return dispatch(self.base_dir, options, project_options, returncode) + def dispatch(self, options, project_options=None, returncode=0, stdin=None): + return dispatch(self.base_dir, options, project_options, returncode, stdin) def execute(self, container, cmd): # Remove once Hijack and CloseNotifier sign a peace treaty @@ -241,6 +242,17 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/v2-full' assert self.dispatch(['config', '--quiet']).stdout == '' + def test_config_stdin(self): + config = b"""version: "3.7" +services: + web: + image: nginx + other: + image: alpine +""" + result = self.dispatch(['-f', '-', 'config', '--services'], stdin=config) + assert set(result.stdout.rstrip().split('\n')) == {'web', 'other'} + def test_config_with_hash_option(self): self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config', '--hash=*']) @@ -661,13 +673,6 @@ class CLITestCase(DockerClientTestCase): 'image library/nonexisting-image:latest not found' in result.stderr or 'pull access denied for nonexisting-image' in result.stderr) - def test_pull_with_build(self): - result = self.dispatch(['-f', 'pull-with-build.yml', 'pull']) - - assert 'Pulling simple' not in result.stderr - assert 'Pulling from_simple' not in result.stderr - assert 'Pulling another ...' in result.stderr - def test_pull_with_quiet(self): assert self.dispatch(['pull', '--quiet']).stderr == '' assert self.dispatch(['pull', '--quiet']).stdout == '' @@ -689,6 +694,14 @@ class CLITestCase(DockerClientTestCase): result.stderr ) + def test_pull_can_build(self): + result = self.dispatch([ + '-f', 'can-build-pull-failures.yml', 'pull'], + returncode=0 + ) + assert 'Some service image(s) must be built from source' in result.stderr + assert 'docker-compose build can_build' in result.stderr + def test_pull_with_no_deps(self): self.base_dir = 'tests/fixtures/links-composefile' result = self.dispatch(['pull', '--no-parallel', 'web']) diff --git a/tests/fixtures/simple-composefile/can-build-pull-failures.yml b/tests/fixtures/simple-composefile/can-build-pull-failures.yml new file mode 100644 index 000000000..1ffe8e0fb --- /dev/null +++ b/tests/fixtures/simple-composefile/can-build-pull-failures.yml @@ -0,0 +1,6 @@ +version: '3' +services: + can_build: + image: nonexisting-image-but-can-build:latest + build: . + command: top