Merge pull request #7071 from docker/bump-1.25.1-rc1

Bump 1.25.1-rc1

.circleci/config.yml

@@ -32,6 +32,9 @@ jobs:
       - store_artifacts:
           path: dist/docker-compose-Darwin-x86_64
           destination: docker-compose-Darwin-x86_64
+      - store_artifacts:
+          path: dist/docker-compose-Darwin-x86_64.tgz
+          destination: docker-compose-Darwin-x86_64.tgz
       - deploy:
           name: Deploy binary to bintray
           command: |

.github/CODEOWNERS

@@ -0,0 +1,6 @@
+# GitHub code owners
+# See https://help.github.com/articles/about-codeowners/
+#
+# KEEP THIS FILE SORTED. Order is important. Last match takes precedence.
+
+*       @ndeloof @rumpl @ulyssessouza

CHANGELOG.md

@@ -1,6 +1,19 @@
 Change log
 ==========
 
+1.25.1-rc1 (2019-11-29)
+-----------------------
+
+### Bugfixes
+
+- Discard label `com.docker.compose.filepaths` having `None` as value. Typically, when coming from stdin
+
+- Add OSX binary as a directory to solve slow start up time caused by MacOS Catalina binary scan
+
+- Pass in HOME env-var in container mode (running with `script/run/run.sh`)
+
+- Revert behavior of "only pull images that we can't build" and replace by a warning informing the image we can't pull and must be built
+
 1.25.0 (2019-11-18)
 -------------------
 

MAINTAINERS

@@ -11,6 +11,7 @@
 [Org]
 	[Org."Core maintainers"]
 		people = [
+			"ndeloof",
 			"rumpl",
 			"ulyssessouza",
 		]
@@ -77,6 +78,11 @@
 	Email = "mazz@houseofmnowster.com"
 	GitHub = "mnowster"
 
+	[people.ndeloof]
+	Name = "Nicolas De Loof"
+	Email = "nicolas.deloof@gmail.com"
+	GitHub = "ndeloof"
+
 	[people.rumpl]
 	Name = "Djordje Lukic"
 	Email = "djordje.lukic@docker.com"

README.md

@@ -2,6 +2,8 @@ Docker Compose
 ==============
 ![Docker Compose](logo.png?raw=true "Docker Compose Logo")
 
+## :exclamation: The docker-compose project announces that as Python 2 reaches it's EOL, versions 1.25.x will be the last to support it. For more information, please refer to this [issue](https://github.com/docker/compose/issues/6890).
+
 Compose is a tool for defining and running multi-container Docker applications.
 With Compose, you use a Compose file to configure your application's services.
 Then, using a single command, you create and start all the services

compose/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-__version__ = '1.25.0'
+__version__ = '1.25.1-rc1'

compose/cli/command.py

@@ -159,15 +159,25 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False,
 
 def execution_context_labels(config_details, environment_file):
     extra_labels = [
-        '{0}={1}'.format(LABEL_WORKING_DIR, os.path.abspath(config_details.working_dir)),
+        '{0}={1}'.format(LABEL_WORKING_DIR, os.path.abspath(config_details.working_dir))
-        '{0}={1}'.format(LABEL_CONFIG_FILES, config_files_label(config_details)),
     ]
+
+    if not use_config_from_stdin(config_details):
+        extra_labels.append('{0}={1}'.format(LABEL_CONFIG_FILES, config_files_label(config_details)))
+
     if environment_file is not None:
         extra_labels.append('{0}={1}'.format(LABEL_ENVIRONMENT_FILE,
                                              os.path.normpath(environment_file)))
     return extra_labels
 
 
+def use_config_from_stdin(config_details):
+    for c in config_details.config_files:
+        if not c.filename:
+            return True
+    return False
+
+
 def config_files_label(config_details):
     return ",".join(
         map(str, (os.path.normpath(c.filename) for c in config_details.config_files)))

compose/progress_stream.py

@@ -114,3 +114,13 @@ def get_digest_from_push(events):
         if digest:
             return digest
     return None
+
+
+def read_status(event):
+    status = event['status'].lower()
+    if 'progressDetail' in event:
+        detail = event['progressDetail']
+        if 'current' in detail and 'total' in detail:
+            percentage = float(detail['current']) / float(detail['total'])
+            status = '{} ({:.1%})'.format(status, percentage)
+    return status

compose/project.py

@@ -11,6 +11,8 @@ from os import path
 import enum
 import six
 from docker.errors import APIError
+from docker.errors import ImageNotFound
+from docker.errors import NotFound
 from docker.utils import version_lt
 
 from . import parallel
@@ -25,6 +27,7 @@ from .container import Container
 from .network import build_networks
 from .network import get_networks
 from .network import ProjectNetworks
+from .progress_stream import read_status
 from .service import BuildAction
 from .service import ContainerNetworkMode
 from .service import ContainerPidMode
@@ -619,49 +622,68 @@ class Project(object):
     def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False,
              include_deps=False):
         services = self.get_services(service_names, include_deps)
-        images_to_build = {service.image_name for service in services if service.can_be_built()}
-        services_to_pull = [service for service in services if service.image_name not in images_to_build]
-
-        msg = not silent and 'Pulling' or None
 
         if parallel_pull:
-            def pull_service(service):
+            self.parallel_pull(services, silent=silent)
-                strm = service.pull(ignore_pull_failures, True, stream=True)
-                if strm is None:  # Attempting to pull service with no `image` key is a no-op
-                    return
 
+        else:
+            must_build = []
+            for service in services:
+                try:
+                    service.pull(ignore_pull_failures, silent=silent)
+                except (ImageNotFound, NotFound):
+                    if service.can_be_built():
+                        must_build.append(service.name)
+                    else:
+                        raise
+
+            if len(must_build):
+                log.warning('Some service image(s) must be built from source by running:\n'
+                            '    docker-compose build {}'
+                            .format(' '.join(must_build)))
+
+    def parallel_pull(self, services, ignore_pull_failures=False, silent=False):
+        msg = 'Pulling' if not silent else None
+        must_build = []
+
+        def pull_service(service):
+            strm = service.pull(ignore_pull_failures, True, stream=True)
+
+            if strm is None:  # Attempting to pull service with no `image` key is a no-op
+                return
+
+            try:
                 writer = parallel.get_stream_writer()
-
                 for event in strm:
                     if 'status' not in event:
                         continue
-                    status = event['status'].lower()
+                    status = read_status(event)
-                    if 'progressDetail' in event:
-                        detail = event['progressDetail']
-                        if 'current' in detail and 'total' in detail:
-                            percentage = float(detail['current']) / float(detail['total'])
-                            status = '{} ({:.1%})'.format(status, percentage)
-
                     writer.write(
                         msg, service.name, truncate_string(status), lambda s: s
                     )
+            except (ImageNotFound, NotFound):
+                if service.can_be_built():
+                    must_build.append(service.name)
+                else:
+                    raise
 
-            _, errors = parallel.parallel_execute(
+        _, errors = parallel.parallel_execute(
-                services_to_pull,
+            services,
-                pull_service,
+            pull_service,
-                operator.attrgetter('name'),
+            operator.attrgetter('name'),
-                msg,
+            msg,
-                limit=5,
+            limit=5,
-            )
+        )
-            if len(errors):
-                combined_errors = '\n'.join([
-                    e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
-                ])
-                raise ProjectError(combined_errors)
 
-        else:
+        if len(must_build):
-            for service in services_to_pull:
+            log.warning('Some service image(s) must be built from source by running:\n'
-                service.pull(ignore_pull_failures, silent=silent)
+                        '    docker-compose build {}'
+                        .format(' '.join(must_build)))
+        if len(errors):
+            combined_errors = '\n'.join([
+                e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values()
+            ])
+            raise ProjectError(combined_errors)
 
     def push(self, service_names=None, ignore_push_failures=False):
         unique_images = set()

docker-compose_darwin.spec

@@ -0,0 +1,108 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+block_cipher = None
+
+a = Analysis(['bin/docker-compose'],
+             pathex=['.'],
+             hiddenimports=[],
+             hookspath=[],
+             runtime_hooks=[],
+             cipher=block_cipher)
+
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+
+exe = EXE(pyz,
+          a.scripts,
+          exclude_binaries=True,
+          name='docker-compose',
+          debug=False,
+          strip=False,
+          upx=True,
+          console=True,
+          bootloader_ignore_signals=True)
+coll = COLLECT(exe,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          [
+            (
+                'compose/config/config_schema_v1.json',
+                'compose/config/config_schema_v1.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v2.0.json',
+                'compose/config/config_schema_v2.0.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v2.1.json',
+                'compose/config/config_schema_v2.1.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v2.2.json',
+                'compose/config/config_schema_v2.2.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v2.3.json',
+                'compose/config/config_schema_v2.3.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v2.4.json',
+                'compose/config/config_schema_v2.4.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.0.json',
+                'compose/config/config_schema_v3.0.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.1.json',
+                'compose/config/config_schema_v3.1.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.2.json',
+                'compose/config/config_schema_v3.2.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.3.json',
+                'compose/config/config_schema_v3.3.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.4.json',
+                'compose/config/config_schema_v3.4.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.5.json',
+                'compose/config/config_schema_v3.5.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.6.json',
+                'compose/config/config_schema_v3.6.json',
+                'DATA'
+            ),
+            (
+                'compose/config/config_schema_v3.7.json',
+                'compose/config/config_schema_v3.7.json',
+                'DATA'
+            ),
+            (
+                'compose/GITSHA',
+                'compose/GITSHA',
+                'DATA'
+            )
+          ],
+          strip=False,
+          upx=True,
+          upx_exclude=[],
+          name='docker-compose-Darwin-x86_64')

script/build/osx

@@ -11,6 +11,14 @@ venv/bin/pip install -r requirements-build.txt
 venv/bin/pip install --no-deps .
 DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)"
 echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA
+
+# Build as a folder for macOS Catalina.
+venv/bin/pyinstaller docker-compose_darwin.spec
+dist/docker-compose-Darwin-x86_64/docker-compose version
+(cd dist/docker-compose-Darwin-x86_64/ && tar zcvf ../docker-compose-Darwin-x86_64.tgz .)
+rm -rf dist/docker-compose-Darwin-x86_64
+
+# Build static binary for legacy.
 venv/bin/pyinstaller docker-compose.spec
 mv dist/docker-compose dist/docker-compose-Darwin-x86_64
 dist/docker-compose-Darwin-x86_64 version

script/circle/bintray-deploy.sh

@@ -25,3 +25,11 @@ curl -f -T dist/docker-compose-${OS_NAME}-x86_64 -u$BINTRAY_USERNAME:$BINTRAY_AP
   -H "X-Bintray-Package: ${PKG_NAME}" -H "X-Bintray-Version: $CIRCLE_BRANCH" \
   -H "X-Bintray-Override: 1" -H "X-Bintray-Publish: 1" -X PUT \
   https://api.bintray.com/content/docker-compose/${CIRCLE_BRANCH}/docker-compose-${OS_NAME}-x86_64 || exit 1
+
+# Upload folder format of docker-compose for macOS in addition to binary.
+if [ "${OS_NAME}" == "Darwin" ]; then
+  curl -f -T dist/docker-compose-${OS_NAME}-x86_64.tgz -u$BINTRAY_USERNAME:$BINTRAY_API_KEY \
+    -H "X-Bintray-Package: ${PKG_NAME}" -H "X-Bintray-Version: $CIRCLE_BRANCH" \
+    -H "X-Bintray-Override: 1" -H "X-Bintray-Publish: 1" -X PUT \
+    https://api.bintray.com/content/docker-compose/${CIRCLE_BRANCH}/docker-compose-${OS_NAME}-x86_64.tgz || exit 1
+fi

script/release/release/downloader.py

@@ -55,6 +55,7 @@ class BinaryDownloader(requests.Session):
 
     def download_all(self, version):
         files = {
+            'docker-compose-Darwin-x86_64.tgz': None,
             'docker-compose-Darwin-x86_64': None,
             'docker-compose-Linux-x86_64': None,
             'docker-compose-Windows-x86_64.exe': None,

script/run/run.sh

@@ -15,7 +15,7 @@
 
 set -e
 
-VERSION="1.25.0"
+VERSION="1.25.1-rc1"
 IMAGE="docker/compose:$VERSION"
 
 
@@ -36,18 +36,18 @@ if [ "$(pwd)" != '/' ]; then
 fi
 if [ -n "$COMPOSE_FILE" ]; then
     COMPOSE_OPTIONS="$COMPOSE_OPTIONS -e COMPOSE_FILE=$COMPOSE_FILE"
-    compose_dir=$(realpath $(dirname $COMPOSE_FILE))
+    compose_dir=$(realpath "$(dirname "$COMPOSE_FILE")")
 fi
 # TODO: also check --file argument
 if [ -n "$compose_dir" ]; then
     VOLUMES="$VOLUMES -v $compose_dir:$compose_dir"
 fi
 if [ -n "$HOME" ]; then
-    VOLUMES="$VOLUMES -v $HOME:$HOME -v $HOME:/root" # mount $HOME in /root to share docker.config
+    VOLUMES="$VOLUMES -v $HOME:$HOME -e HOME" # Pass in HOME to share docker.config and allow ~/-relative paths to work.
 fi
 
 # Only allocate tty if we detect one
-if [ -t 0 -a -t 1 ]; then
+if [ -t 0 ] && [ -t 1 ]; then
     DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t"
 fi
 
@@ -56,8 +56,9 @@ DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i"
 
 
 # Handle userns security
-if [ ! -z "$(docker info 2>/dev/null | grep userns)" ]; then
+if docker info --format '{{json .SecurityOptions}}' 2>/dev/null | grep -q 'name=userns'; then
     DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS --userns=host"
 fi
 
+# shellcheck disable=SC2086
 exec docker run --rm $DOCKER_RUN_OPTIONS $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w "$(pwd)" $IMAGE "$@"

script/setup/osx

@@ -36,7 +36,7 @@ if ! [ -x "$(command -v python3)" ]; then
   brew install python3
 fi
 if ! [ -x "$(command -v virtualenv)" ]; then
-  pip install virtualenv==16.2.0
+  pip3 install virtualenv==16.2.0
 fi
 
 #

tests/acceptance/cli_test.py

@@ -48,6 +48,7 @@ BUILD_PULL_TEXT = 'Status: Image is up to date for busybox:1.27.2'
 def start_process(base_dir, options):
     proc = subprocess.Popen(
         ['docker-compose'] + options,
+        stdin=subprocess.PIPE,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         cwd=base_dir)
@@ -55,8 +56,8 @@ def start_process(base_dir, options):
     return proc
 
 
-def wait_on_process(proc, returncode=0):
+def wait_on_process(proc, returncode=0, stdin=None):
-    stdout, stderr = proc.communicate()
+    stdout, stderr = proc.communicate(input=stdin)
     if proc.returncode != returncode:
         print("Stderr: {}".format(stderr))
         print("Stdout: {}".format(stdout))
@@ -64,10 +65,10 @@ def wait_on_process(proc, returncode=0):
     return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8'))
 
 
-def dispatch(base_dir, options, project_options=None, returncode=0):
+def dispatch(base_dir, options, project_options=None, returncode=0, stdin=None):
     project_options = project_options or []
     proc = start_process(base_dir, project_options + options)
-    return wait_on_process(proc, returncode=returncode)
+    return wait_on_process(proc, returncode=returncode, stdin=stdin)
 
 
 def wait_on_condition(condition, delay=0.1, timeout=40):
@@ -156,8 +157,8 @@ class CLITestCase(DockerClientTestCase):
             self._project = get_project(self.base_dir, override_dir=self.override_dir)
         return self._project
 
-    def dispatch(self, options, project_options=None, returncode=0):
+    def dispatch(self, options, project_options=None, returncode=0, stdin=None):
-        return dispatch(self.base_dir, options, project_options, returncode)
+        return dispatch(self.base_dir, options, project_options, returncode, stdin)
 
     def execute(self, container, cmd):
         # Remove once Hijack and CloseNotifier sign a peace treaty
@@ -241,6 +242,17 @@ class CLITestCase(DockerClientTestCase):
         self.base_dir = 'tests/fixtures/v2-full'
         assert self.dispatch(['config', '--quiet']).stdout == ''
 
+    def test_config_stdin(self):
+        config = b"""version: "3.7"
+services:
+  web:
+    image: nginx
+  other:
+    image: alpine
+"""
+        result = self.dispatch(['-f', '-', 'config', '--services'], stdin=config)
+        assert set(result.stdout.rstrip().split('\n')) == {'web', 'other'}
+
     def test_config_with_hash_option(self):
         self.base_dir = 'tests/fixtures/v2-full'
         result = self.dispatch(['config', '--hash=*'])
@@ -661,13 +673,6 @@ class CLITestCase(DockerClientTestCase):
                 'image library/nonexisting-image:latest not found' in result.stderr or
                 'pull access denied for nonexisting-image' in result.stderr)
 
-    def test_pull_with_build(self):
-        result = self.dispatch(['-f', 'pull-with-build.yml', 'pull'])
-
-        assert 'Pulling simple' not in result.stderr
-        assert 'Pulling from_simple' not in result.stderr
-        assert 'Pulling another ...' in result.stderr
-
     def test_pull_with_quiet(self):
         assert self.dispatch(['pull', '--quiet']).stderr == ''
         assert self.dispatch(['pull', '--quiet']).stdout == ''
@@ -689,6 +694,14 @@ class CLITestCase(DockerClientTestCase):
             result.stderr
         )
 
+    def test_pull_can_build(self):
+        result = self.dispatch([
+            '-f', 'can-build-pull-failures.yml', 'pull'],
+            returncode=0
+        )
+        assert 'Some service image(s) must be built from source' in result.stderr
+        assert 'docker-compose build can_build' in result.stderr
+
     def test_pull_with_no_deps(self):
         self.base_dir = 'tests/fixtures/links-composefile'
         result = self.dispatch(['pull', '--no-parallel', 'web'])

tests/fixtures/simple-composefile/can-build-pull-failures.yml

@@ -0,0 +1,6 @@
+version: '3'
+services:
+  can_build:
+    image: nonexisting-image-but-can-build:latest
+    build: .
+    command: top