From 66856e884c0e7337046b0f0fa359ca17a2a96b53 Mon Sep 17 00:00:00 2001 From: Antonio Gutierrez Date: Tue, 2 Apr 2019 15:04:26 +0200 Subject: [PATCH 01/40] requirements: update jsonschema dependency Fixes: https://github.com/docker/compose/issues/6347 Signed-off-by: Antonio Gutierrez --- requirements-build.txt | 2 +- requirements.txt | 4 ++-- script/build/linux-entrypoint | 7 ++++--- setup.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/requirements-build.txt b/requirements-build.txt index 9161fadf9..2a1cd7d6b 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -pyinstaller==3.4 +pyinstaller==3.5 diff --git a/requirements.txt b/requirements.txt index e5b6883e9..726b43b03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,14 +11,14 @@ enum34==1.1.6; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' idna==2.5 ipaddress==1.0.18 -jsonschema==2.6.0 +jsonschema==3.0.1 paramiko==2.4.2 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.6.7 PyYAML==4.2b1 requests==2.22.0 -six==1.10.0 +six==1.12.0 texttable==1.6.2 urllib3==1.24.2; python_version == '3.3' websocket-client==0.32.0 diff --git a/script/build/linux-entrypoint b/script/build/linux-entrypoint index 1c5438d8e..d607dd5c2 100755 --- a/script/build/linux-entrypoint +++ b/script/build/linux-entrypoint @@ -20,10 +20,11 @@ echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA export PATH="${CODE_PATH}/pyinstaller:${PATH}" if [ ! -z "${BUILD_BOOTLOADER}" ]; then - # Build bootloader for alpine - git clone --single-branch --branch master https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller + # Build bootloader for alpine; develop is the main branch + git clone --single-branch --branch develop https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller cd /tmp/pyinstaller/bootloader - git checkout v3.4 + # Checkout commit corresponding to version in requirements-build + git checkout v3.5 "${VENV}"/bin/python3 ./waf configure --no-lsb all "${VENV}"/bin/pip3 install .. cd "${CODE_PATH}" diff --git a/setup.py b/setup.py index a4020df46..4b47c8bf1 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ install_requires = [ 'docker[ssh] >= 3.7.0, < 5', 'dockerpty >= 0.4.1, < 1', 'six >= 1.3.0, < 2', - 'jsonschema >= 2.5.1, < 3', + 'jsonschema >= 2.5.1, < 4', ] From 60dcf87cc08bef7d45f1b376fe54374c3c1f99a1 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 19 Aug 2019 17:30:28 +0200 Subject: [PATCH 02/40] update alpine version to 3.10.1 Signed-off-by: aiordache --- Dockerfile | 2 +- Dockerfile.s390x | 2 +- tests/acceptance/cli_test.py | 4 ++-- tests/fixtures/compatibility-mode/docker-compose.yml | 2 +- tests/fixtures/default-env-file/alt/.env | 2 +- tests/fixtures/networks/docker-compose.yml | 6 +++--- tests/unit/config/config_test.py | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index ed9d74e5e..a45b1dd64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG DOCKER_VERSION=18.09.7 ARG PYTHON_VERSION=3.7.4 ARG BUILD_ALPINE_VERSION=3.10 ARG BUILD_DEBIAN_VERSION=slim-stretch -ARG RUNTIME_ALPINE_VERSION=3.10.0 +ARG RUNTIME_ALPINE_VERSION=3.10.1 ARG RUNTIME_DEBIAN_VERSION=stretch-20190708-slim ARG BUILD_PLATFORM=alpine diff --git a/Dockerfile.s390x b/Dockerfile.s390x index 3b19bb390..9bae72d67 100644 --- a/Dockerfile.s390x +++ b/Dockerfile.s390x @@ -1,4 +1,4 @@ -FROM s390x/alpine:3.6 +FROM s390x/alpine:3.10.1 ARG COMPOSE_VERSION=1.16.1 diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 77b46c279..14dbb7d6c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -360,7 +360,7 @@ class CLITestCase(DockerClientTestCase): 'services': { 'web': { 'command': 'echo uwu', - 'image': 'alpine:3.4', + 'image': 'alpine:3.10.1', 'ports': ['3341/tcp', '4449/tcp'] } }, @@ -559,7 +559,7 @@ class CLITestCase(DockerClientTestCase): 'services': { 'foo': { 'command': '/bin/true', - 'image': 'alpine:3.7', + 'image': 'alpine:3.10.1', 'scale': 3, 'restart': 'always:7', 'mem_limit': '300M', diff --git a/tests/fixtures/compatibility-mode/docker-compose.yml b/tests/fixtures/compatibility-mode/docker-compose.yml index 8187b110c..4b63fadfb 100644 --- a/tests/fixtures/compatibility-mode/docker-compose.yml +++ b/tests/fixtures/compatibility-mode/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.5' services: foo: - image: alpine:3.7 + image: alpine:3.10.1 command: /bin/true deploy: replicas: 3 diff --git a/tests/fixtures/default-env-file/alt/.env b/tests/fixtures/default-env-file/alt/.env index 163668d22..981c7207b 100644 --- a/tests/fixtures/default-env-file/alt/.env +++ b/tests/fixtures/default-env-file/alt/.env @@ -1,4 +1,4 @@ -IMAGE=alpine:3.4 +IMAGE=alpine:3.10.1 COMMAND=echo uwu PORT1=3341 PORT2=4449 diff --git a/tests/fixtures/networks/docker-compose.yml b/tests/fixtures/networks/docker-compose.yml index 275376aef..b911c752b 100644 --- a/tests/fixtures/networks/docker-compose.yml +++ b/tests/fixtures/networks/docker-compose.yml @@ -2,17 +2,17 @@ version: "2" services: web: - image: alpine:3.7 + image: alpine:3.10.1 command: top networks: ["front"] app: - image: alpine:3.7 + image: alpine:3.10.1 command: top networks: ["front", "back"] links: - "db:database" db: - image: alpine:3.7 + image: alpine:3.10.1 command: top networks: ["back"] diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index b583422f5..5ad1a2334 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3620,7 +3620,7 @@ class InterpolationTest(unittest.TestCase): 'version': '3.5', 'services': { 'foo': { - 'image': 'alpine:3.7', + 'image': 'alpine:3.10.1', 'deploy': { 'replicas': 3, 'restart_policy': { @@ -3646,7 +3646,7 @@ class InterpolationTest(unittest.TestCase): service_dict = cfg.services[0] assert service_dict == { - 'image': 'alpine:3.7', + 'image': 'alpine:3.10.1', 'scale': 3, 'restart': {'MaximumRetryCount': 7, 'Name': 'always'}, 'mem_limit': '300M', From 672ced8742f517b7d7e17c50edd50f3748575a05 Mon Sep 17 00:00:00 2001 From: Samuel Searles-Bryant Date: Thu, 22 Aug 2019 11:42:48 +0100 Subject: [PATCH 03/40] Change Formatter.table method to staticmethod Make this a staticmethod so it's easier to use without needing to init a Formatter object first. Signed-off-by: Samuel Searles-Bryant --- compose/cli/formatter.py | 6 ++++-- compose/cli/main.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/cli/formatter.py b/compose/cli/formatter.py index 6c0a3695a..13794b89a 100644 --- a/compose/cli/formatter.py +++ b/compose/cli/formatter.py @@ -18,9 +18,11 @@ def get_tty_width(): return int(width) -class Formatter(object): +class Formatter: """Format tabular data for printing.""" - def table(self, headers, rows): + + @staticmethod + def table(headers, rows): table = texttable.Texttable(max_width=get_tty_width()) table.set_cols_dtype(['t' for h in headers]) table.add_rows([headers] + rows) diff --git a/compose/cli/main.py b/compose/cli/main.py index 477b57b52..86da056ea 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -613,7 +613,7 @@ class TopLevelCommand(object): image_id, size ]) - print(Formatter().table(headers, rows)) + print(Formatter.table(headers, rows)) def kill(self, options): """ @@ -747,7 +747,7 @@ class TopLevelCommand(object): container.human_readable_state, container.human_readable_ports, ]) - print(Formatter().table(headers, rows)) + print(Formatter.table(headers, rows)) def pull(self, options): """ @@ -987,7 +987,7 @@ class TopLevelCommand(object): rows.append(process) print(container.name) - print(Formatter().table(headers, rows)) + print(Formatter.table(headers, rows)) def unpause(self, options): """ From cacbcccc0c68bfcd33f4707bd388b1441523c521 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Fri, 15 Mar 2019 20:22:14 +0900 Subject: [PATCH 04/40] Add support to CLI build This includes can be enabled by setting the env var `COMPOSE_NATIVE_BUILDER=1`. Signed-off-by: Nao YONASHIRO Signed-off-by: Ulysses Souza --- compose/cli/main.py | 14 +++-- compose/project.py | 13 ++++- compose/service.py | 135 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 150 insertions(+), 12 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 477b57b52..a9f9da1c4 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -263,14 +263,14 @@ class TopLevelCommand(object): Usage: build [options] [--build-arg key=val...] [SERVICE...] Options: + --build-arg key=val Set build-time variables for services. --compress Compress the build context using gzip. --force-rm Always remove intermediate containers. + -m, --memory MEM Sets memory limit for the build container. --no-cache Do not use cache when building the image. --no-rm Do not remove intermediate containers after a successful build. - --pull Always attempt to pull a newer version of the image. - -m, --memory MEM Sets memory limit for the build container. - --build-arg key=val Set build-time variables for services. --parallel Build images in parallel. + --pull Always attempt to pull a newer version of the image. -q, --quiet Don't print anything to STDOUT """ service_names = options['SERVICE'] @@ -283,6 +283,8 @@ class TopLevelCommand(object): ) build_args = resolve_build_args(build_args, self.toplevel_environment) + native_builder = self.toplevel_environment.get_boolean('COMPOSE_NATIVE_BUILDER') + self.project.build( service_names=options['SERVICE'], no_cache=bool(options.get('--no-cache', False)), @@ -293,7 +295,8 @@ class TopLevelCommand(object): build_args=build_args, gzip=options.get('--compress', False), parallel_build=options.get('--parallel', False), - silent=options.get('--quiet', False) + silent=options.get('--quiet', False), + cli=native_builder, ) def bundle(self, options): @@ -1071,6 +1074,8 @@ class TopLevelCommand(object): for excluded in [x for x in opts if options.get(x) and no_start]: raise UserError('--no-start and {} cannot be combined.'.format(excluded)) + native_builder = self.toplevel_environment.get_boolean('COMPOSE_NATIVE_BUILDER') + with up_shutdown_context(self.project, service_names, timeout, detached): warn_for_swarm_mode(self.project.client) @@ -1090,6 +1095,7 @@ class TopLevelCommand(object): reset_container_image=rebuild, renew_anonymous_volumes=options.get('--renew-anon-volumes'), silent=options.get('--quiet-pull'), + cli=native_builder, ) try: diff --git a/compose/project.py b/compose/project.py index a608ffd71..b4f55ac95 100644 --- a/compose/project.py +++ b/compose/project.py @@ -355,7 +355,7 @@ class Project(object): return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None, - build_args=None, gzip=False, parallel_build=False, rm=True, silent=False): + build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False): services = [] for service in self.get_services(service_names): @@ -364,8 +364,11 @@ class Project(object): elif not silent: log.info('%s uses an image, skipping' % service.name) + if cli: + log.warning("Native build is an experimental feature and could change at any time") + def build_service(service): - service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent) + service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli) if parallel_build: _, errors = parallel.parallel_execute( services, @@ -509,8 +512,12 @@ class Project(object): reset_container_image=False, renew_anonymous_volumes=False, silent=False, + cli=False, ): + if cli: + log.warning("Native build is an experimental feature and could change at any time") + self.initialize() if not ignore_orphans: self.find_orphan_containers(remove_orphans) @@ -523,7 +530,7 @@ class Project(object): include_deps=start_deps) for svc in services: - svc.ensure_image_exists(do_build=do_build, silent=silent) + svc.ensure_image_exists(do_build=do_build, silent=silent, cli=cli) plans = self._get_convergence_plans( services, strategy, always_recreate_deps=always_recreate_deps) diff --git a/compose/service.py b/compose/service.py index 0db35438d..9457bfd2b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -2,10 +2,13 @@ from __future__ import absolute_import from __future__ import unicode_literals import itertools +import json import logging import os import re +import subprocess import sys +import tempfile from collections import namedtuple from collections import OrderedDict from operator import attrgetter @@ -338,9 +341,9 @@ class Service(object): raise OperationFailedError("Cannot create container for service %s: %s" % (self.name, ex.explanation)) - def ensure_image_exists(self, do_build=BuildAction.none, silent=False): + def ensure_image_exists(self, do_build=BuildAction.none, silent=False, cli=False): if self.can_be_built() and do_build == BuildAction.force: - self.build() + self.build(cli=cli) return try: @@ -356,7 +359,7 @@ class Service(object): if do_build == BuildAction.skip: raise NeedsBuildError(self) - self.build() + self.build(cli=cli) log.warning( "Image for service {} was built because it did not already exist. To " "rebuild this image you must use `docker-compose build` or " @@ -1049,7 +1052,7 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None, - gzip=False, rm=True, silent=False): + gzip=False, rm=True, silent=False, cli=False): output_stream = open(os.devnull, 'w') if not silent: output_stream = sys.stdout @@ -1070,7 +1073,8 @@ class Service(object): 'Impossible to perform platform-targeted builds for API version < 1.35' ) - build_output = self.client.build( + build_image = self.client.build if not cli else cli_build + build_output = build_image( path=path, tag=self.image_name, rm=rm, @@ -1701,3 +1705,124 @@ def rewrite_build_path(path): path = WINDOWS_LONGPATH_PREFIX + os.path.normpath(path) return path + + +def cli_build(path, tag=None, quiet=False, fileobj=None, + nocache=False, rm=False, timeout=None, + custom_context=False, encoding=None, pull=False, + forcerm=False, dockerfile=None, container_limits=None, + decode=False, buildargs=None, gzip=False, shmsize=None, + labels=None, cache_from=None, target=None, network_mode=None, + squash=None, extra_hosts=None, platform=None, isolation=None, + use_config_proxy=True): + """ + Args: + path (str): Path to the directory containing the Dockerfile + buildargs (dict): A dictionary of build arguments + cache_from (:py:class:`list`): A list of images used for build + cache resolution + container_limits (dict): A dictionary of limits applied to each + container created by the build process. Valid keys: + - memory (int): set memory limit for build + - memswap (int): Total memory (memory + swap), -1 to disable + swap + - cpushares (int): CPU shares (relative weight) + - cpusetcpus (str): CPUs in which to allow execution, e.g., + ``"0-3"``, ``"0,1"`` + custom_context (bool): Optional if using ``fileobj`` + decode (bool): If set to ``True``, the returned stream will be + decoded into dicts on the fly. Default ``False`` + dockerfile (str): path within the build context to the Dockerfile + encoding (str): The encoding for a stream. Set to ``gzip`` for + compressing + extra_hosts (dict): Extra hosts to add to /etc/hosts in building + containers, as a mapping of hostname to IP address. + fileobj: A file object to use as the Dockerfile. (Or a file-like + object) + forcerm (bool): Always remove intermediate containers, even after + unsuccessful builds + isolation (str): Isolation technology used during build. + Default: `None`. + labels (dict): A dictionary of labels to set on the image + network_mode (str): networking mode for the run commands during + build + nocache (bool): Don't use the cache when set to ``True`` + platform (str): Platform in the format ``os[/arch[/variant]]`` + pull (bool): Downloads any updates to the FROM image in Dockerfiles + quiet (bool): Whether to return the status + rm (bool): Remove intermediate containers. The ``docker build`` + command now defaults to ``--rm=true``, but we have kept the old + default of `False` to preserve backward compatibility + shmsize (int): Size of `/dev/shm` in bytes. The size must be + greater than 0. If omitted the system uses 64MB + squash (bool): Squash the resulting images layers into a + single layer. + tag (str): A tag to add to the final image + target (str): Name of the build-stage to build in a multi-stage + Dockerfile + timeout (int): HTTP timeout + use_config_proxy (bool): If ``True``, and if the docker client + configuration file (``~/.docker/config.json`` by default) + contains a proxy configuration, the corresponding environment + variables will be set in the container being built. + Returns: + A generator for the build output. + """ + if dockerfile: + dockerfile = os.path.join(path, dockerfile) + iidfile = tempfile.mktemp() + + command_builder = _CommandBuilder() + command_builder.add_params("--build-arg", buildargs) + command_builder.add_arg("--file", dockerfile) + command_builder.add_flag("--force-rm", forcerm) + command_builder.add_arg("--memory", container_limits.get("memory")) + command_builder.add_flag("--no-cache", nocache) + command_builder.add_flag("--pull", pull) + command_builder.add_arg("--iidfile", iidfile) + args = command_builder.build([path]) + + magic_word = "Successfully built " + appear = False + with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p: + while True: + line = p.stdout.readline() + if not line: + break + if line.startswith(magic_word): + appear = True + yield json.dumps({"stream": line}) + + with open(iidfile) as f: + line = f.readline() + image_id = line.split(":")[1].strip() + os.remove(iidfile) + + if not appear: + yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)}) + + +class _CommandBuilder(object): + def __init__(self): + self._args = ["docker", "build"] + + def add_arg(self, name, value): + if value: + self._args.extend([name, str(value)]) + + def add_flag(self, name, flag): + if flag: + self._args.extend([name]) + + def add_params(self, name, params): + if params: + for key, val in params.items(): + self._args.extend([name, "{}={}".format(key, val)]) + + def add_list(self, name, values): + if values: + for val in values: + self._args.extend([name, val]) + + def build(self, args): + return self._args + args From 862a13b8f3f9eceadef80d47a671247d7ba63669 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Sun, 25 Aug 2019 21:04:44 +0900 Subject: [PATCH 05/40] fix: add build flags Signed-off-by: Nao YONASHIRO --- compose/service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compose/service.py b/compose/service.py index 9457bfd2b..4688d6c1d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1774,11 +1774,14 @@ def cli_build(path, tag=None, quiet=False, fileobj=None, command_builder = _CommandBuilder() command_builder.add_params("--build-arg", buildargs) + command_builder.add_list("--cache-from", cache_from) command_builder.add_arg("--file", dockerfile) command_builder.add_flag("--force-rm", forcerm) command_builder.add_arg("--memory", container_limits.get("memory")) command_builder.add_flag("--no-cache", nocache) command_builder.add_flag("--pull", pull) + command_builder.add_arg("--tag", tag) + command_builder.add_arg("--target", target) command_builder.add_arg("--iidfile", iidfile) args = command_builder.build([path]) From 81e223d499f5cfd4935f322b2006d4c41067bca0 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Wed, 28 Aug 2019 05:34:47 +0900 Subject: [PATCH 06/40] feat: add --progress flag Signed-off-by: Nao YONASHIRO --- compose/cli/main.py | 4 + compose/project.py | 5 +- compose/service.py | 181 +++++++++++++++++++++++--------------------- 3 files changed, 100 insertions(+), 90 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index a9f9da1c4..0850f73b0 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -270,6 +270,9 @@ class TopLevelCommand(object): --no-cache Do not use cache when building the image. --no-rm Do not remove intermediate containers after a successful build. --parallel Build images in parallel. + --progress string Set type of progress output (auto, plain, tty). + EXPERIMENTAL flag for native builder. + To enable, run with COMPOSE_NATIVE_BUILDER=1) --pull Always attempt to pull a newer version of the image. -q, --quiet Don't print anything to STDOUT """ @@ -297,6 +300,7 @@ class TopLevelCommand(object): parallel_build=options.get('--parallel', False), silent=options.get('--quiet', False), cli=native_builder, + progress=options.get('--progress'), ) def bundle(self, options): diff --git a/compose/project.py b/compose/project.py index b4f55ac95..d41bcc8af 100644 --- a/compose/project.py +++ b/compose/project.py @@ -355,7 +355,8 @@ class Project(object): return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None, - build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False): + build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False, + progress=None): services = [] for service in self.get_services(service_names): @@ -368,7 +369,7 @@ class Project(object): log.warning("Native build is an experimental feature and could change at any time") def build_service(service): - service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli) + service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress) if parallel_build: _, errors = parallel.parallel_execute( services, diff --git a/compose/service.py b/compose/service.py index 4688d6c1d..73d38287c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1052,7 +1052,7 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None, - gzip=False, rm=True, silent=False, cli=False): + gzip=False, rm=True, silent=False, cli=False, progress=None): output_stream = open(os.devnull, 'w') if not silent: output_stream = sys.stdout @@ -1073,8 +1073,8 @@ class Service(object): 'Impossible to perform platform-targeted builds for API version < 1.35' ) - build_image = self.client.build if not cli else cli_build - build_output = build_image( + builder = self.client if not cli else _CLIBuilder(progress) + build_output = builder.build( path=path, tag=self.image_name, rm=rm, @@ -1707,7 +1707,11 @@ def rewrite_build_path(path): return path -def cli_build(path, tag=None, quiet=False, fileobj=None, +class _CLIBuilder(object): + def __init__(self, progress): + self._progress = progress + + def build(self, path, tag=None, quiet=False, fileobj=None, nocache=False, rm=False, timeout=None, custom_context=False, encoding=None, pull=False, forcerm=False, dockerfile=None, container_limits=None, @@ -1715,94 +1719,95 @@ def cli_build(path, tag=None, quiet=False, fileobj=None, labels=None, cache_from=None, target=None, network_mode=None, squash=None, extra_hosts=None, platform=None, isolation=None, use_config_proxy=True): - """ - Args: - path (str): Path to the directory containing the Dockerfile - buildargs (dict): A dictionary of build arguments - cache_from (:py:class:`list`): A list of images used for build - cache resolution - container_limits (dict): A dictionary of limits applied to each - container created by the build process. Valid keys: - - memory (int): set memory limit for build - - memswap (int): Total memory (memory + swap), -1 to disable - swap - - cpushares (int): CPU shares (relative weight) - - cpusetcpus (str): CPUs in which to allow execution, e.g., - ``"0-3"``, ``"0,1"`` - custom_context (bool): Optional if using ``fileobj`` - decode (bool): If set to ``True``, the returned stream will be - decoded into dicts on the fly. Default ``False`` - dockerfile (str): path within the build context to the Dockerfile - encoding (str): The encoding for a stream. Set to ``gzip`` for - compressing - extra_hosts (dict): Extra hosts to add to /etc/hosts in building - containers, as a mapping of hostname to IP address. - fileobj: A file object to use as the Dockerfile. (Or a file-like - object) - forcerm (bool): Always remove intermediate containers, even after - unsuccessful builds - isolation (str): Isolation technology used during build. - Default: `None`. - labels (dict): A dictionary of labels to set on the image - network_mode (str): networking mode for the run commands during - build - nocache (bool): Don't use the cache when set to ``True`` - platform (str): Platform in the format ``os[/arch[/variant]]`` - pull (bool): Downloads any updates to the FROM image in Dockerfiles - quiet (bool): Whether to return the status - rm (bool): Remove intermediate containers. The ``docker build`` - command now defaults to ``--rm=true``, but we have kept the old - default of `False` to preserve backward compatibility - shmsize (int): Size of `/dev/shm` in bytes. The size must be - greater than 0. If omitted the system uses 64MB - squash (bool): Squash the resulting images layers into a - single layer. - tag (str): A tag to add to the final image - target (str): Name of the build-stage to build in a multi-stage - Dockerfile - timeout (int): HTTP timeout - use_config_proxy (bool): If ``True``, and if the docker client - configuration file (``~/.docker/config.json`` by default) - contains a proxy configuration, the corresponding environment - variables will be set in the container being built. - Returns: - A generator for the build output. - """ - if dockerfile: - dockerfile = os.path.join(path, dockerfile) - iidfile = tempfile.mktemp() + """ + Args: + path (str): Path to the directory containing the Dockerfile + buildargs (dict): A dictionary of build arguments + cache_from (:py:class:`list`): A list of images used for build + cache resolution + container_limits (dict): A dictionary of limits applied to each + container created by the build process. Valid keys: + - memory (int): set memory limit for build + - memswap (int): Total memory (memory + swap), -1 to disable + swap + - cpushares (int): CPU shares (relative weight) + - cpusetcpus (str): CPUs in which to allow execution, e.g., + ``"0-3"``, ``"0,1"`` + custom_context (bool): Optional if using ``fileobj`` + decode (bool): If set to ``True``, the returned stream will be + decoded into dicts on the fly. Default ``False`` + dockerfile (str): path within the build context to the Dockerfile + encoding (str): The encoding for a stream. Set to ``gzip`` for + compressing + extra_hosts (dict): Extra hosts to add to /etc/hosts in building + containers, as a mapping of hostname to IP address. + fileobj: A file object to use as the Dockerfile. (Or a file-like + object) + forcerm (bool): Always remove intermediate containers, even after + unsuccessful builds + isolation (str): Isolation technology used during build. + Default: `None`. + labels (dict): A dictionary of labels to set on the image + network_mode (str): networking mode for the run commands during + build + nocache (bool): Don't use the cache when set to ``True`` + platform (str): Platform in the format ``os[/arch[/variant]]`` + pull (bool): Downloads any updates to the FROM image in Dockerfiles + quiet (bool): Whether to return the status + rm (bool): Remove intermediate containers. The ``docker build`` + command now defaults to ``--rm=true``, but we have kept the old + default of `False` to preserve backward compatibility + shmsize (int): Size of `/dev/shm` in bytes. The size must be + greater than 0. If omitted the system uses 64MB + squash (bool): Squash the resulting images layers into a + single layer. + tag (str): A tag to add to the final image + target (str): Name of the build-stage to build in a multi-stage + Dockerfile + timeout (int): HTTP timeout + use_config_proxy (bool): If ``True``, and if the docker client + configuration file (``~/.docker/config.json`` by default) + contains a proxy configuration, the corresponding environment + variables will be set in the container being built. + Returns: + A generator for the build output. + """ + if dockerfile: + dockerfile = os.path.join(path, dockerfile) + iidfile = tempfile.mktemp() - command_builder = _CommandBuilder() - command_builder.add_params("--build-arg", buildargs) - command_builder.add_list("--cache-from", cache_from) - command_builder.add_arg("--file", dockerfile) - command_builder.add_flag("--force-rm", forcerm) - command_builder.add_arg("--memory", container_limits.get("memory")) - command_builder.add_flag("--no-cache", nocache) - command_builder.add_flag("--pull", pull) - command_builder.add_arg("--tag", tag) - command_builder.add_arg("--target", target) - command_builder.add_arg("--iidfile", iidfile) - args = command_builder.build([path]) + command_builder = _CommandBuilder() + command_builder.add_params("--build-arg", buildargs) + command_builder.add_list("--cache-from", cache_from) + command_builder.add_arg("--file", dockerfile) + command_builder.add_flag("--force-rm", forcerm) + command_builder.add_arg("--memory", container_limits.get("memory")) + command_builder.add_flag("--no-cache", nocache) + command_builder.add_flag("--progress", self._progress) + command_builder.add_flag("--pull", pull) + command_builder.add_arg("--tag", tag) + command_builder.add_arg("--target", target) + command_builder.add_arg("--iidfile", iidfile) + args = command_builder.build([path]) - magic_word = "Successfully built " - appear = False - with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p: - while True: - line = p.stdout.readline() - if not line: - break - if line.startswith(magic_word): - appear = True - yield json.dumps({"stream": line}) + magic_word = "Successfully built " + appear = False + with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p: + while True: + line = p.stdout.readline() + if not line: + break + if line.startswith(magic_word): + appear = True + yield json.dumps({"stream": line}) - with open(iidfile) as f: - line = f.readline() - image_id = line.split(":")[1].strip() - os.remove(iidfile) + with open(iidfile) as f: + line = f.readline() + image_id = line.split(":")[1].strip() + os.remove(iidfile) - if not appear: - yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)}) + if not appear: + yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)}) class _CommandBuilder(object): From 15e8edca3c6853a7c48cffce9eee00486a8a84c0 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Wed, 28 Aug 2019 05:53:46 +0900 Subject: [PATCH 07/40] feat: add a warning if someone uses the --compress or --parallel flag Signed-off-by: Nao YONASHIRO --- compose/project.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compose/project.py b/compose/project.py index d41bcc8af..bbaa1094e 100644 --- a/compose/project.py +++ b/compose/project.py @@ -367,6 +367,10 @@ class Project(object): if cli: log.warning("Native build is an experimental feature and could change at any time") + if parallel_build: + log.warning("unavailable --parallel on COMPOSE_NATIVE_BUILDER=1") + if gzip: + log.warning("unavailable --compress on COMPOSE_NATIVE_BUILDER=1") def build_service(service): service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress) From ee8ca5d6f8d929f52d4ddecef897ff31009f8aea Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 28 Aug 2019 14:08:32 +0200 Subject: [PATCH 08/40] Rephrase warnings when building with the cli Signed-off-by: Ulysses Souza --- compose/project.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compose/project.py b/compose/project.py index bbaa1094e..c84fa77b1 100644 --- a/compose/project.py +++ b/compose/project.py @@ -368,9 +368,11 @@ class Project(object): if cli: log.warning("Native build is an experimental feature and could change at any time") if parallel_build: - log.warning("unavailable --parallel on COMPOSE_NATIVE_BUILDER=1") + log.warning("Flag '--parallel' is ignored when building with " + "COMPOSE_NATIVE_BUILDER=1") if gzip: - log.warning("unavailable --compress on COMPOSE_NATIVE_BUILDER=1") + log.warning("Flag '--compress' is ignored when building with " + "COMPOSE_NATIVE_BUILDER=1") def build_service(service): service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress) From bbdb3cab881692c0785b1bd61217db9822db76f1 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 28 Aug 2019 15:14:24 +0200 Subject: [PATCH 09/40] Add integration tests to native builder Signed-off-by: Ulysses Souza --- tests/integration/service_test.py | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 9750f581c..738391c05 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -38,6 +38,8 @@ from compose.container import Container from compose.errors import OperationFailedError from compose.parallel import ParallelStreamWriter from compose.project import OneOffFilter +from compose.project import Project +from compose.service import BuildAction from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy from compose.service import NetworkMode @@ -966,6 +968,43 @@ class ServiceTest(DockerClientTestCase): assert self.client.inspect_image('composetest_web') + def test_build_cli(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + + service = self.create_service('web', + build={'context': base_dir}, + environment={ + 'COMPOSE_NATIVE_BUILDER': '1', + 'DOCKER_BUILDKIT': '1', + }) + service.build(cli=True) + self.addCleanup(self.client.remove_image, service.image_name) + assert self.client.inspect_image('composetest_web') + + def test_up_build_cli(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + + web = self.create_service('web', + build={'context': base_dir}, + environment={ + 'COMPOSE_NATIVE_BUILDER': '1', + 'DOCKER_BUILDKIT': '1', + }) + project = Project('composetest', [web], self.client) + project.up(do_build=BuildAction.force) + + containers = project.containers(['web']) + assert len(containers) == 1 + assert containers[0].name.startswith('composetest_web_') + def test_build_non_ascii_filename(self): base_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, base_dir) From 719a1b05818cc0dcb4515dc3a704e468061b2455 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Thu, 29 Aug 2019 19:56:31 +0900 Subject: [PATCH 10/40] fix: use subprocess32 for python2 Signed-off-by: Nao YONASHIRO --- compose/service.py | 6 +++++- setup.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 73d38287c..987ba64f1 100644 --- a/compose/service.py +++ b/compose/service.py @@ -6,7 +6,6 @@ import json import logging import os import re -import subprocess import sys import tempfile from collections import namedtuple @@ -62,6 +61,11 @@ from .utils import parse_seconds_float from .utils import truncate_id from .utils import unique_everseen +if six.PY2: + import subprocess32 as subprocess +else: + import subprocess + log = logging.getLogger(__name__) diff --git a/setup.py b/setup.py index 4b47c8bf1..c3e214c22 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ if sys.version_info[:2] < (3, 4): tests_require.append('mock >= 1.0.1, < 4') extras_require = { + ':python_version < "3.2"': ['subprocess32 >= 3.5.4, < 4'], ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5, < 4'], ':python_version < "3.3"': ['ipaddress >= 1.0.16, < 2'], From 9d7ad3bac17a174a85f6d9f4e20d04b2e65b8c66 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 29 Aug 2019 16:29:09 +0200 Subject: [PATCH 11/40] Add comment on native build and fix typo Signed-off-by: Ulysses Souza --- compose/cli/main.py | 2 +- compose/service.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 0850f73b0..244ac64f0 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -266,7 +266,7 @@ class TopLevelCommand(object): --build-arg key=val Set build-time variables for services. --compress Compress the build context using gzip. --force-rm Always remove intermediate containers. - -m, --memory MEM Sets memory limit for the build container. + -m, --memory MEM Set memory limit for the build container. --no-cache Do not use cache when building the image. --no-rm Do not remove intermediate containers after a successful build. --parallel Build images in parallel. diff --git a/compose/service.py b/compose/service.py index 987ba64f1..55d2e9cd1 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1810,6 +1810,10 @@ class _CLIBuilder(object): image_id = line.split(":")[1].strip() os.remove(iidfile) + # In case of `DOCKER_BUILDKIT=1` + # there is no success message already present in the output. + # Since that's the way `Service::build` gets the `image_id` + # it has to be added `manually` if not appear: yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)}) From 0c6fce271e949955693ed86461d8a0e6adcd18d4 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 29 Aug 2019 17:45:21 +0200 Subject: [PATCH 12/40] Bump runtime debian From `stretch-20190708-slim` to `stretch-20190812-slim` Signed-off-by: Ulysses Souza --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a45b1dd64..a5398413c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG PYTHON_VERSION=3.7.4 ARG BUILD_ALPINE_VERSION=3.10 ARG BUILD_DEBIAN_VERSION=slim-stretch ARG RUNTIME_ALPINE_VERSION=3.10.1 -ARG RUNTIME_DEBIAN_VERSION=stretch-20190708-slim +ARG RUNTIME_DEBIAN_VERSION=stretch-20190812-slim ARG BUILD_PLATFORM=alpine From 5add9192ac52a5c72ecc1495aa68cbfeb5a8e863 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 30 Aug 2019 12:11:09 +0200 Subject: [PATCH 13/40] Rename envvar switch to COMPOSE_DOCKER_CLI_BUILD From `COMPOSE_NATIVE_BUILDER` to `COMPOSE_DOCKER_CLI_BUILD` Signed-off-by: Ulysses Souza --- compose/cli/main.py | 6 +++--- compose/project.py | 4 ++-- tests/integration/service_test.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 244ac64f0..b94f41eec 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -272,7 +272,7 @@ class TopLevelCommand(object): --parallel Build images in parallel. --progress string Set type of progress output (auto, plain, tty). EXPERIMENTAL flag for native builder. - To enable, run with COMPOSE_NATIVE_BUILDER=1) + To enable, run with COMPOSE_DOCKER_CLI_BUILD=1) --pull Always attempt to pull a newer version of the image. -q, --quiet Don't print anything to STDOUT """ @@ -286,7 +286,7 @@ class TopLevelCommand(object): ) build_args = resolve_build_args(build_args, self.toplevel_environment) - native_builder = self.toplevel_environment.get_boolean('COMPOSE_NATIVE_BUILDER') + native_builder = self.toplevel_environment.get_boolean('COMPOSE_DOCKER_CLI_BUILD') self.project.build( service_names=options['SERVICE'], @@ -1078,7 +1078,7 @@ class TopLevelCommand(object): for excluded in [x for x in opts if options.get(x) and no_start]: raise UserError('--no-start and {} cannot be combined.'.format(excluded)) - native_builder = self.toplevel_environment.get_boolean('COMPOSE_NATIVE_BUILDER') + native_builder = self.toplevel_environment.get_boolean('COMPOSE_DOCKER_CLI_BUILD') with up_shutdown_context(self.project, service_names, timeout, detached): warn_for_swarm_mode(self.project.client) diff --git a/compose/project.py b/compose/project.py index c84fa77b1..69e273c4b 100644 --- a/compose/project.py +++ b/compose/project.py @@ -369,10 +369,10 @@ class Project(object): log.warning("Native build is an experimental feature and could change at any time") if parallel_build: log.warning("Flag '--parallel' is ignored when building with " - "COMPOSE_NATIVE_BUILDER=1") + "COMPOSE_DOCKER_CLI_BUILD=1") if gzip: log.warning("Flag '--compress' is ignored when building with " - "COMPOSE_NATIVE_BUILDER=1") + "COMPOSE_DOCKER_CLI_BUILD=1") def build_service(service): service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 738391c05..c50aab08b 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -978,7 +978,7 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('web', build={'context': base_dir}, environment={ - 'COMPOSE_NATIVE_BUILDER': '1', + 'COMPOSE_DOCKER_CLI_BUILD': '1', 'DOCKER_BUILDKIT': '1', }) service.build(cli=True) @@ -995,7 +995,7 @@ class ServiceTest(DockerClientTestCase): web = self.create_service('web', build={'context': base_dir}, environment={ - 'COMPOSE_NATIVE_BUILDER': '1', + 'COMPOSE_DOCKER_CLI_BUILD': '1', 'DOCKER_BUILDKIT': '1', }) project = Project('composetest', [web], self.client) From 47d170b06a0f04ca77c4083a151d48a714508a52 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 4 Sep 2019 17:52:31 +0200 Subject: [PATCH 14/40] Fix race condition on watch_events Avoid to attach to restarting containers and ignore race conditions when trying to attach to already dead containers Signed-off-by: Ulysses Souza --- compose/cli/log_printer.py | 8 +++++++- tests/unit/cli/log_printer_test.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 8aa93a844..6940a74c8 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -230,7 +230,13 @@ def watch_events(thread_map, event_stream, presenters, thread_args): # Container crashed so we should reattach to it if event['id'] in crashed_containers: - event['container'].attach_log_stream() + container = event['container'] + if not container.is_restarting: + try: + container.attach_log_stream() + except APIError: + # Just ignore errors when reattaching to already crashed containers + pass crashed_containers.remove(event['id']) thread_map[event['id']] = build_thread( diff --git a/tests/unit/cli/log_printer_test.py b/tests/unit/cli/log_printer_test.py index 6db24e464..5e387241d 100644 --- a/tests/unit/cli/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -152,6 +152,17 @@ class TestWatchEvents(object): *thread_args) assert container_id in thread_map + def test_container_attach_event(self, thread_map, mock_presenters): + container_id = 'abcd' + mock_container = mock.Mock(is_restarting=False) + mock_container.attach_log_stream.side_effect = APIError("race condition") + event_die = {'action': 'die', 'id': container_id} + event_start = {'action': 'start', 'id': container_id, 'container': mock_container} + event_stream = [event_die, event_start] + thread_args = 'foo', 'bar' + watch_events(thread_map, event_stream, mock_presenters, thread_args) + assert mock_container.attach_log_stream.called + def test_other_event(self, thread_map, mock_presenters): container_id = 'abcd' event_stream = [{'action': 'create', 'id': container_id}] From a5fbf91b72f1f32fe61b2d7b3e0cd5d53fbf2aac Mon Sep 17 00:00:00 2001 From: Danil Kister Date: Wed, 21 Aug 2019 18:51:44 +0200 Subject: [PATCH 15/40] Prevent KeyError when remote network labels are None. Signed-off-by: Danil Kister --- compose/network.py | 2 +- tests/unit/network_test.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/compose/network.py b/compose/network.py index e0d711ff7..84531ecc7 100644 --- a/compose/network.py +++ b/compose/network.py @@ -226,7 +226,7 @@ def check_remote_network_config(remote, local): raise NetworkConfigChangedError(local.true_name, 'enable_ipv6') local_labels = local.labels or {} - remote_labels = remote.get('Labels', {}) + remote_labels = remote.get('Labels') or {} for k in set.union(set(remote_labels.keys()), set(local_labels.keys())): if k.startswith('com.docker.'): # We are only interested in user-specified labels continue diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index 82cfb3be2..b829de196 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -168,3 +168,8 @@ class NetworkTest(unittest.TestCase): mock_log.warning.assert_called_once_with(mock.ANY) _, args, kwargs = mock_log.warning.mock_calls[0] assert 'label "com.project.touhou.character" has changed' in args[0] + + def test_remote_config_labels_none(self): + remote = {'Labels': None} + local = Network(None, 'test_project', 'test_network') + check_remote_network_config(remote, local) From b9092cacdb51bb4077aee5a18e0bb750833ca4ab Mon Sep 17 00:00:00 2001 From: Marian Gappa Date: Fri, 10 Aug 2018 00:06:47 +0200 Subject: [PATCH 16/40] Fix missing secret error message Add a warning message when the secret file doesn't exist Fixes #5920 Signed-off-by: Marian Gappa --- compose/project.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 69e273c4b..b2305e67f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -6,6 +6,7 @@ import logging import operator import re from functools import reduce +from os import path import enum import six @@ -807,7 +808,15 @@ def get_secrets(service, service_secrets, secret_defs): ) ) - secrets.append({'secret': secret, 'file': secret_def.get('file')}) + secret_file = secret_def.get('file') + if not path.isfile(str(secret_file)): + log.warn( + "Service \"{service}\" uses an undefined secret file \"{secret_file}\", " + "the following folder is created \"{secret_file}\"".format( + service=service, secret_file=secret_file + ) + ) + secrets.append({'secret': secret, 'file': secret_file}) return secrets From 70ead597d271bf87ec0b564f3901ae5809abdfbc Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Mon, 9 Sep 2019 10:03:43 +0200 Subject: [PATCH 17/40] Add tests to 'get_secret' warnings Signed-off-by: Ulysses Souza --- compose/project.py | 2 +- tests/unit/project_test.py | 84 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index b2305e67f..478e50b55 100644 --- a/compose/project.py +++ b/compose/project.py @@ -810,7 +810,7 @@ def get_secrets(service, service_secrets, secret_defs): secret_file = secret_def.get('file') if not path.isfile(str(secret_file)): - log.warn( + log.warning( "Service \"{service}\" uses an undefined secret file \"{secret_file}\", " "the following folder is created \"{secret_file}\"".format( service=service, secret_file=secret_file diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 93a9aa292..e1453e7f6 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -3,6 +3,8 @@ from __future__ import absolute_import from __future__ import unicode_literals import datetime +import os +import tempfile import docker import pytest @@ -11,6 +13,7 @@ from docker.errors import NotFound from .. import mock from .. import unittest from ..helpers import BUSYBOX_IMAGE_WITH_TAG +from compose.config import ConfigurationError from compose.config.config import Config from compose.config.types import VolumeFromSpec from compose.const import COMPOSEFILE_V1 as V1 @@ -21,6 +24,7 @@ from compose.const import DEFAULT_TIMEOUT from compose.const import LABEL_SERVICE from compose.container import Container from compose.errors import OperationFailedError +from compose.project import get_secrets from compose.project import NoSuchService from compose.project import Project from compose.project import ProjectError @@ -841,3 +845,83 @@ class ProjectTest(unittest.TestCase): with mock.patch('compose.service.Service.push') as fake_push: project.push() assert fake_push.call_count == 2 + + def test_get_secrets_no_secret_def(self): + service = 'foo' + secret_source = 'bar' + + secret_defs = mock.Mock() + secret_defs.get.return_value = None + secret = mock.Mock(source=secret_source) + + with self.assertRaises(ConfigurationError): + get_secrets(service, [secret], secret_defs) + + def test_get_secrets_external_warning(self): + service = 'foo' + secret_source = 'bar' + + secret_def = mock.Mock() + secret_def.get.return_value = True + + secret_defs = mock.Mock() + secret_defs.get.side_effect = secret_def + secret = mock.Mock(source=secret_source) + + with mock.patch('compose.project.log') as mock_log: + get_secrets(service, [secret], secret_defs) + + mock_log.warning.assert_called_with("Service \"{service}\" uses secret \"{secret}\" " + "which is external. External secrets are not available" + " to containers created by docker-compose." + .format(service=service, secret=secret_source)) + + def test_get_secrets_uid_gid_mode_warning(self): + service = 'foo' + secret_source = 'bar' + + _, filename_path = tempfile.mkstemp() + self.addCleanup(os.remove, filename_path) + + def mock_get(key): + return {'external': False, 'file': filename_path}[key] + + secret_def = mock.MagicMock() + secret_def.get = mock.MagicMock(side_effect=mock_get) + + secret_defs = mock.Mock() + secret_defs.get.return_value = secret_def + + secret = mock.Mock(uid=True, gid=True, mode=True, source=secret_source) + + with mock.patch('compose.project.log') as mock_log: + get_secrets(service, [secret], secret_defs) + + mock_log.warning.assert_called_with("Service \"{service}\" uses secret \"{secret}\" with uid, " + "gid, or mode. These fields are not supported by this " + "implementation of the Compose file" + .format(service=service, secret=secret_source)) + + def test_get_secrets_secret_file_warning(self): + service = 'foo' + secret_source = 'bar' + not_a_path = 'NOT_A_PATH' + + def mock_get(key): + return {'external': False, 'file': not_a_path}[key] + + secret_def = mock.MagicMock() + secret_def.get = mock.MagicMock(side_effect=mock_get) + + secret_defs = mock.Mock() + secret_defs.get.return_value = secret_def + + secret = mock.Mock(uid=False, gid=False, mode=False, source=secret_source) + + with mock.patch('compose.project.log') as mock_log: + get_secrets(service, [secret], secret_defs) + + mock_log.warning.assert_called_with("Service \"{service}\" uses an undefined secret file " + "\"{secret_file}\", the following folder is created " + "\"{secret_file}\"" + .format(service=service, secret_file=not_a_path)) From 98d7cc8d0c8b884720579ad9a9dc944517fc042d Mon Sep 17 00:00:00 2001 From: Zuhayr Elahi Date: Wed, 28 Aug 2019 11:51:22 -0700 Subject: [PATCH 18/40] ADDED a stage for executing License Scans Signed-off-by: Zuhayr Elahi --- script/Jenkinsfile.fossa | 20 ++++++++++++++++++++ script/fossa.mk | 16 ++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 script/Jenkinsfile.fossa create mode 100644 script/fossa.mk diff --git a/script/Jenkinsfile.fossa b/script/Jenkinsfile.fossa new file mode 100644 index 000000000..480e98efa --- /dev/null +++ b/script/Jenkinsfile.fossa @@ -0,0 +1,20 @@ +pipeline { + agent any + stages { + stage("License Scan") { + agent { + label 'ubuntu-1604-aufs-edge' + } + + steps { + withCredentials([ + string(credentialsId: 'fossa-api-key', variable: 'FOSSA_API_KEY') + ]) { + checkout scm + sh "FOSSA_API_KEY='${FOSSA_API_KEY}' BRANCH_NAME='${env.BRANCH_NAME}' make -f script/fossa.mk fossa-analyze" + sh "FOSSA_API_KEY='${FOSSA_API_KEY}' make -f script/fossa.mk fossa-test" + } + } + } + } +} diff --git a/script/fossa.mk b/script/fossa.mk new file mode 100644 index 000000000..8d7af49d8 --- /dev/null +++ b/script/fossa.mk @@ -0,0 +1,16 @@ +# Variables for Fossa +BUILD_ANALYZER?=docker/fossa-analyzer +FOSSA_OPTS?=--option all-tags:true --option allow-unresolved:true + +fossa-analyze: + docker run --rm -e FOSSA_API_KEY=$(FOSSA_API_KEY) \ + -v $(CURDIR)/$*:/go/src/github.com/docker/compose \ + -w /go/src/github.com/docker/compose \ + $(BUILD_ANALYZER) analyze ${FOSSA_OPTS} --branch ${BRANCH_NAME} + + # This command is used to run the fossa test command +fossa-test: + docker run -i -e FOSSA_API_KEY=$(FOSSA_API_KEY) \ + -v $(CURDIR)/$*:/go/src/github.com/docker/compose \ + -w /go/src/github.com/docker/compose \ + $(BUILD_ANALYZER) test From 475f8199f773936e2504c0977b970c2f9f432140 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Tue, 24 Sep 2019 13:31:30 +0300 Subject: [PATCH 19/40] Fixing features broken link Signed-off-by: Dimitar Dimitrov --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dd4003048..c9b87daba 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Compose is a tool for defining and running multi-container Docker applications. With Compose, you use a Compose file to configure your application's services. Then, using a single command, you create and start all the services from your configuration. To learn more about all the features of Compose -see [the list of features](https://github.com/docker/docker.github.io/blob/master/compose/overview.md#features). +see [the list of features](https://github.com/docker/docker.github.io/blob/master/compose/index.md#features). Compose is great for development, testing, and staging environments, as well as CI workflows. You can learn more about each case in From 32ac6edb862ba6d4f3bb8cb4869d8b2f0b27ca88 Mon Sep 17 00:00:00 2001 From: Lukas Hettwer Date: Tue, 24 Sep 2019 16:02:12 +0200 Subject: [PATCH 20/40] Fix --progress arg when run docker-compose build --progress is no longer processed as flag but as argument with value. Signed-off-by: Lukas Hettwer Resolve: [#6913] --- compose/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 55d2e9cd1..50d2c4b52 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1787,7 +1787,7 @@ class _CLIBuilder(object): command_builder.add_flag("--force-rm", forcerm) command_builder.add_arg("--memory", container_limits.get("memory")) command_builder.add_flag("--no-cache", nocache) - command_builder.add_flag("--progress", self._progress) + command_builder.add_arg("--progress", self._progress) command_builder.add_flag("--pull", pull) command_builder.add_arg("--tag", tag) command_builder.add_arg("--target", target) From eca358e2f0583d6e85aedc58b45aaebfd38e55f0 Mon Sep 17 00:00:00 2001 From: ulyssessouza Date: Fri, 27 Sep 2019 09:10:49 +0200 Subject: [PATCH 21/40] Fix secret missing warning Signed-off-by: ulyssessouza --- compose/project.py | 2 +- tests/unit/project_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/project.py b/compose/project.py index 478e50b55..092c1e123 100644 --- a/compose/project.py +++ b/compose/project.py @@ -812,7 +812,7 @@ def get_secrets(service, service_secrets, secret_defs): if not path.isfile(str(secret_file)): log.warning( "Service \"{service}\" uses an undefined secret file \"{secret_file}\", " - "the following folder is created \"{secret_file}\"".format( + "the following file should be created \"{secret_file}\"".format( service=service, secret_file=secret_file ) ) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index e1453e7f6..de16febf5 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -922,6 +922,6 @@ class ProjectTest(unittest.TestCase): get_secrets(service, [secret], secret_defs) mock_log.warning.assert_called_with("Service \"{service}\" uses an undefined secret file " - "\"{secret_file}\", the following folder is created " + "\"{secret_file}\", the following file should be created " "\"{secret_file}\"" .format(service=service, secret_file=not_a_path)) From 70f8e38b1dbc8d34e1af4274b7e3bc756f521f99 Mon Sep 17 00:00:00 2001 From: Guillaume LOURS Date: Tue, 8 Oct 2019 11:05:30 +0200 Subject: [PATCH 22/40] Add automatic labeling of bug, feature & question issues Signed-off-by: Guillaume Lours --- .github/ISSUE_TEMPLATE/bug_report.md | 3 +++ .github/ISSUE_TEMPLATE/feature_request.md | 3 +++ .github/ISSUE_TEMPLATE/question-about-using-compose.md | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 49d4691fb..2f3012f61 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,6 +1,9 @@ --- name: Bug report about: Report a bug encountered while using docker-compose +title: '' +labels: kind/bug +assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index d53c49a79..603d34c38 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,6 +1,9 @@ --- name: Feature request about: Suggest an idea to improve Compose +title: '' +labels: kind/feature +assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/question-about-using-compose.md b/.github/ISSUE_TEMPLATE/question-about-using-compose.md index 11ef65ccf..ccb4e9b33 100644 --- a/.github/ISSUE_TEMPLATE/question-about-using-compose.md +++ b/.github/ISSUE_TEMPLATE/question-about-using-compose.md @@ -1,6 +1,9 @@ --- name: Question about using Compose about: This is not the appropriate channel +title: '' +labels: kind/question +assignees: '' --- From 74f892de955759cbcead836780126850be488a36 Mon Sep 17 00:00:00 2001 From: Aleksandr Mezin Date: Fri, 21 Dec 2018 15:07:39 +0600 Subject: [PATCH 23/40] Add test to verify same file 'extends' optimization Signed-off-by: Aleksandr Mezin --- tests/unit/config/config_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 5ad1a2334..0d3f49b99 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -18,6 +18,7 @@ from ...helpers import build_config_details from ...helpers import BUSYBOX_IMAGE_WITH_TAG from compose.config import config from compose.config import types +from compose.config.config import ConfigFile from compose.config.config import resolve_build_args from compose.config.config import resolve_environment from compose.config.environment import Environment @@ -4887,6 +4888,11 @@ class ExtendsTest(unittest.TestCase): assert types.SecurityOpt.parse('apparmor:unconfined') in svc['security_opt'] assert types.SecurityOpt.parse('seccomp:unconfined') in svc['security_opt'] + @mock.patch.object(ConfigFile, 'from_filename', wraps=ConfigFile.from_filename) + def test_extends_same_file_optimization(self, from_filename_mock): + load_from_filename('tests/fixtures/extends/no-file-specified.yml') + from_filename_mock.assert_called_once() + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From c24b7b6464b302477728b0882d2fb6bf647f2e79 Mon Sep 17 00:00:00 2001 From: Aleksandr Mezin Date: Fri, 21 Dec 2018 15:03:44 +0600 Subject: [PATCH 24/40] Fix same file 'extends' optimization Signed-off-by: Aleksandr Mezin --- compose/config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 5202d0025..f64dc04a0 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -615,7 +615,7 @@ class ServiceExtendsResolver(object): config_path = self.get_extended_config_path(extends) service_name = extends['service'] - if config_path == self.config_file.filename: + if config_path == os.path.abspath(self.config_file.filename): try: service_config = self.config_file.get_service(service_name) except KeyError: From 79f29dda2370b7eec9942405d45541f1f569d0f9 Mon Sep 17 00:00:00 2001 From: Stefan Scherer Date: Sat, 21 Sep 2019 17:40:58 +0200 Subject: [PATCH 25/40] Add dependencies for ARM build Signed-off-by: Stefan Scherer --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a5398413c..64de77890 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,15 +30,18 @@ RUN apk add --no-cache \ ENV BUILD_BOOTLOADER=1 FROM python:${PYTHON_VERSION}-${BUILD_DEBIAN_VERSION} AS build-debian -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install --no-install-recommends -y \ curl \ gcc \ git \ libc-dev \ + libffi-dev \ libgcc-6-dev \ + libssl-dev \ make \ openssl \ - python2.7-dev + python2.7-dev \ + zlib1g-dev FROM build-${BUILD_PLATFORM} AS build COPY docker-compose-entrypoint.sh /usr/local/bin/ From ce52f597a0f0a509912b91befe59991906391028 Mon Sep 17 00:00:00 2001 From: Stefan Scherer Date: Mon, 23 Sep 2019 10:36:47 +0200 Subject: [PATCH 26/40] Enhance build script for different CPU architectures Signed-off-by: Stefan Scherer --- script/build/linux | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/build/linux b/script/build/linux index 28065da08..ca5620b85 100755 --- a/script/build/linux +++ b/script/build/linux @@ -12,6 +12,7 @@ docker build -t "${TAG}" . \ --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}" TMP_CONTAINER=$(docker create "${TAG}") mkdir -p dist -docker cp "${TMP_CONTAINER}":/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64 +ARCH=$(uname -m) +docker cp "${TMP_CONTAINER}":/usr/local/bin/docker-compose "dist/docker-compose-Linux-${ARCH}" docker container rm -f "${TMP_CONTAINER}" docker image rm -f "${TAG}" From 37be2ad9cd54e61de07a43c14ca768b2d07da851 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Sirot Date: Wed, 9 Oct 2019 10:51:17 +0200 Subject: [PATCH 27/40] Remove set -x to make this script less verbose Signed-off-by: Jean-Christophe Sirot --- script/circle/bintray-deploy.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/script/circle/bintray-deploy.sh b/script/circle/bintray-deploy.sh index 8c8871aa6..d508da365 100755 --- a/script/circle/bintray-deploy.sh +++ b/script/circle/bintray-deploy.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -x - curl -f -u$BINTRAY_USERNAME:$BINTRAY_API_KEY -X GET \ https://api.bintray.com/repos/docker-compose/${CIRCLE_BRANCH} From 9375c15bad6931be9cfadf198b2932f0079dc03c Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Wed, 9 Oct 2019 16:16:57 +0200 Subject: [PATCH 28/40] Add config file for @probot/stale Signed-off-by: Guillaume Lours --- .github/stale.yml | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/stale.yml diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 000000000..10660d759 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,58 @@ +# Configuration for probot-stale - https://github.com/probot/stale + +# Number of days of inactivity before an Issue or Pull Request becomes stale +daysUntilStale: 180 + +# Number of days of inactivity before an Issue or Pull Request with the stale label is closed. +# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. +daysUntilClose: 7 + +# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled) +onlyLabels: [] + +# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable +exemptLabels: [] + +# Set to true to ignore issues in a project (defaults to false) +exemptProjects: false + +# Set to true to ignore issues in a milestone (defaults to false) +exemptMilestones: false + +# Set to true to ignore issues with an assignee (defaults to false) +exemptAssignees: true + +# Label to use when marking as stale +staleLabel: stale + +# Comment to post when marking as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you + for your contributions. + +# Comment to post when removing the stale label. +unmarkComment: > + This issue has been automatically marked as not stale anymore due to the recent activity. + +# Comment to post when closing a stale Issue or Pull Request. +closeComment: > + This issue has been automatically closed because it had not recent activity during the stale period. + +# Limit the number of actions per hour, from 1-30. Default is 30 +limitPerRun: 30 + +# Limit to only `issues` or `pulls` + only: issues + +# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls': +# pulls: +# daysUntilStale: 30 +# markComment: > +# This pull request has been automatically marked as stale because it has not had +# recent activity. It will be closed if no further activity occurs. Thank you +# for your contributions. + +# issues: +# exemptLabels: +# - confirmed From 79bf9ed652470e34bbca7fee390cf7306a710cf7 Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Wed, 9 Oct 2019 21:10:18 +0200 Subject: [PATCH 29/40] correct invalid yaml indentation Signed-off-by: Guillaume Lours --- .github/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/stale.yml b/.github/stale.yml index 10660d759..0e42a89cb 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -43,7 +43,7 @@ closeComment: > limitPerRun: 30 # Limit to only `issues` or `pulls` - only: issues +only: issues # Optionally, specify configuration settings that are specific to just 'issues' or 'pulls': # pulls: From cdae06a89ced31765391a7ab2aa70b8bb75214f9 Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Wed, 9 Oct 2019 21:51:34 +0200 Subject: [PATCH 30/40] exclude issue flagged with kind/feature from stale process Signed-off-by: Guillaume Lours --- .github/stale.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/stale.yml b/.github/stale.yml index 0e42a89cb..6de76aef9 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -11,7 +11,8 @@ daysUntilClose: 7 onlyLabels: [] # Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable -exemptLabels: [] +exemptLabels: + - kind/feature # Set to true to ignore issues in a project (defaults to false) exemptProjects: false From 3135a0a8392e4a70d98bde16e58fd28d17904e1d Mon Sep 17 00:00:00 2001 From: Zuhayr Elahi Date: Tue, 8 Oct 2019 16:27:56 -0700 Subject: [PATCH 31/40] Added log message to check compose file Signed-off-by: Zuhayr Elahi --- compose/service.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compose/service.py b/compose/service.py index 50d2c4b52..f036826f6 100644 --- a/compose/service.py +++ b/compose/service.py @@ -623,6 +623,10 @@ class Service(object): try: container.start() except APIError as ex: + if "driver failed programming external connectivity" in ex.explanation: + log.warn( + "Port is already in use, check the docker-compose file to see if the same" + + " port was allocated to multiple services") raise OperationFailedError("Cannot start service %s: %s" % (self.name, ex.explanation)) return container From 8835056ce46dd680ecb5d6015e183b17c3815772 Mon Sep 17 00:00:00 2001 From: Zuhayr Elahi Date: Wed, 9 Oct 2019 12:02:18 -0700 Subject: [PATCH 32/40] UPDATED log message Signed-off-by: Zuhayr Elahi --- compose/service.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index f036826f6..638cd71e5 100644 --- a/compose/service.py +++ b/compose/service.py @@ -624,9 +624,7 @@ class Service(object): container.start() except APIError as ex: if "driver failed programming external connectivity" in ex.explanation: - log.warn( - "Port is already in use, check the docker-compose file to see if the same" + - " port was allocated to multiple services") + log.warn("Host is already in use by another container") raise OperationFailedError("Cannot start service %s: %s" % (self.name, ex.explanation)) return container From 8973a940e6e28c3b23648dc60528cb690766fdc3 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 10 Oct 2019 08:12:11 +0200 Subject: [PATCH 33/40] Bump paramiko to 2.6.0 close #6953 Signed-off-by: Nicolas De Loof --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 726b43b03..ed1c87693 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ functools32==3.2.3.post2; python_version < '3.2' idna==2.5 ipaddress==1.0.18 jsonschema==3.0.1 -paramiko==2.4.2 +paramiko==2.6.0 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.6.7 From 1678a4fbe45d9e0d2a7eb0b055787f6bf25bd2a1 Mon Sep 17 00:00:00 2001 From: Guillaume Rose Date: Mon, 14 Oct 2019 21:17:15 +0200 Subject: [PATCH 34/40] Run CI on amd64 Signed-off-by: Guillaume Rose --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4de276ada..1d7c348e3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ def buildImage = { String baseImage -> def image - wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) { + wrappedNode(label: "ubuntu && amd64 && !zfs", cleanWorkspace: true) { stage("build image for \"${baseImage}\"") { checkout(scm) def imageName = "dockerbuildbot/compose:${baseImage}-${gitCommit()}" @@ -29,7 +29,7 @@ def buildImage = { String baseImage -> def get_versions = { String imageId, int number -> def docker_versions - wrappedNode(label: "ubuntu && !zfs") { + wrappedNode(label: "ubuntu && amd64 && !zfs") { def result = sh(script: """docker run --rm \\ --entrypoint=/code/.tox/py27/bin/python \\ ${imageId} \\ @@ -55,7 +55,7 @@ def runTests = { Map settings -> } { -> - wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) { + wrappedNode(label: "ubuntu && amd64 && !zfs", cleanWorkspace: true) { stage("test python=${pythonVersions} / docker=${dockerVersions} / baseImage=${baseImage}") { checkout(scm) def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim() From dbe4d7323eb8d38c98cec2810469bcd65266edb0 Mon Sep 17 00:00:00 2001 From: Guillaume Rose Date: Mon, 14 Oct 2019 14:55:41 +0200 Subject: [PATCH 35/40] Add working dir, config files and env file in service labels Signed-off-by: Guillaume Rose --- compose/cli/command.py | 30 ++++++++++++++++++--- compose/const.py | 3 +++ compose/project.py | 3 ++- compose/service.py | 59 ++++++++++++++++++++++-------------------- 4 files changed, 63 insertions(+), 32 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 2f38fe5af..c3a10a043 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -13,6 +13,9 @@ from .. import config from .. import parallel from ..config.environment import Environment from ..const import API_VERSIONS +from ..const import LABEL_CONFIG_FILES +from ..const import LABEL_ENVIRONMENT_FILE +from ..const import LABEL_WORKING_DIR from ..project import Project from .docker_client import docker_client from .docker_client import get_tls_version @@ -57,7 +60,8 @@ def project_from_options(project_dir, options, additional_options={}): environment=environment, override_dir=override_dir, compatibility=options.get('--compatibility'), - interpolate=(not additional_options.get('--no-interpolate')) + interpolate=(not additional_options.get('--no-interpolate')), + environment_file=environment_file ) @@ -125,7 +129,7 @@ def get_client(environment, verbose=False, version=None, tls_config=None, host=N def get_project(project_dir, config_path=None, project_name=None, verbose=False, host=None, tls_config=None, environment=None, override_dir=None, - compatibility=False, interpolate=True): + compatibility=False, interpolate=True, environment_file=None): if not environment: environment = Environment.from_env_file(project_dir) config_details = config.find(project_dir, config_path, environment, override_dir) @@ -145,10 +149,30 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, with errors.handle_connection_errors(client): return Project.from_config( - project_name, config_data, client, environment.get('DOCKER_DEFAULT_PLATFORM') + project_name, + config_data, + client, + environment.get('DOCKER_DEFAULT_PLATFORM'), + execution_context_labels(config_details, environment_file), ) +def execution_context_labels(config_details, environment_file): + extra_labels = [ + '{0}={1}'.format(LABEL_WORKING_DIR, os.path.abspath(config_details.working_dir)), + '{0}={1}'.format(LABEL_CONFIG_FILES, config_files_label(config_details)), + ] + if environment_file is not None: + extra_labels.append('{0}={1}'.format(LABEL_ENVIRONMENT_FILE, + os.path.normpath(environment_file))) + return extra_labels + + +def config_files_label(config_details): + return ",".join( + map(str, (os.path.normpath(c.filename) for c in config_details.config_files))) + + def get_project_name(working_dir, project_name=None, environment=None): def normalize_name(name): return re.sub(r'[^-_a-z0-9]', '', name.lower()) diff --git a/compose/const.py b/compose/const.py index 46d81ae71..ab0389ce0 100644 --- a/compose/const.py +++ b/compose/const.py @@ -11,6 +11,9 @@ IS_WINDOWS_PLATFORM = (sys.platform == "win32") LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' LABEL_ONE_OFF = 'com.docker.compose.oneoff' LABEL_PROJECT = 'com.docker.compose.project' +LABEL_WORKING_DIR = 'com.docker.compose.project.working_dir' +LABEL_CONFIG_FILES = 'com.docker.compose.project.config_files' +LABEL_ENVIRONMENT_FILE = 'com.docker.compose.project.environment_file' LABEL_SERVICE = 'com.docker.compose.service' LABEL_NETWORK = 'com.docker.compose.network' LABEL_VERSION = 'com.docker.compose.version' diff --git a/compose/project.py b/compose/project.py index 092c1e123..094ce4d7a 100644 --- a/compose/project.py +++ b/compose/project.py @@ -83,7 +83,7 @@ class Project(object): return labels @classmethod - def from_config(cls, name, config_data, client, default_platform=None): + def from_config(cls, name, config_data, client, default_platform=None, extra_labels=[]): """ Construct a Project from a config.Config object. """ @@ -136,6 +136,7 @@ class Project(object): pid_mode=pid_mode, platform=service_dict.pop('platform', None), default_platform=default_platform, + extra_labels=extra_labels, **service_dict) ) diff --git a/compose/service.py b/compose/service.py index 638cd71e5..ae4e7665c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -68,7 +68,6 @@ else: log = logging.getLogger(__name__) - HOST_CONFIG_KEYS = [ 'cap_add', 'cap_drop', @@ -137,7 +136,6 @@ class NoSuchImageError(Exception): ServiceName = namedtuple('ServiceName', 'project service number') - ConvergencePlan = namedtuple('ConvergencePlan', 'action containers') @@ -173,20 +171,21 @@ class BuildAction(enum.Enum): class Service(object): def __init__( - self, - name, - client=None, - project='default', - use_networking=False, - links=None, - volumes_from=None, - network_mode=None, - networks=None, - secrets=None, - scale=1, - pid_mode=None, - default_platform=None, - **options + self, + name, + client=None, + project='default', + use_networking=False, + links=None, + volumes_from=None, + network_mode=None, + networks=None, + secrets=None, + scale=1, + pid_mode=None, + default_platform=None, + extra_labels=[], + **options ): self.name = name self.client = client @@ -201,6 +200,7 @@ class Service(object): self.scale_num = scale self.default_platform = default_platform self.options = options + self.extra_labels = extra_labels def __repr__(self): return ''.format(self.name) @@ -215,7 +215,7 @@ class Service(object): for container in self.client.containers( all=stopped, filters=filters)]) - ) + ) if result: return result @@ -404,8 +404,8 @@ class Service(object): return ConvergencePlan('start', containers) if ( - strategy is ConvergenceStrategy.always or - self._containers_have_diverged(containers) + strategy is ConvergenceStrategy.always or + self._containers_have_diverged(containers) ): return ConvergencePlan('recreate', containers) @@ -482,6 +482,7 @@ class Service(object): container, timeout=timeout, attach_logs=not detached, start_new_container=start, renew_anonymous_volumes=renew_anonymous_volumes ) + containers, errors = parallel_execute( containers, recreate, @@ -705,11 +706,11 @@ class Service(object): net_name = self.network_mode.service_name pid_namespace = self.pid_mode.service_name return ( - self.get_linked_service_names() + - self.get_volumes_from_names() + - ([net_name] if net_name else []) + - ([pid_namespace] if pid_namespace else []) + - list(self.options.get('depends_on', {}).keys()) + self.get_linked_service_names() + + self.get_volumes_from_names() + + ([net_name] if net_name else []) + + ([pid_namespace] if pid_namespace else []) + + list(self.options.get('depends_on', {}).keys()) ) def get_dependency_configs(self): @@ -899,7 +900,7 @@ class Service(object): container_options['labels'] = build_container_labels( container_options.get('labels', {}), - self.labels(one_off=one_off), + self.labels(one_off=one_off) + self.extra_labels, number, self.config_hash if add_config_hash else None, slug @@ -1552,9 +1553,9 @@ def warn_on_masked_volume(volumes_option, container_volumes, service): for volume in volumes_option: if ( - volume.external and - volume.internal in container_volumes and - container_volumes.get(volume.internal) != volume.external + volume.external and + volume.internal in container_volumes and + container_volumes.get(volume.internal) != volume.external ): log.warning(( "Service \"{service}\" is using volume \"{volume}\" from the " @@ -1601,6 +1602,7 @@ def build_mount(mount_spec): read_only=mount_spec.read_only, consistency=mount_spec.consistency, **kwargs ) + # Labels @@ -1655,6 +1657,7 @@ def format_environment(environment): if isinstance(value, six.binary_type): value = value.decode('utf-8') return '{key}={value}'.format(key=key, value=value) + return [format_env(*item) for item in environment.items()] From 452880af7cd3c59f3bbe9a7714a0176f52f2d12d Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 11 Oct 2019 17:27:08 +0200 Subject: [PATCH 36/40] Use python Posix support to get tty size stty is not portable outside *nix Note: shutil.get_terminal_size require python 3.3 Signed-off-by: Nicolas De Loof --- compose/cli/formatter.py | 15 ++++++++++----- requirements.txt | 1 + setup.py | 3 ++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/compose/cli/formatter.py b/compose/cli/formatter.py index 13794b89a..c1f43ed7a 100644 --- a/compose/cli/formatter.py +++ b/compose/cli/formatter.py @@ -2,20 +2,25 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging -import os +import shutil import six import texttable from compose.cli import colors +if hasattr(shutil, "get_terminal_size"): + from shutil import get_terminal_size +else: + from backports.shutil_get_terminal_size import get_terminal_size + def get_tty_width(): - tty_size = os.popen('stty size 2> /dev/null', 'r').read().split() - if len(tty_size) != 2: + try: + width, _ = get_terminal_size() + return int(width) + except OSError: return 0 - _, width = tty_size - return int(width) class Formatter: diff --git a/requirements.txt b/requirements.txt index ed1c87693..f2e66b34d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +backports.shutil_get_terminal_size==1.0.0 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 diff --git a/setup.py b/setup.py index c3e214c22..23ae08a12 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,8 @@ extras_require = { ':python_version < "3.2"': ['subprocess32 >= 3.5.4, < 4'], ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5, < 4'], - ':python_version < "3.3"': ['ipaddress >= 1.0.16, < 2'], + ':python_version < "3.3"': ['backports.shutil_get_terminal_size == 1.0.0', + 'ipaddress >= 1.0.16, < 2'], ':sys_platform == "win32"': ['colorama >= 0.4, < 1'], 'socks': ['PySocks >= 1.5.6, != 1.5.7, < 2'], } From 1ca10f90fb280016a14158ff702524d6bb4ae2fd Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Tue, 15 Oct 2019 19:45:54 +0200 Subject: [PATCH 37/40] Fix acceptance tests tty is now (correclty) reported to have 80 columns, which split service ID in two lines Signed-off-by: Nicolas De Loof --- tests/acceptance/cli_test.py | 6 +++--- tests/fixtures/images-service-tag/docker-compose.yml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 14dbb7d6c..a03d56567 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2816,8 +2816,8 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['images']) assert 'busybox' in result.stdout - assert 'multiple-composefiles_another_1' in result.stdout - assert 'multiple-composefiles_simple_1' in result.stdout + assert '_another_1' in result.stdout + assert '_simple_1' in result.stdout @mock.patch.dict(os.environ) def test_images_tagless_image(self): @@ -2865,4 +2865,4 @@ class CLITestCase(DockerClientTestCase): assert re.search(r'foo1.+test[ \t]+dev', result.stdout) is not None assert re.search(r'foo2.+test[ \t]+prod', result.stdout) is not None - assert re.search(r'foo3.+_foo3[ \t]+latest', result.stdout) is not None + assert re.search(r'foo3.+test[ \t]+latest', result.stdout) is not None diff --git a/tests/fixtures/images-service-tag/docker-compose.yml b/tests/fixtures/images-service-tag/docker-compose.yml index aff3cf285..a46b32bf5 100644 --- a/tests/fixtures/images-service-tag/docker-compose.yml +++ b/tests/fixtures/images-service-tag/docker-compose.yml @@ -8,3 +8,4 @@ services: image: test:prod foo3: build: . + image: test:latest From 17bbbba7d67f3756ea8bae580f962490d59abab8 Mon Sep 17 00:00:00 2001 From: okor Date: Thu, 17 Oct 2019 12:50:01 -0600 Subject: [PATCH 38/40] update docker-py Signed-off-by: Jason Ormand --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f2e66b34d..1627cca9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.4.0; sys_platform == 'win32' -docker==4.0.1 +docker==4.1.0 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 From 386bdda2468056b755af186bc80386e565a9785d Mon Sep 17 00:00:00 2001 From: Guillaume Lours Date: Fri, 18 Oct 2019 11:49:30 +0200 Subject: [PATCH 39/40] Format image size as decimal to be align with Docker CLI Signed-off-by: Guillaume Lours --- compose/cli/utils.py | 4 ++-- tests/unit/cli/utils_test.py | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index bd06beef8..931487a6c 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -133,12 +133,12 @@ def generate_user_agent(): def human_readable_file_size(size): suffixes = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', ] - order = int(math.log(size, 2) / 10) if size else 0 + order = int(math.log(size, 1000)) if size else 0 if order >= len(suffixes): order = len(suffixes) - 1 return '{0:.4g} {1}'.format( - size / float(1 << (order * 10)), + size / pow(10, order * 3), suffixes[order] ) diff --git a/tests/unit/cli/utils_test.py b/tests/unit/cli/utils_test.py index b340fb947..7a7628903 100644 --- a/tests/unit/cli/utils_test.py +++ b/tests/unit/cli/utils_test.py @@ -29,16 +29,20 @@ class HumanReadableFileSizeTest(unittest.TestCase): assert human_readable_file_size(100) == '100 B' def test_1kb(self): - assert human_readable_file_size(1024) == '1 kB' + assert human_readable_file_size(1000) == '1 kB' + assert human_readable_file_size(1024) == '1.024 kB' def test_1023b(self): - assert human_readable_file_size(1023) == '1023 B' + assert human_readable_file_size(1023) == '1.023 kB' + + def test_999b(self): + assert human_readable_file_size(999) == '999 B' def test_units(self): - assert human_readable_file_size((2 ** 10) ** 0) == '1 B' - assert human_readable_file_size((2 ** 10) ** 1) == '1 kB' - assert human_readable_file_size((2 ** 10) ** 2) == '1 MB' - assert human_readable_file_size((2 ** 10) ** 3) == '1 GB' - assert human_readable_file_size((2 ** 10) ** 4) == '1 TB' - assert human_readable_file_size((2 ** 10) ** 5) == '1 PB' - assert human_readable_file_size((2 ** 10) ** 6) == '1 EB' + assert human_readable_file_size((10 ** 3) ** 0) == '1 B' + assert human_readable_file_size((10 ** 3) ** 1) == '1 kB' + assert human_readable_file_size((10 ** 3) ** 2) == '1 MB' + assert human_readable_file_size((10 ** 3) ** 3) == '1 GB' + assert human_readable_file_size((10 ** 3) ** 4) == '1 TB' + assert human_readable_file_size((10 ** 3) ** 5) == '1 PB' + assert human_readable_file_size((10 ** 3) ** 6) == '1 EB' From c7e82489f4bf8ebef4afc22848d121b39aa2e769 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Mon, 28 Oct 2019 11:33:59 +0100 Subject: [PATCH 40/40] "Bump 1.25.0-rc3" Signed-off-by: Djordje Lukic --- CHANGELOG.md | 83 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a3ad8bfd..210aecd4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,89 @@ Change log ========== +1.25.0-rc3 (2019-10-28) +------------------- + +### Features + +- Add BuildKit support, use `DOCKER_BUILDKIT=1` and `COMPOSE_NATIVE_BUILDER=1` + +- Bump paramiko to 2.6.0 + +- Add working dir, config files and env file in service labels + +- Add tag `docker-compose:latest` + +- Add `docker-compose:-alpine` image/tag + +- Add `docker-compose:-debian` image/tag + +- Bumped `docker-py` 4.1.0 + +- Supports `requests` up to 2.22.0 version + +- Drops empty tag on `build:cache_from` + +- `Dockerfile` now generates `libmusl` binaries for alpine + +- Only pull images that can't be built + +- Attribute `scale` can now accept `0` as a value + +- Added `--quiet` build flag + +- Added `--no-interpolate` to `docker-compose config` + +- Bump OpenSSL for macOS build (`1.1.0j` to `1.1.1c`) + +- Added `--no-rm` to `build` command + +- Added support for `credential_spec` + +- Resolve digests without pulling image + +- Upgrade `pyyaml` to `4.2b1` + +- Lowered severity to `warning` if `down` tries to remove nonexisting image + +- Use improved API fields for project events when possible + +- Update `setup.py` for modern `pypi/setuptools` and remove `pandoc` dependencies + +- Removed `Dockerfile.armhf` which is no longer needed + +### Bugfixes + +- Fix same file 'extends' optimization + +- Use python POSIX support to get tty size + +- Format image size as decimal to be align with Docker CLI + +- Fixed stdin_open + +- Fixed `--remove-orphans` when used with `up --no-start` + +- Fixed `docker-compose ps --all` + +- Fixed `depends_on` dependency recreation behavior + +- Fixed bash completion for `build --memory` + +- Fixed misleading warning concerning env vars when performing an `exec` command + +- Fixed failure check in parallel_execute_watch + +- Fixed race condition after pulling image + +- Fixed error on duplicate mount points. + +- Fixed merge on networks section + +- Always connect Compose container to `stdin` + +- Fixed the presentation of failed services on 'docker-compose start' when containers are not available + 1.24.0 (2019-03-28) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 0042896b6..552f91b2d 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.25.0dev' +__version__ = '1.25.0-rc3' diff --git a/script/run/run.sh b/script/run/run.sh index f3456720f..f7d6eb35f 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.24.0" +VERSION="1.25.0-rc3" IMAGE="docker/compose:$VERSION"