From 86428af5bcb96975e4379074c54c1e6c088603d5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Feb 2018 14:44:21 -0800 Subject: [PATCH 01/13] Bump 1.20.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 77 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 79 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4287de493..e662d40a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,83 @@ Change log ========== +1.20.0 (2018-03-07) +------------------- + +### New features + +#### Compose file version 3.6 + +- Introduced version 3.6 of the `docker-compose.yml` specification. + This version requires to be used with Docker Engine 18.02.0 or above. + +- Added support for the `tmpfs.size` property in volume mappings + +#### Compose file version 3.2 and up + +- The `--build-arg` option can now be used without specifying a service + in `docker-compose build` + +#### Compose file version 2.3 + +- Added support for `device_cgroup_rules` in service definitions + +- Added support for the `tmpfs.size` property in long-form volume mappings + +- The `--build-arg` option can now be used without specifying a service + in `docker-compose build` + +#### All formats + +- Added a `--log-level` option to the top-level `docker-compose` command. + Accepted values are `debug`, `info`, `warning`, `error`, `critical`. + Default log level is `info` + +- `docker-compose run` now allows users to unset the container's entrypoint + +- Proxy configuration found in the `~/.docker/config.json` file now populates + environment and build args for containers created by Compose + +- Added a `--use-aliases` flag to `docker-compose run`, indicating that + network aliases declared in the service's config should be used for the + running container + +- `docker-compose run` now kills and removes the running container upon + receiving `SIGHUP` + +- `docker-compose ps` now shows the containers' health status if available + +- Added the long-form `--detach` option to the `exec`, `run` and `up` + commands + +### Bugfixes + +- Fixed `.dockerignore` handling, notably with regard to absolute paths + and last-line precedence rules + +- Fixed a bug introduced in 1.19.0 which caused the default certificate path + to not be honored by Compose + +- Fixed a bug where Compose would incorrectly check whether a symlink's + destination was accessible when part of a build context + +- Fixed a bug where `.dockerignore` files containing lines of whitespace + caused Compose to error out on Windows + +- Fixed a bug where `--tls*` and `--host` options wouldn't be properly honored + for interactive `run` and `exec` commands + +- A `seccomp:` entry in the `security_opt` config now correctly + sends the contents of the file to the engine + +- Improved support for non-unicode locales + +- Fixed a crash occurring on Windows when the user's home directory name + contained non-ASCII characters + +- Fixed a bug occurring during builds caused by files with a negative `mtime` + values in the build context + 1.19.0 (2018-02-07) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index f0ad67347..2090f10c8 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.20.0dev' +__version__ = '1.20.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index ae55ff759..2adcc98f8 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.19.0" +VERSION="1.20.0-rc1" IMAGE="docker/compose:$VERSION" From d36f222c7e922b9c9acd3275fdca20cd1b694afe Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Thu, 1 Mar 2018 14:09:11 +0100 Subject: [PATCH 02/13] Fix a race condition in ParallelStreamWriter. Signed-off-by: Matthieu Nottale --- compose/parallel.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compose/parallel.py b/compose/parallel.py index 341ca2f5e..dd83c70cd 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import logging import operator import sys +from threading import Lock from threading import Semaphore from threading import Thread @@ -251,6 +252,7 @@ class ParallelStreamWriter(object): """ noansi = False + lock = Lock() @classmethod def set_noansi(cls, value=True): @@ -274,6 +276,7 @@ class ParallelStreamWriter(object): self.stream.flush() def _write_ansi(self, obj_index, status): + self.lock.acquire() position = self.lines.index(obj_index) diff = len(self.lines) - position # move up @@ -285,6 +288,7 @@ class ParallelStreamWriter(object): # move back down self.stream.write("%c[%dB" % (27, diff)) self.stream.flush() + self.lock.release() def _write_noansi(self, obj_index, status): self.stream.write("{} {:<{width}} ... {}\r\n".format(self.msg, obj_index, From 9e42b5006332b98a5a2ad361dc202ad1660878af Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 1 Mar 2018 15:32:41 +0100 Subject: [PATCH 03/13] Add support for options added in 1.20.0 to bash completion New options: - `docker-compose --log-level` - `docker-compose pull --include-deps` - `docker-compose run --use-aliases` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 30 +++++++++++++++++--------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 29853b089..90c9ce5fc 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -179,18 +179,22 @@ _docker_compose_docker_compose() { _filedir "y?(a)ml" return ;; + --log-level) + COMPREPLY=( $( compgen -W "debug info warning error critical" -- "$cur" ) ) + return + ;; --project-directory) _filedir -d return ;; - $(__docker_compose_to_extglob "$top_level_options_with_args") ) + $(__docker_compose_to_extglob "$daemon_options_with_args") ) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "$top_level_boolean_options $top_level_options_with_args --help -h --no-ansi --verbose --version -v" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "$daemon_boolean_options $daemon_options_with_args $top_level_options_with_args --help -h --no-ansi --verbose --version -v" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -375,7 +379,7 @@ _docker_compose_ps() { _docker_compose_pull() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --parallel --quiet -q" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --ignore-pull-failures --include-deps --parallel --quiet -q" -- "$cur" ) ) ;; *) __docker_compose_services_from_image @@ -444,7 +448,7 @@ _docker_compose_run() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --detach --entrypoint -e --help --label -l --name --no-deps --publish -p --rm --service-ports -T --user -u --volume -v --workdir -w" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--detach -d --entrypoint -e --help --label -l --name --no-deps --publish -p --rm --service-ports -T --use-aliases --user -u --volume -v --workdir -w" -- "$cur" ) ) ;; *) __docker_compose_services_all @@ -605,14 +609,12 @@ _docker_compose() { # Options for the docker daemon that have to be passed to secondary calls to # docker-compose executed by this script. - # Other global otions that are not relevant for secondary calls are defined in - # `_docker_compose_docker_compose`. - local top_level_boolean_options=" + local daemon_boolean_options=" --skip-hostname-check --tls --tlsverify " - local top_level_options_with_args=" + local daemon_options_with_args=" --file -f --host -H --project-directory @@ -622,6 +624,11 @@ _docker_compose() { --tlskey " + # These options are require special treatment when searching the command. + local top_level_options_with_args=" + --log-level + " + COMPREPLY=() local cur prev words cword _get_comp_words_by_ref -n : cur prev words cword @@ -634,15 +641,18 @@ _docker_compose() { while [ $counter -lt $cword ]; do case "${words[$counter]}" in - $(__docker_compose_to_extglob "$top_level_boolean_options") ) + $(__docker_compose_to_extglob "$daemon_boolean_options") ) local opt=${words[counter]} top_level_options+=($opt) ;; - $(__docker_compose_to_extglob "$top_level_options_with_args") ) + $(__docker_compose_to_extglob "$daemon_options_with_args") ) local opt=${words[counter]} local arg=${words[++counter]} top_level_options+=($opt $arg) ;; + $(__docker_compose_to_extglob "$top_level_options_with_args") ) + (( counter++ )) + ;; -*) ;; *) From 4d1dad24ae2fa88a6429b7a6277da7eeff408e13 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Mar 2018 15:38:42 -0800 Subject: [PATCH 04/13] SDK version 3.1.1 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index da05e4212..33462d496 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.1.0 +docker==3.1.1 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index d1788df0a..cf8f6dc13 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.1.0, < 4.0', + 'docker >= 3.1.1, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From d2b5d59dd8a9e83642799557b73d92ecbd4045e0 Mon Sep 17 00:00:00 2001 From: Matthieu Nottale Date: Mon, 5 Mar 2018 14:28:46 +0100 Subject: [PATCH 05/13] Revamp ParallelStreamWriter to fix display issues. Signed-off-by: Matthieu Nottale --- compose/parallel.py | 112 ++++++++++++++++-------------- compose/service.py | 3 +- tests/integration/service_test.py | 3 + tests/unit/parallel_test.py | 3 + tests/unit/service_test.py | 2 + 5 files changed, 69 insertions(+), 54 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index dd83c70cd..5d4791f97 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -43,7 +43,36 @@ class GlobalLimit(object): cls.global_limiter = Semaphore(value) -def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, parent_objects=None): +def parallel_execute_watch(events, writer, errors, results, msg, get_name): + """ Watch events from a parallel execution, update status and fill errors and results. + Returns exception to re-raise. + """ + error_to_reraise = None + for obj, result, exception in events: + if exception is None: + writer.write(msg, get_name(obj), 'done', green) + results.append(result) + elif isinstance(exception, ImageNotFound): + # This is to bubble up ImageNotFound exceptions to the client so we + # can prompt the user if they want to rebuild. + errors[get_name(obj)] = exception.explanation + writer.write(msg, get_name(obj), 'error', red) + error_to_reraise = exception + elif isinstance(exception, APIError): + errors[get_name(obj)] = exception.explanation + writer.write(msg, get_name(obj), 'error', red) + elif isinstance(exception, (OperationFailedError, HealthCheckFailed, NoHealthCheckConfigured)): + errors[get_name(obj)] = exception.msg + writer.write(msg, get_name(obj), 'error', red) + elif isinstance(exception, UpstreamError): + writer.write(msg, get_name(obj), 'error', red) + else: + errors[get_name(obj)] = exception + error_to_reraise = exception + return error_to_reraise + + +def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): """Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. @@ -53,45 +82,21 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, pa objects = list(objects) stream = get_output_stream(sys.stderr) - writer = ParallelStreamWriter(stream, msg) + if ParallelStreamWriter.instance: + writer = ParallelStreamWriter.instance + else: + writer = ParallelStreamWriter(stream) - display_objects = list(parent_objects) if parent_objects else objects - - for obj in display_objects: - writer.add_object(get_name(obj)) - - # write data in a second loop to consider all objects for width alignment - # and avoid duplicates when parent_objects exists for obj in objects: - writer.write_initial(get_name(obj)) + writer.add_object(msg, get_name(obj)) + for obj in objects: + writer.write_initial(msg, get_name(obj)) events = parallel_execute_iter(objects, func, get_deps, limit) errors = {} results = [] - error_to_reraise = None - - for obj, result, exception in events: - if exception is None: - writer.write(get_name(obj), 'done', green) - results.append(result) - elif isinstance(exception, ImageNotFound): - # This is to bubble up ImageNotFound exceptions to the client so we - # can prompt the user if they want to rebuild. - errors[get_name(obj)] = exception.explanation - writer.write(get_name(obj), 'error', red) - error_to_reraise = exception - elif isinstance(exception, APIError): - errors[get_name(obj)] = exception.explanation - writer.write(get_name(obj), 'error', red) - elif isinstance(exception, (OperationFailedError, HealthCheckFailed, NoHealthCheckConfigured)): - errors[get_name(obj)] = exception.msg - writer.write(get_name(obj), 'error', red) - elif isinstance(exception, UpstreamError): - writer.write(get_name(obj), 'error', red) - else: - errors[get_name(obj)] = exception - error_to_reraise = exception + error_to_reraise = parallel_execute_watch(events, writer, errors, results, msg, get_name) for obj_name, error in errors.items(): stream.write("\nERROR: for {} {}\n".format(obj_name, error)) @@ -253,55 +258,58 @@ class ParallelStreamWriter(object): noansi = False lock = Lock() + instance = None @classmethod def set_noansi(cls, value=True): cls.noansi = value - def __init__(self, stream, msg): + def __init__(self, stream): self.stream = stream - self.msg = msg self.lines = [] self.width = 0 + ParallelStreamWriter.instance = self - def add_object(self, obj_index): - self.lines.append(obj_index) - self.width = max(self.width, len(obj_index)) - - def write_initial(self, obj_index): - if self.msg is None: + def add_object(self, msg, obj_index): + if msg is None: return - self.stream.write("{} {:<{width}} ... \r\n".format( - self.msg, self.lines[self.lines.index(obj_index)], width=self.width)) + self.lines.append(msg + obj_index) + self.width = max(self.width, len(msg + ' ' + obj_index)) + + def write_initial(self, msg, obj_index): + if msg is None: + return + self.stream.write("{:<{width}} ... \r\n".format( + msg + ' ' + obj_index, width=self.width)) self.stream.flush() - def _write_ansi(self, obj_index, status): + def _write_ansi(self, msg, obj_index, status): self.lock.acquire() - position = self.lines.index(obj_index) + position = self.lines.index(msg + obj_index) diff = len(self.lines) - position # move up self.stream.write("%c[%dA" % (27, diff)) # erase self.stream.write("%c[2K\r" % 27) - self.stream.write("{} {:<{width}} ... {}\r".format(self.msg, obj_index, + self.stream.write("{:<{width}} ... {}\r".format(msg + ' ' + obj_index, status, width=self.width)) # move back down self.stream.write("%c[%dB" % (27, diff)) self.stream.flush() self.lock.release() - def _write_noansi(self, obj_index, status): - self.stream.write("{} {:<{width}} ... {}\r\n".format(self.msg, obj_index, + def _write_noansi(self, msg, obj_index, status): + self.stream.write("{:<{width}} ... {}\r\n".format(msg + ' ' + obj_index, status, width=self.width)) self.stream.flush() - def write(self, obj_index, status, color_func): - if self.msg is None: + def write(self, msg, obj_index, status, color_func): + if msg is None: return if self.noansi: - self._write_noansi(obj_index, status) + self._write_noansi(msg, obj_index, status) else: - self._write_ansi(obj_index, color_func(status)) + self._write_ansi(msg, obj_index, color_func(status)) def parallel_operation(containers, operation, options, message): diff --git a/compose/service.py b/compose/service.py index b9f9af2cd..652080478 100644 --- a/compose/service.py +++ b/compose/service.py @@ -402,8 +402,7 @@ class Service(object): [ServiceName(self.project, self.name, index) for index in range(i, i + scale)], lambda service_name: create_and_start(self, service_name.number), lambda service_name: self.get_container_name(service_name.service, service_name.number), - "Creating", - parent_objects=project_services + "Creating" ) for error in errors.values(): raise OperationFailedError(error) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2b6b7711e..6e86a02d4 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -35,6 +35,7 @@ from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION from compose.container import Container from compose.errors import OperationFailedError +from compose.parallel import ParallelStreamWriter from compose.project import OneOffFilter from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy @@ -1197,6 +1198,7 @@ class ServiceTest(DockerClientTestCase): service.create_container(number=next_number) service.create_container(number=next_number + 1) + ParallelStreamWriter.instance = None with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: service.scale(2) for container in service.containers(): @@ -1220,6 +1222,7 @@ class ServiceTest(DockerClientTestCase): for container in service.containers(): assert not container.is_running + ParallelStreamWriter.instance = None with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: service.scale(2) diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 4ebc24d8c..0735bfccb 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -143,6 +143,7 @@ class ParallelTest(unittest.TestCase): def test_parallel_execute_alignment(capsys): + ParallelStreamWriter.instance = None results, errors = parallel_execute( objects=["short", "a very long name"], func=lambda x: x, @@ -158,6 +159,7 @@ def test_parallel_execute_alignment(capsys): def test_parallel_execute_ansi(capsys): + ParallelStreamWriter.instance = None ParallelStreamWriter.set_noansi(value=False) results, errors = parallel_execute( objects=["something", "something more"], @@ -173,6 +175,7 @@ def test_parallel_execute_ansi(capsys): def test_parallel_execute_noansi(capsys): + ParallelStreamWriter.instance = None ParallelStreamWriter.set_noansi() results, errors = parallel_execute( objects=["something", "something more"], diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c315dcc4d..9128b9550 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -20,6 +20,7 @@ from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import SECRETS_PATH from compose.container import Container +from compose.parallel import ParallelStreamWriter from compose.project import OneOffFilter from compose.service import build_ulimits from compose.service import build_volume_binding @@ -727,6 +728,7 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.log', autospec=True) def test_only_log_warning_when_host_ports_clash(self, mock_log): self.mock_client.inspect_image.return_value = {'Id': 'abcd'} + ParallelStreamWriter.instance = None name = 'foo' service = Service( name, From ca012640c1c476cc1d4bfd44176cc0f24f6e87aa Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Mar 2018 18:03:54 -0800 Subject: [PATCH 06/13] Preserve security_opt values in extends Signed-off-by: Joffrey F --- compose/config/config.py | 3 ++- compose/config/types.py | 6 ++++++ tests/unit/config/config_test.py | 23 +++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0b5c7df0a..508ceafbf 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1039,6 +1039,7 @@ def merge_service_dicts(base, override, version): md.merge_sequence('links', ServiceLink.parse) md.merge_sequence('secrets', types.ServiceSecret.parse) md.merge_sequence('configs', types.ServiceConfig.parse) + md.merge_sequence('security_opt', types.SecurityOpt.parse) md.merge_mapping('extra_hosts', parse_extra_hosts) for field in ['volumes', 'devices']: @@ -1046,7 +1047,7 @@ def merge_service_dicts(base, override, version): for field in [ 'cap_add', 'cap_drop', 'expose', 'external_links', - 'security_opt', 'volumes_from', 'device_cgroup_rules', + 'volumes_from', 'device_cgroup_rules', ]: md.merge_field(field, merge_unique_items_lists, default=[]) diff --git a/compose/config/types.py b/compose/config/types.py index 47e7222a3..ff9875218 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -464,6 +464,8 @@ def normalize_port_dict(port): class SecurityOpt(namedtuple('_SecurityOpt', 'value src_file')): @classmethod def parse(cls, value): + if not isinstance(value, six.string_types): + return value # based on https://github.com/docker/cli/blob/9de1b162f/cli/command/container/opts.go#L673-L697 con = value.split('=', 2) if len(con) == 1 and con[0] != 'no-new-privileges': @@ -486,3 +488,7 @@ class SecurityOpt(namedtuple('_SecurityOpt', 'value src_file')): if self.src_file is not None: return 'seccomp:{}'.format(self.src_file) return self.value + + @property + def merge_field(self): + return self.value diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e03298221..7a9bb944d 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -4508,6 +4508,29 @@ class ExtendsTest(unittest.TestCase): for svc in services: assert svc['ports'] == [types.ServicePort('80', None, None, None, None)] + def test_extends_with_security_opt(self): + tmpdir = py.test.ensuretemp('test_extends_with_ports') + self.addCleanup(tmpdir.remove) + tmpdir.join('docker-compose.yml').write(""" + version: '2' + + services: + a: + image: nginx + security_opt: + - apparmor:unconfined + - seccomp:unconfined + + b: + extends: + service: a + """) + services = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + assert len(services) == 2 + for svc in services: + assert types.SecurityOpt.parse('apparmor:unconfined') in svc['security_opt'] + assert types.SecurityOpt.parse('seccomp:unconfined') in svc['security_opt'] + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From 742979371d55aa4f2ada056c0447f3e0c1485090 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 2 Mar 2018 15:45:37 -0800 Subject: [PATCH 07/13] Install both versions of python Signed-off-by: Joffrey F --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7661c6470..6ee2a60e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: - checkout - run: name: install python3 - command: brew update > /dev/null && brew install python3 + command: brew update > /dev/null && brew upgrade python - run: name: install tox command: sudo pip install --upgrade tox==2.1.1 From 768e28ee303d2ef6a6102fd1394b527fcdf10f2a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Mar 2018 15:44:42 -0800 Subject: [PATCH 08/13] Update Dockerfile.run to produce smaller image Signed-off-by: Joffrey F --- Dockerfile.run | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/Dockerfile.run b/Dockerfile.run index b3f9a01f6..04acf01c3 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,42 +1,20 @@ -FROM sgerrand/glibc-builder as glibc -RUN apt-get install -yq bison - -ENV PKGDIR /pkgdata - -RUN mkdir -p /usr/glibc-compat/etc && touch /usr/glibc-compat/etc/ld.so.conf -RUN /builder 2.27 /usr/glibc-compat || true -RUN mkdir -p $PKGDIR -RUN tar -xf /glibc-bin-2.27.tar.gz -C $PKGDIR -RUN rm "$PKGDIR"/usr/glibc-compat/etc/rpc && \ - rm -rf "$PKGDIR"/usr/glibc-compat/bin && \ - rm -rf "$PKGDIR"/usr/glibc-compat/sbin && \ - rm -rf "$PKGDIR"/usr/glibc-compat/lib/gconv && \ - rm -rf "$PKGDIR"/usr/glibc-compat/lib/getconf && \ - rm -rf "$PKGDIR"/usr/glibc-compat/lib/audit && \ - rm -rf "$PKGDIR"/usr/glibc-compat/share && \ - rm -rf "$PKGDIR"/usr/glibc-compat/var - - FROM alpine:3.6 -RUN apk update && apk add --no-cache openssl ca-certificates -COPY --from=glibc /pkgdata/ / +ENV GLIBC 2.27-r0 +ENV DOCKERBINS_SHA 1270dce1bd7e1838d62ae21d2505d87f16efc1d9074645571daaefdfd0c14054 -RUN mkdir -p /lib /lib64 /usr/glibc-compat/lib/locale /etc && \ +RUN apk update && apk add --no-cache openssl ca-certificates curl && \ + curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \ + curl -fsSL -o glibc-$GLIBC.apk https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ + apk add --no-cache glibc-$GLIBC.apk && \ ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ - ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib/ld-linux-x86-64.so.2 && \ - ln -s /usr/glibc-compat/lib/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 && \ - ln -s /usr/glibc-compat/etc/ld.so.cache /etc/ld.so.cache - -RUN apk add --no-cache curl && \ - curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.0-ce.tgz" && \ - SHA256=692e1c72937f6214b1038def84463018d8e320c8eaf8530546c84c2f8f9c767d; \ - echo "${SHA256} dockerbins.tgz" | sha256sum -c - && \ + curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.1-ce.tgz" && \ + echo "${DOCKERBINS_SHA} dockerbins.tgz" | sha256sum -c - && \ tar xvf dockerbins.tgz docker/docker --strip-components 1 && \ mv docker /usr/local/bin/docker && \ chmod +x /usr/local/bin/docker && \ - rm dockerbins.tgz && \ + rm dockerbins.tgz /etc/apk/keys/sgerrand.rsa.pub glibc-$GLIBC.apk && \ apk del curl COPY dist/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose From 8c4af54257daae922010f75dac3f259fb0f3905d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Mar 2018 18:29:48 -0800 Subject: [PATCH 09/13] Bump 1.20.0-rc2 Signed-off-by: Joffrey F --- CHANGELOG.md | 10 +++++++++- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e662d40a6..c113572ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,10 +38,12 @@ Change log - Proxy configuration found in the `~/.docker/config.json` file now populates environment and build args for containers created by Compose -- Added a `--use-aliases` flag to `docker-compose run`, indicating that +- Added the `--use-aliases` flag to `docker-compose run`, indicating that network aliases declared in the service's config should be used for the running container +- Added the `--include-deps` flag to `docker-compose pull` + - `docker-compose run` now kills and removes the running container upon receiving `SIGHUP` @@ -55,6 +57,9 @@ Change log - Fixed `.dockerignore` handling, notably with regard to absolute paths and last-line precedence rules +- Fixed an issue where Compose would make costly DNS lookups when connecting + to the Engine when using Docker For Mac + - Fixed a bug introduced in 1.19.0 which caused the default certificate path to not be honored by Compose @@ -70,6 +75,9 @@ Change log - A `seccomp:` entry in the `security_opt` config now correctly sends the contents of the file to the engine +- ANSI output for `up` and `down` operations should no longer affect the wrong + lines + - Improved support for non-unicode locales - Fixed a crash occurring on Windows when the user's home directory name diff --git a/compose/__init__.py b/compose/__init__.py index 2090f10c8..d2e3f696b 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.20.0-rc1' +__version__ = '1.20.0-rc2' diff --git a/script/run/run.sh b/script/run/run.sh index 2adcc98f8..dff9d982a 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.20.0-rc1" +VERSION="1.20.0-rc2" IMAGE="docker/compose:$VERSION" From fd94fab264ee56509730dce184a30e159d4cc074 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 15 Mar 2018 16:22:28 -0700 Subject: [PATCH 10/13] Manage encoding errors in progress_stream Signed-off-by: Joffrey F --- compose/progress_stream.py | 32 +++++++++++++++++++----------- tests/unit/progress_stream_test.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 5314f89fd..5e709770a 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -8,6 +8,14 @@ class StreamOutputError(Exception): pass +def write_to_stream(s, stream): + try: + stream.write(s) + except UnicodeEncodeError: + encoding = getattr(stream, 'encoding', 'ascii') + stream.write(s.encode(encoding, errors='replace').decode(encoding)) + + def stream_output(output, stream): is_terminal = hasattr(stream, 'isatty') and stream.isatty() stream = utils.get_output_stream(stream) @@ -34,18 +42,18 @@ def stream_output(output, stream): if image_id not in lines: lines[image_id] = len(lines) - stream.write("\n") + write_to_stream("\n", stream) diff = len(lines) - lines[image_id] # move cursor up `diff` rows - stream.write("%c[%dA" % (27, diff)) + write_to_stream("%c[%dA" % (27, diff), stream) print_output_event(event, stream, is_terminal) if 'id' in event: # move cursor back down - stream.write("%c[%dB" % (27, diff)) + write_to_stream("%c[%dB" % (27, diff), stream) stream.flush() @@ -60,36 +68,36 @@ def print_output_event(event, stream, is_terminal): if is_terminal and 'stream' not in event: # erase current line - stream.write("%c[2K\r" % 27) + write_to_stream("%c[2K\r" % 27, stream) terminator = "\r" elif 'progressDetail' in event: return if 'time' in event: - stream.write("[%s] " % event['time']) + write_to_stream("[%s] " % event['time'], stream) if 'id' in event: - stream.write("%s: " % event['id']) + write_to_stream("%s: " % event['id'], stream) if 'from' in event: - stream.write("(from %s) " % event['from']) + write_to_stream("(from %s) " % event['from'], stream) status = event.get('status', '') if 'progress' in event: - stream.write("%s %s%s" % (status, event['progress'], terminator)) + write_to_stream("%s %s%s" % (status, event['progress'], terminator), stream) elif 'progressDetail' in event: detail = event['progressDetail'] total = detail.get('total') if 'current' in detail and total: percentage = float(detail['current']) / float(total) * 100 - stream.write('%s (%.1f%%)%s' % (status, percentage, terminator)) + write_to_stream('%s (%.1f%%)%s' % (status, percentage, terminator), stream) else: - stream.write('%s%s' % (status, terminator)) + write_to_stream('%s%s' % (status, terminator), stream) elif 'stream' in event: - stream.write("%s%s" % (event['stream'], terminator)) + write_to_stream("%s%s" % (event['stream'], terminator), stream) else: - stream.write("%s%s\n" % (status, terminator)) + write_to_stream("%s%s\n" % (status, terminator), stream) def get_digest_from_pull(events): diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index 22a6e081b..f4a0ab063 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -1,6 +1,13 @@ +# ~*~ encoding: utf-8 ~*~ from __future__ import absolute_import from __future__ import unicode_literals +import io +import os +import random +import shutil +import tempfile + from six import StringIO from compose import progress_stream @@ -66,6 +73,30 @@ class ProgressStreamTestCase(unittest.TestCase): events = progress_stream.stream_output(events, output) assert len(output.getvalue()) > 0 + def test_mismatched_encoding_stream_write(self): + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir, True) + + def mktempfile(encoding): + fname = os.path.join(tmpdir, hex(random.getrandbits(128))[2:-1]) + return io.open(fname, mode='w+', encoding=encoding) + + text = '就吃饭' + with mktempfile(encoding='utf-8') as tf: + progress_stream.write_to_stream(text, tf) + tf.seek(0) + assert tf.read() == text + + with mktempfile(encoding='utf-32') as tf: + progress_stream.write_to_stream(text, tf) + tf.seek(0) + assert tf.read() == text + + with mktempfile(encoding='ascii') as tf: + progress_stream.write_to_stream(text, tf) + tf.seek(0) + assert tf.read() == '???' + def test_get_digest_from_push(): digest = "sha256:abcd" From 538e982c93ca6c2c281e2dea116790d8d6c57e49 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Mar 2018 12:15:51 +0100 Subject: [PATCH 11/13] Bump Docker SDK -> 3.1.3 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 33462d496..743272d4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 -docker==3.1.1 +docker==3.1.3 docker-pycreds==0.2.1 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index cf8f6dc13..e24736df0 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.19', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.1.1, < 4.0', + 'docker >= 3.1.3, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 80ce4eafa722ee9fc312ad0d14d2eaa5d7f17a22 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Mar 2018 15:10:39 +0100 Subject: [PATCH 12/13] pip3? Signed-off-by: Joffrey F --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6ee2a60e5..66a87d1d0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: command: brew update > /dev/null && brew upgrade python - run: name: install tox - command: sudo pip install --upgrade tox==2.1.1 + command: sudo pip3 install --upgrade tox==2.1.1 - run: name: unit tests command: tox -e py27,py36 -- tests/unit From ca8d3c6c18d314a7314e383b53472127e00a523f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 20 Mar 2018 14:55:44 +0100 Subject: [PATCH 13/13] Bump 1.20.0 Signed-off-by: Joffrey F --- CHANGELOG.md | 2 ++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c113572ca..ab0a2c354 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,8 @@ Change log - Fixed a bug occurring during builds caused by files with a negative `mtime` values in the build context +- Fixed an encoding bug when streaming build progress + 1.19.0 (2018-02-07) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index d2e3f696b..1c5a409f1 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.20.0-rc2' +__version__ = '1.20.0' diff --git a/script/run/run.sh b/script/run/run.sh index dff9d982a..a739edb35 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.20.0-rc2" +VERSION="1.20.0" IMAGE="docker/compose:$VERSION"