From 14115a3570f51f5fe391f02583b8b452ba86136e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Mar 2017 17:27:11 -0700 Subject: [PATCH 1/7] Recognize COMPOSE_TLS_VERSION env var in tls_config_from_options Signed-off-by: Joffrey F --- compose/cli/command.py | 19 +------------------ compose/cli/docker_client.py | 27 ++++++++++++++++++++++++--- tests/unit/cli/command_test.py | 20 -------------------- tests/unit/cli/docker_client_test.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 41 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 4e2722648..ccc76ceb4 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals import logging import os import re -import ssl import six @@ -15,6 +14,7 @@ from ..config.environment import Environment from ..const import API_VERSIONS from ..project import Project from .docker_client import docker_client +from .docker_client import get_tls_version from .docker_client import tls_config_from_options from .utils import get_version_info @@ -60,23 +60,6 @@ def get_config_path_from_options(base_dir, options, environment): return None -def get_tls_version(environment): - compose_tls_version = environment.get('COMPOSE_TLS_VERSION', None) - if not compose_tls_version: - return None - - tls_attr_name = "PROTOCOL_{}".format(compose_tls_version) - if not hasattr(ssl, tls_attr_name): - log.warn( - 'The "{}" protocol is unavailable. You may need to update your ' - 'version of Python or OpenSSL. Falling back to TLSv1 (default).' - .format(compose_tls_version) - ) - return None - - return getattr(ssl, tls_attr_name) - - def get_client(environment, verbose=False, version=None, tls_config=None, host=None, tls_version=None): diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 018d24513..44c7ad91d 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging +import ssl from docker import APIClient from docker.errors import TLSParameterError @@ -16,7 +17,24 @@ from .utils import unquote_path log = logging.getLogger(__name__) -def tls_config_from_options(options): +def get_tls_version(environment): + compose_tls_version = environment.get('COMPOSE_TLS_VERSION', None) + if not compose_tls_version: + return None + + tls_attr_name = "PROTOCOL_{}".format(compose_tls_version) + if not hasattr(ssl, tls_attr_name): + log.warn( + 'The "{}" protocol is unavailable. You may need to update your ' + 'version of Python or OpenSSL. Falling back to TLSv1 (default).' + .format(compose_tls_version) + ) + return None + + return getattr(ssl, tls_attr_name) + + +def tls_config_from_options(options, environment=None): tls = options.get('--tls', False) ca_cert = unquote_path(options.get('--tlscacert')) cert = unquote_path(options.get('--tlscert')) @@ -24,7 +42,9 @@ def tls_config_from_options(options): verify = options.get('--tlsverify') skip_hostname_check = options.get('--skip-hostname-check', False) - advanced_opts = any([ca_cert, cert, key, verify]) + tls_version = get_tls_version(environment or {}) + + advanced_opts = any([ca_cert, cert, key, verify, tls_version]) if tls is True and not advanced_opts: return True @@ -35,7 +55,8 @@ def tls_config_from_options(options): return TLSConfig( client_cert=client_cert, verify=verify, ca_cert=ca_cert, - assert_hostname=False if skip_hostname_check else None + assert_hostname=False if skip_hostname_check else None, + ssl_version=tls_version ) return None diff --git a/tests/unit/cli/command_test.py b/tests/unit/cli/command_test.py index 3655c432e..c64a0401b 100644 --- a/tests/unit/cli/command_test.py +++ b/tests/unit/cli/command_test.py @@ -2,12 +2,10 @@ from __future__ import absolute_import from __future__ import unicode_literals import os -import ssl import pytest from compose.cli.command import get_config_path_from_options -from compose.cli.command import get_tls_version from compose.config.environment import Environment from compose.const import IS_WINDOWS_PLATFORM from tests import mock @@ -57,21 +55,3 @@ class TestGetConfigPathFromOptions(object): def test_no_path(self): environment = Environment.from_env_file('.') assert not get_config_path_from_options('.', {}, environment) - - -class TestGetTlsVersion(object): - def test_get_tls_version_default(self): - environment = {} - assert get_tls_version(environment) is None - - @pytest.mark.skipif(not hasattr(ssl, 'PROTOCOL_TLSv1_2'), reason='TLS v1.2 unsupported') - def test_get_tls_version_upgrade(self): - environment = {'COMPOSE_TLS_VERSION': 'TLSv1_2'} - assert get_tls_version(environment) == ssl.PROTOCOL_TLSv1_2 - - def test_get_tls_version_unavailable(self): - environment = {'COMPOSE_TLS_VERSION': 'TLSv5_5'} - with mock.patch('compose.cli.command.log') as mock_log: - tls_version = get_tls_version(environment) - mock_log.warn.assert_called_once_with(mock.ANY) - assert tls_version is None diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index aaa935afa..482ad9850 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import os import platform +import ssl import docker import pytest @@ -10,6 +11,7 @@ import pytest import compose from compose.cli import errors from compose.cli.docker_client import docker_client +from compose.cli.docker_client import get_tls_version from compose.cli.docker_client import tls_config_from_options from tests import mock from tests import unittest @@ -157,3 +159,29 @@ class TLSConfigTestCase(unittest.TestCase): assert result.cert == (self.client_cert, self.key) assert result.ca_cert == self.ca_cert assert result.verify is True + + def test_tls_simple_with_tls_version(self): + tls_version = 'TLSv1' + options = {'--tls': True} + environment = {'COMPOSE_TLS_VERSION': tls_version} + result = tls_config_from_options(options, environment) + assert isinstance(result, docker.tls.TLSConfig) + assert result.ssl_version == ssl.PROTOCOL_TLSv1 + + +class TestGetTlsVersion(object): + def test_get_tls_version_default(self): + environment = {} + assert get_tls_version(environment) is None + + @pytest.mark.skipif(not hasattr(ssl, 'PROTOCOL_TLSv1_2'), reason='TLS v1.2 unsupported') + def test_get_tls_version_upgrade(self): + environment = {'COMPOSE_TLS_VERSION': 'TLSv1_2'} + assert get_tls_version(environment) == ssl.PROTOCOL_TLSv1_2 + + def test_get_tls_version_unavailable(self): + environment = {'COMPOSE_TLS_VERSION': 'TLSv5_5'} + with mock.patch('compose.cli.docker_client.log') as mock_log: + tls_version = get_tls_version(environment) + mock_log.warn.assert_called_once_with(mock.ANY) + assert tls_version is None From 6cbedb78ccaf4d9ca38d6969970172af41f61518 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Mar 2017 14:09:50 -0700 Subject: [PATCH 2/7] The interval is too damn small Signed-off-by: Joffrey F --- tests/integration/project_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index e8dbe8fbf..455189851 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1419,7 +1419,7 @@ class ProjectTest(DockerClientTestCase): 'test': 'exit 0', 'retries': 1, 'timeout': '10s', - 'interval': '0.1s' + 'interval': '1s' }, }, 'svc2': { @@ -1456,7 +1456,7 @@ class ProjectTest(DockerClientTestCase): 'test': 'exit 1', 'retries': 1, 'timeout': '10s', - 'interval': '0.1s' + 'interval': '1s' }, }, 'svc2': { From 0a55b070025a974841b610a68f201cc337e8db64 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 21 Mar 2017 17:23:29 -0700 Subject: [PATCH 3/7] Fix ports reparsing for service extends Signed-off-by: Joffrey F --- compose/config/types.py | 4 ++++ tests/unit/config/config_test.py | 24 +++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/compose/config/types.py b/compose/config/types.py index a8d366fba..96846b5ba 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -266,6 +266,10 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext @classmethod def parse(cls, spec): + if isinstance(spec, cls): + # WHen extending a service with ports, the port definitions have already been parsed + return [spec] + if not isinstance(spec, dict): result = [] for k, v in build_port_bindings([spec]).items(): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 195efe3b9..4db87ecb6 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3403,7 +3403,7 @@ class ExtendsTest(unittest.TestCase): self.assertEqual(service[0]['command'], "top") def test_extends_with_depends_on(self): - tmpdir = py.test.ensuretemp('test_extends_with_defined_version') + tmpdir = py.test.ensuretemp('test_extends_with_depends_on') self.addCleanup(tmpdir.remove) tmpdir.join('docker-compose.yml').write(""" version: "2" @@ -3435,6 +3435,28 @@ class ExtendsTest(unittest.TestCase): } }] + def test_extends_with_ports(self): + tmpdir = py.test.ensuretemp('test_extends_with_ports') + self.addCleanup(tmpdir.remove) + tmpdir.join('docker-compose.yml').write(""" + version: '2' + + services: + a: + image: nginx + ports: + - 80 + + b: + extends: + service: a + """) + services = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + + assert len(services) == 2 + for svc in services: + assert svc['ports'] == [types.ServicePort('80', None, None, None, None)] + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From 747186f9008609d7ff2ddfede71ab0916c4a5718 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Mar 2017 17:04:26 -0700 Subject: [PATCH 4/7] Support 'nocopy' mode for expanded volume syntax Signed-off-by: Joffrey F --- compose/config/config.py | 7 +++++-- tests/acceptance/cli_test.py | 3 ++- tests/fixtures/v3-full/docker-compose.yml | 5 +++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 8cbaae272..72687d756 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1033,8 +1033,11 @@ def resolve_volume_path(working_dir, volume): if isinstance(volume, dict): host_path = volume.get('source') container_path = volume.get('target') - if host_path and volume.get('read_only'): - container_path += ':ro' + if host_path: + if volume.get('read_only'): + container_path += ':ro' + if volume.get('volume', {}).get('nocopy'): + container_path += ':nocopy' else: container_path, host_path = split_path_mapping(volume) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 14e6f7336..bceb102a2 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -374,7 +374,8 @@ class CLITestCase(DockerClientTestCase): 'volumes': [ '/host/path:/container/path:ro', 'foobar:/container/volumepath:rw', - '/anonymous' + '/anonymous', + 'foobar:/container/volumepath2:nocopy' ], 'stop_grace_period': '20s', diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml index 27f3c6e04..2bc0e248d 100644 --- a/tests/fixtures/v3-full/docker-compose.yml +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -44,6 +44,11 @@ services: target: /container/volumepath - type: volume target: /anonymous + - type: volume + source: foobar + target: /container/volumepath2 + volume: + nocopy: true stop_grace_period: 20s volumes: From da466b391470333492a56395569812653ed6658f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 21 Mar 2017 16:28:53 -0700 Subject: [PATCH 5/7] Change docker-py dependency error to a warning, update fix command Signed-off-by: Joffrey F --- compose/cli/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index c5db44558..4061e7091 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -20,16 +20,15 @@ try: list(filter(lambda p: p.startswith(b'docker-py=='), packages)) ) > 0 if dockerpy_installed: - from .colors import red + from .colors import yellow print( - red('ERROR:'), + yellow('WARNING:'), "Dependency conflict: an older version of the 'docker-py' package " - "is polluting the namespace. " - "Run the following command to remedy the issue:\n" - "pip uninstall docker docker-py; pip install docker", + "may be polluting the namespace. " + "If you're experiencing crashes, run the following command to remedy the issue:\n" + "pip uninstall docker-py; pip uninstall docker; pip install docker", file=sys.stderr ) - sys.exit(1) except OSError: # pip command is not available, which indicates it's probably the binary From 7575f4006939bdb346a793e6747d3f132c00ecb8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 23 Mar 2017 16:43:15 -0700 Subject: [PATCH 6/7] Ignore unicode error in subprocess call Signed-off-by: Joffrey F --- compose/cli/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index 4061e7091..1fe9aab8d 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -34,3 +34,9 @@ except OSError: # pip command is not available, which indicates it's probably the binary # distribution of Compose which is not affected pass +except UnicodeDecodeError: + # ref: https://github.com/docker/compose/issues/4663 + # This could be caused by a number of things, but it seems to be a + # python 2 + MacOS interaction. It's not ideal to ignore this, but at least + # it doesn't make the program unusable. + pass From 08dc2a41524e2bb476aec34a9864e13e325665c3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 23 Mar 2017 15:21:40 -0700 Subject: [PATCH 7/7] Bump 1.12.0-rc2 Signed-off-by: Joffrey F --- CHANGELOG.md | 3 +++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 581d45bbf..0aa3acf65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,9 @@ Change log - Fixed a bug where `docker-compose` would crash when trying to write into a closed pipe +- Fixed an issue where Compose would not pick up on the value of + COMPOSE_TLS_VERSION when used in combination with command-line TLS flags + 1.11.2 (2017-02-17) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 502e9cc4f..4399b28af 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.12.0-rc1' +__version__ = '1.12.0-rc2' diff --git a/script/run/run.sh b/script/run/run.sh index 192b31219..62c065bbc 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.12.0-rc1" +VERSION="1.12.0-rc2" IMAGE="docker/compose:$VERSION"