From 1ac33ea7e5f5c2c4d4facd5b52143f0a962515bc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 15 Mar 2016 14:19:25 -0700 Subject: [PATCH 1/6] Add support for TLS config command-line options Signed-off-by: Joffrey F --- compose/cli/command.py | 15 +++++++++---- compose/cli/docker_client.py | 41 +++++++++++++++++++++++++++++++++++- compose/cli/main.py | 7 ++++++ 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 55f6df01a..730cd1153 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -12,6 +12,7 @@ from .. import config from ..const import API_VERSIONS from ..project import Project from .docker_client import docker_client +from .docker_client import TLSArgs from .utils import get_version_info log = logging.getLogger(__name__) @@ -23,6 +24,8 @@ def project_from_options(project_dir, options): get_config_path_from_options(options), project_name=options.get('--project-name'), verbose=options.get('--verbose'), + host=options.get('--host'), + tls_args=TLSArgs.from_options(options), ) @@ -37,8 +40,8 @@ def get_config_path_from_options(options): return None -def get_client(verbose=False, version=None): - client = docker_client(version=version) +def get_client(verbose=False, version=None, tls_args=None, host=None): + client = docker_client(version=version, tls_args=tls_args, host=host) if verbose: version_info = six.iteritems(client.version()) log.info(get_version_info('full')) @@ -49,7 +52,8 @@ def get_client(verbose=False, version=None): return client -def get_project(project_dir, config_path=None, project_name=None, verbose=False): +def get_project(project_dir, config_path=None, project_name=None, verbose=False, + host=None, tls_args=None): config_details = config.find(project_dir, config_path) project_name = get_project_name(config_details.working_dir, project_name) config_data = config.load(config_details) @@ -57,7 +61,10 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False) api_version = os.environ.get( 'COMPOSE_API_VERSION', API_VERSIONS[config_data.version]) - client = get_client(verbose=verbose, version=api_version) + client = get_client( + verbose=verbose, version=api_version, tls_args=tls_args, + host=host + ) return Project.from_config(project_name, config_data, client) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 9e79fe777..cff28f8c7 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -3,9 +3,11 @@ from __future__ import unicode_literals import logging import os +from collections import namedtuple from docker import Client from docker.errors import TLSParameterError +from docker.tls import TLSConfig from docker.utils import kwargs_from_env from ..const import HTTP_TIMEOUT @@ -14,7 +16,24 @@ from .errors import UserError log = logging.getLogger(__name__) -def docker_client(version=None): +class TLSArgs(namedtuple('_TLSArgs', 'tls cert key ca_cert verify')): + @classmethod + def from_options(cls, options): + return cls( + tls=options.get('--tls', False), + ca_cert=options.get('--tlscacert'), + cert=options.get('--tlscert'), + key=options.get('--tlskey'), + verify=options.get('--tlsverify') + ) + + # def has_config(self): + # return ( + # self.tls or self.ca_cert or self.cert or self.key or self.verify + # ) + + +def docker_client(version=None, tls_args=None, host=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. @@ -31,6 +50,26 @@ def docker_client(version=None): "and DOCKER_CERT_PATH are set correctly.\n" "You might need to run `eval \"$(docker-machine env default)\"`") + if host: + kwargs['base_url'] = host + if tls_args and any(tls_args): + if tls_args.tls is True: + kwargs['tls'] = True + else: + client_cert = None + if tls_args.cert or tls_args.key: + client_cert = (tls_args.cert, tls_args.key) + try: + kwargs['tls'] = TLSConfig( + client_cert=client_cert, verify=tls_args.verify, + ca_cert=tls_args.ca_cert + ) + except TLSParameterError as e: + raise UserError( + "TLS configuration is invalid. Please double-check the " + "TLS command-line arguments. ({0})".format(e) + ) + if version: kwargs['version'] = version diff --git a/compose/cli/main.py b/compose/cli/main.py index a978579c0..17c2ac45f 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -149,6 +149,13 @@ class TopLevelCommand(object): -p, --project-name NAME Specify an alternate project name (default: directory name) --verbose Show more output -v, --version Print version and exit + -H, --host HOST Daemon socket to connect to + + --tls Use TLS; implied by --tlsverify + --tlsacert Trust certs signed only by this CA + --tlscert Path to TLS certificate file + --tlskey Path to TLS key file + --tlsverify Use TLS and verify the remote Commands: build Build or rebuild services From 7166408d2a9f9972e4a7f60f30228808f2260117 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 16 Mar 2016 13:40:13 -0700 Subject: [PATCH 2/6] Fixed typos + simplified TLSConfig creation process. Signed-off-by: Joffrey F --- compose/cli/command.py | 12 ++++---- compose/cli/docker_client.py | 53 ++++++++++++++---------------------- compose/cli/main.py | 20 +++++++------- 3 files changed, 36 insertions(+), 49 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 730cd1153..63d387f0c 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -12,7 +12,7 @@ from .. import config from ..const import API_VERSIONS from ..project import Project from .docker_client import docker_client -from .docker_client import TLSArgs +from .docker_client import tls_config_from_options from .utils import get_version_info log = logging.getLogger(__name__) @@ -25,7 +25,7 @@ def project_from_options(project_dir, options): project_name=options.get('--project-name'), verbose=options.get('--verbose'), host=options.get('--host'), - tls_args=TLSArgs.from_options(options), + tls_config=tls_config_from_options(options), ) @@ -40,8 +40,8 @@ def get_config_path_from_options(options): return None -def get_client(verbose=False, version=None, tls_args=None, host=None): - client = docker_client(version=version, tls_args=tls_args, host=host) +def get_client(verbose=False, version=None, tls_config=None, host=None): + client = docker_client(version=version, tls_config=tls_config, host=host) if verbose: version_info = six.iteritems(client.version()) log.info(get_version_info('full')) @@ -53,7 +53,7 @@ def get_client(verbose=False, version=None, tls_args=None, host=None): def get_project(project_dir, config_path=None, project_name=None, verbose=False, - host=None, tls_args=None): + host=None, tls_config=None): config_details = config.find(project_dir, config_path) project_name = get_project_name(config_details.working_dir, project_name) config_data = config.load(config_details) @@ -62,7 +62,7 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, 'COMPOSE_API_VERSION', API_VERSIONS[config_data.version]) client = get_client( - verbose=verbose, version=api_version, tls_args=tls_args, + verbose=verbose, version=api_version, tls_config=tls_config, host=host ) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index cff28f8c7..c8159ad49 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import logging import os -from collections import namedtuple from docker import Client from docker.errors import TLSParameterError @@ -16,24 +15,27 @@ from .errors import UserError log = logging.getLogger(__name__) -class TLSArgs(namedtuple('_TLSArgs', 'tls cert key ca_cert verify')): - @classmethod - def from_options(cls, options): - return cls( - tls=options.get('--tls', False), - ca_cert=options.get('--tlscacert'), - cert=options.get('--tlscert'), - key=options.get('--tlskey'), - verify=options.get('--tlsverify') +def tls_config_from_options(options): + tls = options.get('--tls', False) + ca_cert = options.get('--tlscacert') + cert = options.get('--tlscert') + key = options.get('--tlskey') + verify = options.get('--tlsverify') + + if tls is True: + return True + elif any([ca_cert, cert, key, verify]): + client_cert = None + if cert or key: + client_cert = (cert, key) + return TLSConfig( + client_cert=client_cert, verify=verify, ca_cert=ca_cert ) - - # def has_config(self): - # return ( - # self.tls or self.ca_cert or self.cert or self.key or self.verify - # ) + else: + return None -def docker_client(version=None, tls_args=None, host=None): +def docker_client(version=None, tls_config=None, host=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. @@ -52,23 +54,8 @@ def docker_client(version=None, tls_args=None, host=None): if host: kwargs['base_url'] = host - if tls_args and any(tls_args): - if tls_args.tls is True: - kwargs['tls'] = True - else: - client_cert = None - if tls_args.cert or tls_args.key: - client_cert = (tls_args.cert, tls_args.key) - try: - kwargs['tls'] = TLSConfig( - client_cert=client_cert, verify=tls_args.verify, - ca_cert=tls_args.ca_cert - ) - except TLSParameterError as e: - raise UserError( - "TLS configuration is invalid. Please double-check the " - "TLS command-line arguments. ({0})".format(e) - ) + if tls_config: + kwargs['tls'] = tls_config if version: kwargs['version'] = version diff --git a/compose/cli/main.py b/compose/cli/main.py index 17c2ac45f..331476e21 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -145,17 +145,17 @@ class TopLevelCommand(object): docker-compose -h|--help Options: - -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) - -p, --project-name NAME Specify an alternate project name (default: directory name) - --verbose Show more output - -v, --version Print version and exit - -H, --host HOST Daemon socket to connect to + -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) + -p, --project-name NAME Specify an alternate project name (default: directory name) + --verbose Show more output + -v, --version Print version and exit + -H, --host HOST Daemon socket to connect to - --tls Use TLS; implied by --tlsverify - --tlsacert Trust certs signed only by this CA - --tlscert Path to TLS certificate file - --tlskey Path to TLS key file - --tlsverify Use TLS and verify the remote + --tls Use TLS; implied by --tlsverify + --tlscacert CA_PATH Trust certs signed only by this CA + --tlscert CLIENT_CERT_PATH Path to TLS certificate file + --tlskey TLS_KEY_PATH Path to TLS key file + --tlsverify Use TLS and verify the remote Commands: build Build or rebuild services From 26f3861791a82ddee9171a6710f595b0136c4ab3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 17 Mar 2016 16:09:45 -0700 Subject: [PATCH 3/6] Specifying --tls no longer overrides all other TLS options Add an option to skip hostname verification Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 9 ++++++--- compose/cli/main.py | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index c8159ad49..e2848a90b 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -22,14 +22,17 @@ def tls_config_from_options(options): key = options.get('--tlskey') verify = options.get('--tlsverify') - if tls is True: + advanced_opts = any([ca_cert, cert, key, verify]) + + if tls is True and not advanced_opts: return True - elif any([ca_cert, cert, key, verify]): + elif advanced_opts: client_cert = None if cert or key: client_cert = (cert, key) return TLSConfig( - client_cert=client_cert, verify=verify, ca_cert=ca_cert + client_cert=client_cert, verify=verify, ca_cert=ca_cert, + assert_hostname=options.get('--skip-hostname-check') ) else: return None diff --git a/compose/cli/main.py b/compose/cli/main.py index 331476e21..6eada097f 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -156,6 +156,9 @@ class TopLevelCommand(object): --tlscert CLIENT_CERT_PATH Path to TLS certificate file --tlskey TLS_KEY_PATH Path to TLS key file --tlsverify Use TLS and verify the remote + --skip-hostname-check Don't check the daemon's hostname against the name specified + in the client certificate (for example if your docker host + is an IP address) Commands: build Build or rebuild services From 442dff72b4568656821189e3d45617e3f87f63c9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 21 Mar 2016 12:06:29 -0700 Subject: [PATCH 4/6] Improve assert_hostname setting in tls_config_from_options Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index e2848a90b..d47bd2dbb 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -8,6 +8,7 @@ from docker import Client from docker.errors import TLSParameterError from docker.tls import TLSConfig from docker.utils import kwargs_from_env +from requests.utils import urlparse from ..const import HTTP_TIMEOUT from .errors import UserError @@ -21,6 +22,7 @@ def tls_config_from_options(options): cert = options.get('--tlscert') key = options.get('--tlskey') verify = options.get('--tlsverify') + hostname = urlparse(options.get('--host', '')).hostname advanced_opts = any([ca_cert, cert, key, verify]) @@ -32,7 +34,9 @@ def tls_config_from_options(options): client_cert = (cert, key) return TLSConfig( client_cert=client_cert, verify=verify, ca_cert=ca_cert, - assert_hostname=options.get('--skip-hostname-check') + assert_hostname=( + hostname or not options.get('--skip-hostname-check', False) + ) ) else: return None From 472711531749aa3e9909ec8e386e4bd73027530b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 21 Mar 2016 12:10:00 -0700 Subject: [PATCH 5/6] Bump docker-py version to include tcp host fix Signed-off-by: Joffrey F --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2b7c85e6a..883673538 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.2.0 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4 -git+https://github.com/docker/docker-py.git@8c4546f8c8f52bb2923834783a17beb5bb89a724#egg=docker-py +git+https://github.com/docker/docker-py.git@5c1c42397cf0fdb74182df2d69822b82df8f2a6a#egg=docker-py jsonschema==2.5.1 requests==2.7.0 six==1.7.3 From 2cc87555cb1e1c1c8322ca4dcae21f00025800aa Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 21 Mar 2016 14:23:31 -0700 Subject: [PATCH 6/6] tls_config_from_options unit tests Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 2 +- tests/fixtures/tls/ca.pem | 0 tests/fixtures/tls/cert.pem | 0 tests/fixtures/tls/key.key | 0 tests/unit/cli/docker_client_test.py | 89 +++++++++++++++++++++++++++- 5 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/tls/ca.pem create mode 100644 tests/fixtures/tls/cert.pem create mode 100644 tests/fixtures/tls/key.key diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index d47bd2dbb..deb568660 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -22,7 +22,7 @@ def tls_config_from_options(options): cert = options.get('--tlscert') key = options.get('--tlskey') verify = options.get('--tlsverify') - hostname = urlparse(options.get('--host', '')).hostname + hostname = urlparse(options.get('--host') or '').hostname advanced_opts = any([ca_cert, cert, key, verify]) diff --git a/tests/fixtures/tls/ca.pem b/tests/fixtures/tls/ca.pem new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/tls/cert.pem b/tests/fixtures/tls/cert.pem new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/tls/key.key b/tests/fixtures/tls/key.key new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index d497495b4..b55f1d179 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -3,7 +3,11 @@ from __future__ import unicode_literals import os -from compose.cli import docker_client +import docker +import pytest + +from compose.cli.docker_client import docker_client +from compose.cli.docker_client import tls_config_from_options from tests import mock from tests import unittest @@ -13,10 +17,89 @@ class DockerClientTestCase(unittest.TestCase): def test_docker_client_no_home(self): with mock.patch.dict(os.environ): del os.environ['HOME'] - docker_client.docker_client() + docker_client() def test_docker_client_with_custom_timeout(self): timeout = 300 with mock.patch('compose.cli.docker_client.HTTP_TIMEOUT', 300): - client = docker_client.docker_client() + client = docker_client() self.assertEqual(client.timeout, int(timeout)) + + +class TLSConfigTestCase(unittest.TestCase): + ca_cert = 'tests/fixtures/tls/ca.pem' + client_cert = 'tests/fixtures/tls/cert.pem' + key = 'tests/fixtures/tls/key.key' + + def test_simple_tls(self): + options = {'--tls': True} + result = tls_config_from_options(options) + assert result is True + + def test_tls_ca_cert(self): + options = { + '--tlscacert': self.ca_cert, '--tlsverify': True + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.ca_cert == options['--tlscacert'] + assert result.verify is True + + def test_tls_ca_cert_explicit(self): + options = { + '--tlscacert': self.ca_cert, '--tls': True, + '--tlsverify': True + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.ca_cert == options['--tlscacert'] + assert result.verify is True + + def test_tls_client_cert(self): + options = { + '--tlscert': self.client_cert, '--tlskey': self.key + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.cert == (options['--tlscert'], options['--tlskey']) + + def test_tls_client_cert_explicit(self): + options = { + '--tlscert': self.client_cert, '--tlskey': self.key, + '--tls': True + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.cert == (options['--tlscert'], options['--tlskey']) + + def test_tls_client_and_ca(self): + options = { + '--tlscert': self.client_cert, '--tlskey': self.key, + '--tlsverify': True, '--tlscacert': self.ca_cert + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.cert == (options['--tlscert'], options['--tlskey']) + assert result.ca_cert == options['--tlscacert'] + assert result.verify is True + + def test_tls_client_and_ca_explicit(self): + options = { + '--tlscert': self.client_cert, '--tlskey': self.key, + '--tlsverify': True, '--tlscacert': self.ca_cert, + '--tls': True + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.cert == (options['--tlscert'], options['--tlskey']) + assert result.ca_cert == options['--tlscacert'] + assert result.verify is True + + def test_tls_client_missing_key(self): + options = {'--tlscert': self.client_cert} + with pytest.raises(docker.errors.TLSParameterError): + tls_config_from_options(options) + + options = {'--tlskey': self.key} + with pytest.raises(docker.errors.TLSParameterError): + tls_config_from_options(options)