From d4106679a6883addea940dcc8bbf424b90eed023 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 22 Feb 2018 16:18:44 -0800 Subject: [PATCH 1/2] Use configfile-provided proxy values to populate buildargs and env values Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 5 ++++- compose/service.py | 36 ++++++++++++++++++++++++++++++++++-- tests/unit/cli_test.py | 2 ++ tests/unit/project_test.py | 1 + tests/unit/service_test.py | 31 ++++++++++++++++++++----------- 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index cc8993d7f..a4609780f 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -117,4 +117,7 @@ def docker_client(environment, version=None, tls_config=None, host=None, kwargs['user_agent'] = generate_user_agent() - return APIClient(**kwargs) + client = APIClient(**kwargs) + client._original_base_url = kwargs.get('base_url') + + return client diff --git a/compose/service.py b/compose/service.py index f79a46962..50f9fd71b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -793,8 +793,12 @@ class Service(object): )) container_options['environment'] = merge_environment( - self.options.get('environment'), - override_options.get('environment')) + self._parse_proxy_config(), + merge_environment( + self.options.get('environment'), + override_options.get('environment') + ) + ) container_options['labels'] = merge_labels( self.options.get('labels'), @@ -963,6 +967,9 @@ class Service(object): if build_args_override: build_args.update(build_args_override) + for k, v in self._parse_proxy_config().items(): + build_args.setdefault(k, v) + # python2 os.stat() doesn't support unicode on some UNIX, so we # encode it to a bytestring to be safe path = build_opts.get('context') @@ -1142,6 +1149,31 @@ class Service(object): raise HealthCheckFailed(ctnr.short_id) return result + def _parse_proxy_config(self): + client = self.client + if 'proxies' not in client._general_configs: + return {} + docker_host = getattr(client, '_original_base_url', client.base_url) + proxy_config = client._general_configs['proxies'].get( + docker_host, client._general_configs['proxies'].get('default') + ) or {} + + permitted = { + 'ftpProxy': 'FTP_PROXY', + 'httpProxy': 'HTTP_PROXY', + 'httpsProxy': 'HTTPS_PROXY', + 'noProxy': 'NO_PROXY', + } + + result = {} + + for k, v in proxy_config.items(): + if k not in permitted: + continue + result[permitted[k]] = result[permitted[k].lower()] = v + + return result + def short_id_alias_exists(container, network): aliases = container.get( diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 6399bef89..cef53740d 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -102,6 +102,7 @@ class CLITestCase(unittest.TestCase): os.environ['COMPOSE_INTERACTIVE_NO_CLI'] = 'true' mock_client = mock.create_autospec(docker.APIClient) mock_client.api_version = DEFAULT_DOCKER_API_VERSION + mock_client._general_configs = {} project = Project.from_config( name='composetest', client=mock_client, @@ -136,6 +137,7 @@ class CLITestCase(unittest.TestCase): def test_run_service_with_restart_always(self): mock_client = mock.create_autospec(docker.APIClient) mock_client.api_version = DEFAULT_DOCKER_API_VERSION + mock_client._general_configs = {} project = Project.from_config( name='composetest', diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index a0291d9f9..b4994a99e 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -24,6 +24,7 @@ from compose.service import Service class ProjectTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client._general_configs = {} def test_from_config_v1(self): config = Config( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 002ae0c0d..884598b60 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -43,6 +43,7 @@ class ServiceTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION + self.mock_client._general_configs = {} def test_containers(self): service = Service('db', self.mock_client, 'myproject', image='foo') @@ -743,13 +744,16 @@ class ServiceTest(unittest.TestCase): 'for this service are created on a single host, the port will clash.'.format(name)) -class TestServiceNetwork(object): +class TestServiceNetwork(unittest.TestCase): + def setUp(self): + self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION + self.mock_client._general_configs = {} def test_connect_container_to_networks_short_aliase_exists(self): - mock_client = mock.create_autospec(docker.APIClient) service = Service( 'db', - mock_client, + self.mock_client, 'myproject', image='foo', networks={'project_default': {}}) @@ -768,8 +772,8 @@ class TestServiceNetwork(object): True) service.connect_container_to_networks(container) - assert not mock_client.disconnect_container_from_network.call_count - assert not mock_client.connect_container_to_network.call_count + assert not self.mock_client.disconnect_container_from_network.call_count + assert not self.mock_client.connect_container_to_network.call_count def sort_by_name(dictionary_list): @@ -814,6 +818,10 @@ class BuildUlimitsTestCase(unittest.TestCase): class NetTestCase(unittest.TestCase): + def setUp(self): + self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION + self.mock_client._general_configs = {} def test_network_mode(self): network_mode = NetworkMode('host') @@ -831,12 +839,11 @@ class NetTestCase(unittest.TestCase): def test_network_mode_service(self): container_id = 'bbbb' service_name = 'web' - mock_client = mock.create_autospec(docker.APIClient) - mock_client.containers.return_value = [ + self.mock_client.containers.return_value = [ {'Id': container_id, 'Name': container_id, 'Image': 'abcd'}, ] - service = Service(name=service_name, client=mock_client) + service = Service(name=service_name, client=self.mock_client) network_mode = ServiceNetworkMode(service) assert network_mode.id == service_name @@ -845,10 +852,9 @@ class NetTestCase(unittest.TestCase): def test_network_mode_service_no_containers(self): service_name = 'web' - mock_client = mock.create_autospec(docker.APIClient) - mock_client.containers.return_value = [] + self.mock_client.containers.return_value = [] - service = Service(name=service_name, client=mock_client) + service = Service(name=service_name, client=self.mock_client) network_mode = ServiceNetworkMode(service) assert network_mode.id == service_name @@ -884,6 +890,7 @@ class ServiceVolumesTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION + self.mock_client._general_configs = {} def test_build_volume_binding(self): binding = build_volume_binding(VolumeSpec.parse('/outside:/inside', True)) @@ -1118,6 +1125,8 @@ class ServiceVolumesTest(unittest.TestCase): class ServiceSecretTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.APIClient) + self.mock_client.api_version = DEFAULT_DOCKER_API_VERSION + self.mock_client._general_configs = {} def test_get_secret_volumes(self): secret1 = { From 5e4700b176bf36eaece1cfb6bb5c75a03d069d3f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 23 Feb 2018 14:32:43 -0800 Subject: [PATCH 2/2] Add proxy_config tests Signed-off-by: Joffrey F --- tests/unit/service_test.py | 143 +++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 884598b60..c315dcc4d 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -25,6 +25,7 @@ from compose.service import build_ulimits from compose.service import build_volume_binding from compose.service import BuildAction from compose.service import ContainerNetworkMode +from compose.service import format_environment from compose.service import formatted_ports from compose.service import get_container_data_volumes from compose.service import ImageType @@ -743,6 +744,148 @@ class ServiceTest(unittest.TestCase): 'The "{}" service specifies a port on the host. If multiple containers ' 'for this service are created on a single host, the port will clash.'.format(name)) + def test_parse_proxy_config(self): + default_proxy_config = { + 'httpProxy': 'http://proxy.mycorp.com:3128', + 'httpsProxy': 'https://user:password@proxy.mycorp.com:3129', + 'ftpProxy': 'http://ftpproxy.mycorp.com:21', + 'noProxy': '*.intra.mycorp.com', + } + + self.mock_client.base_url = 'http+docker://localunixsocket' + self.mock_client._general_configs = { + 'proxies': { + 'default': default_proxy_config, + } + } + + service = Service('foo', client=self.mock_client) + + assert service._parse_proxy_config() == { + 'HTTP_PROXY': default_proxy_config['httpProxy'], + 'http_proxy': default_proxy_config['httpProxy'], + 'HTTPS_PROXY': default_proxy_config['httpsProxy'], + 'https_proxy': default_proxy_config['httpsProxy'], + 'FTP_PROXY': default_proxy_config['ftpProxy'], + 'ftp_proxy': default_proxy_config['ftpProxy'], + 'NO_PROXY': default_proxy_config['noProxy'], + 'no_proxy': default_proxy_config['noProxy'], + } + + def test_parse_proxy_config_per_host(self): + default_proxy_config = { + 'httpProxy': 'http://proxy.mycorp.com:3128', + 'httpsProxy': 'https://user:password@proxy.mycorp.com:3129', + 'ftpProxy': 'http://ftpproxy.mycorp.com:21', + 'noProxy': '*.intra.mycorp.com', + } + host_specific_proxy_config = { + 'httpProxy': 'http://proxy.example.com:3128', + 'httpsProxy': 'https://user:password@proxy.example.com:3129', + 'ftpProxy': 'http://ftpproxy.example.com:21', + 'noProxy': '*.intra.example.com' + } + + self.mock_client.base_url = 'http+docker://localunixsocket' + self.mock_client._general_configs = { + 'proxies': { + 'default': default_proxy_config, + 'tcp://example.docker.com:2376': host_specific_proxy_config, + } + } + + service = Service('foo', client=self.mock_client) + + assert service._parse_proxy_config() == { + 'HTTP_PROXY': default_proxy_config['httpProxy'], + 'http_proxy': default_proxy_config['httpProxy'], + 'HTTPS_PROXY': default_proxy_config['httpsProxy'], + 'https_proxy': default_proxy_config['httpsProxy'], + 'FTP_PROXY': default_proxy_config['ftpProxy'], + 'ftp_proxy': default_proxy_config['ftpProxy'], + 'NO_PROXY': default_proxy_config['noProxy'], + 'no_proxy': default_proxy_config['noProxy'], + } + + self.mock_client._original_base_url = 'tcp://example.docker.com:2376' + + assert service._parse_proxy_config() == { + 'HTTP_PROXY': host_specific_proxy_config['httpProxy'], + 'http_proxy': host_specific_proxy_config['httpProxy'], + 'HTTPS_PROXY': host_specific_proxy_config['httpsProxy'], + 'https_proxy': host_specific_proxy_config['httpsProxy'], + 'FTP_PROXY': host_specific_proxy_config['ftpProxy'], + 'ftp_proxy': host_specific_proxy_config['ftpProxy'], + 'NO_PROXY': host_specific_proxy_config['noProxy'], + 'no_proxy': host_specific_proxy_config['noProxy'], + } + + def test_build_service_with_proxy_config(self): + default_proxy_config = { + 'httpProxy': 'http://proxy.mycorp.com:3128', + 'httpsProxy': 'https://user:password@proxy.example.com:3129', + } + buildargs = { + 'HTTPS_PROXY': 'https://rdcf.th08.jp:8911', + 'https_proxy': 'https://rdcf.th08.jp:8911', + } + self.mock_client._general_configs = { + 'proxies': { + 'default': default_proxy_config, + } + } + self.mock_client.base_url = 'http+docker://localunixsocket' + self.mock_client.build.return_value = [ + b'{"stream": "Successfully built 12345"}', + ] + + service = Service('foo', client=self.mock_client, build={'context': '.', 'args': buildargs}) + service.build() + + assert self.mock_client.build.call_count == 1 + assert self.mock_client.build.call_args[1]['buildargs'] == { + 'HTTP_PROXY': default_proxy_config['httpProxy'], + 'http_proxy': default_proxy_config['httpProxy'], + 'HTTPS_PROXY': buildargs['HTTPS_PROXY'], + 'https_proxy': buildargs['HTTPS_PROXY'], + } + + def test_get_create_options_with_proxy_config(self): + default_proxy_config = { + 'httpProxy': 'http://proxy.mycorp.com:3128', + 'httpsProxy': 'https://user:password@proxy.mycorp.com:3129', + 'ftpProxy': 'http://ftpproxy.mycorp.com:21', + } + self.mock_client._general_configs = { + 'proxies': { + 'default': default_proxy_config, + } + } + self.mock_client.base_url = 'http+docker://localunixsocket' + + override_options = { + 'environment': { + 'FTP_PROXY': 'ftp://xdge.exo.au:21', + 'ftp_proxy': 'ftp://xdge.exo.au:21', + } + } + environment = { + 'HTTPS_PROXY': 'https://rdcf.th08.jp:8911', + 'https_proxy': 'https://rdcf.th08.jp:8911', + } + + service = Service('foo', client=self.mock_client, environment=environment) + + create_opts = service._get_container_create_options(override_options, 1) + assert set(create_opts['environment']) == set(format_environment({ + 'HTTP_PROXY': default_proxy_config['httpProxy'], + 'http_proxy': default_proxy_config['httpProxy'], + 'HTTPS_PROXY': environment['HTTPS_PROXY'], + 'https_proxy': environment['HTTPS_PROXY'], + 'FTP_PROXY': override_options['environment']['FTP_PROXY'], + 'ftp_proxy': override_options['environment']['FTP_PROXY'], + })) + class TestServiceNetwork(unittest.TestCase): def setUp(self):