From 8155ddc7add99edd1c9a366f44c65ba4d62589a3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 6 Dec 2017 16:48:14 -0800 Subject: [PATCH] Add support for custom names for networks, secrets, configs Finalize v3.5 schema Signed-off-by: Joffrey F --- compose/config/config.py | 5 +- compose/config/config_schema_v2.1.json | 3 +- compose/config/config_schema_v2.2.json | 3 +- compose/config/config_schema_v2.3.json | 3 +- compose/config/config_schema_v3.5.json | 58 ++++++++++++++----- compose/config/serialize.py | 11 +++- compose/config/types.py | 5 +- compose/network.py | 23 ++++---- compose/project.py | 2 +- docker-compose.spec | 5 ++ tests/acceptance/cli_test.py | 16 +++++ .../networks/external-networks-v3-5.yml | 17 ++++++ tests/integration/project_test.py | 37 ++++++++++++ tests/unit/config/config_test.py | 54 +++++++++++++---- 14 files changed, 198 insertions(+), 44 deletions(-) create mode 100644 tests/fixtures/networks/external-networks-v3-5.yml diff --git a/compose/config/config.py b/compose/config/config.py index 9b4130536..98719d6ba 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -410,12 +410,11 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): external = config.get('external') if external: - name_field = 'name' if entity_type == 'Volume' else 'external_name' validate_external(entity_type, name, config, config_file.version) if isinstance(external, dict): - config[name_field] = external.get('name') + config['name'] = external.get('name') elif not config.get('name'): - config[name_field] = name + config['name'] = name if 'driver_opts' in config: config['driver_opts'] = build_string_dict( diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 6b74f0ed6..15b78e5db 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -350,7 +350,8 @@ }, "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v2.2.json b/compose/config/config_schema_v2.2.json index 21343b893..7a3eed0a9 100644 --- a/compose/config/config_schema_v2.2.json +++ b/compose/config/config_schema_v2.2.json @@ -357,7 +357,8 @@ }, "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index d50df3e81..7c0e54807 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -393,7 +393,8 @@ }, "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, - "labels": {"$ref": "#/definitions/list_or_dict"} + "labels": {"$ref": "#/definitions/list_or_dict"}, + "name": {"type": "string"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json index d94b3feb0..565da0193 100644 --- a/compose/config/config_schema_v3.5.json +++ b/compose/config/config_schema_v3.5.json @@ -64,6 +64,7 @@ } }, + "patternProperties": {"^x-": {}}, "additionalProperties": false, "definitions": { @@ -154,6 +155,7 @@ "hostname": {"type": "string"}, "image": {"type": "string"}, "ipc": {"type": "string"}, + "isolation": {"type": "string"}, "labels": {"$ref": "#/definitions/list_or_dict"}, "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, @@ -281,7 +283,6 @@ { "type": "object", "required": ["type"], - "additionalProperties": false, "properties": { "type": {"type": "string"}, "source": {"type": "string"}, @@ -300,7 +301,8 @@ "nocopy": {"type": "boolean"} } } - } + }, + "additionalProperties": false } ], "uniqueItems": true @@ -317,7 +319,7 @@ "additionalProperties": false, "properties": { "disable": {"type": "boolean"}, - "interval": {"type": "string"}, + "interval": {"type": "string", "format": "duration"}, "retries": {"type": "number"}, "test": { "oneOf": [ @@ -325,7 +327,8 @@ {"type": "array", "items": {"type": "string"}} ] }, - "timeout": {"type": "string"} + "timeout": {"type": "string", "format": "duration"}, + "start_period": {"type": "string", "format": "duration"} } }, "deployment": { @@ -353,8 +356,23 @@ "resources": { "type": "object", "properties": { - "limits": {"$ref": "#/definitions/resource"}, - "reservations": {"$ref": "#/definitions/resource"} + "limits": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + "reservations": { + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"}, + "generic_resources": {"$ref": "#/definitions/generic_resources"} + }, + "additionalProperties": false + } }, "additionalProperties": false }, @@ -389,20 +407,30 @@ "additionalProperties": false }, - "resource": { - "id": "#/definitions/resource", - "type": "object", - "properties": { - "cpus": {"type": "string"}, - "memory": {"type": "string"} - }, - "additionalProperties": false + "generic_resources": { + "id": "#/definitions/generic_resources", + "type": "array", + "items": { + "type": "object", + "properties": { + "discrete_resource_spec": { + "type": "object", + "properties": { + "kind": {"type": "string"}, + "value": {"type": "number"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } }, "network": { "id": "#/definitions/network", "type": ["object", "null"], "properties": { + "name": {"type": "string"}, "driver": {"type": "string"}, "driver_opts": { "type": "object", @@ -469,6 +497,7 @@ "id": "#/definitions/secret", "type": "object", "properties": { + "name": {"type": "string"}, "file": {"type": "string"}, "external": { "type": ["boolean", "object"], @@ -485,6 +514,7 @@ "id": "#/definitions/config", "type": "object", "properties": { + "name": {"type": "string"}, "file": {"type": "string"}, "external": { "type": ["boolean", "object"], diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 4982f8e34..3ab43fc59 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -11,6 +11,7 @@ from compose.const import COMPOSEFILE_V2_3 as V2_3 from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_4 as V3_4 +from compose.const import COMPOSEFILE_V3_5 as V3_5 def serialize_config_type(dumper, data): @@ -58,6 +59,7 @@ def denormalize_config(config, image_digests=None): service_dict.pop('name'): service_dict for service_dict in denormalized_services } + for key in ('networks', 'volumes', 'secrets', 'configs'): config_dict = getattr(config, key) if not config_dict: @@ -68,7 +70,8 @@ def denormalize_config(config, image_digests=None): del conf['external_name'] if 'name' in conf: - if config.version < V2_1 or (config.version >= V3_0 and config.version < V3_4): + if config.version < V2_1 or ( + config.version >= V3_0 and config.version < v3_introduced_name_key(key)): del conf['name'] elif 'external' in conf: conf['external'] = True @@ -76,6 +79,12 @@ def denormalize_config(config, image_digests=None): return result +def v3_introduced_name_key(key): + if key == 'volumes': + return V3_4 + return V3_5 + + def serialize_config(config, image_digests=None): return yaml.safe_dump( denormalize_config(config, image_digests), diff --git a/compose/config/types.py b/compose/config/types.py index c134bd7ca..daf25f700 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -293,17 +293,18 @@ class ServiceLink(namedtuple('_ServiceLink', 'target alias')): return self.alias -class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode')): +class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode name')): @classmethod def parse(cls, spec): if isinstance(spec, six.string_types): - return cls(spec, None, None, None, None) + return cls(spec, None, None, None, None, None) return cls( spec.get('source'), spec.get('target'), spec.get('uid'), spec.get('gid'), spec.get('mode'), + spec.get('name') ) @property diff --git a/compose/network.py b/compose/network.py index ee5939c15..95e2bf60e 100644 --- a/compose/network.py +++ b/compose/network.py @@ -25,21 +25,22 @@ OPTS_EXCEPTIONS = [ class Network(object): def __init__(self, client, project, name, driver=None, driver_opts=None, - ipam=None, external_name=None, internal=False, enable_ipv6=False, - labels=None): + ipam=None, external=False, internal=False, enable_ipv6=False, + labels=None, custom_name=False): self.client = client self.project = project self.name = name self.driver = driver self.driver_opts = driver_opts self.ipam = create_ipam_config_from_dict(ipam) - self.external_name = external_name + self.external = external self.internal = internal self.enable_ipv6 = enable_ipv6 self.labels = labels + self.custom_name = custom_name def ensure(self): - if self.external_name: + if self.external: try: self.inspect() log.debug( @@ -51,7 +52,7 @@ class Network(object): 'Network {name} declared as external, but could' ' not be found. Please create the network manually' ' using `{command} {name}` and try again.'.format( - name=self.external_name, + name=self.full_name, command='docker network create' ) ) @@ -83,7 +84,7 @@ class Network(object): ) def remove(self): - if self.external_name: + if self.external: log.info("Network %s is external, skipping", self.full_name) return @@ -95,8 +96,8 @@ class Network(object): @property def full_name(self): - if self.external_name: - return self.external_name + if self.custom_name: + return self.name return '{0}_{1}'.format(self.project, self.name) @property @@ -203,14 +204,16 @@ def build_networks(name, config_data, client): network_config = config_data.networks or {} networks = { network_name: Network( - client=client, project=name, name=network_name, + client=client, project=name, + name=data.get('name', network_name), driver=data.get('driver'), driver_opts=data.get('driver_opts'), ipam=data.get('ipam'), - external_name=data.get('external_name'), + external=bool(data.get('external', False)), internal=data.get('internal'), enable_ipv6=data.get('enable_ipv6'), labels=data.get('labels'), + custom_name=data.get('name') is not None, ) for network_name, data in network_config.items() } diff --git a/compose/project.py b/compose/project.py index 411576386..11ee4a0b7 100644 --- a/compose/project.py +++ b/compose/project.py @@ -648,7 +648,7 @@ def get_secrets(service, service_secrets, secret_defs): "Service \"{service}\" uses an undefined secret \"{secret}\" " .format(service=service, secret=secret.source)) - if secret_def.get('external_name'): + if secret_def.get('external'): log.warn("Service \"{service}\" uses secret \"{secret}\" which is external. " "External secrets are not available to containers created by " "docker-compose.".format(service=service, secret=secret.source)) diff --git a/docker-compose.spec b/docker-compose.spec index 9c46421f0..83d7389f3 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -67,6 +67,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.4.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.5.json', + 'compose/config/config_schema_v3.5.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ce0e75055..502907fe7 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -350,6 +350,22 @@ class CLITestCase(DockerClientTestCase): } } + def test_config_external_network_v3_5(self): + self.base_dir = 'tests/fixtures/networks' + result = self.dispatch(['-f', 'external-networks-v3-5.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'networks' in json_result + assert json_result['networks'] == { + 'foo': { + 'external': True, + 'name': 'some_foo', + }, + 'bar': { + 'external': True, + 'name': 'some_bar', + }, + } + def test_config_v1(self): self.base_dir = 'tests/fixtures/v1-config' result = self.dispatch(['config']) diff --git a/tests/fixtures/networks/external-networks-v3-5.yml b/tests/fixtures/networks/external-networks-v3-5.yml new file mode 100644 index 000000000..9ac7b14b5 --- /dev/null +++ b/tests/fixtures/networks/external-networks-v3-5.yml @@ -0,0 +1,17 @@ +version: "3.5" + +services: + web: + image: busybox + command: top + networks: + - foo + - bar + +networks: + foo: + external: true + name: some_foo + bar: + external: + name: some_bar diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 6686d96cc..82e0adab3 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -953,6 +953,43 @@ class ProjectTest(DockerClientTestCase): assert 'LinkLocalIPs' in ipam_config assert ipam_config['LinkLocalIPs'] == ['169.254.8.8'] + @v2_1_only() + def test_up_with_custom_name_resources(self): + config_data = build_config( + version=V2_2, + services=[{ + 'name': 'web', + 'volumes': [VolumeSpec.parse('foo:/container-path')], + 'networks': {'foo': {}}, + 'image': 'busybox:latest' + }], + networks={ + 'foo': { + 'name': 'zztop', + 'labels': {'com.docker.compose.test_value': 'sharpdressedman'} + } + }, + volumes={ + 'foo': { + 'name': 'acdc', + 'labels': {'com.docker.compose.test_value': 'thefuror'} + } + } + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data + ) + + project.up(detached=True) + network = [n for n in self.client.networks() if n['Name'] == 'zztop'][0] + volume = [v for v in self.client.volumes()['Volumes'] if v['Name'] == 'acdc'][0] + + assert network['Labels']['com.docker.compose.test_value'] == 'sharpdressedman' + assert volume['Labels']['com.docker.compose.test_value'] == 'thefuror' + @v2_1_only() def test_up_with_isolation(self): self.require_api_version('1.24') diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d519deb90..7029fcb08 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -432,6 +432,40 @@ class ConfigTest(unittest.TestCase): 'label_key': 'label_val' } + def test_load_config_custom_resource_names(self): + base_file = config.ConfigFile( + 'base.yaml', { + 'version': '3.5', + 'volumes': { + 'abc': { + 'name': 'xyz' + } + }, + 'networks': { + 'abc': { + 'name': 'xyz' + } + }, + 'secrets': { + 'abc': { + 'name': 'xyz' + } + }, + 'configs': { + 'abc': { + 'name': 'xyz' + } + } + } + ) + details = config.ConfigDetails('.', [base_file]) + loaded_config = config.load(details) + + assert loaded_config.networks['abc'] == {'name': 'xyz'} + assert loaded_config.volumes['abc'] == {'name': 'xyz'} + assert loaded_config.secrets['abc']['name'] == 'xyz' + assert loaded_config.configs['abc']['name'] == 'xyz' + def test_load_config_volume_and_network_labels(self): base_file = config.ConfigFile( 'base.yaml', @@ -2539,8 +2573,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'secrets': [ - types.ServiceSecret('one', None, None, None, None), - types.ServiceSecret('source', 'target', '100', '200', 0o777), + types.ServiceSecret('one', None, None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -2586,8 +2620,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'secrets': [ - types.ServiceSecret('one', None, None, None, None), - types.ServiceSecret('source', 'target', '100', '200', 0o777), + types.ServiceSecret('one', None, None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -2624,8 +2658,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'configs': [ - types.ServiceConfig('one', None, None, None, None), - types.ServiceConfig('source', 'target', '100', '200', 0o777), + types.ServiceConfig('one', None, None, None, None, None), + types.ServiceConfig('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -2671,8 +2705,8 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'image': 'example/web', 'configs': [ - types.ServiceConfig('one', None, None, None, None), - types.ServiceConfig('source', 'target', '100', '200', 0o777), + types.ServiceConfig('one', None, None, None, None, None), + types.ServiceConfig('source', 'target', '100', '200', 0o777, None), ], }, ] @@ -3131,7 +3165,7 @@ class InterpolationTest(unittest.TestCase): assert config_dict.secrets == { 'secretdata': { 'external': {'name': 'baz.bar'}, - 'external_name': 'baz.bar' + 'name': 'baz.bar' } } @@ -3149,7 +3183,7 @@ class InterpolationTest(unittest.TestCase): assert config_dict.configs == { 'configdata': { 'external': {'name': 'baz.bar'}, - 'external_name': 'baz.bar' + 'name': 'baz.bar' } }