Merge pull request #5448 from docker/custom_resource_names

Add support for custom names for networks, secrets, configs

compose/config/config.py

@@ -410,12 +410,11 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None):
 
             external = config.get('external')
             if external:
-                name_field = 'name' if entity_type == 'Volume' else 'external_name'
                 validate_external(entity_type, name, config, config_file.version)
                 if isinstance(external, dict):
-                    config[name_field] = external.get('name')
+                    config['name'] = external.get('name')
                 elif not config.get('name'):
-                    config[name_field] = name
+                    config['name'] = name
 
             if 'driver_opts' in config:
                 config['driver_opts'] = build_string_dict(

compose/config/config_schema_v2.1.json

@@ -350,7 +350,8 @@
         },
         "internal": {"type": "boolean"},
         "enable_ipv6": {"type": "boolean"},
-        "labels": {"$ref": "#/definitions/list_or_dict"}
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "name": {"type": "string"}
       },
       "additionalProperties": false
     },

compose/config/config_schema_v2.2.json

@@ -357,7 +357,8 @@
         },
         "internal": {"type": "boolean"},
         "enable_ipv6": {"type": "boolean"},
-        "labels": {"$ref": "#/definitions/list_or_dict"}
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "name": {"type": "string"}
       },
       "additionalProperties": false
     },

compose/config/config_schema_v2.3.json

@@ -393,7 +393,8 @@
         },
         "internal": {"type": "boolean"},
         "enable_ipv6": {"type": "boolean"},
-        "labels": {"$ref": "#/definitions/list_or_dict"}
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "name": {"type": "string"}
       },
       "additionalProperties": false
     },

compose/config/config_schema_v3.5.json

@@ -64,6 +64,7 @@
     }
   },
 
+  "patternProperties": {"^x-": {}},
   "additionalProperties": false,
 
   "definitions": {
@@ -154,6 +155,7 @@
         "hostname": {"type": "string"},
         "image": {"type": "string"},
         "ipc": {"type": "string"},
+        "isolation": {"type": "string"},
         "labels": {"$ref": "#/definitions/list_or_dict"},
         "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
 
@@ -281,7 +283,6 @@
               {
                 "type": "object",
                 "required": ["type"],
-                "additionalProperties": false,
                 "properties": {
                   "type": {"type": "string"},
                   "source": {"type": "string"},
@@ -300,7 +301,8 @@
                       "nocopy": {"type": "boolean"}
                     }
                   }
-                }
+                },
+                "additionalProperties": false
               }
             ],
             "uniqueItems": true
@@ -317,7 +319,7 @@
       "additionalProperties": false,
       "properties": {
         "disable": {"type": "boolean"},
-        "interval": {"type": "string"},
+        "interval": {"type": "string", "format": "duration"},
         "retries": {"type": "number"},
         "test": {
           "oneOf": [
@@ -325,7 +327,8 @@
             {"type": "array", "items": {"type": "string"}}
           ]
         },
-        "timeout": {"type": "string"}
+        "timeout": {"type": "string", "format": "duration"},
+        "start_period": {"type": "string", "format": "duration"}
       }
     },
     "deployment": {
@@ -353,8 +356,23 @@
         "resources": {
           "type": "object",
           "properties": {
-            "limits": {"$ref": "#/definitions/resource"},
+            "limits": {
-            "reservations": {"$ref": "#/definitions/resource"}
+              "type": "object",
+              "properties": {
+                "cpus": {"type": "string"},
+                "memory": {"type": "string"}
+              },
+              "additionalProperties": false
+            },
+            "reservations": {
+              "type": "object",
+              "properties": {
+                "cpus": {"type": "string"},
+                "memory": {"type": "string"},
+                "generic_resources": {"$ref": "#/definitions/generic_resources"}
+              },
+              "additionalProperties": false
+            }
           },
           "additionalProperties": false
         },
@@ -389,20 +407,30 @@
       "additionalProperties": false
     },
 
-    "resource": {
+    "generic_resources": {
-      "id": "#/definitions/resource",
+      "id": "#/definitions/generic_resources",
+      "type": "array",
+      "items": {
         "type": "object",
         "properties": {
-        "cpus": {"type": "string"},
+          "discrete_resource_spec": {
-        "memory": {"type": "string"}
+            "type": "object",
+            "properties": {
+              "kind": {"type": "string"},
+              "value": {"type": "number"}
             },
             "additionalProperties": false
+          }
+        },
+        "additionalProperties": false
+      }
     },
 
     "network": {
       "id": "#/definitions/network",
       "type": ["object", "null"],
       "properties": {
+        "name": {"type": "string"},
         "driver": {"type": "string"},
         "driver_opts": {
           "type": "object",
@@ -469,6 +497,7 @@
       "id": "#/definitions/secret",
       "type": "object",
       "properties": {
+        "name": {"type": "string"},
         "file": {"type": "string"},
         "external": {
           "type": ["boolean", "object"],
@@ -485,6 +514,7 @@
       "id": "#/definitions/config",
       "type": "object",
       "properties": {
+        "name": {"type": "string"},
         "file": {"type": "string"},
         "external": {
           "type": ["boolean", "object"],

compose/config/serialize.py

@@ -11,6 +11,7 @@ from compose.const import COMPOSEFILE_V2_3 as V2_3
 from compose.const import COMPOSEFILE_V3_0 as V3_0
 from compose.const import COMPOSEFILE_V3_2 as V3_2
 from compose.const import COMPOSEFILE_V3_4 as V3_4
+from compose.const import COMPOSEFILE_V3_5 as V3_5
 
 
 def serialize_config_type(dumper, data):
@@ -58,6 +59,7 @@ def denormalize_config(config, image_digests=None):
         service_dict.pop('name'): service_dict
         for service_dict in denormalized_services
     }
+
     for key in ('networks', 'volumes', 'secrets', 'configs'):
         config_dict = getattr(config, key)
         if not config_dict:
@@ -68,7 +70,8 @@ def denormalize_config(config, image_digests=None):
                 del conf['external_name']
 
             if 'name' in conf:
-                if config.version < V2_1 or (config.version >= V3_0 and config.version < V3_4):
+                if config.version < V2_1 or (
+                        config.version >= V3_0 and config.version < v3_introduced_name_key(key)):
                     del conf['name']
                 elif 'external' in conf:
                     conf['external'] = True
@@ -76,6 +79,12 @@ def denormalize_config(config, image_digests=None):
     return result
 
 
+def v3_introduced_name_key(key):
+    if key == 'volumes':
+        return V3_4
+    return V3_5
+
+
 def serialize_config(config, image_digests=None):
     return yaml.safe_dump(
         denormalize_config(config, image_digests),

compose/config/types.py

@@ -293,17 +293,18 @@ class ServiceLink(namedtuple('_ServiceLink', 'target alias')):
         return self.alias
 
 
-class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode')):
+class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode name')):
     @classmethod
     def parse(cls, spec):
         if isinstance(spec, six.string_types):
-            return cls(spec, None, None, None, None)
+            return cls(spec, None, None, None, None, None)
         return cls(
             spec.get('source'),
             spec.get('target'),
             spec.get('uid'),
             spec.get('gid'),
             spec.get('mode'),
+            spec.get('name')
         )
 
     @property

compose/network.py

@@ -25,21 +25,22 @@ OPTS_EXCEPTIONS = [
 
 class Network(object):
     def __init__(self, client, project, name, driver=None, driver_opts=None,
-                 ipam=None, external_name=None, internal=False, enable_ipv6=False,
+                 ipam=None, external=False, internal=False, enable_ipv6=False,
-                 labels=None):
+                 labels=None, custom_name=False):
         self.client = client
         self.project = project
         self.name = name
         self.driver = driver
         self.driver_opts = driver_opts
         self.ipam = create_ipam_config_from_dict(ipam)
-        self.external_name = external_name
+        self.external = external
         self.internal = internal
         self.enable_ipv6 = enable_ipv6
         self.labels = labels
+        self.custom_name = custom_name
 
     def ensure(self):
-        if self.external_name:
+        if self.external:
             try:
                 self.inspect()
                 log.debug(
@@ -51,7 +52,7 @@ class Network(object):
                     'Network {name} declared as external, but could'
                     ' not be found. Please create the network manually'
                     ' using `{command} {name}` and try again.'.format(
-                        name=self.external_name,
+                        name=self.full_name,
                         command='docker network create'
                     )
                 )
@@ -83,7 +84,7 @@ class Network(object):
             )
 
     def remove(self):
-        if self.external_name:
+        if self.external:
             log.info("Network %s is external, skipping", self.full_name)
             return
 
@@ -95,8 +96,8 @@ class Network(object):
 
     @property
     def full_name(self):
-        if self.external_name:
+        if self.custom_name:
-            return self.external_name
+            return self.name
         return '{0}_{1}'.format(self.project, self.name)
 
     @property
@@ -203,14 +204,16 @@ def build_networks(name, config_data, client):
     network_config = config_data.networks or {}
     networks = {
         network_name: Network(
-            client=client, project=name, name=network_name,
+            client=client, project=name,
+            name=data.get('name', network_name),
             driver=data.get('driver'),
             driver_opts=data.get('driver_opts'),
             ipam=data.get('ipam'),
-            external_name=data.get('external_name'),
+            external=bool(data.get('external', False)),
             internal=data.get('internal'),
             enable_ipv6=data.get('enable_ipv6'),
             labels=data.get('labels'),
+            custom_name=data.get('name') is not None,
         )
         for network_name, data in network_config.items()
     }

compose/project.py

@@ -648,7 +648,7 @@ def get_secrets(service, service_secrets, secret_defs):
                 "Service \"{service}\" uses an undefined secret \"{secret}\" "
                 .format(service=service, secret=secret.source))
 
-        if secret_def.get('external_name'):
+        if secret_def.get('external'):
             log.warn("Service \"{service}\" uses secret \"{secret}\" which is external. "
                      "External secrets are not available to containers created by "
                      "docker-compose.".format(service=service, secret=secret.source))

docker-compose.spec

@@ -67,6 +67,11 @@ exe = EXE(pyz,
                 'compose/config/config_schema_v3.4.json',
                 'DATA'
             ),
+            (
+                'compose/config/config_schema_v3.5.json',
+                'compose/config/config_schema_v3.5.json',
+                'DATA'
+            ),
             (
                 'compose/GITSHA',
                 'compose/GITSHA',

tests/acceptance/cli_test.py

@@ -350,6 +350,22 @@ class CLITestCase(DockerClientTestCase):
             }
         }
 
+    def test_config_external_network_v3_5(self):
+        self.base_dir = 'tests/fixtures/networks'
+        result = self.dispatch(['-f', 'external-networks-v3-5.yml', 'config'])
+        json_result = yaml.load(result.stdout)
+        assert 'networks' in json_result
+        assert json_result['networks'] == {
+            'foo': {
+                'external': True,
+                'name': 'some_foo',
+            },
+            'bar': {
+                'external': True,
+                'name': 'some_bar',
+            },
+        }
+
     def test_config_v1(self):
         self.base_dir = 'tests/fixtures/v1-config'
         result = self.dispatch(['config'])

tests/fixtures/networks/external-networks-v3-5.yml

@@ -0,0 +1,17 @@
+version: "3.5"
+
+services:
+  web:
+    image: busybox
+    command: top
+    networks:
+      - foo
+      - bar
+
+networks:
+  foo:
+    external: true
+    name: some_foo
+  bar:
+    external:
+      name: some_bar

tests/integration/project_test.py

@@ -953,6 +953,43 @@ class ProjectTest(DockerClientTestCase):
         assert 'LinkLocalIPs' in ipam_config
         assert ipam_config['LinkLocalIPs'] == ['169.254.8.8']
 
+    @v2_1_only()
+    def test_up_with_custom_name_resources(self):
+        config_data = build_config(
+            version=V2_2,
+            services=[{
+                'name': 'web',
+                'volumes': [VolumeSpec.parse('foo:/container-path')],
+                'networks': {'foo': {}},
+                'image': 'busybox:latest'
+            }],
+            networks={
+                'foo': {
+                    'name': 'zztop',
+                    'labels': {'com.docker.compose.test_value': 'sharpdressedman'}
+                }
+            },
+            volumes={
+                'foo': {
+                    'name': 'acdc',
+                    'labels': {'com.docker.compose.test_value': 'thefuror'}
+                }
+            }
+        )
+
+        project = Project.from_config(
+            client=self.client,
+            name='composetest',
+            config_data=config_data
+        )
+
+        project.up(detached=True)
+        network = [n for n in self.client.networks() if n['Name'] == 'zztop'][0]
+        volume = [v for v in self.client.volumes()['Volumes'] if v['Name'] == 'acdc'][0]
+
+        assert network['Labels']['com.docker.compose.test_value'] == 'sharpdressedman'
+        assert volume['Labels']['com.docker.compose.test_value'] == 'thefuror'
+
     @v2_1_only()
     def test_up_with_isolation(self):
         self.require_api_version('1.24')

tests/unit/config/config_test.py

@@ -432,6 +432,40 @@ class ConfigTest(unittest.TestCase):
                 'label_key': 'label_val'
             }
 
+    def test_load_config_custom_resource_names(self):
+        base_file = config.ConfigFile(
+            'base.yaml', {
+                'version': '3.5',
+                'volumes': {
+                    'abc': {
+                        'name': 'xyz'
+                    }
+                },
+                'networks': {
+                    'abc': {
+                        'name': 'xyz'
+                    }
+                },
+                'secrets': {
+                    'abc': {
+                        'name': 'xyz'
+                    }
+                },
+                'configs': {
+                    'abc': {
+                        'name': 'xyz'
+                    }
+                }
+            }
+        )
+        details = config.ConfigDetails('.', [base_file])
+        loaded_config = config.load(details)
+
+        assert loaded_config.networks['abc'] == {'name': 'xyz'}
+        assert loaded_config.volumes['abc'] == {'name': 'xyz'}
+        assert loaded_config.secrets['abc']['name'] == 'xyz'
+        assert loaded_config.configs['abc']['name'] == 'xyz'
+
     def test_load_config_volume_and_network_labels(self):
         base_file = config.ConfigFile(
             'base.yaml',
@@ -2539,8 +2573,8 @@ class ConfigTest(unittest.TestCase):
                 'name': 'web',
                 'image': 'example/web',
                 'secrets': [
-                    types.ServiceSecret('one', None, None, None, None),
+                    types.ServiceSecret('one', None, None, None, None, None),
-                    types.ServiceSecret('source', 'target', '100', '200', 0o777),
+                    types.ServiceSecret('source', 'target', '100', '200', 0o777, None),
                 ],
             },
         ]
@@ -2586,8 +2620,8 @@ class ConfigTest(unittest.TestCase):
                 'name': 'web',
                 'image': 'example/web',
                 'secrets': [
-                    types.ServiceSecret('one', None, None, None, None),
+                    types.ServiceSecret('one', None, None, None, None, None),
-                    types.ServiceSecret('source', 'target', '100', '200', 0o777),
+                    types.ServiceSecret('source', 'target', '100', '200', 0o777, None),
                 ],
             },
         ]
@@ -2624,8 +2658,8 @@ class ConfigTest(unittest.TestCase):
                 'name': 'web',
                 'image': 'example/web',
                 'configs': [
-                    types.ServiceConfig('one', None, None, None, None),
+                    types.ServiceConfig('one', None, None, None, None, None),
-                    types.ServiceConfig('source', 'target', '100', '200', 0o777),
+                    types.ServiceConfig('source', 'target', '100', '200', 0o777, None),
                 ],
             },
         ]
@@ -2671,8 +2705,8 @@ class ConfigTest(unittest.TestCase):
                 'name': 'web',
                 'image': 'example/web',
                 'configs': [
-                    types.ServiceConfig('one', None, None, None, None),
+                    types.ServiceConfig('one', None, None, None, None, None),
-                    types.ServiceConfig('source', 'target', '100', '200', 0o777),
+                    types.ServiceConfig('source', 'target', '100', '200', 0o777, None),
                 ],
             },
         ]
@@ -3131,7 +3165,7 @@ class InterpolationTest(unittest.TestCase):
         assert config_dict.secrets == {
             'secretdata': {
                 'external': {'name': 'baz.bar'},
-                'external_name': 'baz.bar'
+                'name': 'baz.bar'
             }
         }
 
@@ -3149,7 +3183,7 @@ class InterpolationTest(unittest.TestCase):
         assert config_dict.configs == {
             'configdata': {
                 'external': {'name': 'baz.bar'},
-                'external_name': 'baz.bar'
+                'name': 'baz.bar'
             }
         }