Advanced merge for deploy dict in v3 files

Signed-off-by: Joffrey F <joffrey@docker.com>

compose/config/config.py

@@ -19,6 +19,7 @@ from ..const import COMPOSEFILE_V2_1 as V2_1
 from ..const import COMPOSEFILE_V3_0 as V3_0
 from ..const import COMPOSEFILE_V3_4 as V3_4
 from ..utils import build_string_dict
+from ..utils import json_hash
 from ..utils import parse_bytes
 from ..utils import parse_nanoseconds_int
 from ..utils import splitdrive
@@ -922,10 +923,14 @@ class MergeDict(dict):
             self.base.get(field, default),
             self.override.get(field, default))
 
-    def merge_mapping(self, field, parse_func):
+    def merge_mapping(self, field, parse_func=None):
         if not self.needs_merge(field):
             return
 
+        if parse_func is None:
+            def parse_func(m):
+                return m or {}
+
         self[field] = parse_func(self.base.get(field))
         self[field].update(parse_func(self.override.get(field)))
 
@@ -957,7 +962,6 @@ def merge_service_dicts(base, override, version):
     md.merge_sequence('links', ServiceLink.parse)
     md.merge_sequence('secrets', types.ServiceSecret.parse)
     md.merge_sequence('configs', types.ServiceConfig.parse)
-    md.merge_mapping('deploy', parse_deploy)
     md.merge_mapping('extra_hosts', parse_extra_hosts)
 
     for field in ['volumes', 'devices']:
@@ -976,6 +980,7 @@ def merge_service_dicts(base, override, version):
     merge_ports(md, base, override)
     md.merge_field('blkio_config', merge_blkio_config, default={})
     md.merge_field('healthcheck', merge_healthchecks, default={})
+    md.merge_field('deploy', merge_deploy, default={})
 
     for field in set(ALLOWED_KEYS) - set(md):
         md.merge_scalar(field)
@@ -1039,6 +1044,41 @@ def merge_build(output, base, override):
     return dict(md)
 
 
+def merge_deploy(base, override):
+    md = MergeDict(base or {}, override or {})
+    md.merge_scalar('mode')
+    md.merge_scalar('endpoint_mode')
+    md.merge_scalar('replicas')
+    md.merge_mapping('labels', parse_labels)
+    md.merge_mapping('update_config')
+    md.merge_mapping('restart_policy')
+    if md.needs_merge('resources'):
+        resources_md = MergeDict(md.base.get('resources') or {}, md.override.get('resources') or {})
+        resources_md.merge_mapping('limits')
+        resources_md.merge_field('reservations', merge_reservations, default={})
+        md['resources'] = dict(resources_md)
+    if md.needs_merge('placement'):
+        placement_md = MergeDict(md.base.get('placement') or {}, md.override.get('placement') or {})
+        placement_md.merge_field('constraints', merge_unique_items_lists, default=[])
+        placement_md.merge_field('preferences', merge_unique_objects_lists, default=[])
+        md['placement'] = dict(placement_md)
+
+    return dict(md)
+
+
+def merge_reservations(base, override):
+    md = MergeDict(base, override)
+    md.merge_scalar('cpus')
+    md.merge_scalar('memory')
+    md.merge_sequence('generic_resources', types.GenericResource.parse)
+    return dict(md)
+
+
+def merge_unique_objects_lists(base, override):
+    result = dict((json_hash(i), i) for i in base + override)
+    return [i[1] for i in sorted([(k, v) for k, v in result.items()], key=lambda x: x[0])]
+
+
 def merge_blkio_config(base, override):
     md = MergeDict(base, override)
     md.merge_scalar('weight')
@@ -1125,7 +1165,6 @@ parse_sysctls = functools.partial(parse_dict_or_list, split_kv, 'sysctls')
 parse_depends_on = functools.partial(
     parse_dict_or_list, lambda k: (k, {'condition': 'service_started'}), 'depends_on'
 )
-parse_deploy = functools.partial(parse_dict_or_list, split_kv, 'deploy')
 
 
 def parse_flat_dict(d):

compose/config/types.py

@@ -413,6 +413,35 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext
         return normalize_port_dict(self.repr())
 
 
+class GenericResource(namedtuple('_GenericResource', 'kind value')):
+    @classmethod
+    def parse(cls, dct):
+        if 'discrete_resource_spec' not in dct:
+            raise ConfigurationError(
+                'generic_resource entry must include a discrete_resource_spec key'
+            )
+        if 'kind' not in dct['discrete_resource_spec']:
+            raise ConfigurationError(
+                'generic_resource entry must include a discrete_resource_spec.kind subkey'
+            )
+        return cls(
+            dct['discrete_resource_spec']['kind'],
+            dct['discrete_resource_spec'].get('value')
+        )
+
+    def repr(self):
+        return {
+            'discrete_resource_spec': {
+                'kind': self.kind,
+                'value': self.value,
+            }
+        }
+
+    @property
+    def merge_field(self):
+        return self.kind
+
+
 def normalize_port_dict(port):
     return '{external_ip}{has_ext_ip}{published}{is_pub}{target}/{protocol}'.format(
         published=port.get('published', ''),

tests/unit/config/config_test.py

@@ -33,6 +33,7 @@ from compose.const import COMPOSEFILE_V3_0 as V3_0
 from compose.const import COMPOSEFILE_V3_1 as V3_1
 from compose.const import COMPOSEFILE_V3_2 as V3_2
 from compose.const import COMPOSEFILE_V3_3 as V3_3
+from compose.const import COMPOSEFILE_V3_5 as V3_5
 from compose.const import IS_WINDOWS_PLATFORM
 from tests import mock
 from tests import unittest
@@ -2300,37 +2301,96 @@ class ConfigTest(unittest.TestCase):
 
     def test_merge_deploy_override(self):
         base = {
-            'image': 'busybox',
             'deploy': {
-                'mode': 'global',
+                'endpoint_mode': 'vip',
-                'restart_policy': {
+                'labels': ['com.docker.compose.a=1', 'com.docker.compose.b=2'],
-                    'condition': 'on-failure'
+                'mode': 'replicated',
-                },
                 'placement': {
                     'constraints': [
-                        'node.role == manager'
+                        'node.role == manager', 'engine.labels.aws == true'
+                    ],
+                    'preferences': [
+                        {'spread': 'node.labels.zone'}, {'spread': 'x.d.z'}
                     ]
+                },
+                'replicas': 3,
+                'resources': {
+                    'limits': {'cpus': '0.50', 'memory': '50m'},
+                    'reservations': {
+                        'cpus': '0.1',
+                        'generic_resources': [
+                            {'discrete_resource_spec': {'kind': 'abc', 'value': 123}}
+                        ],
+                        'memory': '15m'
                     }
-            }
+                },
+                'restart_policy': {'condition': 'any', 'delay': '10s'},
+                'update_config': {'delay': '10s', 'max_failure_ratio': 0.3}
+            },
+            'image': 'hello-world'
         }
         override = {
             'deploy': {
-                'mode': 'replicated',
+                'labels': {
-                'restart_policy': {
+                    'com.docker.compose.b': '21', 'com.docker.compose.c': '3'
-                    'condition': 'any'
+                },
+                'placement': {
+                    'constraints': ['node.role == worker', 'engine.labels.dev == true'],
+                    'preferences': [{'spread': 'node.labels.zone'}, {'spread': 'x.d.s'}]
+                },
+                'resources': {
+                    'limits': {'memory': '200m'},
+                    'reservations': {
+                        'cpus': '0.78',
+                        'generic_resources': [
+                            {'discrete_resource_spec': {'kind': 'abc', 'value': 134}},
+                            {'discrete_resource_spec': {'kind': 'xyz', 'value': 0.1}}
+                        ]
+                    }
+                },
+                'restart_policy': {'condition': 'on-failure', 'max_attempts': 42},
+                'update_config': {'max_failure_ratio': 0.712, 'parallelism': 4}
             }
         }
-        }
+        actual = config.merge_service_dicts(base, override, V3_5)
-        actual = config.merge_service_dicts(base, override, V3_0)
         assert actual['deploy'] == {
             'mode': 'replicated',
-            'restart_policy': {
+            'endpoint_mode': 'vip',
-                'condition': 'any'
+            'labels': {
+                'com.docker.compose.a': '1',
+                'com.docker.compose.b': '21',
+                'com.docker.compose.c': '3'
             },
             'placement': {
                 'constraints': [
-                    'node.role == manager'
+                    'engine.labels.aws == true', 'engine.labels.dev == true',
+                    'node.role == manager', 'node.role == worker'
+                ],
+                'preferences': [
+                    {'spread': 'node.labels.zone'}, {'spread': 'x.d.s'}, {'spread': 'x.d.z'}
                 ]
+            },
+            'replicas': 3,
+            'resources': {
+                'limits': {'cpus': '0.50', 'memory': '200m'},
+                'reservations': {
+                    'cpus': '0.78',
+                    'memory': '15m',
+                    'generic_resources': [
+                        {'discrete_resource_spec': {'kind': 'abc', 'value': 134}},
+                        {'discrete_resource_spec': {'kind': 'xyz', 'value': 0.1}},
+                    ]
+                }
+            },
+            'restart_policy': {
+                'condition': 'on-failure',
+                'delay': '10s',
+                'max_attempts': 42,
+            },
+            'update_config': {
+                'max_failure_ratio': 0.712,
+                'delay': '10s',
+                'parallelism': 4
             }
         }