diff --git a/compose/config/config.py b/compose/config/config.py index 2a6e8437b..960c3c678 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -19,6 +19,7 @@ from ..const import COMPOSEFILE_V2_1 as V2_1 from ..const import COMPOSEFILE_V3_0 as V3_0 from ..const import COMPOSEFILE_V3_4 as V3_4 from ..utils import build_string_dict +from ..utils import json_hash from ..utils import parse_bytes from ..utils import parse_nanoseconds_int from ..utils import splitdrive @@ -922,10 +923,14 @@ class MergeDict(dict): self.base.get(field, default), self.override.get(field, default)) - def merge_mapping(self, field, parse_func): + def merge_mapping(self, field, parse_func=None): if not self.needs_merge(field): return + if parse_func is None: + def parse_func(m): + return m or {} + self[field] = parse_func(self.base.get(field)) self[field].update(parse_func(self.override.get(field))) @@ -957,7 +962,6 @@ def merge_service_dicts(base, override, version): md.merge_sequence('links', ServiceLink.parse) md.merge_sequence('secrets', types.ServiceSecret.parse) md.merge_sequence('configs', types.ServiceConfig.parse) - md.merge_mapping('deploy', parse_deploy) md.merge_mapping('extra_hosts', parse_extra_hosts) for field in ['volumes', 'devices']: @@ -976,6 +980,7 @@ def merge_service_dicts(base, override, version): merge_ports(md, base, override) md.merge_field('blkio_config', merge_blkio_config, default={}) md.merge_field('healthcheck', merge_healthchecks, default={}) + md.merge_field('deploy', merge_deploy, default={}) for field in set(ALLOWED_KEYS) - set(md): md.merge_scalar(field) @@ -1039,6 +1044,41 @@ def merge_build(output, base, override): return dict(md) +def merge_deploy(base, override): + md = MergeDict(base or {}, override or {}) + md.merge_scalar('mode') + md.merge_scalar('endpoint_mode') + md.merge_scalar('replicas') + md.merge_mapping('labels', parse_labels) + md.merge_mapping('update_config') + md.merge_mapping('restart_policy') + if md.needs_merge('resources'): + resources_md = MergeDict(md.base.get('resources') or {}, md.override.get('resources') or {}) + resources_md.merge_mapping('limits') + resources_md.merge_field('reservations', merge_reservations, default={}) + md['resources'] = dict(resources_md) + if md.needs_merge('placement'): + placement_md = MergeDict(md.base.get('placement') or {}, md.override.get('placement') or {}) + placement_md.merge_field('constraints', merge_unique_items_lists, default=[]) + placement_md.merge_field('preferences', merge_unique_objects_lists, default=[]) + md['placement'] = dict(placement_md) + + return dict(md) + + +def merge_reservations(base, override): + md = MergeDict(base, override) + md.merge_scalar('cpus') + md.merge_scalar('memory') + md.merge_sequence('generic_resources', types.GenericResource.parse) + return dict(md) + + +def merge_unique_objects_lists(base, override): + result = dict((json_hash(i), i) for i in base + override) + return [i[1] for i in sorted([(k, v) for k, v in result.items()], key=lambda x: x[0])] + + def merge_blkio_config(base, override): md = MergeDict(base, override) md.merge_scalar('weight') @@ -1125,7 +1165,6 @@ parse_sysctls = functools.partial(parse_dict_or_list, split_kv, 'sysctls') parse_depends_on = functools.partial( parse_dict_or_list, lambda k: (k, {'condition': 'service_started'}), 'depends_on' ) -parse_deploy = functools.partial(parse_dict_or_list, split_kv, 'deploy') def parse_flat_dict(d): diff --git a/compose/config/types.py b/compose/config/types.py index 72e68d345..b896b883f 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -413,6 +413,35 @@ class ServicePort(namedtuple('_ServicePort', 'target published protocol mode ext return normalize_port_dict(self.repr()) +class GenericResource(namedtuple('_GenericResource', 'kind value')): + @classmethod + def parse(cls, dct): + if 'discrete_resource_spec' not in dct: + raise ConfigurationError( + 'generic_resource entry must include a discrete_resource_spec key' + ) + if 'kind' not in dct['discrete_resource_spec']: + raise ConfigurationError( + 'generic_resource entry must include a discrete_resource_spec.kind subkey' + ) + return cls( + dct['discrete_resource_spec']['kind'], + dct['discrete_resource_spec'].get('value') + ) + + def repr(self): + return { + 'discrete_resource_spec': { + 'kind': self.kind, + 'value': self.value, + } + } + + @property + def merge_field(self): + return self.kind + + def normalize_port_dict(port): return '{external_ip}{has_ext_ip}{published}{is_pub}{target}/{protocol}'.format( published=port.get('published', ''), diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 7cb74c00a..a33080726 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -33,6 +33,7 @@ from compose.const import COMPOSEFILE_V3_0 as V3_0 from compose.const import COMPOSEFILE_V3_1 as V3_1 from compose.const import COMPOSEFILE_V3_2 as V3_2 from compose.const import COMPOSEFILE_V3_3 as V3_3 +from compose.const import COMPOSEFILE_V3_5 as V3_5 from compose.const import IS_WINDOWS_PLATFORM from tests import mock from tests import unittest @@ -2300,37 +2301,96 @@ class ConfigTest(unittest.TestCase): def test_merge_deploy_override(self): base = { - 'image': 'busybox', 'deploy': { - 'mode': 'global', - 'restart_policy': { - 'condition': 'on-failure' - }, + 'endpoint_mode': 'vip', + 'labels': ['com.docker.compose.a=1', 'com.docker.compose.b=2'], + 'mode': 'replicated', 'placement': { 'constraints': [ - 'node.role == manager' + 'node.role == manager', 'engine.labels.aws == true' + ], + 'preferences': [ + {'spread': 'node.labels.zone'}, {'spread': 'x.d.z'} ] - } - } + }, + 'replicas': 3, + 'resources': { + 'limits': {'cpus': '0.50', 'memory': '50m'}, + 'reservations': { + 'cpus': '0.1', + 'generic_resources': [ + {'discrete_resource_spec': {'kind': 'abc', 'value': 123}} + ], + 'memory': '15m' + } + }, + 'restart_policy': {'condition': 'any', 'delay': '10s'}, + 'update_config': {'delay': '10s', 'max_failure_ratio': 0.3} + }, + 'image': 'hello-world' } override = { 'deploy': { - 'mode': 'replicated', - 'restart_policy': { - 'condition': 'any' - } + 'labels': { + 'com.docker.compose.b': '21', 'com.docker.compose.c': '3' + }, + 'placement': { + 'constraints': ['node.role == worker', 'engine.labels.dev == true'], + 'preferences': [{'spread': 'node.labels.zone'}, {'spread': 'x.d.s'}] + }, + 'resources': { + 'limits': {'memory': '200m'}, + 'reservations': { + 'cpus': '0.78', + 'generic_resources': [ + {'discrete_resource_spec': {'kind': 'abc', 'value': 134}}, + {'discrete_resource_spec': {'kind': 'xyz', 'value': 0.1}} + ] + } + }, + 'restart_policy': {'condition': 'on-failure', 'max_attempts': 42}, + 'update_config': {'max_failure_ratio': 0.712, 'parallelism': 4} } } - actual = config.merge_service_dicts(base, override, V3_0) + actual = config.merge_service_dicts(base, override, V3_5) assert actual['deploy'] == { 'mode': 'replicated', - 'restart_policy': { - 'condition': 'any' + 'endpoint_mode': 'vip', + 'labels': { + 'com.docker.compose.a': '1', + 'com.docker.compose.b': '21', + 'com.docker.compose.c': '3' }, 'placement': { 'constraints': [ - 'node.role == manager' + 'engine.labels.aws == true', 'engine.labels.dev == true', + 'node.role == manager', 'node.role == worker' + ], + 'preferences': [ + {'spread': 'node.labels.zone'}, {'spread': 'x.d.s'}, {'spread': 'x.d.z'} ] + }, + 'replicas': 3, + 'resources': { + 'limits': {'cpus': '0.50', 'memory': '200m'}, + 'reservations': { + 'cpus': '0.78', + 'memory': '15m', + 'generic_resources': [ + {'discrete_resource_spec': {'kind': 'abc', 'value': 134}}, + {'discrete_resource_spec': {'kind': 'xyz', 'value': 0.1}}, + ] + } + }, + 'restart_policy': { + 'condition': 'on-failure', + 'delay': '10s', + 'max_attempts': 42, + }, + 'update_config': { + 'max_failure_ratio': 0.712, + 'delay': '10s', + 'parallelism': 4 } }