diff --git a/compose/config/config.py b/compose/config/config.py index d5aaf9538..a9f82a29d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -519,13 +519,13 @@ def process_config_file(config_file, environment, service_name=None): processed_config['secrets'] = interpolate_config_section( config_file, config_file.get_secrets(), - 'secrets', + 'secret', environment) if config_file.version >= const.COMPOSEFILE_V3_3: processed_config['configs'] = interpolate_config_section( config_file, config_file.get_configs(), - 'configs', + 'config', environment ) else: diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index df9c988e7..9d7e428c9 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging +import re from string import Template import six @@ -44,9 +45,13 @@ def interpolate_environment_variables(version, config, section, environment): ) +def get_config_path(config_key, section, name): + return '{}.{}.{}'.format(section, name, config_key) + + def interpolate_value(name, config_key, value, section, interpolator): try: - return recursive_interpolate(value, interpolator) + return recursive_interpolate(value, interpolator, get_config_path(config_key, section, name)) except InvalidInterpolation as e: raise ConfigurationError( 'Invalid interpolation format for "{config_key}" option ' @@ -57,16 +62,19 @@ def interpolate_value(name, config_key, value, section, interpolator): string=e.string)) -def recursive_interpolate(obj, interpolator): +def recursive_interpolate(obj, interpolator, config_path): + def append(config_path, key): + return '{}.{}'.format(config_path, key) + if isinstance(obj, six.string_types): - return interpolator.interpolate(obj) + return converter.convert(config_path, interpolator.interpolate(obj)) if isinstance(obj, dict): return dict( - (key, recursive_interpolate(val, interpolator)) + (key, recursive_interpolate(val, interpolator, append(config_path, key))) for (key, val) in obj.items() ) if isinstance(obj, list): - return [recursive_interpolate(val, interpolator) for val in obj] + return [recursive_interpolate(val, interpolator, config_path) for val in obj] return obj @@ -100,3 +108,80 @@ class TemplateWithDefaults(Template): class InvalidInterpolation(Exception): def __init__(self, string): self.string = string + + +PATH_JOKER = '[^.]+' + + +def re_path(*args): + return re.compile('^{}$'.format('.'.join(args))) + + +def re_path_basic(section, name): + return re_path(section, PATH_JOKER, name) + + +def service_path(*args): + return re_path('service', PATH_JOKER, *args) + + +def to_boolean(s): + s = s.lower() + if s in ['y', 'yes', 'true', 'on']: + return True + elif s in ['n', 'no', 'false', 'off']: + return False + raise ValueError('"{}" is not a valid boolean value'.format(s)) + + +def to_int(s): + # We must be able to handle octal representation for `mode` values notably + if six.PY3 and re.match('^0[0-9]+$', s.strip()): + s = '0o' + s[1:] + return int(s, base=0) + + +class ConversionMap(object): + map = { + service_path('blkio_config', 'weight'): to_int, + service_path('blkio_config', 'weight_device', 'weight'): to_int, + service_path('cpus'): float, + service_path('cpu_count'): to_int, + service_path('configs', 'mode'): to_int, + service_path('secrets', 'mode'): to_int, + service_path('healthcheck', 'retries'): to_int, + service_path('healthcheck', 'disable'): to_boolean, + service_path('deploy', 'replicas'): to_int, + service_path('deploy', 'update_config', 'parallelism'): to_int, + service_path('deploy', 'update_config', 'max_failure_ratio'): float, + service_path('deploy', 'restart_policy', 'max_attempts'): to_int, + service_path('mem_swappiness'): to_int, + service_path('oom_score_adj'): to_int, + service_path('ports', 'target'): to_int, + service_path('ports', 'published'): to_int, + service_path('scale'): to_int, + service_path('ulimits', PATH_JOKER): to_int, + service_path('ulimits', PATH_JOKER, 'soft'): to_int, + service_path('ulimits', PATH_JOKER, 'hard'): to_int, + service_path('privileged'): to_boolean, + service_path('read_only'): to_boolean, + service_path('stdin_open'): to_boolean, + service_path('tty'): to_boolean, + service_path('volumes', 'read_only'): to_boolean, + service_path('volumes', 'volume', 'nocopy'): to_boolean, + re_path_basic('network', 'attachable'): to_boolean, + re_path_basic('network', 'external'): to_boolean, + re_path_basic('network', 'internal'): to_boolean, + re_path_basic('volume', 'external'): to_boolean, + re_path_basic('secret', 'external'): to_boolean, + re_path_basic('config', 'external'): to_boolean, + } + + def convert(self, path, value): + for rexp in self.map.keys(): + if rexp.match(path): + return self.map[rexp](value) + return value + + +converter = ConversionMap() diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index 018a5621a..516f5c9e9 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -9,12 +9,22 @@ from compose.config.interpolation import Interpolator from compose.config.interpolation import InvalidInterpolation from compose.config.interpolation import TemplateWithDefaults from compose.const import COMPOSEFILE_V2_0 as V2_0 -from compose.const import COMPOSEFILE_V3_1 as V3_1 +from compose.const import COMPOSEFILE_V2_3 as V2_3 +from compose.const import COMPOSEFILE_V3_4 as V3_4 @pytest.fixture def mock_env(): - return Environment({'USER': 'jenny', 'FOO': 'bar'}) + return Environment({ + 'USER': 'jenny', + 'FOO': 'bar', + 'TRUE': 'True', + 'FALSE': 'OFF', + 'POSINT': '50', + 'NEGINT': '-200', + 'FLOAT': '0.145', + 'MODE': '0600', + }) @pytest.fixture @@ -102,7 +112,189 @@ def test_interpolate_environment_variables_in_secrets(mock_env): }, 'other': {}, } - value = interpolate_environment_variables(V3_1, secrets, 'volume', mock_env) + value = interpolate_environment_variables(V3_4, secrets, 'secret', mock_env) + assert value == expected + + +def test_interpolate_environment_services_convert_types_v2(mock_env): + entry = { + 'service1': { + 'blkio_config': { + 'weight': '${POSINT}', + 'weight_device': [{'file': '/dev/sda1', 'weight': '${POSINT}'}] + }, + 'cpus': '${FLOAT}', + 'cpu_count': '$POSINT', + 'healthcheck': { + 'retries': '${POSINT:-3}', + 'disable': '${FALSE}', + 'command': 'true' + }, + 'mem_swappiness': '${DEFAULT:-127}', + 'oom_score_adj': '${NEGINT}', + 'scale': '${POSINT}', + 'ulimits': { + 'nproc': '${POSINT}', + 'nofile': { + 'soft': '${POSINT}', + 'hard': '${DEFAULT:-40000}' + }, + }, + 'privileged': '${TRUE}', + 'read_only': '${DEFAULT:-no}', + 'tty': '${DEFAULT:-N}', + 'stdin_open': '${DEFAULT-on}', + } + } + + expected = { + 'service1': { + 'blkio_config': { + 'weight': 50, + 'weight_device': [{'file': '/dev/sda1', 'weight': 50}] + }, + 'cpus': 0.145, + 'cpu_count': 50, + 'healthcheck': { + 'retries': 50, + 'disable': False, + 'command': 'true' + }, + 'mem_swappiness': 127, + 'oom_score_adj': -200, + 'scale': 50, + 'ulimits': { + 'nproc': 50, + 'nofile': { + 'soft': 50, + 'hard': 40000 + }, + }, + 'privileged': True, + 'read_only': False, + 'tty': False, + 'stdin_open': True, + } + } + + value = interpolate_environment_variables(V2_3, entry, 'service', mock_env) + assert value == expected + + +def test_interpolate_environment_services_convert_types_v3(mock_env): + entry = { + 'service1': { + 'healthcheck': { + 'retries': '${POSINT:-3}', + 'disable': '${FALSE}', + 'command': 'true' + }, + 'ulimits': { + 'nproc': '${POSINT}', + 'nofile': { + 'soft': '${POSINT}', + 'hard': '${DEFAULT:-40000}' + }, + }, + 'privileged': '${TRUE}', + 'read_only': '${DEFAULT:-no}', + 'tty': '${DEFAULT:-N}', + 'stdin_open': '${DEFAULT-on}', + 'deploy': { + 'update_config': { + 'parallelism': '${DEFAULT:-2}', + 'max_failure_ratio': '${FLOAT}', + }, + 'restart_policy': { + 'max_attempts': '$POSINT', + }, + 'replicas': '${DEFAULT-3}' + }, + 'ports': [{'target': '${POSINT}', 'published': '${DEFAULT:-5000}'}], + 'configs': [{'mode': '${MODE}', 'source': 'config1'}], + 'secrets': [{'mode': '${MODE}', 'source': 'secret1'}], + } + } + + expected = { + 'service1': { + 'healthcheck': { + 'retries': 50, + 'disable': False, + 'command': 'true' + }, + 'ulimits': { + 'nproc': 50, + 'nofile': { + 'soft': 50, + 'hard': 40000 + }, + }, + 'privileged': True, + 'read_only': False, + 'tty': False, + 'stdin_open': True, + 'deploy': { + 'update_config': { + 'parallelism': 2, + 'max_failure_ratio': 0.145, + }, + 'restart_policy': { + 'max_attempts': 50, + }, + 'replicas': 3 + }, + 'ports': [{'target': 50, 'published': 5000}], + 'configs': [{'mode': 0o600, 'source': 'config1'}], + 'secrets': [{'mode': 0o600, 'source': 'secret1'}], + } + } + + value = interpolate_environment_variables(V3_4, entry, 'service', mock_env) + assert value == expected + + +def test_interpolate_environment_network_convert_types(mock_env): + entry = { + 'network1': { + 'external': '${FALSE}', + 'attachable': '${TRUE}', + 'internal': '${DEFAULT:-false}' + } + } + + expected = { + 'network1': { + 'external': False, + 'attachable': True, + 'internal': False, + } + } + + value = interpolate_environment_variables(V3_4, entry, 'network', mock_env) + assert value == expected + + +def test_interpolate_environment_external_resource_convert_types(mock_env): + entry = { + 'resource1': { + 'external': '${TRUE}', + } + } + + expected = { + 'resource1': { + 'external': True, + } + } + + value = interpolate_environment_variables(V3_4, entry, 'network', mock_env) + assert value == expected + value = interpolate_environment_variables(V3_4, entry, 'volume', mock_env) + assert value == expected + value = interpolate_environment_variables(V3_4, entry, 'secret', mock_env) + assert value == expected + value = interpolate_environment_variables(V3_4, entry, 'config', mock_env) assert value == expected