Support multiple config files

Signed-off-by: Daniel Nephin <dnephin@docker.com>
2025-04-08 17:05:13 +02:00 · 2015-09-11 20:18:45 -04:00 · 2015-09-11 20:18:45 -04:00 · 10b3188214
commit 10b3188214
parent 258d0fa0c6
3 changed files with 105 additions and 79 deletions
--- a/compose/cli/command.py
+++ b/compose/cli/command.py
@ -51,24 +51,26 @@ class Command(DocoptCommand):
            handler(None, command_options)
            return

-        if 'FIG_FILE' in os.environ:
-            log.warn('The FIG_FILE environment variable is deprecated.')
-            log.warn('Please use COMPOSE_FILE instead.')
-
-        explicit_config_path = (
-            options.get('--file') or
-            os.environ.get('COMPOSE_FILE') or
-            os.environ.get('FIG_FILE'))
-
        project = get_project(
            self.base_dir,
-            explicit_config_path,
+            get_config_path(options.get('--file')),
            project_name=options.get('--project-name'),
            verbose=options.get('--verbose'))

        handler(project, command_options)


+def get_config_path(file_option):
+    if file_option:
+        return file_option
+
+    if 'FIG_FILE' in os.environ:
+        log.warn('The FIG_FILE environment variable is deprecated.')
+        log.warn('Please use COMPOSE_FILE instead.')
+
+    return [os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')]
+
+
 def get_client(verbose=False):
    client = docker_client()
    if verbose:
--- a/compose/config/config.py
+++ b/compose/config/config.py
@ -2,6 +2,7 @@ import logging
 import os
 import sys
 from collections import namedtuple
+from functools import reduce

 import six
 import yaml
@ -88,18 +89,24 @@ PATH_START_CHARS = [
 log = logging.getLogger(__name__)


-ConfigDetails = namedtuple('ConfigDetails', 'config working_dir filename')
+ConfigDetails = namedtuple('ConfigDetails', 'working_dir configs')
+
+ConfigFile = namedtuple('ConfigFile', 'filename config')


-def find(base_dir, filename):
-    if filename == '-':
-        return ConfigDetails(yaml.safe_load(sys.stdin), os.getcwd(), None)
+def find(base_dir, filenames):
+    if filenames == ['-']:
+        return ConfigDetails(
+            os.getcwd(),
+            [ConfigFile(None, yaml.safe_load(sys.stdin))])

-    if filename:
-        filename = os.path.join(base_dir, filename)
+    if filenames:
+        filenames = [os.path.join(base_dir, f) for f in filenames]
    else:
-        filename = get_config_path(base_dir)
-    return ConfigDetails(load_yaml(filename), os.path.dirname(filename), filename)
+        filenames = [get_config_path(base_dir)]
+    return ConfigDetails(
+        os.path.dirname(filenames[0]),
+        [ConfigFile(f, load_yaml(f)) for f in filenames])


 def get_config_path(base_dir):
@ -133,29 +140,40 @@ def pre_process_config(config):
    Pre validation checks and processing of the config file to interpolate env
    vars returning a config dict ready to be tested against the schema.
    """
-    config = interpolate_environment_variables(config)
-    return config
+    return interpolate_environment_variables(config)


 def load(config_details):
-    config, working_dir, filename = config_details
+    working_dir, configs = config_details

-    processed_config = pre_process_config(config)
-    validate_against_fields_schema(processed_config)
-
-    service_dicts = []
-
-    for service_name, service_dict in list(processed_config.items()):
-        loader = ServiceLoader(
-            working_dir=working_dir,
-            filename=filename,
-            service_name=service_name,
-            service_dict=service_dict)
+    def build_service(filename, service_name, service_dict):
+        loader = ServiceLoader(working_dir, filename, service_name, service_dict)
        service_dict = loader.make_service_dict()
        validate_paths(service_dict)
-        service_dicts.append(service_dict)
+        return service_dict

-    return service_dicts
+    def load_file(filename, config):
+        processed_config = pre_process_config(config)
+        validate_against_fields_schema(processed_config)
+        return [
+            build_service(filename, name, service_config)
+            for name, service_config in processed_config.items()
+        ]
+
+    def merge_services(base, override):
+        return {
+            name: merge_service_dicts(base.get(name, {}), override.get(name, {}))
+            for name in set(base) | set(override)
+        }
+
+    def combine_configs(override, base):
+        service_dicts = load_file(base.filename, base.config)
+        if not override:
+            return service_dicts
+
+        return merge_service_dicts(base.config, override.config)
+
+    return reduce(combine_configs, configs, None)


 class ServiceLoader(object):
--- a/tests/unit/config_test.py
+++ b/tests/unit/config_test.py
@ -26,10 +26,16 @@ def service_sort(services):
    return sorted(services, key=itemgetter('name'))


+def build_config_details(contents, working_dir, filename):
+    return config.ConfigDetails(
+        working_dir,
+        [config.ConfigFile(filename, contents)])
+
+
 class ConfigTest(unittest.TestCase):
    def test_load(self):
        service_dicts = config.load(
-            config.ConfigDetails(
+            build_config_details(
                {
                    'foo': {'image': 'busybox'},
                    'bar': {'image': 'busybox', 'environment': ['FOO=1']},
@ -57,7 +63,7 @@ class ConfigTest(unittest.TestCase):
    def test_load_throws_error_when_not_dict(self):
        with self.assertRaises(ConfigurationError):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {'web': 'busybox:latest'},
                    'working_dir',
                    'filename.yml'
@ -68,7 +74,7 @@ class ConfigTest(unittest.TestCase):
        with self.assertRaises(ConfigurationError):
            for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']:
                config.load(
-                    config.ConfigDetails(
+                    build_config_details(
                        {invalid_name: {'image': 'busybox'}},
                        'working_dir',
                        'filename.yml'
@ -79,7 +85,7 @@ class ConfigTest(unittest.TestCase):
        expected_error_msg = "Service name: 1 needs to be a string, eg '1'"
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {1: {'image': 'busybox'}},
                    'working_dir',
                    'filename.yml'
@ -89,7 +95,7 @@ class ConfigTest(unittest.TestCase):
    def test_config_valid_service_names(self):
        for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']:
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {valid_name: {'image': 'busybox'}},
                    'tests/fixtures/extends',
                    'common.yml'
@ -101,7 +107,7 @@ class ConfigTest(unittest.TestCase):
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]:
                config.load(
-                    config.ConfigDetails(
+                    build_config_details(
                        {'web': {'image': 'busybox', 'ports': invalid_ports}},
                        'working_dir',
                        'filename.yml'
@ -112,7 +118,7 @@ class ConfigTest(unittest.TestCase):
        valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], [8000], ["49153-49154:3002-3003"]]
        for ports in valid_ports:
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {'web': {'image': 'busybox', 'ports': ports}},
                    'working_dir',
                    'filename.yml'
@ -123,7 +129,7 @@ class ConfigTest(unittest.TestCase):
        expected_error_msg = "(did you mean 'privileged'?)"
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'foo': {'image': 'busybox', 'privilige': 'something'},
                    },
@ -136,7 +142,7 @@ class ConfigTest(unittest.TestCase):
        expected_error_msg = "Service 'foo' has both an image and build path specified."
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'foo': {'image': 'busybox', 'build': '.'},
                    },
@ -149,7 +155,7 @@ class ConfigTest(unittest.TestCase):
        expected_error_msg = "Service 'foo' configuration key 'links' contains an invalid type, it should be an array"
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'foo': {'image': 'busybox', 'links': 'an_link'},
                    },
@ -162,7 +168,7 @@ class ConfigTest(unittest.TestCase):
        expected_error_msg = "Top level object needs to be a dictionary."
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    ['foo', 'lol'],
                    'tests/fixtures/extends',
                    'filename.yml'
@ -173,7 +179,7 @@ class ConfigTest(unittest.TestCase):
        expected_error_msg = "has non-unique elements"
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'web': {'build': '.', 'devices': ['/dev/foo:/dev/foo', '/dev/foo:/dev/foo']}
                    },
@ -187,7 +193,7 @@ class ConfigTest(unittest.TestCase):
        expected_error_msg += ", which is an invalid type, it should be a string"
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'web': {'build': '.', 'command': [1]}
                    },
@ -200,7 +206,7 @@ class ConfigTest(unittest.TestCase):
        expected_error_msg = "Service 'web' has both an image and alternate Dockerfile."
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {'web': {'image': 'busybox', 'dockerfile': 'Dockerfile.alt'}},
                    'working_dir',
                    'filename.yml'
@ -212,7 +218,7 @@ class ConfigTest(unittest.TestCase):

        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {'web': {
                        'image': 'busybox',
                        'extra_hosts': 'somehost:162.242.195.82'
@ -227,7 +233,7 @@ class ConfigTest(unittest.TestCase):

        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {'web': {
                        'image': 'busybox',
                        'extra_hosts': [
@ -244,7 +250,7 @@ class ConfigTest(unittest.TestCase):
        expose_values = [["8000"], [8000]]
        for expose in expose_values:
            service = config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {'web': {
                        'image': 'busybox',
                        'expose': expose
@ -259,7 +265,7 @@ class ConfigTest(unittest.TestCase):
        entrypoint_values = [["sh"], "sh"]
        for entrypoint in entrypoint_values:
            service = config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {'web': {
                        'image': 'busybox',
                        'entrypoint': entrypoint
@ -331,16 +337,16 @@ class InterpolationTest(unittest.TestCase):
    def test_unset_variable_produces_warning(self):
        os.environ.pop('FOO', None)
        os.environ.pop('BAR', None)
-        config_details = config.ConfigDetails(
-            config={
+        config_details = build_config_details(
+            {
                'web': {
                    'image': '${FOO}',
                    'command': '${BAR}',
                    'container_name': '${BAR}',
                },
            },
-            working_dir='.',
-            filename=None,
+            '.',
+            None,
        )

        with mock.patch('compose.config.interpolation.log') as log:
@ -355,7 +361,7 @@ class InterpolationTest(unittest.TestCase):
    def test_invalid_interpolation(self):
        with self.assertRaises(config.ConfigurationError) as cm:
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {'web': {'image': '${'}},
                    'working_dir',
                    'filename.yml'
@ -371,10 +377,10 @@ class InterpolationTest(unittest.TestCase):
    def test_volume_binding_with_environment_variable(self):
        os.environ['VOLUME_PATH'] = '/host/path'
        d = config.load(
-            config.ConfigDetails(
-                config={'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}},
-                working_dir='.',
-                filename=None,
+            build_config_details(
+                {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}},
+                '.',
+                None,
            )
        )[0]
        self.assertEqual(d['volumes'], ['/host/path:/container/path'])
@ -649,7 +655,7 @@ class MemoryOptionsTest(unittest.TestCase):
        )
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'foo': {'image': 'busybox', 'memswap_limit': 2000000},
                    },
@ -660,7 +666,7 @@ class MemoryOptionsTest(unittest.TestCase):

    def test_validation_with_correct_memswap_values(self):
        service_dict = config.load(
-            config.ConfigDetails(
+            build_config_details(
                {'foo': {'image': 'busybox', 'mem_limit': 1000000, 'memswap_limit': 2000000}},
                'tests/fixtures/extends',
                'common.yml'
@ -670,7 +676,7 @@ class MemoryOptionsTest(unittest.TestCase):

    def test_memswap_can_be_a_string(self):
        service_dict = config.load(
-            config.ConfigDetails(
+            build_config_details(
                {'foo': {'image': 'busybox', 'mem_limit': "1G", 'memswap_limit': "512M"}},
                'tests/fixtures/extends',
                'common.yml'
@ -780,26 +786,26 @@ class EnvTest(unittest.TestCase):
        os.environ['CONTAINERENV'] = '/host/tmp'

        service_dict = config.load(
-            config.ConfigDetails(
-                config={'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}},
-                working_dir="tests/fixtures/env",
-                filename=None,
+            build_config_details(
+                {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}},
+                "tests/fixtures/env",
+                None,
            )
        )[0]
        self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp']))

        service_dict = config.load(
-            config.ConfigDetails(
-                config={'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}},
-                working_dir="tests/fixtures/env",
-                filename=None,
+            build_config_details(
+                {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}},
+                "tests/fixtures/env",
+                None,
            )
        )[0]
        self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp']))


 def load_from_filename(filename):
-    return config.load(config.find('.', filename))
+    return config.load(config.find('.', [filename]))


 class ExtendsTest(unittest.TestCase):
@ -885,7 +891,7 @@ class ExtendsTest(unittest.TestCase):
    def test_extends_validation_empty_dictionary(self):
        with self.assertRaisesRegexp(ConfigurationError, 'service'):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'web': {'image': 'busybox', 'extends': {}},
                    },
@ -897,7 +903,7 @@ class ExtendsTest(unittest.TestCase):
    def test_extends_validation_missing_service_key(self):
        with self.assertRaisesRegexp(ConfigurationError, "'service' is a required property"):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'web': {'image': 'busybox', 'extends': {'file': 'common.yml'}},
                    },
@ -910,7 +916,7 @@ class ExtendsTest(unittest.TestCase):
        expected_error_msg = "Unsupported config option for 'web' service: 'rogue_key'"
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'web': {
                            'image': 'busybox',
@ -930,7 +936,7 @@ class ExtendsTest(unittest.TestCase):
        expected_error_msg = "Service 'web' configuration key 'extends' 'file' contains an invalid type"
        with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'web': {
                            'image': 'busybox',
@ -955,7 +961,7 @@ class ExtendsTest(unittest.TestCase):

    def test_extends_validation_valid_config(self):
        service = config.load(
-            config.ConfigDetails(
+            build_config_details(
                {
                    'web': {'image': 'busybox', 'extends': {'service': 'web', 'file': 'common.yml'}},
                },
@ -1093,7 +1099,7 @@ class BuildPathTest(unittest.TestCase):
    def test_nonexistent_path(self):
        with self.assertRaises(ConfigurationError):
            config.load(
-                config.ConfigDetails(
+                build_config_details(
                    {
                        'foo': {'build': 'nonexistent.path'},
                    },