From 2121f5117ea035c83d4ace97fad8f2db6582afc9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 18:20:35 -0400 Subject: [PATCH 01/12] Add docopt support for multiple files Signed-off-by: Daniel Nephin --- compose/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 61461ae7b..3dd0c9fae 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -96,7 +96,7 @@ class TopLevelCommand(Command): """Define and run multi-container applications with Docker. Usage: - docker-compose [options] [COMMAND] [ARGS...] + docker-compose [-f=...] [options] [COMMAND] [ARGS...] docker-compose -h|--help Options: From 258d0fa0c660813d8b6b3d8d17731cc56a5da321 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 19:51:50 -0400 Subject: [PATCH 02/12] Remove some functions from Command class Signed-off-by: Daniel Nephin --- compose/cli/command.py | 82 +++++++++++++++++++++++------------------- tests/unit/cli_test.py | 39 +++++++++----------- 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 67176df27..70b129d29 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -55,53 +55,61 @@ class Command(DocoptCommand): log.warn('The FIG_FILE environment variable is deprecated.') log.warn('Please use COMPOSE_FILE instead.') - explicit_config_path = options.get('--file') or os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE') - project = self.get_project( + explicit_config_path = ( + options.get('--file') or + os.environ.get('COMPOSE_FILE') or + os.environ.get('FIG_FILE')) + + project = get_project( + self.base_dir, explicit_config_path, project_name=options.get('--project-name'), verbose=options.get('--verbose')) handler(project, command_options) - def get_client(self, verbose=False): - client = docker_client() - if verbose: - version_info = six.iteritems(client.version()) - log.info("Compose version %s", __version__) - log.info("Docker base_url: %s", client.base_url) - log.info("Docker version: %s", - ", ".join("%s=%s" % item for item in version_info)) - return verbose_proxy.VerboseProxy('docker', client) - return client - def get_project(self, config_path=None, project_name=None, verbose=False): - config_details = config.find(self.base_dir, config_path) +def get_client(verbose=False): + client = docker_client() + if verbose: + version_info = six.iteritems(client.version()) + log.info("Compose version %s", __version__) + log.info("Docker base_url: %s", client.base_url) + log.info("Docker version: %s", + ", ".join("%s=%s" % item for item in version_info)) + return verbose_proxy.VerboseProxy('docker', client) + return client - try: - return Project.from_dicts( - self.get_project_name(config_details.working_dir, project_name), - config.load(config_details), - self.get_client(verbose=verbose)) - except ConfigError as e: - raise errors.UserError(six.text_type(e)) - def get_project_name(self, working_dir, project_name=None): - def normalize_name(name): - return re.sub(r'[^a-z0-9]', '', name.lower()) +def get_project(base_dir, config_path=None, project_name=None, verbose=False): + config_details = config.find(base_dir, config_path) - if 'FIG_PROJECT_NAME' in os.environ: - log.warn('The FIG_PROJECT_NAME environment variable is deprecated.') - log.warn('Please use COMPOSE_PROJECT_NAME instead.') + try: + return Project.from_dicts( + get_project_name(config_details.working_dir, project_name), + config.load(config_details), + get_client(verbose=verbose)) + except ConfigError as e: + raise errors.UserError(six.text_type(e)) - project_name = ( - project_name or - os.environ.get('COMPOSE_PROJECT_NAME') or - os.environ.get('FIG_PROJECT_NAME')) - if project_name is not None: - return normalize_name(project_name) - project = os.path.basename(os.path.abspath(working_dir)) - if project: - return normalize_name(project) +def get_project_name(working_dir, project_name=None): + def normalize_name(name): + return re.sub(r'[^a-z0-9]', '', name.lower()) - return 'default' + if 'FIG_PROJECT_NAME' in os.environ: + log.warn('The FIG_PROJECT_NAME environment variable is deprecated.') + log.warn('Please use COMPOSE_PROJECT_NAME instead.') + + project_name = ( + project_name or + os.environ.get('COMPOSE_PROJECT_NAME') or + os.environ.get('FIG_PROJECT_NAME')) + if project_name is not None: + return normalize_name(project_name) + + project = os.path.basename(os.path.abspath(working_dir)) + if project: + return normalize_name(project) + + return 'default' diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index d12f41955..321df97a5 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -4,9 +4,12 @@ from __future__ import unicode_literals import os import docker +import py from .. import mock from .. import unittest +from compose.cli.command import get_project +from compose.cli.command import get_project_name from compose.cli.docopt_command import NoSuchCommand from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand @@ -14,55 +17,45 @@ from compose.service import Service class CLITestCase(unittest.TestCase): - def test_default_project_name(self): - cwd = os.getcwd() - try: - os.chdir('tests/fixtures/simple-composefile') - command = TopLevelCommand() - project_name = command.get_project_name('.') + def test_default_project_name(self): + test_dir = py._path.local.LocalPath('tests/fixtures/simple-composefile') + with test_dir.as_cwd(): + project_name = get_project_name('.') self.assertEquals('simplecomposefile', project_name) - finally: - os.chdir(cwd) def test_project_name_with_explicit_base_dir(self): - command = TopLevelCommand() - command.base_dir = 'tests/fixtures/simple-composefile' - project_name = command.get_project_name(command.base_dir) + base_dir = 'tests/fixtures/simple-composefile' + project_name = get_project_name(base_dir) self.assertEquals('simplecomposefile', project_name) def test_project_name_with_explicit_uppercase_base_dir(self): - command = TopLevelCommand() - command.base_dir = 'tests/fixtures/UpperCaseDir' - project_name = command.get_project_name(command.base_dir) + base_dir = 'tests/fixtures/UpperCaseDir' + project_name = get_project_name(base_dir) self.assertEquals('uppercasedir', project_name) def test_project_name_with_explicit_project_name(self): - command = TopLevelCommand() name = 'explicit-project-name' - project_name = command.get_project_name(None, project_name=name) + project_name = get_project_name(None, project_name=name) self.assertEquals('explicitprojectname', project_name) def test_project_name_from_environment_old_var(self): - command = TopLevelCommand() name = 'namefromenv' with mock.patch.dict(os.environ): os.environ['FIG_PROJECT_NAME'] = name - project_name = command.get_project_name(None) + project_name = get_project_name(None) self.assertEquals(project_name, name) def test_project_name_from_environment_new_var(self): - command = TopLevelCommand() name = 'namefromenv' with mock.patch.dict(os.environ): os.environ['COMPOSE_PROJECT_NAME'] = name - project_name = command.get_project_name(None) + project_name = get_project_name(None) self.assertEquals(project_name, name) def test_get_project(self): - command = TopLevelCommand() - command.base_dir = 'tests/fixtures/longer-filename-composefile' - project = command.get_project() + base_dir = 'tests/fixtures/longer-filename-composefile' + project = get_project(base_dir) self.assertEqual(project.name, 'longerfilenamecomposefile') self.assertTrue(project.client) self.assertTrue(project.services) From 10b3188214fc6716387339bb0146d8d901962e93 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 20:18:45 -0400 Subject: [PATCH 03/12] Support multiple config files Signed-off-by: Daniel Nephin --- compose/cli/command.py | 22 +++++---- compose/config/config.py | 66 +++++++++++++++++---------- tests/unit/config_test.py | 96 +++++++++++++++++++++------------------ 3 files changed, 105 insertions(+), 79 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 70b129d29..2120ec4db 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -51,24 +51,26 @@ class Command(DocoptCommand): handler(None, command_options) return - if 'FIG_FILE' in os.environ: - log.warn('The FIG_FILE environment variable is deprecated.') - log.warn('Please use COMPOSE_FILE instead.') - - explicit_config_path = ( - options.get('--file') or - os.environ.get('COMPOSE_FILE') or - os.environ.get('FIG_FILE')) - project = get_project( self.base_dir, - explicit_config_path, + get_config_path(options.get('--file')), project_name=options.get('--project-name'), verbose=options.get('--verbose')) handler(project, command_options) +def get_config_path(file_option): + if file_option: + return file_option + + if 'FIG_FILE' in os.environ: + log.warn('The FIG_FILE environment variable is deprecated.') + log.warn('Please use COMPOSE_FILE instead.') + + return [os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')] + + def get_client(verbose=False): client = docker_client() if verbose: diff --git a/compose/config/config.py b/compose/config/config.py index 840a28a1b..204f70b66 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -2,6 +2,7 @@ import logging import os import sys from collections import namedtuple +from functools import reduce import six import yaml @@ -88,18 +89,24 @@ PATH_START_CHARS = [ log = logging.getLogger(__name__) -ConfigDetails = namedtuple('ConfigDetails', 'config working_dir filename') +ConfigDetails = namedtuple('ConfigDetails', 'working_dir configs') + +ConfigFile = namedtuple('ConfigFile', 'filename config') -def find(base_dir, filename): - if filename == '-': - return ConfigDetails(yaml.safe_load(sys.stdin), os.getcwd(), None) +def find(base_dir, filenames): + if filenames == ['-']: + return ConfigDetails( + os.getcwd(), + [ConfigFile(None, yaml.safe_load(sys.stdin))]) - if filename: - filename = os.path.join(base_dir, filename) + if filenames: + filenames = [os.path.join(base_dir, f) for f in filenames] else: - filename = get_config_path(base_dir) - return ConfigDetails(load_yaml(filename), os.path.dirname(filename), filename) + filenames = [get_config_path(base_dir)] + return ConfigDetails( + os.path.dirname(filenames[0]), + [ConfigFile(f, load_yaml(f)) for f in filenames]) def get_config_path(base_dir): @@ -133,29 +140,40 @@ def pre_process_config(config): Pre validation checks and processing of the config file to interpolate env vars returning a config dict ready to be tested against the schema. """ - config = interpolate_environment_variables(config) - return config + return interpolate_environment_variables(config) def load(config_details): - config, working_dir, filename = config_details + working_dir, configs = config_details - processed_config = pre_process_config(config) - validate_against_fields_schema(processed_config) - - service_dicts = [] - - for service_name, service_dict in list(processed_config.items()): - loader = ServiceLoader( - working_dir=working_dir, - filename=filename, - service_name=service_name, - service_dict=service_dict) + def build_service(filename, service_name, service_dict): + loader = ServiceLoader(working_dir, filename, service_name, service_dict) service_dict = loader.make_service_dict() validate_paths(service_dict) - service_dicts.append(service_dict) + return service_dict - return service_dicts + def load_file(filename, config): + processed_config = pre_process_config(config) + validate_against_fields_schema(processed_config) + return [ + build_service(filename, name, service_config) + for name, service_config in processed_config.items() + ] + + def merge_services(base, override): + return { + name: merge_service_dicts(base.get(name, {}), override.get(name, {})) + for name in set(base) | set(override) + } + + def combine_configs(override, base): + service_dicts = load_file(base.filename, base.config) + if not override: + return service_dicts + + return merge_service_dicts(base.config, override.config) + + return reduce(combine_configs, configs, None) class ServiceLoader(object): diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index ff80270e6..0347e443f 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -26,10 +26,16 @@ def service_sort(services): return sorted(services, key=itemgetter('name')) +def build_config_details(contents, working_dir, filename): + return config.ConfigDetails( + working_dir, + [config.ConfigFile(filename, contents)]) + + class ConfigTest(unittest.TestCase): def test_load(self): service_dicts = config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox'}, 'bar': {'image': 'busybox', 'environment': ['FOO=1']}, @@ -57,7 +63,7 @@ class ConfigTest(unittest.TestCase): def test_load_throws_error_when_not_dict(self): with self.assertRaises(ConfigurationError): config.load( - config.ConfigDetails( + build_config_details( {'web': 'busybox:latest'}, 'working_dir', 'filename.yml' @@ -68,7 +74,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaises(ConfigurationError): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: config.load( - config.ConfigDetails( + build_config_details( {invalid_name: {'image': 'busybox'}}, 'working_dir', 'filename.yml' @@ -79,7 +85,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service name: 1 needs to be a string, eg '1'" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {1: {'image': 'busybox'}}, 'working_dir', 'filename.yml' @@ -89,7 +95,7 @@ class ConfigTest(unittest.TestCase): def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( - config.ConfigDetails( + build_config_details( {valid_name: {'image': 'busybox'}}, 'tests/fixtures/extends', 'common.yml' @@ -101,7 +107,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]: config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': 'busybox', 'ports': invalid_ports}}, 'working_dir', 'filename.yml' @@ -112,7 +118,7 @@ class ConfigTest(unittest.TestCase): valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], [8000], ["49153-49154:3002-3003"]] for ports in valid_ports: config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': 'busybox', 'ports': ports}}, 'working_dir', 'filename.yml' @@ -123,7 +129,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "(did you mean 'privileged'?)" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'privilige': 'something'}, }, @@ -136,7 +142,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service 'foo' has both an image and build path specified." with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'build': '.'}, }, @@ -149,7 +155,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service 'foo' configuration key 'links' contains an invalid type, it should be an array" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'links': 'an_link'}, }, @@ -162,7 +168,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Top level object needs to be a dictionary." with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( ['foo', 'lol'], 'tests/fixtures/extends', 'filename.yml' @@ -173,7 +179,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "has non-unique elements" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'build': '.', 'devices': ['/dev/foo:/dev/foo', '/dev/foo:/dev/foo']} }, @@ -187,7 +193,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg += ", which is an invalid type, it should be a string" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'build': '.', 'command': [1]} }, @@ -200,7 +206,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service 'web' has both an image and alternate Dockerfile." with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': 'busybox', 'dockerfile': 'Dockerfile.alt'}}, 'working_dir', 'filename.yml' @@ -212,7 +218,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'extra_hosts': 'somehost:162.242.195.82' @@ -227,7 +233,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'extra_hosts': [ @@ -244,7 +250,7 @@ class ConfigTest(unittest.TestCase): expose_values = [["8000"], [8000]] for expose in expose_values: service = config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'expose': expose @@ -259,7 +265,7 @@ class ConfigTest(unittest.TestCase): entrypoint_values = [["sh"], "sh"] for entrypoint in entrypoint_values: service = config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'entrypoint': entrypoint @@ -331,16 +337,16 @@ class InterpolationTest(unittest.TestCase): def test_unset_variable_produces_warning(self): os.environ.pop('FOO', None) os.environ.pop('BAR', None) - config_details = config.ConfigDetails( - config={ + config_details = build_config_details( + { 'web': { 'image': '${FOO}', 'command': '${BAR}', 'container_name': '${BAR}', }, }, - working_dir='.', - filename=None, + '.', + None, ) with mock.patch('compose.config.interpolation.log') as log: @@ -355,7 +361,7 @@ class InterpolationTest(unittest.TestCase): def test_invalid_interpolation(self): with self.assertRaises(config.ConfigurationError) as cm: config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': '${'}}, 'working_dir', 'filename.yml' @@ -371,10 +377,10 @@ class InterpolationTest(unittest.TestCase): def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' d = config.load( - config.ConfigDetails( - config={'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, - working_dir='.', - filename=None, + build_config_details( + {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, + '.', + None, ) )[0] self.assertEqual(d['volumes'], ['/host/path:/container/path']) @@ -649,7 +655,7 @@ class MemoryOptionsTest(unittest.TestCase): ) with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'memswap_limit': 2000000}, }, @@ -660,7 +666,7 @@ class MemoryOptionsTest(unittest.TestCase): def test_validation_with_correct_memswap_values(self): service_dict = config.load( - config.ConfigDetails( + build_config_details( {'foo': {'image': 'busybox', 'mem_limit': 1000000, 'memswap_limit': 2000000}}, 'tests/fixtures/extends', 'common.yml' @@ -670,7 +676,7 @@ class MemoryOptionsTest(unittest.TestCase): def test_memswap_can_be_a_string(self): service_dict = config.load( - config.ConfigDetails( + build_config_details( {'foo': {'image': 'busybox', 'mem_limit': "1G", 'memswap_limit': "512M"}}, 'tests/fixtures/extends', 'common.yml' @@ -780,26 +786,26 @@ class EnvTest(unittest.TestCase): os.environ['CONTAINERENV'] = '/host/tmp' service_dict = config.load( - config.ConfigDetails( - config={'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, - working_dir="tests/fixtures/env", - filename=None, + build_config_details( + {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, + "tests/fixtures/env", + None, ) )[0] self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp'])) service_dict = config.load( - config.ConfigDetails( - config={'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, - working_dir="tests/fixtures/env", - filename=None, + build_config_details( + {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, + "tests/fixtures/env", + None, ) )[0] self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp'])) def load_from_filename(filename): - return config.load(config.find('.', filename)) + return config.load(config.find('.', [filename])) class ExtendsTest(unittest.TestCase): @@ -885,7 +891,7 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation_empty_dictionary(self): with self.assertRaisesRegexp(ConfigurationError, 'service'): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'image': 'busybox', 'extends': {}}, }, @@ -897,7 +903,7 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation_missing_service_key(self): with self.assertRaisesRegexp(ConfigurationError, "'service' is a required property"): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'image': 'busybox', 'extends': {'file': 'common.yml'}}, }, @@ -910,7 +916,7 @@ class ExtendsTest(unittest.TestCase): expected_error_msg = "Unsupported config option for 'web' service: 'rogue_key'" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': { 'image': 'busybox', @@ -930,7 +936,7 @@ class ExtendsTest(unittest.TestCase): expected_error_msg = "Service 'web' configuration key 'extends' 'file' contains an invalid type" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': { 'image': 'busybox', @@ -955,7 +961,7 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation_valid_config(self): service = config.load( - config.ConfigDetails( + build_config_details( { 'web': {'image': 'busybox', 'extends': {'service': 'web', 'file': 'common.yml'}}, }, @@ -1093,7 +1099,7 @@ class BuildPathTest(unittest.TestCase): def test_nonexistent_path(self): with self.assertRaises(ConfigurationError): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'build': 'nonexistent.path'}, }, From c0c9a7c1e4d22980afb6e22817a960f7424f0eae Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 20:50:31 -0400 Subject: [PATCH 04/12] Update integration tests for multiple file support Signed-off-by: Daniel Nephin --- compose/cli/command.py | 3 ++- tests/integration/cli_test.py | 7 ++++--- tests/integration/project_test.py | 7 +++++-- tests/integration/state_test.py | 8 +++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 2120ec4db..950cb166e 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -68,7 +68,8 @@ def get_config_path(file_option): log.warn('The FIG_FILE environment variable is deprecated.') log.warn('Please use COMPOSE_FILE instead.') - return [os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')] + config_file = os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE') + return [config_file] if config_file else None def get_client(verbose=False): diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 4a80d3369..8688fb8b4 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -9,6 +9,7 @@ from six import StringIO from .. import mock from .testcases import DockerClientTestCase +from compose.cli.command import get_project from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand from compose.project import NoSuchService @@ -38,7 +39,7 @@ class CLITestCase(DockerClientTestCase): if hasattr(self, '_project'): return self._project - return self.command.get_project() + return get_project(self.command.base_dir) def test_help(self): old_base_dir = self.command.base_dir @@ -72,7 +73,7 @@ class CLITestCase(DockerClientTestCase): def test_ps_alternate_composefile(self, mock_stdout): config_path = os.path.abspath( 'tests/fixtures/multiple-composefiles/compose2.yml') - self._project = self.command.get_project(config_path) + self._project = get_project(self.command.base_dir, [config_path]) self.command.base_dir = 'tests/fixtures/multiple-composefiles' self.command.dispatch(['-f', 'compose2.yml', 'up', '-d'], None) @@ -571,7 +572,7 @@ class CLITestCase(DockerClientTestCase): def test_env_file_relative_to_compose_file(self): config_path = os.path.abspath('tests/fixtures/env-file/docker-compose.yml') self.command.dispatch(['-f', config_path, 'up', '-d'], None) - self._project = self.command.get_project(config_path) + self._project = get_project(self.command.base_dir, [config_path]) containers = self.project.containers(stopped=True) self.assertEqual(len(containers), 1) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ad49ad10a..bd7ecccbe 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals from .testcases import DockerClientTestCase -from compose import config +from compose.config import config from compose.const import LABEL_PROJECT from compose.container import Container from compose.project import Project @@ -9,7 +9,10 @@ from compose.service import ConvergenceStrategy def build_service_dicts(service_config): - return config.load(config.ConfigDetails(service_config, 'working_dir', None)) + return config.load( + config.ConfigDetails( + 'working_dir', + [config.ConfigFile(None, service_config)])) class ProjectTest(DockerClientTestCase): diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 93d0572a0..ef7276bd8 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -9,7 +9,7 @@ import shutil import tempfile from .testcases import DockerClientTestCase -from compose import config +from compose.config import config from compose.const import LABEL_CONFIG_HASH from compose.project import Project from compose.service import ConvergenceStrategy @@ -24,11 +24,13 @@ class ProjectTestCase(DockerClientTestCase): return set(project.containers(stopped=True)) def make_project(self, cfg): + details = config.ConfigDetails( + 'working_dir', + [config.ConfigFile(None, cfg)]) return Project.from_dicts( name='composetest', client=self.client, - service_dicts=config.load(config.ConfigDetails(cfg, 'working_dir', None)) - ) + service_dicts=config.load(details)) class BasicProjectTest(ProjectTestCase): From 831276f53163c0999ec635d92629e6e1b4ba2683 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 14 Sep 2015 21:30:32 -0400 Subject: [PATCH 05/12] Move config_test to the correct package name. Signed-off-by: Daniel Nephin --- tests/unit/config/__init__.py | 0 tests/unit/{ => config}/config_test.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 tests/unit/config/__init__.py rename tests/unit/{ => config}/config_test.py (99%) diff --git a/tests/unit/config/__init__.py b/tests/unit/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/config_test.py b/tests/unit/config/config_test.py similarity index 99% rename from tests/unit/config_test.py rename to tests/unit/config/config_test.py index 0347e443f..3542f272b 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config/config_test.py @@ -280,7 +280,7 @@ class ConfigTest(unittest.TestCase): def test_logs_warning_for_boolean_in_environment(self, mock_logging): expected_warning_msg = "Warning: There is a boolean value, True in the 'environment' key." config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'environment': {'SHOW_STUFF': True} @@ -298,7 +298,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'environment': {'---': 'nope'} From 89be7f1fa76f53dbc082715eadec65e08f992e8a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 14 Sep 2015 21:35:41 -0400 Subject: [PATCH 06/12] Unit tests for multiple files Signed-off-by: Daniel Nephin --- compose/config/config.py | 8 ++++--- tests/unit/config/config_test.py | 41 ++++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 204f70b66..058183d97 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -166,14 +166,16 @@ def load(config_details): for name in set(base) | set(override) } - def combine_configs(override, base): + def combine_configs(base, override): service_dicts = load_file(base.filename, base.config) if not override: return service_dicts - return merge_service_dicts(base.config, override.config) + return ConfigFile( + override.filename, + merge_services(base.config, override.config)) - return reduce(combine_configs, configs, None) + return reduce(combine_configs, configs + [None]) class ServiceLoader(object): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 3542f272b..60f4bbe22 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -5,10 +5,10 @@ import shutil import tempfile from operator import itemgetter -from .. import mock -from .. import unittest from compose.config import config from compose.config.errors import ConfigurationError +from tests import mock +from tests import unittest def make_service_dict(name, service_dict, working_dir, filename=None): @@ -92,6 +92,43 @@ class ConfigTest(unittest.TestCase): ) ) + def test_load_with_multiple_files(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'web': { + 'image': 'example/web', + 'links': ['db'], + }, + 'db': { + 'image': 'example/db', + }, + }) + override_file = config.ConfigFile( + 'override.yaml', + { + 'web': { + 'build': '/', + 'volumes': ['/home/user/project:/code'], + }, + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + service_dicts = config.load(details) + expected = [ + { + 'name': 'web', + 'build': '/', + 'links': ['db'], + 'volumes': ['/home/user/project:/code'], + }, + { + 'name': 'db', + 'image': 'example/db', + }, + ] + self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( From fe5daf860dfb341ba894d79d225689ed2e981064 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 15 Sep 2015 14:17:27 -0400 Subject: [PATCH 07/12] Move find_candidates_in_parent_dirs() into a config module so that config doesn't import from cli. Signed-off-by: Daniel Nephin --- compose/cli/utils.py | 19 ------------------- compose/config/config.py | 24 +++++++++++++++++++++--- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 0b7ac683d..0a4416c0f 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -36,25 +36,6 @@ def yesno(prompt, default=None): return None -def find_candidates_in_parent_dirs(filenames, path): - """ - Given a directory path to start, looks for filenames in the - directory, and then each parent directory successively, - until found. - - Returns tuple (candidates, path). - """ - candidates = [filename for filename in filenames - if os.path.exists(os.path.join(path, filename))] - - if len(candidates) == 0: - parent_dir = os.path.join(path, '..') - if os.path.abspath(parent_dir) != os.path.abspath(path): - return find_candidates_in_parent_dirs(filenames, parent_dir) - - return (candidates, path) - - def split_buffer(reader, separator): """ Given a generator which yields strings and a separator string, diff --git a/compose/config/config.py b/compose/config/config.py index 058183d97..2e4d0a751 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -17,7 +17,6 @@ from .validation import validate_extended_service_exists from .validation import validate_extends_file_path from .validation import validate_service_names from .validation import validate_top_level_object -from compose.cli.utils import find_candidates_in_parent_dirs DOCKER_CONFIG_KEYS = [ @@ -103,13 +102,13 @@ def find(base_dir, filenames): if filenames: filenames = [os.path.join(base_dir, f) for f in filenames] else: - filenames = [get_config_path(base_dir)] + filenames = get_default_config_path(base_dir) return ConfigDetails( os.path.dirname(filenames[0]), [ConfigFile(f, load_yaml(f)) for f in filenames]) -def get_config_path(base_dir): +def get_default_config_path(base_dir): (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir) if len(candidates) == 0: @@ -133,6 +132,25 @@ def get_config_path(base_dir): return os.path.join(path, winner) +def find_candidates_in_parent_dirs(filenames, path): + """ + Given a directory path to start, looks for filenames in the + directory, and then each parent directory successively, + until found. + + Returns tuple (candidates, path). + """ + candidates = [filename for filename in filenames + if os.path.exists(os.path.join(path, filename))] + + if len(candidates) == 0: + parent_dir = os.path.join(path, '..') + if os.path.abspath(parent_dir) != os.path.abspath(path): + return find_candidates_in_parent_dirs(filenames, parent_dir) + + return (candidates, path) + + @validate_top_level_object @validate_service_names def pre_process_config(config): From 39ae85db8ad4aa6429d6c4863d67b21d1c93aac7 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 15 Sep 2015 14:43:18 -0400 Subject: [PATCH 08/12] Support a default docker-compose.override.yml for overrides Signed-off-by: Daniel Nephin --- compose/config/config.py | 16 +++++--- .../docker-compose.override.yml | 6 +++ .../override-files/docker-compose.yml | 10 +++++ tests/fixtures/override-files/extra.yml | 9 +++++ tests/integration/cli_test.py | 39 ++++++++++++++++++- tests/unit/config/config_test.py | 3 +- 6 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/override-files/docker-compose.override.yml create mode 100644 tests/fixtures/override-files/docker-compose.yml create mode 100644 tests/fixtures/override-files/extra.yml diff --git a/compose/config/config.py b/compose/config/config.py index 2e4d0a751..3ecdd29d7 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -77,6 +77,7 @@ SUPPORTED_FILENAMES = [ 'fig.yaml', ] +DEFAULT_OVERRIDE_FILENAME = 'docker-compose.override.yml' PATH_START_CHARS = [ '/', @@ -102,16 +103,16 @@ def find(base_dir, filenames): if filenames: filenames = [os.path.join(base_dir, f) for f in filenames] else: - filenames = get_default_config_path(base_dir) + filenames = get_default_config_files(base_dir) return ConfigDetails( os.path.dirname(filenames[0]), [ConfigFile(f, load_yaml(f)) for f in filenames]) -def get_default_config_path(base_dir): +def get_default_config_files(base_dir): (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir) - if len(candidates) == 0: + if not candidates: raise ComposeFileNotFound(SUPPORTED_FILENAMES) winner = candidates[0] @@ -129,7 +130,12 @@ def get_default_config_path(base_dir): log.warn("%s is deprecated and will not be supported in future. " "Please rename your config file to docker-compose.yml\n" % winner) - return os.path.join(path, winner) + return [os.path.join(path, winner)] + get_default_override_file(path) + + +def get_default_override_file(path): + override_filename = os.path.join(path, DEFAULT_OVERRIDE_FILENAME) + return [override_filename] if os.path.exists(override_filename) else [] def find_candidates_in_parent_dirs(filenames, path): @@ -143,7 +149,7 @@ def find_candidates_in_parent_dirs(filenames, path): candidates = [filename for filename in filenames if os.path.exists(os.path.join(path, filename))] - if len(candidates) == 0: + if not candidates: parent_dir = os.path.join(path, '..') if os.path.abspath(parent_dir) != os.path.abspath(path): return find_candidates_in_parent_dirs(filenames, parent_dir) diff --git a/tests/fixtures/override-files/docker-compose.override.yml b/tests/fixtures/override-files/docker-compose.override.yml new file mode 100644 index 000000000..a03d3d6f5 --- /dev/null +++ b/tests/fixtures/override-files/docker-compose.override.yml @@ -0,0 +1,6 @@ + +web: + command: "top" + +db: + command: "top" diff --git a/tests/fixtures/override-files/docker-compose.yml b/tests/fixtures/override-files/docker-compose.yml new file mode 100644 index 000000000..8eb43ddb0 --- /dev/null +++ b/tests/fixtures/override-files/docker-compose.yml @@ -0,0 +1,10 @@ + +web: + image: busybox:latest + command: "sleep 200" + links: + - db + +db: + image: busybox:latest + command: "sleep 200" diff --git a/tests/fixtures/override-files/extra.yml b/tests/fixtures/override-files/extra.yml new file mode 100644 index 000000000..7b3ade9c2 --- /dev/null +++ b/tests/fixtures/override-files/extra.yml @@ -0,0 +1,9 @@ + +web: + links: + - db + - other + +other: + image: busybox:latest + command: "top" diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 8688fb8b4..33fdda3be 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -549,7 +549,6 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(get_port(3002), "0.0.0.0:49153") def test_port_with_scale(self): - self.command.base_dir = 'tests/fixtures/ports-composefile-scale' self.command.dispatch(['scale', 'simple=2'], None) containers = sorted( @@ -593,6 +592,44 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(components[-2:] == ['home-dir', 'my-volume'], msg="Last two components differ: %s, %s" % (actual_host_path, expected_host_path)) + def test_up_with_default_override_file(self): + self.command.base_dir = 'tests/fixtures/override-files' + self.command.dispatch(['up', '-d'], None) + + containers = self.project.containers() + self.assertEqual(len(containers), 2) + + web, db = containers + self.assertEqual(web.human_readable_command, 'top') + self.assertEqual(db.human_readable_command, 'top') + + def test_up_with_multiple_files(self): + self.command.base_dir = 'tests/fixtures/override-files' + config_paths = [ + 'docker-compose.yml', + 'docker-compose.override.yml', + 'extra.yml', + + ] + self._project = get_project(self.command.base_dir, config_paths) + self.command.dispatch( + [ + '-f', config_paths[0], + '-f', config_paths[1], + '-f', config_paths[2], + 'up', '-d', + ], + None) + + containers = self.project.containers() + self.assertEqual(len(containers), 3) + + web, other, db = containers + self.assertEqual(web.human_readable_command, 'top') + self.assertTrue({'db', 'other'} <= set(web.links())) + self.assertEqual(db.human_readable_command, 'top') + self.assertEqual(other.human_readable_command, 'top') + def test_up_with_extends(self): self.command.base_dir = 'tests/fixtures/extends' self.command.dispatch(['up', '-d'], None) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 60f4bbe22..38eb3de23 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1213,6 +1213,7 @@ def get_config_filename_for_files(filenames, subdir=None): base_dir = tempfile.mkdtemp(dir=project_dir) else: base_dir = project_dir - return os.path.basename(config.get_config_path(base_dir)) + filename, = config.get_default_config_files(base_dir) + return os.path.basename(filename) finally: shutil.rmtree(project_dir) From be0611bf3e435c9431d023b7f1fdef0e487554b0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 15 Sep 2015 14:47:33 -0400 Subject: [PATCH 09/12] Cleanup get_default_config_files tests. Signed-off-by: Daniel Nephin --- tests/unit/config/config_test.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 38eb3de23..79864ec78 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1167,7 +1167,7 @@ class BuildPathTest(unittest.TestCase): self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}]) -class GetConfigPathTestCase(unittest.TestCase): +class GetDefaultConfigFilesTestCase(unittest.TestCase): files = [ 'docker-compose.yml', @@ -1177,25 +1177,21 @@ class GetConfigPathTestCase(unittest.TestCase): ] def test_get_config_path_default_file_in_basedir(self): - files = self.files - self.assertEqual('docker-compose.yml', get_config_filename_for_files(files[0:])) - self.assertEqual('docker-compose.yaml', get_config_filename_for_files(files[1:])) - self.assertEqual('fig.yml', get_config_filename_for_files(files[2:])) - self.assertEqual('fig.yaml', get_config_filename_for_files(files[3:])) + for index, filename in enumerate(self.files): + self.assertEqual( + filename, + get_config_filename_for_files(self.files[index:])) with self.assertRaises(config.ComposeFileNotFound): get_config_filename_for_files([]) def test_get_config_path_default_file_in_parent_dir(self): """Test with files placed in the subdir""" - files = self.files def get_config_in_subdir(files): return get_config_filename_for_files(files, subdir=True) - self.assertEqual('docker-compose.yml', get_config_in_subdir(files[0:])) - self.assertEqual('docker-compose.yaml', get_config_in_subdir(files[1:])) - self.assertEqual('fig.yml', get_config_in_subdir(files[2:])) - self.assertEqual('fig.yaml', get_config_in_subdir(files[3:])) + for index, filename in enumerate(self.files): + self.assertEqual(filename, get_config_in_subdir(self.files[index:])) with self.assertRaises(config.ComposeFileNotFound): get_config_in_subdir([]) From fd75e4bf6385d33165f1c91af2f63d9a8201e530 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 16 Sep 2015 15:03:55 -0400 Subject: [PATCH 10/12] Update docs about using multiple -f arguments Signed-off-by: Daniel Nephin --- docs/reference/docker-compose.md | 62 ++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index b43055fbe..32fcbe706 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -14,7 +14,7 @@ weight=-2 ``` Usage: - docker-compose [options] [COMMAND] [ARGS...] + docker-compose [-f=...] [options] [COMMAND] [ARGS...] docker-compose -h|--help Options: @@ -41,20 +41,62 @@ Commands: unpause Unpause services up Create and start containers migrate-to-labels Recreate containers to add labels + version Show the Docker-Compose version information ``` -The Docker Compose binary. You use this command to build and manage multiple services in Docker containers. +The Docker Compose binary. You use this command to build and manage multiple +services in Docker containers. -Use the `-f` flag to specify the location of a Compose configuration file. This -flag is optional. If you don't provide this flag. Compose looks for a file named -`docker-compose.yml` in the working directory. If the file is not found, -Compose looks in each parent directory successively, until it finds the file. +Use the `-f` flag to specify the location of a Compose configuration file. You +can supply multiple `-f` configuration files. When you supply multiple files, +Compose combines them into a single configuration. Compose builds the +configuration in the order you supply the files. Subsequent files override and +add to their successors. -Use a `-` as the filename to read configuration file from stdin. When stdin is -used all paths in the configuration are relative to the current working -directory. +For example, consider this command line: + +``` +$ docker-compose -f docker-compose.yml -f docker-compose.admin.yml run backup_db` +``` + +The `docker-compose.yml` file might specify a `webapp` service. + +``` +webapp: + image: examples/web + ports: + - "8000:8000" + volumes: + - "/data" +``` + +If the `docker-compose.admin.yml` also specifies this same service, any matching +fields will override the previous file. New values, add to the `webapp` service +configuration. + +``` +webapp: + build: . + environment: + - DEBUG=1 +``` + +Use a `-f` with `-` (dash) as the filename to read the configuration from +stdin. When stdin is used all paths in the configuration are +relative to the current working directory. + +The `-f` flag is optional. If you don't provide this flag on the command line, +Compose traverses the working directory and its subdirectories looking for a +`docker-compose.yml` and a `docker-compose.override.yml` file. You must supply +at least the `docker-compose.yml` file. If both files are present, Compose +combines the two files into a single configuration. The configuration in the +`docker-compose.override.yml` file is applied over and in addition to the values +in the `docker-compose.yml` file. + +Each configuration has a project name. If you supply a `-p` flag, you can +specify a project name. If you don't specify the flag, Compose uses the current +directory name. -Each configuration can has a project name. If you supply a `-p` flag, you can specify a project name. If you don't specify the flag, Compose uses the current directory name. ## Where to go next From 577439ea7f6b506e6905ca097abeff7ba82af5e6 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 17 Sep 2015 14:28:16 -0400 Subject: [PATCH 11/12] Add a debug log message for config filenames. Signed-off-by: Daniel Nephin --- compose/config/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index 3ecdd29d7..56e6e796b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -104,6 +104,8 @@ def find(base_dir, filenames): filenames = [os.path.join(base_dir, f) for f in filenames] else: filenames = get_default_config_files(base_dir) + + log.debug("Using configuration files: {}".format(",".join(filenames))) return ConfigDetails( os.path.dirname(filenames[0]), [ConfigFile(f, load_yaml(f)) for f in filenames]) From 22bc174650fddb9b53ece787c489df7dabcef8d9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 17 Sep 2015 15:07:53 -0400 Subject: [PATCH 12/12] Refactor config.load() to remove reduce() and document some types. Signed-off-by: Daniel Nephin --- compose/config/config.py | 49 ++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 56e6e796b..94c5ab95a 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -2,7 +2,6 @@ import logging import os import sys from collections import namedtuple -from functools import reduce import six import yaml @@ -89,9 +88,22 @@ PATH_START_CHARS = [ log = logging.getLogger(__name__) -ConfigDetails = namedtuple('ConfigDetails', 'working_dir configs') +class ConfigDetails(namedtuple('_ConfigDetails', 'working_dir config_files')): + """ + :param working_dir: the directory to use for relative paths in the config + :type working_dir: string + :param config_files: list of configuration files to load + :type config_files: list of :class:`ConfigFile` + """ -ConfigFile = namedtuple('ConfigFile', 'filename config') + +class ConfigFile(namedtuple('_ConfigFile', 'filename config')): + """ + :param filename: filename of the config file + :type filename: string + :param config: contents of the config file + :type config: :class:`dict` + """ def find(base_dir, filenames): @@ -170,10 +182,19 @@ def pre_process_config(config): def load(config_details): - working_dir, configs = config_details + """Load the configuration from a working directory and a list of + configuration files. Files are loaded in order, and merged on top + of each other to create the final configuration. + + Return a fully interpolated, extended and validated configuration. + """ def build_service(filename, service_name, service_dict): - loader = ServiceLoader(working_dir, filename, service_name, service_dict) + loader = ServiceLoader( + config_details.working_dir, + filename, + service_name, + service_dict) service_dict = loader.make_service_dict() validate_paths(service_dict) return service_dict @@ -187,21 +208,19 @@ def load(config_details): ] def merge_services(base, override): + all_service_names = set(base) | set(override) return { name: merge_service_dicts(base.get(name, {}), override.get(name, {})) - for name in set(base) | set(override) + for name in all_service_names } - def combine_configs(base, override): - service_dicts = load_file(base.filename, base.config) - if not override: - return service_dicts + config_file = config_details.config_files[0] + for next_file in config_details.config_files[1:]: + config_file = ConfigFile( + config_file.filename, + merge_services(config_file.config, next_file.config)) - return ConfigFile( - override.filename, - merge_services(base.config, override.config)) - - return reduce(combine_configs, configs + [None]) + return load_file(config_file.filename, config_file.config) class ServiceLoader(object):