tyler.durden/compose
1
0
Fork 0
mirror of https://github.com/docker/compose.git synced 2025-07-28 16:14:06 +02:00
Code Issues Packages Projects Releases Wiki Activity

Support multiple config files

Browse Source 
Signed-off-by: Daniel Nephin <dnephin@docker.com>
This commit is contained in:
Daniel Nephin 2015-09-11 20:18:45 -04:00
parent 258d0fa0c6
commit 10b3188214
3 changed files with 105 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
compose/cli/command.py
View File

@ -51,24 +51,26 @@ class Command(DocoptCommand):
handler(None, command_options) handler(None, command_options)
return return
if 'FIG_FILE' in os.environ:
log.warn('The FIG_FILE environment variable is deprecated.')
log.warn('Please use COMPOSE_FILE instead.')
explicit_config_path = (
options.get('--file') or
os.environ.get('COMPOSE_FILE') or
os.environ.get('FIG_FILE'))
project = get_project( project = get_project(
self.base_dir, self.base_dir,
explicit_config_path, get_config_path(options.get('--file')),
project_name=options.get('--project-name'), project_name=options.get('--project-name'),
verbose=options.get('--verbose')) verbose=options.get('--verbose'))
handler(project, command_options) handler(project, command_options)
def get_config_path(file_option):
if file_option:
return file_option
if 'FIG_FILE' in os.environ:
log.warn('The FIG_FILE environment variable is deprecated.')
log.warn('Please use COMPOSE_FILE instead.')
return [os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')]
def get_client(verbose=False): def get_client(verbose=False):
client = docker_client() client = docker_client()
if verbose: if verbose:

64
compose/config/config.py
View File

@ -2,6 +2,7 @@ import logging
import os import os
import sys import sys
from collections import namedtuple from collections import namedtuple
from functools import reduce
import six import six
import yaml import yaml
@ -88,18 +89,24 @@ PATH_START_CHARS = [
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
ConfigDetails = namedtuple('ConfigDetails', 'config working_dir filename') ConfigDetails = namedtuple('ConfigDetails', 'working_dir configs')
ConfigFile = namedtuple('ConfigFile', 'filename config')
def find(base_dir, filename): def find(base_dir, filenames):
if filename == '-': if filenames == ['-']:
return ConfigDetails(yaml.safe_load(sys.stdin), os.getcwd(), None) return ConfigDetails(
os.getcwd(),
[ConfigFile(None, yaml.safe_load(sys.stdin))])
if filename: if filenames:
filename = os.path.join(base_dir, filename) filenames = [os.path.join(base_dir, f) for f in filenames]
else: else:
filename = get_config_path(base_dir) filenames = [get_config_path(base_dir)]
return ConfigDetails(load_yaml(filename), os.path.dirname(filename), filename) return ConfigDetails(
os.path.dirname(filenames[0]),
[ConfigFile(f, load_yaml(f)) for f in filenames])
def get_config_path(base_dir): def get_config_path(base_dir):
@ -133,30 +140,41 @@ def pre_process_config(config):
Pre validation checks and processing of the config file to interpolate env Pre validation checks and processing of the config file to interpolate env
vars returning a config dict ready to be tested against the schema. vars returning a config dict ready to be tested against the schema.
""" """
config = interpolate_environment_variables(config) return interpolate_environment_variables(config)
return config
def load(config_details): def load(config_details):
config, working_dir, filename = config_details working_dir, configs = config_details
processed_config = pre_process_config(config) def build_service(filename, service_name, service_dict):
validate_against_fields_schema(processed_config) loader = ServiceLoader(working_dir, filename, service_name, service_dict)
service_dicts = []
for service_name, service_dict in list(processed_config.items()):
loader = ServiceLoader(
working_dir=working_dir,
filename=filename,
service_name=service_name,
service_dict=service_dict)
service_dict = loader.make_service_dict() service_dict = loader.make_service_dict()
validate_paths(service_dict) validate_paths(service_dict)
service_dicts.append(service_dict) return service_dict
def load_file(filename, config):
processed_config = pre_process_config(config)
validate_against_fields_schema(processed_config)
return [
build_service(filename, name, service_config)
for name, service_config in processed_config.items()
]
def merge_services(base, override):
return {
name: merge_service_dicts(base.get(name, {}), override.get(name, {}))
for name in set(base) | set(override)
}
def combine_configs(override, base):
service_dicts = load_file(base.filename, base.config)
if not override:
return service_dicts return service_dicts
return merge_service_dicts(base.config, override.config)
return reduce(combine_configs, configs, None)
class ServiceLoader(object): class ServiceLoader(object):
def __init__(self, working_dir, filename, service_name, service_dict, already_seen=None): def __init__(self, working_dir, filename, service_name, service_dict, already_seen=None):

96
tests/unit/config_test.py
View File

@ -26,10 +26,16 @@ def service_sort(services):
return sorted(services, key=itemgetter('name')) return sorted(services, key=itemgetter('name'))
def build_config_details(contents, working_dir, filename):
return config.ConfigDetails(
working_dir,
[config.ConfigFile(filename, contents)])
class ConfigTest(unittest.TestCase): class ConfigTest(unittest.TestCase):
def test_load(self): def test_load(self):
service_dicts = config.load( service_dicts = config.load(
config.ConfigDetails( build_config_details(
{ {
'foo': {'image': 'busybox'}, 'foo': {'image': 'busybox'},
'bar': {'image': 'busybox', 'environment': ['FOO=1']}, 'bar': {'image': 'busybox', 'environment': ['FOO=1']},
@ -57,7 +63,7 @@ class ConfigTest(unittest.TestCase):
def test_load_throws_error_when_not_dict(self): def test_load_throws_error_when_not_dict(self):
with self.assertRaises(ConfigurationError): with self.assertRaises(ConfigurationError):
config.load( config.load(
config.ConfigDetails( build_config_details(
{'web': 'busybox:latest'}, {'web': 'busybox:latest'},
'working_dir', 'working_dir',
'filename.yml' 'filename.yml'
@ -68,7 +74,7 @@ class ConfigTest(unittest.TestCase):
with self.assertRaises(ConfigurationError): with self.assertRaises(ConfigurationError):
for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']:
config.load( config.load(
config.ConfigDetails( build_config_details(
{invalid_name: {'image': 'busybox'}}, {invalid_name: {'image': 'busybox'}},
'working_dir', 'working_dir',
'filename.yml' 'filename.yml'
@ -79,7 +85,7 @@ class ConfigTest(unittest.TestCase):
expected_error_msg = "Service name: 1 needs to be a string, eg '1'" expected_error_msg = "Service name: 1 needs to be a string, eg '1'"
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{1: {'image': 'busybox'}}, {1: {'image': 'busybox'}},
'working_dir', 'working_dir',
'filename.yml' 'filename.yml'
@ -89,7 +95,7 @@ class ConfigTest(unittest.TestCase):
def test_config_valid_service_names(self): def test_config_valid_service_names(self):
for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']:
config.load( config.load(
config.ConfigDetails( build_config_details(
{valid_name: {'image': 'busybox'}}, {valid_name: {'image': 'busybox'}},
'tests/fixtures/extends', 'tests/fixtures/extends',
'common.yml' 'common.yml'
@ -101,7 +107,7 @@ class ConfigTest(unittest.TestCase):
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]: for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]:
config.load( config.load(
config.ConfigDetails( build_config_details(
{'web': {'image': 'busybox', 'ports': invalid_ports}}, {'web': {'image': 'busybox', 'ports': invalid_ports}},
'working_dir', 'working_dir',
'filename.yml' 'filename.yml'
@ -112,7 +118,7 @@ class ConfigTest(unittest.TestCase):
valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], [8000], ["49153-49154:3002-3003"]] valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], [8000], ["49153-49154:3002-3003"]]
for ports in valid_ports: for ports in valid_ports:
config.load( config.load(
config.ConfigDetails( build_config_details(
{'web': {'image': 'busybox', 'ports': ports}}, {'web': {'image': 'busybox', 'ports': ports}},
'working_dir', 'working_dir',
'filename.yml' 'filename.yml'
@ -123,7 +129,7 @@ class ConfigTest(unittest.TestCase):
expected_error_msg = "(did you mean 'privileged'?)" expected_error_msg = "(did you mean 'privileged'?)"
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'foo': {'image': 'busybox', 'privilige': 'something'}, 'foo': {'image': 'busybox', 'privilige': 'something'},
}, },
@ -136,7 +142,7 @@ class ConfigTest(unittest.TestCase):
expected_error_msg = "Service 'foo' has both an image and build path specified." expected_error_msg = "Service 'foo' has both an image and build path specified."
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'foo': {'image': 'busybox', 'build': '.'}, 'foo': {'image': 'busybox', 'build': '.'},
}, },
@ -149,7 +155,7 @@ class ConfigTest(unittest.TestCase):
expected_error_msg = "Service 'foo' configuration key 'links' contains an invalid type, it should be an array" expected_error_msg = "Service 'foo' configuration key 'links' contains an invalid type, it should be an array"
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'foo': {'image': 'busybox', 'links': 'an_link'}, 'foo': {'image': 'busybox', 'links': 'an_link'},
}, },
@ -162,7 +168,7 @@ class ConfigTest(unittest.TestCase):
expected_error_msg = "Top level object needs to be a dictionary." expected_error_msg = "Top level object needs to be a dictionary."
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
['foo', 'lol'], ['foo', 'lol'],
'tests/fixtures/extends', 'tests/fixtures/extends',
'filename.yml' 'filename.yml'
@ -173,7 +179,7 @@ class ConfigTest(unittest.TestCase):
expected_error_msg = "has non-unique elements" expected_error_msg = "has non-unique elements"
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'web': {'build': '.', 'devices': ['/dev/foo:/dev/foo', '/dev/foo:/dev/foo']} 'web': {'build': '.', 'devices': ['/dev/foo:/dev/foo', '/dev/foo:/dev/foo']}
}, },
@ -187,7 +193,7 @@ class ConfigTest(unittest.TestCase):
expected_error_msg += ", which is an invalid type, it should be a string" expected_error_msg += ", which is an invalid type, it should be a string"
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'web': {'build': '.', 'command': [1]} 'web': {'build': '.', 'command': [1]}
}, },
@ -200,7 +206,7 @@ class ConfigTest(unittest.TestCase):
expected_error_msg = "Service 'web' has both an image and alternate Dockerfile." expected_error_msg = "Service 'web' has both an image and alternate Dockerfile."
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{'web': {'image': 'busybox', 'dockerfile': 'Dockerfile.alt'}}, {'web': {'image': 'busybox', 'dockerfile': 'Dockerfile.alt'}},
'working_dir', 'working_dir',
'filename.yml' 'filename.yml'
@ -212,7 +218,7 @@ class ConfigTest(unittest.TestCase):
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{'web': { {'web': {
'image': 'busybox', 'image': 'busybox',
'extra_hosts': 'somehost:162.242.195.82' 'extra_hosts': 'somehost:162.242.195.82'
@ -227,7 +233,7 @@ class ConfigTest(unittest.TestCase):
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{'web': { {'web': {
'image': 'busybox', 'image': 'busybox',
'extra_hosts': [ 'extra_hosts': [
@ -244,7 +250,7 @@ class ConfigTest(unittest.TestCase):
expose_values = [["8000"], [8000]] expose_values = [["8000"], [8000]]
for expose in expose_values: for expose in expose_values:
service = config.load( service = config.load(
config.ConfigDetails( build_config_details(
{'web': { {'web': {
'image': 'busybox', 'image': 'busybox',
'expose': expose 'expose': expose
@ -259,7 +265,7 @@ class ConfigTest(unittest.TestCase):
entrypoint_values = [["sh"], "sh"] entrypoint_values = [["sh"], "sh"]
for entrypoint in entrypoint_values: for entrypoint in entrypoint_values:
service = config.load( service = config.load(
config.ConfigDetails( build_config_details(
{'web': { {'web': {
'image': 'busybox', 'image': 'busybox',
'entrypoint': entrypoint 'entrypoint': entrypoint
@ -331,16 +337,16 @@ class InterpolationTest(unittest.TestCase):
def test_unset_variable_produces_warning(self): def test_unset_variable_produces_warning(self):
os.environ.pop('FOO', None) os.environ.pop('FOO', None)
os.environ.pop('BAR', None) os.environ.pop('BAR', None)
config_details = config.ConfigDetails( config_details = build_config_details(
config={ {
'web': { 'web': {
'image': '${FOO}', 'image': '${FOO}',
'command': '${BAR}', 'command': '${BAR}',
'container_name': '${BAR}', 'container_name': '${BAR}',
}, },
}, },
working_dir='.', '.',
filename=None, None,
) )
with mock.patch('compose.config.interpolation.log') as log: with mock.patch('compose.config.interpolation.log') as log:
@ -355,7 +361,7 @@ class InterpolationTest(unittest.TestCase):
def test_invalid_interpolation(self): def test_invalid_interpolation(self):
with self.assertRaises(config.ConfigurationError) as cm: with self.assertRaises(config.ConfigurationError) as cm:
config.load( config.load(
config.ConfigDetails( build_config_details(
{'web': {'image': '${'}}, {'web': {'image': '${'}},
'working_dir', 'working_dir',
'filename.yml' 'filename.yml'
@ -371,10 +377,10 @@ class InterpolationTest(unittest.TestCase):
def test_volume_binding_with_environment_variable(self): def test_volume_binding_with_environment_variable(self):
os.environ['VOLUME_PATH'] = '/host/path' os.environ['VOLUME_PATH'] = '/host/path'
d = config.load( d = config.load(
config.ConfigDetails( build_config_details(
config={'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}},
working_dir='.', '.',
filename=None, None,
) )
)[0] )[0]
self.assertEqual(d['volumes'], ['/host/path:/container/path']) self.assertEqual(d['volumes'], ['/host/path:/container/path'])
@ -649,7 +655,7 @@ class MemoryOptionsTest(unittest.TestCase):
) )
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'foo': {'image': 'busybox', 'memswap_limit': 2000000}, 'foo': {'image': 'busybox', 'memswap_limit': 2000000},
}, },
@ -660,7 +666,7 @@ class MemoryOptionsTest(unittest.TestCase):
def test_validation_with_correct_memswap_values(self): def test_validation_with_correct_memswap_values(self):
service_dict = config.load( service_dict = config.load(
config.ConfigDetails( build_config_details(
{'foo': {'image': 'busybox', 'mem_limit': 1000000, 'memswap_limit': 2000000}}, {'foo': {'image': 'busybox', 'mem_limit': 1000000, 'memswap_limit': 2000000}},
'tests/fixtures/extends', 'tests/fixtures/extends',
'common.yml' 'common.yml'
@ -670,7 +676,7 @@ class MemoryOptionsTest(unittest.TestCase):
def test_memswap_can_be_a_string(self): def test_memswap_can_be_a_string(self):
service_dict = config.load( service_dict = config.load(
config.ConfigDetails( build_config_details(
{'foo': {'image': 'busybox', 'mem_limit': "1G", 'memswap_limit': "512M"}}, {'foo': {'image': 'busybox', 'mem_limit': "1G", 'memswap_limit': "512M"}},
'tests/fixtures/extends', 'tests/fixtures/extends',
'common.yml' 'common.yml'
@ -780,26 +786,26 @@ class EnvTest(unittest.TestCase):
os.environ['CONTAINERENV'] = '/host/tmp' os.environ['CONTAINERENV'] = '/host/tmp'
service_dict = config.load( service_dict = config.load(
config.ConfigDetails( build_config_details(
config={'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}},
working_dir="tests/fixtures/env", "tests/fixtures/env",
filename=None, None,
) )
)[0] )[0]
self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp'])) self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp']))
service_dict = config.load( service_dict = config.load(
config.ConfigDetails( build_config_details(
config={'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}},
working_dir="tests/fixtures/env", "tests/fixtures/env",
filename=None, None,
) )
)[0] )[0]
self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp'])) self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp']))
def load_from_filename(filename): def load_from_filename(filename):
return config.load(config.find('.', filename)) return config.load(config.find('.', [filename]))
class ExtendsTest(unittest.TestCase): class ExtendsTest(unittest.TestCase):
@ -885,7 +891,7 @@ class ExtendsTest(unittest.TestCase):
def test_extends_validation_empty_dictionary(self): def test_extends_validation_empty_dictionary(self):
with self.assertRaisesRegexp(ConfigurationError, 'service'): with self.assertRaisesRegexp(ConfigurationError, 'service'):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'web': {'image': 'busybox', 'extends': {}}, 'web': {'image': 'busybox', 'extends': {}},
}, },
@ -897,7 +903,7 @@ class ExtendsTest(unittest.TestCase):
def test_extends_validation_missing_service_key(self): def test_extends_validation_missing_service_key(self):
with self.assertRaisesRegexp(ConfigurationError, "'service' is a required property"): with self.assertRaisesRegexp(ConfigurationError, "'service' is a required property"):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'web': {'image': 'busybox', 'extends': {'file': 'common.yml'}}, 'web': {'image': 'busybox', 'extends': {'file': 'common.yml'}},
}, },
@ -910,7 +916,7 @@ class ExtendsTest(unittest.TestCase):
expected_error_msg = "Unsupported config option for 'web' service: 'rogue_key'" expected_error_msg = "Unsupported config option for 'web' service: 'rogue_key'"
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'web': { 'web': {
'image': 'busybox', 'image': 'busybox',
@ -930,7 +936,7 @@ class ExtendsTest(unittest.TestCase):
expected_error_msg = "Service 'web' configuration key 'extends' 'file' contains an invalid type" expected_error_msg = "Service 'web' configuration key 'extends' 'file' contains an invalid type"
with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'web': { 'web': {
'image': 'busybox', 'image': 'busybox',
@ -955,7 +961,7 @@ class ExtendsTest(unittest.TestCase):
def test_extends_validation_valid_config(self): def test_extends_validation_valid_config(self):
service = config.load( service = config.load(
config.ConfigDetails( build_config_details(
{ {
'web': {'image': 'busybox', 'extends': {'service': 'web', 'file': 'common.yml'}}, 'web': {'image': 'busybox', 'extends': {'service': 'web', 'file': 'common.yml'}},
}, },
@ -1093,7 +1099,7 @@ class BuildPathTest(unittest.TestCase):
def test_nonexistent_path(self): def test_nonexistent_path(self):
with self.assertRaises(ConfigurationError): with self.assertRaises(ConfigurationError):
config.load( config.load(
config.ConfigDetails( build_config_details(
{ {
'foo': {'build': 'nonexistent.path'}, 'foo': {'build': 'nonexistent.path'},
}, },