From 4c582e4352f056c70b87abb4cb2d51bc231dec74 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 18 Mar 2015 13:51:27 -0700 Subject: [PATCH 1/2] Implement `extends` Signed-off-by: Aanand Prasad --- compose/config.py | 163 ++++++++++++++++-- compose/project.py | 13 +- docs/yml.md | 77 +++++++++ tests/fixtures/extends/circle-1.yml | 12 ++ tests/fixtures/extends/circle-2.yml | 12 ++ tests/fixtures/extends/common.yml | 6 + tests/fixtures/extends/docker-compose.yml | 16 ++ .../fixtures/extends/nested-intermediate.yml | 6 + tests/fixtures/extends/nested.yml | 6 + tests/integration/cli_test.py | 27 +++ tests/unit/config_test.py | 111 ++++++++++++ 11 files changed, 421 insertions(+), 28 deletions(-) create mode 100644 tests/fixtures/extends/circle-1.yml create mode 100644 tests/fixtures/extends/circle-2.yml create mode 100644 tests/fixtures/extends/common.yml create mode 100644 tests/fixtures/extends/docker-compose.yml create mode 100644 tests/fixtures/extends/nested-intermediate.yml create mode 100644 tests/fixtures/extends/nested.yml diff --git a/compose/config.py b/compose/config.py index a4e3a991f..cfa5ce44a 100644 --- a/compose/config.py +++ b/compose/config.py @@ -52,34 +52,110 @@ DOCKER_CONFIG_HINTS = { def load(filename): working_dir = os.path.dirname(filename) - return from_dictionary(load_yaml(filename), working_dir=working_dir) + return from_dictionary(load_yaml(filename), working_dir=working_dir, filename=filename) -def load_yaml(filename): - try: - with open(filename, 'r') as fh: - return yaml.safe_load(fh) - except IOError as e: - raise ConfigurationError(six.text_type(e)) - - -def from_dictionary(dictionary, working_dir=None): +def from_dictionary(dictionary, working_dir=None, filename=None): service_dicts = [] for service_name, service_dict in list(dictionary.items()): if not isinstance(service_dict, dict): raise ConfigurationError('Service "%s" doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.' % service_name) - service_dict = make_service_dict(service_name, service_dict, working_dir=working_dir) + loader = ServiceLoader(working_dir=working_dir, filename=filename) + service_dict = loader.make_service_dict(service_name, service_dict) service_dicts.append(service_dict) return service_dicts -def make_service_dict(name, options, working_dir=None): - service_dict = options.copy() - service_dict['name'] = name - service_dict = resolve_environment(service_dict, working_dir=working_dir) - return process_container_options(service_dict, working_dir=working_dir) +def make_service_dict(name, service_dict, working_dir=None): + return ServiceLoader(working_dir=working_dir).make_service_dict(name, service_dict) + + +class ServiceLoader(object): + def __init__(self, working_dir, filename=None, already_seen=None): + self.working_dir = working_dir + self.filename = filename + self.already_seen = already_seen or [] + + def make_service_dict(self, name, service_dict): + if self.signature(name) in self.already_seen: + raise CircularReference(self.already_seen) + + service_dict = service_dict.copy() + service_dict['name'] = name + service_dict = resolve_environment(service_dict, working_dir=self.working_dir) + service_dict = self.resolve_extends(service_dict) + return process_container_options(service_dict, working_dir=self.working_dir) + + def resolve_extends(self, service_dict): + if 'extends' not in service_dict: + return service_dict + + extends_options = process_extends_options(service_dict['name'], service_dict['extends']) + + if self.working_dir is None: + raise Exception("No working_dir passed to ServiceLoader()") + + other_config_path = expand_path(self.working_dir, extends_options['file']) + other_working_dir = os.path.dirname(other_config_path) + other_already_seen = self.already_seen + [self.signature(service_dict['name'])] + other_loader = ServiceLoader( + working_dir=other_working_dir, + filename=other_config_path, + already_seen=other_already_seen, + ) + + other_config = load_yaml(other_config_path) + other_service_dict = other_config[extends_options['service']] + other_service_dict = other_loader.make_service_dict( + service_dict['name'], + other_service_dict, + ) + validate_extended_service_dict( + other_service_dict, + filename=other_config_path, + service=extends_options['service'], + ) + + return merge_service_dicts(other_service_dict, service_dict) + + def signature(self, name): + return (self.filename, name) + + +def process_extends_options(service_name, extends_options): + error_prefix = "Invalid 'extends' configuration for %s:" % service_name + + if not isinstance(extends_options, dict): + raise ConfigurationError("%s must be a dictionary" % error_prefix) + + if 'service' not in extends_options: + raise ConfigurationError( + "%s you need to specify a service, e.g. 'service: web'" % error_prefix + ) + + for k, _ in extends_options.items(): + if k not in ['file', 'service']: + raise ConfigurationError( + "%s unsupported configuration option '%s'" % (error_prefix, k) + ) + + return extends_options + + +def validate_extended_service_dict(service_dict, filename, service): + error_prefix = "Cannot extend service '%s' in %s:" % (service, filename) + + if 'links' in service_dict: + raise ConfigurationError("%s services with 'links' cannot be extended" % error_prefix) + + if 'volumes_from' in service_dict: + raise ConfigurationError("%s services with 'volumes_from' cannot be extended" % error_prefix) + + if 'net' in service_dict: + if get_service_name_from_net(service_dict['net']) is not None: + raise ConfigurationError("%s services with 'net: container' cannot be extended" % error_prefix) def process_container_options(service_dict, working_dir=None): @@ -93,6 +169,29 @@ def process_container_options(service_dict, working_dir=None): return service_dict +def merge_service_dicts(base, override): + d = base.copy() + + if 'environment' in base or 'environment' in override: + d['environment'] = merge_environment( + base.get('environment'), + override.get('environment'), + ) + + for k in ALLOWED_KEYS: + if k not in ['environment']: + if k in override: + d[k] = override[k] + + return d + + +def merge_environment(base, override): + env = parse_environment(base) + env.update(parse_environment(override)) + return env + + def parse_links(links): return dict(parse_link(l) for l in links) @@ -190,9 +289,41 @@ def expand_path(working_dir, path): return os.path.abspath(os.path.join(working_dir, path)) +def get_service_name_from_net(net_config): + if not net_config: + return + + if not net_config.startswith('container:'): + return + + _, net_name = net_config.split(':', 1) + return net_name + + +def load_yaml(filename): + try: + with open(filename, 'r') as fh: + return yaml.safe_load(fh) + except IOError as e: + raise ConfigurationError(six.text_type(e)) + + class ConfigurationError(Exception): def __init__(self, msg): self.msg = msg def __str__(self): return self.msg + + +class CircularReference(ConfigurationError): + def __init__(self, trail): + self.trail = trail + + @property + def msg(self): + lines = [ + "{} in {}".format(service_name, filename) + for (filename, service_name) in self.trail + ] + return "Circular reference:\n {}".format("\n extends ".join(lines)) diff --git a/compose/project.py b/compose/project.py index 881d8eb0a..7c0d19da3 100644 --- a/compose/project.py +++ b/compose/project.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import logging from functools import reduce -from .config import ConfigurationError +from .config import get_service_name_from_net, ConfigurationError from .service import Service from .container import Container from docker.errors import APIError @@ -11,17 +11,6 @@ from docker.errors import APIError log = logging.getLogger(__name__) -def get_service_name_from_net(net_config): - if not net_config: - return - - if not net_config.startswith('container:'): - return - - _, net_name = net_config.split(':', 1) - return net_name - - def sort_service_dicts(services): # Topological sort (Cormen/Tarjan algorithm). unmarked = services[:] diff --git a/docs/yml.md b/docs/yml.md index 52be706a0..157ba4e67 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -176,6 +176,83 @@ env_file: RACK_ENV: development ``` +### extends + +Extend another service, in the current file or another, optionally overriding +configuration. + +Here's a simple example. Suppose we have 2 files - **common.yml** and +**development.yml**. We can use `extends` to define a service in +**development.yml** which uses configuration defined in **common.yml**: + +**common.yml** + +``` +webapp: + build: ./webapp + environment: + - DEBUG=false + - SEND_EMAILS=false +``` + +**development.yml** + +``` +web: + extends: + file: common.yml + service: webapp + ports: + - "8000:8000" + links: + - db + environment: + - DEBUG=true +db: + image: postgres +``` + +Here, the `web` service in **development.yml** inherits the configuration of +the `webapp` service in **common.yml** - the `build` and `environment` keys - +and adds `ports` and `links` configuration. It overrides one of the defined +environment variables (DEBUG) with a new value, and the other one +(SEND_EMAILS) is left untouched. It's exactly as if you defined `web` like +this: + +```yaml +web: + build: ./webapp + ports: + - "8000:8000" + links: + - db + environment: + - DEBUG=true + - SEND_EMAILS=false +``` + +The `extends` option is great for sharing configuration between different +apps, or for configuring the same app differently for different environments. +You could write a new file for a staging environment, **staging.yml**, which +binds to a different port and doesn't turn on debugging: + +``` +web: + extends: + file: common.yml + service: webapp + ports: + - "80:8000" + links: + - db +db: + image: postgres +``` + +> **Note:** When you extend a service, `links` and `volumes_from` +> configuration options are **not** inherited - you will have to define +> those manually each time you extend it. + ### net Networking mode. Use the same values as the docker client `--net` parameter. diff --git a/tests/fixtures/extends/circle-1.yml b/tests/fixtures/extends/circle-1.yml new file mode 100644 index 000000000..a034e9619 --- /dev/null +++ b/tests/fixtures/extends/circle-1.yml @@ -0,0 +1,12 @@ +foo: + image: busybox +bar: + image: busybox +web: + extends: + file: circle-2.yml + service: web +baz: + image: busybox +quux: + image: busybox diff --git a/tests/fixtures/extends/circle-2.yml b/tests/fixtures/extends/circle-2.yml new file mode 100644 index 000000000..fa6ddefcc --- /dev/null +++ b/tests/fixtures/extends/circle-2.yml @@ -0,0 +1,12 @@ +foo: + image: busybox +bar: + image: busybox +web: + extends: + file: circle-1.yml + service: web +baz: + image: busybox +quux: + image: busybox diff --git a/tests/fixtures/extends/common.yml b/tests/fixtures/extends/common.yml new file mode 100644 index 000000000..358ef5bcc --- /dev/null +++ b/tests/fixtures/extends/common.yml @@ -0,0 +1,6 @@ +web: + image: busybox + command: /bin/true + environment: + - FOO=1 + - BAR=1 diff --git a/tests/fixtures/extends/docker-compose.yml b/tests/fixtures/extends/docker-compose.yml new file mode 100644 index 000000000..0ae92d2a5 --- /dev/null +++ b/tests/fixtures/extends/docker-compose.yml @@ -0,0 +1,16 @@ +myweb: + extends: + file: common.yml + service: web + command: sleep 300 + links: + - "mydb:db" + environment: + # leave FOO alone + # override BAR + BAR: "2" + # add BAZ + BAZ: "2" +mydb: + image: busybox + command: sleep 300 diff --git a/tests/fixtures/extends/nested-intermediate.yml b/tests/fixtures/extends/nested-intermediate.yml new file mode 100644 index 000000000..c2dd8c943 --- /dev/null +++ b/tests/fixtures/extends/nested-intermediate.yml @@ -0,0 +1,6 @@ +webintermediate: + extends: + file: common.yml + service: web + environment: + - "FOO=2" diff --git a/tests/fixtures/extends/nested.yml b/tests/fixtures/extends/nested.yml new file mode 100644 index 000000000..6025e6d53 --- /dev/null +++ b/tests/fixtures/extends/nested.yml @@ -0,0 +1,6 @@ +myweb: + extends: + file: nested-intermediate.yml + service: webintermediate + environment: + - "BAR=2" diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 2f961d2b2..df3eec66d 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -427,3 +427,30 @@ class CLITestCase(DockerClientTestCase): containers = self.project.containers(stopped=True) self.assertEqual(len(containers), 1) self.assertIn("FOO=1", containers[0].get('Config.Env')) + + def test_up_with_extends(self): + self.command.base_dir = 'tests/fixtures/extends' + self.command.dispatch(['up', '-d'], None) + + self.assertEqual( + set([s.name for s in self.project.services]), + set(['mydb', 'myweb']), + ) + + # Sort by name so we get [db, web] + containers = sorted( + self.project.containers(stopped=True), + key=lambda c: c.name, + ) + + self.assertEqual(len(containers), 2) + web = containers[1] + + self.assertEqual(set(web.links()), set(['db', 'mydb_1', 'extends_mydb_1'])) + + expected_env = set([ + "FOO=1", + "BAR=2", + "BAZ=2", + ]) + self.assertTrue(expected_env <= set(web.get('Config.Env'))) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 4ff08a9ef..7e18e2e92 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -38,6 +38,8 @@ class ConfigTest(unittest.TestCase): ) config.make_service_dict('foo', {'ports': ['8000']}) + +class EnvTest(unittest.TestCase): def test_parse_environment_as_list(self): environment =[ 'NORMAL=F1', @@ -130,3 +132,112 @@ class ConfigTest(unittest.TestCase): service_dict['environment'], {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}, ) + + +class ExtendsTest(unittest.TestCase): + def test_extends(self): + service_dicts = config.load('tests/fixtures/extends/docker-compose.yml') + + service_dicts = sorted( + service_dicts, + key=lambda sd: sd['name'], + ) + + self.assertEqual(service_dicts, [ + { + 'name': 'mydb', + 'image': 'busybox', + 'command': 'sleep 300', + }, + { + 'name': 'myweb', + 'image': 'busybox', + 'command': 'sleep 300', + 'links': ['mydb:db'], + 'environment': { + "FOO": "1", + "BAR": "2", + "BAZ": "2", + }, + } + ]) + + def test_nested(self): + service_dicts = config.load('tests/fixtures/extends/nested.yml') + + self.assertEqual(service_dicts, [ + { + 'name': 'myweb', + 'image': 'busybox', + 'command': '/bin/true', + 'environment': { + "FOO": "2", + "BAR": "2", + }, + }, + ]) + + def test_circular(self): + try: + config.load('tests/fixtures/extends/circle-1.yml') + raise Exception("Expected config.CircularReference to be raised") + except config.CircularReference as e: + self.assertEqual( + [(os.path.basename(filename), service_name) for (filename, service_name) in e.trail], + [ + ('circle-1.yml', 'web'), + ('circle-2.yml', 'web'), + ('circle-1.yml', 'web'), + ], + ) + + + def test_extends_validation(self): + dictionary = {'extends': None} + load_config = lambda: config.make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') + + self.assertRaisesRegexp(config.ConfigurationError, 'dictionary', load_config) + + dictionary['extends'] = {} + self.assertRaises(config.ConfigurationError, load_config) + + dictionary['extends']['file'] = 'common.yml' + self.assertRaisesRegexp(config.ConfigurationError, 'service', load_config) + + dictionary['extends']['service'] = 'web' + self.assertIsInstance(load_config(), dict) + + dictionary['extends']['what'] = 'is this' + self.assertRaisesRegexp(config.ConfigurationError, 'what', load_config) + + def test_blacklisted_options(self): + def load_config(): + return config.make_service_dict('myweb', { + 'extends': { + 'file': 'whatever', + 'service': 'web', + } + }, '.') + + with self.assertRaisesRegexp(config.ConfigurationError, 'links'): + other_config = {'web': {'links': ['db']}} + + with mock.patch.object(config, 'load_yaml', return_value=other_config): + print load_config() + + with self.assertRaisesRegexp(config.ConfigurationError, 'volumes_from'): + other_config = {'web': {'volumes_from': ['db']}} + + with mock.patch.object(config, 'load_yaml', return_value=other_config): + print load_config() + + with self.assertRaisesRegexp(config.ConfigurationError, 'net'): + other_config = {'web': {'net': 'container:db'}} + + with mock.patch.object(config, 'load_yaml', return_value=other_config): + print load_config() + + other_config = {'web': {'net': 'host'}} + + with mock.patch.object(config, 'load_yaml', return_value=other_config): + print load_config() From 37efdb1f8bfa5952fd15a3050f46c00c3fa582f0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 18 Mar 2015 18:23:17 -0700 Subject: [PATCH 2/2] Make volume host paths relative to file, merge volumes when extending Signed-off-by: Aanand Prasad --- compose/config.py | 49 ++++++++++++++++++- .../fixtures/volume-path/common/services.yml | 5 ++ tests/fixtures/volume-path/docker-compose.yml | 6 +++ tests/integration/project_test.py | 21 ++++---- tests/unit/config_test.py | 11 ++++- 5 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 tests/fixtures/volume-path/common/services.yml create mode 100644 tests/fixtures/volume-path/docker-compose.yml diff --git a/compose/config.py b/compose/config.py index cfa5ce44a..668d2b726 100644 --- a/compose/config.py +++ b/compose/config.py @@ -166,6 +166,11 @@ def process_container_options(service_dict, working_dir=None): msg += " (did you mean '%s'?)" % DOCKER_CONFIG_HINTS[k] raise ConfigurationError(msg) + service_dict = service_dict.copy() + + if 'volumes' in service_dict: + service_dict['volumes'] = resolve_host_paths(service_dict['volumes'], working_dir=working_dir) + return service_dict @@ -178,8 +183,14 @@ def merge_service_dicts(base, override): override.get('environment'), ) + if 'volumes' in base or 'volumes' in override: + d['volumes'] = merge_volumes( + base.get('volumes'), + override.get('volumes'), + ) + for k in ALLOWED_KEYS: - if k not in ['environment']: + if k not in ['environment', 'volumes']: if k in override: d[k] = override[k] @@ -285,6 +296,42 @@ def env_vars_from_file(filename): return env +def resolve_host_paths(volumes, working_dir=None): + if working_dir is None: + raise Exception("No working_dir passed to resolve_host_paths()") + + return [resolve_host_path(v, working_dir) for v in volumes] + + +def resolve_host_path(volume, working_dir): + container_path, host_path = split_volume(volume) + if host_path is not None: + return "%s:%s" % (expand_path(working_dir, host_path), container_path) + else: + return container_path + + +def merge_volumes(base, override): + d = dict_from_volumes(base) + d.update(dict_from_volumes(override)) + return volumes_from_dict(d) + + +def dict_from_volumes(volumes): + return dict(split_volume(v) for v in volumes) + + +def split_volume(volume): + if ':' in volume: + return reversed(volume.split(':', 1)) + else: + return (volume, None) + + +def volumes_from_dict(d): + return ["%s:%s" % (host_path, container_path) for (container_path, host_path) in d.items()] + + def expand_path(working_dir, path): return os.path.abspath(os.path.join(working_dir, path)) diff --git a/tests/fixtures/volume-path/common/services.yml b/tests/fixtures/volume-path/common/services.yml new file mode 100644 index 000000000..2dbf75961 --- /dev/null +++ b/tests/fixtures/volume-path/common/services.yml @@ -0,0 +1,5 @@ +db: + image: busybox + volumes: + - ./foo:/foo + - ./bar:/bar diff --git a/tests/fixtures/volume-path/docker-compose.yml b/tests/fixtures/volume-path/docker-compose.yml new file mode 100644 index 000000000..af433c52f --- /dev/null +++ b/tests/fixtures/volume-path/docker-compose.yml @@ -0,0 +1,6 @@ +db: + extends: + file: common/services.yml + service: db + volumes: + - ./bar:/bar diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 3212b0be9..a585740f4 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -7,18 +7,19 @@ from .testcases import DockerClientTestCase class ProjectTest(DockerClientTestCase): def test_volumes_from_service(self): + service_dicts = config.from_dictionary({ + 'data': { + 'image': 'busybox:latest', + 'volumes': ['/var/data'], + }, + 'db': { + 'image': 'busybox:latest', + 'volumes_from': ['data'], + }, + }, working_dir='.') project = Project.from_dicts( name='composetest', - service_dicts=config.from_dictionary({ - 'data': { - 'image': 'busybox:latest', - 'volumes': ['/var/data'], - }, - 'db': { - 'image': 'busybox:latest', - 'volumes_from': ['data'], - }, - }), + service_dicts=service_dicts, client=self.client, ) db = project.get_service('db') diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 7e18e2e92..013ad5031 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -133,7 +133,6 @@ class EnvTest(unittest.TestCase): {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}, ) - class ExtendsTest(unittest.TestCase): def test_extends(self): service_dicts = config.load('tests/fixtures/extends/docker-compose.yml') @@ -241,3 +240,13 @@ class ExtendsTest(unittest.TestCase): with mock.patch.object(config, 'load_yaml', return_value=other_config): print load_config() + + def test_volume_path(self): + dicts = config.load('tests/fixtures/volume-path/docker-compose.yml') + + paths = [ + '%s:/foo' % os.path.abspath('tests/fixtures/volume-path/common/foo'), + '%s:/bar' % os.path.abspath('tests/fixtures/volume-path/bar'), + ] + + self.assertEqual(set(dicts[0]['volumes']), set(paths))