tyler.durden
/
compose
mirror of https://github.com/docker/compose.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for declaring named volumes in compose files 

* Bump default API version to 1.21 (required for named volume management)
* Introduce new, versioned compose file format while maintaining support
  for current (legacy) format
* Test updates to reflect changes made to the internal API

Signed-off-by: Joffrey F <joffrey@docker.com>
This commit is contained in:
Joffrey F 2015-11-16 19:21:56 -08:00
parent 4bf2f8c4f9
commit b4be7b870f
14 changed files with 262 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
compose/cli/command.py
View File

@ -80,12 +80,13 @@ def get_project(base_dir, config_path=None, project_name=None, verbose=False,
config_details = config.find(base_dir, config_path)
api_version = '1.21' if use_networking else None
return Project.from_dicts(
return Project.from_config(
get_project_name(config_details.working_dir, project_name),
config.load(config_details),
get_client(verbose=verbose, version=api_version),
use_networking=use_networking,
network_driver=network_driver)
network_driver=network_driver
)
def get_project_name(working_dir, project_name=None):

3
compose/cli/docker_client.py
View File

@ -8,8 +8,7 @@ from ..const import HTTP_TIMEOUT
log = logging.getLogger(__name__)
DEFAULT_API_VERSION = '1.20'
DEFAULT_API_VERSION = '1.21'
def docker_client(version=None):

79
compose/config/config.py
View File

@ -117,6 +117,17 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
return cls(filename, load_yaml(filename))
class Config(namedtuple('_Config', 'version services volumes')):
"""
:param version: configuration version
:type version: int
:param services: List of service description dictionaries
:type services: :class:`list`
:param volumes: List of volume description dictionaries
:type volumes: :class:`list`
"""
class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name config')):
@classmethod
@ -148,6 +159,24 @@ def find(base_dir, filenames):
[ConfigFile.from_filename(f) for f in filenames])
def get_config_version(config_details):
def get_version(config):
validate_top_level_object(config)
return config.config.get('version')
main_file = config_details.config_files[0]
version = get_version(main_file)
for next_file in config_details.config_files[1:]:
next_file_version = get_version(next_file)
if version != next_file_version:
raise ConfigurationError(
"Version mismatch: main file {0} specifies version {1} but "
"extension file {2} uses version {3}".format(
main_file.filename, version, next_file.filename, next_file_version
)
)
return version
def get_default_config_files(base_dir):
(candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir)
@ -194,10 +223,46 @@ def load(config_details):
Return a fully interpolated, extended and validated configuration.
"""
version = get_config_version(config_details)
processed_files = []
for config_file in config_details.config_files:
processed_files.append(
process_config_file(config_file, version=version)
)
config_details = config_details._replace(config_files=processed_files)
if not version or isinstance(version, dict):
service_dicts = load_services(
config_details.working_dir, config_details.config_files
)
volumes = {}
elif version == 2:
config_files = [
ConfigFile(f.filename, f.config.get('services', {}))
for f in config_details.config_files
]
service_dicts = load_services(
config_details.working_dir, config_files
)
volumes = load_volumes(config_details.config_files)
else:
raise ConfigurationError('Invalid config version provided: {0}'.format(version))
return Config(version, service_dicts, volumes)
def load_volumes(config_files):
volumes = {}
for config_file in config_files:
for name, volume_config in config_file.config.get('volumes', {}).items():
volumes.update({name: volume_config})
return volumes
def load_services(working_dir, config_files):
def build_service(filename, service_name, service_dict):
service_config = ServiceConfig.with_abs_paths(
config_details.working_dir,
working_dir,
filename,
service_name,
service_dict)
@ -227,20 +292,20 @@ def load(config_details):
for name in all_service_names
}
config_file = process_config_file(config_details.config_files[0])
for next_file in config_details.config_files[1:]:
next_file = process_config_file(next_file)
config_file = config_files[0]
for next_file in config_files[1:]:
config = merge_services(config_file.config, next_file.config)
config_file = config_file._replace(config=config)
return build_services(config_file)
def process_config_file(config_file, service_name=None):
def process_config_file(config_file, service_name=None, version=None):
validate_top_level_object(config_file)
processed_config = interpolate_environment_variables(config_file.config)
validate_against_fields_schema(processed_config, config_file.filename)
validate_against_fields_schema(
processed_config, config_file.filename, version
)
if service_name and service_name not in processed_config:
raise ConfigurationError(

43
compose/config/fields_schema_v2.json Normal file
View File

@ -0,0 +1,43 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"version": {
"enum": [2]
},
"services": {
"type": "object",
"patternProperties": {
"^[a-zA-Z0-9._-]+$": {
"$ref": "fields_schema.json#/definitions/service"
}
}
},
"volumes": {
"type": "object",
"patternProperties": {
"^[a-zA-Z0-9._-]+$": {
"$ref": "#/definitions/volume"
}
}
}
},
"definitions": {
"volume": {
"type": "object",
"properties": {
"driver": {"type": "string"},
"driver_opts": {
"type": "object",
"patternProperties": {
"^.+$": {"type": ["boolean", "string", "number"]}
},
"additionalProperties": false
}
}
}
},
"additionalProperties": false
}

9
compose/config/validation.py
View File

@ -281,11 +281,14 @@ def process_errors(errors, service_name=None):
return '\n'.join(format_error_message(error, service_name) for error in errors)
def validate_against_fields_schema(config, filename):
def validate_against_fields_schema(config, filename, version=None):
schema_filename = "fields_schema.json"
if version:
schema_filename = "fields_schema_v{0}.json".format(version)
_validate_against_schema(
config,
"fields_schema.json",
format_checker=["ports", "expose", "bool-value-in-mapping"],
schema_filename,
format_checker=["ports", "environment", "bool-value-in-mapping"],
filename=filename)

31
compose/project.py
View File

@ -20,6 +20,7 @@ from .service import ConvergenceStrategy
from .service import Net
from .service import Service
from .service import ServiceNet
from .volume import Volume
log = logging.getLogger(__name__)
@ -29,12 +30,13 @@ class Project(object):
"""
A collection of services.
"""
def __init__(self, name, services, client, use_networking=False, network_driver=None):
def __init__(self, name, services, client, volumes=None, use_networking=False, network_driver=None):
self.name = name
self.services = services
self.client = client
self.use_networking = use_networking
self.network_driver = network_driver
self.volumes = volumes or []
def labels(self, one_off=False):
return [
@ -43,16 +45,16 @@ class Project(object):
]
@classmethod
def from_dicts(cls, name, service_dicts, client, use_networking=False, network_driver=None):
def from_config(cls, name, config_data, client, use_networking=False, network_driver=None):
"""
Construct a ServiceCollection from a list of dicts representing services.
Construct a Project from a config.Config object.
"""
project = cls(name, [], client, use_networking=use_networking, network_driver=network_driver)
if use_networking:
remove_links(service_dicts)
remove_links(config_data.services)
for service_dict in service_dicts:
for service_dict in config_data.services:
links = project.get_links(service_dict)
volumes_from = project.get_volumes_from(service_dict)
net = project.get_net(service_dict)
@ -66,6 +68,14 @@ class Project(object):
net=net,
volumes_from=volumes_from,
**service_dict))
if config_data.volumes:
for vol_name, data in config_data.volumes.items():
project.volumes.append(
Volume(
client=client, project=name, name=vol_name,
driver=data.get('driver'), driver_opts=data.get('driver_opts')
)
)
return project
@property
@ -218,6 +228,15 @@ class Project(object):
def remove_stopped(self, service_names=None, **options):
parallel.parallel_remove(self.containers(service_names, stopped=True), options)
def initialize_volumes(self):
try:
for volume in self.volumes:
volume.create()
except NotFound:
raise ConfigurationError(
'Volume %s sepcifies nonexistent driver %s' % (volume.name, volume.driver)
)
def restart(self, service_names=None, **options):
containers = self.containers(service_names, stopped=True)
parallel.parallel_restart(containers, options)
@ -253,6 +272,8 @@ class Project(object):
if self.use_networking and self.uses_default_network():
self.ensure_network_exists()
self.initialize_volumes()
return [
container
for service in services

19
compose/volume.py Normal file
View File

@ -0,0 +1,19 @@
from __future__ import unicode_literals
class Volume(object):
def __init__(self, client, project, name, driver=None, driver_opts=None):
self.client = client
self.project = project
self.name = name
self.driver = driver
self.driver_opts = driver_opts
def create(self):
return self.client.create_volume(self.name, self.driver, self.driver_opts)
def remove(self):
return self.client.remove_volume(self.name)
def inspect(self):
return self.client.inspect_volume(self.name)

3
requirements.txt
View File

@ -1,5 +1,6 @@
-e git://github.com/docker/docker-py.git@881e24c231ab9921eb0cbd475e85706137983f89#egg=docker-py
PyYAML==3.11
docker-py==1.5.0
# docker-py==1.5.1
dockerpty==0.3.4
docopt==0.6.1
enum34==1.0.4

25
tests/integration/project_test.py
View File

@ -69,9 +69,9 @@ class ProjectTest(DockerClientTestCase):
'volumes_from': ['data'],
},
})
project = Project.from_dicts(
project = Project.from_config(
name='composetest',
service_dicts=service_dicts,
config_data=service_dicts,
client=self.client,
)
db = project.get_service('db')
@ -86,9 +86,9 @@ class ProjectTest(DockerClientTestCase):
name='composetest_data_container',
labels={LABEL_PROJECT: 'composetest'},
)
project = Project.from_dicts(
project = Project.from_config(
name='composetest',
service_dicts=build_service_dicts({
config_data=build_service_dicts({
'db': {
'image': 'busybox:latest',
'volumes_from': ['composetest_data_container'],
@ -117,9 +117,9 @@ class ProjectTest(DockerClientTestCase):
assert project.get_network()['Name'] == network_name
def test_net_from_service(self):
project = Project.from_dicts(
project = Project.from_config(
name='composetest',
service_dicts=build_service_dicts({
config_data=build_service_dicts({
'net': {
'image': 'busybox:latest',
'command': ["top"]
@ -149,9 +149,9 @@ class ProjectTest(DockerClientTestCase):
)
net_container.start()
project = Project.from_dicts(
project = Project.from_config(
name='composetest',
service_dicts=build_service_dicts({
config_data=build_service_dicts({
'web': {
'image': 'busybox:latest',
'net': 'container:composetest_net_container'
@ -331,7 +331,6 @@ class ProjectTest(DockerClientTestCase):
project.up(['db'])
self.assertEqual(len(project.containers()), 1)
old_db_id = project.containers()[0].id
container, = project.containers()
db_volume_path = container.get_mount('/var/db')['Source']
@ -401,9 +400,9 @@ class ProjectTest(DockerClientTestCase):
self.assertEqual(len(console.containers()), 0)
def test_project_up_starts_depends(self):
project = Project.from_dicts(
project = Project.from_config(
name='composetest',
service_dicts=build_service_dicts({
config_data=build_service_dicts({
'console': {
'image': 'busybox:latest',
'command': ["top"],
@ -436,9 +435,9 @@ class ProjectTest(DockerClientTestCase):
self.assertEqual(len(project.get_service('console').containers()), 0)
def test_project_up_with_no_deps(self):
project = Project.from_dicts(
project = Project.from_config(
name='composetest',
service_dicts=build_service_dicts({
config_data=build_service_dicts({
'console': {
'image': 'busybox:latest',
'command': ["top"],

1
tests/integration/service_test.py
View File

@ -163,6 +163,7 @@ class ServiceTest(DockerClientTestCase):
# Match the last component ("host-path"), because boot2docker symlinks /tmp
actual_host_path = container.get_mount(container_path)['Source']
self.assertTrue(path.basename(actual_host_path) == path.basename(host_path),
msg=("Last component differs: %s, %s" % (actual_host_path, host_path)))

4
tests/integration/state_test.py
View File

@ -26,10 +26,10 @@ class ProjectTestCase(DockerClientTestCase):
details = config.ConfigDetails(
'working_dir',
[config.ConfigFile(None, cfg)])
return Project.from_dicts(
return Project.from_config(
name='composetest',
client=self.client,
service_dicts=config.load(details))
config_data=config.load(details))
class BasicProjectTest(ProjectTestCase):

4
tests/integration/testcases.py
View File

@ -39,6 +39,10 @@ class DockerClientTestCase(unittest.TestCase):
for i in self.client.images(
filters={'label': 'com.docker.compose.test_image'}):
self.client.remove_image(i)
volumes = self.client.volumes().get('Volumes') or []
for v in volumes:
if 'composetests_' in v['Name']:
self.client.remove_volume(v['Name'])
def create_service(self, name, **kwargs):
if 'image' not in kwargs and 'build' not in kwargs:

53
tests/unit/config/config_test.py
View File

@ -51,7 +51,7 @@ class ConfigTest(unittest.TestCase):
'tests/fixtures/extends',
'common.yml'
)
)
).services
self.assertEqual(
service_sort(service_dicts),
@ -143,7 +143,7 @@ class ConfigTest(unittest.TestCase):
})
details = config.ConfigDetails('.', [base_file, override_file])
service_dicts = config.load(details)
service_dicts = config.load(details).services
expected = [
{
'name': 'web',
@ -207,7 +207,7 @@ class ConfigTest(unittest.TestCase):
labels: ['label=one']
""")
with tmpdir.as_cwd():
service_dicts = config.load(details)
service_dicts = config.load(details).services
expected = [
{
@ -260,7 +260,7 @@ class ConfigTest(unittest.TestCase):
build_config_details(
{valid_name: {'image': 'busybox'}},
'tests/fixtures/extends',
'common.yml'))
'common.yml')).services
assert services[0]['name'] == valid_name
def test_config_hint(self):
@ -451,7 +451,7 @@ class ConfigTest(unittest.TestCase):
'working_dir',
'filename.yml'
)
)
).services
self.assertEqual(service[0]['expose'], expose)
def test_valid_config_oneof_string_or_list(self):
@ -466,7 +466,7 @@ class ConfigTest(unittest.TestCase):
'working_dir',
'filename.yml'
)
)
).services
self.assertEqual(service[0]['entrypoint'], entrypoint)
@mock.patch('compose.config.validation.log')
@ -496,7 +496,7 @@ class ConfigTest(unittest.TestCase):
'working_dir',
'filename.yml'
)
)
).services
self.assertEqual(services[0]['environment']['SPRING_JPA_HIBERNATE_DDL-AUTO'], 'none')
def test_load_yaml_with_yaml_error(self):
@ -655,7 +655,7 @@ class InterpolationTest(unittest.TestCase):
service_dicts = config.load(
config.find('tests/fixtures/environment-interpolation', None),
)
).services
self.assertEqual(service_dicts, [
{
@ -722,7 +722,7 @@ class InterpolationTest(unittest.TestCase):
'.',
None,
)
)[0]
).services[0]
self.assertEquals(service_dict['environment']['POSTGRES_PASSWORD'], '')
@ -734,11 +734,15 @@ class VolumeConfigTest(unittest.TestCase):
@mock.patch.dict(os.environ)
def test_volume_binding_with_environment_variable(self):
os.environ['VOLUME_PATH'] = '/host/path'
d = config.load(build_config_details(
{'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}},
'.',
))[0]
self.assertEqual(d['volumes'], [VolumeSpec.parse('/host/path:/container/path')])
d = config.load(
build_config_details(
{'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}},
'.',
None,
)
).services[0]
self.assertEqual(d['volumes'], ['/host/path:/container/path'])
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths')
@mock.patch.dict(os.environ)
@ -1012,7 +1016,7 @@ class MemoryOptionsTest(unittest.TestCase):
'tests/fixtures/extends',
'common.yml'
)
)
).services
self.assertEqual(service_dict[0]['memswap_limit'], 2000000)
def test_memswap_can_be_a_string(self):
@ -1022,7 +1026,7 @@ class MemoryOptionsTest(unittest.TestCase):
'tests/fixtures/extends',
'common.yml'
)
)
).services
self.assertEqual(service_dict[0]['memswap_limit'], "512M")
@ -1126,24 +1130,21 @@ class EnvTest(unittest.TestCase):
{'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}},
"tests/fixtures/env",
)
)[0]
self.assertEqual(
set(service_dict['volumes']),
set([VolumeSpec.parse('/tmp:/host/tmp')]))
).services[0]
self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp']))
service_dict = config.load(
build_config_details(
{'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}},
"tests/fixtures/env",
)
)[0]
self.assertEqual(
set(service_dict['volumes']),
set([VolumeSpec.parse('/opt/tmp:/opt/host/tmp')]))
).services[0]
self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp']))
def load_from_filename(filename):
return config.load(config.find('.', [filename]))
return config.load(config.find('.', [filename])).services
class ExtendsTest(unittest.TestCase):
@ -1313,7 +1314,7 @@ class ExtendsTest(unittest.TestCase):
'tests/fixtures/extends',
'common.yml'
)
)
).services
self.assertEquals(len(service), 1)
self.assertIsInstance(service[0], dict)

64
tests/unit/project_test.py
View File

@ -4,6 +4,7 @@ import docker
from .. import mock
from .. import unittest
from compose.config.config import Config
from compose.config.types import VolumeFromSpec
from compose.const import LABEL_SERVICE
from compose.container import Container
@ -18,7 +19,7 @@ class ProjectTest(unittest.TestCase):
self.mock_client = mock.create_autospec(docker.Client)
def test_from_dict(self):
project = Project.from_dicts('composetest', [
project = Project.from_config('composetest', Config(None, [
{
'name': 'web',
'image': 'busybox:latest'
@ -27,15 +28,38 @@ class ProjectTest(unittest.TestCase):
'name': 'db',
'image': 'busybox:latest'
},
], None)
], None), None)
self.assertEqual(len(project.services), 2)
self.assertEqual(project.get_service('web').name, 'web')
self.assertEqual(project.get_service('web').options['image'], 'busybox:latest')
self.assertEqual(project.get_service('db').name, 'db')
self.assertEqual(project.get_service('db').options['image'], 'busybox:latest')
def test_from_dict_sorts_in_dependency_order(self):
project = Project.from_config('composetest', Config(None, [
{
'name': 'web',
'image': 'busybox:latest',
'links': ['db'],
},
{
'name': 'db',
'image': 'busybox:latest',
'volumes_from': ['volume']
},
{
'name': 'volume',
'image': 'busybox:latest',
'volumes': ['/tmp'],
}
], None), None)
self.assertEqual(project.services[0].name, 'volume')
self.assertEqual(project.services[1].name, 'db')
self.assertEqual(project.services[2].name, 'web')
def test_from_config(self):
dicts = [
dicts = Config(None, [
{
'name': 'web',
'image': 'busybox:latest',
@ -44,8 +68,8 @@ class ProjectTest(unittest.TestCase):
'name': 'db',
'image': 'busybox:latest',
},
]
project = Project.from_dicts('composetest', dicts, None)
], None)
project = Project.from_config('composetest', dicts, None)
self.assertEqual(len(project.services), 2)
self.assertEqual(project.get_service('web').name, 'web')
self.assertEqual(project.get_service('web').options['image'], 'busybox:latest')
@ -141,13 +165,13 @@ class ProjectTest(unittest.TestCase):
container_id = 'aabbccddee'
container_dict = dict(Name='aaa', Id=container_id)
self.mock_client.inspect_container.return_value = container_dict
project = Project.from_dicts('test', [
project = Project.from_config('test', Config(None, [
{
'name': 'test',
'image': 'busybox:latest',
'volumes_from': [VolumeFromSpec('aaa', 'rw')]
}
], self.mock_client)
], None), self.mock_client)
self.assertEqual(project.get_service('test')._get_volumes_from(), [container_id + ":rw"])
def test_use_volumes_from_service_no_container(self):
@ -160,7 +184,7 @@ class ProjectTest(unittest.TestCase):
"Image": 'busybox:latest'
}
]
project = Project.from_dicts('test', [
project = Project.from_config('test', Config(None, [
{
'name': 'vol',
'image': 'busybox:latest'
@ -170,13 +194,13 @@ class ProjectTest(unittest.TestCase):
'image': 'busybox:latest',
'volumes_from': [VolumeFromSpec('vol', 'rw')]
}
], self.mock_client)
], None), self.mock_client)
self.assertEqual(project.get_service('test')._get_volumes_from(), [container_name + ":rw"])
def test_use_volumes_from_service_container(self):
container_ids = ['aabbccddee', '12345']
project = Project.from_dicts('test', [
project = Project.from_config('test', Config(None, [
{
'name': 'vol',
'image': 'busybox:latest'
@ -186,7 +210,7 @@ class ProjectTest(unittest.TestCase):
'image': 'busybox:latest',
'volumes_from': [VolumeFromSpec('vol', 'rw')]
}
], None)
], None), None)
with mock.patch.object(Service, 'containers') as mock_return:
mock_return.return_value = [
mock.Mock(id=container_id, spec=Container)
@ -196,12 +220,12 @@ class ProjectTest(unittest.TestCase):
[container_ids[0] + ':rw'])
def test_net_unset(self):
project = Project.from_dicts('test', [
project = Project.from_config('test', Config(None, [
{
'name': 'test',
'image': 'busybox:latest',
}
], self.mock_client)
], None), self.mock_client)
service = project.get_service('test')
self.assertEqual(service.net.id, None)
self.assertNotIn('NetworkMode', service._get_container_host_config({}))
@ -210,13 +234,13 @@ class ProjectTest(unittest.TestCase):
container_id = 'aabbccddee'
container_dict = dict(Name='aaa', Id=container_id)
self.mock_client.inspect_container.return_value = container_dict
project = Project.from_dicts('test', [
project = Project.from_config('test', Config(None, [
{
'name': 'test',
'image': 'busybox:latest',
'net': 'container:aaa'
}
], self.mock_client)
], None), self.mock_client)
service = project.get_service('test')
self.assertEqual(service.net.mode, 'container:' + container_id)
@ -230,7 +254,7 @@ class ProjectTest(unittest.TestCase):
"Image": 'busybox:latest'
}
]
project = Project.from_dicts('test', [
project = Project.from_config('test', Config(None, [
{
'name': 'aaa',
'image': 'busybox:latest'
@ -240,7 +264,7 @@ class ProjectTest(unittest.TestCase):
'image': 'busybox:latest',
'net': 'container:aaa'
}
], self.mock_client)
], None), self.mock_client)
service = project.get_service('test')
self.assertEqual(service.net.mode, 'container:' + container_name)
@ -285,12 +309,12 @@ class ProjectTest(unittest.TestCase):
},
},
}
project = Project.from_dicts(
project = Project.from_config(
'test',
[{
Config(None, [{
'name': 'web',
'image': 'busybox:latest',
}],
}], None),
self.mock_client,
)
self.assertEqual([c.id for c in project.containers()], ['1'])