From e1b27acd0275593081fb3db58949b34ad4b91a00 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 19 Mar 2015 18:05:45 -0700 Subject: [PATCH 001/118] Add contributing section to readme Signed-off-by: Ben Firshman --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e76431b06..ce89d5aa2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ Docker Compose ============== -[![Build Status](http://jenkins.dockerproject.com/buildStatus/icon?job=Compose Master)](http://jenkins.dockerproject.com/job/Compose%20Master/) *(Previously known as Fig)* Compose is a tool for defining and running complex applications with Docker. @@ -53,3 +52,11 @@ Installation and documentation - Full documentation is available on [Docker's website](http://docs.docker.com/compose/). - Hop into #docker-compose on Freenode if you have any questions. + +Contributing +------------ + +[![Build Status](http://jenkins.dockerproject.com/buildStatus/icon?job=Compose Master)](http://jenkins.dockerproject.com/job/Compose%20Master/) + +Want to help build Compose? Check out our [contributing documentation](https://github.com/docker/compose/blob/master/CONTRIBUTING.md). + From 99f7eba9305968d8eb73d1fd0da5d082fafd380c Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 25 Mar 2015 14:48:36 -0700 Subject: [PATCH 002/118] Add Docker 1.6 RC2 to tested versions Signed-off-by: Aanand Prasad --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d7a6019aa..c9dba927d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,13 +15,15 @@ RUN set -ex; \ ; \ rm -rf /var/lib/apt/lists/* -ENV ALL_DOCKER_VERSIONS 1.3.3 1.4.1 1.5.0 +ENV ALL_DOCKER_VERSIONS 1.3.3 1.4.1 1.5.0 1.6.0-rc2 RUN set -ex; \ - for v in ${ALL_DOCKER_VERSIONS}; do \ + for v in 1.3.3 1.4.1 1.5.0; do \ curl https://get.docker.com/builds/Linux/x86_64/docker-$v -o /usr/local/bin/docker-$v; \ chmod +x /usr/local/bin/docker-$v; \ - done + done; \ + curl https://test.docker.com/builds/Linux/x86_64/docker-1.6.0-rc2 -o /usr/local/bin/docker-1.6.0-rc2; \ + chmod +x /usr/local/bin/docker-1.6.0-rc2 RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ From f9ea5ecf40113fca8a09d8c96a3a00d77c7bf6c3 Mon Sep 17 00:00:00 2001 From: akoskaaa Date: Wed, 25 Mar 2015 20:13:01 -0700 Subject: [PATCH 003/118] [pep8] make test files and config files pep8 valid Signed-off-by: akoskaaa --- compose/config.py | 14 +++++----- setup.py | 3 ++- tests/__init__.py | 3 +-- tests/integration/project_test.py | 8 +++--- tests/integration/service_test.py | 20 ++++++--------- tests/unit/cli/docker_client_test.py | 2 +- tests/unit/cli_test.py | 1 - tests/unit/config_test.py | 10 ++++---- tests/unit/container_test.py | 25 +++++++++--------- tests/unit/progress_stream_test.py | 2 +- tests/unit/project_test.py | 5 ++-- tests/unit/service_test.py | 38 +++++++++++++++------------- tests/unit/split_buffer_test.py | 1 + 13 files changed, 66 insertions(+), 66 deletions(-) diff --git a/compose/config.py b/compose/config.py index 0cd7c1ae6..c89f6de55 100644 --- a/compose/config.py +++ b/compose/config.py @@ -39,14 +39,14 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ ] DOCKER_CONFIG_HINTS = { - 'cpu_share' : 'cpu_shares', - 'link' : 'links', - 'port' : 'ports', - 'privilege' : 'privileged', + 'cpu_share': 'cpu_shares', + 'link': 'links', + 'port': 'ports', + 'privilege': 'privileged', 'priviliged': 'privileged', - 'privilige' : 'privileged', - 'volume' : 'volumes', - 'workdir' : 'working_dir', + 'privilige': 'privileged', + 'volume': 'volumes', + 'workdir': 'working_dir', } diff --git a/setup.py b/setup.py index a8f7d98ea..39ac0f6f5 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ install_requires = [ 'six >= 1.3.0, < 2', ] + tests_require = [ 'mock >= 1.0.1', 'nose', @@ -54,7 +55,7 @@ setup( url='https://www.docker.com/', author='Docker, Inc.', license='Apache License 2.0', - packages=find_packages(exclude=[ 'tests.*', 'tests' ]), + packages=find_packages(exclude=['tests.*', 'tests']), include_package_data=True, test_suite='nose.collector', install_requires=install_requires, diff --git a/tests/__init__.py b/tests/__init__.py index b4d38cccb..68116d58e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,6 @@ import sys -if sys.version_info >= (2,7): +if sys.version_info >= (2, 7): import unittest else: import unittest2 as unittest - diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index a585740f4..00d156b37 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -70,7 +70,7 @@ class ProjectTest(DockerClientTestCase): web = project.get_service('web') net = project.get_service('net') - self.assertEqual(web._get_net(), 'container:'+net.containers()[0].id) + self.assertEqual(web._get_net(), 'container:' + net.containers()[0].id) project.kill() project.remove_stopped() @@ -98,7 +98,7 @@ class ProjectTest(DockerClientTestCase): project.up() web = project.get_service('web') - self.assertEqual(web._get_net(), 'container:'+net_container.id) + self.assertEqual(web._get_net(), 'container:' + net_container.id) project.kill() project.remove_stopped() @@ -266,7 +266,7 @@ class ProjectTest(DockerClientTestCase): 'image': 'busybox:latest', 'command': ["/bin/sleep", "300"], }, - 'data' : { + 'data': { 'image': 'busybox:latest', 'command': ["/bin/sleep", "300"] }, @@ -304,7 +304,7 @@ class ProjectTest(DockerClientTestCase): 'image': 'busybox:latest', 'command': ["/bin/sleep", "300"], }, - 'data' : { + 'data': { 'image': 'busybox:latest', 'command': ["/bin/sleep", "300"] }, diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index f0fb771d9..148ac4caf 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -121,7 +121,7 @@ class ServiceTest(DockerClientTestCase): # Match the last component ("host-path"), because boot2docker symlinks /tmp actual_host_path = volumes[container_path] self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), - msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) + msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) def test_create_container_with_volumes_from(self): volume_service = self.create_service('data') @@ -187,7 +187,6 @@ class ServiceTest(DockerClientTestCase): service.recreate_containers() self.assertEqual(len(service.containers(stopped=True)), 1) - def test_recreate_containers_with_image_declared_volume(self): service = Service( project='composetest', @@ -229,8 +228,7 @@ class ServiceTest(DockerClientTestCase): set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', - 'db', - ]), + 'db']) ) def test_start_container_creates_links_with_names(self): @@ -246,8 +244,7 @@ class ServiceTest(DockerClientTestCase): set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', - 'custom_link_name', - ]), + 'custom_link_name']) ) def test_start_container_with_external_links(self): @@ -291,8 +288,7 @@ class ServiceTest(DockerClientTestCase): set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', - 'db', - ]), + 'db']) ) def test_start_container_builds_images(self): @@ -331,7 +327,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(container['HostConfig']['Privileged'], False) def test_start_container_becomes_priviliged(self): - service = self.create_service('web', privileged = True) + service = self.create_service('web', privileged=True) container = create_and_start_container(service).inspect() self.assertEqual(container['HostConfig']['Privileged'], True) @@ -483,13 +479,13 @@ class ServiceTest(DockerClientTestCase): def test_split_env(self): service = self.create_service('web', environment=['NORMAL=F1', 'CONTAINS_EQUALS=F=2', 'TRAILING_EQUALS=']) env = create_and_start_container(service).environment - for k,v in {'NORMAL': 'F1', 'CONTAINS_EQUALS': 'F=2', 'TRAILING_EQUALS': ''}.items(): + for k, v in {'NORMAL': 'F1', 'CONTAINS_EQUALS': 'F=2', 'TRAILING_EQUALS': ''}.items(): self.assertEqual(env[k], v) def test_env_from_file_combined_with_env(self): service = self.create_service('web', environment=['ONE=1', 'TWO=2', 'THREE=3'], env_file=['tests/fixtures/env/one.env', 'tests/fixtures/env/two.env']) env = create_and_start_container(service).environment - for k,v in {'ONE': '1', 'TWO': '2', 'THREE': '3', 'FOO': 'baz', 'DOO': 'dah'}.items(): + for k, v in {'ONE': '1', 'TWO': '2', 'THREE': '3', 'FOO': 'baz', 'DOO': 'dah'}.items(): self.assertEqual(env[k], v) @mock.patch.dict(os.environ) @@ -499,5 +495,5 @@ class ServiceTest(DockerClientTestCase): os.environ['ENV_DEF'] = 'E3' service = self.create_service('web', environment={'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': None, 'NO_DEF': None}) env = create_and_start_container(service).environment - for k,v in {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}.items(): + for k, v in {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}.items(): self.assertEqual(env[k], v) diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index 184aff4de..44bdbb291 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -5,7 +5,7 @@ import os import mock from tests import unittest -from compose.cli import docker_client +from compose.cli import docker_client class DockerClientTestCase(unittest.TestCase): diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 0cb7a1d59..2ed771f02 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -151,4 +151,3 @@ def make_files(dirname, filenames): for fname in filenames: with open(os.path.join(dirname, fname), 'w') as f: f.write('') - diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 8deb457af..1eaeee203 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -4,6 +4,7 @@ from .. import unittest from compose import config + class ConfigTest(unittest.TestCase): def test_from_dictionary(self): service_dicts = config.from_dictionary({ @@ -82,7 +83,7 @@ class MergeTest(unittest.TestCase): class EnvTest(unittest.TestCase): def test_parse_environment_as_list(self): - environment =[ + environment = [ 'NORMAL=F1', 'CONTAINS_EQUALS=F=2', 'TRAILING_EQUALS=', @@ -114,9 +115,8 @@ class EnvTest(unittest.TestCase): os.environ['ENV_DEF'] = 'E3' service_dict = config.make_service_dict( - 'foo', - { - 'environment': { + 'foo', { + 'environment': { 'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': None, @@ -174,6 +174,7 @@ class EnvTest(unittest.TestCase): {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}, ) + class ExtendsTest(unittest.TestCase): def test_extends(self): service_dicts = config.load('tests/fixtures/extends/docker-compose.yml') @@ -231,7 +232,6 @@ class ExtendsTest(unittest.TestCase): ], ) - def test_extends_validation(self): dictionary = {'extends': None} load_config = lambda: config.make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index a5f3f7d34..7637adf58 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -9,7 +9,6 @@ from compose.container import Container class ContainerTest(unittest.TestCase): - def setUp(self): self.container_dict = { "Id": "abc", @@ -30,11 +29,13 @@ class ContainerTest(unittest.TestCase): container = Container.from_ps(None, self.container_dict, has_been_inspected=True) - self.assertEqual(container.dictionary, { - "Id": "abc", - "Image":"busybox:latest", - "Name": "/composetest_db_1", - }) + self.assertEqual( + container.dictionary, + { + "Id": "abc", + "Image": "busybox:latest", + "Name": "/composetest_db_1", + }) def test_from_ps_prefixed(self): self.container_dict['Names'] = ['/swarm-host-1' + n for n in self.container_dict['Names']] @@ -44,7 +45,7 @@ class ContainerTest(unittest.TestCase): has_been_inspected=True) self.assertEqual(container.dictionary, { "Id": "abc", - "Image":"busybox:latest", + "Image": "busybox:latest", "Name": "/composetest_db_1", }) @@ -100,7 +101,7 @@ class ContainerTest(unittest.TestCase): def test_human_readable_ports_public_and_private(self): self.container_dict['NetworkSettings']['Ports'].update({ - "45454/tcp": [ { "HostIp": "0.0.0.0", "HostPort": "49197" } ], + "45454/tcp": [{"HostIp": "0.0.0.0", "HostPort": "49197"}], "45453/tcp": [], }) container = Container(None, self.container_dict, has_been_inspected=True) @@ -110,7 +111,7 @@ class ContainerTest(unittest.TestCase): def test_get_local_port(self): self.container_dict['NetworkSettings']['Ports'].update({ - "45454/tcp": [ { "HostIp": "0.0.0.0", "HostPort": "49197" } ], + "45454/tcp": [{"HostIp": "0.0.0.0", "HostPort": "49197"}], }) container = Container(None, self.container_dict, has_been_inspected=True) @@ -120,12 +121,12 @@ class ContainerTest(unittest.TestCase): def test_get(self): container = Container(None, { - "Status":"Up 8 seconds", + "Status": "Up 8 seconds", "HostConfig": { - "VolumesFrom": ["volume_id",] + "VolumesFrom": ["volume_id"] }, }, has_been_inspected=True) self.assertEqual(container.get('Status'), "Up 8 seconds") - self.assertEqual(container.get('HostConfig.VolumesFrom'), ["volume_id",]) + self.assertEqual(container.get('HostConfig.VolumesFrom'), ["volume_id"]) self.assertEqual(container.get('Foo.Bar.DoesNotExist'), None) diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index b53f2eb9a..694f120ed 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -5,7 +5,7 @@ from tests import unittest import mock from six import StringIO -from compose import progress_stream +from compose import progress_stream class ProgressStreamTestCase(unittest.TestCase): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index c995d432f..d5c5acb78 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -8,6 +8,7 @@ from compose import config import mock import docker + class ProjectTest(unittest.TestCase): def test_from_dict(self): project = Project.from_dicts('composetest', [ @@ -211,7 +212,7 @@ class ProjectTest(unittest.TestCase): } ], mock_client) service = project.get_service('test') - self.assertEqual(service._get_net(), 'container:'+container_id) + self.assertEqual(service._get_net(), 'container:' + container_id) def test_use_net_from_service(self): container_name = 'test_aaa_1' @@ -237,4 +238,4 @@ class ProjectTest(unittest.TestCase): ], mock_client) service = project.get_service('test') - self.assertEqual(service._get_net(), 'container:'+container_name) + self.assertEqual(service._get_net(), 'container:' + container_name) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c70c30bfa..39a6f5c10 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -146,16 +146,16 @@ class ServiceTest(unittest.TestCase): def test_build_port_bindings_with_one_port(self): port_bindings = build_port_bindings(["127.0.0.1:1000:1000"]) - self.assertEqual(port_bindings["1000"],[("127.0.0.1","1000")]) + self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) def test_build_port_bindings_with_matching_internal_ports(self): - port_bindings = build_port_bindings(["127.0.0.1:1000:1000","127.0.0.1:2000:1000"]) - self.assertEqual(port_bindings["1000"],[("127.0.0.1","1000"),("127.0.0.1","2000")]) + port_bindings = build_port_bindings(["127.0.0.1:1000:1000", "127.0.0.1:2000:1000"]) + self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000"), ("127.0.0.1", "2000")]) def test_build_port_bindings_with_nonmatching_internal_ports(self): - port_bindings = build_port_bindings(["127.0.0.1:1000:1000","127.0.0.1:2000:2000"]) - self.assertEqual(port_bindings["1000"],[("127.0.0.1","1000")]) - self.assertEqual(port_bindings["2000"],[("127.0.0.1","2000")]) + port_bindings = build_port_bindings(["127.0.0.1:1000:1000", "127.0.0.1:2000:2000"]) + self.assertEqual(port_bindings["1000"], [("127.0.0.1", "1000")]) + self.assertEqual(port_bindings["2000"], [("127.0.0.1", "2000")]) def test_split_domainname_none(self): service = Service('foo', hostname='name', client=self.mock_client) @@ -165,29 +165,32 @@ class ServiceTest(unittest.TestCase): self.assertFalse('domainname' in opts, 'domainname') def test_split_domainname_fqdn(self): - service = Service('foo', - hostname='name.domain.tld', - client=self.mock_client) + service = Service( + 'foo', + hostname='name.domain.tld', + client=self.mock_client) self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'image': 'foo'}) self.assertEqual(opts['hostname'], 'name', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') def test_split_domainname_both(self): - service = Service('foo', - hostname='name', - domainname='domain.tld', - client=self.mock_client) + service = Service( + 'foo', + hostname='name', + domainname='domain.tld', + client=self.mock_client) self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'image': 'foo'}) self.assertEqual(opts['hostname'], 'name', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') def test_split_domainname_weird(self): - service = Service('foo', - hostname='name.sub', - domainname='domain.tld', - client=self.mock_client) + service = Service( + 'foo', + hostname='name.sub', + domainname='domain.tld', + client=self.mock_client) self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'image': 'foo'}) self.assertEqual(opts['hostname'], 'name.sub', 'hostname') @@ -315,4 +318,3 @@ class ServiceVolumesTest(unittest.TestCase): self.assertEqual( binding, ('/home/user', dict(bind='/home/user', ro=False))) - diff --git a/tests/unit/split_buffer_test.py b/tests/unit/split_buffer_test.py index 3322fb55f..8eb54177a 100644 --- a/tests/unit/split_buffer_test.py +++ b/tests/unit/split_buffer_test.py @@ -3,6 +3,7 @@ from __future__ import absolute_import from compose.cli.utils import split_buffer from .. import unittest + class SplitBufferTest(unittest.TestCase): def test_single_line_chunks(self): def reader(): From fa2fb6bd38904e73e52c0bde2abb5de3914ff254 Mon Sep 17 00:00:00 2001 From: akoskaaa Date: Wed, 25 Mar 2015 23:15:34 -0700 Subject: [PATCH 004/118] [pep8] flake8 run for everything, fix items from this change Signed-off-by: akoskaaa --- script/test-versions | 2 +- tests/__init__.py | 4 ++-- tests/integration/service_test.py | 5 ++--- tests/unit/cli_test.py | 1 - tests/unit/config_test.py | 4 +++- tests/unit/progress_stream_test.py | 1 - tox.ini | 2 +- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/script/test-versions b/script/test-versions index a9e3bc4c7..e5174200b 100755 --- a/script/test-versions +++ b/script/test-versions @@ -8,7 +8,7 @@ set -e script/validate-dco >&2 echo "Running lint checks" -flake8 compose +flake8 if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="1.5.0" diff --git a/tests/__init__.py b/tests/__init__.py index 68116d58e..08a7865e9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,6 +1,6 @@ import sys if sys.version_info >= (2, 7): - import unittest + import unittest # NOQA else: - import unittest2 as unittest + import unittest2 as unittest # NOQA diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 148ac4caf..066f8b095 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -182,7 +182,7 @@ class ServiceTest(DockerClientTestCase): entrypoint=['sleep'], command=['300'] ) - old_container = service.create_container() + service.create_container() self.assertEqual(len(service.containers(stopped=True)), 1) service.recreate_containers() self.assertEqual(len(service.containers(stopped=True)), 1) @@ -262,8 +262,7 @@ class ServiceTest(DockerClientTestCase): set([ 'composetest_db_1', 'composetest_db_2', - 'db_3', - ]), + 'db_3']), ) def test_start_normal_container_does_not_create_links_to_its_own_service(self): diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 2ed771f02..fcb55a673 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -8,7 +8,6 @@ from .. import unittest import docker import mock -from six import StringIO from compose.cli import main from compose.cli.main import TopLevelCommand diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 1eaeee203..8644d354b 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -234,7 +234,9 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation(self): dictionary = {'extends': None} - load_config = lambda: config.make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') + + def load_config(): + return config.make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') self.assertRaisesRegexp(config.ConfigurationError, 'dictionary', load_config) diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index 694f120ed..142560681 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals from __future__ import absolute_import from tests import unittest -import mock from six import StringIO from compose import progress_stream diff --git a/tox.ini b/tox.ini index 6e83fc414..76a1b3297 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ deps = -rrequirements-dev.txt commands = nosetests -v {posargs} - flake8 compose + flake8 [flake8] # ignore line-length for now From 826b8ca4d35efee112ad9adf80e488d212f27a34 Mon Sep 17 00:00:00 2001 From: funkyfuture Date: Thu, 26 Mar 2015 13:11:05 +0100 Subject: [PATCH 005/118] Reformat CONTRIBUTING.md - some reformatting to make it better readable in smaller terminals - adds a note that suggests validating DCO before pushing Signed-off-by: funkyfuture --- CONTRIBUTING.md | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 22cbdcf80..0cca17b00 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,8 @@ # Contributing to Compose -Compose is a part of the Docker project, and follows the same rules and principles. Take a read of [Docker's contributing guidelines](https://github.com/docker/docker/blob/master/CONTRIBUTING.md) to get an overview. +Compose is a part of the Docker project, and follows the same rules and +principles. Take a read of [Docker's contributing guidelines](https://github.com/docker/docker/blob/master/CONTRIBUTING.md) +to get an overview. ## TL;DR @@ -17,28 +19,40 @@ If you're looking contribute to Compose but you're new to the project or maybe even to Python, here are the steps that should get you started. -1. Fork [https://github.com/docker/compose](https://github.com/docker/compose) to your username. -1. Clone your forked repository locally `git clone git@github.com:yourusername/compose.git`. -1. Enter the local directory `cd compose`. -1. Set up a development environment by running `python setup.py develop`. This will install the dependencies and set up a symlink from your `docker-compose` executable to the checkout of the repository. When you now run `docker-compose` from anywhere on your machine, it will run your development version of Compose. +1. Fork [https://github.com/docker/compose](https://github.com/docker/compose) + to your username. +2. Clone your forked repository locally `git clone git@github.com:yourusername/compose.git`. +3. Enter the local directory `cd compose`. +4. Set up a development environment by running `python setup.py develop`. This + will install the dependencies and set up a symlink from your `docker-compose` + executable to the checkout of the repository. When you now run + `docker-compose` from anywhere on your machine, it will run your development + version of Compose. ## Running the test suite -Use the test script to run linting checks and then the full test suite: +Use the test script to run DCO check, linting checks and then the full test +suite against different Python interpreters: $ script/test -Tests are run against a Docker daemon inside a container, so that we can test against multiple Docker versions. By default they'll run against only the latest Docker version - set the `DOCKER_VERSIONS` environment variable to "all" to run against all supported versions: +Tests are run against a Docker daemon inside a container, so that we can test +against multiple Docker versions. By default they'll run against only the latest +Docker version - set the `DOCKER_VERSIONS` environment variable to "all" to run +against all supported versions: $ DOCKER_VERSIONS=all script/test -Arguments to `script/test` are passed through to the `nosetests` executable, so you can specify a test directory, file, module, class or method: +Arguments to `script/test` are passed through to the `nosetests` executable, so +you can specify a test directory, file, module, class or method: $ script/test tests/unit $ script/test tests/unit/cli_test.py $ script/test tests.integration.service_test $ script/test tests.integration.service_test:ServiceTest.test_containers +Before pushing a commit you can check the DCO by invoking `script/validate-dco`. + ## Building binaries Linux: @@ -49,27 +63,23 @@ OS X: $ script/build-osx -Note that this only works on Mountain Lion, not Mavericks, due to a [bug in PyInstaller](http://www.pyinstaller.org/ticket/807). +Note that this only works on Mountain Lion, not Mavericks, due to a +[bug in PyInstaller](http://www.pyinstaller.org/ticket/807). ## Release process 1. Open pull request that: - - Updates the version in `compose/__init__.py` - Updates the binary URL in `docs/install.md` - Updates the script URL in `docs/completion.md` - Adds release notes to `CHANGES.md` - 2. Create unpublished GitHub release with release notes - -3. Build Linux version on any Docker host with `script/build-linux` and attach to release - -4. Build OS X version on Mountain Lion with `script/build-osx` and attach to release as `docker-compose-Darwin-x86_64` and `docker-compose-Linux-x86_64`. - +3. Build Linux version on any Docker host with `script/build-linux` and attach + to release +4. Build OS X version on Mountain Lion with `script/build-osx` and attach to + release as `docker-compose-Darwin-x86_64` and `docker-compose-Linux-x86_64`. 5. Publish GitHub release, creating tag - 6. Update website with `script/deploy-docs` - 7. Upload PyPi package $ git checkout $VERSION From 4e0f555c585fd22d2152075e97626abb1d75d1a2 Mon Sep 17 00:00:00 2001 From: akoskaaa Date: Thu, 26 Mar 2015 09:09:15 -0700 Subject: [PATCH 006/118] make flake8 a bit more specific Signed-off-by: akoskaaa --- script/test-versions | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/script/test-versions b/script/test-versions index e5174200b..166c1ef4e 100755 --- a/script/test-versions +++ b/script/test-versions @@ -8,7 +8,7 @@ set -e script/validate-dco >&2 echo "Running lint checks" -flake8 +flake8 compose tests setup.py if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="1.5.0" diff --git a/tox.ini b/tox.ini index 76a1b3297..33cdee167 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ deps = -rrequirements-dev.txt commands = nosetests -v {posargs} - flake8 + flake8 compose tests setup.py [flake8] # ignore line-length for now From 59f04c6e29b52a0461cf8fdfbd5e47363b5c9c47 Mon Sep 17 00:00:00 2001 From: Pascal Borreli Date: Thu, 26 Mar 2015 01:43:23 +0000 Subject: [PATCH 007/118] Fixed typo Signed-off-by: Pascal Borreli --- docs/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install.md b/docs/install.md index 0e60e1f18..6017dd190 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,5 +1,5 @@ page_title: Installing Compose -page_description: How to intall Docker Compose +page_description: How to install Docker Compose page_keywords: compose, orchestration, install, installation, docker, documentation @@ -31,7 +31,7 @@ Compose can also be installed as a Python package: $ sudo pip install -U docker-compose -No further steps are required; Compose should now be successfully installed. +No further steps are required; Compose should now be successfully installed. You can test the installation by running `docker-compose --version`. ## Compose documentation From c441ac90d67bf10ceb368915d9cd8c669c98ca58 Mon Sep 17 00:00:00 2001 From: Patrick Chanezon Date: Thu, 26 Mar 2015 16:35:53 -0700 Subject: [PATCH 008/118] paulczar fixes plus example file Signed-off-by: Patrick Chanezon --- docs/index.md | 80 +++++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/docs/index.md b/docs/index.md index a75e7285a..9e67b6907 100644 --- a/docs/index.md +++ b/docs/index.md @@ -15,34 +15,28 @@ recommend that you use it in production yet. Using Compose is basically a three-step process. -First, you define your app's environment with a `Dockerfile` so it can be -reproduced anywhere: - -```Dockerfile -FROM python:2.7 -WORKDIR /code -ADD requirements.txt /code/ -RUN pip install -r requirements.txt -ADD . /code -CMD python app.py -``` - -Next, you define the services that make up your app in `docker-compose.yml` so +1. Define your app's environment with a `Dockerfile` so it can be +reproduced anywhere. +2. Define the services that make up your app in `docker-compose.yml` so they can be run together in an isolated environment: +3. Lastly, run `docker-compose up` and Compose will start and run your entire app. + +A `docker-compose.yml` looks like this: ```yaml web: build: . - links: - - db + command: python app.py ports: - - "8000:8000" -db: - image: postgres + - "5000:5000" + volumes: + - .:/code + links: + - redis +redis: + image: redis ``` -Lastly, run `docker-compose up` and Compose will start and run your entire app. - Compose has commands for managing the whole lifecycle of your application: * Start, stop and rebuild services @@ -108,13 +102,18 @@ specify how to build the image using a file called ADD . /code WORKDIR /code RUN pip install -r requirements.txt + CMD python app.py -This tells Docker to include Python, your code, and your Python dependencies in -a Docker image. For more information on how to write Dockerfiles, see the -[Docker user -guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) -and the -[Dockerfile reference](http://docs.docker.com/reference/builder/). +This tells Docker to: + +* Build an image starting with the Python 2.7 image. +* Add the curret directory `.` into the path `/code` in the image. +* Set the working directory to `/code`. +* Install your Python dependencies. + +For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + +You can test that this builds by running `docker build -t web .`. ### Define services @@ -134,19 +133,21 @@ Next, define a set of services using `docker-compose.yml`: This defines two services: - - `web`, which is built from the `Dockerfile` in the current directory. It also - says to run the command `python app.py` inside the image, forward the exposed - port 5000 on the container to port 5000 on the host machine, connect up the - Redis service, and mount the current directory inside the container so we can - work on code without having to rebuild the image. - - `redis`, which uses the public image - [redis](https://registry.hub.docker.com/_/redis/), which gets pulled from the - Docker Hub registry. +#### web + +* Builds from the `Dockerfile` in the current directory. +* Defines to run the command `python app.py` inside the image on start. +* Forwards the exposed port 5000 on the container to port 5000 on the host machine. +* Connects the web container to the Redis service via a link. +* Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. + +#### redis + +* Uses the public [redis](https://registry.hub.docker.com/_/redis/) image which gets pulled from the Docker Hub registry. ### Build and run your app with Compose -Now, when you run `docker-compose up`, Compose will pull a Redis image, build an -image for your code, and start everything up: +Now, when you run `docker-compose up`, Compose will pull a Redis image, build an image for your code, and start everything up: $ docker-compose up Pulling image redis... @@ -157,7 +158,12 @@ image for your code, and start everything up: web_1 | * Running on http://0.0.0.0:5000/ The web app should now be listening on port 5000 on your Docker daemon host (if -you're using Boot2docker, `boot2docker ip` will tell you its address). +you're using Boot2docker, `boot2docker ip` will tell you its address). In a browser, +open `http://ip-from-boot2docker:5000` and you should get a message in your browser saying: + +`Hello World! I have been seen 1 times.` + +Refreshing the page will see the number increment. If you want to run your services in the background, you can pass the `-d` flag (for daemon mode) to `docker-compose up` and use `docker-compose ps` to see what From 98dd0cd1f892b357b8b17ef4e301a9cd104f44ff Mon Sep 17 00:00:00 2001 From: Patrick Chanezon Date: Fri, 27 Mar 2015 13:26:51 -0700 Subject: [PATCH 009/118] implemented @aanand comments Signed-off-by: Patrick Chanezon --- docs/index.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/index.md b/docs/index.md index 9e67b6907..a8ee926bd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,7 +26,6 @@ A `docker-compose.yml` looks like this: ```yaml web: build: . - command: python app.py ports: - "5000:5000" volumes: @@ -107,9 +106,10 @@ specify how to build the image using a file called This tells Docker to: * Build an image starting with the Python 2.7 image. -* Add the curret directory `.` into the path `/code` in the image. +* Add the current directory `.` into the path `/code` in the image. * Set the working directory to `/code`. -* Install your Python dependencies. +* Install your Python dependencies. +* Set the default command for the container to `python app.py` For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). @@ -121,7 +121,6 @@ Next, define a set of services using `docker-compose.yml`: web: build: . - command: python app.py ports: - "5000:5000" volumes: @@ -135,10 +134,9 @@ This defines two services: #### web -* Builds from the `Dockerfile` in the current directory. -* Defines to run the command `python app.py` inside the image on start. +* Builds from the `Dockerfile` in the current directory. * Forwards the exposed port 5000 on the container to port 5000 on the host machine. -* Connects the web container to the Redis service via a link. +* Connects the web container to the Redis service via a link. * Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. #### redis From db852e14e4d8ebc1f0a477360ceef7694970dc59 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 23 Mar 2015 14:16:14 -0700 Subject: [PATCH 010/118] Add script/ci Signed-off-by: Aanand Prasad --- Dockerfile | 3 +++ script/build-linux | 18 +++++++++++------- script/build-linux-inner | 10 ++++++++++ script/ci | 18 ++++++++++++++++++ script/test-versions | 5 +---- script/wrapdocker | 2 +- 6 files changed, 44 insertions(+), 12 deletions(-) create mode 100755 script/build-linux-inner create mode 100755 script/ci diff --git a/Dockerfile b/Dockerfile index c9dba927d..594e321f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,9 @@ RUN set -ex; \ curl https://test.docker.com/builds/Linux/x86_64/docker-1.6.0-rc2 -o /usr/local/bin/docker-1.6.0-rc2; \ chmod +x /usr/local/bin/docker-1.6.0-rc2 +# Set the default Docker to be run +RUN ln -s /usr/local/bin/docker-1.3.3 /usr/local/bin/docker + RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ diff --git a/script/build-linux b/script/build-linux index 07c9d7ec6..5e4a9470e 100755 --- a/script/build-linux +++ b/script/build-linux @@ -1,8 +1,12 @@ -#!/bin/sh +#!/bin/bash + set -ex -mkdir -p `pwd`/dist -chmod 777 `pwd`/dist -docker build -t docker-compose . -docker run -u user -v `pwd`/dist:/code/dist --rm --entrypoint pyinstaller docker-compose -F bin/docker-compose -mv dist/docker-compose dist/docker-compose-Linux-x86_64 -docker run -u user -v `pwd`/dist:/code/dist --rm --entrypoint dist/docker-compose-Linux-x86_64 docker-compose --version + +TAG="docker-compose" +docker build -t "$TAG" . +docker run \ + --rm \ + --user=user \ + --volume="$(pwd):/code" \ + --entrypoint="script/build-linux-inner" \ + "$TAG" diff --git a/script/build-linux-inner b/script/build-linux-inner new file mode 100755 index 000000000..34b0c06fd --- /dev/null +++ b/script/build-linux-inner @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +mkdir -p `pwd`/dist +chmod 777 `pwd`/dist + +pyinstaller -F bin/docker-compose +mv dist/docker-compose dist/docker-compose-Linux-x86_64 +dist/docker-compose-Linux-x86_64 --version diff --git a/script/ci b/script/ci new file mode 100755 index 000000000..a1391c627 --- /dev/null +++ b/script/ci @@ -0,0 +1,18 @@ +#!/bin/bash +# This should be run inside a container built from the Dockerfile +# at the root of the repo: +# +# $ TAG="docker-compose:$(git rev-parse --short HEAD)" +# $ docker build -t "$TAG" . +# $ docker run --rm --volume="/var/run/docker.sock:/var/run/docker.sock" --volume="$(pwd)/.git:/code/.git" -e "TAG=$TAG" --entrypoint="script/ci" "$TAG" + +set -e + +>&2 echo "Validating DCO" +script/validate-dco + +export DOCKER_VERSIONS=all +. script/test-versions + +>&2 echo "Building Linux binary" +su -c script/build-linux-inner user diff --git a/script/test-versions b/script/test-versions index 166c1ef4e..dc54d7f26 100755 --- a/script/test-versions +++ b/script/test-versions @@ -4,9 +4,6 @@ set -e ->&2 echo "Validating DCO" -script/validate-dco - >&2 echo "Running lint checks" flake8 compose tests setup.py @@ -18,7 +15,7 @@ fi for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" - docker-1.5.0 run \ + docker run \ --rm \ --privileged \ --volume="/var/lib/docker" \ diff --git a/script/wrapdocker b/script/wrapdocker index 20dc9e3ce..7b699688a 100755 --- a/script/wrapdocker +++ b/script/wrapdocker @@ -4,7 +4,7 @@ if [ "$DOCKER_VERSION" == "" ]; then DOCKER_VERSION="1.5.0" fi -ln -s "/usr/local/bin/docker-$DOCKER_VERSION" "/usr/local/bin/docker" +ln -fs "/usr/local/bin/docker-$DOCKER_VERSION" "/usr/local/bin/docker" # If a pidfile is still around (for example after a container restart), # delete it so that docker can start. From cec6dc28bbeac1ee9ab85bb5d0df029749116fc2 Mon Sep 17 00:00:00 2001 From: Patrick Chanezon Date: Fri, 27 Mar 2015 17:12:29 -0700 Subject: [PATCH 011/118] implemented @fredl suggestions Signed-off-by: Patrick Chanezon --- docs/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index a8ee926bd..627745687 100644 --- a/docs/index.md +++ b/docs/index.md @@ -141,7 +141,7 @@ This defines two services: #### redis -* Uses the public [redis](https://registry.hub.docker.com/_/redis/) image which gets pulled from the Docker Hub registry. +* Uses the public [Redis](https://registry.hub.docker.com/_/redis/) image which gets pulled from the Docker Hub registry. ### Build and run your app with Compose @@ -161,7 +161,7 @@ open `http://ip-from-boot2docker:5000` and you should get a message in your brow `Hello World! I have been seen 1 times.` -Refreshing the page will see the number increment. +Refreshing the page will increment the number. If you want to run your services in the background, you can pass the `-d` flag (for daemon mode) to `docker-compose up` and use `docker-compose ps` to see what From 0b89ae6f208eff3606854034a97bfa20d8ad15fd Mon Sep 17 00:00:00 2001 From: Joseph Page Date: Fri, 27 Mar 2015 11:31:43 +0100 Subject: [PATCH 012/118] [cli] run --rm overrides restart: always docker/compose#1013 Signed-off-by: Joseph Page --- compose/cli/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index 95dfb6cbd..85e675568 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -325,6 +325,9 @@ class TopLevelCommand(Command): if options['--entrypoint']: container_options['entrypoint'] = options.get('--entrypoint') + if options['--rm']: + container_options['restart'] = None + if options['--user']: container_options['user'] = options.get('--user') From 1a14449fe6a84f39548d45cbea235ae2520cba48 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 30 Mar 2015 11:33:21 -0400 Subject: [PATCH 013/118] Update Swarm doc - Co-scheduling will now work, so we can remove the stuff about `volumes_from` and `net` and manual affinity filters. - Added a section about building. Signed-off-by: Aanand Prasad --- SWARM.md | 45 +++++++++++++-------------------------------- 1 file changed, 13 insertions(+), 32 deletions(-) diff --git a/SWARM.md b/SWARM.md index 3e549e560..6cb24b601 100644 --- a/SWARM.md +++ b/SWARM.md @@ -9,43 +9,24 @@ Still, Compose and Swarm can be useful in a “batch processing” scenario (whe A number of things need to happen before full integration is achieved, which are documented below. -Re-deploying containers with `docker-compose up` ------------------------------------------------- - -Repeated invocations of `docker-compose up` will not work reliably when used against a Swarm cluster because of an under-the-hood design problem; [this will be fixed](https://github.com/docker/fig/pull/972) in the next version of Compose. For now, containers must be completely removed and re-created: - - $ docker-compose kill - $ docker-compose rm --force - $ docker-compose up - Links and networking -------------------- The primary thing stopping multi-container apps from working seamlessly on Swarm is getting them to talk to one another: enabling private communication between containers on different hosts hasn’t been solved in a non-hacky way. -Long-term, networking is [getting overhauled](https://github.com/docker/docker/issues/9983) in such a way that it’ll fit the multi-host model much better. For now, containers on different hosts cannot be linked. In the next version of Compose, linked services will be automatically scheduled on the same host; for now, this must be done manually (see “Co-scheduling containers” below). +Long-term, networking is [getting overhauled](https://github.com/docker/docker/issues/9983) in such a way that it’ll fit the multi-host model much better. For now, **linked containers are automatically scheduled on the same host**. -`volumes_from` and `net: container` ------------------------------------ +Building +-------- -For containers to share volumes or a network namespace, they must be scheduled on the same host - this is, after all, inherent to how both volumes and network namespaces work. In the next version of Compose, this co-scheduling will be automatic whenever `volumes_from` or `net: "container:..."` is specified; for now, containers which share volumes or a network namespace must be co-scheduled manually (see “Co-scheduling containers” below). +`docker build` against a Swarm cluster is not implemented, so for now the `build` option will not work - you will need to manually build your service's image, push it somewhere and use `image` to instruct Compose to pull it. Here's an example using the Docker Hub: -Co-scheduling containers ------------------------- - -For now, containers can be manually scheduled on the same host using Swarm’s [affinity filters](https://github.com/docker/swarm/blob/master/scheduler/filter/README.md#affinity-filter). Here’s a simple example: - -```yaml -web: - image: my-web-image - links: ["db"] - environment: - - "affinity:container==myproject_db_*" -db: - image: postgres -``` - -Here, we express an affinity filter on all web containers, saying that each one must run alongside a container whose name begins with `myproject_db_`. - -- `myproject` is the common prefix Compose gives to all containers in your project, which is either generated from the name of the current directory or specified with `-p` or the `DOCKER_COMPOSE_PROJECT_NAME` environment variable. -- `*` is a wildcard, which works just like filename wildcards in a Unix shell. + $ docker build -t myusername/web . + $ docker push myusername/web + $ cat docker-compose.yml + web: + image: myusername/web + links: ["db"] + db: + image: postgres + $ docker-compose up -d From 2a415ede088a3cb04443cc4d91b804593d17a396 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 30 Mar 2015 17:14:19 -0400 Subject: [PATCH 014/118] When extending, `build` replaces `image` and vice versa Signed-off-by: Aanand Prasad --- compose/config.py | 6 ++++++ tests/unit/config_test.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/compose/config.py b/compose/config.py index c89f6de55..dea734d51 100644 --- a/compose/config.py +++ b/compose/config.py @@ -189,6 +189,12 @@ def merge_service_dicts(base, override): override.get('volumes'), ) + if 'image' in override and 'build' in d: + del d['build'] + + if 'build' in override and 'image' in d: + del d['image'] + for k in ALLOWED_KEYS: if k not in ['environment', 'volumes']: if k in override: diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 8644d354b..280034449 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -80,6 +80,39 @@ class MergeTest(unittest.TestCase): ) self.assertEqual(set(service_dict['volumes']), set(['/bar:/code', '/data'])) + def test_merge_build_or_image_no_override(self): + self.assertEqual( + config.merge_service_dicts({'build': '.'}, {}), + {'build': '.'}, + ) + + self.assertEqual( + config.merge_service_dicts({'image': 'redis'}, {}), + {'image': 'redis'}, + ) + + def test_merge_build_or_image_override_with_same(self): + self.assertEqual( + config.merge_service_dicts({'build': '.'}, {'build': './web'}), + {'build': './web'}, + ) + + self.assertEqual( + config.merge_service_dicts({'image': 'redis'}, {'image': 'postgres'}), + {'image': 'postgres'}, + ) + + def test_merge_build_or_image_override_with_other(self): + self.assertEqual( + config.merge_service_dicts({'build': '.'}, {'image': 'redis'}), + {'image': 'redis'} + ) + + self.assertEqual( + config.merge_service_dicts({'image': 'redis'}, {'build': '.'}), + {'build': '.'} + ) + class EnvTest(unittest.TestCase): def test_parse_environment_as_list(self): From 907918b492deb7543a2ce4385ea5a3a3228ff93d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 30 Mar 2015 18:20:34 -0400 Subject: [PATCH 015/118] Merge multi-value options when extending Closes #1143. Signed-off-by: Aanand Prasad --- compose/config.py | 30 ++++++++++++++--- tests/unit/config_test.py | 71 +++++++++++++++++++++++++++++++++++---- 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/compose/config.py b/compose/config.py index dea734d51..022069fdf 100644 --- a/compose/config.py +++ b/compose/config.py @@ -195,10 +195,23 @@ def merge_service_dicts(base, override): if 'build' in override and 'image' in d: del d['image'] - for k in ALLOWED_KEYS: - if k not in ['environment', 'volumes']: - if k in override: - d[k] = override[k] + list_keys = ['ports', 'expose', 'external_links'] + + for key in list_keys: + if key in base or key in override: + d[key] = base.get(key, []) + override.get(key, []) + + list_or_string_keys = ['dns', 'dns_search'] + + for key in list_or_string_keys: + if key in base or key in override: + d[key] = to_list(base.get(key)) + to_list(override.get(key)) + + already_merged_keys = ['environment', 'volumes'] + list_keys + list_or_string_keys + + for k in set(ALLOWED_KEYS) - set(already_merged_keys): + if k in override: + d[k] = override[k] return d @@ -354,6 +367,15 @@ def expand_path(working_dir, path): return os.path.abspath(os.path.join(working_dir, path)) +def to_list(value): + if value is None: + return [] + elif isinstance(value, six.string_types): + return [value] + else: + return value + + def get_service_name_from_net(net_config): if not net_config: return diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 280034449..af3bebb33 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -40,40 +40,40 @@ class ConfigTest(unittest.TestCase): config.make_service_dict('foo', {'ports': ['8000']}) -class MergeTest(unittest.TestCase): - def test_merge_volumes_empty(self): +class MergeVolumesTest(unittest.TestCase): + def test_empty(self): service_dict = config.merge_service_dicts({}, {}) self.assertNotIn('volumes', service_dict) - def test_merge_volumes_no_override(self): + def test_no_override(self): service_dict = config.merge_service_dicts( {'volumes': ['/foo:/code', '/data']}, {}, ) self.assertEqual(set(service_dict['volumes']), set(['/foo:/code', '/data'])) - def test_merge_volumes_no_base(self): + def test_no_base(self): service_dict = config.merge_service_dicts( {}, {'volumes': ['/bar:/code']}, ) self.assertEqual(set(service_dict['volumes']), set(['/bar:/code'])) - def test_merge_volumes_override_explicit_path(self): + def test_override_explicit_path(self): service_dict = config.merge_service_dicts( {'volumes': ['/foo:/code', '/data']}, {'volumes': ['/bar:/code']}, ) self.assertEqual(set(service_dict['volumes']), set(['/bar:/code', '/data'])) - def test_merge_volumes_add_explicit_path(self): + def test_add_explicit_path(self): service_dict = config.merge_service_dicts( {'volumes': ['/foo:/code', '/data']}, {'volumes': ['/bar:/code', '/quux:/data']}, ) self.assertEqual(set(service_dict['volumes']), set(['/bar:/code', '/quux:/data'])) - def test_merge_volumes_remove_explicit_path(self): + def test_remove_explicit_path(self): service_dict = config.merge_service_dicts( {'volumes': ['/foo:/code', '/quux:/data']}, {'volumes': ['/bar:/code', '/data']}, @@ -114,6 +114,63 @@ class MergeTest(unittest.TestCase): ) +class MergeListsTest(unittest.TestCase): + def test_empty(self): + service_dict = config.merge_service_dicts({}, {}) + self.assertNotIn('ports', service_dict) + + def test_no_override(self): + service_dict = config.merge_service_dicts( + {'ports': ['10:8000', '9000']}, + {}, + ) + self.assertEqual(set(service_dict['ports']), set(['10:8000', '9000'])) + + def test_no_base(self): + service_dict = config.merge_service_dicts( + {}, + {'ports': ['10:8000', '9000']}, + ) + self.assertEqual(set(service_dict['ports']), set(['10:8000', '9000'])) + + def test_add_item(self): + service_dict = config.merge_service_dicts( + {'ports': ['10:8000', '9000']}, + {'ports': ['20:8000']}, + ) + self.assertEqual(set(service_dict['ports']), set(['10:8000', '9000', '20:8000'])) + + +class MergeStringsOrListsTest(unittest.TestCase): + def test_no_override(self): + service_dict = config.merge_service_dicts( + {'dns': '8.8.8.8'}, + {}, + ) + self.assertEqual(set(service_dict['dns']), set(['8.8.8.8'])) + + def test_no_base(self): + service_dict = config.merge_service_dicts( + {}, + {'dns': '8.8.8.8'}, + ) + self.assertEqual(set(service_dict['dns']), set(['8.8.8.8'])) + + def test_add_string(self): + service_dict = config.merge_service_dicts( + {'dns': ['8.8.8.8']}, + {'dns': '9.9.9.9'}, + ) + self.assertEqual(set(service_dict['dns']), set(['8.8.8.8', '9.9.9.9'])) + + def test_add_list(self): + service_dict = config.merge_service_dicts( + {'dns': '8.8.8.8'}, + {'dns': ['9.9.9.9']}, + ) + self.assertEqual(set(service_dict['dns']), set(['8.8.8.8', '9.9.9.9'])) + + class EnvTest(unittest.TestCase): def test_parse_environment_as_list(self): environment = [ From 0650c4485a85e350a76fbaa6d167c0ee26b08cbd Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 31 Mar 2015 16:04:22 -0400 Subject: [PATCH 016/118] Test against Docker 1.6 RC2 only Signed-off-by: Aanand Prasad --- Dockerfile | 8 ++------ script/test-versions | 2 +- script/wrapdocker | 6 ++---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 594e321f8..75de99326 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,18 +15,14 @@ RUN set -ex; \ ; \ rm -rf /var/lib/apt/lists/* -ENV ALL_DOCKER_VERSIONS 1.3.3 1.4.1 1.5.0 1.6.0-rc2 +ENV ALL_DOCKER_VERSIONS 1.6.0-rc2 RUN set -ex; \ - for v in 1.3.3 1.4.1 1.5.0; do \ - curl https://get.docker.com/builds/Linux/x86_64/docker-$v -o /usr/local/bin/docker-$v; \ - chmod +x /usr/local/bin/docker-$v; \ - done; \ curl https://test.docker.com/builds/Linux/x86_64/docker-1.6.0-rc2 -o /usr/local/bin/docker-1.6.0-rc2; \ chmod +x /usr/local/bin/docker-1.6.0-rc2 # Set the default Docker to be run -RUN ln -s /usr/local/bin/docker-1.3.3 /usr/local/bin/docker +RUN ln -s /usr/local/bin/docker-1.6.0-rc2 /usr/local/bin/docker RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ diff --git a/script/test-versions b/script/test-versions index dc54d7f26..7f1a14a9b 100755 --- a/script/test-versions +++ b/script/test-versions @@ -8,7 +8,7 @@ set -e flake8 compose tests setup.py if [ "$DOCKER_VERSIONS" == "" ]; then - DOCKER_VERSIONS="1.5.0" + DOCKER_VERSIONS="default" elif [ "$DOCKER_VERSIONS" == "all" ]; then DOCKER_VERSIONS="$ALL_DOCKER_VERSIONS" fi diff --git a/script/wrapdocker b/script/wrapdocker index 7b699688a..2e07bdadf 100755 --- a/script/wrapdocker +++ b/script/wrapdocker @@ -1,11 +1,9 @@ #!/bin/bash -if [ "$DOCKER_VERSION" == "" ]; then - DOCKER_VERSION="1.5.0" +if [ "$DOCKER_VERSION" != "" ] && [ "$DOCKER_VERSION" != "default" ]; then + ln -fs "/usr/local/bin/docker-$DOCKER_VERSION" "/usr/local/bin/docker" fi -ln -fs "/usr/local/bin/docker-$DOCKER_VERSION" "/usr/local/bin/docker" - # If a pidfile is still around (for example after a container restart), # delete it so that docker can start. rm -rf /var/run/docker.pid From 8584525e8d2a3b3c48abece0d1ecc38ec808fe18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moys=C3=A9s=20Borges?= Date: Sun, 29 Mar 2015 17:06:35 -0300 Subject: [PATCH 017/118] Interpret 'build:' as relative to the yml file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * This fix introduces one side-effect: the build parameter is now validated early, when the service dicionary is first constructed. That leads to less scary stack traces when the path is not valid. * The tests for the changes introduced here alter the fixtures of those (otherwise unrelated) tests that make use of the 'build:' parameter) Signed-off-by: Moysés Borges Furtado --- compose/config.py | 14 ++++++++ docs/yml.md | 5 +-- tests/fixtures/build-ctx/Dockerfile | 2 ++ tests/fixtures/build-path/docker-compose.yml | 2 ++ .../docker-compose.yml | 2 +- .../simple-dockerfile/docker-compose.yml | 2 +- tests/unit/config_test.py | 33 +++++++++++++++++++ 7 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/build-ctx/Dockerfile create mode 100644 tests/fixtures/build-path/docker-compose.yml diff --git a/compose/config.py b/compose/config.py index 022069fdf..2c2ddf633 100644 --- a/compose/config.py +++ b/compose/config.py @@ -171,6 +171,9 @@ def process_container_options(service_dict, working_dir=None): if 'volumes' in service_dict: service_dict['volumes'] = resolve_host_paths(service_dict['volumes'], working_dir=working_dir) + if 'build' in service_dict: + service_dict['build'] = resolve_build_path(service_dict['build'], working_dir=working_dir) + return service_dict @@ -330,6 +333,17 @@ def resolve_host_path(volume, working_dir): return container_path +def resolve_build_path(build_path, working_dir=None): + if working_dir is None: + raise Exception("No working_dir passed to resolve_build_path") + + _path = expand_path(working_dir, build_path) + if not os.path.exists(_path) or not os.access(_path, os.R_OK): + raise ConfigurationError("build path %s either does not exist or is not accessible." % _path) + else: + return _path + + def merge_volumes(base, override): d = dict_from_volumes(base) d.update(dict_from_volumes(override)) diff --git a/docs/yml.md b/docs/yml.md index 157ba4e67..a9909e816 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -29,8 +29,9 @@ image: a4bc65fd ### build -Path to a directory containing a Dockerfile. This directory is also the -build context that is sent to the Docker daemon. +Path to a directory containing a Dockerfile. When the value supplied is a +relative path, it is interpreted as relative to the location of the yml file +itself. This directory is also the build context that is sent to the Docker daemon. Compose will build and tag it with a generated name, and use that image thereafter. diff --git a/tests/fixtures/build-ctx/Dockerfile b/tests/fixtures/build-ctx/Dockerfile new file mode 100644 index 000000000..d1ceac6b7 --- /dev/null +++ b/tests/fixtures/build-ctx/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:latest +CMD echo "success" diff --git a/tests/fixtures/build-path/docker-compose.yml b/tests/fixtures/build-path/docker-compose.yml new file mode 100644 index 000000000..66e8916e9 --- /dev/null +++ b/tests/fixtures/build-path/docker-compose.yml @@ -0,0 +1,2 @@ +foo: + build: ../build-ctx/ diff --git a/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml b/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml index a10381187..786315020 100644 --- a/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml +++ b/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml @@ -1,2 +1,2 @@ service: - build: tests/fixtures/dockerfile_with_entrypoint + build: . diff --git a/tests/fixtures/simple-dockerfile/docker-compose.yml b/tests/fixtures/simple-dockerfile/docker-compose.yml index a3f56d46f..b0357541e 100644 --- a/tests/fixtures/simple-dockerfile/docker-compose.yml +++ b/tests/fixtures/simple-dockerfile/docker-compose.yml @@ -1,2 +1,2 @@ simple: - build: tests/fixtures/simple-dockerfile + build: . diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index af3bebb33..ea7503430 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -383,3 +383,36 @@ class ExtendsTest(unittest.TestCase): ] self.assertEqual(set(dicts[0]['volumes']), set(paths)) + + +class BuildPathTest(unittest.TestCase): + def setUp(self): + self.abs_context_path = os.path.join(os.getcwd(), 'tests/fixtures/build-ctx') + + def test_nonexistent_path(self): + options = {'build': 'nonexistent.path'} + self.assertRaises( + config.ConfigurationError, + lambda: config.make_service_dict('foo', options, 'tests/fixtures/build-path'), + ) + + def test_relative_path(self): + relative_build_path = '../build-ctx/' + service_dict = config.make_service_dict( + 'relpath', + {'build': relative_build_path}, + working_dir='tests/fixtures/build-path' + ) + self.assertEquals(service_dict['build'], self.abs_context_path) + + def test_absolute_path(self): + service_dict = config.make_service_dict( + 'abspath', + {'build': self.abs_context_path}, + working_dir='tests/fixtures/build-path' + ) + self.assertEquals(service_dict['build'], self.abs_context_path) + + def test_from_file(self): + service_dict = config.load('tests/fixtures/build-path/docker-compose.yml') + self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}]) From e3cff5d17db64921f98d087a21df2391780d1cb6 Mon Sep 17 00:00:00 2001 From: Laurent Arnoud Date: Tue, 24 Mar 2015 15:22:11 +0100 Subject: [PATCH 018/118] Docs: fix env_file example Thanks-to: @aanand Signed-off-by: Laurent Arnoud --- docs/yml.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/yml.md b/docs/yml.md index a85f0923f..2272e381f 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -172,8 +172,12 @@ env_file: - /opt/secrets.env ``` +Compose expects each line in an env file to be in `VAR=VAL` format. Lines +beginning with `#` (i.e. comments) are ignored, as are blank lines. + ``` -RACK_ENV: development +# Set Rails/Rack environment +RACK_ENV=development ``` ### extends From b3382ffd4f455745c0401a0c2d6d74c2a182bc6f Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 1 Apr 2015 14:59:21 -0400 Subject: [PATCH 019/118] Use Docker 1.6 RC4 Signed-off-by: Aanand Prasad --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 75de99326..7438d6b1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,14 +15,14 @@ RUN set -ex; \ ; \ rm -rf /var/lib/apt/lists/* -ENV ALL_DOCKER_VERSIONS 1.6.0-rc2 +ENV ALL_DOCKER_VERSIONS 1.6.0-rc4 RUN set -ex; \ - curl https://test.docker.com/builds/Linux/x86_64/docker-1.6.0-rc2 -o /usr/local/bin/docker-1.6.0-rc2; \ - chmod +x /usr/local/bin/docker-1.6.0-rc2 + curl https://test.docker.com/builds/Linux/x86_64/docker-1.6.0-rc4 -o /usr/local/bin/docker-1.6.0-rc4; \ + chmod +x /usr/local/bin/docker-1.6.0-rc4 # Set the default Docker to be run -RUN ln -s /usr/local/bin/docker-1.6.0-rc2 /usr/local/bin/docker +RUN ln -s /usr/local/bin/docker-1.6.0-rc4 /usr/local/bin/docker RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ From d866415b9a5bd5c01f834173fe709f699026263e Mon Sep 17 00:00:00 2001 From: Roland Cooper Date: Thu, 26 Mar 2015 19:17:36 -0500 Subject: [PATCH 020/118] Update install docs for permission denied error Signed-off-by: Roland Cooper --- docs/install.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/install.md b/docs/install.md index 6017dd190..7ee6f1b9d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -23,6 +23,8 @@ To install Compose, run the following commands: curl -L https://github.com/docker/compose/releases/download/1.1.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose +> Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands above, then `exit`. + Optionally, you can also install [command completion](completion.md) for the bash shell. From 502d58abe6e017f26a9d1d8360f9f298dc5eda5f Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 3 Apr 2015 18:26:02 -0400 Subject: [PATCH 021/118] Add guide to using Compose in production Signed-off-by: Aanand Prasad --- docs/mkdocs.yml | 1 + docs/production.md | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 docs/production.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 14335873d..aa7413846 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,5 +1,6 @@ - ['compose/index.md', 'User Guide', 'Docker Compose' ] +- ['compose/production.md', 'User Guide', 'Using Compose in production' ] - ['compose/install.md', 'Installation', 'Docker Compose'] - ['compose/cli.md', 'Reference', 'Compose command line'] - ['compose/yml.md', 'Reference', 'Compose yml'] diff --git a/docs/production.md b/docs/production.md new file mode 100644 index 000000000..d267ed41f --- /dev/null +++ b/docs/production.md @@ -0,0 +1,77 @@ +page_title: Using Compose in production +page_description: Guide to using Docker Compose in production +page_keywords: documentation, docs, docker, compose, orchestration, containers, production + + +## Using Compose in production + +While **Compose is not yet considered production-ready**, you can try using it +for production deployments if you're feeling brave. Production-readiness is an +active, ongoing project - see the +[roadmap](https://github.com/docker/compose/blob/master/ROADMAP.md) for details +on how it's coming along and what needs to be done. + +When deploying to production, you'll almost certainly want to make changes to +your app configuration that are more appropriate to a live environment. This may +include: + +- Removing any volume bindings for application code, so that code stays inside + the container and can't be changed from outside +- Binding to different ports on the host +- Setting environment variables differently (e.g. to decrease the verbosity of + logging, or to enable email sending) +- Specifying a restart policy (e.g. `restart: always`) to avoid downtime +- Adding extra services (e.g. a log aggregator) + +For this reason, you'll probably want to define a separate Compose file, say +`production.yml`, which specifies production-appropriate configuration. + + + +Once you've got an alternate configuration file, you can make Compose use it +by setting the `COMPOSE_FILE` environment variable: + + $ COMPOSE_FILE=production.yml + $ docker-compose up -d + +> **Note:** You can also use the file for a one-off command without setting +> an environment variable by passing the `-f` flag, e.g. +> `docker-compose -f production.yml up -d`. + +### Deploying changes + +When you make changes to your app code, you'll need to rebuild your image and +recreate your app containers. If the service you want to redeploy is called +`web`, this will look like: + + $ docker-compose build web + $ docker-compose up --no-deps -d web + +This will first rebuild the image for `web` and then stop, destroy and recreate +*just* the `web` service. The `--no-deps` flag prevents Compose from also +recreating any services which `web` depends on. + +### Run Compose on a single server + +You can use Compose to deploy an app to a remote Docker host by setting the +`DOCKER_HOST`, `DOCKER_TLS_VERIFY` and `DOCKER_CERT_PATH` environment variables +appropriately. [Docker Machine](https://docs.docker.com/machine) makes managing +local and remote Docker hosts very easy, and is recommended even if you're not +deploying remotely. + +Once you've set up your environment variables, all the normal `docker-compose` +commands will work with no extra configuration. + +### Run Compose on a Swarm cluster + +[Docker Swarm](https://docs.docker.com/swarm), a Docker-native clustering +system, exposes the same API as a single Docker host, which means you can use +Compose against a Swarm instance and run your apps across multiple hosts. + +Compose/Swarm integration is still in the experimental stage, and Swarm is still +in beta, but if you're interested to try it out, check out the +[integration guide](https://github.com/docker/compose/blob/master/SWARM.md). From 11a2100d537319361f9515414e10ebd55bbb9ac4 Mon Sep 17 00:00:00 2001 From: Steven Dake Date: Tue, 24 Feb 2015 06:39:06 -0700 Subject: [PATCH 022/118] Add a --pid=host feature to expose the host PID space to the container Docker 1.5.0+ introduces a --pid=host feature which allows sharing of PID namespaces between baremetal and containers. This is useful for atomic upgrades, atomic rollbacks, and monitoring. For more details of a real-life use case, check out: http://sdake.io/2015/01/28/an-atomic-upgrade-process-for-openstack-compute-nodes/ Signed-off-by: Steven Dake --- docs/yml.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/yml.md b/docs/yml.md index a9909e816..f8191766a 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -264,6 +264,16 @@ net: "none" net: "container:[name or id]" net: "host" ``` +### pid + +``` +pid: "host" +``` + +Sets the PID mode to the host PID mode. This turns on sharing between +container and the host operating system the PID address space. Containers +launched with this flag will be able to access and manipulate other +containers in the bare-metal machine's namespace and vise-versa. ### dns From 94277a3eb052c1bef77e95f0d12bcf5f3c327038 Mon Sep 17 00:00:00 2001 From: Steven Dake Date: Tue, 24 Feb 2015 22:42:23 -0700 Subject: [PATCH 023/118] Add --pid=host support Allow docker-compsoe to use the docker --pid=host API available in 1.17 Signed-off-by: Steven Dake --- compose/config.py | 1 + compose/service.py | 3 +++ tests/integration/service_test.py | 11 +++++++++++ 3 files changed, 15 insertions(+) diff --git a/compose/config.py b/compose/config.py index 2c2ddf633..6ef637c5a 100644 --- a/compose/config.py +++ b/compose/config.py @@ -20,6 +20,7 @@ DOCKER_CONFIG_KEYS = [ 'links', 'mem_limit', 'net', + 'pid', 'ports', 'privileged', 'restart', diff --git a/compose/service.py b/compose/service.py index 936e3f9d0..20955d235 100644 --- a/compose/service.py +++ b/compose/service.py @@ -25,6 +25,7 @@ DOCKER_START_KEYS = [ 'dns_search', 'env_file', 'net', + 'pid', 'privileged', 'restart', ] @@ -434,6 +435,7 @@ class Service(object): privileged = options.get('privileged', False) cap_add = options.get('cap_add', None) cap_drop = options.get('cap_drop', None) + pid = options.get('pid', None) dns = options.get('dns', None) if isinstance(dns, six.string_types): @@ -457,6 +459,7 @@ class Service(object): restart_policy=restart, cap_add=cap_add, cap_drop=cap_drop, + pid_mode=pid ) def _get_image_name(self, image): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 066f8b095..85f6db9db 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -419,6 +419,17 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.NetworkMode'), 'host') + def test_pid_mode_none_defined(self): + service = self.create_service('web', pid=None) + container = create_and_start_container(service) + print 'STEAK %s' % (container.get('HostConfig.PidMode')) + self.assertEqual(container.get('HostConfig.PidMode'), '') + + def test_pid_mode_host(self): + service = self.create_service('web', pid='host') + container = create_and_start_container(service) + self.assertEqual(container.get('HostConfig.PidMode'), 'host') + def test_dns_no_value(self): service = self.create_service('web') container = create_and_start_container(service) From 947742852e5d6f1f8ddd20a764165531b510d2f3 Mon Sep 17 00:00:00 2001 From: Fred Lifton Date: Mon, 6 Apr 2015 16:47:07 -0700 Subject: [PATCH 024/118] Prepping for 1.6 release. Adds release notes and edits/revises new Compose in production doc. --- docs/index.md | 16 ++++++++++++++++ docs/production.md | 48 +++++++++++++++++++++++----------------------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/docs/index.md b/docs/index.md index a75e7285a..da5a8efdb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,6 +5,8 @@ page_keywords: documentation, docs, docker, compose, orchestration, containers # Docker Compose +## Overview + Compose is a tool for defining and running complex applications with Docker. With Compose, you define a multi-container application in a single file, then spin your application up in a single command which does everything that needs to @@ -191,3 +193,17 @@ At this point, you have seen the basics of how Compose works. [Rails](rails.md), or [Wordpress](wordpress.md). - See the reference guides for complete details on the [commands](cli.md), the [configuration file](yml.md) and [environment variables](env.md). + +## Release Notes + +### Version 1.2.0 (April 7, 2015) + +For complete information on this release, see the [1.2.0 Milestone project page](https://github.com/docker/compose/wiki/1.2.0-Milestone-Project-Page). +In addition to bug fixes and refinements, this release adds the following: + +* The `extends` keyword, which adds the ability to extend services by sharing common configurations. For details, see +[PR #972](https://github.com/docker/compose/pull/1088). + +* Better integration with Swarm. Swarm will now schedule inter-dependent +containers on the same host. For details, see +[PR #972](https://github.com/docker/compose/pull/972). diff --git a/docs/production.md b/docs/production.md index d267ed41f..8524c99b8 100644 --- a/docs/production.md +++ b/docs/production.md @@ -5,73 +5,73 @@ page_keywords: documentation, docs, docker, compose, orchestration, containers, ## Using Compose in production -While **Compose is not yet considered production-ready**, you can try using it -for production deployments if you're feeling brave. Production-readiness is an -active, ongoing project - see the +While **Compose is not yet considered production-ready**, if you'd like to experiment and learn more about using it in production deployments, this guide +can help. +The project is actively working towards becoming +production-ready; to learn more about the progress being made, check out the [roadmap](https://github.com/docker/compose/blob/master/ROADMAP.md) for details -on how it's coming along and what needs to be done. +on how it's coming along and what still needs to be done. When deploying to production, you'll almost certainly want to make changes to -your app configuration that are more appropriate to a live environment. This may -include: +your app configuration that are more appropriate to a live environment. These +changes may include: - Removing any volume bindings for application code, so that code stays inside the container and can't be changed from outside - Binding to different ports on the host -- Setting environment variables differently (e.g. to decrease the verbosity of +- Setting environment variables differently (e.g., to decrease the verbosity of logging, or to enable email sending) -- Specifying a restart policy (e.g. `restart: always`) to avoid downtime -- Adding extra services (e.g. a log aggregator) +- Specifying a restart policy (e.g., `restart: always`) to avoid downtime +- Adding extra services (e.g., a log aggregator) For this reason, you'll probably want to define a separate Compose file, say `production.yml`, which specifies production-appropriate configuration. - -Once you've got an alternate configuration file, you can make Compose use it +Once you've got an alternate configuration file, make Compose use it by setting the `COMPOSE_FILE` environment variable: $ COMPOSE_FILE=production.yml $ docker-compose up -d > **Note:** You can also use the file for a one-off command without setting -> an environment variable by passing the `-f` flag, e.g. +> an environment variable. You do this by passing the `-f` flag, e.g., > `docker-compose -f production.yml up -d`. ### Deploying changes When you make changes to your app code, you'll need to rebuild your image and -recreate your app containers. If the service you want to redeploy is called -`web`, this will look like: +recreate your app's containers. To redeploy a service called +`web`, you would use: $ docker-compose build web $ docker-compose up --no-deps -d web -This will first rebuild the image for `web` and then stop, destroy and recreate +This will first rebuild the image for `web` and then stop, destroy, and recreate *just* the `web` service. The `--no-deps` flag prevents Compose from also recreating any services which `web` depends on. -### Run Compose on a single server +### Running Compose on a single server You can use Compose to deploy an app to a remote Docker host by setting the -`DOCKER_HOST`, `DOCKER_TLS_VERIFY` and `DOCKER_CERT_PATH` environment variables -appropriately. [Docker Machine](https://docs.docker.com/machine) makes managing -local and remote Docker hosts very easy, and is recommended even if you're not -deploying remotely. +`DOCKER_HOST`, `DOCKER_TLS_VERIFY`, and `DOCKER_CERT_PATH` environment variables +appropriately. For tasks like this, +[Docker Machine](https://docs.docker.com/machine) makes managing local and +remote Docker hosts very easy, and is recommended even if you're not deploying +remotely. Once you've set up your environment variables, all the normal `docker-compose` -commands will work with no extra configuration. +commands will work with no further configuration. -### Run Compose on a Swarm cluster +### Running Compose on a Swarm cluster [Docker Swarm](https://docs.docker.com/swarm), a Docker-native clustering system, exposes the same API as a single Docker host, which means you can use Compose against a Swarm instance and run your apps across multiple hosts. Compose/Swarm integration is still in the experimental stage, and Swarm is still -in beta, but if you're interested to try it out, check out the +in beta, but if you'd like to explore and experiment, check out the [integration guide](https://github.com/docker/compose/blob/master/SWARM.md). From 0b48e137e898f55d395cd1a06d7d3a08db75c6d8 Mon Sep 17 00:00:00 2001 From: Joseph Page Date: Tue, 7 Apr 2015 10:11:36 +0200 Subject: [PATCH 025/118] add unit tests for run --rm with restart Signed-off-by: Joseph Page --- tests/unit/cli_test.py | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index fcb55a673..240069adb 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -134,6 +134,55 @@ class CLITestCase(unittest.TestCase): call_kwargs['environment'], {'FOO': 'ONE', 'BAR': 'NEW', 'OTHER': 'THREE'}) + def test_run_service_with_restart_always(self): + command = TopLevelCommand() + mock_client = mock.create_autospec(docker.Client) + mock_project = mock.Mock() + mock_project.get_service.return_value = Service( + 'service', + client=mock_client, + restart='always', + image='someimage') + command.run(mock_project, { + 'SERVICE': 'service', + 'COMMAND': None, + '-e': [], + '--user': None, + '--no-deps': None, + '--allow-insecure-ssl': None, + '-d': True, + '-T': None, + '--entrypoint': None, + '--service-ports': None, + '--rm': None, + }) + _, _, call_kwargs = mock_client.create_container.mock_calls[0] + self.assertEquals(call_kwargs['host_config']['RestartPolicy']['Name'], 'always') + + command = TopLevelCommand() + mock_client = mock.create_autospec(docker.Client) + mock_project = mock.Mock() + mock_project.get_service.return_value = Service( + 'service', + client=mock_client, + restart='always', + image='someimage') + command.run(mock_project, { + 'SERVICE': 'service', + 'COMMAND': None, + '-e': [], + '--user': None, + '--no-deps': None, + '--allow-insecure-ssl': None, + '-d': True, + '-T': None, + '--entrypoint': None, + '--service-ports': None, + '--rm': True, + }) + _, _, call_kwargs = mock_client.create_container.mock_calls[0] + self.assertFalse('RestartPolicy' in call_kwargs['host_config']) + def get_config_filename_for_files(filenames): project_dir = tempfile.mkdtemp() From f3f7f000fec841a62cc1849462f851230614af85 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 3 Apr 2015 16:31:12 -0400 Subject: [PATCH 026/118] Add tutorial and reference for `extends` Signed-off-by: Aanand Prasad --- docs/extends.md | 364 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/mkdocs.yml | 1 + docs/yml.md | 38 +---- 3 files changed, 368 insertions(+), 35 deletions(-) create mode 100644 docs/extends.md diff --git a/docs/extends.md b/docs/extends.md new file mode 100644 index 000000000..2393ca6ae --- /dev/null +++ b/docs/extends.md @@ -0,0 +1,364 @@ +page_title: Extending services in Compose +page_description: How to use Docker Compose's "extends" keyword to share configuration between files and projects +page_keywords: fig, composition, compose, docker, orchestration, documentation, docs + + +## Extending services in Compose + +Docker Compose's `extends` keyword enables sharing of common configurations +among different files, or even different projects entirely. Extending services +is useful if you have several applications that reuse commonly-defined services. +Using `extends` you can define a service in one place and refer to it from +anywhere. + +Alternatively, you can deploy the same application to multiple environments with +a slightly different set of services in each case (or with changes to the +configuration of some services). Moreover, you can do so without copy-pasting +the configuration around. + +### Understand the extends configuration + +When defining any service in `docker-compose.yml`, you can declare that you are +extending another service like this: + +```yaml +web: + extends: + file: common-services.yml + service: webapp +``` + +This instructs Compose to re-use the configuration for the `webapp` service +defined in the `common-services.yml` file. Suppose that `common-services.yml` +looks like this: + +```yaml +webapp: + build: . + ports: + - "8000:8000" + volumes: + - "/data" +``` + +In this case, you'll get exactly the same result as if you wrote +`docker-compose.yml` with that `build`, `ports` and `volumes` configuration +defined directly under `web`. + +You can go further and define (or re-define) configuration locally in +`docker-compose.yml`: + +```yaml +web: + extends: + file: common-services.yml + service: webapp + environment: + - DEBUG=1 + cpu_shares: 5 +``` + +You can also write other services and link your `web` service to them: + +```yaml +web: + extends: + file: common-services.yml + service: webapp + environment: + - DEBUG=1 + cpu_shares: 5 + links: + - db +db: + image: postgres +``` + +For full details on how to use `extends`, refer to the [reference](#reference). + +### Example use case + +In this example, you’ll repurpose the example app from the [quick start +guide](index.md). (If you're not familiar with Compose, it's recommended that +you go through the quick start first.) This example assumes you want to use +Compose both to develop an application locally and then deploy it to a +production environment. + +The local and production environments are similar, but there are some +differences. In development, you mount the application code as a volume so that +it can pick up changes; in production, the code should be immutable from the +outside. This ensures it’s not accidentally changed. The development environment +uses a local Redis container, but in production another team manages the Redis +service, which is listening at `redis-production.example.com`. + +To configure with `extends` for this sample, you must: + +1. Define the web application as a Docker image in `Dockerfile` and a Compose + service in `common.yml`. + +2. Define the development environment in the standard Compose file, + `docker-compose.yml`. + + - Use `extends` to pull in the web service. + - Configure a volume to enable code reloading. + - Create an additional Redis service for the application to use locally. + +3. Define the production environment in a third Compose file, `production.yml`. + + - Use `extends` to pull in the web service. + - Configure the web service to talk to the external, production Redis service. + +#### Define the web app + +Defining the web application requires the following: + +1. Create an `app.py` file. + + This file contains a simple Python application that uses Flask to serve HTTP + and increments a counter in Redis: + + from flask import Flask + from redis import Redis + import os + + app = Flask(__name__) + redis = Redis(host=os.environ['REDIS_HOST'], port=6379) + + @app.route('/') + def hello(): + redis.incr('hits') + return 'Hello World! I have been seen %s times.\n' % redis.get('hits') + + if __name__ == "__main__": + app.run(host="0.0.0.0", debug=True) + + This code uses a `REDIS_HOST` environment variable to determine where to + find Redis. + +2. Define the Python dependencies in a `requirements.txt` file: + + flask + redis + +3. Create a `Dockerfile` to build an image containing the app: + + FROM python:2.7 + ADD . /code + WORKDIR /code + RUN pip install -r + requirements.txt + CMD python app.py + +4. Create a Compose configuration file called `common.yml`: + + This configuration defines how to run the app. + + web: + build: . + ports: + - "5000:5000" + + Typically, you would have dropped this configuration into + `docker-compose.yml` file, but in order to pull it into multiple files with + `extends`, it needs to be in a separate file. + +#### Define the development environment + +1. Create a `docker-compose.yml` file. + + The `extends` option pulls in the `web` service from the `common.yml` file + you created in the previous section. + + web: + extends: + file: common.yml + service: web + volumes: + - .:/code + links: + - redis + environment: + - REDIS_HOST=redis + redis: + image: redis + + The new addition defines a `web` service that: + + - Fetches the base configuration for `web` out of `common.yml`. + - Adds `volumes` and `links` configuration to the base (`common.yml`) + configuration. + - Sets the `REDIS_HOST` environment variable to point to the linked redis + container. This environment uses a stock `redis` image from the Docker Hub. + +2. Run `docker-compose up`. + + Compose creates, links, and starts a web and redis container linked together. + It mounts your application code inside the web container. + +3. Verify that the code is mounted by changing the message in + `app.py`—say, from `Hello world!` to `Hello from Compose!`. + + Don't forget to refresh your browser to see the change! + +#### Define the production environment + +You are almost done. Now, define your production environment: + +1. Create a `production.yml` file. + + As with `docker-compose.yml`, the `extends` option pulls in the `web` service + from `common.yml`. + + web: + extends: + file: common.yml + service: web + environment: + - REDIS_HOST=redis-production.example.com + +2. Run `docker-compose -f production.yml up`. + + Compose creates *just* a web container and configures the Redis connection via + the `REDIS_HOST` environment variable. This variable points to the production + Redis instance. + + > **Note**: If you try to load up the webapp in your browser you'll get an + > error—`redis-production.example.com` isn't actually a Redis server. + +You've now done a basic `extends` configuration. As your application develops, +you can make any necessary changes to the web service in `common.yml`. Compose +picks up both the development and production environments when you next run +`docker-compose`. You don't have to do any copy-and-paste, and you don't have to +manually keep both environments in sync. + + +### Reference + +You can use `extends` on any service together with other configuration keys. It +always expects a dictionary that should always contain two keys: `file` and +`service`. + +The `file` key specifies which file to look in. It can be an absolute path or a +relative one—if relative, it's treated as relative to the current file. + +The `service` key specifies the name of the service to extend, for example `web` +or `database`. + +You can extend a service that itself extends another. You can extend +indefinitely. Compose does not support circular references and `docker-compose` +returns an error if it encounters them. + +#### Adding and overriding configuration + +Compose copies configurations from the original service over to the local one, +**except** for `links` and `volumes_from`. These exceptions exist to avoid +implicit dependencies—you always define `links` and `volumes_from` +locally. This ensures dependencies between services are clearly visible when +reading the current file. Defining these locally also ensures changes to the +referenced file don't result in breakage. + +If a configuration option is defined in both the original service and the local +service, the local value either *override*s or *extend*s the definition of the +original service. This works differently for other configuration options. + +For single-value options like `image`, `command` or `mem_limit`, the new value +replaces the old value. **This is the default behaviour - all exceptions are +listed below.** + +```yaml +# original service +command: python app.py + +# local service +command: python otherapp.py + +# result +command: python otherapp.py +``` + +In the case of `build` and `image`, using one in the local service causes +Compose to discard the other, if it was defined in the original service. + +```yaml +# original service +build: . + +# local service +image: redis + +# result +image: redis +``` + +```yaml +# original service +image: redis + +# local service +build: . + +# result +build: . +``` + +For the **multi-value options** `ports`, `expose`, `external_links`, `dns` and +`dns_search`, Compose concatenates both sets of values: + +```yaml +# original service +expose: + - "3000" + +# local service +expose: + - "4000" + - "5000" + +# result +expose: + - "3000" + - "4000" + - "5000" +``` + +In the case of `environment`, Compose "merges" entries together with +locally-defined values taking precedence: + +```yaml +# original service +environment: + - FOO=original + - BAR=original + +# local service +environment: + - BAR=local + - BAZ=local + +# result +environment: + - FOO=original + - BAR=local + - BAZ=local +``` + +Finally, for `volumes`, Compose "merges" entries together with locally-defined +bindings taking precedence: + +```yaml +# original service +volumes: + - /original-dir/foo:/foo + - /original-dir/bar:/bar + +# local service +volumes: + - /local-dir/bar:/bar + - /local-dir/baz/:baz + +# result +volumes: + - /original-dir/foo:/foo + - /local-dir/bar:/bar + - /local-dir/baz/:baz +``` \ No newline at end of file diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index aa7413846..428439bc4 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,6 +1,7 @@ - ['compose/index.md', 'User Guide', 'Docker Compose' ] - ['compose/production.md', 'User Guide', 'Using Compose in production' ] +- ['compose/extends.md', 'User Guide', 'Extending services in Compose'] - ['compose/install.md', 'Installation', 'Docker Compose'] - ['compose/cli.md', 'Reference', 'Compose command line'] - ['compose/yml.md', 'Reference', 'Compose yml'] diff --git a/docs/yml.md b/docs/yml.md index a9909e816..140a26e1d 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -217,42 +217,10 @@ Here, the `web` service in **development.yml** inherits the configuration of the `webapp` service in **common.yml** - the `build` and `environment` keys - and adds `ports` and `links` configuration. It overrides one of the defined environment variables (DEBUG) with a new value, and the other one -(SEND_EMAILS) is left untouched. It's exactly as if you defined `web` like -this: +(SEND_EMAILS) is left untouched. -```yaml -web: - build: ./webapp - ports: - - "8000:8000" - links: - - db - environment: - - DEBUG=true - - SEND_EMAILS=false -``` - -The `extends` option is great for sharing configuration between different -apps, or for configuring the same app differently for different environments. -You could write a new file for a staging environment, **staging.yml**, which -binds to a different port and doesn't turn on debugging: - -``` -web: - extends: - file: common.yml - service: webapp - ports: - - "80:8000" - links: - - db -db: - image: postgres -``` - -> **Note:** When you extend a service, `links` and `volumes_from` -> configuration options are **not** inherited - you will have to define -> those manually each time you extend it. +For more on `extends`, see the [tutorial](extends.md#example) and +[reference](extends.md#reference). ### net From fd568b389ddaf1a07d5c1e5e7aebc53b96dc4b04 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 7 Apr 2015 12:59:47 +0100 Subject: [PATCH 027/118] Fix home directory and env expansion in volume paths Signed-off-by: Aanand Prasad --- compose/config.py | 2 ++ compose/service.py | 4 +--- tests/integration/service_test.py | 18 ++++++++++++++++++ tests/unit/config_test.py | 14 ++++++++++++++ tests/unit/service_test.py | 15 --------------- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/compose/config.py b/compose/config.py index 2c2ddf633..d3300d511 100644 --- a/compose/config.py +++ b/compose/config.py @@ -328,6 +328,8 @@ def resolve_host_paths(volumes, working_dir=None): def resolve_host_path(volume, working_dir): container_path, host_path = split_volume(volume) if host_path is not None: + host_path = os.path.expanduser(host_path) + host_path = os.path.expandvars(host_path) return "%s:%s" % (expand_path(working_dir, host_path), container_path) else: return container_path diff --git a/compose/service.py b/compose/service.py index 936e3f9d0..86427a1ea 100644 --- a/compose/service.py +++ b/compose/service.py @@ -3,7 +3,6 @@ from __future__ import absolute_import from collections import namedtuple import logging import re -import os from operator import attrgetter import sys import six @@ -586,8 +585,7 @@ def parse_repository_tag(s): def build_volume_binding(volume_spec): internal = {'bind': volume_spec.internal, 'ro': volume_spec.mode == 'ro'} - external = os.path.expanduser(volume_spec.external) - return os.path.abspath(os.path.expandvars(external)), internal + return volume_spec.external, internal def build_port_bindings(ports): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 066f8b095..544e2def0 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -123,6 +123,24 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) + @mock.patch.dict(os.environ) + def test_create_container_with_home_and_env_var_in_volume_path(self): + os.environ['VOLUME_NAME'] = 'my-volume' + os.environ['HOME'] = '/tmp/home-dir' + expected_host_path = os.path.join(os.environ['HOME'], os.environ['VOLUME_NAME']) + + host_path = '~/${VOLUME_NAME}' + container_path = '/container-path' + + service = self.create_service('db', volumes=['%s:%s' % (host_path, container_path)]) + container = service.create_container() + service.start_container(container) + + actual_host_path = container.get('Volumes')[container_path] + components = actual_host_path.split('/') + self.assertTrue(components[-2:] == ['home-dir', 'my-volume'], + msg="Last two components differ: %s, %s" % (actual_host_path, expected_host_path)) + def test_create_container_with_volumes_from(self): volume_service = self.create_service('data') volume_container_1 = volume_service.create_container() diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index ea7503430..97bd1b91d 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -40,6 +40,20 @@ class ConfigTest(unittest.TestCase): config.make_service_dict('foo', {'ports': ['8000']}) +class VolumePathTest(unittest.TestCase): + @mock.patch.dict(os.environ) + def test_volume_binding_with_environ(self): + os.environ['VOLUME_PATH'] = '/host/path' + d = config.make_service_dict('foo', {'volumes': ['${VOLUME_PATH}:/container/path']}, working_dir='.') + self.assertEqual(d['volumes'], ['/host/path:/container/path']) + + @mock.patch.dict(os.environ) + def test_volume_binding_with_home(self): + os.environ['HOME'] = '/home/user' + d = config.make_service_dict('foo', {'volumes': ['~:/container/path']}, working_dir='.') + self.assertEqual(d['volumes'], ['/home/user:/container/path']) + + class MergeVolumesTest(unittest.TestCase): def test_empty(self): service_dict = config.merge_service_dicts({}, {}) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 39a6f5c10..a3a94048c 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals from __future__ import absolute_import -import os from .. import unittest import mock @@ -304,17 +303,3 @@ class ServiceVolumesTest(unittest.TestCase): self.assertEqual( binding, ('/outside', dict(bind='/inside', ro=False))) - - @mock.patch.dict(os.environ) - def test_build_volume_binding_with_environ(self): - os.environ['VOLUME_PATH'] = '/opt' - binding = build_volume_binding(parse_volume_spec('${VOLUME_PATH}:/opt')) - self.assertEqual(binding, ('/opt', dict(bind='/opt', ro=False))) - - @mock.patch.dict(os.environ) - def test_building_volume_binding_with_home(self): - os.environ['HOME'] = '/home/user' - binding = build_volume_binding(parse_volume_spec('~:/home/user')) - self.assertEqual( - binding, - ('/home/user', dict(bind='/home/user', ro=False))) From 1d7247b67e1f83c3534d80fe3f2addf4dbd8ccc7 Mon Sep 17 00:00:00 2001 From: Steven Dake Date: Wed, 8 Apr 2015 12:49:37 -0700 Subject: [PATCH 028/118] Remove stray print A previous commit introduced a stray print operation. Remove it. Signed-off-by: Steven Dake --- tests/integration/service_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 85f6db9db..38b994ba8 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -422,7 +422,6 @@ class ServiceTest(DockerClientTestCase): def test_pid_mode_none_defined(self): service = self.create_service('web', pid=None) container = create_and_start_container(service) - print 'STEAK %s' % (container.get('HostConfig.PidMode')) self.assertEqual(container.get('HostConfig.PidMode'), '') def test_pid_mode_host(self): From ceff5cb9cabc7b5e7f9ae6adc8dbd48abe340b85 Mon Sep 17 00:00:00 2001 From: Aleksandr Vinokurov Date: Tue, 31 Mar 2015 20:21:04 +0000 Subject: [PATCH 029/118] Add parent directories search for default compose-files Does not change directory to the parent with the compose-file found. Works like passing '--file' or setting 'COMPOSE_FILE' with absolute path. Resolves issue #946. Signed-off-by: Aleksandr Vinokurov --- compose/cli/command.py | 25 +++++++++++-------------- compose/cli/errors.py | 2 +- compose/cli/utils.py | 19 +++++++++++++++++++ docs/cli.md | 9 +++++++-- tests/unit/cli_test.py | 41 +++++++++++++++++++++++------------------ 5 files changed, 61 insertions(+), 35 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index e829b25b2..bd6b2dc84 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -10,7 +10,7 @@ from .. import config from ..project import Project from ..service import ConfigError from .docopt_command import DocoptCommand -from .utils import call_silently, is_mac, is_ubuntu +from .utils import call_silently, is_mac, is_ubuntu, find_candidates_in_parent_dirs from .docker_client import docker_client from . import verbose_proxy from . import errors @@ -18,6 +18,13 @@ from .. import __version__ log = logging.getLogger(__name__) +SUPPORTED_FILENAMES = [ + 'docker-compose.yml', + 'docker-compose.yaml', + 'fig.yml', + 'fig.yaml', +] + class Command(DocoptCommand): base_dir = '.' @@ -100,20 +107,10 @@ class Command(DocoptCommand): if file_path: return os.path.join(self.base_dir, file_path) - supported_filenames = [ - 'docker-compose.yml', - 'docker-compose.yaml', - 'fig.yml', - 'fig.yaml', - ] - - def expand(filename): - return os.path.join(self.base_dir, filename) - - candidates = [filename for filename in supported_filenames if os.path.exists(expand(filename))] + (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, self.base_dir) if len(candidates) == 0: - raise errors.ComposeFileNotFound(supported_filenames) + raise errors.ComposeFileNotFound(SUPPORTED_FILENAMES) winner = candidates[0] @@ -130,4 +127,4 @@ class Command(DocoptCommand): log.warning("%s is deprecated and will not be supported in future. " "Please rename your config file to docker-compose.yml\n" % winner) - return expand(winner) + return os.path.join(path, winner) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index d439aa61c..9a909e469 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -58,7 +58,7 @@ class ConnectionErrorGeneric(UserError): class ComposeFileNotFound(UserError): def __init__(self, supported_filenames): super(ComposeFileNotFound, self).__init__(""" - Can't find a suitable configuration file. Are you in the right directory? + Can't find a suitable configuration file in this directory or any parent. Are you in the right directory? Supported filenames: %s """ % ", ".join(supported_filenames)) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index d64eef4bc..5f5fed64e 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -62,6 +62,25 @@ def mkdir(path, permissions=0o700): return path +def find_candidates_in_parent_dirs(filenames, path): + """ + Given a directory path to start, looks for filenames in the + directory, and then each parent directory successively, + until found. + + Returns tuple (candidates, path). + """ + candidates = [filename for filename in filenames + if os.path.exists(os.path.join(path, filename))] + + if len(candidates) == 0: + parent_dir = os.path.join(path, '..') + if os.path.abspath(parent_dir) != os.path.abspath(path): + return find_candidates_in_parent_dirs(filenames, parent_dir) + + return (candidates, path) + + def split_buffer(reader, separator): """ Given a generator which yields strings and a separator string, diff --git a/docs/cli.md b/docs/cli.md index 30f821771..1b0fa852e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -136,7 +136,10 @@ By default, if there are existing containers for a service, `docker-compose up` ### -f, --file FILE - Specifies an alternate Compose yaml file (default: `docker-compose.yml`) + Specify what file to read configuration from. If not provided, Compose will look + for `docker-compose.yml` in the current working directory, and then each parent + directory successively, until found. + ### -p, --project-name NAME @@ -157,7 +160,9 @@ Sets the project name, which is prepended to the name of every container started ### COMPOSE\_FILE -Sets the path to the `docker-compose.yml` to use. Defaults to `docker-compose.yml` in the current working directory. +Specify what file to read configuration from. If not provided, Compose will look +for `docker-compose.yml` in the current working directory, and then each parent +directory successively, until found. ### DOCKER\_HOST diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index fcb55a673..bc49be4b8 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -62,30 +62,32 @@ class CLITestCase(unittest.TestCase): self.assertEquals(project_name, name) def test_filename_check(self): - self.assertEqual('docker-compose.yml', get_config_filename_for_files([ + files = [ 'docker-compose.yml', 'docker-compose.yaml', 'fig.yml', 'fig.yaml', - ])) + ] - self.assertEqual('docker-compose.yaml', get_config_filename_for_files([ - 'docker-compose.yaml', - 'fig.yml', - 'fig.yaml', - ])) - - self.assertEqual('fig.yml', get_config_filename_for_files([ - 'fig.yml', - 'fig.yaml', - ])) - - self.assertEqual('fig.yaml', get_config_filename_for_files([ - 'fig.yaml', - ])) + """Test with files placed in the basedir""" + self.assertEqual('docker-compose.yml', get_config_filename_for_files(files[0:])) + self.assertEqual('docker-compose.yaml', get_config_filename_for_files(files[1:])) + self.assertEqual('fig.yml', get_config_filename_for_files(files[2:])) + self.assertEqual('fig.yaml', get_config_filename_for_files(files[3:])) self.assertRaises(ComposeFileNotFound, lambda: get_config_filename_for_files([])) + """Test with files placed in the subdir""" + + def get_config_filename_for_files_in_subdir(files): + return get_config_filename_for_files(files, subdir=True) + + self.assertEqual('docker-compose.yml', get_config_filename_for_files_in_subdir(files[0:])) + self.assertEqual('docker-compose.yaml', get_config_filename_for_files_in_subdir(files[1:])) + self.assertEqual('fig.yml', get_config_filename_for_files_in_subdir(files[2:])) + self.assertEqual('fig.yaml', get_config_filename_for_files_in_subdir(files[3:])) + self.assertRaises(ComposeFileNotFound, lambda: get_config_filename_for_files_in_subdir([])) + def test_get_project(self): command = TopLevelCommand() command.base_dir = 'tests/fixtures/longer-filename-composefile' @@ -135,12 +137,15 @@ class CLITestCase(unittest.TestCase): {'FOO': 'ONE', 'BAR': 'NEW', 'OTHER': 'THREE'}) -def get_config_filename_for_files(filenames): +def get_config_filename_for_files(filenames, subdir=None): project_dir = tempfile.mkdtemp() try: make_files(project_dir, filenames) command = TopLevelCommand() - command.base_dir = project_dir + if subdir: + command.base_dir = tempfile.mkdtemp(dir=project_dir) + else: + command.base_dir = project_dir return os.path.basename(command.get_config_path()) finally: shutil.rmtree(project_dir) From 2a442ec6d98bb014bee381dfcb9ef7a69a0a3d4a Mon Sep 17 00:00:00 2001 From: Fred Lifton Date: Thu, 9 Apr 2015 16:23:25 -0700 Subject: [PATCH 030/118] Adds Where to Get Help section Signed-off-by: Fred Lifton --- docs/index.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/index.md b/docs/index.md index da5a8efdb..78d9de281 100644 --- a/docs/index.md +++ b/docs/index.md @@ -207,3 +207,17 @@ In addition to bug fixes and refinements, this release adds the following: * Better integration with Swarm. Swarm will now schedule inter-dependent containers on the same host. For details, see [PR #972](https://github.com/docker/compose/pull/972). + +## Getting help + +Docker Compose is still in its infancy and under active development. If you need +help, would like to contribute, or simply want to talk about the project with +like-minded individuals, we have a number of open channels for communication. + +* To report bugs or file feature requests: please use the [issue tracker on Github](https://github.com/docker/compose/issues). + +* To talk about the project with people in real time: please join the `#docker-compose` channel on IRC. + +* To contribute code or documentation changes: please submit a [pull request on Github](https://github.com/docker/compose/pulls). + +For more information and resources, please visit the [Getting Help project page](https://docs.docker.com/project/get-help/). From 5f17423d3ebe45b43dc8d59b56ab781cc54c89a3 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Fri, 10 Apr 2015 19:51:45 +0200 Subject: [PATCH 031/118] Add bash completion for docker-compose run --user Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index af3368036..548773d61 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -232,14 +232,14 @@ _docker-compose_run() { compopt -o nospace return ;; - --entrypoint) + --entrypoint|--user|-u) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "--allow-insecure-ssl -d --entrypoint -e --no-deps --rm --service-ports -T" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--allow-insecure-ssl -d --entrypoint -e --no-deps --rm --service-ports -T --user -u" -- "$cur" ) ) ;; *) __docker-compose_services_all From 15b763acdbeb07c8039fe940771c762b8377d71c Mon Sep 17 00:00:00 2001 From: Michael Chase-Salerno Date: Wed, 15 Apr 2015 02:03:02 +0000 Subject: [PATCH 032/118] Fix for #1224, check that image or build is specified Signed-off-by: Michael Chase-Salerno --- compose/service.py | 2 ++ tests/unit/project_test.py | 12 +++++++++++- tests/unit/service_test.py | 30 +++++++++++++++++------------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/compose/service.py b/compose/service.py index a58ab5ff4..5afaa30fa 100644 --- a/compose/service.py +++ b/compose/service.py @@ -60,6 +60,8 @@ class Service(object): raise ConfigError('Invalid project name "%s" - only %s are allowed' % (project, VALID_NAME_CHARS)) if 'image' in options and 'build' in options: raise ConfigError('Service %s has both an image and build path specified. A service can either be built to image or use an existing image, not both.' % name) + if 'image' not in options and 'build' not in options: + raise ConfigError('Service %s has neither an image nor a build path specified. Exactly one must be provided.' % name) self.name = name self.client = client diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index d5c5acb78..fc49e9b88 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -80,10 +80,12 @@ class ProjectTest(unittest.TestCase): web = Service( project='composetest', name='web', + image='foo', ) console = Service( project='composetest', name='console', + image='foo', ) project = Project('test', [web, console], None) self.assertEqual(project.get_services(), [web, console]) @@ -92,10 +94,12 @@ class ProjectTest(unittest.TestCase): web = Service( project='composetest', name='web', + image='foo', ) console = Service( project='composetest', name='console', + image='foo', ) project = Project('test', [web, console], None) self.assertEqual(project.get_services(['console']), [console]) @@ -104,19 +108,23 @@ class ProjectTest(unittest.TestCase): db = Service( project='composetest', name='db', + image='foo', ) web = Service( project='composetest', name='web', + image='foo', links=[(db, 'database')] ) cache = Service( project='composetest', - name='cache' + name='cache', + image='foo' ) console = Service( project='composetest', name='console', + image='foo', links=[(web, 'web')] ) project = Project('test', [web, db, cache, console], None) @@ -129,10 +137,12 @@ class ProjectTest(unittest.TestCase): db = Service( project='composetest', name='db', + image='foo', ) web = Service( project='composetest', name='web', + image='foo', links=[(db, 'database')] ) project = Project('test', [web, db], None) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index a3a94048c..96b081072 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -38,12 +38,12 @@ class ServiceTest(unittest.TestCase): self.assertRaises(ConfigError, lambda: Service(name='foo_bar')) self.assertRaises(ConfigError, lambda: Service(name='__foo_bar__')) - Service('a') - Service('foo') + Service('a', image='foo') + Service('foo', image='foo') def test_project_validation(self): - self.assertRaises(ConfigError, lambda: Service(name='foo', project='_')) - Service(name='foo', project='bar') + self.assertRaises(ConfigError, lambda: Service(name='foo', project='_', image='foo')) + Service(name='foo', project='bar', image='foo') def test_get_container_name(self): self.assertIsNone(get_container_name({})) @@ -52,7 +52,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual(get_container_name({'Names': ['/swarm-host-1/myproject_db_1', '/swarm-host-1/myproject_web_1/db']}), 'myproject_db_1') def test_containers(self): - service = Service('db', client=self.mock_client, project='myproject') + service = Service('db', client=self.mock_client, image='foo', project='myproject') self.mock_client.containers.return_value = [] self.assertEqual(service.containers(), []) @@ -66,7 +66,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual([c.id for c in service.containers()], ['IN_1']) def test_containers_prefixed(self): - service = Service('db', client=self.mock_client, project='myproject') + service = Service('db', client=self.mock_client, image='foo', project='myproject') self.mock_client.containers.return_value = [ {'Image': 'busybox', 'Id': 'OUT_1', 'Names': ['/swarm-host-1/myproject', '/swarm-host-1/foo/bar']}, @@ -80,14 +80,15 @@ class ServiceTest(unittest.TestCase): container_id = 'aabbccddee' service = Service( 'test', + image='foo', volumes_from=[mock.Mock(id=container_id, spec=Container)]) self.assertEqual(service._get_volumes_from(), [container_id]) def test_get_volumes_from_intermediate_container(self): container_id = 'aabbccddee' - service = Service('test') - container = mock.Mock(id=container_id, spec=Container) + service = Service('test', image='foo') + container = mock.Mock(id=container_id, spec=Container, image='foo') self.assertEqual(service._get_volumes_from(container), [container_id]) @@ -98,7 +99,7 @@ class ServiceTest(unittest.TestCase): mock.Mock(id=container_id, spec=Container) for container_id in container_ids ] - service = Service('test', volumes_from=[from_service]) + service = Service('test', volumes_from=[from_service], image='foo') self.assertEqual(service._get_volumes_from(), container_ids) @@ -109,7 +110,7 @@ class ServiceTest(unittest.TestCase): from_service.create_container.return_value = mock.Mock( id=container_id, spec=Container) - service = Service('test', volumes_from=[from_service]) + service = Service('test', image='foo', volumes_from=[from_service]) self.assertEqual(service._get_volumes_from(), [container_id]) from_service.create_container.assert_called_once_with() @@ -157,7 +158,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual(port_bindings["2000"], [("127.0.0.1", "2000")]) def test_split_domainname_none(self): - service = Service('foo', hostname='name', client=self.mock_client) + service = Service('foo', image='foo', hostname='name', client=self.mock_client) self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'image': 'foo'}) self.assertEqual(opts['hostname'], 'name', 'hostname') @@ -167,6 +168,7 @@ class ServiceTest(unittest.TestCase): service = Service( 'foo', hostname='name.domain.tld', + image='foo', client=self.mock_client) self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'image': 'foo'}) @@ -177,6 +179,7 @@ class ServiceTest(unittest.TestCase): service = Service( 'foo', hostname='name', + image='foo', domainname='domain.tld', client=self.mock_client) self.mock_client.containers.return_value = [] @@ -189,6 +192,7 @@ class ServiceTest(unittest.TestCase): 'foo', hostname='name.sub', domainname='domain.tld', + image='foo', client=self.mock_client) self.mock_client.containers.return_value = [] opts = service._get_container_create_options({'image': 'foo'}) @@ -197,7 +201,7 @@ class ServiceTest(unittest.TestCase): def test_get_container_not_found(self): self.mock_client.containers.return_value = [] - service = Service('foo', client=self.mock_client) + service = Service('foo', client=self.mock_client, image='foo') self.assertRaises(ValueError, service.get_container) @@ -205,7 +209,7 @@ class ServiceTest(unittest.TestCase): def test_get_container(self, mock_container_class): container_dict = dict(Name='default_foo_2') self.mock_client.containers.return_value = [container_dict] - service = Service('foo', client=self.mock_client) + service = Service('foo', image='foo', client=self.mock_client) container = service.get_container(number=2) self.assertEqual(container, mock_container_class.from_ps.return_value) From 24a6c240fcc74025ec703f7a9232d346cee4a58f Mon Sep 17 00:00:00 2001 From: Michael Chase-Salerno Date: Wed, 15 Apr 2015 21:38:24 +0000 Subject: [PATCH 033/118] Testcase for #1224, check that image or build is specified Signed-off-by: Michael Chase-Salerno --- tests/unit/service_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 96b081072..ec17018ed 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -42,6 +42,7 @@ class ServiceTest(unittest.TestCase): Service('foo', image='foo') def test_project_validation(self): + self.assertRaises(ConfigError, lambda: Service('bar')) self.assertRaises(ConfigError, lambda: Service(name='foo', project='_', image='foo')) Service(name='foo', project='bar', image='foo') From 8b5015c10fa6f441a7ac5337ae9af8ca083fac95 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 23 Mar 2015 10:40:23 -0700 Subject: [PATCH 034/118] Bump 1.2.0 Signed-off-by: Aanand Prasad --- CHANGES.md | 23 +++++++++++++++++++++++ compose/__init__.py | 2 +- docs/completion.md | 2 +- docs/install.md | 2 +- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 75c130906..277a188a3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,29 @@ Change log ========== +1.2.0 (2015-04-16) +------------------ + +- `docker-compose.yml` now supports an `extends` option, which enables a service to inherit configuration from another service in another configuration file. This is really good for sharing common configuration between apps, or for configuring the same app for different environments. Here's the [documentation](https://github.com/docker/compose/blob/master/docs/yml.md#extends). + +- When using Compose with a Swarm cluster, containers that depend on one another will be co-scheduled on the same node. This means that most Compose apps will now work out of the box, as long as they don't use `build`. + +- Repeated invocations of `docker-compose up` when using Compose with a Swarm cluster now work reliably. + +- Directories passed to `build`, filenames passed to `env_file` and volume host paths passed to `volumes` are now treated as relative to the *directory of the configuration file*, not the directory that `docker-compose` is being run in. In the majority of cases, those are the same, but if you use the `-f|--file` argument to specify a configuration file in another directory, **this is a breaking change**. + +- A service can now share another service's network namespace with `net: container:`. + +- `volumes_from` and `net: container:` entries are taken into account when resolving dependencies, so `docker-compose up ` will correctly start all dependencies of ``. + +- `docker-compose run` now accepts a `--user` argument to specify a user to run the command as, just like `docker run`. + +- The `up`, `stop` and `restart` commands now accept a `--timeout` (or `-t`) argument to specify how long to wait when attempting to gracefully stop containers, just like `docker stop`. + +- `docker-compose rm` now accepts `-f` as a shorthand for `--force`, just like `docker rm`. + +Thanks, @abesto, @albers, @alunduil, @dnephin, @funkyfuture, @gilclark, @IanVS, @KingsleyKelly, @knutwalker, @thaJeztah and @vmalloc! + 1.1.0 (2015-02-25) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index c770b3950..2c426c781 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import unicode_literals from .service import Service # noqa:flake8 -__version__ = '1.1.0' +__version__ = '1.2.0' diff --git a/docs/completion.md b/docs/completion.md index d9b94f6cf..6ac95c2ef 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -17,7 +17,7 @@ On a Mac, install with `brew install bash-completion` Place the completion script in `/etc/bash_completion.d/` (`/usr/local/etc/bash_completion.d/` on a Mac), using e.g. - curl -L https://raw.githubusercontent.com/docker/compose/1.1.0/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose + curl -L https://raw.githubusercontent.com/docker/compose/1.2.0/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose Completion will be available upon next login. diff --git a/docs/install.md b/docs/install.md index 7ee6f1b9d..24928d74c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -20,7 +20,7 @@ First, install Docker version 1.3 or greater: To install Compose, run the following commands: - curl -L https://github.com/docker/compose/releases/download/1.1.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.2.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands above, then `exit`. From 310c7623f9c4c9a6928f8f2e0cb471c22c396cdf Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 16 Apr 2015 17:54:18 +0100 Subject: [PATCH 035/118] Bump 1.3.0dev Signed-off-by: Aanand Prasad --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index 2c426c781..2de2a7f8b 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import unicode_literals from .service import Service # noqa:flake8 -__version__ = '1.2.0' +__version__ = '1.3.0dev' From 2291fa2d45ad38d6804e988e001761f4d8a29650 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 21 Apr 2015 11:58:44 +0100 Subject: [PATCH 036/118] Fix --timeout flag on restart, add tests for stop and restart Signed-off-by: Aanand Prasad --- compose/container.py | 4 ++-- tests/integration/cli_test.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/compose/container.py b/compose/container.py index 1d044a421..e10f13850 100644 --- a/compose/container.py +++ b/compose/container.py @@ -126,8 +126,8 @@ class Container(object): def kill(self, **options): return self.client.kill(self.id, **options) - def restart(self): - return self.client.restart(self.id) + def restart(self, **options): + return self.client.restart(self.id, **options) def remove(self, **options): return self.client.remove_container(self.id, **options) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index df3eec66d..c7e2ea343 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -332,6 +332,17 @@ class CLITestCase(DockerClientTestCase): self.command.dispatch(['rm', '-f'], None) self.assertEqual(len(service.containers(stopped=True)), 0) + def test_stop(self): + self.command.dispatch(['up', '-d'], None) + service = self.project.get_service('simple') + self.assertEqual(len(service.containers()), 1) + self.assertTrue(service.containers()[0].is_running) + + self.command.dispatch(['stop', '-t', '1'], None) + + self.assertEqual(len(service.containers(stopped=True)), 1) + self.assertFalse(service.containers(stopped=True)[0].is_running) + def test_kill(self): self.command.dispatch(['up', '-d'], None) service = self.project.get_service('simple') @@ -371,7 +382,7 @@ class CLITestCase(DockerClientTestCase): container = service.create_container() service.start_container(container) started_at = container.dictionary['State']['StartedAt'] - self.command.dispatch(['restart'], None) + self.command.dispatch(['restart', '-t', '1'], None) container.inspect() self.assertNotEqual( container.dictionary['State']['FinishedAt'], From fb81c37ca643876dff1e3f9a4feee5882a32cc1e Mon Sep 17 00:00:00 2001 From: Sam Wing Date: Thu, 15 Jan 2015 12:58:17 -0800 Subject: [PATCH 037/118] added the extra_hosts option to the yml configuration which exposes the --add-host flag from the docker client Signed-off-by: Sam Wing --- compose/config.py | 2 ++ compose/service.py | 14 ++++++++++++++ docs/yml.md | 17 +++++++++++++++++ tests/integration/service_test.py | 14 ++++++++++++++ 4 files changed, 47 insertions(+) diff --git a/compose/config.py b/compose/config.py index f87da1d8c..87d610c82 100644 --- a/compose/config.py +++ b/compose/config.py @@ -15,6 +15,7 @@ DOCKER_CONFIG_KEYS = [ 'entrypoint', 'env_file', 'environment', + 'extra_hosts', 'hostname', 'image', 'links', @@ -41,6 +42,7 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ DOCKER_CONFIG_HINTS = { 'cpu_share': 'cpu_shares', + 'add_host': 'extra_hosts', 'link': 'links', 'port': 'ports', 'privilege': 'privileged', diff --git a/compose/service.py b/compose/service.py index 5afaa30fa..dfc7a71f0 100644 --- a/compose/service.py +++ b/compose/service.py @@ -23,6 +23,7 @@ DOCKER_START_KEYS = [ 'dns', 'dns_search', 'env_file', + 'extra_hosts', 'net', 'pid', 'privileged', @@ -448,6 +449,8 @@ class Service(object): restart = parse_restart_spec(options.get('restart', None)) + extra_hosts = build_extra_hosts(options.get('extra_hosts', None)) + return create_host_config( links=self._get_links(link_to_self=one_off), port_bindings=port_bindings, @@ -460,6 +463,7 @@ class Service(object): restart_policy=restart, cap_add=cap_add, cap_drop=cap_drop, + extra_hosts=extra_hosts, pid_mode=pid ) @@ -619,3 +623,13 @@ def split_port(port): external_ip, external_port, internal_port = parts return internal_port, (external_ip, external_port or None) + + +def build_extra_hosts(extra_hosts_config): + if extra_hosts_config is None: + return None + + if isinstance(extra_hosts_config, list): + return dict(r.split(':') for r in extra_hosts_config) + else: + return dict([extra_hosts_config.split(':')]) diff --git a/docs/yml.md b/docs/yml.md index c375648df..8756b2020 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -87,6 +87,23 @@ external_links: - project_db_1:postgresql ``` +### extra_hosts + +Add hostname mappings. Use the same values as the docker client `--add-hosts` parameter. + +``` +extra_hosts: + - docker: 162.242.195.82 + - fig: 50.31.209.229 +``` + +An entry with the ip address and hostname will be created in `/etc/hosts` inside containers for this service, e.g: + +``` +162.242.195.82 docker +50.31.209.229 fig +``` + ### ports Expose ports. Either specify both ports (`HOST:CONTAINER`), or just the container diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 4abd4a909..5bc877d03 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -107,6 +107,20 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(container.inspect()['Config']['CpuShares'], 73) + def test_create_container_with_extra_hosts_list(self): + extra_hosts = ['docker:162.242.195.82', 'fig:50.31.209.229'] + service = self.create_service('db', extra_hosts=extra_hosts) + container = service.create_container() + service.start_container(container) + self.assertEqual(container.get('HostConfig.ExtraHosts'), extra_hosts) + + def test_create_container_with_extra_hosts_string(self): + extra_hosts = 'docker:162.242.195.82' + service = self.create_service('db', extra_hosts=extra_hosts) + container = service.create_container() + service.start_container(container) + self.assertEqual(container.get('HostConfig.ExtraHosts'), [extra_hosts]) + def test_create_container_with_specified_volume(self): host_path = '/tmp/host-path' container_path = '/container-path' From 8098b65576b5c8e69705d533580c1d37a09632ad Mon Sep 17 00:00:00 2001 From: Thomas Desvenain Date: Wed, 21 Jan 2015 20:33:51 +0100 Subject: [PATCH 038/118] Fix when pyyaml has interpreted line as a dictionary Added unit tests in build_extra_hosts + fix Signed-off-by: CJ --- compose/service.py | 18 ++++++++++++++---- tests/integration/service_test.py | 23 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/compose/service.py b/compose/service.py index dfc7a71f0..f2dbd746e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -629,7 +629,17 @@ def build_extra_hosts(extra_hosts_config): if extra_hosts_config is None: return None - if isinstance(extra_hosts_config, list): - return dict(r.split(':') for r in extra_hosts_config) - else: - return dict([extra_hosts_config.split(':')]) + if isinstance(extra_hosts_config, basestring): + extra_hosts_config = [extra_hosts_config] + + extra_hosts_dict = {} + for extra_hosts_line in extra_hosts_config: + if isinstance(extra_hosts_line, dict): + # already interpreted as a dict (depends on pyyaml version) + extra_hosts_dict.update(extra_hosts_line) + else: + # not already interpreted as a dict + host, ip = extra_hosts_line.split(':') + extra_hosts_dict.update({host.strip(): ip.strip()}) + + return extra_hosts_dict diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 5bc877d03..f71d609da 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -6,6 +6,7 @@ import mock from compose import Service from compose.service import CannotBeScaledError +from compose.service import build_extra_hosts from compose.container import Container from docker.errors import APIError from .testcases import DockerClientTestCase @@ -107,6 +108,28 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(container.inspect()['Config']['CpuShares'], 73) + def test_build_extra_hosts(self): + # string + self.assertEqual(build_extra_hosts("www.example.com: 192.168.0.17"), + {'www.example.com': '192.168.0.17'}) + + # list of strings + self.assertEqual(build_extra_hosts( + ["www.example.com: 192.168.0.17"]), + {'www.example.com': '192.168.0.17'}) + self.assertEqual(build_extra_hosts( + ["www.example.com: 192.168.0.17", + "api.example.com: 192.168.0.18"]), + {'www.example.com': '192.168.0.17', + 'api.example.com': '192.168.0.18'}) + # list of dictionaries + self.assertEqual(build_extra_hosts( + [{'www.example.com': '192.168.0.17'}, + {'api.example.com': '192.168.0.18'} + ]), + {'www.example.com': '192.168.0.17', + 'api.example.com': '192.168.0.18'}) + def test_create_container_with_extra_hosts_list(self): extra_hosts = ['docker:162.242.195.82', 'fig:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) From 25ee3f0033f349df9692cac943065b32be1fa4e2 Mon Sep 17 00:00:00 2001 From: CJ Date: Tue, 24 Mar 2015 18:25:09 +0800 Subject: [PATCH 039/118] Remove extra s from --add-host linting... six.string_types list-of-strings in examples disallow extra_hosts support for list-of-dicts A more thorough sets of tests for extra_hosts Provide better examples As per @aanand's [comment](https://github.com/docker/compose/pull/1158/files#r28326312) I think it'd be better to check `if not isinstance(extra_hosts_line, six.string_types)` and raise an error saying `extra_hosts_config must be either a list of strings or a string->string mapping`. We shouldn't need to do anything special with the list-of-dicts case. order result to work with assert use set() instead of sort() Signed-off-by: CJ --- compose/service.py | 31 +++++++++------- docs/yml.md | 10 ++--- tests/integration/service_test.py | 61 +++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 37 deletions(-) diff --git a/compose/service.py b/compose/service.py index f2dbd746e..e668dc493 100644 --- a/compose/service.py +++ b/compose/service.py @@ -626,20 +626,25 @@ def split_port(port): def build_extra_hosts(extra_hosts_config): - if extra_hosts_config is None: - return None + if not extra_hosts_config: + return {} - if isinstance(extra_hosts_config, basestring): - extra_hosts_config = [extra_hosts_config] - - extra_hosts_dict = {} - for extra_hosts_line in extra_hosts_config: - if isinstance(extra_hosts_line, dict): - # already interpreted as a dict (depends on pyyaml version) - extra_hosts_dict.update(extra_hosts_line) - else: - # not already interpreted as a dict + if isinstance(extra_hosts_config, list): + extra_hosts_dict = {} + for extra_hosts_line in extra_hosts_config: + if not isinstance(extra_hosts_line, six.string_types): + raise ConfigError( + "extra_hosts_config \"%s\" must be either a list of strings or a string->string mapping," % + extra_hosts_config + ) host, ip = extra_hosts_line.split(':') extra_hosts_dict.update({host.strip(): ip.strip()}) + extra_hosts_config = extra_hosts_dict - return extra_hosts_dict + if isinstance(extra_hosts_config, dict): + return extra_hosts_config + + raise ConfigError( + "extra_hosts_config \"%s\" must be either a list of strings or a string->string mapping," % + extra_hosts_config + ) diff --git a/docs/yml.md b/docs/yml.md index 8756b2020..82aeed128 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -89,19 +89,19 @@ external_links: ### extra_hosts -Add hostname mappings. Use the same values as the docker client `--add-hosts` parameter. +Add hostname mappings. Use the same values as the docker client `--add-host` parameter. ``` extra_hosts: - - docker: 162.242.195.82 - - fig: 50.31.209.229 + - "somehost:162.242.195.82" + - "otherhost:50.31.209.229" ``` An entry with the ip address and hostname will be created in `/etc/hosts` inside containers for this service, e.g: ``` -162.242.195.82 docker -50.31.209.229 fig +162.242.195.82 somehost +50.31.209.229 otherhost ``` ### ports diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index f71d609da..3fbf546c0 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -5,8 +5,11 @@ from os import path import mock from compose import Service -from compose.service import CannotBeScaledError -from compose.service import build_extra_hosts +from compose.service import ( + CannotBeScaledError, + build_extra_hosts, + ConfigError, +) from compose.container import Container from docker.errors import APIError from .testcases import DockerClientTestCase @@ -110,39 +113,59 @@ class ServiceTest(DockerClientTestCase): def test_build_extra_hosts(self): # string - self.assertEqual(build_extra_hosts("www.example.com: 192.168.0.17"), - {'www.example.com': '192.168.0.17'}) + self.assertRaises(ConfigError, lambda: build_extra_hosts("www.example.com: 192.168.0.17")) # list of strings self.assertEqual(build_extra_hosts( - ["www.example.com: 192.168.0.17"]), - {'www.example.com': '192.168.0.17'}) + ["www.example.com:192.168.0.17"]), + {'www.example.com': '192.168.0.17'}) self.assertEqual(build_extra_hosts( - ["www.example.com: 192.168.0.17", - "api.example.com: 192.168.0.18"]), - {'www.example.com': '192.168.0.17', - 'api.example.com': '192.168.0.18'}) + ["www.example.com: 192.168.0.17"]), + {'www.example.com': '192.168.0.17'}) + self.assertEqual(build_extra_hosts( + ["www.example.com: 192.168.0.17", + "static.example.com:192.168.0.19", + "api.example.com: 192.168.0.18"]), + {'www.example.com': '192.168.0.17', + 'static.example.com': '192.168.0.19', + 'api.example.com': '192.168.0.18'}) + # list of dictionaries + self.assertRaises(ConfigError, lambda: build_extra_hosts( + [{'www.example.com': '192.168.0.17'}, + {'api.example.com': '192.168.0.18'}])) + + # dictionaries self.assertEqual(build_extra_hosts( - [{'www.example.com': '192.168.0.17'}, - {'api.example.com': '192.168.0.18'} - ]), - {'www.example.com': '192.168.0.17', - 'api.example.com': '192.168.0.18'}) + {'www.example.com': '192.168.0.17', + 'api.example.com': '192.168.0.18'}), + {'www.example.com': '192.168.0.17', + 'api.example.com': '192.168.0.18'}) def test_create_container_with_extra_hosts_list(self): - extra_hosts = ['docker:162.242.195.82', 'fig:50.31.209.229'] + extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) container = service.create_container() service.start_container(container) - self.assertEqual(container.get('HostConfig.ExtraHosts'), extra_hosts) + self.assertEqual(set(container.get('HostConfig.ExtraHosts')), set(extra_hosts)) def test_create_container_with_extra_hosts_string(self): - extra_hosts = 'docker:162.242.195.82' + extra_hosts = 'somehost:162.242.195.82' + service = self.create_service('db', extra_hosts=extra_hosts) + self.assertRaises(ConfigError, lambda: service.create_container()) + + def test_create_container_with_extra_hosts_list_of_dicts(self): + extra_hosts = [{'somehost': '162.242.195.82'}, {'otherhost': '50.31.209.229'}] + service = self.create_service('db', extra_hosts=extra_hosts) + self.assertRaises(ConfigError, lambda: service.create_container()) + + def test_create_container_with_extra_hosts_dicts(self): + extra_hosts = {'somehost': '162.242.195.82', 'otherhost': '50.31.209.229'} + extra_hosts_list = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) container = service.create_container() service.start_container(container) - self.assertEqual(container.get('HostConfig.ExtraHosts'), [extra_hosts]) + self.assertEqual(set(container.get('HostConfig.ExtraHosts')), set(extra_hosts_list)) def test_create_container_with_specified_volume(self): host_path = '/tmp/host-path' From d17c4d27fa5259fb8d853ccedc82d28fd199ff8f Mon Sep 17 00:00:00 2001 From: Kyle Walker Date: Fri, 6 Mar 2015 12:33:56 -0800 Subject: [PATCH 040/118] Support alternate Dockerfile name. Signed-off-by: Kyle James Walker --- compose/cli/docker_client.py | 2 +- compose/config.py | 1 + compose/service.py | 1 + docs/yml.md | 10 ++++++++++ requirements.txt | 4 ++-- setup.py | 4 ++-- 6 files changed, 17 insertions(+), 5 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 20acbdebc..7bbe0ebfa 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -32,4 +32,4 @@ def docker_client(): ) timeout = int(os.environ.get('DOCKER_CLIENT_TIMEOUT', 60)) - return Client(base_url=base_url, tls=tls_config, version='1.15', timeout=timeout) + return Client(base_url=base_url, tls=tls_config, version='1.17', timeout=timeout) diff --git a/compose/config.py b/compose/config.py index 2c2ddf633..049c9cb32 100644 --- a/compose/config.py +++ b/compose/config.py @@ -33,6 +33,7 @@ DOCKER_CONFIG_KEYS = [ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'build', + 'dockerfile', 'expose', 'external_links', 'name', diff --git a/compose/service.py b/compose/service.py index 936e3f9d0..7dbbfe7d0 100644 --- a/compose/service.py +++ b/compose/service.py @@ -474,6 +474,7 @@ class Service(object): stream=True, rm=True, nocache=no_cache, + dockerfile=self.options.get('dockerfile', None), ) try: diff --git a/docs/yml.md b/docs/yml.md index a9909e816..9dc2884ba 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -39,6 +39,16 @@ Compose will build and tag it with a generated name, and use that image thereaft build: /path/to/build/dir ``` +### dockerfile + +Alternate Dockerfile. + +Compose will use an alternate file to build with. + +``` +dockerfile: Dockerfile-alternate +``` + ### command Override the default command. diff --git a/requirements.txt b/requirements.txt index 4c4113ab9..65f075442 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ PyYAML==3.10 -docker-py==1.0.0 +docker-py==1.1.0 dockerpty==0.3.2 docopt==0.6.1 -requests==2.2.1 +requests==2.6.1 six==1.7.3 texttable==0.8.2 websocket-client==0.11.0 diff --git a/setup.py b/setup.py index 39ac0f6f5..c02a31f4f 100644 --- a/setup.py +++ b/setup.py @@ -27,10 +27,10 @@ def find_version(*file_paths): install_requires = [ 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', - 'requests >= 2.2.1, < 2.6', + 'requests >= 2.6.1, < 2.7', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', - 'docker-py >= 1.0.0, < 1.2', + 'docker-py >= 1.1.0, < 1.2', 'dockerpty >= 0.3.2, < 0.4', 'six >= 1.3.0, < 2', ] From 9a44708081406304325c45571026c1ca3ba1f944 Mon Sep 17 00:00:00 2001 From: Michael Chase-Salerno Date: Fri, 24 Apr 2015 20:45:18 +0000 Subject: [PATCH 041/118] Fix for #1301, Alphabetize Commands Signed-off-by: Michael Chase-Salerno --- compose/cli/main.py | 6 +++--- docs/cli.md | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 85e675568..92a7c5f31 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -75,10 +75,10 @@ class TopLevelCommand(Command): docker-compose -h|--help Options: - --verbose Show more output - --version Print version and exit -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) -p, --project-name NAME Specify an alternate project name (default: directory name) + --verbose Show more output + -v, --version Print version and exit Commands: build Build or rebuild services @@ -88,12 +88,12 @@ class TopLevelCommand(Command): port Print the public port for a port binding ps List containers pull Pulls service images + restart Restart services rm Remove stopped containers run Run a one-off command scale Set number of containers for a service start Start services stop Stop services - restart Restart services up Create and start containers """ diff --git a/docs/cli.md b/docs/cli.md index 1b0fa852e..62287f138 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -47,6 +47,10 @@ Lists containers. Pulls service images. +### restart + +Restarts services. + ### rm Removes stopped service containers. @@ -130,7 +134,7 @@ By default, if there are existing containers for a service, `docker-compose up` Shows more output -### --version +### -v, --version Prints version and exits From 688f82c1cf12a7eb771ef543d4744fd89497f8d3 Mon Sep 17 00:00:00 2001 From: xuxinkun Date: Thu, 23 Apr 2015 09:33:46 +0800 Subject: [PATCH 042/118] Add cpuset config. Signed-off-by: xuxinkun --- compose/config.py | 1 + docs/yml.md | 3 ++- tests/integration/service_test.py | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/compose/config.py b/compose/config.py index 6455d7842..c50ae211a 100644 --- a/compose/config.py +++ b/compose/config.py @@ -7,6 +7,7 @@ DOCKER_CONFIG_KEYS = [ 'cap_add', 'cap_drop', 'cpu_shares', + 'cpuset', 'command', 'detach', 'dns', diff --git a/docs/yml.md b/docs/yml.md index 101c2cf27..848c34a3f 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -310,13 +310,14 @@ dns_search: - dc2.example.com ``` -### working\_dir, entrypoint, user, hostname, domainname, mem\_limit, privileged, restart, stdin\_open, tty, cpu\_shares +### working\_dir, entrypoint, user, hostname, domainname, mem\_limit, privileged, restart, stdin\_open, tty, cpu\_shares, cpuset Each of these is a single value, analogous to its [docker run](https://docs.docker.com/reference/run/) counterpart. ``` cpu_shares: 73 +cpuset: 0,1 working_dir: /code entrypoint: /code/entrypoint.sh diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3fbf546c0..c3d121952 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -167,6 +167,12 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(set(container.get('HostConfig.ExtraHosts')), set(extra_hosts_list)) + def test_create_container_with_cpu_set(self): + service = self.create_service('db', cpuset='0') + container = service.create_container() + service.start_container(container) + self.assertEqual(container.inspect()['Config']['Cpuset'], '0') + def test_create_container_with_specified_volume(self): host_path = '/tmp/host-path' container_path = '/container-path' From 0ca9fa8b2b8fe08b639a07b15d828b22d32807a0 Mon Sep 17 00:00:00 2001 From: xwisen Date: Sun, 26 Apr 2015 12:49:27 +0800 Subject: [PATCH 043/118] modified the release notes section the first[PR #972]to[PR #1088] Signed-off-by: xwisen --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 78d9de281..5ddf1bbc4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -202,7 +202,7 @@ For complete information on this release, see the [1.2.0 Milestone project page] In addition to bug fixes and refinements, this release adds the following: * The `extends` keyword, which adds the ability to extend services by sharing common configurations. For details, see -[PR #972](https://github.com/docker/compose/pull/1088). +[PR #1088](https://github.com/docker/compose/pull/1088). * Better integration with Swarm. Swarm will now schedule inter-dependent containers on the same host. For details, see From 86a08c00f247131bfebcf750f37866eb0fcf9458 Mon Sep 17 00:00:00 2001 From: CJ Date: Mon, 27 Apr 2015 14:07:21 +0800 Subject: [PATCH 044/118] See https://github.com/docker/compose/pull/1158#discussion_r29063218 Signed-off-by: CJ --- compose/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/config.py b/compose/config.py index 87d610c82..95948ef8a 100644 --- a/compose/config.py +++ b/compose/config.py @@ -43,6 +43,8 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ DOCKER_CONFIG_HINTS = { 'cpu_share': 'cpu_shares', 'add_host': 'extra_hosts', + 'hosts': 'extra_hosts', + 'extra_host': 'extra_hosts', 'link': 'links', 'port': 'ports', 'privilege': 'privileged', From 2e19887bf102cbc77f668a2abff9e5b3035e5746 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 27 Apr 2015 14:58:20 +0100 Subject: [PATCH 045/118] Update README.md with changes to docs/index.md Signed-off-by: Aanand Prasad --- README.md | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ce89d5aa2..60b577094 100644 --- a/README.md +++ b/README.md @@ -12,33 +12,24 @@ recommend that you use it in production yet. Using Compose is basically a three-step process. -First, you define your app's environment with a `Dockerfile` so it can be -reproduced anywhere: - -```Dockerfile -FROM python:2.7 -WORKDIR /code -ADD requirements.txt /code/ -RUN pip install -r requirements.txt -ADD . /code -CMD python app.py -``` - -Next, you define the services that make up your app in `docker-compose.yml` so +1. Define your app's environment with a `Dockerfile` so it can be +reproduced anywhere. +2. Define the services that make up your app in `docker-compose.yml` so they can be run together in an isolated environment: +3. Lastly, run `docker-compose up` and Compose will start and run your entire app. -```yaml -web: - build: . - links: - - db - ports: - - "8000:8000" -db: - image: postgres -``` +A `docker-compose.yml` looks like this: -Lastly, run `docker-compose up` and Compose will start and run your entire app. + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + links: + - redis + redis: + image: redis Compose has commands for managing the whole lifecycle of your application: From 240495f07f54574cbc83030f472948cca7fbd93e Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 27 Apr 2015 15:00:50 +0100 Subject: [PATCH 046/118] Remove DCO validation from CI script Signed-off-by: Aanand Prasad --- CONTRIBUTING.md | 6 ++--- script/.validate | 33 -------------------------- script/ci | 3 --- script/validate-dco | 58 --------------------------------------------- 4 files changed, 2 insertions(+), 98 deletions(-) delete mode 100644 script/.validate delete mode 100755 script/validate-dco diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0cca17b00..373c8dc6f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,8 +31,8 @@ that should get you started. ## Running the test suite -Use the test script to run DCO check, linting checks and then the full test -suite against different Python interpreters: +Use the test script to run linting checks and then the full test suite against +different Python interpreters: $ script/test @@ -51,8 +51,6 @@ you can specify a test directory, file, module, class or method: $ script/test tests.integration.service_test $ script/test tests.integration.service_test:ServiceTest.test_containers -Before pushing a commit you can check the DCO by invoking `script/validate-dco`. - ## Building binaries Linux: diff --git a/script/.validate b/script/.validate deleted file mode 100644 index 244cbe49c..000000000 --- a/script/.validate +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -if [ -z "$VALIDATE_UPSTREAM" ]; then - # this is kind of an expensive check, so let's not do this twice if we - # are running more than one validate bundlescript - - VALIDATE_REPO='https://github.com/docker/fig.git' - VALIDATE_BRANCH='master' - - if [ "$TRAVIS" = 'true' -a "$TRAVIS_PULL_REQUEST" != 'false' ]; then - VALIDATE_REPO="https://github.com/${TRAVIS_REPO_SLUG}.git" - VALIDATE_BRANCH="${TRAVIS_BRANCH}" - fi - - VALIDATE_HEAD="$(git rev-parse --verify HEAD)" - - git fetch -q "$VALIDATE_REPO" "refs/heads/$VALIDATE_BRANCH" - VALIDATE_UPSTREAM="$(git rev-parse --verify FETCH_HEAD)" - - VALIDATE_COMMIT_LOG="$VALIDATE_UPSTREAM..$VALIDATE_HEAD" - VALIDATE_COMMIT_DIFF="$VALIDATE_UPSTREAM...$VALIDATE_HEAD" - - validate_diff() { - if [ "$VALIDATE_UPSTREAM" != "$VALIDATE_HEAD" ]; then - git diff "$VALIDATE_COMMIT_DIFF" "$@" - fi - } - validate_log() { - if [ "$VALIDATE_UPSTREAM" != "$VALIDATE_HEAD" ]; then - git log "$VALIDATE_COMMIT_LOG" "$@" - fi - } -fi diff --git a/script/ci b/script/ci index a1391c627..2e4ec9197 100755 --- a/script/ci +++ b/script/ci @@ -8,9 +8,6 @@ set -e ->&2 echo "Validating DCO" -script/validate-dco - export DOCKER_VERSIONS=all . script/test-versions diff --git a/script/validate-dco b/script/validate-dco deleted file mode 100755 index 701ac5e46..000000000 --- a/script/validate-dco +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -set -e - -source "$(dirname "$BASH_SOURCE")/.validate" - -adds=$(validate_diff --numstat | awk '{ s += $1 } END { print s }') -dels=$(validate_diff --numstat | awk '{ s += $2 } END { print s }') -notDocs="$(validate_diff --numstat | awk '$3 !~ /^docs\// { print $3 }')" - -: ${adds:=0} -: ${dels:=0} - -# "Username may only contain alphanumeric characters or dashes and cannot begin with a dash" -githubUsernameRegex='[a-zA-Z0-9][a-zA-Z0-9-]+' - -# https://github.com/docker/docker/blob/master/CONTRIBUTING.md#sign-your-work -dcoPrefix='Signed-off-by:' -dcoRegex="^(Docker-DCO-1.1-)?$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)>( \\(github: ($githubUsernameRegex)\\))?$" - -check_dco() { - grep -qE "$dcoRegex" -} - -if [ $adds -eq 0 -a $dels -eq 0 ]; then - echo '0 adds, 0 deletions; nothing to validate! :)' -elif [ -z "$notDocs" -a $adds -le 1 -a $dels -le 1 ]; then - echo 'Congratulations! DCO small-patch-exception material!' -else - commits=( $(validate_log --format='format:%H%n') ) - badCommits=() - for commit in "${commits[@]}"; do - if [ -z "$(git log -1 --format='format:' --name-status "$commit")" ]; then - # no content (ie, Merge commit, etc) - continue - fi - if ! git log -1 --format='format:%B' "$commit" | check_dco; then - badCommits+=( "$commit" ) - fi - done - if [ ${#badCommits[@]} -eq 0 ]; then - echo "Congratulations! All commits are properly signed with the DCO!" - else - { - echo "These commits do not have a proper '$dcoPrefix' marker:" - for commit in "${badCommits[@]}"; do - echo " - $commit" - done - echo - echo 'Please amend each commit to include a properly formatted DCO marker.' - echo - echo 'Visit the following URL for information about the Docker DCO:' - echo ' https://github.com/docker/docker/blob/master/CONTRIBUTING.md#sign-your-work' - echo - } >&2 - false - fi -fi From 7d617d60bc2d4e0ed27a7c27378a84bb6485acbe Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 27 Apr 2015 15:10:01 +0100 Subject: [PATCH 047/118] Remove wercker.yml Signed-off-by: Aanand Prasad --- wercker.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 wercker.yml diff --git a/wercker.yml b/wercker.yml deleted file mode 100644 index 96fb22b57..000000000 --- a/wercker.yml +++ /dev/null @@ -1,12 +0,0 @@ -box: wercker-labs/docker -build: - steps: - - script: - name: validate DCO - code: script/validate-dco - - script: - name: run tests - code: script/test - - script: - name: build binary - code: script/build-linux From 855855a0e64d2304f8a9a1218783e82570ed7aa1 Mon Sep 17 00:00:00 2001 From: Timothy Van Heest Date: Mon, 27 Apr 2015 08:17:53 -0400 Subject: [PATCH 048/118] Fix for #1350, nonexisting build path in parent section causes extending section to fail Signed-off-by: Timothy Van Heest --- compose/config.py | 13 +++++++----- .../extends/nonexistent-path-base.yml | 6 ++++++ .../extends/nonexistent-path-child.yml | 8 ++++++++ tests/unit/config_test.py | 20 ++++++++++++++++++- 4 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/extends/nonexistent-path-base.yml create mode 100644 tests/fixtures/extends/nonexistent-path-child.yml diff --git a/compose/config.py b/compose/config.py index f87da1d8c..50ac5b606 100644 --- a/compose/config.py +++ b/compose/config.py @@ -64,6 +64,7 @@ def from_dictionary(dictionary, working_dir=None, filename=None): raise ConfigurationError('Service "%s" doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.' % service_name) loader = ServiceLoader(working_dir=working_dir, filename=filename) service_dict = loader.make_service_dict(service_name, service_dict) + validate_paths(service_dict) service_dicts.append(service_dict) return service_dicts @@ -339,12 +340,14 @@ def resolve_host_path(volume, working_dir): def resolve_build_path(build_path, working_dir=None): if working_dir is None: raise Exception("No working_dir passed to resolve_build_path") + return expand_path(working_dir, build_path) - _path = expand_path(working_dir, build_path) - if not os.path.exists(_path) or not os.access(_path, os.R_OK): - raise ConfigurationError("build path %s either does not exist or is not accessible." % _path) - else: - return _path + +def validate_paths(service_dict): + if 'build' in service_dict: + build_path = service_dict['build'] + if not os.path.exists(build_path) or not os.access(build_path, os.R_OK): + raise ConfigurationError("build path %s either does not exist or is not accessible." % build_path) def merge_volumes(base, override): diff --git a/tests/fixtures/extends/nonexistent-path-base.yml b/tests/fixtures/extends/nonexistent-path-base.yml new file mode 100644 index 000000000..1cf9a304a --- /dev/null +++ b/tests/fixtures/extends/nonexistent-path-base.yml @@ -0,0 +1,6 @@ +dnebase: + build: nonexistent.path + command: /bin/true + environment: + - FOO=1 + - BAR=1 \ No newline at end of file diff --git a/tests/fixtures/extends/nonexistent-path-child.yml b/tests/fixtures/extends/nonexistent-path-child.yml new file mode 100644 index 000000000..aab11459b --- /dev/null +++ b/tests/fixtures/extends/nonexistent-path-child.yml @@ -0,0 +1,8 @@ +dnechild: + extends: + file: nonexistent-path-base.yml + service: dnebase + image: busybox + command: /bin/true + environment: + - BAR=2 \ No newline at end of file diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 97bd1b91d..12610f102 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -398,6 +398,21 @@ class ExtendsTest(unittest.TestCase): self.assertEqual(set(dicts[0]['volumes']), set(paths)) + def test_parent_build_path_dne(self): + child = config.load('tests/fixtures/extends/nonexistent-path-child.yml') + + self.assertEqual(child, [ + { + 'name': 'dnechild', + 'image': 'busybox', + 'command': '/bin/true', + 'environment': { + "FOO": "1", + "BAR": "2", + }, + }, + ]) + class BuildPathTest(unittest.TestCase): def setUp(self): @@ -407,7 +422,10 @@ class BuildPathTest(unittest.TestCase): options = {'build': 'nonexistent.path'} self.assertRaises( config.ConfigurationError, - lambda: config.make_service_dict('foo', options, 'tests/fixtures/build-path'), + lambda: config.from_dictionary({ + 'foo': options, + 'working_dir': 'tests/fixtures/build-path' + }) ) def test_relative_path(self): From e5a118e3cedaf7a3a77967ba9201665d64eae069 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Fri, 24 Apr 2015 23:26:56 +0100 Subject: [PATCH 049/118] Use cool new IRCCloud links for IRC channel Signed-off-by: Ben Firshman --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ce89d5aa2..f62f0d86b 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Installation and documentation ------------------------------ - Full documentation is available on [Docker's website](http://docs.docker.com/compose/). -- Hop into #docker-compose on Freenode if you have any questions. +- If you have any questions, you can talk in real-time with other developers in the #docker-compose IRC channel on Freenode. [Click here to join using IRCCloud.](https://www.irccloud.com/invite?hostname=irc.freenode.net&channel=%23docker-compose) Contributing ------------ From 021bf465572c2dae7e002a98f415fc61fa864691 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 27 Apr 2015 15:18:00 +0100 Subject: [PATCH 050/118] Update Docker version to 1.6 stable Signed-off-by: Aanand Prasad --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7438d6b1b..b2ae0063c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,14 +15,14 @@ RUN set -ex; \ ; \ rm -rf /var/lib/apt/lists/* -ENV ALL_DOCKER_VERSIONS 1.6.0-rc4 +ENV ALL_DOCKER_VERSIONS 1.6.0 RUN set -ex; \ - curl https://test.docker.com/builds/Linux/x86_64/docker-1.6.0-rc4 -o /usr/local/bin/docker-1.6.0-rc4; \ - chmod +x /usr/local/bin/docker-1.6.0-rc4 + curl https://get.docker.com/builds/Linux/x86_64/docker-1.6.0 -o /usr/local/bin/docker-1.6.0; \ + chmod +x /usr/local/bin/docker-1.6.0 # Set the default Docker to be run -RUN ln -s /usr/local/bin/docker-1.6.0-rc4 /usr/local/bin/docker +RUN ln -s /usr/local/bin/docker-1.6.0 /usr/local/bin/docker RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ From 3dd860f0ba67ac57ef9889eb0dd6ad58784a3030 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 24 Apr 2015 11:11:01 -0400 Subject: [PATCH 051/118] Fix #923, support image with ids instead of names. Signed-off-by: Daniel Nephin --- compose/service.py | 35 +++++++++++------------------ tests/integration/service_test.py | 5 +++++ tests/integration/testcases.py | 2 +- tests/unit/service_test.py | 37 +++++++++++++++++++++++++------ 4 files changed, 49 insertions(+), 30 deletions(-) diff --git a/compose/service.py b/compose/service.py index 5afaa30fa..eeda8f32b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -193,13 +193,7 @@ class Service(object): return Container.create(self.client, **container_options) except APIError as e: if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation): - log.info('Pulling image %s...' % container_options['image']) - output = self.client.pull( - container_options['image'], - stream=True, - insecure_registry=insecure_registry - ) - stream_output(output, sys.stdout) + self.pull(insecure_registry=insecure_registry) return Container.create(self.client, **container_options) raise @@ -413,8 +407,6 @@ class Service(object): if self.can_be_built(): container_options['image'] = self.full_name - else: - container_options['image'] = self._get_image_name(container_options['image']) # Delete options which are only used when starting for key in DOCKER_START_KEYS: @@ -463,12 +455,6 @@ class Service(object): pid_mode=pid ) - def _get_image_name(self, image): - repo, tag = parse_repository_tag(image) - if tag == "": - tag = "latest" - return '%s:%s' % (repo, tag) - def build(self, no_cache=False): log.info('Building %s...' % self.name) @@ -515,13 +501,18 @@ class Service(object): return True def pull(self, insecure_registry=False): - if 'image' in self.options: - image_name = self._get_image_name(self.options['image']) - log.info('Pulling %s (%s)...' % (self.name, image_name)) - self.client.pull( - image_name, - insecure_registry=insecure_registry - ) + if 'image' not in self.options: + return + + repo, tag = parse_repository_tag(self.options['image']) + tag = tag or 'latest' + log.info('Pulling %s (%s:%s)...' % (self.name, repo, tag)) + output = self.client.pull( + repo, + tag=tag, + stream=True, + insecure_registry=insecure_registry) + stream_output(output, sys.stdout) NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$') diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 4abd4a909..891caae58 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -391,6 +391,11 @@ class ServiceTest(DockerClientTestCase): ], }) + def test_start_with_image_id(self): + # Image id for the current busybox:latest + service = self.create_service('foo', image='8c2e06607696') + self.assertTrue(service.start_or_create_containers()) + def test_scale(self): service = self.create_service('web') service.scale(1) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index d5ca1debc..715b135c4 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -22,7 +22,7 @@ class DockerClientTestCase(unittest.TestCase): self.client.remove_image(i) def create_service(self, name, **kwargs): - kwargs['image'] = "busybox:latest" + kwargs['image'] = kwargs.pop('image', 'busybox:latest') if 'command' not in kwargs: kwargs['command'] = ["/bin/sleep", "300"] diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index ec17018ed..b9f968db1 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -221,9 +221,22 @@ class ServiceTest(unittest.TestCase): def test_pull_image(self, mock_log): service = Service('foo', client=self.mock_client, image='someimage:sometag') service.pull(insecure_registry=True) - self.mock_client.pull.assert_called_once_with('someimage:sometag', insecure_registry=True) + self.mock_client.pull.assert_called_once_with( + 'someimage', + tag='sometag', + insecure_registry=True, + stream=True) mock_log.info.assert_called_once_with('Pulling foo (someimage:sometag)...') + def test_pull_image_no_tag(self): + service = Service('foo', client=self.mock_client, image='ababab') + service.pull() + self.mock_client.pull.assert_called_once_with( + 'ababab', + tag='latest', + insecure_registry=False, + stream=True) + @mock.patch('compose.service.Container', autospec=True) @mock.patch('compose.service.log', autospec=True) def test_create_container_from_insecure_registry( @@ -243,11 +256,12 @@ class ServiceTest(unittest.TestCase): service.create_container(insecure_registry=True) self.mock_client.pull.assert_called_once_with( - 'someimage:sometag', + 'someimage', + tag='sometag', insecure_registry=True, stream=True) mock_log.info.assert_called_once_with( - 'Pulling image someimage:sometag...') + 'Pulling foo (someimage:sometag)...') def test_parse_repository_tag(self): self.assertEqual(parse_repository_tag("root"), ("root", "")) @@ -257,11 +271,20 @@ class ServiceTest(unittest.TestCase): self.assertEqual(parse_repository_tag("url:5000/repo"), ("url:5000/repo", "")) self.assertEqual(parse_repository_tag("url:5000/repo:tag"), ("url:5000/repo", "tag")) - def test_latest_is_used_when_tag_is_not_specified(self): + @mock.patch('compose.service.Container', autospec=True) + def test_create_container_latest_is_used_when_no_tag_specified(self, mock_container): + mock_container.create.side_effect = APIError( + "oops", + mock.Mock(status_code=404), + "No such image") service = Service('foo', client=self.mock_client, image='someimage') - Container.create = mock.Mock() - service.create_container() - self.assertEqual(Container.create.call_args[1]['image'], 'someimage:latest') + with self.assertRaises(APIError): + service.create_container() + self.mock_client.pull.assert_called_once_with( + 'someimage', + tag='latest', + insecure_registry=False, + stream=True) def test_create_container_with_build(self): self.mock_client.images.return_value = [] From 2e6bc078fbc502965ac5d3a3ec0be13aafcfeb09 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 19 Mar 2015 16:10:27 -0700 Subject: [PATCH 052/118] Implement 'labels' option Signed-off-by: Aanand Prasad --- compose/cli/docker_client.py | 2 +- compose/config.py | 41 ++++++++++++++++++++++++++++++- compose/container.py | 4 +++ docs/extends.md | 4 +-- docs/install.md | 2 +- docs/yml.md | 18 ++++++++++++++ requirements.txt | 2 +- setup.py | 2 +- tests/integration/service_test.py | 27 ++++++++++++++++++++ tests/unit/config_test.py | 41 +++++++++++++++++++++++++++++++ 10 files changed, 136 insertions(+), 7 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 7bbe0ebfa..e513182fb 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -32,4 +32,4 @@ def docker_client(): ) timeout = int(os.environ.get('DOCKER_CLIENT_TIMEOUT', 60)) - return Client(base_url=base_url, tls=tls_config, version='1.17', timeout=timeout) + return Client(base_url=base_url, tls=tls_config, version='1.18', timeout=timeout) diff --git a/compose/config.py b/compose/config.py index c28703961..d5a82114a 100644 --- a/compose/config.py +++ b/compose/config.py @@ -18,6 +18,7 @@ DOCKER_CONFIG_KEYS = [ 'extra_hosts', 'hostname', 'image', + 'labels', 'links', 'mem_limit', 'net', @@ -180,6 +181,9 @@ def process_container_options(service_dict, working_dir=None): if 'build' in service_dict: service_dict['build'] = resolve_build_path(service_dict['build'], working_dir=working_dir) + if 'labels' in service_dict: + service_dict['labels'] = parse_labels(service_dict['labels']) + return service_dict @@ -198,6 +202,12 @@ def merge_service_dicts(base, override): override.get('volumes'), ) + if 'labels' in base or 'labels' in override: + d['labels'] = merge_labels( + base.get('labels'), + override.get('labels'), + ) + if 'image' in override and 'build' in d: del d['build'] @@ -216,7 +226,7 @@ def merge_service_dicts(base, override): if key in base or key in override: d[key] = to_list(base.get(key)) + to_list(override.get(key)) - already_merged_keys = ['environment', 'volumes'] + list_keys + list_or_string_keys + already_merged_keys = ['environment', 'volumes', 'labels'] + list_keys + list_or_string_keys for k in set(ALLOWED_KEYS) - set(already_merged_keys): if k in override: @@ -385,6 +395,35 @@ def join_volume(pair): return ":".join((host, container)) +def merge_labels(base, override): + labels = parse_labels(base) + labels.update(parse_labels(override)) + return labels + + +def parse_labels(labels): + if not labels: + return {} + + if isinstance(labels, list): + return dict(split_label(e) for e in labels) + + if isinstance(labels, dict): + return labels + + raise ConfigurationError( + "labels \"%s\" must be a list or mapping" % + labels + ) + + +def split_label(label): + if '=' in label: + return label.split('=', 1) + else: + return label, '' + + def expand_path(working_dir, path): return os.path.abspath(os.path.join(working_dir, path)) diff --git a/compose/container.py b/compose/container.py index e10f13850..9439a7087 100644 --- a/compose/container.py +++ b/compose/container.py @@ -79,6 +79,10 @@ class Container(object): return ', '.join(format_port(*item) for item in sorted(six.iteritems(self.ports))) + @property + def labels(self): + return self.get('Config.Labels') or {} + @property def human_readable_state(self): if self.is_running: diff --git a/docs/extends.md b/docs/extends.md index 2393ca6ae..06c08f25e 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -321,8 +321,8 @@ expose: - "5000" ``` -In the case of `environment`, Compose "merges" entries together with -locally-defined values taking precedence: +In the case of `environment` and `labels`, Compose "merges" entries together +with locally-defined values taking precedence: ```yaml # original service diff --git a/docs/install.md b/docs/install.md index 24928d74c..a3524c603 100644 --- a/docs/install.md +++ b/docs/install.md @@ -10,7 +10,7 @@ Compose with a `curl` command. ### Install Docker -First, install Docker version 1.3 or greater: +First, install Docker version 1.6 or greater: - [Instructions for Mac OS X](http://docs.docker.com/installation/mac/) - [Instructions for Ubuntu](http://docs.docker.com/installation/ubuntulinux/) diff --git a/docs/yml.md b/docs/yml.md index 101c2cf27..964cf5f20 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -253,6 +253,24 @@ environment variables (DEBUG) with a new value, and the other one For more on `extends`, see the [tutorial](extends.md#example) and [reference](extends.md#reference). +### labels + +Add metadata to containers using [Docker labels](http://docs.docker.com/userguide/labels-custom-metadata/). You can use either an array or a dictionary. + +It's recommended that you use reverse-DNS notation to prevent your labels from conflicting with those used by other software. + +``` +labels: + com.example.description: "Accounting webapp" + com.example.department: "Finance" + com.example.label-with-empty-value: "" + +labels: + - "com.example.description=Accounting webapp" + - "com.example.department=Finance" + - "com.example.label-with-empty-value" +``` + ### net Networking mode. Use the same values as the docker client `--net` parameter. diff --git a/requirements.txt b/requirements.txt index 65f075442..ed09cccac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.10 -docker-py==1.1.0 +docker-py==1.2.1 dockerpty==0.3.2 docopt==0.6.1 requests==2.6.1 diff --git a/setup.py b/setup.py index c02a31f4f..46193eeef 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.7', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', - 'docker-py >= 1.1.0, < 1.2', + 'docker-py >= 1.2.0, < 1.3', 'dockerpty >= 0.3.2, < 0.4', 'six >= 1.3.0, < 2', ] diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3fbf546c0..df5b2b9d3 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -584,3 +584,30 @@ class ServiceTest(DockerClientTestCase): env = create_and_start_container(service).environment for k, v in {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}.items(): self.assertEqual(env[k], v) + + def test_labels(self): + labels_dict = { + 'com.example.description': "Accounting webapp", + 'com.example.department': "Finance", + 'com.example.label-with-empty-value': "", + } + + service = self.create_service('web', labels=labels_dict) + labels = create_and_start_container(service).labels.items() + for pair in labels_dict.items(): + self.assertIn(pair, labels) + + labels_list = ["%s=%s" % pair for pair in labels_dict.items()] + + service = self.create_service('web', labels=labels_list) + labels = create_and_start_container(service).labels.items() + for pair in labels_dict.items(): + self.assertIn(pair, labels) + + def test_empty_labels(self): + labels_list = ['foo', 'bar'] + + service = self.create_service('web', labels=labels_list) + labels = create_and_start_container(service).labels.items() + for name in labels_list: + self.assertIn((name, ''), labels) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 97bd1b91d..c478a2182 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -185,6 +185,47 @@ class MergeStringsOrListsTest(unittest.TestCase): self.assertEqual(set(service_dict['dns']), set(['8.8.8.8', '9.9.9.9'])) +class MergeLabelsTest(unittest.TestCase): + def test_empty(self): + service_dict = config.merge_service_dicts({}, {}) + self.assertNotIn('labels', service_dict) + + def test_no_override(self): + service_dict = config.merge_service_dicts( + config.make_service_dict('foo', {'labels': ['foo=1', 'bar']}), + config.make_service_dict('foo', {}), + ) + self.assertEqual(service_dict['labels'], {'foo': '1', 'bar': ''}) + + def test_no_base(self): + service_dict = config.merge_service_dicts( + config.make_service_dict('foo', {}), + config.make_service_dict('foo', {'labels': ['foo=2']}), + ) + self.assertEqual(service_dict['labels'], {'foo': '2'}) + + def test_override_explicit_value(self): + service_dict = config.merge_service_dicts( + config.make_service_dict('foo', {'labels': ['foo=1', 'bar']}), + config.make_service_dict('foo', {'labels': ['foo=2']}), + ) + self.assertEqual(service_dict['labels'], {'foo': '2', 'bar': ''}) + + def test_add_explicit_value(self): + service_dict = config.merge_service_dicts( + config.make_service_dict('foo', {'labels': ['foo=1', 'bar']}), + config.make_service_dict('foo', {'labels': ['bar=2']}), + ) + self.assertEqual(service_dict['labels'], {'foo': '1', 'bar': '2'}) + + def test_remove_explicit_value(self): + service_dict = config.merge_service_dicts( + config.make_service_dict('foo', {'labels': ['foo=1', 'bar=2']}), + config.make_service_dict('foo', {'labels': ['bar']}), + ) + self.assertEqual(service_dict['labels'], {'foo': '1', 'bar': ''}) + + class EnvTest(unittest.TestCase): def test_parse_environment_as_list(self): environment = [ From 1a77feea3f6f938d0511a2dcc5f2f8c7b14a720c Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 29 Apr 2015 17:54:33 +0100 Subject: [PATCH 053/118] Close connection before attaching on 'up' and 'run' This ensures that the connection is not recycled, which can cause the Docker daemon to complain if we've already performed another streaming call such as doing a build. Signed-off-by: Aanand Prasad --- compose/service.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compose/service.py b/compose/service.py index 43230be3d..b81922dd1 100644 --- a/compose/service.py +++ b/compose/service.py @@ -476,6 +476,11 @@ class Service(object): except StreamOutputError as e: raise BuildError(self, unicode(e)) + # Ensure the HTTP connection is not reused for another + # streaming command, as the Docker daemon can sometimes + # complain about it + self.client.close() + image_id = None for event in all_events: From 4f366d83556f23a34f5a1a59cc45fb7eada3c5c2 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 30 Apr 2015 11:57:46 +0100 Subject: [PATCH 054/118] Make sure the build path we pass to docker-py is a binary string Signed-off-by: Aanand Prasad --- compose/service.py | 4 +++- tests/integration/service_test.py | 27 +++++++++++++++++++++++++++ tests/integration/testcases.py | 3 ++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index b81922dd1..c66344833 100644 --- a/compose/service.py +++ b/compose/service.py @@ -462,8 +462,10 @@ class Service(object): def build(self, no_cache=False): log.info('Building %s...' % self.name) + path = six.binary_type(self.options['build']) + build_output = self.client.build( - self.options['build'], + path=path, tag=self.full_name, stream=True, rm=True, diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 6e63bf365..8b2a1b1a7 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -4,6 +4,10 @@ import os from os import path import mock +import tempfile +import shutil +import six + from compose import Service from compose.service import ( CannotBeScaledError, @@ -404,6 +408,29 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(list(container['NetworkSettings']['Ports'].keys()), ['8000/tcp']) self.assertNotEqual(container['NetworkSettings']['Ports']['8000/tcp'][0]['HostPort'], '8000') + def test_build(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + + self.create_service('web', build=base_dir).build() + self.assertEqual(len(self.client.images(name='composetest_web')), 1) + + def test_build_non_ascii_filename(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + + with open(os.path.join(base_dir, b'foo\xE2bar'), 'w') as f: + f.write("hello world\n") + + self.create_service('web', build=six.text_type(base_dir)).build() + self.assertEqual(len(self.client.images(name='composetest_web')), 1) + def test_start_container_stays_unpriviliged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 715b135c4..31281a1d7 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -22,7 +22,8 @@ class DockerClientTestCase(unittest.TestCase): self.client.remove_image(i) def create_service(self, name, **kwargs): - kwargs['image'] = kwargs.pop('image', 'busybox:latest') + if 'image' not in kwargs and 'build' not in kwargs: + kwargs['image'] = 'busybox:latest' if 'command' not in kwargs: kwargs['command'] = ["/bin/sleep", "300"] From b8e0aed21cf4a13d11c49b8f09a73e7e20c4ea46 Mon Sep 17 00:00:00 2001 From: Simon Herter Date: Fri, 1 May 2015 18:51:20 -0400 Subject: [PATCH 055/118] Show proper command in help text of build subcommand The help text of the build subcommand suggested to use 'compose build' (instead of 'docker-compose build') to rebuild images. Signed-off-by: Simon Herter --- compose/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 92a7c5f31..e941c005c 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -108,7 +108,7 @@ class TopLevelCommand(Command): Services are built once and then tagged as `project_service`, e.g. `composetest_db`. If you change a service's `Dockerfile` or the - contents of its build directory, you can run `compose build` to rebuild it. + contents of its build directory, you can run `docker-compose build` to rebuild it. Usage: build [options] [SERVICE...] From b06294399a4f7a4895a3ac1ea324b235f1763377 Mon Sep 17 00:00:00 2001 From: CJ Date: Sun, 29 Mar 2015 23:28:57 +0800 Subject: [PATCH 056/118] See #1335: Added --read-only Signed-off-by: CJ --- compose/config.py | 1 + compose/service.py | 3 +++ docs/yml.md | 3 ++- tests/integration/service_test.py | 7 +++++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/compose/config.py b/compose/config.py index d54215dd2..8ebcef010 100644 --- a/compose/config.py +++ b/compose/config.py @@ -17,6 +17,7 @@ DOCKER_CONFIG_KEYS = [ 'env_file', 'environment', 'extra_hosts', + 'read_only', 'hostname', 'image', 'labels', diff --git a/compose/service.py b/compose/service.py index c66344833..6250e5351 100644 --- a/compose/service.py +++ b/compose/service.py @@ -24,6 +24,7 @@ DOCKER_START_KEYS = [ 'dns_search', 'env_file', 'extra_hosts', + 'read_only', 'net', 'pid', 'privileged', @@ -442,6 +443,7 @@ class Service(object): restart = parse_restart_spec(options.get('restart', None)) extra_hosts = build_extra_hosts(options.get('extra_hosts', None)) + read_only = options.get('read_only', None) return create_host_config( links=self._get_links(link_to_self=one_off), @@ -456,6 +458,7 @@ class Service(object): cap_add=cap_add, cap_drop=cap_drop, extra_hosts=extra_hosts, + read_only=read_only, pid_mode=pid ) diff --git a/docs/yml.md b/docs/yml.md index d7196b40e..1e910d184 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -328,7 +328,7 @@ dns_search: - dc2.example.com ``` -### working\_dir, entrypoint, user, hostname, domainname, mem\_limit, privileged, restart, stdin\_open, tty, cpu\_shares, cpuset +### working\_dir, entrypoint, user, hostname, domainname, mem\_limit, privileged, restart, stdin\_open, tty, cpu\_shares, cpuset, read\_only Each of these is a single value, analogous to its [docker run](https://docs.docker.com/reference/run/) counterpart. @@ -351,6 +351,7 @@ restart: always stdin_open: true tty: true +read_only: true ``` ## Compose documentation diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index f7190df64..edda71dec 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -177,6 +177,13 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(container.inspect()['Config']['Cpuset'], '0') + def test_create_container_with_read_only_root_fs(self): + read_only = True + service = self.create_service('db', read_only=read_only) + container = service.create_container() + service.start_container(container) + self.assertEqual(container.get('HostConfig.ReadonlyRootfs'), read_only, container.get('HostConfig')) + def test_create_container_with_specified_volume(self): host_path = '/tmp/host-path' container_path = '/container-path' From 1579a125a3b50acb9d34b116078c133ace28fded Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Wed, 6 May 2015 09:33:22 +0200 Subject: [PATCH 057/118] Ensure that exglob is set in bash completion Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 548773d61..ec0f23481 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -94,7 +94,7 @@ _docker-compose_build() { _docker-compose_docker-compose() { case "$prev" in --file|-f) - _filedir y?(a)ml + _filedir "y?(a)ml" return ;; --project-name|-p) @@ -303,6 +303,9 @@ _docker-compose_up() { _docker-compose() { + local previous_extglob_setting=$(shopt -p extglob) + shopt -s extglob + local commands=( build help @@ -352,6 +355,7 @@ _docker-compose() { local completions_func=_docker-compose_${command} declare -F $completions_func >/dev/null && $completions_func + eval "$previous_extglob_setting" return 0 } From f626fc5ce8f25f82b47d503c781855d9481e2b19 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Wed, 6 May 2015 13:18:58 +0200 Subject: [PATCH 058/118] Add support for log-driver in docker-compose.yml Closes #1303 Signed-off-by: Vincent Demeester --- compose/config.py | 1 + compose/container.py | 4 ++++ compose/service.py | 5 ++++- docs/yml.md | 14 ++++++++++++++ tests/integration/service_test.py | 18 ++++++++++++++++++ 5 files changed, 41 insertions(+), 1 deletion(-) diff --git a/compose/config.py b/compose/config.py index d54215dd2..5f9ed22b3 100644 --- a/compose/config.py +++ b/compose/config.py @@ -23,6 +23,7 @@ DOCKER_CONFIG_KEYS = [ 'links', 'mem_limit', 'net', + 'log_driver', 'pid', 'ports', 'privileged', diff --git a/compose/container.py b/compose/container.py index 9439a7087..fc3370d9e 100644 --- a/compose/container.py +++ b/compose/container.py @@ -83,6 +83,10 @@ class Container(object): def labels(self): return self.get('Config.Labels') or {} + @property + def log_config(self): + return self.get('HostConfig.LogConfig') or None + @property def human_readable_state(self): if self.is_running: diff --git a/compose/service.py b/compose/service.py index c66344833..ed87bc7ed 100644 --- a/compose/service.py +++ b/compose/service.py @@ -8,7 +8,7 @@ import sys import six from docker.errors import APIError -from docker.utils import create_host_config +from docker.utils import create_host_config, LogConfig from .config import DOCKER_CONFIG_KEYS from .container import Container, get_container_name @@ -25,6 +25,7 @@ DOCKER_START_KEYS = [ 'env_file', 'extra_hosts', 'net', + 'log_driver', 'pid', 'privileged', 'restart', @@ -429,6 +430,7 @@ class Service(object): privileged = options.get('privileged', False) cap_add = options.get('cap_add', None) cap_drop = options.get('cap_drop', None) + log_config = LogConfig(type=options.get('log_driver', 'json-file')) pid = options.get('pid', None) dns = options.get('dns', None) @@ -455,6 +457,7 @@ class Service(object): restart_policy=restart, cap_add=cap_add, cap_drop=cap_drop, + log_config=log_config, extra_hosts=extra_hosts, pid_mode=pid ) diff --git a/docs/yml.md b/docs/yml.md index d7196b40e..40b6c6053 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -271,6 +271,20 @@ labels: - "com.example.label-with-empty-value" ``` +### log driver + +Specify a logging driver for the service's containers, as with the ``--log-driver`` option for docker run ([documented here](http://docs.docker.com/reference/run/#logging-drivers-log-driver)). + +Allowed values are currently ``json-file``, ``syslog`` and ``none``. The list will change over time as more drivers are added to the Docker engine. + +The default value is json-file. + +``` +log_driver: "json-file" +log_driver: "syslog" +log_driver: "none" +``` + ### net Networking mode. Use the same values as the docker client `--net` parameter. diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index f7190df64..d25408bc5 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -649,3 +649,21 @@ class ServiceTest(DockerClientTestCase): labels = create_and_start_container(service).labels.items() for name in labels_list: self.assertIn((name, ''), labels) + + def test_log_drive_invalid(self): + service = self.create_service('web', log_driver='xxx') + self.assertRaises(ValueError, lambda: create_and_start_container(service)) + + def test_log_drive_empty_default_jsonfile(self): + service = self.create_service('web') + log_config = create_and_start_container(service).log_config + + self.assertEqual('json-file', log_config['Type']) + self.assertFalse(log_config['Config']) + + def test_log_drive_none(self): + service = self.create_service('web', log_driver='none') + log_config = create_and_start_container(service).log_config + + self.assertEqual('none', log_config['Type']) + self.assertFalse(log_config['Config']) From d6223371d68b1a88dda67ab47cfca902fcbb58d8 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 7 May 2015 03:22:11 -0700 Subject: [PATCH 059/118] Fix markdown formatting issue Signed-off-by: Harald Albers --- docs/completion.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/completion.md b/docs/completion.md index 6ac95c2ef..96b5e8742 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -3,14 +3,12 @@ layout: default title: Command Completion --- -Command Completion -================== +#Command Completion Compose comes with [command completion](http://en.wikipedia.org/wiki/Command-line_completion) for the bash shell. -Installing Command Completion ------------------------------ +##Installing Command Completion Make sure bash completion is installed. If you use a current Linux in a non-minimal installation, bash completion should be available. On a Mac, install with `brew install bash-completion` @@ -21,8 +19,8 @@ Place the completion script in `/etc/bash_completion.d/` (`/usr/local/etc/bash_c Completion will be available upon next login. -Available completions ---------------------- +##Available completions + Depending on what you typed on the command line so far, it will complete - available docker-compose commands From 6829efd4d3200ab67bef6bfb959f089f2dafcb45 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 24 Apr 2015 14:05:17 -0400 Subject: [PATCH 060/118] Resolves #874, Rename instead of use an intermediate. Signed-off-by: Daniel Nephin --- compose/project.py | 18 ++++---- compose/service.py | 74 ++++++++++++++----------------- tests/integration/service_test.py | 11 ++--- tests/unit/service_test.py | 16 ++++++- 4 files changed, 60 insertions(+), 59 deletions(-) diff --git a/compose/project.py b/compose/project.py index 7c0d19da3..2f3675ffb 100644 --- a/compose/project.py +++ b/compose/project.py @@ -202,17 +202,15 @@ class Project(object): running_containers = [] for service in self.get_services(service_names, include_deps=start_deps): if recreate: - for (_, container) in service.recreate_containers( - insecure_registry=insecure_registry, - detach=detach, - do_build=do_build): - running_containers.append(container) + create_func = service.recreate_containers else: - for container in service.start_or_create_containers( - insecure_registry=insecure_registry, - detach=detach, - do_build=do_build): - running_containers.append(container) + create_func = service.start_or_create_containers + + for container in create_func( + insecure_registry=insecure_registry, + detach=detach, + do_build=do_build): + running_containers.append(container) return running_containers diff --git a/compose/service.py b/compose/service.py index ee47142f2..a1c0f9258 100644 --- a/compose/service.py +++ b/compose/service.py @@ -30,6 +30,7 @@ DOCKER_START_KEYS = [ 'pid', 'privileged', 'restart', + 'volumes_from', ] VALID_NAME_CHARS = '[a-zA-Z0-9]' @@ -175,16 +176,16 @@ class Service(object): one_off=False, insecure_registry=False, do_build=True, - intermediate_container=None, + previous_container=None, **override_options): """ Create a container for this service. If the image doesn't exist, attempt to pull it. """ + override_options['volumes_from'] = self._get_volumes_from(previous_container) container_options = self._get_container_create_options( override_options, one_off=one_off, - intermediate_container=intermediate_container, ) if (do_build and @@ -213,21 +214,24 @@ class Service(object): do_build=do_build, **override_options) self.start_container(container) - return [(None, container)] - else: - tuples = [] + return [container] - for c in containers: - log.info("Recreating %s..." % c.name) - tuples.append(self.recreate_container(c, insecure_registry=insecure_registry, **override_options)) - - return tuples + return [ + self.recreate_container( + c, + insecure_registry=insecure_registry, + **override_options) + for c in containers + ] def recreate_container(self, container, **override_options): - """Recreate a container. An intermediate container is created so that - the new container has the same name, while still supporting - `volumes-from` the original container. + """Recreate a container. + + The original container is renamed to a temporary name so that data + volumes can be copied to the new container, before the original + container is removed. """ + log.info("Recreating %s..." % container.name) try: container.stop() except APIError as e: @@ -238,29 +242,17 @@ class Service(object): else: raise - intermediate_container = Container.create( - self.client, - image=container.image, - entrypoint=['/bin/echo'], - command=[], - detach=True, - host_config=create_host_config(volumes_from=[container.id]), - ) - intermediate_container.start() - intermediate_container.wait() - container.remove() - - options = dict(override_options) + # Use a hopefully unique container name by prepending the short id + self.client.rename( + container.id, + '%s_%s' % (container.short_id, container.name)) new_container = self.create_container( do_build=False, - intermediate_container=intermediate_container, - **options - ) + previous_container=container, + **override_options) self.start_container(new_container) - - intermediate_container.remove() - - return (intermediate_container, new_container) + container.remove() + return new_container def start_container_if_stopped(self, container): if container.is_running: @@ -333,7 +325,7 @@ class Service(object): links.append((external_link, link_name)) return links - def _get_volumes_from(self, intermediate_container=None): + def _get_volumes_from(self, previous_container=None): volumes_from = [] for volume_source in self.volumes_from: if isinstance(volume_source, Service): @@ -346,8 +338,8 @@ class Service(object): elif isinstance(volume_source, Container): volumes_from.append(volume_source.id) - if intermediate_container: - volumes_from.append(intermediate_container.id) + if previous_container: + volumes_from.append(previous_container.id) return volumes_from @@ -370,7 +362,7 @@ class Service(object): return net - def _get_container_create_options(self, override_options, one_off=False, intermediate_container=None): + def _get_container_create_options(self, override_options, one_off=False): container_options = dict( (k, self.options[k]) for k in DOCKER_CONFIG_KEYS if k in self.options) @@ -415,11 +407,13 @@ class Service(object): for key in DOCKER_START_KEYS: container_options.pop(key, None) - container_options['host_config'] = self._get_container_host_config(override_options, one_off=one_off, intermediate_container=intermediate_container) + container_options['host_config'] = self._get_container_host_config( + override_options, + one_off=one_off) return container_options - def _get_container_host_config(self, override_options, one_off=False, intermediate_container=None): + def _get_container_host_config(self, override_options, one_off=False): options = dict(self.options, **override_options) port_bindings = build_port_bindings(options.get('ports') or []) @@ -451,7 +445,7 @@ class Service(object): links=self._get_links(link_to_self=one_off), port_bindings=port_bindings, binds=volume_bindings, - volumes_from=self._get_volumes_from(intermediate_container), + volumes_from=options.get('volumes_from'), privileged=privileged, network_mode=self._get_net(), dns=dns, diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 678aacdd0..dbb4a609c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -249,25 +249,20 @@ class ServiceTest(DockerClientTestCase): num_containers_before = len(self.client.containers(all=True)) service.options['environment']['FOO'] = '2' - tuples = service.recreate_containers() - self.assertEqual(len(tuples), 1) - - intermediate_container = tuples[0][0] - new_container = tuples[0][1] - self.assertEqual(intermediate_container.dictionary['Config']['Entrypoint'], ['/bin/echo']) + new_container, = service.recreate_containers() self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['sleep']) self.assertEqual(new_container.dictionary['Config']['Cmd'], ['300']) self.assertIn('FOO=2', new_container.dictionary['Config']['Env']) self.assertEqual(new_container.name, 'composetest_db_1') self.assertEqual(new_container.inspect()['Volumes']['/etc'], volume_path) - self.assertIn(intermediate_container.id, new_container.dictionary['HostConfig']['VolumesFrom']) + self.assertIn(old_container.id, new_container.dictionary['HostConfig']['VolumesFrom']) self.assertEqual(len(self.client.containers(all=True)), num_containers_before) self.assertNotEqual(old_container.id, new_container.id) self.assertRaises(APIError, self.client.inspect_container, - intermediate_container.id) + old_container.id) def test_recreate_containers_when_containers_are_stopped(self): service = self.create_service( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index b9f968db1..583f72ef0 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -86,7 +86,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual(service._get_volumes_from(), [container_id]) - def test_get_volumes_from_intermediate_container(self): + def test_get_volumes_from_previous_container(self): container_id = 'aabbccddee' service = Service('test', image='foo') container = mock.Mock(id=container_id, spec=Container, image='foo') @@ -263,6 +263,20 @@ class ServiceTest(unittest.TestCase): mock_log.info.assert_called_once_with( 'Pulling foo (someimage:sometag)...') + @mock.patch('compose.service.Container', autospec=True) + def test_recreate_container(self, _): + mock_container = mock.create_autospec(Container) + service = Service('foo', client=self.mock_client, image='someimage') + new_container = service.recreate_container(mock_container) + + mock_container.stop.assert_called_once_with() + self.mock_client.rename.assert_called_once_with( + mock_container.id, + '%s_%s' % (mock_container.short_id, mock_container.name)) + + new_container.start.assert_called_once_with() + mock_container.remove.assert_called_once_with() + def test_parse_repository_tag(self): self.assertEqual(parse_repository_tag("root"), ("root", "")) self.assertEqual(parse_repository_tag("root:tag"), ("root", "tag")) From df87bd91c86ba089667c771a5bd8e0ad5957de39 Mon Sep 17 00:00:00 2001 From: "delbert@umn.edu" Date: Fri, 8 May 2015 18:14:32 -0500 Subject: [PATCH 061/118] Added devices configuration option Signed-off-by: Dan Elbert --- compose/config.py | 41 +++++++++++++++------------- compose/service.py | 4 +++ docs/extends.md | 6 ++--- docs/yml.md | 14 ++++++++-- tests/integration/service_test.py | 13 +++++++++ tests/unit/config_test.py | 45 ++++++++++++++++++++----------- 6 files changed, 85 insertions(+), 38 deletions(-) diff --git a/compose/config.py b/compose/config.py index 3241bb80e..1919ef5a3 100644 --- a/compose/config.py +++ b/compose/config.py @@ -10,6 +10,7 @@ DOCKER_CONFIG_KEYS = [ 'cpuset', 'command', 'detach', + 'devices', 'dns', 'dns_search', 'domainname', @@ -50,6 +51,7 @@ DOCKER_CONFIG_HINTS = { 'add_host': 'extra_hosts', 'hosts': 'extra_hosts', 'extra_host': 'extra_hosts', + 'device': 'devices', 'link': 'links', 'port': 'ports', 'privilege': 'privileged', @@ -200,11 +202,14 @@ def merge_service_dicts(base, override): override.get('environment'), ) - if 'volumes' in base or 'volumes' in override: - d['volumes'] = merge_volumes( - base.get('volumes'), - override.get('volumes'), - ) + path_mapping_keys = ['volumes', 'devices'] + + for key in path_mapping_keys: + if key in base or key in override: + d[key] = merge_path_mappings( + base.get(key), + override.get(key), + ) if 'labels' in base or 'labels' in override: d['labels'] = merge_labels( @@ -230,7 +235,7 @@ def merge_service_dicts(base, override): if key in base or key in override: d[key] = to_list(base.get(key)) + to_list(override.get(key)) - already_merged_keys = ['environment', 'volumes', 'labels'] + list_keys + list_or_string_keys + already_merged_keys = ['environment', 'labels'] + path_mapping_keys + list_keys + list_or_string_keys for k in set(ALLOWED_KEYS) - set(already_merged_keys): if k in override: @@ -346,7 +351,7 @@ def resolve_host_paths(volumes, working_dir=None): def resolve_host_path(volume, working_dir): - container_path, host_path = split_volume(volume) + container_path, host_path = split_path_mapping(volume) if host_path is not None: host_path = os.path.expanduser(host_path) host_path = os.path.expandvars(host_path) @@ -368,24 +373,24 @@ def validate_paths(service_dict): raise ConfigurationError("build path %s either does not exist or is not accessible." % build_path) -def merge_volumes(base, override): - d = dict_from_volumes(base) - d.update(dict_from_volumes(override)) - return volumes_from_dict(d) +def merge_path_mappings(base, override): + d = dict_from_path_mappings(base) + d.update(dict_from_path_mappings(override)) + return path_mappings_from_dict(d) -def dict_from_volumes(volumes): - if volumes: - return dict(split_volume(v) for v in volumes) +def dict_from_path_mappings(path_mappings): + if path_mappings: + return dict(split_path_mapping(v) for v in path_mappings) else: return {} -def volumes_from_dict(d): - return [join_volume(v) for v in d.items()] +def path_mappings_from_dict(d): + return [join_path_mapping(v) for v in d.items()] -def split_volume(string): +def split_path_mapping(string): if ':' in string: (host, container) = string.split(':', 1) return (container, host) @@ -393,7 +398,7 @@ def split_volume(string): return (string, None) -def join_volume(pair): +def join_path_mapping(pair): (container, host) = pair if host is None: return container diff --git a/compose/service.py b/compose/service.py index a1c0f9258..20f8db0a4 100644 --- a/compose/service.py +++ b/compose/service.py @@ -20,6 +20,7 @@ log = logging.getLogger(__name__) DOCKER_START_KEYS = [ 'cap_add', 'cap_drop', + 'devices', 'dns', 'dns_search', 'env_file', @@ -441,6 +442,8 @@ class Service(object): extra_hosts = build_extra_hosts(options.get('extra_hosts', None)) read_only = options.get('read_only', None) + devices = options.get('devices', None) + return create_host_config( links=self._get_links(link_to_self=one_off), port_bindings=port_bindings, @@ -448,6 +451,7 @@ class Service(object): volumes_from=options.get('volumes_from'), privileged=privileged, network_mode=self._get_net(), + devices=devices, dns=dns, dns_search=dns_search, restart_policy=restart, diff --git a/docs/extends.md b/docs/extends.md index 06c08f25e..a4768b8f5 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -342,8 +342,8 @@ environment: - BAZ=local ``` -Finally, for `volumes`, Compose "merges" entries together with locally-defined -bindings taking precedence: +Finally, for `volumes` and `devices`, Compose "merges" entries together with +locally-defined bindings taking precedence: ```yaml # original service @@ -361,4 +361,4 @@ volumes: - /original-dir/foo:/foo - /local-dir/bar:/bar - /local-dir/baz/:baz -``` \ No newline at end of file +``` diff --git a/docs/yml.md b/docs/yml.md index 96a478bb2..0b8d4313b 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -29,8 +29,8 @@ image: a4bc65fd ### build -Path to a directory containing a Dockerfile. When the value supplied is a -relative path, it is interpreted as relative to the location of the yml file +Path to a directory containing a Dockerfile. When the value supplied is a +relative path, it is interpreted as relative to the location of the yml file itself. This directory is also the build context that is sent to the Docker daemon. Compose will build and tag it with a generated name, and use that image thereafter. @@ -342,6 +342,16 @@ dns_search: - dc2.example.com ``` +### devices + +List of device mappings. Uses the same format as the `--device` docker +client create option. + +``` +devices: + - "/dev/ttyUSB0:/dev/ttyUSB0" +``` + ### working\_dir, entrypoint, user, hostname, domainname, mem\_limit, privileged, restart, stdin\_open, tty, cpu\_shares, cpuset, read\_only Each of these is a single value, analogous to its diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index dbb4a609c..08e92a57f 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -669,3 +669,16 @@ class ServiceTest(DockerClientTestCase): self.assertEqual('none', log_config['Type']) self.assertFalse(log_config['Config']) + + def test_devices(self): + service = self.create_service('web', devices=["/dev/random:/dev/mapped-random"]) + device_config = create_and_start_container(service).get('HostConfig.Devices') + + device_dict = { + 'PathOnHost': '/dev/random', + 'CgroupPermissions': 'rwm', + 'PathInContainer': '/dev/mapped-random' + } + + self.assertEqual(1, len(device_config)) + self.assertDictEqual(device_dict, device_config[0]) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index fcd417b06..0a48dfefe 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -54,46 +54,61 @@ class VolumePathTest(unittest.TestCase): self.assertEqual(d['volumes'], ['/home/user:/container/path']) -class MergeVolumesTest(unittest.TestCase): +class MergePathMappingTest(object): + def config_name(self): + return "" + def test_empty(self): service_dict = config.merge_service_dicts({}, {}) - self.assertNotIn('volumes', service_dict) + self.assertNotIn(self.config_name(), service_dict) def test_no_override(self): service_dict = config.merge_service_dicts( - {'volumes': ['/foo:/code', '/data']}, + {self.config_name(): ['/foo:/code', '/data']}, {}, ) - self.assertEqual(set(service_dict['volumes']), set(['/foo:/code', '/data'])) + self.assertEqual(set(service_dict[self.config_name()]), set(['/foo:/code', '/data'])) def test_no_base(self): service_dict = config.merge_service_dicts( {}, - {'volumes': ['/bar:/code']}, + {self.config_name(): ['/bar:/code']}, ) - self.assertEqual(set(service_dict['volumes']), set(['/bar:/code'])) + self.assertEqual(set(service_dict[self.config_name()]), set(['/bar:/code'])) def test_override_explicit_path(self): service_dict = config.merge_service_dicts( - {'volumes': ['/foo:/code', '/data']}, - {'volumes': ['/bar:/code']}, + {self.config_name(): ['/foo:/code', '/data']}, + {self.config_name(): ['/bar:/code']}, ) - self.assertEqual(set(service_dict['volumes']), set(['/bar:/code', '/data'])) + self.assertEqual(set(service_dict[self.config_name()]), set(['/bar:/code', '/data'])) def test_add_explicit_path(self): service_dict = config.merge_service_dicts( - {'volumes': ['/foo:/code', '/data']}, - {'volumes': ['/bar:/code', '/quux:/data']}, + {self.config_name(): ['/foo:/code', '/data']}, + {self.config_name(): ['/bar:/code', '/quux:/data']}, ) - self.assertEqual(set(service_dict['volumes']), set(['/bar:/code', '/quux:/data'])) + self.assertEqual(set(service_dict[self.config_name()]), set(['/bar:/code', '/quux:/data'])) def test_remove_explicit_path(self): service_dict = config.merge_service_dicts( - {'volumes': ['/foo:/code', '/quux:/data']}, - {'volumes': ['/bar:/code', '/data']}, + {self.config_name(): ['/foo:/code', '/quux:/data']}, + {self.config_name(): ['/bar:/code', '/data']}, ) - self.assertEqual(set(service_dict['volumes']), set(['/bar:/code', '/data'])) + self.assertEqual(set(service_dict[self.config_name()]), set(['/bar:/code', '/data'])) + +class MergeVolumesTest(unittest.TestCase, MergePathMappingTest): + def config_name(self): + return 'volumes' + + +class MergeDevicesTest(unittest.TestCase, MergePathMappingTest): + def config_name(self): + return 'devices' + + +class BuildOrImageMergeTest(unittest.TestCase): def test_merge_build_or_image_no_override(self): self.assertEqual( config.merge_service_dicts({'build': '.'}, {}), From 417d9c2d51ae305742330314b124573512b54f51 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sat, 9 May 2015 19:38:53 -0400 Subject: [PATCH 062/118] Use individual volumes for recreate instead of volumes_from Signed-off-by: Daniel Nephin --- compose/cli/main.py | 5 +- compose/container.py | 4 ++ compose/service.py | 86 +++++++++++++++++++++++++------ tests/integration/service_test.py | 27 +++++----- tests/unit/service_test.py | 70 ++++++++++++++++++++++--- 5 files changed, 152 insertions(+), 40 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index e941c005c..a2375516e 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -317,10 +317,7 @@ class TopLevelCommand(Command): } if options['-e']: - # Merge environment from config with -e command line - container_options['environment'] = dict( - parse_environment(service.options.get('environment')), - **parse_environment(options['-e'])) + container_options['environment'] = parse_environment(options['-e']) if options['--entrypoint']: container_options['entrypoint'] = options.get('--entrypoint') diff --git a/compose/container.py b/compose/container.py index fc3370d9e..6388ca80c 100644 --- a/compose/container.py +++ b/compose/container.py @@ -44,6 +44,10 @@ class Container(object): def image(self): return self.dictionary['Image'] + @property + def image_config(self): + return self.client.inspect_image(self.image) + @property def short_id(self): return self.id[:10] diff --git a/compose/service.py b/compose/service.py index 20f8db0a4..08d203274 100644 --- a/compose/service.py +++ b/compose/service.py @@ -3,14 +3,14 @@ from __future__ import absolute_import from collections import namedtuple import logging import re -from operator import attrgetter import sys -import six +from operator import attrgetter +import six from docker.errors import APIError from docker.utils import create_host_config, LogConfig -from .config import DOCKER_CONFIG_KEYS +from .config import DOCKER_CONFIG_KEYS, merge_environment from .container import Container, get_container_name from .progress_stream import stream_output, StreamOutputError @@ -183,10 +183,10 @@ class Service(object): Create a container for this service. If the image doesn't exist, attempt to pull it. """ - override_options['volumes_from'] = self._get_volumes_from(previous_container) container_options = self._get_container_create_options( override_options, one_off=one_off, + previous_container=previous_container, ) if (do_build and @@ -247,6 +247,12 @@ class Service(object): self.client.rename( container.id, '%s_%s' % (container.short_id, container.name)) + + override_options = dict( + override_options, + environment=merge_environment( + override_options.get('environment'), + {'affinity:container': '=' + container.id})) new_container = self.create_container( do_build=False, previous_container=container, @@ -326,7 +332,7 @@ class Service(object): links.append((external_link, link_name)) return links - def _get_volumes_from(self, previous_container=None): + def _get_volumes_from(self): volumes_from = [] for volume_source in self.volumes_from: if isinstance(volume_source, Service): @@ -339,9 +345,6 @@ class Service(object): elif isinstance(volume_source, Container): volumes_from.append(volume_source.id) - if previous_container: - volumes_from.append(previous_container.id) - return volumes_from def _get_net(self): @@ -363,7 +366,11 @@ class Service(object): return net - def _get_container_create_options(self, override_options, one_off=False): + def _get_container_create_options( + self, + override_options, + one_off=False, + previous_container=None): container_options = dict( (k, self.options[k]) for k in DOCKER_CONFIG_KEYS if k in self.options) @@ -396,11 +403,19 @@ class Service(object): ports.append(port) container_options['ports'] = ports + override_options['binds'] = merge_volume_bindings( + container_options.get('volumes') or [], + previous_container) + if 'volumes' in container_options: container_options['volumes'] = dict( (parse_volume_spec(v).internal, {}) for v in container_options['volumes']) + container_options['environment'] = merge_environment( + self.options.get('environment'), + override_options.get('environment')) + if self.can_be_built(): container_options['image'] = self.full_name @@ -418,11 +433,6 @@ class Service(object): options = dict(self.options, **override_options) port_bindings = build_port_bindings(options.get('ports') or []) - volume_bindings = dict( - build_volume_binding(parse_volume_spec(volume)) - for volume in options.get('volumes') or [] - if ':' in volume) - privileged = options.get('privileged', False) cap_add = options.get('cap_add', None) cap_drop = options.get('cap_drop', None) @@ -447,8 +457,8 @@ class Service(object): return create_host_config( links=self._get_links(link_to_self=one_off), port_bindings=port_bindings, - binds=volume_bindings, - volumes_from=options.get('volumes_from'), + binds=options.get('binds'), + volumes_from=self._get_volumes_from(), privileged=privileged, network_mode=self._get_net(), devices=devices, @@ -531,6 +541,50 @@ class Service(object): stream_output(output, sys.stdout) +def get_container_data_volumes(container, volumes_option): + """Find the container data volumes that are in `volumes_option`, and return + a mapping of volume bindings for those volumes. + """ + volumes = [] + + volumes_option = volumes_option or [] + container_volumes = container.get('Volumes') or {} + image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {} + + for volume in set(volumes_option + image_volumes.keys()): + volume = parse_volume_spec(volume) + # No need to preserve host volumes + if volume.external: + continue + + volume_path = container_volumes.get(volume.internal) + # New volume, doesn't exist in the old container + if not volume_path: + continue + + # Copy existing volume from old container + volume = volume._replace(external=volume_path) + volumes.append(build_volume_binding(volume)) + + return dict(volumes) + + +def merge_volume_bindings(volumes_option, previous_container): + """Return a list of volume bindings for a container. Container data volumes + are replaced by those from the previous container. + """ + volume_bindings = dict( + build_volume_binding(parse_volume_spec(volume)) + for volume in volumes_option or [] + if ':' in volume) + + if previous_container: + volume_bindings.update( + get_container_data_volumes(previous_container, volumes_option)) + + return volume_bindings + + NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$') diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 08e92a57f..bc21ab018 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -107,7 +107,7 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('db', volumes=['/var/db']) container = service.create_container() service.start_container(container) - self.assertIn('/var/db', container.inspect()['Volumes']) + self.assertIn('/var/db', container.get('Volumes')) def test_create_container_with_cpu_shares(self): service = self.create_service('db', cpu_shares=73) @@ -239,24 +239,27 @@ class ServiceTest(DockerClientTestCase): command=['300'] ) old_container = service.create_container() - self.assertEqual(old_container.dictionary['Config']['Entrypoint'], ['sleep']) - self.assertEqual(old_container.dictionary['Config']['Cmd'], ['300']) - self.assertIn('FOO=1', old_container.dictionary['Config']['Env']) + self.assertEqual(old_container.get('Config.Entrypoint'), ['sleep']) + self.assertEqual(old_container.get('Config.Cmd'), ['300']) + self.assertIn('FOO=1', old_container.get('Config.Env')) self.assertEqual(old_container.name, 'composetest_db_1') service.start_container(old_container) - volume_path = old_container.inspect()['Volumes']['/etc'] + old_container.inspect() # reload volume data + volume_path = old_container.get('Volumes')['/etc'] num_containers_before = len(self.client.containers(all=True)) service.options['environment']['FOO'] = '2' new_container, = service.recreate_containers() - self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['sleep']) - self.assertEqual(new_container.dictionary['Config']['Cmd'], ['300']) - self.assertIn('FOO=2', new_container.dictionary['Config']['Env']) + self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep']) + self.assertEqual(new_container.get('Config.Cmd'), ['300']) + self.assertIn('FOO=2', new_container.get('Config.Env')) self.assertEqual(new_container.name, 'composetest_db_1') - self.assertEqual(new_container.inspect()['Volumes']['/etc'], volume_path) - self.assertIn(old_container.id, new_container.dictionary['HostConfig']['VolumesFrom']) + self.assertEqual(new_container.get('Volumes')['/etc'], volume_path) + self.assertIn( + 'affinity:container==%s' % old_container.id, + new_container.get('Config.Env')) self.assertEqual(len(self.client.containers(all=True)), num_containers_before) self.assertNotEqual(old_container.id, new_container.id) @@ -289,9 +292,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(old_container.get('Volumes').keys(), ['/data']) volume_path = old_container.get('Volumes')['/data'] - service.recreate_containers() - new_container = service.containers()[0] - service.start_container(new_container) + new_container = service.recreate_containers()[0] self.assertEqual(new_container.get('Volumes').keys(), ['/data']) self.assertEqual(new_container.get('Volumes')['/data'], volume_path) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 583f72ef0..2ea94edbd 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -14,7 +14,9 @@ from compose.service import ( ConfigError, build_port_bindings, build_volume_binding, + get_container_data_volumes, get_container_name, + merge_volume_bindings, parse_repository_tag, parse_volume_spec, split_port, @@ -86,13 +88,6 @@ class ServiceTest(unittest.TestCase): self.assertEqual(service._get_volumes_from(), [container_id]) - def test_get_volumes_from_previous_container(self): - container_id = 'aabbccddee' - service = Service('test', image='foo') - container = mock.Mock(id=container_id, spec=Container, image='foo') - - self.assertEqual(service._get_volumes_from(container), [container_id]) - def test_get_volumes_from_service_container_exists(self): container_ids = ['aabbccddee', '12345'] from_service = mock.create_autospec(Service) @@ -320,6 +315,9 @@ class ServiceTest(unittest.TestCase): class ServiceVolumesTest(unittest.TestCase): + def setUp(self): + self.mock_client = mock.create_autospec(docker.Client) + def test_parse_volume_spec_only_one_path(self): spec = parse_volume_spec('/the/volume') self.assertEqual(spec, (None, '/the/volume', 'rw')) @@ -345,3 +343,61 @@ class ServiceVolumesTest(unittest.TestCase): self.assertEqual( binding, ('/outside', dict(bind='/inside', ro=False))) + + def test_get_container_data_volumes(self): + options = [ + '/host/volume:/host/volume:ro', + '/new/volume', + '/existing/volume', + ] + + self.mock_client.inspect_image.return_value = { + 'ContainerConfig': { + 'Volumes': { + '/mnt/image/data': {}, + } + } + } + container = Container(self.mock_client, { + 'Image': 'ababab', + 'Volumes': { + '/host/volume': '/host/volume', + '/existing/volume': '/var/lib/docker/aaaaaaaa', + '/removed/volume': '/var/lib/docker/bbbbbbbb', + '/mnt/image/data': '/var/lib/docker/cccccccc', + }, + }, has_been_inspected=True) + + expected = { + '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False}, + '/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False}, + } + + binds = get_container_data_volumes(container, options) + self.assertEqual(binds, expected) + + def test_merge_volume_bindings(self): + options = [ + '/host/volume:/host/volume:ro', + '/host/rw/volume:/host/rw/volume', + '/new/volume', + '/existing/volume', + ] + + self.mock_client.inspect_image.return_value = { + 'ContainerConfig': {'Volumes': {}} + } + + intermediate_container = Container(self.mock_client, { + 'Image': 'ababab', + 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'}, + }, has_been_inspected=True) + + expected = { + '/host/volume': {'bind': '/host/volume', 'ro': True}, + '/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False}, + '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False}, + } + + binds = merge_volume_bindings(options, intermediate_container) + self.assertEqual(binds, expected) From 4d745ab87a1c1e3bfe9259907cbf29a025ced3e5 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 12 May 2015 12:44:43 +0100 Subject: [PATCH 063/118] Link to getting started guides from each page These are really hard to find. Signed-off-by: Ben Firshman --- docs/cli.md | 5 ++++- docs/completion.md | 5 ++++- docs/django.md | 5 ++++- docs/env.md | 5 ++++- docs/extends.md | 11 +++++++++++ docs/index.md | 3 +++ docs/install.md | 3 +++ docs/production.md | 12 ++++++++++++ docs/rails.md | 5 ++++- docs/wordpress.md | 5 ++++- docs/yml.md | 5 ++++- 11 files changed, 57 insertions(+), 7 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 62287f138..e5594871d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -183,8 +183,11 @@ Configures the path to the `ca.pem`, `cert.pem`, and `key.pem` files used for TL ## Compose documentation -- [Installing Compose](install.md) - [User guide](index.md) +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) diff --git a/docs/completion.md b/docs/completion.md index 96b5e8742..35c53b55f 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -32,8 +32,11 @@ Enjoy working with Compose faster and with less typos! ## Compose documentation -- [Installing Compose](install.md) - [User guide](index.md) +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) - [Command line reference](cli.md) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/django.md b/docs/django.md index 0605c86b6..4cbebe041 100644 --- a/docs/django.md +++ b/docs/django.md @@ -119,8 +119,11 @@ example, run `docker-compose up` and in another terminal run: ## More Compose documentation -- [Installing Compose](install.md) - [User guide](index.md) +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) - [Command line reference](cli.md) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/env.md b/docs/env.md index 3fc7b95aa..a4b543ae3 100644 --- a/docs/env.md +++ b/docs/env.md @@ -34,8 +34,11 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` ## Compose documentation -- [Installing Compose](install.md) - [User guide](index.md) +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) - [Command line reference](cli.md) - [Yaml file reference](yml.md) - [Compose command line completion](completion.md) diff --git a/docs/extends.md b/docs/extends.md index a4768b8f5..84fd1609f 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -362,3 +362,14 @@ volumes: - /local-dir/bar:/bar - /local-dir/baz/:baz ``` + +## Compose documentation + +- [User guide](index.md) +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) +- [Command line reference](cli.md) +- [Yaml file reference](yml.md) +- [Compose command line completion](completion.md) diff --git a/docs/index.md b/docs/index.md index b3190fca4..44f56ae96 100644 --- a/docs/index.md +++ b/docs/index.md @@ -48,6 +48,9 @@ Compose has commands for managing the whole lifecycle of your application: ## Compose documentation - [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) - [Command line reference](cli.md) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/install.md b/docs/install.md index a3524c603..a521ec06c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -39,6 +39,9 @@ You can test the installation by running `docker-compose --version`. ## Compose documentation - [User guide](index.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) - [Command line reference](cli.md) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/production.md b/docs/production.md index 8524c99b8..60a6873da 100644 --- a/docs/production.md +++ b/docs/production.md @@ -75,3 +75,15 @@ Compose against a Swarm instance and run your apps across multiple hosts. Compose/Swarm integration is still in the experimental stage, and Swarm is still in beta, but if you'd like to explore and experiment, check out the [integration guide](https://github.com/docker/compose/blob/master/SWARM.md). + +## Compose documentation + +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) +- [Command line reference](cli.md) +- [Yaml file reference](yml.md) +- [Compose environment variables](env.md) +- [Compose command line completion](completion.md) + diff --git a/docs/rails.md b/docs/rails.md index 0671d0624..aedb4c6e7 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -119,8 +119,11 @@ you're using Boot2docker, `boot2docker ip` will tell you its address). ## More Compose documentation -- [Installing Compose](install.md) - [User guide](index.md) +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) - [Command line reference](cli.md) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index 5a9c37a8d..b40d1a9f0 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -114,8 +114,11 @@ address). ## More Compose documentation -- [Installing Compose](install.md) - [User guide](index.md) +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) - [Command line reference](cli.md) - [Yaml file reference](yml.md) - [Compose environment variables](env.md) diff --git a/docs/yml.md b/docs/yml.md index 0b8d4313b..41247c703 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -380,8 +380,11 @@ read_only: true ## Compose documentation -- [Installing Compose](install.md) - [User guide](index.md) +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with Wordpress](wordpress.md) - [Command line reference](cli.md) - [Compose environment variables](env.md) - [Compose command line completion](completion.md) From 1dccd58209a61b4001a7034c5e0b7472fc6878f9 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 May 2015 18:51:45 +0100 Subject: [PATCH 064/118] Update docker-py to 1.2.2 Signed-off-by: Aanand Prasad --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ed09cccac..43907e1dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.10 -docker-py==1.2.1 +docker-py==1.2.2 dockerpty==0.3.2 docopt==0.6.1 requests==2.6.1 diff --git a/setup.py b/setup.py index 46193eeef..5668cf138 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.7', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', - 'docker-py >= 1.2.0, < 1.3', + 'docker-py >= 1.2.2, < 1.3', 'dockerpty >= 0.3.2, < 0.4', 'six >= 1.3.0, < 2', ] From ad9c5ad938254aa742900b3e80515fa7ebb52ae1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 14 May 2015 10:48:35 +0100 Subject: [PATCH 065/118] Fix typo in extends.md Signed-off-by: Aanand Prasad --- docs/extends.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/extends.md b/docs/extends.md index 84fd1609f..fd372ce2d 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -145,8 +145,7 @@ Defining the web application requires the following: FROM python:2.7 ADD . /code WORKDIR /code - RUN pip install -r - requirements.txt + RUN pip install -r requirements.txt CMD python app.py 4. Create a Compose configuration file called `common.yml`: From 9bbf1a33d103bddbadbfe174cd4095b758e3d18a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 14 May 2015 19:59:52 +0100 Subject: [PATCH 066/118] Update dockerpty to 0.3.3 Signed-off-by: Aanand Prasad --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 43907e1dc..b93988480 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PyYAML==3.10 docker-py==1.2.2 -dockerpty==0.3.2 +dockerpty==0.3.3 docopt==0.6.1 requests==2.6.1 six==1.7.3 diff --git a/setup.py b/setup.py index 5668cf138..153275f69 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ install_requires = [ 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', 'docker-py >= 1.2.2, < 1.3', - 'dockerpty >= 0.3.2, < 0.4', + 'dockerpty >= 0.3.3, < 0.4', 'six >= 1.3.0, < 2', ] From 862971cffa9db04f597acf0cd01e55b66681579e Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 15 May 2015 12:16:24 +0100 Subject: [PATCH 067/118] Fix race condition in `docker-compose run` We shouldn't start the container before handing it off to dockerpty - dockerpty will start it after attaching, which is the correct order. Otherwise the container might exit before we attach to it, which can lead to weird bugs. Signed-off-by: Aanand Prasad --- compose/cli/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index a2375516e..19e4e9378 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -341,7 +341,6 @@ class TopLevelCommand(Command): service.start_container(container) print(container.name) else: - service.start_container(container) dockerpty.start(project.client, container.id, interactive=not options['-T']) exit_code = container.wait() if options['--rm']: From 28d2aff8b8f65666d4659b9a25671228396dc7dc Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sun, 26 Apr 2015 21:21:55 -0400 Subject: [PATCH 068/118] Fix teardown for integration tests. Signed-off-by: Daniel Nephin --- tests/integration/cli_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index c7e2ea343..92789363e 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -21,6 +21,8 @@ class CLITestCase(DockerClientTestCase): sys.exit = self.old_sys_exit self.project.kill() self.project.remove_stopped() + for container in self.project.containers(stopped=True, one_off=True): + container.remove(force=True) @property def project(self): @@ -62,6 +64,10 @@ class CLITestCase(DockerClientTestCase): @patch('sys.stdout', new_callable=StringIO) def test_ps_alternate_composefile(self, mock_stdout): + config_path = os.path.abspath( + 'tests/fixtures/multiple-composefiles/compose2.yml') + self._project = self.command.get_project(config_path) + self.command.base_dir = 'tests/fixtures/multiple-composefiles' self.command.dispatch(['-f', 'compose2.yml', 'up', '-d'], None) self.command.dispatch(['-f', 'compose2.yml', 'ps'], None) @@ -416,7 +422,6 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(project.get_service('another').containers()), 0) def test_port(self): - self.command.base_dir = 'tests/fixtures/ports-composefile' self.command.dispatch(['up', '-d'], None) container = self.project.get_service('simple').get_container() From ed50a0a3a02821f5a6e50fc46db864e386258921 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sun, 26 Apr 2015 17:09:20 -0400 Subject: [PATCH 069/118] Resolves #1066, use labels to identify containers Signed-off-by: Daniel Nephin --- compose/__init__.py | 1 - compose/const.py | 6 ++ compose/container.py | 10 ++-- compose/project.py | 13 +++- compose/service.py | 98 +++++++++++++++++-------------- tests/integration/service_test.py | 33 +++++++---- tests/integration/testcases.py | 1 + tests/unit/container_test.py | 29 ++++++--- tests/unit/service_test.py | 49 +++++++--------- 9 files changed, 138 insertions(+), 102 deletions(-) create mode 100644 compose/const.py diff --git a/compose/__init__.py b/compose/__init__.py index 2de2a7f8b..045e79144 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,3 @@ from __future__ import unicode_literals -from .service import Service # noqa:flake8 __version__ = '1.3.0dev' diff --git a/compose/const.py b/compose/const.py new file mode 100644 index 000000000..0714a6dbf --- /dev/null +++ b/compose/const.py @@ -0,0 +1,6 @@ + +LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' +LABEL_ONE_OFF = 'com.docker.compose.oneoff' +LABEL_PROJECT = 'com.docker.compose.project' +LABEL_SERVICE = 'com.docker.compose.service' +LABEL_VERSION = 'com.docker.compose.version' diff --git a/compose/container.py b/compose/container.py index 6388ca80c..183d5fde8 100644 --- a/compose/container.py +++ b/compose/container.py @@ -4,6 +4,8 @@ from __future__ import absolute_import import six from functools import reduce +from .const import LABEL_CONTAINER_NUMBER, LABEL_SERVICE + class Container(object): """ @@ -58,14 +60,11 @@ class Container(object): @property def name_without_project(self): - return '_'.join(self.dictionary['Name'].split('_')[1:]) + return '{0}_{1}'.format(self.labels.get(LABEL_SERVICE), self.number) @property def number(self): - try: - return int(self.name.split('_')[-1]) - except ValueError: - return None + return int(self.labels.get(LABEL_CONTAINER_NUMBER) or 0) @property def ports(self): @@ -159,6 +158,7 @@ class Container(object): self.has_been_inspected = True return self.dictionary + # TODO: only used by tests, move to test module def links(self): links = [] for container in self.client.containers(): diff --git a/compose/project.py b/compose/project.py index 2f3675ffb..d22bdf4dc 100644 --- a/compose/project.py +++ b/compose/project.py @@ -4,6 +4,7 @@ import logging from functools import reduce from .config import get_service_name_from_net, ConfigurationError +from .const import LABEL_PROJECT, LABEL_ONE_OFF from .service import Service from .container import Container from docker.errors import APIError @@ -60,6 +61,12 @@ class Project(object): self.services = services self.client = client + def labels(self, one_off=False): + return [ + '{0}={1}'.format(LABEL_PROJECT, self.name), + '{0}={1}'.format(LABEL_ONE_OFF, "True" if one_off else "False"), + ] + @classmethod def from_dicts(cls, name, service_dicts, client): """ @@ -224,9 +231,9 @@ class Project(object): def containers(self, service_names=None, stopped=False, one_off=False): return [Container.from_ps(self.client, container) - for container in self.client.containers(all=stopped) - for service in self.get_services(service_names) - if service.has_container(container, one_off=one_off)] + for container in self.client.containers( + all=stopped, + filters={'label': self.labels(one_off=one_off)})] def _inject_deps(self, acc, service): net_name = service.get_net_name() diff --git a/compose/service.py b/compose/service.py index 08d203274..3c62dbebb 100644 --- a/compose/service.py +++ b/compose/service.py @@ -10,8 +10,16 @@ import six from docker.errors import APIError from docker.utils import create_host_config, LogConfig +from . import __version__ from .config import DOCKER_CONFIG_KEYS, merge_environment -from .container import Container, get_container_name +from .const import ( + LABEL_CONTAINER_NUMBER, + LABEL_ONE_OFF, + LABEL_PROJECT, + LABEL_SERVICE, + LABEL_VERSION, +) +from .container import Container from .progress_stream import stream_output, StreamOutputError log = logging.getLogger(__name__) @@ -79,27 +87,17 @@ class Service(object): def containers(self, stopped=False, one_off=False): return [Container.from_ps(self.client, container) - for container in self.client.containers(all=stopped) - if self.has_container(container, one_off=one_off)] - - def has_container(self, container, one_off=False): - """Return True if `container` was created to fulfill this service.""" - name = get_container_name(container) - if not name or not is_valid_name(name, one_off): - return False - project, name, _number = parse_name(name) - return project == self.project and name == self.name + for container in self.client.containers( + all=stopped, + filters={'label': self.labels(one_off=one_off)})] def get_container(self, number=1): """Return a :class:`compose.container.Container` for this service. The container must be active, and match `number`. """ - for container in self.client.containers(): - if not self.has_container(container): - continue - _, _, container_number = parse_name(get_container_name(container)) - if container_number == number: - return Container.from_ps(self.client, container) + labels = self.labels() + ['{0}={1}'.format(LABEL_CONTAINER_NUMBER, number)] + for container in self.client.containers(filters={'label': labels}): + return Container.from_ps(self.client, container) raise ValueError("No container found for %s_%s" % (self.name, number)) @@ -138,7 +136,6 @@ class Service(object): # Create enough containers containers = self.containers(stopped=True) while len(containers) < desired_num: - log.info("Creating %s..." % self._next_container_name(containers)) containers.append(self.create_container(detach=True)) running_containers = [] @@ -178,6 +175,7 @@ class Service(object): insecure_registry=False, do_build=True, previous_container=None, + number=None, **override_options): """ Create a container for this service. If the image doesn't exist, attempt to pull @@ -185,6 +183,7 @@ class Service(object): """ container_options = self._get_container_create_options( override_options, + number or self._next_container_number(one_off=one_off), one_off=one_off, previous_container=previous_container, ) @@ -209,7 +208,6 @@ class Service(object): """ containers = self.containers(stopped=True) if not containers: - log.info("Creating %s..." % self._next_container_name(containers)) container = self.create_container( insecure_registry=insecure_registry, do_build=do_build, @@ -256,6 +254,7 @@ class Service(object): new_container = self.create_container( do_build=False, previous_container=container, + number=container.labels.get(LABEL_CONTAINER_NUMBER), **override_options) self.start_container(new_container) container.remove() @@ -280,7 +279,6 @@ class Service(object): containers = self.containers(stopped=True) if not containers: - log.info("Creating %s..." % self._next_container_name(containers)) new_container = self.create_container( insecure_registry=insecure_registry, detach=detach, @@ -302,14 +300,19 @@ class Service(object): else: return - def _next_container_name(self, all_containers, one_off=False): - bits = [self.project, self.name] - if one_off: - bits.append('run') - return '_'.join(bits + [str(self._next_container_number(all_containers))]) + def get_container_name(self, number, one_off=False): + # TODO: Implement issue #652 here + return build_container_name(self.project, self.name, number, one_off) - def _next_container_number(self, all_containers): - numbers = [parse_name(c.name).number for c in all_containers] + # TODO: this would benefit from github.com/docker/docker/pull/11943 + # to remove the need to inspect every container + def _next_container_number(self, one_off=False): + numbers = [ + Container.from_ps(self.client, container).number + for container in self.client.containers( + all=True, + filters={'label': self.labels(one_off=one_off)}) + ] return 1 if not numbers else max(numbers) + 1 def _get_links(self, link_to_self): @@ -369,6 +372,7 @@ class Service(object): def _get_container_create_options( self, override_options, + number, one_off=False, previous_container=None): container_options = dict( @@ -376,9 +380,7 @@ class Service(object): for k in DOCKER_CONFIG_KEYS if k in self.options) container_options.update(override_options) - container_options['name'] = self._next_container_name( - self.containers(stopped=True, one_off=one_off), - one_off) + container_options['name'] = self.get_container_name(number, one_off) # If a qualified hostname was given, split it into an # unqualified hostname and a domainname unless domainname @@ -419,6 +421,11 @@ class Service(object): if self.can_be_built(): container_options['image'] = self.full_name + container_options['labels'] = build_container_labels( + container_options.get('labels', {}), + self.labels(one_off=one_off), + number) + # Delete options which are only used when starting for key in DOCKER_START_KEYS: container_options.pop(key, None) @@ -520,6 +527,13 @@ class Service(object): """ return '%s_%s' % (self.project, self.name) + def labels(self, one_off=False): + return [ + '{0}={1}'.format(LABEL_PROJECT, self.project), + '{0}={1}'.format(LABEL_SERVICE, self.name), + '{0}={1}'.format(LABEL_ONE_OFF, "True" if one_off else "False") + ] + def can_be_scaled(self): for port in self.options.get('ports', []): if ':' in str(port): @@ -585,23 +599,19 @@ def merge_volume_bindings(volumes_option, previous_container): return volume_bindings -NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$') - - -def is_valid_name(name, one_off=False): - match = NAME_RE.match(name) - if match is None: - return False +def build_container_name(project, service, number, one_off=False): + bits = [project, service] if one_off: - return match.group(3) == 'run_' - else: - return match.group(3) is None + bits.append('run') + return '_'.join(bits + [str(number)]) -def parse_name(name): - match = NAME_RE.match(name) - (project, service_name, _, suffix) = match.groups() - return ServiceName(project, service_name, int(suffix)) +def build_container_labels(label_options, service_labels, number, one_off=False): + labels = label_options or {} + labels.update(label.split('=', 1) for label in service_labels) + labels[LABEL_CONTAINER_NUMBER] = str(number) + labels[LABEL_VERSION] = __version__ + return labels def parse_restart_spec(restart_config): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index bc21ab018..47c826ec5 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -8,11 +8,19 @@ import tempfile import shutil import six -from compose import Service +from compose import __version__ +from compose.const import ( + LABEL_CONTAINER_NUMBER, + LABEL_ONE_OFF, + LABEL_PROJECT, + LABEL_SERVICE, + LABEL_VERSION, +) from compose.service import ( CannotBeScaledError, - build_extra_hosts, ConfigError, + Service, + build_extra_hosts, ) from compose.container import Container from docker.errors import APIError @@ -633,17 +641,18 @@ class ServiceTest(DockerClientTestCase): 'com.example.label-with-empty-value': "", } + compose_labels = { + LABEL_CONTAINER_NUMBER: '1', + LABEL_ONE_OFF: 'False', + LABEL_PROJECT: 'composetest', + LABEL_SERVICE: 'web', + LABEL_VERSION: __version__, + } + expected = dict(labels_dict, **compose_labels) + service = self.create_service('web', labels=labels_dict) - labels = create_and_start_container(service).labels.items() - for pair in labels_dict.items(): - self.assertIn(pair, labels) - - labels_list = ["%s=%s" % pair for pair in labels_dict.items()] - - service = self.create_service('web', labels=labels_list) - labels = create_and_start_container(service).labels.items() - for pair in labels_dict.items(): - self.assertIn(pair, labels) + labels = create_and_start_container(service).labels + self.assertEqual(labels, expected) def test_empty_labels(self): labels_list = ['foo', 'bar'] diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 31281a1d7..4a0f7248a 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -12,6 +12,7 @@ class DockerClientTestCase(unittest.TestCase): def setUpClass(cls): cls.client = docker_client() + # TODO: update to use labels in #652 def setUp(self): for c in self.client.containers(all=True): if c['Names'] and 'composetest' in c['Names'][0]: diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 7637adf58..b04df6592 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -5,6 +5,7 @@ import mock import docker from compose.container import Container +from compose.container import get_container_name class ContainerTest(unittest.TestCase): @@ -23,6 +24,13 @@ class ContainerTest(unittest.TestCase): "NetworkSettings": { "Ports": {}, }, + "Config": { + "Labels": { + "com.docker.compose.project": "composetest", + "com.docker.compose.service": "web", + "com.docker.compose.container_number": 7, + }, + } } def test_from_ps(self): @@ -65,10 +73,8 @@ class ContainerTest(unittest.TestCase): }) def test_number(self): - container = Container.from_ps(None, - self.container_dict, - has_been_inspected=True) - self.assertEqual(container.number, 1) + container = Container(None, self.container_dict, has_been_inspected=True) + self.assertEqual(container.number, 7) def test_name(self): container = Container.from_ps(None, @@ -77,10 +83,8 @@ class ContainerTest(unittest.TestCase): self.assertEqual(container.name, "composetest_db_1") def test_name_without_project(self): - container = Container.from_ps(None, - self.container_dict, - has_been_inspected=True) - self.assertEqual(container.name_without_project, "db_1") + container = Container(None, self.container_dict, has_been_inspected=True) + self.assertEqual(container.name_without_project, "web_7") def test_inspect_if_not_inspected(self): mock_client = mock.create_autospec(docker.Client) @@ -130,3 +134,12 @@ class ContainerTest(unittest.TestCase): self.assertEqual(container.get('Status'), "Up 8 seconds") self.assertEqual(container.get('HostConfig.VolumesFrom'), ["volume_id"]) self.assertEqual(container.get('Foo.Bar.DoesNotExist'), None) + + +class GetContainerNameTestCase(unittest.TestCase): + + def test_get_container_name(self): + self.assertIsNone(get_container_name({})) + self.assertEqual(get_container_name({'Name': 'myproject_db_1'}), 'myproject_db_1') + self.assertEqual(get_container_name({'Names': ['/myproject_db_1', '/myproject_web_1/db']}), 'myproject_db_1') + self.assertEqual(get_container_name({'Names': ['/swarm-host-1/myproject_db_1', '/swarm-host-1/myproject_web_1/db']}), 'myproject_db_1') diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 2ea94edbd..fa252062c 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -7,15 +7,15 @@ import mock import docker from requests import Response -from compose import Service +from compose.service import Service from compose.container import Container +from compose.const import LABEL_SERVICE, LABEL_PROJECT, LABEL_ONE_OFF from compose.service import ( APIError, ConfigError, build_port_bindings, build_volume_binding, get_container_data_volumes, - get_container_name, merge_volume_bindings, parse_repository_tag, parse_volume_spec, @@ -48,36 +48,27 @@ class ServiceTest(unittest.TestCase): self.assertRaises(ConfigError, lambda: Service(name='foo', project='_', image='foo')) Service(name='foo', project='bar', image='foo') - def test_get_container_name(self): - self.assertIsNone(get_container_name({})) - self.assertEqual(get_container_name({'Name': 'myproject_db_1'}), 'myproject_db_1') - self.assertEqual(get_container_name({'Names': ['/myproject_db_1', '/myproject_web_1/db']}), 'myproject_db_1') - self.assertEqual(get_container_name({'Names': ['/swarm-host-1/myproject_db_1', '/swarm-host-1/myproject_web_1/db']}), 'myproject_db_1') - def test_containers(self): - service = Service('db', client=self.mock_client, image='foo', project='myproject') - + service = Service('db', self.mock_client, 'myproject', image='foo') self.mock_client.containers.return_value = [] self.assertEqual(service.containers(), []) + def test_containers_with_containers(self): self.mock_client.containers.return_value = [ - {'Image': 'busybox', 'Id': 'OUT_1', 'Names': ['/myproject', '/foo/bar']}, - {'Image': 'busybox', 'Id': 'OUT_2', 'Names': ['/myproject_db']}, - {'Image': 'busybox', 'Id': 'OUT_3', 'Names': ['/db_1']}, - {'Image': 'busybox', 'Id': 'IN_1', 'Names': ['/myproject_db_1', '/myproject_web_1/db']}, + dict(Name=str(i), Image='foo', Id=i) for i in range(3) ] - self.assertEqual([c.id for c in service.containers()], ['IN_1']) + service = Service('db', self.mock_client, 'myproject', image='foo') + self.assertEqual([c.id for c in service.containers()], range(3)) - def test_containers_prefixed(self): - service = Service('db', client=self.mock_client, image='foo', project='myproject') - - self.mock_client.containers.return_value = [ - {'Image': 'busybox', 'Id': 'OUT_1', 'Names': ['/swarm-host-1/myproject', '/swarm-host-1/foo/bar']}, - {'Image': 'busybox', 'Id': 'OUT_2', 'Names': ['/swarm-host-1/myproject_db']}, - {'Image': 'busybox', 'Id': 'OUT_3', 'Names': ['/swarm-host-1/db_1']}, - {'Image': 'busybox', 'Id': 'IN_1', 'Names': ['/swarm-host-1/myproject_db_1', '/swarm-host-1/myproject_web_1/db']}, + expected_labels = [ + '{0}=myproject'.format(LABEL_PROJECT), + '{0}=db'.format(LABEL_SERVICE), + '{0}=False'.format(LABEL_ONE_OFF), ] - self.assertEqual([c.id for c in service.containers()], ['IN_1']) + + self.mock_client.containers.assert_called_once_with( + all=False, + filters={'label': expected_labels}) def test_get_volumes_from_container(self): container_id = 'aabbccddee' @@ -156,7 +147,7 @@ class ServiceTest(unittest.TestCase): def test_split_domainname_none(self): service = Service('foo', image='foo', hostname='name', client=self.mock_client) self.mock_client.containers.return_value = [] - opts = service._get_container_create_options({'image': 'foo'}) + opts = service._get_container_create_options({'image': 'foo'}, 1) self.assertEqual(opts['hostname'], 'name', 'hostname') self.assertFalse('domainname' in opts, 'domainname') @@ -167,7 +158,7 @@ class ServiceTest(unittest.TestCase): image='foo', client=self.mock_client) self.mock_client.containers.return_value = [] - opts = service._get_container_create_options({'image': 'foo'}) + opts = service._get_container_create_options({'image': 'foo'}, 1) self.assertEqual(opts['hostname'], 'name', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') @@ -179,7 +170,7 @@ class ServiceTest(unittest.TestCase): domainname='domain.tld', client=self.mock_client) self.mock_client.containers.return_value = [] - opts = service._get_container_create_options({'image': 'foo'}) + opts = service._get_container_create_options({'image': 'foo'}, 1) self.assertEqual(opts['hostname'], 'name', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') @@ -191,7 +182,7 @@ class ServiceTest(unittest.TestCase): image='foo', client=self.mock_client) self.mock_client.containers.return_value = [] - opts = service._get_container_create_options({'image': 'foo'}) + opts = service._get_container_create_options({'image': 'foo'}, 1) self.assertEqual(opts['hostname'], 'name.sub', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') @@ -255,7 +246,7 @@ class ServiceTest(unittest.TestCase): tag='sometag', insecure_registry=True, stream=True) - mock_log.info.assert_called_once_with( + mock_log.info.assert_called_with( 'Pulling foo (someimage:sometag)...') @mock.patch('compose.service.Container', autospec=True) From 62059d55e60e01d395fb8a5e99cda364ce517b49 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sat, 9 May 2015 12:53:59 -0400 Subject: [PATCH 070/118] Add migration warning and option to migrate to labels. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 33 ++++++++++++--------- compose/container.py | 6 +++- compose/migration.py | 35 ++++++++++++++++++++++ compose/project.py | 32 +++++++++++++++----- compose/service.py | 46 +++++++++++++++++++++++++---- tests/integration/migration_test.py | 23 +++++++++++++++ tests/unit/container_test.py | 2 +- 7 files changed, 148 insertions(+), 29 deletions(-) create mode 100644 compose/migration.py create mode 100644 tests/integration/migration_test.py diff --git a/compose/cli/main.py b/compose/cli/main.py index a2375516e..0c5d5a7f6 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -11,6 +11,7 @@ from docker.errors import APIError import dockerpty from .. import __version__ +from .. import migration from ..project import NoSuchService, ConfigurationError from ..service import BuildError, CannotBeScaledError from ..config import parse_environment @@ -81,20 +82,21 @@ class TopLevelCommand(Command): -v, --version Print version and exit Commands: - build Build or rebuild services - help Get help on a command - kill Kill containers - logs View output from containers - port Print the public port for a port binding - ps List containers - pull Pulls service images - restart Restart services - rm Remove stopped containers - run Run a one-off command - scale Set number of containers for a service - start Start services - stop Stop services - up Create and start containers + build Build or rebuild services + help Get help on a command + kill Kill containers + logs View output from containers + port Print the public port for a port binding + ps List containers + pull Pulls service images + restart Restart services + rm Remove stopped containers + run Run a one-off command + scale Set number of containers for a service + start Start services + stop Stop services + up Create and start containers + migrate_to_labels Recreate containers to add labels """ def docopt_options(self): @@ -483,6 +485,9 @@ class TopLevelCommand(Command): params = {} if timeout is None else {'timeout': int(timeout)} project.stop(service_names=service_names, **params) + def migrate_to_labels(self, project, _options): + migration.migrate_project_to_labels(project) + def list_containers(containers): return ", ".join(c.name for c in containers) diff --git a/compose/container.py b/compose/container.py index 183d5fde8..3e462088f 100644 --- a/compose/container.py +++ b/compose/container.py @@ -64,7 +64,11 @@ class Container(object): @property def number(self): - return int(self.labels.get(LABEL_CONTAINER_NUMBER) or 0) + number = self.labels.get(LABEL_CONTAINER_NUMBER) + if not number: + raise ValueError("Container {0} does not have a {1} label".format( + self.short_id, LABEL_CONTAINER_NUMBER)) + return int(number) @property def ports(self): diff --git a/compose/migration.py b/compose/migration.py new file mode 100644 index 000000000..16b5dd167 --- /dev/null +++ b/compose/migration.py @@ -0,0 +1,35 @@ +import logging +import re + +from .container import get_container_name, Container + + +log = logging.getLogger(__name__) + + +# TODO: remove this section when migrate_project_to_labels is removed +NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$') + + +def is_valid_name(name): + match = NAME_RE.match(name) + return match is not None + + +def add_labels(project, container, name): + project_name, service_name, one_off, number = NAME_RE.match(name).groups() + if project_name != project.name or service_name not in project.service_names: + return + service = project.get_service(service_name) + service.recreate_container(container) + + +def migrate_project_to_labels(project): + log.info("Running migration to labels for project %s", project.name) + + client = project.client + for container in client.containers(all=True): + name = get_container_name(container) + if not is_valid_name(name): + continue + add_labels(project, Container.from_ps(client, container), name) diff --git a/compose/project.py b/compose/project.py index d22bdf4dc..8ca144813 100644 --- a/compose/project.py +++ b/compose/project.py @@ -1,13 +1,14 @@ from __future__ import unicode_literals from __future__ import absolute_import import logging - from functools import reduce + +from docker.errors import APIError + from .config import get_service_name_from_net, ConfigurationError from .const import LABEL_PROJECT, LABEL_ONE_OFF -from .service import Service +from .service import Service, check_for_legacy_containers from .container import Container -from docker.errors import APIError log = logging.getLogger(__name__) @@ -82,6 +83,10 @@ class Project(object): volumes_from=volumes_from, **service_dict)) return project + @property + def service_names(self): + return [service.name for service in self.services] + def get_service(self, name): """ Retrieve a service by name. Raises NoSuchService @@ -109,7 +114,7 @@ class Project(object): """ if service_names is None or len(service_names) == 0: return self.get_services( - service_names=[s.name for s in self.services], + service_names=self.service_names, include_deps=include_deps ) else: @@ -230,10 +235,21 @@ class Project(object): service.remove_stopped(**options) def containers(self, service_names=None, stopped=False, one_off=False): - return [Container.from_ps(self.client, container) - for container in self.client.containers( - all=stopped, - filters={'label': self.labels(one_off=one_off)})] + containers = [ + Container.from_ps(self.client, container) + for container in self.client.containers( + all=stopped, + filters={'label': self.labels(one_off=one_off)})] + + if not containers: + check_for_legacy_containers( + self.client, + self.name, + self.service_names, + stopped=stopped, + one_off=one_off) + + return containers def _inject_deps(self, acc, service): net_name = service.get_net_name() diff --git a/compose/service.py b/compose/service.py index 3c62dbebb..dc34a9bc2 100644 --- a/compose/service.py +++ b/compose/service.py @@ -19,7 +19,7 @@ from .const import ( LABEL_SERVICE, LABEL_VERSION, ) -from .container import Container +from .container import Container, get_container_name from .progress_stream import stream_output, StreamOutputError log = logging.getLogger(__name__) @@ -86,10 +86,21 @@ class Service(object): self.options = options def containers(self, stopped=False, one_off=False): - return [Container.from_ps(self.client, container) - for container in self.client.containers( - all=stopped, - filters={'label': self.labels(one_off=one_off)})] + containers = [ + Container.from_ps(self.client, container) + for container in self.client.containers( + all=stopped, + filters={'label': self.labels(one_off=one_off)})] + + if not containers: + check_for_legacy_containers( + self.client, + self.project, + [self.name], + stopped=stopped, + one_off=one_off) + + return containers def get_container(self, number=1): """Return a :class:`compose.container.Container` for this service. The @@ -614,6 +625,31 @@ def build_container_labels(label_options, service_labels, number, one_off=False) return labels +def check_for_legacy_containers( + client, + project, + services, + stopped=False, + one_off=False): + """Check if there are containers named using the old naming convention + and warn the user that those containers may need to be migrated to + using labels, so that compose can find them. + """ + for container in client.containers(all=stopped): + name = get_container_name(container) + for service in services: + prefix = '%s_%s_%s' % (project, service, 'run_' if one_off else '') + if not name.startswith(prefix): + continue + + log.warn( + "Compose found a found a container named %s without any " + "labels. As of compose 1.3.0 containers are identified with " + "labels instead of naming convention. If you'd like compose " + "to use this container, please run " + "`docker-compose --migrate-to-labels`" % (name,)) + + def parse_restart_spec(restart_config): if not restart_config: return None diff --git a/tests/integration/migration_test.py b/tests/integration/migration_test.py new file mode 100644 index 000000000..133d23148 --- /dev/null +++ b/tests/integration/migration_test.py @@ -0,0 +1,23 @@ +import mock + +from compose import service, migration +from compose.project import Project +from .testcases import DockerClientTestCase + + +class ProjectTest(DockerClientTestCase): + + def test_migration_to_labels(self): + web = self.create_service('web') + db = self.create_service('db') + project = Project('composetest', [web, db], self.client) + + self.client.create_container(name='composetest_web_1', **web.options) + self.client.create_container(name='composetest_db_1', **db.options) + + with mock.patch.object(service, 'log', autospec=True) as mock_log: + self.assertEqual(project.containers(stopped=True), []) + self.assertEqual(mock_log.warn.call_count, 2) + + migration.migrate_project_to_labels(project) + self.assertEqual(len(project.containers(stopped=True)), 2) diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index b04df6592..2313d4b8e 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -28,7 +28,7 @@ class ContainerTest(unittest.TestCase): "Labels": { "com.docker.compose.project": "composetest", "com.docker.compose.service": "web", - "com.docker.compose.container_number": 7, + "com.docker.compose.container-number": 7, }, } } From 3304c68891a16ad5d3972a95688f2c4ed426d9c6 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 May 2015 11:11:36 +0100 Subject: [PATCH 071/118] Only set AttachStdin/out/err for one-off containers If we're just streaming logs from `docker-compose up`, we don't need to set AttachStdin/out/err, and doing so results in containers with different configuration depending on whether `up` or `run` were invoked with `-d` or not. Signed-off-by: Aanand Prasad --- compose/cli/main.py | 2 -- compose/project.py | 2 -- compose/service.py | 7 ++++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 8dae737b8..9379c79e5 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -299,7 +299,6 @@ class TopLevelCommand(Command): start_deps=True, recreate=False, insecure_registry=insecure_registry, - detach=options['-d'] ) tty = True @@ -461,7 +460,6 @@ class TopLevelCommand(Command): start_deps=start_deps, recreate=recreate, insecure_registry=insecure_registry, - detach=detached, do_build=not options['--no-build'], ) diff --git a/compose/project.py b/compose/project.py index 8ca144813..41cd14e0d 100644 --- a/compose/project.py +++ b/compose/project.py @@ -209,7 +209,6 @@ class Project(object): start_deps=True, recreate=True, insecure_registry=False, - detach=False, do_build=True): running_containers = [] for service in self.get_services(service_names, include_deps=start_deps): @@ -220,7 +219,6 @@ class Project(object): for container in create_func( insecure_registry=insecure_registry, - detach=detach, do_build=do_build): running_containers.append(container) diff --git a/compose/service.py b/compose/service.py index dc34a9bc2..001d36b41 100644 --- a/compose/service.py +++ b/compose/service.py @@ -147,7 +147,7 @@ class Service(object): # Create enough containers containers = self.containers(stopped=True) while len(containers) < desired_num: - containers.append(self.create_container(detach=True)) + containers.append(self.create_container()) running_containers = [] stopped_containers = [] @@ -285,14 +285,12 @@ class Service(object): def start_or_create_containers( self, insecure_registry=False, - detach=False, do_build=True): containers = self.containers(stopped=True) if not containers: new_container = self.create_container( insecure_registry=insecure_registry, - detach=detach, do_build=do_build, ) return [self.start_container(new_container)] @@ -393,6 +391,9 @@ class Service(object): container_options['name'] = self.get_container_name(number, one_off) + if 'detach' not in container_options: + container_options['detach'] = True + # If a qualified hostname was given, split it into an # unqualified hostname and a domainname unless domainname # was also given explicitly. This matches the behavior of From 82bc7cd5ba84bcb6cc14ba9f8c8775dbfb8fa474 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 May 2015 14:35:44 +0100 Subject: [PATCH 072/118] Remove override_options arg from recreate_container(s) Signed-off-by: Aanand Prasad --- compose/service.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/compose/service.py b/compose/service.py index 001d36b41..306b81833 100644 --- a/compose/service.py +++ b/compose/service.py @@ -212,7 +212,7 @@ class Service(object): return Container.create(self.client, **container_options) raise - def recreate_containers(self, insecure_registry=False, do_build=True, **override_options): + def recreate_containers(self, insecure_registry=False, do_build=True): """ If a container for this service doesn't exist, create and start one. If there are any, stop them, create+start new ones, and remove the old containers. @@ -221,20 +221,16 @@ class Service(object): if not containers: container = self.create_container( insecure_registry=insecure_registry, - do_build=do_build, - **override_options) + do_build=do_build) self.start_container(container) return [container] return [ - self.recreate_container( - c, - insecure_registry=insecure_registry, - **override_options) + self.recreate_container(c, insecure_registry=insecure_registry) for c in containers ] - def recreate_container(self, container, **override_options): + def recreate_container(self, container, insecure_registry=False): """Recreate a container. The original container is renamed to a temporary name so that data @@ -257,16 +253,12 @@ class Service(object): container.id, '%s_%s' % (container.short_id, container.name)) - override_options = dict( - override_options, - environment=merge_environment( - override_options.get('environment'), - {'affinity:container': '=' + container.id})) new_container = self.create_container( + insecure_registry=insecure_registry, do_build=False, previous_container=container, number=container.labels.get(LABEL_CONTAINER_NUMBER), - **override_options) + ) self.start_container(new_container) container.remove() return new_container @@ -430,8 +422,10 @@ class Service(object): self.options.get('environment'), override_options.get('environment')) - if self.can_be_built(): - container_options['image'] = self.full_name + if previous_container: + container_options['environment']['affinity:container'] = ('=' + previous_container.id) + + container_options['image'] = self.image_name container_options['labels'] = build_container_labels( container_options.get('labels', {}), From ef4eb66723318af8aa189ce93fdd525a1f1d427d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 May 2015 11:11:36 +0100 Subject: [PATCH 073/118] Implement smart recreate behind an experimental CLI flag Signed-off-by: Aanand Prasad --- compose/cli/main.py | 15 +- compose/const.py | 1 + compose/container.py | 5 +- compose/project.py | 64 ++++++-- compose/service.py | 176 ++++++++++++++++++-- compose/state.py | 0 compose/utils.py | 9 + tests/integration/project_test.py | 7 +- tests/integration/service_test.py | 27 ++- tests/integration/state_test.py | 263 ++++++++++++++++++++++++++++++ tests/unit/service_test.py | 86 +++++----- 11 files changed, 563 insertions(+), 90 deletions(-) create mode 100644 compose/state.py create mode 100644 compose/utils.py create mode 100644 tests/integration/state_test.py diff --git a/compose/cli/main.py b/compose/cli/main.py index 9379c79e5..557dc6367 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -13,7 +13,7 @@ import dockerpty from .. import __version__ from .. import migration from ..project import NoSuchService, ConfigurationError -from ..service import BuildError, CannotBeScaledError +from ..service import BuildError, CannotBeScaledError, NeedsBuildError from ..config import parse_environment from .command import Command from .docopt_command import NoSuchCommand @@ -47,6 +47,9 @@ def main(): except BuildError as e: log.error("Service '%s' failed to build: %s" % (e.service.name, e.reason)) sys.exit(1) + except NeedsBuildError as e: + log.error("Service '%s' needs to be built, but --no-build was passed." % e.service.name) + sys.exit(1) def setup_logging(): @@ -297,7 +300,7 @@ class TopLevelCommand(Command): project.up( service_names=deps, start_deps=True, - recreate=False, + allow_recreate=False, insecure_registry=insecure_registry, ) @@ -440,6 +443,8 @@ class TopLevelCommand(Command): print new container names. --no-color Produce monochrome output. --no-deps Don't start linked services. + --x-smart-recreate Only recreate containers whose configuration or + image needs to be updated. (EXPERIMENTAL) --no-recreate If containers already exist, don't recreate them. --no-build Don't build an image, even if it's missing -t, --timeout TIMEOUT When attached, use this timeout in seconds @@ -452,13 +457,15 @@ class TopLevelCommand(Command): monochrome = options['--no-color'] start_deps = not options['--no-deps'] - recreate = not options['--no-recreate'] + allow_recreate = not options['--no-recreate'] + smart_recreate = options['--x-smart-recreate'] service_names = options['SERVICE'] project.up( service_names=service_names, start_deps=start_deps, - recreate=recreate, + allow_recreate=allow_recreate, + smart_recreate=smart_recreate, insecure_registry=insecure_registry, do_build=not options['--no-build'], ) diff --git a/compose/const.py b/compose/const.py index 0714a6dbf..f76fb572c 100644 --- a/compose/const.py +++ b/compose/const.py @@ -4,3 +4,4 @@ LABEL_ONE_OFF = 'com.docker.compose.oneoff' LABEL_PROJECT = 'com.docker.compose.project' LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' +LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' diff --git a/compose/container.py b/compose/container.py index 3e462088f..719514971 100644 --- a/compose/container.py +++ b/compose/container.py @@ -179,13 +179,16 @@ class Container(object): return self.client.attach_socket(self.id, **kwargs) def __repr__(self): - return '' % self.name + return '' % (self.name, self.id[:6]) def __eq__(self, other): if type(self) != type(other): return False return self.id == other.id + def __hash__(self): + return self.id.__hash__() + def get_container_name(container): if not container.get('Name') and not container.get('Names'): diff --git a/compose/project.py b/compose/project.py index 41cd14e0d..c37175ae0 100644 --- a/compose/project.py +++ b/compose/project.py @@ -207,22 +207,59 @@ class Project(object): def up(self, service_names=None, start_deps=True, - recreate=True, + allow_recreate=True, + smart_recreate=False, insecure_registry=False, do_build=True): - running_containers = [] - for service in self.get_services(service_names, include_deps=start_deps): - if recreate: - create_func = service.recreate_containers + + services = self.get_services(service_names, include_deps=start_deps) + + plans = self._get_convergence_plans( + services, + allow_recreate=allow_recreate, + smart_recreate=smart_recreate, + ) + + return [ + container + for service in services + for container in service.execute_convergence_plan( + plans[service.name], + insecure_registry=insecure_registry, + do_build=do_build, + ) + ] + + def _get_convergence_plans(self, + services, + allow_recreate=True, + smart_recreate=False): + + plans = {} + + for service in services: + updated_dependencies = [ + name + for name in service.get_dependency_names() + if name in plans + and plans[name].action == 'recreate' + ] + + if updated_dependencies: + log.debug( + '%s has not changed but its dependencies (%s) have, so recreating', + service.name, ", ".join(updated_dependencies), + ) + plan = service.recreate_plan() else: - create_func = service.start_or_create_containers + plan = service.convergence_plan( + allow_recreate=allow_recreate, + smart_recreate=smart_recreate, + ) - for container in create_func( - insecure_registry=insecure_registry, - do_build=do_build): - running_containers.append(container) + plans[service.name] = plan - return running_containers + return plans def pull(self, service_names=None, insecure_registry=False): for service in self.get_services(service_names, include_deps=True): @@ -250,10 +287,7 @@ class Project(object): return containers def _inject_deps(self, acc, service): - net_name = service.get_net_name() - dep_names = (service.get_linked_names() + - service.get_volumes_from_names() + - ([net_name] if net_name else [])) + dep_names = service.get_dependency_names() if len(dep_names) > 0: dep_services = self.get_services( diff --git a/compose/service.py b/compose/service.py index 306b81833..0c03648c4 100644 --- a/compose/service.py +++ b/compose/service.py @@ -18,9 +18,11 @@ from .const import ( LABEL_PROJECT, LABEL_SERVICE, LABEL_VERSION, + LABEL_CONFIG_HASH, ) from .container import Container, get_container_name from .progress_stream import stream_output, StreamOutputError +from .utils import json_hash log = logging.getLogger(__name__) @@ -59,12 +61,20 @@ class ConfigError(ValueError): pass +class NeedsBuildError(Exception): + def __init__(self, service): + self.service = service + + VolumeSpec = namedtuple('VolumeSpec', 'external internal mode') ServiceName = namedtuple('ServiceName', 'project service number') +ConvergencePlan = namedtuple('ConvergencePlan', 'action containers') + + class Service(object): def __init__(self, name, client=None, project='default', links=None, external_links=None, volumes_from=None, net=None, **options): if not re.match('^%s+$' % VALID_NAME_CHARS, name): @@ -192,6 +202,11 @@ class Service(object): Create a container for this service. If the image doesn't exist, attempt to pull it. """ + self.ensure_image_exists( + do_build=do_build, + insecure_registry=insecure_registry, + ) + container_options = self._get_container_create_options( override_options, number or self._next_container_number(one_off=one_off), @@ -199,38 +214,142 @@ class Service(object): previous_container=previous_container, ) - if (do_build and - self.can_be_built() and - not self.client.images(name=self.full_name)): - self.build() + return Container.create(self.client, **container_options) + def ensure_image_exists(self, + do_build=True, + insecure_registry=False): + + if self.image(): + return + + if self.can_be_built(): + if do_build: + self.build() + else: + raise NeedsBuildError(self) + else: + self.pull(insecure_registry=insecure_registry) + + def image(self): try: - return Container.create(self.client, **container_options) + return self.client.inspect_image(self.image_name) except APIError as e: if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation): - self.pull(insecure_registry=insecure_registry) - return Container.create(self.client, **container_options) - raise + return None + else: + raise - def recreate_containers(self, insecure_registry=False, do_build=True): + @property + def image_name(self): + if self.can_be_built(): + return self.full_name + else: + return self.options['image'] + + def converge(self, + allow_recreate=True, + smart_recreate=False, + insecure_registry=False, + do_build=True): """ If a container for this service doesn't exist, create and start one. If there are any, stop them, create+start new ones, and remove the old containers. """ + plan = self.convergence_plan( + allow_recreate=allow_recreate, + smart_recreate=smart_recreate, + ) + + return self.execute_convergence_plan( + plan, + insecure_registry=insecure_registry, + do_build=do_build, + ) + + def convergence_plan(self, + allow_recreate=True, + smart_recreate=False): + containers = self.containers(stopped=True) + if not containers: + return ConvergencePlan('create', []) + + if smart_recreate and not self._containers_have_diverged(containers): + stopped = [c for c in containers if not c.is_running] + + if stopped: + return ConvergencePlan('start', stopped) + + return ConvergencePlan('noop', containers) + + if not allow_recreate: + return ConvergencePlan('start', containers) + + return ConvergencePlan('recreate', containers) + + def recreate_plan(self): + containers = self.containers(stopped=True) + return ConvergencePlan('recreate', containers) + + def _containers_have_diverged(self, containers): + config_hash = self.config_hash() + has_diverged = False + + for c in containers: + container_config_hash = c.labels.get(LABEL_CONFIG_HASH, None) + if container_config_hash != config_hash: + log.debug( + '%s has diverged: %s != %s', + c.name, container_config_hash, config_hash, + ) + has_diverged = True + + return has_diverged + + def execute_convergence_plan(self, + plan, + insecure_registry=False, + do_build=True): + (action, containers) = plan + + if action == 'create': container = self.create_container( insecure_registry=insecure_registry, - do_build=do_build) + do_build=do_build, + ) self.start_container(container) + return [container] - return [ - self.recreate_container(c, insecure_registry=insecure_registry) - for c in containers - ] + elif action == 'recreate': + return [ + self.recreate_container( + c, + insecure_registry=insecure_registry, + ) + for c in containers + ] - def recreate_container(self, container, insecure_registry=False): + elif action == 'start': + for c in containers: + self.start_container_if_stopped(c) + + return containers + + elif action == 'noop': + for c in containers: + log.info("%s is up-to-date" % c.name) + + return containers + + else: + raise Exception("Invalid action: {}".format(action)) + + def recreate_container(self, + container, + insecure_registry=False): """Recreate a container. The original container is renamed to a temporary name so that data @@ -289,6 +408,21 @@ class Service(object): else: return [self.start_container_if_stopped(c) for c in containers] + def config_hash(self): + return json_hash(self.config_dict()) + + def config_dict(self): + return { + 'options': self.options, + 'image_id': self.image()['Id'], + } + + def get_dependency_names(self): + net_name = self.get_net_name() + return (self.get_linked_names() + + self.get_volumes_from_names() + + ([net_name] if net_name else [])) + def get_linked_names(self): return [s.name for (s, _) in self.links] @@ -376,6 +510,9 @@ class Service(object): number, one_off=False, previous_container=None): + + add_config_hash = (not one_off and not override_options) + container_options = dict( (k, self.options[k]) for k in DOCKER_CONFIG_KEYS if k in self.options) @@ -383,6 +520,13 @@ class Service(object): container_options['name'] = self.get_container_name(number, one_off) + if add_config_hash: + config_hash = self.config_hash() + if 'labels' not in container_options: + container_options['labels'] = {} + container_options['labels'][LABEL_CONFIG_HASH] = config_hash + log.debug("Added config hash: %s" % config_hash) + if 'detach' not in container_options: container_options['detach'] = True @@ -493,7 +637,7 @@ class Service(object): build_output = self.client.build( path=path, - tag=self.full_name, + tag=self.image_name, stream=True, rm=True, nocache=no_cache, diff --git a/compose/state.py b/compose/state.py new file mode 100644 index 000000000..e69de29bb diff --git a/compose/utils.py b/compose/utils.py new file mode 100644 index 000000000..d441a2dae --- /dev/null +++ b/compose/utils.py @@ -0,0 +1,9 @@ +import json +import hashlib + + +def json_hash(obj): + dump = json.dumps(obj, sort_keys=True) + h = hashlib.sha256() + h.update(dump) + return h.hexdigest() diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 00d156b37..b6dcecbc6 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -185,7 +185,7 @@ class ProjectTest(DockerClientTestCase): old_db_id = project.containers()[0].id db_volume_path = project.containers()[0].inspect()['Volumes']['/var/db'] - project.up(recreate=False) + project.up(allow_recreate=False) self.assertEqual(len(project.containers()), 2) db_container = [c for c in project.containers() if 'db' in c.name][0] @@ -204,7 +204,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(project.containers()), 0) project.up(['db']) - project.stop() + project.kill() old_containers = project.containers(stopped=True) @@ -212,10 +212,11 @@ class ProjectTest(DockerClientTestCase): old_db_id = old_containers[0].id db_volume_path = old_containers[0].inspect()['Volumes']['/var/db'] - project.up(recreate=False) + project.up(allow_recreate=False) new_containers = project.containers(stopped=True) self.assertEqual(len(new_containers), 2) + self.assertEqual([c.is_running for c in new_containers], [True, True]) db_container = [c for c in new_containers if 'db' in c.name][0] self.assertEqual(db_container.id, old_db_id) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 47c826ec5..26f02d4a9 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -238,7 +238,7 @@ class ServiceTest(DockerClientTestCase): self.assertIn(volume_container_2.id, host_container.get('HostConfig.VolumesFrom')) - def test_recreate_containers(self): + def test_converge(self): service = self.create_service( 'db', environment={'FOO': '1'}, @@ -258,7 +258,7 @@ class ServiceTest(DockerClientTestCase): num_containers_before = len(self.client.containers(all=True)) service.options['environment']['FOO'] = '2' - new_container, = service.recreate_containers() + new_container = service.converge()[0] self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep']) self.assertEqual(new_container.get('Config.Cmd'), ['300']) @@ -275,7 +275,7 @@ class ServiceTest(DockerClientTestCase): self.client.inspect_container, old_container.id) - def test_recreate_containers_when_containers_are_stopped(self): + def test_converge_when_containers_are_stopped(self): service = self.create_service( 'db', environment={'FOO': '1'}, @@ -285,10 +285,10 @@ class ServiceTest(DockerClientTestCase): ) service.create_container() self.assertEqual(len(service.containers(stopped=True)), 1) - service.recreate_containers() + service.converge() self.assertEqual(len(service.containers(stopped=True)), 1) - def test_recreate_containers_with_image_declared_volume(self): + def test_converge_with_image_declared_volume(self): service = Service( project='composetest', name='db', @@ -300,7 +300,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(old_container.get('Volumes').keys(), ['/data']) volume_path = old_container.get('Volumes')['/data'] - new_container = service.recreate_containers()[0] + new_container = service.converge()[0] self.assertEqual(new_container.get('Volumes').keys(), ['/data']) self.assertEqual(new_container.get('Volumes')['/data'], volume_path) @@ -651,8 +651,19 @@ class ServiceTest(DockerClientTestCase): expected = dict(labels_dict, **compose_labels) service = self.create_service('web', labels=labels_dict) - labels = create_and_start_container(service).labels - self.assertEqual(labels, expected) + labels = create_and_start_container(service).labels.items() + for pair in expected.items(): + self.assertIn(pair, labels) + + service.kill() + service.remove_stopped() + + labels_list = ["%s=%s" % pair for pair in labels_dict.items()] + + service = self.create_service('web', labels=labels_list) + labels = create_and_start_container(service).labels.items() + for pair in expected.items(): + self.assertIn(pair, labels) def test_empty_labels(self): labels_list = ['foo', 'bar'] diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py new file mode 100644 index 000000000..3c0b2530f --- /dev/null +++ b/tests/integration/state_test.py @@ -0,0 +1,263 @@ +from __future__ import unicode_literals +import tempfile +import shutil +import os + +from compose import config +from compose.project import Project +from compose.const import LABEL_CONFIG_HASH + +from .testcases import DockerClientTestCase + + +class ProjectTestCase(DockerClientTestCase): + def run_up(self, cfg, **kwargs): + if 'smart_recreate' not in kwargs: + kwargs['smart_recreate'] = True + + project = self.make_project(cfg) + project.up(**kwargs) + return set(project.containers(stopped=True)) + + def make_project(self, cfg): + return Project.from_dicts( + name='composetest', + client=self.client, + service_dicts=config.from_dictionary(cfg), + ) + + +class BasicProjectTest(ProjectTestCase): + def setUp(self): + super(BasicProjectTest, self).setUp() + + self.cfg = { + 'db': {'image': 'busybox:latest'}, + 'web': {'image': 'busybox:latest'}, + } + + def test_no_change(self): + old_containers = self.run_up(self.cfg) + self.assertEqual(len(old_containers), 2) + + new_containers = self.run_up(self.cfg) + self.assertEqual(len(new_containers), 2) + + self.assertEqual(old_containers, new_containers) + + def test_partial_change(self): + old_containers = self.run_up(self.cfg) + old_db = [c for c in old_containers if c.name_without_project == 'db_1'][0] + old_web = [c for c in old_containers if c.name_without_project == 'web_1'][0] + + self.cfg['web']['command'] = '/bin/true' + + new_containers = self.run_up(self.cfg) + self.assertEqual(len(new_containers), 2) + + preserved = list(old_containers & new_containers) + self.assertEqual(preserved, [old_db]) + + removed = list(old_containers - new_containers) + self.assertEqual(removed, [old_web]) + + created = list(new_containers - old_containers) + self.assertEqual(len(created), 1) + self.assertEqual(created[0].name_without_project, 'web_1') + self.assertEqual(created[0].get('Config.Cmd'), ['/bin/true']) + + def test_all_change(self): + old_containers = self.run_up(self.cfg) + self.assertEqual(len(old_containers), 2) + + self.cfg['web']['command'] = '/bin/true' + self.cfg['db']['command'] = '/bin/true' + + new_containers = self.run_up(self.cfg) + self.assertEqual(len(new_containers), 2) + + unchanged = old_containers & new_containers + self.assertEqual(len(unchanged), 0) + + new = new_containers - old_containers + self.assertEqual(len(new), 2) + + +class ProjectWithDependenciesTest(ProjectTestCase): + def setUp(self): + super(ProjectWithDependenciesTest, self).setUp() + + self.cfg = { + 'db': { + 'image': 'busybox:latest', + 'command': 'tail -f /dev/null', + }, + 'web': { + 'image': 'busybox:latest', + 'command': 'tail -f /dev/null', + 'links': ['db'], + }, + 'nginx': { + 'image': 'busybox:latest', + 'command': 'tail -f /dev/null', + 'links': ['web'], + }, + } + + def test_up(self): + containers = self.run_up(self.cfg) + self.assertEqual( + set(c.name_without_project for c in containers), + set(['db_1', 'web_1', 'nginx_1']), + ) + + def test_change_leaf(self): + old_containers = self.run_up(self.cfg) + + self.cfg['nginx']['environment'] = {'NEW_VAR': '1'} + new_containers = self.run_up(self.cfg) + + self.assertEqual( + set(c.name_without_project for c in new_containers - old_containers), + set(['nginx_1']), + ) + + def test_change_middle(self): + old_containers = self.run_up(self.cfg) + + self.cfg['web']['environment'] = {'NEW_VAR': '1'} + new_containers = self.run_up(self.cfg) + + self.assertEqual( + set(c.name_without_project for c in new_containers - old_containers), + set(['web_1', 'nginx_1']), + ) + + def test_change_root(self): + old_containers = self.run_up(self.cfg) + + self.cfg['db']['environment'] = {'NEW_VAR': '1'} + new_containers = self.run_up(self.cfg) + + self.assertEqual( + set(c.name_without_project for c in new_containers - old_containers), + set(['db_1', 'web_1', 'nginx_1']), + ) + + def test_change_root_no_recreate(self): + old_containers = self.run_up(self.cfg) + + self.cfg['db']['environment'] = {'NEW_VAR': '1'} + new_containers = self.run_up(self.cfg, allow_recreate=False) + + self.assertEqual(new_containers - old_containers, set()) + + +class ServiceStateTest(DockerClientTestCase): + def test_trigger_create(self): + web = self.create_service('web') + self.assertEqual(('create', []), web.convergence_plan(smart_recreate=True)) + + def test_trigger_noop(self): + web = self.create_service('web') + container = web.create_container() + web.start() + + web = self.create_service('web') + self.assertEqual(('noop', [container]), web.convergence_plan(smart_recreate=True)) + + def test_trigger_start(self): + options = dict(command=["/bin/sleep", "300"]) + + web = self.create_service('web', **options) + web.scale(2) + + containers = web.containers(stopped=True) + containers[0].stop() + containers[0].inspect() + + self.assertEqual([c.is_running for c in containers], [False, True]) + + web = self.create_service('web', **options) + self.assertEqual( + ('start', containers[0:1]), + web.convergence_plan(smart_recreate=True), + ) + + def test_trigger_recreate_with_config_change(self): + web = self.create_service('web', command=["/bin/sleep", "300"]) + container = web.create_container() + + web = self.create_service('web', command=["/bin/sleep", "400"]) + self.assertEqual(('recreate', [container]), web.convergence_plan(smart_recreate=True)) + + def test_trigger_recreate_with_image_change(self): + repo = 'composetest_myimage' + tag = 'latest' + image = '{}:{}'.format(repo, tag) + + image_id = self.client.images(name='busybox')[0]['Id'] + self.client.tag(image_id, repository=repo, tag=tag) + + try: + web = self.create_service('web', image=image) + container = web.create_container() + + # update the image + c = self.client.create_container(image, ['touch', '/hello.txt']) + self.client.commit(c, repository=repo, tag=tag) + self.client.remove_container(c) + + web = self.create_service('web', image=image) + self.assertEqual(('recreate', [container]), web.convergence_plan(smart_recreate=True)) + + finally: + self.client.remove_image(image) + + def test_trigger_recreate_with_build(self): + context = tempfile.mkdtemp() + + try: + dockerfile = os.path.join(context, 'Dockerfile') + + with open(dockerfile, 'w') as f: + f.write('FROM busybox\n') + + web = self.create_service('web', build=context) + container = web.create_container() + + with open(dockerfile, 'w') as f: + f.write('FROM busybox\nCMD echo hello world\n') + web.build() + + web = self.create_service('web', build=context) + self.assertEqual(('recreate', [container]), web.convergence_plan(smart_recreate=True)) + finally: + shutil.rmtree(context) + + +class ConfigHashTest(DockerClientTestCase): + def test_no_config_hash_when_one_off(self): + web = self.create_service('web') + container = web.create_container(one_off=True) + self.assertNotIn(LABEL_CONFIG_HASH, container.labels) + + def test_no_config_hash_when_overriding_options(self): + web = self.create_service('web') + container = web.create_container(environment={'FOO': '1'}) + self.assertNotIn(LABEL_CONFIG_HASH, container.labels) + + def test_config_hash_with_custom_labels(self): + web = self.create_service('web', labels={'foo': '1'}) + container = web.converge()[0] + self.assertIn(LABEL_CONFIG_HASH, container.labels) + self.assertIn('foo', container.labels) + + def test_config_hash_sticks_around(self): + web = self.create_service('web', command=["/bin/sleep", "300"]) + container = web.converge()[0] + self.assertIn(LABEL_CONFIG_HASH, container.labels) + + web = self.create_service('web', command=["/bin/sleep", "400"]) + container = web.converge()[0] + self.assertIn(LABEL_CONFIG_HASH, container.labels) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index fa252062c..add48086d 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -5,14 +5,13 @@ from .. import unittest import mock import docker -from requests import Response from compose.service import Service from compose.container import Container from compose.const import LABEL_SERVICE, LABEL_PROJECT, LABEL_ONE_OFF from compose.service import ( - APIError, ConfigError, + NeedsBuildError, build_port_bindings, build_volume_binding, get_container_data_volumes, @@ -223,36 +222,28 @@ class ServiceTest(unittest.TestCase): insecure_registry=False, stream=True) - @mock.patch('compose.service.Container', autospec=True) - @mock.patch('compose.service.log', autospec=True) - def test_create_container_from_insecure_registry( - self, - mock_log, - mock_container): + def test_create_container_from_insecure_registry(self): service = Service('foo', client=self.mock_client, image='someimage:sometag') - mock_response = mock.Mock(Response) - mock_response.status_code = 404 - mock_response.reason = "Not Found" - mock_container.create.side_effect = APIError( - 'Mock error', mock_response, "No such image") + images = [] - # We expect the APIError because our service requires a - # non-existent image. - with self.assertRaises(APIError): - service.create_container(insecure_registry=True) + def pull(repo, tag=None, insecure_registry=False, **kwargs): + self.assertEqual('someimage', repo) + self.assertEqual('sometag', tag) + self.assertTrue(insecure_registry) + images.append({'Id': 'abc123'}) + return [] - self.mock_client.pull.assert_called_once_with( - 'someimage', - tag='sometag', - insecure_registry=True, - stream=True) - mock_log.info.assert_called_with( - 'Pulling foo (someimage:sometag)...') + service.image = lambda: images[0] if images else None + self.mock_client.pull = pull + + service.create_container(insecure_registry=True) + self.assertEqual(1, len(images)) @mock.patch('compose.service.Container', autospec=True) def test_recreate_container(self, _): mock_container = mock.create_autospec(Container) service = Service('foo', client=self.mock_client, image='someimage') + service.image = lambda: {'Id': 'abc123'} new_container = service.recreate_container(mock_container) mock_container.stop.assert_called_once_with() @@ -273,36 +264,45 @@ class ServiceTest(unittest.TestCase): @mock.patch('compose.service.Container', autospec=True) def test_create_container_latest_is_used_when_no_tag_specified(self, mock_container): - mock_container.create.side_effect = APIError( - "oops", - mock.Mock(status_code=404), - "No such image") service = Service('foo', client=self.mock_client, image='someimage') - with self.assertRaises(APIError): - service.create_container() - self.mock_client.pull.assert_called_once_with( - 'someimage', - tag='latest', - insecure_registry=False, - stream=True) + images = [] + + def pull(repo, tag=None, **kwargs): + self.assertEqual('someimage', repo) + self.assertEqual('latest', tag) + images.append({'Id': 'abc123'}) + return [] + + service.image = lambda: images[0] if images else None + self.mock_client.pull = pull + + service.create_container() + self.assertEqual(1, len(images)) def test_create_container_with_build(self): - self.mock_client.images.return_value = [] service = Service('foo', client=self.mock_client, build='.') - service.build = mock.create_autospec(service.build) - service.create_container(do_build=True) - self.mock_client.images.assert_called_once_with(name=service.full_name) - service.build.assert_called_once_with() + images = [] + service.image = lambda *args, **kwargs: images[0] if images else None + service.build = lambda: images.append({'Id': 'abc123'}) + + service.create_container(do_build=True) + self.assertEqual(1, len(images)) def test_create_container_no_build(self): - self.mock_client.images.return_value = [] service = Service('foo', client=self.mock_client, build='.') - service.create_container(do_build=False) + service.image = lambda: {'Id': 'abc123'} - self.assertFalse(self.mock_client.images.called) + service.create_container(do_build=False) self.assertFalse(self.mock_client.build.called) + def test_create_container_no_build_but_needs_build(self): + service = Service('foo', client=self.mock_client, build='.') + service.image = lambda: None + + with self.assertRaises(NeedsBuildError): + service.create_container(do_build=False) + class ServiceVolumesTest(unittest.TestCase): From 41315b32cbe40ec3649b484ea8e3cba197469275 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Tue, 19 May 2015 16:37:50 +0200 Subject: [PATCH 074/118] Fix #1426 - migrate_to_labels not found Signed-off-by: Harald Albers --- compose/cli/main.py | 5 +++++ compose/service.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 557dc6367..a2dca65db 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -490,6 +490,11 @@ class TopLevelCommand(Command): project.stop(service_names=service_names, **params) def migrate_to_labels(self, project, _options): + """ + Recreate containers to add labels + + Usage: migrate_to_labels + """ migration.migrate_project_to_labels(project) diff --git a/compose/service.py b/compose/service.py index 0c03648c4..e351fa60b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -786,7 +786,7 @@ def check_for_legacy_containers( "labels. As of compose 1.3.0 containers are identified with " "labels instead of naming convention. If you'd like compose " "to use this container, please run " - "`docker-compose --migrate-to-labels`" % (name,)) + "`docker-compose migrate_to_labels`" % (name,)) def parse_restart_spec(restart_config): From ea7ee301c04127ff5f1b0faa661e5faea38ce1af Mon Sep 17 00:00:00 2001 From: lsowen Date: Fri, 17 Apr 2015 01:34:42 +0000 Subject: [PATCH 075/118] Add security_opt as a docker-compose.yml option Signed-off-by: Logan Owen --- compose/config.py | 1 + compose/service.py | 5 ++++- docs/yml.md | 10 ++++++++++ tests/integration/service_test.py | 7 +++++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/compose/config.py b/compose/config.py index 1919ef5a3..efc50075e 100644 --- a/compose/config.py +++ b/compose/config.py @@ -30,6 +30,7 @@ DOCKER_CONFIG_KEYS = [ 'ports', 'privileged', 'restart', + 'security_opt', 'stdin_open', 'tty', 'user', diff --git a/compose/service.py b/compose/service.py index 0c03648c4..8efa6e9f3 100644 --- a/compose/service.py +++ b/compose/service.py @@ -42,6 +42,7 @@ DOCKER_START_KEYS = [ 'privileged', 'restart', 'volumes_from', + 'security_opt', ] VALID_NAME_CHARS = '[a-zA-Z0-9]' @@ -595,6 +596,7 @@ class Service(object): cap_drop = options.get('cap_drop', None) log_config = LogConfig(type=options.get('log_driver', 'json-file')) pid = options.get('pid', None) + security_opt = options.get('security_opt', None) dns = options.get('dns', None) if isinstance(dns, six.string_types): @@ -627,7 +629,8 @@ class Service(object): log_config=log_config, extra_hosts=extra_hosts, read_only=read_only, - pid_mode=pid + pid_mode=pid, + security_opt=security_opt ) def build(self, no_cache=False): diff --git a/docs/yml.md b/docs/yml.md index 41247c703..df791bc98 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -352,6 +352,16 @@ devices: - "/dev/ttyUSB0:/dev/ttyUSB0" ``` +### security_opt + +Override the default labeling scheme for each container. + +``` +security_opt: + - label:user:USER + - label:role:ROLE +``` + ### working\_dir, entrypoint, user, hostname, domainname, mem\_limit, privileged, restart, stdin\_open, tty, cpu\_shares, cpuset, read\_only Each of these is a single value, analogous to its diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 26f02d4a9..b6cde37cc 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -192,6 +192,13 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertEqual(container.get('HostConfig.ReadonlyRootfs'), read_only, container.get('HostConfig')) + def test_create_container_with_security_opt(self): + security_opt = ['label:disable'] + service = self.create_service('db', security_opt=security_opt) + container = service.create_container() + service.start_container(container) + self.assertEqual(set(container.get('HostConfig.SecurityOpt')), set(security_opt)) + def test_create_container_with_specified_volume(self): host_path = '/tmp/host-path' container_path = '/container-path' From f5ac1fa0738384fada4abb979ba25dddecc56372 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 20 May 2015 16:02:08 +0100 Subject: [PATCH 076/118] Remove whitespace from json hash Reasoning: https://github.com/aanand/fig/commit/e5d8447f063498164f12567554a2eec16b4a3c88#commitcomment-11243708 Signed-off-by: Ben Firshman --- compose/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/utils.py b/compose/utils.py index d441a2dae..76a4c6b93 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -3,7 +3,7 @@ import hashlib def json_hash(obj): - dump = json.dumps(obj, sort_keys=True) + dump = json.dumps(obj, sort_keys=True, separators=(',', ':')) h = hashlib.sha256() h.update(dump) return h.hexdigest() From 022f81711eca255a6ba248a1dc62d807dccfb7dd Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 20 May 2015 20:45:48 -0400 Subject: [PATCH 077/118] Fixes #1434, Project.containers with service_names. Signed-off-by: Daniel Nephin --- compose/project.py | 9 +++++++-- tests/integration/project_test.py | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/compose/project.py b/compose/project.py index c37175ae0..a13b8a1fb 100644 --- a/compose/project.py +++ b/compose/project.py @@ -6,7 +6,7 @@ from functools import reduce from docker.errors import APIError from .config import get_service_name_from_net, ConfigurationError -from .const import LABEL_PROJECT, LABEL_ONE_OFF +from .const import LABEL_PROJECT, LABEL_SERVICE, LABEL_ONE_OFF from .service import Service, check_for_legacy_containers from .container import Container @@ -276,6 +276,11 @@ class Project(object): all=stopped, filters={'label': self.labels(one_off=one_off)})] + def matches_service_names(container): + if not service_names: + return True + return container.labels.get(LABEL_SERVICE) in service_names + if not containers: check_for_legacy_containers( self.client, @@ -284,7 +289,7 @@ class Project(object): stopped=stopped, one_off=one_off) - return containers + return filter(matches_service_names, containers) def _inject_deps(self, acc, service): dep_names = service.get_dependency_names() diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index b6dcecbc6..6e315e84a 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -6,6 +6,29 @@ from .testcases import DockerClientTestCase class ProjectTest(DockerClientTestCase): + + def test_containers(self): + web = self.create_service('web') + db = self.create_service('db') + project = Project('composetest', [web, db], self.client) + + project.up() + + containers = project.containers() + self.assertEqual(len(containers), 2) + + def test_containers_with_service_names(self): + web = self.create_service('web') + db = self.create_service('db') + project = Project('composetest', [web, db], self.client) + + project.up() + + containers = project.containers(['web']) + self.assertEqual( + [c.name for c in containers], + ['composetest_web_1']) + def test_volumes_from_service(self): service_dicts = config.from_dictionary({ 'data': { From 3080244c0bcad4c9b70def823acf8f963b4975ce Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 May 2015 14:54:41 +0100 Subject: [PATCH 078/118] Rename migrate_to_labels -> migrate-to-labels Signed-off-by: Aanand Prasad --- compose/cli/docopt_command.py | 2 ++ compose/cli/main.py | 4 ++-- compose/service.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/compose/cli/docopt_command.py b/compose/cli/docopt_command.py index 8105d3b3f..ee6947012 100644 --- a/compose/cli/docopt_command.py +++ b/compose/cli/docopt_command.py @@ -33,6 +33,8 @@ class DocoptCommand(object): if command is None: raise SystemExit(getdoc(self)) + command = command.replace('-', '_') + if not hasattr(self, command): raise NoSuchCommand(command, self) diff --git a/compose/cli/main.py b/compose/cli/main.py index a2dca65db..cf7d83114 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -99,7 +99,7 @@ class TopLevelCommand(Command): start Start services stop Stop services up Create and start containers - migrate_to_labels Recreate containers to add labels + migrate-to-labels Recreate containers to add labels """ def docopt_options(self): @@ -493,7 +493,7 @@ class TopLevelCommand(Command): """ Recreate containers to add labels - Usage: migrate_to_labels + Usage: migrate-to-labels """ migration.migrate_project_to_labels(project) diff --git a/compose/service.py b/compose/service.py index 7e2bca244..e10758574 100644 --- a/compose/service.py +++ b/compose/service.py @@ -789,7 +789,7 @@ def check_for_legacy_containers( "labels. As of compose 1.3.0 containers are identified with " "labels instead of naming convention. If you'd like compose " "to use this container, please run " - "`docker-compose migrate_to_labels`" % (name,)) + "`docker-compose migrate-to-labels`" % (name,)) def parse_restart_spec(restart_config): From b0cb31c18635f5bb277c51b656974ce6abef5205 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 May 2015 16:19:15 +0100 Subject: [PATCH 079/118] Use 'top' instead of 'sleep' as a dummy command Signed-off-by: Aanand Prasad --- .../fixtures/UpperCaseDir/docker-compose.yml | 4 ++-- .../dockerfile-with-volume/Dockerfile | 2 +- .../docker-compose.yml | 2 +- tests/fixtures/extends/docker-compose.yml | 4 ++-- .../links-composefile/docker-compose.yml | 6 ++--- .../docker-compose.yaml | 2 +- .../multiple-composefiles/compose2.yml | 2 +- .../multiple-composefiles/docker-compose.yml | 4 ++-- .../ports-composefile/docker-compose.yml | 2 +- .../simple-composefile/docker-compose.yml | 4 ++-- tests/integration/project_test.py | 22 +++++++++---------- tests/integration/service_test.py | 18 +++++++-------- tests/integration/state_test.py | 10 ++++----- tests/integration/testcases.py | 2 +- tests/unit/config_test.py | 4 ++-- tests/unit/container_test.py | 2 +- 16 files changed, 45 insertions(+), 45 deletions(-) diff --git a/tests/fixtures/UpperCaseDir/docker-compose.yml b/tests/fixtures/UpperCaseDir/docker-compose.yml index 3538ab097..b25beaf4b 100644 --- a/tests/fixtures/UpperCaseDir/docker-compose.yml +++ b/tests/fixtures/UpperCaseDir/docker-compose.yml @@ -1,6 +1,6 @@ simple: image: busybox:latest - command: /bin/sleep 300 + command: top another: image: busybox:latest - command: /bin/sleep 300 + command: top diff --git a/tests/fixtures/dockerfile-with-volume/Dockerfile b/tests/fixtures/dockerfile-with-volume/Dockerfile index 2d6437cf4..6e5d0a55e 100644 --- a/tests/fixtures/dockerfile-with-volume/Dockerfile +++ b/tests/fixtures/dockerfile-with-volume/Dockerfile @@ -1,3 +1,3 @@ FROM busybox VOLUME /data -CMD sleep 3000 +CMD top diff --git a/tests/fixtures/environment-composefile/docker-compose.yml b/tests/fixtures/environment-composefile/docker-compose.yml index 924932273..9d99fee08 100644 --- a/tests/fixtures/environment-composefile/docker-compose.yml +++ b/tests/fixtures/environment-composefile/docker-compose.yml @@ -1,6 +1,6 @@ service: image: busybox:latest - command: sleep 5 + command: top environment: foo: bar diff --git a/tests/fixtures/extends/docker-compose.yml b/tests/fixtures/extends/docker-compose.yml index 0ae92d2a5..c51be49ec 100644 --- a/tests/fixtures/extends/docker-compose.yml +++ b/tests/fixtures/extends/docker-compose.yml @@ -2,7 +2,7 @@ myweb: extends: file: common.yml service: web - command: sleep 300 + command: top links: - "mydb:db" environment: @@ -13,4 +13,4 @@ myweb: BAZ: "2" mydb: image: busybox - command: sleep 300 + command: top diff --git a/tests/fixtures/links-composefile/docker-compose.yml b/tests/fixtures/links-composefile/docker-compose.yml index bc5391a99..930fd4c7a 100644 --- a/tests/fixtures/links-composefile/docker-compose.yml +++ b/tests/fixtures/links-composefile/docker-compose.yml @@ -1,11 +1,11 @@ db: image: busybox:latest - command: /bin/sleep 300 + command: top web: image: busybox:latest - command: /bin/sleep 300 + command: top links: - db:db console: image: busybox:latest - command: /bin/sleep 300 + command: top diff --git a/tests/fixtures/longer-filename-composefile/docker-compose.yaml b/tests/fixtures/longer-filename-composefile/docker-compose.yaml index 315289402..b55a9e124 100644 --- a/tests/fixtures/longer-filename-composefile/docker-compose.yaml +++ b/tests/fixtures/longer-filename-composefile/docker-compose.yaml @@ -1,3 +1,3 @@ definedinyamlnotyml: image: busybox:latest - command: /bin/sleep 300 \ No newline at end of file + command: top \ No newline at end of file diff --git a/tests/fixtures/multiple-composefiles/compose2.yml b/tests/fixtures/multiple-composefiles/compose2.yml index 523becac2..568033804 100644 --- a/tests/fixtures/multiple-composefiles/compose2.yml +++ b/tests/fixtures/multiple-composefiles/compose2.yml @@ -1,3 +1,3 @@ yetanother: image: busybox:latest - command: /bin/sleep 300 + command: top diff --git a/tests/fixtures/multiple-composefiles/docker-compose.yml b/tests/fixtures/multiple-composefiles/docker-compose.yml index 3538ab097..b25beaf4b 100644 --- a/tests/fixtures/multiple-composefiles/docker-compose.yml +++ b/tests/fixtures/multiple-composefiles/docker-compose.yml @@ -1,6 +1,6 @@ simple: image: busybox:latest - command: /bin/sleep 300 + command: top another: image: busybox:latest - command: /bin/sleep 300 + command: top diff --git a/tests/fixtures/ports-composefile/docker-compose.yml b/tests/fixtures/ports-composefile/docker-compose.yml index 2474087d0..9496ee082 100644 --- a/tests/fixtures/ports-composefile/docker-compose.yml +++ b/tests/fixtures/ports-composefile/docker-compose.yml @@ -1,7 +1,7 @@ simple: image: busybox:latest - command: /bin/sleep 300 + command: top ports: - '3000' - '49152:3001' diff --git a/tests/fixtures/simple-composefile/docker-compose.yml b/tests/fixtures/simple-composefile/docker-compose.yml index 3538ab097..b25beaf4b 100644 --- a/tests/fixtures/simple-composefile/docker-compose.yml +++ b/tests/fixtures/simple-composefile/docker-compose.yml @@ -1,6 +1,6 @@ simple: image: busybox:latest - command: /bin/sleep 300 + command: top another: image: busybox:latest - command: /bin/sleep 300 + command: top diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 6e315e84a..5e3a40e5b 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -78,12 +78,12 @@ class ProjectTest(DockerClientTestCase): service_dicts=config.from_dictionary({ 'net': { 'image': 'busybox:latest', - 'command': ["/bin/sleep", "300"] + 'command': ["top"] }, 'web': { 'image': 'busybox:latest', 'net': 'container:net', - 'command': ["/bin/sleep", "300"] + 'command': ["top"] }, }), client=self.client, @@ -103,7 +103,7 @@ class ProjectTest(DockerClientTestCase): self.client, image='busybox:latest', name='composetest_net_container', - command='/bin/sleep 300' + command='top' ) net_container.start() @@ -288,20 +288,20 @@ class ProjectTest(DockerClientTestCase): service_dicts=config.from_dictionary({ 'console': { 'image': 'busybox:latest', - 'command': ["/bin/sleep", "300"], + 'command': ["top"], }, 'data': { 'image': 'busybox:latest', - 'command': ["/bin/sleep", "300"] + 'command': ["top"] }, 'db': { 'image': 'busybox:latest', - 'command': ["/bin/sleep", "300"], + 'command': ["top"], 'volumes_from': ['data'], }, 'web': { 'image': 'busybox:latest', - 'command': ["/bin/sleep", "300"], + 'command': ["top"], 'links': ['db'], }, }), @@ -326,20 +326,20 @@ class ProjectTest(DockerClientTestCase): service_dicts=config.from_dictionary({ 'console': { 'image': 'busybox:latest', - 'command': ["/bin/sleep", "300"], + 'command': ["top"], }, 'data': { 'image': 'busybox:latest', - 'command': ["/bin/sleep", "300"] + 'command': ["top"] }, 'db': { 'image': 'busybox:latest', - 'command': ["/bin/sleep", "300"], + 'command': ["top"], 'volumes_from': ['data'], }, 'web': { 'image': 'busybox:latest', - 'command': ["/bin/sleep", "300"], + 'command': ["top"], 'links': ['db'], }, }), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index b6cde37cc..8fd8212ce 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -236,7 +236,7 @@ class ServiceTest(DockerClientTestCase): def test_create_container_with_volumes_from(self): volume_service = self.create_service('data') volume_container_1 = volume_service.create_container() - volume_container_2 = Container.create(self.client, image='busybox:latest', command=["/bin/sleep", "300"]) + volume_container_2 = Container.create(self.client, image='busybox:latest', command=["top"]) host_service = self.create_service('host', volumes_from=[volume_service, volume_container_2]) host_container = host_service.create_container() host_service.start_container(host_container) @@ -250,12 +250,12 @@ class ServiceTest(DockerClientTestCase): 'db', environment={'FOO': '1'}, volumes=['/etc'], - entrypoint=['sleep'], - command=['300'] + entrypoint=['top'], + command=['-d', '1'] ) old_container = service.create_container() - self.assertEqual(old_container.get('Config.Entrypoint'), ['sleep']) - self.assertEqual(old_container.get('Config.Cmd'), ['300']) + self.assertEqual(old_container.get('Config.Entrypoint'), ['top']) + self.assertEqual(old_container.get('Config.Cmd'), ['-d', '1']) self.assertIn('FOO=1', old_container.get('Config.Env')) self.assertEqual(old_container.name, 'composetest_db_1') service.start_container(old_container) @@ -267,8 +267,8 @@ class ServiceTest(DockerClientTestCase): service.options['environment']['FOO'] = '2' new_container = service.converge()[0] - self.assertEqual(new_container.get('Config.Entrypoint'), ['sleep']) - self.assertEqual(new_container.get('Config.Cmd'), ['300']) + self.assertEqual(new_container.get('Config.Entrypoint'), ['top']) + self.assertEqual(new_container.get('Config.Cmd'), ['-d', '1']) self.assertIn('FOO=2', new_container.get('Config.Env')) self.assertEqual(new_container.name, 'composetest_db_1') self.assertEqual(new_container.get('Volumes')['/etc'], volume_path) @@ -287,8 +287,8 @@ class ServiceTest(DockerClientTestCase): 'db', environment={'FOO': '1'}, volumes=['/var/db'], - entrypoint=['sleep'], - command=['300'] + entrypoint=['top'], + command=['-d', '1'] ) service.create_container() self.assertEqual(len(service.containers(stopped=True)), 1) diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 3c0b2530f..7a7d2b58f 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -167,7 +167,7 @@ class ServiceStateTest(DockerClientTestCase): self.assertEqual(('noop', [container]), web.convergence_plan(smart_recreate=True)) def test_trigger_start(self): - options = dict(command=["/bin/sleep", "300"]) + options = dict(command=["top"]) web = self.create_service('web', **options) web.scale(2) @@ -185,10 +185,10 @@ class ServiceStateTest(DockerClientTestCase): ) def test_trigger_recreate_with_config_change(self): - web = self.create_service('web', command=["/bin/sleep", "300"]) + web = self.create_service('web', command=["top"]) container = web.create_container() - web = self.create_service('web', command=["/bin/sleep", "400"]) + web = self.create_service('web', command=["top", "-d", "1"]) self.assertEqual(('recreate', [container]), web.convergence_plan(smart_recreate=True)) def test_trigger_recreate_with_image_change(self): @@ -254,10 +254,10 @@ class ConfigHashTest(DockerClientTestCase): self.assertIn('foo', container.labels) def test_config_hash_sticks_around(self): - web = self.create_service('web', command=["/bin/sleep", "300"]) + web = self.create_service('web', command=["top"]) container = web.converge()[0] self.assertIn(LABEL_CONFIG_HASH, container.labels) - web = self.create_service('web', command=["/bin/sleep", "400"]) + web = self.create_service('web', command=["top", "-d", "1"]) container = web.converge()[0] self.assertIn(LABEL_CONFIG_HASH, container.labels) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 4a0f7248a..48fcf3ef2 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -27,7 +27,7 @@ class DockerClientTestCase(unittest.TestCase): kwargs['image'] = 'busybox:latest' if 'command' not in kwargs: - kwargs['command'] = ["/bin/sleep", "300"] + kwargs['command'] = ["top"] return Service( project='composetest', diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 0a48dfefe..ebd2af7d5 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -348,12 +348,12 @@ class ExtendsTest(unittest.TestCase): { 'name': 'mydb', 'image': 'busybox', - 'command': 'sleep 300', + 'command': 'top', }, { 'name': 'myweb', 'image': 'busybox', - 'command': 'sleep 300', + 'command': 'top', 'links': ['mydb:db'], 'environment': { "FOO": "1", diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 2313d4b8e..c537a8cf5 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -14,7 +14,7 @@ class ContainerTest(unittest.TestCase): self.container_dict = { "Id": "abc", "Image": "busybox:latest", - "Command": "sleep 300", + "Command": "top", "Created": 1387384730, "Status": "Up 8 seconds", "Ports": None, From 0fdb8bf8147d61814cf8007f253a3f6049428d88 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 May 2015 11:25:06 +0100 Subject: [PATCH 080/118] Refactor migration logic - Rename `migration` module to `legacy` to make its legacy-ness explicit - Move `check_for_legacy_containers` into `legacy` module - Fix migration test so it can be run in isolation Signed-off-by: Aanand Prasad --- compose/cli/main.py | 4 ++-- compose/{migration.py => legacy.py} | 25 ++++++++++++++++++++++++ compose/project.py | 3 ++- compose/service.py | 28 ++------------------------- tests/integration/legacy_test.py | 30 +++++++++++++++++++++++++++++ tests/integration/migration_test.py | 23 ---------------------- 6 files changed, 61 insertions(+), 52 deletions(-) rename compose/{migration.py => legacy.py} (50%) create mode 100644 tests/integration/legacy_test.py delete mode 100644 tests/integration/migration_test.py diff --git a/compose/cli/main.py b/compose/cli/main.py index cf7d83114..2b95040ca 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -11,7 +11,7 @@ from docker.errors import APIError import dockerpty from .. import __version__ -from .. import migration +from .. import legacy from ..project import NoSuchService, ConfigurationError from ..service import BuildError, CannotBeScaledError, NeedsBuildError from ..config import parse_environment @@ -495,7 +495,7 @@ class TopLevelCommand(Command): Usage: migrate-to-labels """ - migration.migrate_project_to_labels(project) + legacy.migrate_project_to_labels(project) def list_containers(containers): diff --git a/compose/migration.py b/compose/legacy.py similarity index 50% rename from compose/migration.py rename to compose/legacy.py index 16b5dd167..dc90079da 100644 --- a/compose/migration.py +++ b/compose/legacy.py @@ -16,6 +16,31 @@ def is_valid_name(name): return match is not None +def check_for_legacy_containers( + client, + project, + services, + stopped=False, + one_off=False): + """Check if there are containers named using the old naming convention + and warn the user that those containers may need to be migrated to + using labels, so that compose can find them. + """ + for container in client.containers(all=stopped): + name = get_container_name(container) + for service in services: + prefix = '%s_%s_%s' % (project, service, 'run_' if one_off else '') + if not name.startswith(prefix): + continue + + log.warn( + "Compose found a found a container named %s without any " + "labels. As of compose 1.3.0 containers are identified with " + "labels instead of naming convention. If you'd like compose " + "to use this container, please run " + "`docker-compose migrate-to-labels`" % (name,)) + + def add_labels(project, container, name): project_name, service_name, one_off, number = NAME_RE.match(name).groups() if project_name != project.name or service_name not in project.service_names: diff --git a/compose/project.py b/compose/project.py index a13b8a1fb..6dc926681 100644 --- a/compose/project.py +++ b/compose/project.py @@ -7,8 +7,9 @@ from docker.errors import APIError from .config import get_service_name_from_net, ConfigurationError from .const import LABEL_PROJECT, LABEL_SERVICE, LABEL_ONE_OFF -from .service import Service, check_for_legacy_containers +from .service import Service from .container import Container +from .legacy import check_for_legacy_containers log = logging.getLogger(__name__) diff --git a/compose/service.py b/compose/service.py index e10758574..daf225ce8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -20,7 +20,8 @@ from .const import ( LABEL_VERSION, LABEL_CONFIG_HASH, ) -from .container import Container, get_container_name +from .container import Container +from .legacy import check_for_legacy_containers from .progress_stream import stream_output, StreamOutputError from .utils import json_hash @@ -767,31 +768,6 @@ def build_container_labels(label_options, service_labels, number, one_off=False) return labels -def check_for_legacy_containers( - client, - project, - services, - stopped=False, - one_off=False): - """Check if there are containers named using the old naming convention - and warn the user that those containers may need to be migrated to - using labels, so that compose can find them. - """ - for container in client.containers(all=stopped): - name = get_container_name(container) - for service in services: - prefix = '%s_%s_%s' % (project, service, 'run_' if one_off else '') - if not name.startswith(prefix): - continue - - log.warn( - "Compose found a found a container named %s without any " - "labels. As of compose 1.3.0 containers are identified with " - "labels instead of naming convention. If you'd like compose " - "to use this container, please run " - "`docker-compose migrate-to-labels`" % (name,)) - - def parse_restart_spec(restart_config): if not restart_config: return None diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py new file mode 100644 index 000000000..d39635b7f --- /dev/null +++ b/tests/integration/legacy_test.py @@ -0,0 +1,30 @@ +import mock + +from compose import legacy +from compose.project import Project +from .testcases import DockerClientTestCase + + +class ProjectTest(DockerClientTestCase): + + def test_migration_to_labels(self): + services = [ + self.create_service('web'), + self.create_service('db'), + ] + + project = Project('composetest', services, self.client) + + for service in services: + service.ensure_image_exists() + self.client.create_container( + name='{}_{}_1'.format(project.name, service.name), + **service.options + ) + + with mock.patch.object(legacy, 'log', autospec=True) as mock_log: + self.assertEqual(project.containers(stopped=True), []) + self.assertEqual(mock_log.warn.call_count, 2) + + legacy.migrate_project_to_labels(project) + self.assertEqual(len(project.containers(stopped=True)), 2) diff --git a/tests/integration/migration_test.py b/tests/integration/migration_test.py deleted file mode 100644 index 133d23148..000000000 --- a/tests/integration/migration_test.py +++ /dev/null @@ -1,23 +0,0 @@ -import mock - -from compose import service, migration -from compose.project import Project -from .testcases import DockerClientTestCase - - -class ProjectTest(DockerClientTestCase): - - def test_migration_to_labels(self): - web = self.create_service('web') - db = self.create_service('db') - project = Project('composetest', [web, db], self.client) - - self.client.create_container(name='composetest_web_1', **web.options) - self.client.create_container(name='composetest_db_1', **db.options) - - with mock.patch.object(service, 'log', autospec=True) as mock_log: - self.assertEqual(project.containers(stopped=True), []) - self.assertEqual(mock_log.warn.call_count, 2) - - migration.migrate_project_to_labels(project) - self.assertEqual(len(project.containers(stopped=True)), 2) From b5ce23885b22377aab9a1eff8f27f58474c29c76 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 May 2015 12:12:02 +0100 Subject: [PATCH 081/118] Split out fetching of legacy names so we can test it Signed-off-by: Aanand Prasad --- compose/legacy.py | 33 ++++++++++++++++++++-------- tests/integration/legacy_test.py | 37 +++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/compose/legacy.py b/compose/legacy.py index dc90079da..7f97c3e94 100644 --- a/compose/legacy.py +++ b/compose/legacy.py @@ -26,19 +26,34 @@ def check_for_legacy_containers( and warn the user that those containers may need to be migrated to using labels, so that compose can find them. """ + names = get_legacy_container_names( + client, + project, + services, + stopped=stopped, + one_off=one_off) + + for name in names: + log.warn( + "Compose found a found a container named %s without any " + "labels. As of compose 1.3.0 containers are identified with " + "labels instead of naming convention. If you'd like compose " + "to use this container, please run " + "`docker-compose migrate-to-labels`" % (name,)) + + +def get_legacy_container_names( + client, + project, + services, + stopped=False, + one_off=False): for container in client.containers(all=stopped): name = get_container_name(container) for service in services: prefix = '%s_%s_%s' % (project, service, 'run_' if one_off else '') - if not name.startswith(prefix): - continue - - log.warn( - "Compose found a found a container named %s without any " - "labels. As of compose 1.3.0 containers are identified with " - "labels instead of naming convention. If you'd like compose " - "to use this container, please run " - "`docker-compose migrate-to-labels`" % (name,)) + if name.startswith(prefix): + yield name def add_labels(project, container, name): diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index d39635b7f..85cc40320 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -7,24 +7,41 @@ from .testcases import DockerClientTestCase class ProjectTest(DockerClientTestCase): - def test_migration_to_labels(self): - services = [ + def setUp(self): + super(ProjectTest, self).setUp() + + self.services = [ self.create_service('web'), self.create_service('db'), ] - project = Project('composetest', services, self.client) + self.project = Project('composetest', self.services, self.client) - for service in services: + for service in self.services: service.ensure_image_exists() self.client.create_container( - name='{}_{}_1'.format(project.name, service.name), + name='{}_{}_1'.format(self.project.name, service.name), **service.options ) - with mock.patch.object(legacy, 'log', autospec=True) as mock_log: - self.assertEqual(project.containers(stopped=True), []) - self.assertEqual(mock_log.warn.call_count, 2) + def get_names(self, **kwargs): + if 'stopped' not in kwargs: + kwargs['stopped'] = True - legacy.migrate_project_to_labels(project) - self.assertEqual(len(project.containers(stopped=True)), 2) + return list(legacy.get_legacy_container_names( + self.client, + self.project.name, + [s.name for s in self.services], + **kwargs + )) + + def test_get_legacy_container_names(self): + self.assertEqual(len(self.get_names()), len(self.services)) + + def test_migration_to_labels(self): + with mock.patch.object(legacy, 'log', autospec=True) as mock_log: + self.assertEqual(self.project.containers(stopped=True), []) + self.assertEqual(mock_log.warn.call_count, len(self.services)) + + legacy.migrate_project_to_labels(self.project) + self.assertEqual(len(self.project.containers(stopped=True)), len(self.services)) From 051f56a1e6994fdbe78720000916d18afe59ea0d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 May 2015 12:12:30 +0100 Subject: [PATCH 082/118] Fix bugs with one-off legacy containers - One-off containers were included in the warning log messages, which can make for unreadable output when there are lots (as there often are). - Compose was attempting to recreate one-off containers as normal containers when migrating. Fixed by implementing the exact naming logic from before we used labels. Signed-off-by: Aanand Prasad --- compose/legacy.py | 56 +++++++++++++++++++++----------- tests/integration/legacy_test.py | 10 ++++++ 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/compose/legacy.py b/compose/legacy.py index 7f97c3e94..8deabfa24 100644 --- a/compose/legacy.py +++ b/compose/legacy.py @@ -11,11 +11,6 @@ log = logging.getLogger(__name__) NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$') -def is_valid_name(name): - match = NAME_RE.match(name) - return match is not None - - def check_for_legacy_containers( client, project, @@ -42,20 +37,6 @@ def check_for_legacy_containers( "`docker-compose migrate-to-labels`" % (name,)) -def get_legacy_container_names( - client, - project, - services, - stopped=False, - one_off=False): - for container in client.containers(all=stopped): - name = get_container_name(container) - for service in services: - prefix = '%s_%s_%s' % (project, service, 'run_' if one_off else '') - if name.startswith(prefix): - yield name - - def add_labels(project, container, name): project_name, service_name, one_off, number = NAME_RE.match(name).groups() if project_name != project.name or service_name not in project.service_names: @@ -73,3 +54,40 @@ def migrate_project_to_labels(project): if not is_valid_name(name): continue add_labels(project, Container.from_ps(client, container), name) + + +def get_legacy_container_names( + client, + project, + services, + stopped=False, + one_off=False): + + for container in client.containers(all=stopped): + name = get_container_name(container) + for service in services: + if has_container(project, service, name, one_off=one_off): + yield name + + +def has_container(project, service, name, one_off=False): + if not name or not is_valid_name(name, one_off): + return False + container_project, container_service, _container_number = parse_name(name) + return container_project == project and container_service == service + + +def is_valid_name(name, one_off=False): + match = NAME_RE.match(name) + if match is None: + return False + if one_off: + return match.group(3) == 'run_' + else: + return match.group(3) is None + + +def parse_name(name): + match = NAME_RE.match(name) + (project, service_name, _, suffix) = match.groups() + return (project, service_name, int(suffix)) diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index 85cc40320..8b0a9b7fc 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -17,6 +17,7 @@ class ProjectTest(DockerClientTestCase): self.project = Project('composetest', self.services, self.client) + # Create a legacy container for each service for service in self.services: service.ensure_image_exists() self.client.create_container( @@ -24,6 +25,12 @@ class ProjectTest(DockerClientTestCase): **service.options ) + # Create a single one-off legacy container + self.client.create_container( + name='{}_{}_run_1'.format(self.project.name, self.services[0].name), + **self.services[0].options + ) + def get_names(self, **kwargs): if 'stopped' not in kwargs: kwargs['stopped'] = True @@ -38,6 +45,9 @@ class ProjectTest(DockerClientTestCase): def test_get_legacy_container_names(self): self.assertEqual(len(self.get_names()), len(self.services)) + def test_get_legacy_container_names_one_off(self): + self.assertEqual(len(self.get_names(one_off=True)), 1) + def test_migration_to_labels(self): with mock.patch.object(legacy, 'log', autospec=True) as mock_log: self.assertEqual(self.project.containers(stopped=True), []) From 30c9e7323a1fd0e8f20207860bd9bfd02e4b4e83 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 May 2015 20:05:55 +0100 Subject: [PATCH 083/118] Fix missing logging on container creation Signed-off-by: Aanand Prasad --- compose/service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compose/service.py b/compose/service.py index e10758574..0b1a1453c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -215,6 +215,9 @@ class Service(object): previous_container=previous_container, ) + if 'name' in container_options: + log.info("Creating %s..." % container_options['name']) + return Container.create(self.client, **container_options) def ensure_image_exists(self, From 412034a023b846070a847f92ad952315603d461c Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Wed, 20 May 2015 15:31:10 +0200 Subject: [PATCH 084/118] bash completion for migrate-to-labels Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index ec0f23481..e62b1d8fc 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -311,6 +311,7 @@ _docker-compose() { help kill logs + migrate-to-labels port ps pull From 0b4d9401ee0143c9cba3d25017a35c27c362475b Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 May 2015 12:41:36 +0100 Subject: [PATCH 085/118] Bail out immediately if there are legacy containers Signed-off-by: Aanand Prasad --- compose/cli/main.py | 2 +- compose/legacy.py | 42 +++++++++++++++++++++++++------- tests/integration/legacy_test.py | 10 +++++--- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 2b95040ca..ff30d9701 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -33,7 +33,7 @@ def main(): except KeyboardInterrupt: log.error("\nAborting.") sys.exit(1) - except (UserError, NoSuchService, ConfigurationError) as e: + except (UserError, NoSuchService, ConfigurationError, legacy.LegacyContainersError) as e: log.error(e.msg) sys.exit(1) except NoSuchCommand as e: diff --git a/compose/legacy.py b/compose/legacy.py index 8deabfa24..af0c8700b 100644 --- a/compose/legacy.py +++ b/compose/legacy.py @@ -10,6 +10,20 @@ log = logging.getLogger(__name__) # TODO: remove this section when migrate_project_to_labels is removed NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$') +ERROR_MESSAGE_FORMAT = """ +Compose found the following containers without labels: + +{names_list} + +As of Compose 1.3.0, containers are identified with labels instead of naming convention. If you want to continue using these containers, run: + + $ docker-compose migrate-to-labels + +Alternatively, remove them: + + $ docker rm -f {rm_args} +""" + def check_for_legacy_containers( client, @@ -21,20 +35,30 @@ def check_for_legacy_containers( and warn the user that those containers may need to be migrated to using labels, so that compose can find them. """ - names = get_legacy_container_names( + names = list(get_legacy_container_names( client, project, services, stopped=stopped, - one_off=one_off) + one_off=one_off)) - for name in names: - log.warn( - "Compose found a found a container named %s without any " - "labels. As of compose 1.3.0 containers are identified with " - "labels instead of naming convention. If you'd like compose " - "to use this container, please run " - "`docker-compose migrate-to-labels`" % (name,)) + if names: + raise LegacyContainersError(names) + + +class LegacyContainersError(Exception): + def __init__(self, names): + self.names = names + + self.msg = ERROR_MESSAGE_FORMAT.format( + names_list="\n".join(" {}".format(name) for name in names), + rm_args=" ".join(names), + ) + + def __unicode__(self): + return self.msg + + __str__ = __unicode__ def add_labels(project, container, name): diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index 8b0a9b7fc..f3c33e600 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -1,5 +1,3 @@ -import mock - from compose import legacy from compose.project import Project from .testcases import DockerClientTestCase @@ -49,9 +47,13 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(self.get_names(one_off=True)), 1) def test_migration_to_labels(self): - with mock.patch.object(legacy, 'log', autospec=True) as mock_log: + with self.assertRaises(legacy.LegacyContainersError) as cm: self.assertEqual(self.project.containers(stopped=True), []) - self.assertEqual(mock_log.warn.call_count, len(self.services)) + + self.assertEqual( + set(cm.exception.names), + set(['composetest_web_1', 'composetest_db_1']), + ) legacy.migrate_project_to_labels(self.project) self.assertEqual(len(self.project.containers(stopped=True)), len(self.services)) From 91ceb33d5a70ba54fcc87dd934d35daba77aa40c Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 26 May 2015 15:41:59 +0100 Subject: [PATCH 086/118] Update description of Compose "Define and run multi-container applications with Docker" Not just development environments, and "complex" is not clear and not really true. Signed-off-by: Ben Firshman --- README.md | 9 ++++----- compose/cli/main.py | 2 +- docs/index.md | 10 +++++----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 522488e7a..acd3cbe7a 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ Docker Compose ============== *(Previously known as Fig)* -Compose is a tool for defining and running complex applications with Docker. -With Compose, you define a multi-container application in a single file, then -spin your application up in a single command which does everything that needs to -be done to get it running. +Compose is a tool for defining and running multi-container applications with +Docker. With Compose, you define a multi-container application in a single +file, then spin your application up in a single command which does everything +that needs to be done to get it running. Compose is great for development environments, staging servers, and CI. We don't recommend that you use it in production yet. @@ -50,4 +50,3 @@ Contributing [![Build Status](http://jenkins.dockerproject.com/buildStatus/icon?job=Compose Master)](http://jenkins.dockerproject.com/job/Compose%20Master/) Want to help build Compose? Check out our [contributing documentation](https://github.com/docker/compose/blob/master/CONTRIBUTING.md). - diff --git a/compose/cli/main.py b/compose/cli/main.py index a2dca65db..00fe3115c 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -72,7 +72,7 @@ def parse_doc_section(name, source): class TopLevelCommand(Command): - """Fast, isolated development environments using Docker. + """Define and run multi-container applications with Docker. Usage: docker-compose [options] [COMMAND] [ARGS...] diff --git a/docs/index.md b/docs/index.md index 44f56ae96..981a02702 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,10 +7,10 @@ page_keywords: documentation, docs, docker, compose, orchestration, containers ## Overview -Compose is a tool for defining and running complex applications with Docker. -With Compose, you define a multi-container application in a single file, then -spin your application up in a single command which does everything that needs to -be done to get it running. +Compose is a tool for defining and running multi-container applications with +Docker. With Compose, you define a multi-container application in a single +file, then spin your application up in a single command which does everything +that needs to be done to get it running. Compose is great for development environments, staging servers, and CI. We don't recommend that you use it in production yet. @@ -200,7 +200,7 @@ At this point, you have seen the basics of how Compose works. [Rails](rails.md), or [Wordpress](wordpress.md). - See the reference guides for complete details on the [commands](cli.md), the [configuration file](yml.md) and [environment variables](env.md). - + ## Release Notes ### Version 1.2.0 (April 7, 2015) From 4795fd874f50254af3e9fb6cb60b25410b8456ab Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 May 2015 20:03:02 +0100 Subject: [PATCH 087/118] Fix regression in `docker-compose up` When an upstream dependency (e.g. a db) has a container but a downstream service (e.g. a web app) doesn't, a web container is not created on `docker-compose up`. Signed-off-by: Aanand Prasad --- compose/project.py | 7 +++++-- compose/service.py | 4 ---- tests/integration/project_test.py | 12 ++++++++++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/compose/project.py b/compose/project.py index 6dc926681..d3deeeaf9 100644 --- a/compose/project.py +++ b/compose/project.py @@ -248,10 +248,13 @@ class Project(object): if updated_dependencies: log.debug( - '%s has not changed but its dependencies (%s) have, so recreating', + '%s has upstream changes (%s)', service.name, ", ".join(updated_dependencies), ) - plan = service.recreate_plan() + plan = service.convergence_plan( + allow_recreate=allow_recreate, + smart_recreate=False, + ) else: plan = service.convergence_plan( allow_recreate=allow_recreate, diff --git a/compose/service.py b/compose/service.py index c45a8bdfc..ccfb38511 100644 --- a/compose/service.py +++ b/compose/service.py @@ -294,10 +294,6 @@ class Service(object): return ConvergencePlan('recreate', containers) - def recreate_plan(self): - containers = self.containers(stopped=True) - return ConvergencePlan('recreate', containers) - def _containers_have_diverged(self, containers): config_hash = self.config_hash() has_diverged = False diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 5e3a40e5b..2976af823 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -174,6 +174,18 @@ class ProjectTest(DockerClientTestCase): project.kill() project.remove_stopped() + def test_project_up_starts_uncreated_services(self): + db = self.create_service('db') + web = self.create_service('web', links=[(db, 'db')]) + project = Project('composetest', [db, web], self.client) + project.up(['db']) + self.assertEqual(len(project.containers()), 1) + + project.up() + self.assertEqual(len(project.containers()), 2) + self.assertEqual(len(db.containers()), 1) + self.assertEqual(len(web.containers()), 1) + def test_project_up_recreates_containers(self): web = self.create_service('web') db = self.create_service('db', volumes=['/etc']) From 7da8e6be3b27bbd55a2c818ad1fc4a3bc07ac20a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 May 2015 16:09:06 +0100 Subject: [PATCH 088/118] Migrate containers in dependency order This fixes a bug where migration would fail with an error if a downstream container was migrated before its upstream dependencies, due to `check_for_legacy_containers()` being implicitly called when we fetch `links`, `volumes_from` or `net` dependencies. Signed-off-by: Aanand Prasad --- compose/legacy.py | 37 ++++++++++++++++++-------------- tests/integration/legacy_test.py | 24 ++++++++++----------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/compose/legacy.py b/compose/legacy.py index af0c8700b..340511a76 100644 --- a/compose/legacy.py +++ b/compose/legacy.py @@ -35,15 +35,15 @@ def check_for_legacy_containers( and warn the user that those containers may need to be migrated to using labels, so that compose can find them. """ - names = list(get_legacy_container_names( + containers = list(get_legacy_containers( client, project, services, stopped=stopped, one_off=one_off)) - if names: - raise LegacyContainersError(names) + if containers: + raise LegacyContainersError([c.name for c in containers]) class LegacyContainersError(Exception): @@ -61,8 +61,8 @@ class LegacyContainersError(Exception): __str__ = __unicode__ -def add_labels(project, container, name): - project_name, service_name, one_off, number = NAME_RE.match(name).groups() +def add_labels(project, container): + project_name, service_name, one_off, number = NAME_RE.match(container.name).groups() if project_name != project.name or service_name not in project.service_names: return service = project.get_service(service_name) @@ -72,26 +72,31 @@ def add_labels(project, container, name): def migrate_project_to_labels(project): log.info("Running migration to labels for project %s", project.name) - client = project.client - for container in client.containers(all=True): - name = get_container_name(container) - if not is_valid_name(name): - continue - add_labels(project, Container.from_ps(client, container), name) + containers = get_legacy_containers( + project.client, + project.name, + project.service_names, + stopped=True, + one_off=False) + + for container in containers: + add_labels(project, container) -def get_legacy_container_names( +def get_legacy_containers( client, project, services, stopped=False, one_off=False): - for container in client.containers(all=stopped): - name = get_container_name(container) - for service in services: + containers = client.containers(all=stopped) + + for service in services: + for container in containers: + name = get_container_name(container) if has_container(project, service, name, one_off=one_off): - yield name + yield Container.from_ps(client, container) def has_container(project, service, name, one_off=False): diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index f3c33e600..6c52b68d3 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -8,20 +8,21 @@ class ProjectTest(DockerClientTestCase): def setUp(self): super(ProjectTest, self).setUp() - self.services = [ - self.create_service('web'), - self.create_service('db'), - ] + db = self.create_service('db') + web = self.create_service('web', links=[(db, 'db')]) + nginx = self.create_service('nginx', links=[(web, 'web')]) + self.services = [db, web, nginx] self.project = Project('composetest', self.services, self.client) # Create a legacy container for each service for service in self.services: service.ensure_image_exists() - self.client.create_container( + container = self.client.create_container( name='{}_{}_1'.format(self.project.name, service.name), **service.options ) + self.client.start(container) # Create a single one-off legacy container self.client.create_container( @@ -29,11 +30,8 @@ class ProjectTest(DockerClientTestCase): **self.services[0].options ) - def get_names(self, **kwargs): - if 'stopped' not in kwargs: - kwargs['stopped'] = True - - return list(legacy.get_legacy_container_names( + def get_legacy_containers(self, **kwargs): + return list(legacy.get_legacy_containers( self.client, self.project.name, [s.name for s in self.services], @@ -41,10 +39,10 @@ class ProjectTest(DockerClientTestCase): )) def test_get_legacy_container_names(self): - self.assertEqual(len(self.get_names()), len(self.services)) + self.assertEqual(len(self.get_legacy_containers()), len(self.services)) def test_get_legacy_container_names_one_off(self): - self.assertEqual(len(self.get_names(one_off=True)), 1) + self.assertEqual(len(self.get_legacy_containers(stopped=True, one_off=True)), 1) def test_migration_to_labels(self): with self.assertRaises(legacy.LegacyContainersError) as cm: @@ -52,7 +50,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual( set(cm.exception.names), - set(['composetest_web_1', 'composetest_db_1']), + set(['composetest_db_1', 'composetest_web_1', 'composetest_nginx_1']), ) legacy.migrate_project_to_labels(self.project) From 686c25d50ff3822e0f1515cf6aa0c13de97a4368 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 27 May 2015 15:13:12 +0100 Subject: [PATCH 089/118] Script to prepare OSX build environment Signed-off-by: Aanand Prasad --- CONTRIBUTING.md | 12 ++++++++---- script/build-osx | 2 +- script/prepare-osx | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) create mode 100755 script/prepare-osx diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 373c8dc6f..fddf888dc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,16 +53,20 @@ you can specify a test directory, file, module, class or method: ## Building binaries -Linux: +`script/build-linux` will build the Linux binary inside a Docker container: $ script/build-linux -OS X: +`script/build-osx` will build the Mac OS X binary inside a virtualenv: $ script/build-osx -Note that this only works on Mountain Lion, not Mavericks, due to a -[bug in PyInstaller](http://www.pyinstaller.org/ticket/807). +For official releases, you should build inside a Mountain Lion VM for proper +compatibility. Run the this script first to prepare the environment before +building - it will use Homebrew to make sure Python is installed and +up-to-date. + + $ script/prepare-osx ## Release process diff --git a/script/build-osx b/script/build-osx index 26309744a..6ad00bcdb 100755 --- a/script/build-osx +++ b/script/build-osx @@ -1,7 +1,7 @@ #!/bin/bash set -ex rm -rf venv -virtualenv venv +virtualenv -p /usr/local/bin/python venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-dev.txt venv/bin/pip install . diff --git a/script/prepare-osx b/script/prepare-osx new file mode 100755 index 000000000..69ac56f1c --- /dev/null +++ b/script/prepare-osx @@ -0,0 +1,22 @@ +#!/bin/bash + +set -ex + +if !(which brew); then + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +fi + +brew update + +if [ ! -f /usr/local/bin/python ]; then + brew install python +fi + +if [ -n "$(brew outdated | grep python)" ]; then + brew upgrade python +fi + +if !(which virtualenv); then + pip install virtualenv +fi + From 93a846db318bbf7e332db39f0ed7a764053948d6 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 28 May 2015 17:18:04 +0100 Subject: [PATCH 090/118] Report Python and OpenSSL versions in --version output Signed-off-by: Aanand Prasad Conflicts: compose/cli/utils.py --- compose/cli/main.py | 5 ++--- compose/cli/utils.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index a558e8359..61f3ec3f9 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -10,7 +10,6 @@ import sys from docker.errors import APIError import dockerpty -from .. import __version__ from .. import legacy from ..project import NoSuchService, ConfigurationError from ..service import BuildError, CannotBeScaledError, NeedsBuildError @@ -20,7 +19,7 @@ from .docopt_command import NoSuchCommand from .errors import UserError from .formatter import Formatter from .log_printer import LogPrinter -from .utils import yesno +from .utils import get_version_info, yesno log = logging.getLogger(__name__) @@ -104,7 +103,7 @@ class TopLevelCommand(Command): """ def docopt_options(self): options = super(TopLevelCommand, self).docopt_options() - options['version'] = "docker-compose %s" % __version__ + options['version'] = get_version_info() return options def build(self, project, options): diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 5f5fed64e..93b991038 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -5,6 +5,9 @@ import datetime import os import subprocess import platform +import ssl + +from .. import __version__ def yesno(prompt, default=None): @@ -120,3 +123,11 @@ def is_mac(): def is_ubuntu(): return platform.system() == 'Linux' and platform.linux_distribution()[0] == 'Ubuntu' + + +def get_version_info(): + return '\n'.join([ + 'docker-compose version: %s' % __version__, + "%s version: %s" % (platform.python_implementation(), platform.python_version()), + "OpenSSL version: %s" % ssl.OPENSSL_VERSION, + ]) From f3d0c63db2621a7bbe77164a23d11d3530bd5d19 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 28 May 2015 17:24:03 +0100 Subject: [PATCH 091/118] Make sure we use Python 2.7.9 and OpenSSL 1.0.1 when building OSX binary Signed-off-by: Aanand Prasad --- script/build-osx | 3 +++ script/prepare-osx | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/script/build-osx b/script/build-osx index 6ad00bcdb..d6561aeea 100755 --- a/script/build-osx +++ b/script/build-osx @@ -1,5 +1,8 @@ #!/bin/bash set -ex + +PATH="/usr/local/bin:$PATH" + rm -rf venv virtualenv -p /usr/local/bin/python venv venv/bin/pip install -r requirements.txt diff --git a/script/prepare-osx b/script/prepare-osx index 69ac56f1c..ca2776b64 100755 --- a/script/prepare-osx +++ b/script/prepare-osx @@ -2,20 +2,51 @@ set -ex +python_version() { + python -V 2>&1 +} + +openssl_version() { + python -c "import ssl; print ssl.OPENSSL_VERSION" +} + +desired_python_version="2.7.9" +desired_python_brew_version="2.7.9" +python_formula="https://raw.githubusercontent.com/Homebrew/homebrew/1681e193e4d91c9620c4901efd4458d9b6fcda8e/Library/Formula/python.rb" + +desired_openssl_version="1.0.1j" +desired_openssl_brew_version="1.0.1j_1" +openssl_formula="https://raw.githubusercontent.com/Homebrew/homebrew/62fc2a1a65e83ba9dbb30b2e0a2b7355831c714b/Library/Formula/openssl.rb" + +PATH="/usr/local/bin:$PATH" + if !(which brew); then ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi brew update -if [ ! -f /usr/local/bin/python ]; then - brew install python +if !(python_version | grep "$desired_python_version"); then + if brew list | grep python; then + brew unlink python + fi + + brew install "$python_formula" + brew switch python "$desired_python_brew_version" fi -if [ -n "$(brew outdated | grep python)" ]; then - brew upgrade python +if !(openssl_version | grep "$desired_openssl_version"); then + if brew list | grep openssl; then + brew unlink openssl + fi + + brew install "$openssl_formula" + brew switch openssl "$desired_openssl_brew_version" fi +echo "*** Using $(python_version)" +echo "*** Using $(openssl_version)" + if !(which virtualenv); then pip install virtualenv fi From 8749bc08443ddb344ecf683e796cdb2d814b7f68 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 1 Jun 2015 14:01:30 +0100 Subject: [PATCH 092/118] Build Python 2.7.9 in Docker image Signed-off-by: Aanand Prasad --- Dockerfile | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b2ae0063c..fca5f9803 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,11 @@ FROM debian:wheezy RUN set -ex; \ apt-get update -qq; \ apt-get install -y \ - python \ - python-pip \ - python-dev \ + gcc \ + make \ + zlib1g \ + zlib1g-dev \ + libssl-dev \ git \ apt-transport-https \ ca-certificates \ @@ -15,6 +17,37 @@ RUN set -ex; \ ; \ rm -rf /var/lib/apt/lists/* +# Build Python 2.7.9 from source +RUN set -ex; \ + curl -LO https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz; \ + tar -xzf Python-2.7.9.tgz; \ + cd Python-2.7.9; \ + ./configure --enable-shared; \ + make; \ + make install; \ + cd ..; \ + rm -rf /Python-2.7.9; \ + rm Python-2.7.9.tgz + +# Make libpython findable +ENV LD_LIBRARY_PATH /usr/local/lib + +# Install setuptools +RUN set -ex; \ + curl -LO https://bootstrap.pypa.io/ez_setup.py; \ + python ez_setup.py; \ + rm ez_setup.py + +# Install pip +RUN set -ex; \ + curl -LO https://pypi.python.org/packages/source/p/pip/pip-7.0.1.tar.gz; \ + tar -xzf pip-7.0.1.tar.gz; \ + cd pip-7.0.1; \ + python setup.py install; \ + cd ..; \ + rm -rf pip-7.0.1; \ + rm pip-7.0.1.tar.gz + ENV ALL_DOCKER_VERSIONS 1.6.0 RUN set -ex; \ From 5a5bffebd178670e602e2e9ea8c177bc32ef62b5 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 27 May 2015 12:49:58 +0100 Subject: [PATCH 093/118] Merge pull request #1464 from twhiteman/bug1461 Possible division by zero error when pulling an image - fixes #1463 (cherry picked from commit d0e87929a1f39b4e98c2c8497f3f0ffc09fb9e43) Signed-off-by: Aanand Prasad --- compose/progress_stream.py | 5 +++-- tests/unit/progress_stream_test.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 39aab5ff7..317c6e815 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -74,8 +74,9 @@ def print_output_event(event, stream, is_terminal): stream.write("%s %s%s" % (status, event['progress'], terminator)) elif 'progressDetail' in event: detail = event['progressDetail'] - if 'current' in detail: - percentage = float(detail['current']) / float(detail['total']) * 100 + total = detail.get('total') + if 'current' in detail and total: + percentage = float(detail['current']) / float(total) * 100 stream.write('%s (%.1f%%)%s' % (status, percentage, terminator)) else: stream.write('%s%s' % (status, terminator)) diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index 142560681..317b77e9f 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -17,3 +17,21 @@ class ProgressStreamTestCase(unittest.TestCase): ] events = progress_stream.stream_output(output, StringIO()) self.assertEqual(len(events), 1) + + def test_stream_output_div_zero(self): + output = [ + '{"status": "Downloading", "progressDetail": {"current": ' + '0, "start": 1413653874, "total": 0}, ' + '"progress": "..."}', + ] + events = progress_stream.stream_output(output, StringIO()) + self.assertEqual(len(events), 1) + + def test_stream_output_null_total(self): + output = [ + '{"status": "Downloading", "progressDetail": {"current": ' + '0, "start": 1413653874, "total": null}, ' + '"progress": "..."}', + ] + events = progress_stream.stream_output(output, StringIO()) + self.assertEqual(len(events), 1) From 4f4ea2a402a42c29c9867b02287dd7ded2d5b0d0 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Fri, 29 May 2015 14:45:21 +0100 Subject: [PATCH 094/118] Merge pull request #1325 from sdurrheimer/master Zsh completion for docker-compose (cherry picked from commit b638728d6ca21982e321b4069ef92f8367f069f4) Signed-off-by: Aanand Prasad Conflicts: docs/completion.md --- CONTRIBUTING.md | 1 - contrib/completion/zsh/_docker-compose | 304 +++++++++++++++++++++++++ docs/completion.md | 37 ++- 3 files changed, 333 insertions(+), 9 deletions(-) create mode 100644 contrib/completion/zsh/_docker-compose diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fddf888dc..6914e2159 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,6 @@ up-to-date. 1. Open pull request that: - Updates the version in `compose/__init__.py` - Updates the binary URL in `docs/install.md` - - Updates the script URL in `docs/completion.md` - Adds release notes to `CHANGES.md` 2. Create unpublished GitHub release with release notes 3. Build Linux version on any Docker host with `script/build-linux` and attach diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose new file mode 100644 index 000000000..31052e1e0 --- /dev/null +++ b/contrib/completion/zsh/_docker-compose @@ -0,0 +1,304 @@ +#compdef docker-compose + +# Description +# ----------- +# zsh completion for docker-compose +# https://github.com/sdurrheimer/docker-compose-zsh-completion +# ------------------------------------------------------------------------- +# Version +# ------- +# 0.1.0 +# ------------------------------------------------------------------------- +# Authors +# ------- +# * Steve Durrheimer +# ------------------------------------------------------------------------- +# Inspiration +# ----------- +# * @albers docker-compose bash completion script +# * @felixr docker zsh completion script : https://github.com/felixr/docker-zsh-completion +# ------------------------------------------------------------------------- + +# For compatibility reasons, Compose and therefore its completion supports several +# stack compositon files as listed here, in descending priority. +# Support for these filenames might be dropped in some future version. +__docker-compose_compose_file() { + local file + for file in docker-compose.y{,a}ml fig.y{,a}ml ; do + [ -e $file ] && { + echo $file + return + } + done + echo docker-compose.yml +} + +# Extracts all service names from docker-compose.yml. +___docker-compose_all_services_in_compose_file() { + local already_selected + local -a services + already_selected=$(echo ${words[@]} | tr " " "|") + awk -F: '/^[a-zA-Z0-9]/{print $1}' "${compose_file:-$(__docker-compose_compose_file)}" 2>/dev/null | grep -Ev "$already_selected" +} + +# All services, even those without an existing container +__docker-compose_services_all() { + services=$(___docker-compose_all_services_in_compose_file) + _alternative "args:services:($services)" +} + +# All services that have an entry with the given key in their docker-compose.yml section +___docker-compose_services_with_key() { + local already_selected + local -a buildable + already_selected=$(echo ${words[@]} | tr " " "|") + # flatten sections to one line, then filter lines containing the key and return section name. + awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' "${compose_file:-$(__docker-compose_compose_file)}" 2>/dev/null | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' 2>/dev/null | grep -Ev "$already_selected" +} + +# All services that are defined by a Dockerfile reference +__docker-compose_services_from_build() { + buildable=$(___docker-compose_services_with_key build) + _alternative "args:buildable services:($buildable)" +} + +# All services that are defined by an image +__docker-compose_services_from_image() { + pullable=$(___docker-compose_services_with_key image) + _alternative "args:pullable services:($pullable)" +} + +__docker-compose_get_services() { + local kind expl + declare -a running stopped lines args services + + docker_status=$(docker ps > /dev/null 2>&1) + if [ $? -ne 0 ]; then + _message "Error! Docker is not running." + return 1 + fi + + kind=$1 + shift + [[ $kind = (stopped|all) ]] && args=($args -a) + + lines=(${(f)"$(_call_program commands docker ps ${args})"}) + services=(${(f)"$(_call_program commands docker-compose 2>/dev/null ${compose_file:+-f $compose_file} ${compose_project:+-p $compose_project} ps -q)"}) + + # Parse header line to find columns + local i=1 j=1 k header=${lines[1]} + declare -A begin end + while (( $j < ${#header} - 1 )) { + i=$(( $j + ${${header[$j,-1]}[(i)[^ ]]} - 1)) + j=$(( $i + ${${header[$i,-1]}[(i) ]} - 1)) + k=$(( $j + ${${header[$j,-1]}[(i)[^ ]]} - 2)) + begin[${header[$i,$(($j-1))]}]=$i + end[${header[$i,$(($j-1))]}]=$k + } + lines=(${lines[2,-1]}) + + # Container ID + local line s name + local -a names + for line in $lines; do + if [[ $services == *"${line[${begin[CONTAINER ID]},${end[CONTAINER ID]}]%% ##}"* ]]; then + names=(${(ps:,:)${${line[${begin[NAMES]},-1]}%% *}}) + for name in $names; do + s="${${name%_*}#*_}:${(l:15:: :::)${${line[${begin[CREATED]},${end[CREATED]}]/ ago/}%% ##}}" + s="$s, ${line[${begin[CONTAINER ID]},${end[CONTAINER ID]}]%% ##}" + s="$s, ${${${line[$begin[IMAGE],$end[IMAGE]]}/:/\\:}%% ##}" + if [[ ${line[${begin[STATUS]},${end[STATUS]}]} = Exit* ]]; then + stopped=($stopped $s) + else + running=($running $s) + fi + done + fi + done + + [[ $kind = (running|all) ]] && _describe -t services-running "running services" running + [[ $kind = (stopped|all) ]] && _describe -t services-stopped "stopped services" stopped +} + +__docker-compose_stoppedservices() { + __docker-compose_get_services stopped "$@" +} + +__docker-compose_runningservices() { + __docker-compose_get_services running "$@" +} + +__docker-compose_services () { + __docker-compose_get_services all "$@" +} + +__docker-compose_caching_policy() { + oldp=( "$1"(Nmh+1) ) # 1 hour + (( $#oldp )) +} + +__docker-compose_commands () { + local cache_policy + + zstyle -s ":completion:${curcontext}:" cache-policy cache_policy + if [[ -z "$cache_policy" ]]; then + zstyle ":completion:${curcontext}:" cache-policy __docker-compose_caching_policy + fi + + if ( [[ ${+_docker_compose_subcommands} -eq 0 ]] || _cache_invalid docker_compose_subcommands) \ + && ! _retrieve_cache docker_compose_subcommands; + then + local -a lines + lines=(${(f)"$(_call_program commands docker-compose 2>&1)"}) + _docker_compose_subcommands=(${${${lines[$((${lines[(i)Commands:]} + 1)),${lines[(I) *]}]}## #}/ ##/:}) + _store_cache docker_compose_subcommands _docker_compose_subcommands + fi + _describe -t docker-compose-commands "docker-compose command" _docker_compose_subcommands +} + +__docker-compose_subcommand () { + local -a _command_args + integer ret=1 + case "$words[1]" in + (build) + _arguments \ + '--no-cache[Do not use cache when building the image]' \ + '*:services:__docker-compose_services_from_build' && ret=0 + ;; + (help) + _arguments ':subcommand:__docker-compose_commands' && ret=0 + ;; + (kill) + _arguments \ + '-s[SIGNAL to send to the container. Default signal is SIGKILL.]:signal:_signals' \ + '*:running services:__docker-compose_runningservices' && ret=0 + ;; + (logs) + _arguments \ + '--no-color[Produce monochrome output.]' \ + '*:services:__docker-compose_services_all' && ret=0 + ;; + (migrate-to-labels) + _arguments \ + '(-):Recreate containers to add labels' && ret=0 + ;; + (port) + _arguments \ + '--protocol=-[tcp or udap (defaults to tcp)]:protocol:(tcp udp)' \ + '--index=-[index of the container if there are mutiple instances of a service (defaults to 1)]:index: ' \ + '1:running services:__docker-compose_runningservices' \ + '2:port:_ports' && ret=0 + ;; + (ps) + _arguments \ + '-q[Only display IDs]' \ + '*:services:__docker-compose_services_all' && ret=0 + ;; + (pull) + _arguments \ + '--allow-insecure-ssl[Allow insecure connections to the docker registry]' \ + '*:services:__docker-compose_services_from_image' && ret=0 + ;; + (rm) + _arguments \ + '(-f --force)'{-f,--force}"[Don't ask to confirm removal]" \ + '-v[Remove volumes associated with containers]' \ + '*:stopped services:__docker-compose_stoppedservices' && ret=0 + ;; + (run) + _arguments \ + '--allow-insecure-ssl[Allow insecure connections to the docker registry]' \ + '-d[Detached mode: Run container in the background, print new container name.]' \ + '--entrypoint[Overwrite the entrypoint of the image.]:entry point: ' \ + '*-e[KEY=VAL Set an environment variable (can be used multiple times)]:environment variable KEY=VAL: ' \ + '(-u --user)'{-u,--user=-}'[Run as specified username or uid]:username or uid:_users' \ + "--no-deps[Don't start linked services.]" \ + '--rm[Remove container after run. Ignored in detached mode.]' \ + "--service-ports[Run command with the service's ports enabled and mapped to the host.]" \ + '-T[Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY.]' \ + '(-):services:__docker-compose_services' \ + '(-):command: _command_names -e' \ + '*::arguments: _normal' && ret=0 + ;; + (scale) + _arguments '*:running services:__docker-compose_runningservices' && ret=0 + ;; + (start) + _arguments '*:stopped services:__docker-compose_stoppedservices' && ret=0 + ;; + (stop|restart) + _arguments \ + '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ + '*:running services:__docker-compose_runningservices' && ret=0 + ;; + (up) + _arguments \ + '--allow-insecure-ssl[Allow insecure connections to the docker registry]' \ + '-d[Detached mode: Run containers in the background, print new container names.]' \ + '--no-color[Produce monochrome output.]' \ + "--no-deps[Don't start linked services.]" \ + "--no-recreate[If containers already exist, don't recreate them.]" \ + "--no-build[Don't build an image, even if it's missing]" \ + '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ + "--x-smart-recreate[Only recreate containers whose configuration or image needs to be updated. (EXPERIMENTAL)]" \ + '*:services:__docker-compose_services_all' && ret=0 + ;; + (*) + _message 'Unknown sub command' + esac + + return ret +} + +_docker-compose () { + # Support for subservices, which allows for `compdef _docker docker-shell=_docker_containers`. + # Based on /usr/share/zsh/functions/Completion/Unix/_git without support for `ret`. + if [[ $service != docker-compose ]]; then + _call_function - _$service + return + fi + + local curcontext="$curcontext" state line ret=1 + typeset -A opt_args + + _arguments -C \ + '(- :)'{-h,--help}'[Get help]' \ + '--verbose[Show more output]' \ + '(- :)'{-v,--version}'[Print version and exit]' \ + '(-f --file)'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ + '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ + '(-): :->command' \ + '(-)*:: :->option-or-argument' && ret=0 + + local counter=1 + #local compose_file compose_project + while [ $counter -lt ${#words[@]} ]; do + case "${words[$counter]}" in + -f|--file) + (( counter++ )) + compose_file="${words[$counter]}" + ;; + -p|--project-name) + (( counter++ )) + compose_project="${words[$counter]}" + ;; + *) + ;; + esac + (( counter++ )) + done + + case $state in + (command) + __docker-compose_commands && ret=0 + ;; + (option-or-argument) + curcontext=${curcontext%:*:*}:docker-compose-$words[1]: + __docker-compose_subcommand && ret=0 + ;; + esac + + return ret +} + +_docker-compose "$@" diff --git a/docs/completion.md b/docs/completion.md index 35c53b55f..5168971f8 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -3,23 +3,44 @@ layout: default title: Command Completion --- -#Command Completion +# Command Completion Compose comes with [command completion](http://en.wikipedia.org/wiki/Command-line_completion) -for the bash shell. +for the bash and zsh shell. -##Installing Command Completion +## Installing Command Completion + +### Bash Make sure bash completion is installed. If you use a current Linux in a non-minimal installation, bash completion should be available. On a Mac, install with `brew install bash-completion` - -Place the completion script in `/etc/bash_completion.d/` (`/usr/local/etc/bash_completion.d/` on a Mac), using e.g. - curl -L https://raw.githubusercontent.com/docker/compose/1.2.0/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose - +Place the completion script in `/etc/bash_completion.d/` (`/usr/local/etc/bash_completion.d/` on a Mac), using e.g. + + curl -L https://raw.githubusercontent.com/docker/compose/$(docker-compose --version | awk '{print $2}')/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose + Completion will be available upon next login. -##Available completions +### Zsh + +Place the completion script in your `/path/to/zsh/completion`, using e.g. `~/.zsh/completion/` + + mkdir -p ~/.zsh/completion + curl -L https://raw.githubusercontent.com/docker/compose/$(docker-compose --version | awk '{print $2}')/contrib/completion/zsh/_docker-compose > ~/.zsh/completion/_docker-compose + +Include the directory in your `$fpath`, e.g. by adding in `~/.zshrc` + + fpath=(~/.zsh/completion $fpath) + +Make sure `compinit` is loaded or do it by adding in `~/.zshrc` + + autoload -Uz compinit && compinit -i + +Then reload your shell + + exec $SHELL -l + +## Available completions Depending on what you typed on the command line so far, it will complete From 631f5be02fdc087420989bf820345406e6bc0c7b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sat, 30 May 2015 09:01:39 -0500 Subject: [PATCH 095/118] Merge pull request #1481 from albers/completion-smart-recreate Support --x-smart-recreate in bash completion (cherry picked from commit 9a0bb325f2d1203b7aac915c3bfca4347cc93489) Signed-off-by: Aanand Prasad --- contrib/completion/bash/docker-compose | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index e62b1d8fc..ba3dff352 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -104,7 +104,7 @@ _docker-compose_docker-compose() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help -h --verbose --version --file -f --project-name -p" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help -h --verbose --version -v --file -f --project-name -p" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -293,7 +293,7 @@ _docker-compose_up() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--allow-insecure-ssl -d --no-build --no-color --no-deps --no-recreate -t --timeout" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--allow-insecure-ssl -d --no-build --no-color --no-deps --no-recreate -t --timeout --x-smart-recreate" -- "$cur" ) ) ;; *) __docker-compose_services_all From 8ed7dfef6fb8d3f6eeeb4c515315e9ae43baee29 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 8 Jun 2015 12:48:46 -0400 Subject: [PATCH 096/118] Merge pull request #1525 from aanand/fix-duplicate-logging Fix duplicate logging on up/run (cherry picked from commit e2b790f7328482591863e496de14c825fd3f8a23) Signed-off-by: Aanand Prasad --- compose/cli/main.py | 1 + compose/service.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 61f3ec3f9..fa4013161 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -335,6 +335,7 @@ class TopLevelCommand(Command): container_options['ports'] = [] container = service.create_container( + quiet=True, one_off=True, insecure_registry=insecure_registry, **container_options diff --git a/compose/service.py b/compose/service.py index ccfb38511..dd931beee 100644 --- a/compose/service.py +++ b/compose/service.py @@ -199,6 +199,7 @@ class Service(object): do_build=True, previous_container=None, number=None, + quiet=False, **override_options): """ Create a container for this service. If the image doesn't exist, attempt to pull @@ -216,7 +217,7 @@ class Service(object): previous_container=previous_container, ) - if 'name' in container_options: + if 'name' in container_options and not quiet: log.info("Creating %s..." % container_options['name']) return Container.create(self.client, **container_options) @@ -378,6 +379,7 @@ class Service(object): do_build=False, previous_container=container, number=container.labels.get(LABEL_CONTAINER_NUMBER), + quiet=True, ) self.start_container(new_container) container.remove() From dca3bbdea3eb9991d804cc8b9ac9de34a367b866 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 8 Jun 2015 16:21:02 -0400 Subject: [PATCH 097/118] Merge pull request #1527 from aanand/remove-logging-on-run-rm Remove logging on run --rm (cherry picked from commit 5578ccbb0113e285a20aeeee820c03766ef1ae6e) Signed-off-by: Aanand Prasad --- compose/cli/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index fa4013161..7fde4ebaa 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -348,7 +348,6 @@ class TopLevelCommand(Command): dockerpty.start(project.client, container.id, interactive=not options['-T']) exit_code = container.wait() if options['--rm']: - log.info("Removing %s..." % container.name) project.client.remove_container(container.id) sys.exit(exit_code) From 8212f1bd45ab36d41895fbbd45490cbb68170187 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 9 Jun 2015 18:21:14 -0400 Subject: [PATCH 098/118] Merge pull request #1529 from aanand/update-dockerpty Update dockerpty to 0.3.4 (cherry picked from commit 95b2eaac042bb761b4f94c35a1af539467714098) Signed-off-by: Aanand Prasad --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b93988480..d3909b766 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PyYAML==3.10 docker-py==1.2.2 -dockerpty==0.3.3 +dockerpty==0.3.4 docopt==0.6.1 requests==2.6.1 six==1.7.3 diff --git a/setup.py b/setup.py index 153275f69..9364f57f3 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ install_requires = [ 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', 'docker-py >= 1.2.2, < 1.3', - 'dockerpty >= 0.3.3, < 0.4', + 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', ] From 71514cb380c157bbd7c34ad26697dc0638783d79 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 9 Jun 2015 22:22:56 -0400 Subject: [PATCH 099/118] Merge pull request #1531 from aanand/test-crash-resilience Test that data volumes now survive a crash when recreating (cherry picked from commit 87c30ae6e48c2341593b03770089e3ff86108881) Signed-off-by: Aanand Prasad --- tests/integration/resilience_test.py | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/integration/resilience_test.py diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py new file mode 100644 index 000000000..8229e9d3c --- /dev/null +++ b/tests/integration/resilience_test.py @@ -0,0 +1,37 @@ +from __future__ import unicode_literals +from __future__ import absolute_import + +import mock + +from compose.project import Project +from .testcases import DockerClientTestCase + + +class ResilienceTest(DockerClientTestCase): + def test_recreate_fails(self): + db = self.create_service('db', volumes=['/var/db'], command='top') + project = Project('composetest', [db], self.client) + + container = db.create_container() + db.start_container(container) + host_path = container.get('Volumes')['/var/db'] + + project.up() + container = db.containers()[0] + self.assertEqual(container.get('Volumes')['/var/db'], host_path) + + with mock.patch('compose.service.Service.create_container', crash): + with self.assertRaises(Crash): + project.up() + + project.up() + container = db.containers()[0] + self.assertEqual(container.get('Volumes')['/var/db'], host_path) + + +class Crash(Exception): + pass + + +def crash(*args, **kwargs): + raise Crash() From ca14ed68f7060ffc6e7856a66e7d6f4d3e245a74 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 10 Jun 2015 13:03:55 -0400 Subject: [PATCH 100/118] Merge pull request #1533 from edmorley/update-b2d-shellinit-example Docs: Update boot2docker shellinit example to use 'eval' (cherry picked from commit 17e03b29f9381a10f08e551f0c88899b7961664f) Signed-off-by: Aanand Prasad --- docs/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.md b/docs/cli.md index e5594871d..162189481 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -155,7 +155,7 @@ By default, if there are existing containers for a service, `docker-compose up` Several environment variables are available for you to configure Compose's behaviour. Variables starting with `DOCKER_` are the same as those used to configure the -Docker command-line client. If you're using boot2docker, `$(boot2docker shellinit)` +Docker command-line client. If you're using boot2docker, `eval "$(boot2docker shellinit)"` will set them to their correct values. ### COMPOSE\_PROJECT\_NAME From ad4cc5d6dfc0718d44bbcb6497f3483fc05b09f4 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 10 Jun 2015 17:19:24 -0400 Subject: [PATCH 101/118] Merge pull request #1497 from aanand/use-1.7-rc1 Run tests against Docker 1.7 RC2 (cherry picked from commit 0e9ccd36f3c672902a5241f557ed81df19255ccc) Signed-off-by: Aanand Prasad --- Dockerfile | 6 ++++-- tests/integration/cli_test.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index fca5f9803..1ff2d3825 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,11 +48,13 @@ RUN set -ex; \ rm -rf pip-7.0.1; \ rm pip-7.0.1.tar.gz -ENV ALL_DOCKER_VERSIONS 1.6.0 +ENV ALL_DOCKER_VERSIONS 1.6.0 1.7.0-rc2 RUN set -ex; \ curl https://get.docker.com/builds/Linux/x86_64/docker-1.6.0 -o /usr/local/bin/docker-1.6.0; \ - chmod +x /usr/local/bin/docker-1.6.0 + chmod +x /usr/local/bin/docker-1.6.0; \ + curl https://test.docker.com/builds/Linux/x86_64/docker-1.7.0-rc2 -o /usr/local/bin/docker-1.7.0-rc2; \ + chmod +x /usr/local/bin/docker-1.7.0-rc2 # Set the default Docker to be run RUN ln -s /usr/local/bin/docker-1.6.0 /usr/local/bin/docker diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 92789363e..4d33808cd 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import sys import os +import shlex from six import StringIO from mock import patch @@ -240,8 +241,8 @@ class CLITestCase(DockerClientTestCase): service = self.project.get_service(name) container = service.containers(stopped=True, one_off=True)[0] self.assertEqual( - container.human_readable_command, - u'/bin/echo helloworld' + shlex.split(container.human_readable_command), + [u'/bin/echo', u'helloworld'], ) @patch('dockerpty.start') From b7e8770c4fe8f67c0f50fcb0d39094f5db7e8d3d Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 11 Jun 2015 21:54:53 +0100 Subject: [PATCH 102/118] Merge pull request #1538 from thieman/tnt-serivce-misspelled Correct misspelling of "Service" in an error message (cherry picked from commit bd246fb011aa6805d57eb31d641e3c072c072d63) Signed-off-by: Aanand Prasad --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index d3deeeaf9..bc093628c 100644 --- a/compose/project.py +++ b/compose/project.py @@ -171,7 +171,7 @@ class Project(object): try: net = Container.from_id(self.client, net_name) except APIError: - raise ConfigurationError('Serivce "%s" is trying to use the network of "%s", which is not the name of a service or container.' % (service_dict['name'], net_name)) + raise ConfigurationError('Service "%s" is trying to use the network of "%s", which is not the name of a service or container.' % (service_dict['name'], net_name)) else: net = service_dict['net'] From cd7f67018e9fcb4ff423e344dba55fd96289ce48 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 4 Jun 2015 16:21:01 +0100 Subject: [PATCH 103/118] Merge pull request #1466 from noironetworks/changing-scale-to-warning Modified scale awareness from exception to warning (cherry picked from commit 7d2a89427c59774a8cbf503a57cb9f3b0d47d1fe) Signed-off-by: Aanand Prasad --- compose/cli/main.py | 12 ++---------- compose/service.py | 8 +++----- tests/integration/service_test.py | 5 ----- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 7fde4ebaa..0c3b85e5c 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -12,7 +12,7 @@ import dockerpty from .. import legacy from ..project import NoSuchService, ConfigurationError -from ..service import BuildError, CannotBeScaledError, NeedsBuildError +from ..service import BuildError, NeedsBuildError from ..config import parse_environment from .command import Command from .docopt_command import NoSuchCommand @@ -371,15 +371,7 @@ class TopLevelCommand(Command): except ValueError: raise UserError('Number of containers for service "%s" is not a ' 'number' % service_name) - try: - project.get_service(service_name).scale(num) - except CannotBeScaledError: - raise UserError( - 'Service "%s" cannot be scaled because it specifies a port ' - 'on the host. If multiple containers for this service were ' - 'created, the port would clash.\n\nRemove the ":" from the ' - 'port definition in docker-compose.yml so Docker can choose a random ' - 'port for each container.' % service_name) + project.get_service(service_name).scale(num) def start(self, project, options): """ diff --git a/compose/service.py b/compose/service.py index dd931beee..5d0d171d8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -55,10 +55,6 @@ class BuildError(Exception): self.reason = reason -class CannotBeScaledError(Exception): - pass - - class ConfigError(ValueError): pass @@ -154,7 +150,9 @@ class Service(object): - removes all stopped containers """ if not self.can_be_scaled(): - raise CannotBeScaledError() + log.warn('Service %s specifies a port on the host. If multiple containers ' + 'for this service are created on a single host, the port will clash.' + % self.name) # Create enough containers containers = self.containers(stopped=True) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 8fd8212ce..7e88557f9 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -17,7 +17,6 @@ from compose.const import ( LABEL_VERSION, ) from compose.service import ( - CannotBeScaledError, ConfigError, Service, build_extra_hosts, @@ -526,10 +525,6 @@ class ServiceTest(DockerClientTestCase): service.scale(0) self.assertEqual(len(service.containers()), 0) - def test_scale_on_service_that_cannot_be_scaled(self): - service = self.create_service('web', ports=['8000:8000']) - self.assertRaises(CannotBeScaledError, lambda: service.scale(1)) - def test_scale_sets_ports(self): service = self.create_service('web', ports=['8000']) service.scale(2) From 59d6af73fa964b2e1ce65964561d21b409194724 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 12 Jun 2015 11:56:02 -0400 Subject: [PATCH 104/118] Merge pull request #1539 from bfirsh/add-image-affinity-to-test Add image affinity to test script (cherry picked from commit 4c2112dbfd4da219f2585569b716b59f7562b034) Signed-off-by: Aanand Prasad --- script/test | 1 + 1 file changed, 1 insertion(+) diff --git a/script/test b/script/test index ab0645fdc..700de7779 100755 --- a/script/test +++ b/script/test @@ -12,6 +12,7 @@ docker run \ --volume="$(pwd):/code" \ -e DOCKER_VERSIONS \ -e "TAG=$TAG" \ + -e "affinity:image==$TAG" \ --entrypoint="script/test-versions" \ "$TAG" \ "$@" From 363a6563c7ed80731908658e8cc9cf431885bb1b Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 11 Jun 2015 21:55:33 +0100 Subject: [PATCH 105/118] Merge pull request #1537 from aanand/reorder-service-utils Reorder service.py utility methods (cherry picked from commit e3525d64b55ba6b95adab54ac0b5baf22d7740e0) Signed-off-by: Aanand Prasad --- compose/service.py | 135 ++++++++++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 57 deletions(-) diff --git a/compose/service.py b/compose/service.py index 5d0d171d8..8b4115173 100644 --- a/compose/service.py +++ b/compose/service.py @@ -708,6 +708,47 @@ class Service(object): stream_output(output, sys.stdout) +# Names + + +def build_container_name(project, service, number, one_off=False): + bits = [project, service] + if one_off: + bits.append('run') + return '_'.join(bits + [str(number)]) + + +# Images + + +def parse_repository_tag(s): + if ":" not in s: + return s, "" + repo, tag = s.rsplit(":", 1) + if "/" in tag: + return s, "" + return repo, tag + + +# Volumes + + +def merge_volume_bindings(volumes_option, previous_container): + """Return a list of volume bindings for a container. Container data volumes + are replaced by those from the previous container. + """ + volume_bindings = dict( + build_volume_binding(parse_volume_spec(volume)) + for volume in volumes_option or [] + if ':' in volume) + + if previous_container: + volume_bindings.update( + get_container_data_volumes(previous_container, volumes_option)) + + return volume_bindings + + def get_container_data_volumes(container, volumes_option): """Find the container data volumes that are in `volumes_option`, and return a mapping of volume bindings for those volumes. @@ -736,51 +777,9 @@ def get_container_data_volumes(container, volumes_option): return dict(volumes) -def merge_volume_bindings(volumes_option, previous_container): - """Return a list of volume bindings for a container. Container data volumes - are replaced by those from the previous container. - """ - volume_bindings = dict( - build_volume_binding(parse_volume_spec(volume)) - for volume in volumes_option or [] - if ':' in volume) - - if previous_container: - volume_bindings.update( - get_container_data_volumes(previous_container, volumes_option)) - - return volume_bindings - - -def build_container_name(project, service, number, one_off=False): - bits = [project, service] - if one_off: - bits.append('run') - return '_'.join(bits + [str(number)]) - - -def build_container_labels(label_options, service_labels, number, one_off=False): - labels = label_options or {} - labels.update(label.split('=', 1) for label in service_labels) - labels[LABEL_CONTAINER_NUMBER] = str(number) - labels[LABEL_VERSION] = __version__ - return labels - - -def parse_restart_spec(restart_config): - if not restart_config: - return None - parts = restart_config.split(':') - if len(parts) > 2: - raise ConfigError("Restart %s has incorrect format, should be " - "mode[:max_retry]" % restart_config) - if len(parts) == 2: - name, max_retry_count = parts - else: - name, = parts - max_retry_count = 0 - - return {'Name': name, 'MaximumRetryCount': int(max_retry_count)} +def build_volume_binding(volume_spec): + internal = {'bind': volume_spec.internal, 'ro': volume_spec.mode == 'ro'} + return volume_spec.external, internal def parse_volume_spec(volume_config): @@ -803,18 +802,7 @@ def parse_volume_spec(volume_config): return VolumeSpec(external, internal, mode) -def parse_repository_tag(s): - if ":" not in s: - return s, "" - repo, tag = s.rsplit(":", 1) - if "/" in tag: - return s, "" - return repo, tag - - -def build_volume_binding(volume_spec): - internal = {'bind': volume_spec.internal, 'ro': volume_spec.mode == 'ro'} - return volume_spec.external, internal +# Ports def build_port_bindings(ports): @@ -845,6 +833,39 @@ def split_port(port): return internal_port, (external_ip, external_port or None) +# Labels + + +def build_container_labels(label_options, service_labels, number, one_off=False): + labels = label_options or {} + labels.update(label.split('=', 1) for label in service_labels) + labels[LABEL_CONTAINER_NUMBER] = str(number) + labels[LABEL_VERSION] = __version__ + return labels + + +# Restart policy + + +def parse_restart_spec(restart_config): + if not restart_config: + return None + parts = restart_config.split(':') + if len(parts) > 2: + raise ConfigError("Restart %s has incorrect format, should be " + "mode[:max_retry]" % restart_config) + if len(parts) == 2: + name, max_retry_count = parts + else: + name, = parts + max_retry_count = 0 + + return {'Name': name, 'MaximumRetryCount': int(max_retry_count)} + + +# Extra hosts + + def build_extra_hosts(extra_hosts_config): if not extra_hosts_config: return {} From 8f8693e13ed6c6a3fe518bc1928efa1d536e19e0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 29 May 2015 12:38:40 +0100 Subject: [PATCH 106/118] Merge pull request #1480 from bfirsh/change-sigint-test-to-use-sigstop Change kill SIGINT test to use SIGSTOP (cherry picked from commit a15f996744b4005441b289f6b3fb4eef551b5214) Signed-off-by: Aanand Prasad --- tests/integration/cli_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 4d33808cd..cb7bc17fc 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -361,22 +361,22 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(service.containers(stopped=True)), 1) self.assertFalse(service.containers(stopped=True)[0].is_running) - def test_kill_signal_sigint(self): + def test_kill_signal_sigstop(self): self.command.dispatch(['up', '-d'], None) service = self.project.get_service('simple') self.assertEqual(len(service.containers()), 1) self.assertTrue(service.containers()[0].is_running) - self.command.dispatch(['kill', '-s', 'SIGINT'], None) + self.command.dispatch(['kill', '-s', 'SIGSTOP'], None) self.assertEqual(len(service.containers()), 1) - # The container is still running. It has been only interrupted + # The container is still running. It has only been paused self.assertTrue(service.containers()[0].is_running) - def test_kill_interrupted_service(self): + def test_kill_stopped_service(self): self.command.dispatch(['up', '-d'], None) service = self.project.get_service('simple') - self.command.dispatch(['kill', '-s', 'SIGINT'], None) + self.command.dispatch(['kill', '-s', 'SIGSTOP'], None) self.assertTrue(service.containers()[0].is_running) self.command.dispatch(['kill', '-s', 'SIGKILL'], None) From 4353f7b9f92bc0e6bebd1fa8cf647407890851b1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sat, 30 May 2015 08:43:48 -0500 Subject: [PATCH 107/118] Merge pull request #1475 from fordhurley/patch-1 Fix markdown formatting for `--service-ports` example (cherry picked from commit d64bf88e26f7b1ce097a6b475799364720bcb6cb) Signed-off-by: Aanand Prasad --- docs/cli.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/cli.md b/docs/cli.md index 162189481..1fbd4cb28 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -95,7 +95,9 @@ specify the `--no-deps` flag: Similarly, if you do want the service's ports to be created and mapped to the host, specify the `--service-ports` flag: - $ docker-compose run --service-ports web python manage.py shell + + $ docker-compose run --service-ports web python manage.py shell + ### scale From 58a7844129c9d3797fc11d1680b77b9e9b31577f Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 29 May 2015 17:12:57 +0100 Subject: [PATCH 108/118] Merge pull request #1482 from bfirsh/add-build-and-dist-to-dockerignore Make it possible to run tests remotely (cherry picked from commit c8e096e0895cb3589c4699daa44c299ea23f790c) Signed-off-by: Aanand Prasad --- .dockerignore | 2 ++ script/test | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index f1b636b3e..a03616e53 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,4 @@ .git +build +dist venv diff --git a/script/test b/script/test index 700de7779..625af09b3 100755 --- a/script/test +++ b/script/test @@ -9,7 +9,6 @@ docker build -t "$TAG" . docker run \ --rm \ --volume="/var/run/docker.sock:/var/run/docker.sock" \ - --volume="$(pwd):/code" \ -e DOCKER_VERSIONS \ -e "TAG=$TAG" \ -e "affinity:image==$TAG" \ From 87b4545b44350e7fe5164071ea975d4e4b5a4d91 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 4 Jun 2015 11:18:23 -0500 Subject: [PATCH 109/118] Merge pull request #1508 from thaJeztah/update-dockerproject-links Update dockerproject.com links (cherry picked from commit 417e6ce0c9f67cc719d5f3bfa9e3adbfb16a34eb) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index acd3cbe7a..4b18fc9dc 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,6 @@ Installation and documentation Contributing ------------ -[![Build Status](http://jenkins.dockerproject.com/buildStatus/icon?job=Compose Master)](http://jenkins.dockerproject.com/job/Compose%20Master/) +[![Build Status](http://jenkins.dockerproject.org/buildStatus/icon?job=Compose%20Master)](http://jenkins.dockerproject.org/job/Compose%20Master/) Want to help build Compose? Check out our [contributing documentation](https://github.com/docker/compose/blob/master/CONTRIBUTING.md). From e724a346c7c26b5b1c824ae4760bd414144c56e3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 8 Jun 2015 12:49:32 -0400 Subject: [PATCH 110/118] Merge pull request #1526 from aanand/remove-start-or-create-containers Remove Service.start_or_create_containers() (cherry picked from commit 38a11c4c28b1af644448d519544b876132ae89a8) Signed-off-by: Aanand Prasad --- compose/service.py | 15 --------------- tests/integration/service_test.py | 4 ++-- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/compose/service.py b/compose/service.py index 8b4115173..71edd5e5e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -394,21 +394,6 @@ class Service(object): container.start() return container - def start_or_create_containers( - self, - insecure_registry=False, - do_build=True): - containers = self.containers(stopped=True) - - if not containers: - new_container = self.create_container( - insecure_registry=insecure_registry, - do_build=do_build, - ) - return [self.start_container(new_container)] - else: - return [self.start_container_if_stopped(c) for c in containers] - def config_hash(self): return json_hash(self.config_dict()) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 7e88557f9..32de5fa47 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -501,10 +501,10 @@ class ServiceTest(DockerClientTestCase): ], }) - def test_start_with_image_id(self): + def test_create_with_image_id(self): # Image id for the current busybox:latest service = self.create_service('foo', image='8c2e06607696') - self.assertTrue(service.start_or_create_containers()) + service.create_container() def test_scale(self): service = self.create_service('web') From 67bc3fabe4d045dff44774a1d9681748d8f990e0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sun, 14 Jun 2015 13:28:14 -0400 Subject: [PATCH 111/118] Merge pull request #1544 from aanand/fix-volume-deduping Fix volume binds de-duplication (cherry picked from commit 77e594dc9405707ef8787728ae63ca091593f3ba) Signed-off-by: Aanand Prasad --- compose/service.py | 5 +-- requirements.txt | 2 +- tests/unit/service_test.py | 87 +++++++++++++++++++++++++++++++++----- 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/compose/service.py b/compose/service.py index 71edd5e5e..1e91a9f23 100644 --- a/compose/service.py +++ b/compose/service.py @@ -731,7 +731,7 @@ def merge_volume_bindings(volumes_option, previous_container): volume_bindings.update( get_container_data_volumes(previous_container, volumes_option)) - return volume_bindings + return volume_bindings.values() def get_container_data_volumes(container, volumes_option): @@ -763,8 +763,7 @@ def get_container_data_volumes(container, volumes_option): def build_volume_binding(volume_spec): - internal = {'bind': volume_spec.internal, 'ro': volume_spec.mode == 'ro'} - return volume_spec.external, internal + return volume_spec.internal, "{}:{}:{}".format(*volume_spec) def parse_volume_spec(volume_config): diff --git a/requirements.txt b/requirements.txt index d3909b766..47fa1e05b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.10 -docker-py==1.2.2 +docker-py==1.2.3-rc1 dockerpty==0.3.4 docopt==0.6.1 requests==2.6.1 diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index add48086d..fb3a7fcbb 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -331,9 +331,7 @@ class ServiceVolumesTest(unittest.TestCase): def test_build_volume_binding(self): binding = build_volume_binding(parse_volume_spec('/outside:/inside')) - self.assertEqual( - binding, - ('/outside', dict(bind='/inside', ro=False))) + self.assertEqual(binding, ('/inside', '/outside:/inside:rw')) def test_get_container_data_volumes(self): options = [ @@ -360,8 +358,8 @@ class ServiceVolumesTest(unittest.TestCase): }, has_been_inspected=True) expected = { - '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False}, - '/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False}, + '/existing/volume': '/var/lib/docker/aaaaaaaa:/existing/volume:rw', + '/mnt/image/data': '/var/lib/docker/cccccccc:/mnt/image/data:rw', } binds = get_container_data_volumes(container, options) @@ -384,11 +382,78 @@ class ServiceVolumesTest(unittest.TestCase): 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'}, }, has_been_inspected=True) - expected = { - '/host/volume': {'bind': '/host/volume', 'ro': True}, - '/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False}, - '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False}, - } + expected = [ + '/host/volume:/host/volume:ro', + '/host/rw/volume:/host/rw/volume:rw', + '/var/lib/docker/aaaaaaaa:/existing/volume:rw', + ] binds = merge_volume_bindings(options, intermediate_container) - self.assertEqual(binds, expected) + self.assertEqual(set(binds), set(expected)) + + def test_mount_same_host_path_to_two_volumes(self): + service = Service( + 'web', + image='busybox', + volumes=[ + '/host/path:/data1', + '/host/path:/data2', + ], + client=self.mock_client, + ) + + self.mock_client.inspect_image.return_value = { + 'Id': 'ababab', + 'ContainerConfig': { + 'Volumes': {} + } + } + + create_options = service._get_container_create_options( + override_options={}, + number=1, + ) + + self.assertEqual( + set(create_options['host_config']['Binds']), + set([ + '/host/path:/data1:rw', + '/host/path:/data2:rw', + ]), + ) + + def test_different_host_path_in_container_json(self): + service = Service( + 'web', + image='busybox', + volumes=['/host/path:/data'], + client=self.mock_client, + ) + + self.mock_client.inspect_image.return_value = { + 'Id': 'ababab', + 'ContainerConfig': { + 'Volumes': { + '/data': {}, + } + } + } + + self.mock_client.inspect_container.return_value = { + 'Id': '123123123', + 'Image': 'ababab', + 'Volumes': { + '/data': '/mnt/sda1/host/path', + }, + } + + create_options = service._get_container_create_options( + override_options={}, + number=1, + previous_container=Container(self.mock_client, {'Id': '123123123'}), + ) + + self.assertEqual( + create_options['host_config']['Binds'], + ['/mnt/sda1/host/path:/data:rw'], + ) From 719954b02f8d6f03e20bde2409b074295dc5da98 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 15 Jun 2015 10:36:37 -0700 Subject: [PATCH 112/118] Merge pull request #1545 from moxiegirl/test-tooling Updated for new documentation tooling (cherry picked from commit aaccd12d3df2ab64f44db5c6cd8bae282a314419) Signed-off-by: Aanand Prasad --- docs/Dockerfile | 31 +++++++---- docs/Makefile | 55 ++++++++++++++++++ docs/README.md | 77 ++++++++++++++++++++++++++ docs/cli.md | 17 ++++-- docs/completion.md | 18 ++++-- docs/{index.md => compose-overview.md} | 16 ++++-- docs/django.md | 18 ++++-- docs/env.md | 18 ++++-- docs/extends.md | 17 ++++-- docs/install.md | 21 ++++--- docs/mkdocs.yml | 12 ---- docs/production.md | 13 ++++- docs/rails.md | 19 ++++--- docs/wordpress.md | 21 ++++--- docs/yml.md | 19 ++++--- 15 files changed, 283 insertions(+), 89 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/README.md rename docs/{index.md => compose-overview.md} (96%) delete mode 100644 docs/mkdocs.yml diff --git a/docs/Dockerfile b/docs/Dockerfile index 59ef66cdd..55e7ce700 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,15 +1,24 @@ -FROM docs/base:latest -MAINTAINER Sven Dowideit (@SvenDowideit) +FROM docs/base:hugo +MAINTAINER Mary Anthony (@moxiegirl) -# to get the git info for this repo +# To get the git info for this repo COPY . /src -# Reset the /docs dir so we can replace the theme meta with the new repo's git info -RUN git reset --hard +COPY . /docs/content/compose/ -RUN grep "__version" /src/compose/__init__.py | sed "s/.*'\(.*\)'/\1/" > /docs/VERSION -COPY docs/* /docs/sources/compose/ -COPY docs/mkdocs.yml /docs/mkdocs-compose.yml - -# Then build everything together, ready for mkdocs -RUN /docs/build.sh +# Sed to process GitHub Markdown +# 1-2 Remove comment code from metadata block +# 3 Remove .md extension from link text +# 4 Change ](/ to ](/project/ in links +# 5 Change ](word) to ](/project/word) +# 6 Change ](../../ to ](/project/ +# 7 Change ](../ to ](/project/word) +# +# +RUN find /docs/content/compose -type f -name "*.md" -exec sed -i.old \ + -e '/^/g' \ + -e '/^/g' \ + -e 's/\([(]\)\(.*\)\(\.md\)/\1\2/g' \ + -e 's/\(\]\)\([(]\)\(\/\)/\1\2\/compose\//g' \ + -e 's/\(\][(]\)\([A-z]*[)]\)/\]\(\/compose\/\2/g' \ + -e 's/\(\][(]\)\(\.\.\/\)/\1\/compose\//g' {} \; diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..021e8f6e5 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,55 @@ +.PHONY: all binary build cross default docs docs-build docs-shell shell test test-unit test-integration test-integration-cli test-docker-py validate + +# env vars passed through directly to Docker's build scripts +# to allow things like `make DOCKER_CLIENTONLY=1 binary` easily +# `docs/sources/contributing/devenvironment.md ` and `project/PACKAGERS.md` have some limited documentation of some of these +DOCKER_ENVS := \ + -e BUILDFLAGS \ + -e DOCKER_CLIENTONLY \ + -e DOCKER_EXECDRIVER \ + -e DOCKER_GRAPHDRIVER \ + -e TESTDIRS \ + -e TESTFLAGS \ + -e TIMEOUT +# note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds + +# to allow `make DOCSDIR=docs docs-shell` (to create a bind mount in docs) +DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR)/$(DOCSDIR):/$(DOCSDIR)) + +# to allow `make DOCSPORT=9000 docs` +DOCSPORT := 8000 + +# Get the IP ADDRESS +DOCKER_IP=$(shell python -c "import urlparse ; print urlparse.urlparse('$(DOCKER_HOST)').hostname or ''") +HUGO_BASE_URL=$(shell test -z "$(DOCKER_IP)" && echo localhost || echo "$(DOCKER_IP)") +HUGO_BIND_IP=0.0.0.0 + +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) +DOCKER_IMAGE := docker$(if $(GIT_BRANCH),:$(GIT_BRANCH)) +DOCKER_DOCS_IMAGE := docs-base$(if $(GIT_BRANCH),:$(GIT_BRANCH)) + + +DOCKER_RUN_DOCS := docker run --rm -it $(DOCS_MOUNT) -e AWS_S3_BUCKET -e NOCACHE + +# for some docs workarounds (see below in "docs-build" target) +GITCOMMIT := $(shell git rev-parse --short HEAD 2>/dev/null) + +default: docs + +docs: docs-build + $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) + +docs-draft: docs-build + $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --buildDrafts="true" --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) + + +docs-shell: docs-build + $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 "$(DOCKER_DOCS_IMAGE)" bash + + +docs-build: +# ( git remote | grep -v upstream ) || git diff --name-status upstream/release..upstream/docs ./ > ./changed-files +# echo "$(GIT_BRANCH)" > GIT_BRANCH +# echo "$(AWS_S3_BUCKET)" > AWS_S3_BUCKET +# echo "$(GITCOMMIT)" > GITCOMMIT + docker build -t "$(DOCKER_DOCS_IMAGE)" . diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..00736e476 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,77 @@ +# Contributing to the Docker Compose documentation + +The documentation in this directory is part of the [https://docs.docker.com](https://docs.docker.com) website. Docker uses [the Hugo static generator](http://gohugo.io/overview/introduction/) to convert project Markdown files to a static HTML site. + +You don't need to be a Hugo expert to contribute to the compose documentation. If you are familiar with Markdown, you can modify the content in the `docs` files. + +If you want to add a new file or change the location of the document in the menu, you do need to know a little more. + +## Documentation contributing workflow + +1. Edit a Markdown file in the tree. + +2. Save your changes. + +3. Make sure you in your `docs` subdirectory. + +4. Build the documentation. + + $ make docs + ---> ffcf3f6c4e97 + Removing intermediate container a676414185e8 + Successfully built ffcf3f6c4e97 + docker run --rm -it -e AWS_S3_BUCKET -e NOCACHE -p 8000:8000 -e DOCKERHOST "docs-base:test-tooling" hugo server --port=8000 --baseUrl=192.168.59.103 --bind=0.0.0.0 + ERROR: 2015/06/13 MenuEntry's .Url is deprecated and will be removed in Hugo 0.15. Use .URL instead. + 0 of 4 drafts rendered + 0 future content + 12 pages created + 0 paginator pages created + 0 tags created + 0 categories created + in 55 ms + Serving pages from /docs/public + Web Server is available at http://0.0.0.0:8000/ + Press Ctrl+C to stop + +5. Open the available server in your browser. + + The documentation server has the complete menu but only the Docker Compose + documentation resolves. You can't access the other project docs from this + localized build. + +## Tips on Hugo metadata and menu positioning + +The top of each Docker Compose documentation file contains TOML metadata. The metadata is commented out to prevent it from appears in GitHub. + + + +The metadata alone has this structure: + + +++ + title = "Extending services in Compose" + description = "How to use Docker Compose's extends keyword to share configuration between files and projects" + keywords = ["fig, composition, compose, docker, orchestration, documentation, docs"] + [menu.main] + parent="smn_workw_compose" + weight=2 + +++ + +The `[menu.main]` section refers to navigation defined [in the main Docker menu](https://github.com/docker/docs-base/blob/hugo/config.toml). This metadata says *add a menu item called* Extending services in Compose *to the menu with the* `smn_workdw_compose` *identifier*. If you locate the menu in the configuration, you'll find *Create multi-container applications* is the menu title. + +You can move an article in the tree by specifying a new parent. You can shift the location of the item by changing its weight. Higher numbers are heavier and shift the item to the bottom of menu. Low or no numbers shift it up. + + +## Other key documentation repositories + +The `docker/docs-base` repository contains [the Hugo theme and menu configuration](https://github.com/docker/docs-base). If you open the `Dockerfile` you'll see the `make docs` relies on this as a base image for building the Compose documentation. + +The `docker/docs.docker.com` repository contains [build system for building the Docker documentation site](https://github.com/docker/docs.docker.com). Fork this repository to build the entire documentation site. diff --git a/docs/cli.md b/docs/cli.md index 1fbd4cb28..a2167d9c3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,9 +1,16 @@ -page_title: Compose CLI reference -page_description: Compose CLI reference -page_keywords: fig, composition, compose, docker, orchestration, cli, reference + -# CLI reference +# Compose CLI reference Most Docker Compose commands are run against one or more services. If the service is not specified, the command will apply to all services. @@ -185,7 +192,7 @@ Configures the path to the `ca.pem`, `cert.pem`, and `key.pem` files used for TL ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/completion.md b/docs/completion.md index 5168971f8..7fb696d80 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -1,7 +1,13 @@ ---- -layout: default -title: Command Completion ---- + # Command Completion @@ -53,11 +59,11 @@ Enjoy working with Compose faster and with less typos! ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) - [Command line reference](cli.md) - [Yaml file reference](yml.md) -- [Compose environment variables](env.md) +- [Compose environment variables](env.md) \ No newline at end of file diff --git a/docs/index.md b/docs/compose-overview.md similarity index 96% rename from docs/index.md rename to docs/compose-overview.md index 981a02702..33629957a 100644 --- a/docs/index.md +++ b/docs/compose-overview.md @@ -1,11 +1,15 @@ -page_title: Compose: Multi-container orchestration for Docker -page_description: Introduction and Overview of Compose -page_keywords: documentation, docs, docker, compose, orchestration, containers + -# Docker Compose - -## Overview +# Overview of Docker Compose Compose is a tool for defining and running multi-container applications with Docker. With Compose, you define a multi-container application in a single diff --git a/docs/django.md b/docs/django.md index 4cbebe041..c44329e1c 100644 --- a/docs/django.md +++ b/docs/django.md @@ -1,10 +1,16 @@ -page_title: Quickstart Guide: Compose and Django -page_description: Getting started with Docker Compose and Django -page_keywords: documentation, docs, docker, compose, orchestration, containers, -django + -## Getting started with Compose and Django +## Quickstart Guide: Compose and Django This Quick-start Guide will demonstrate how to use Compose to set up and run a @@ -119,7 +125,7 @@ example, run `docker-compose up` and in another terminal run: ## More Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/env.md b/docs/env.md index a4b543ae3..73496f32f 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1,9 +1,15 @@ ---- -layout: default -title: Compose environment variables reference ---- + -Environment variables reference +# Compose environment variables reference =============================== **Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](yml.md#links) for details. @@ -34,7 +40,7 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/extends.md b/docs/extends.md index fd372ce2d..8527c81b3 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -1,6 +1,13 @@ -page_title: Extending services in Compose -page_description: How to use Docker Compose's "extends" keyword to share configuration between files and projects -page_keywords: fig, composition, compose, docker, orchestration, documentation, docs + ## Extending services in Compose @@ -79,7 +86,7 @@ For full details on how to use `extends`, refer to the [reference](#reference). ### Example use case In this example, you’ll repurpose the example app from the [quick start -guide](index.md). (If you're not familiar with Compose, it's recommended that +guide](compose-overview.md). (If you're not familiar with Compose, it's recommended that you go through the quick start first.) This example assumes you want to use Compose both to develop an application locally and then deploy it to a production environment. @@ -364,7 +371,7 @@ volumes: ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/install.md b/docs/install.md index a521ec06c..ec0e6e4d5 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,14 +1,21 @@ -page_title: Installing Compose -page_description: How to install Docker Compose -page_keywords: compose, orchestration, install, installation, docker, documentation + -## Installing Compose +# Install Docker Compose To install Compose, you'll need to install Docker first. You'll then install Compose with a `curl` command. -### Install Docker +## Install Docker First, install Docker version 1.6 or greater: @@ -16,7 +23,7 @@ First, install Docker version 1.6 or greater: - [Instructions for Ubuntu](http://docs.docker.com/installation/ubuntulinux/) - [Instructions for other systems](http://docs.docker.com/installation/) -### Install Compose +## Install Compose To install Compose, run the following commands: @@ -38,7 +45,7 @@ You can test the installation by running `docker-compose --version`. ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml deleted file mode 100644 index 428439bc4..000000000 --- a/docs/mkdocs.yml +++ /dev/null @@ -1,12 +0,0 @@ - -- ['compose/index.md', 'User Guide', 'Docker Compose' ] -- ['compose/production.md', 'User Guide', 'Using Compose in production' ] -- ['compose/extends.md', 'User Guide', 'Extending services in Compose'] -- ['compose/install.md', 'Installation', 'Docker Compose'] -- ['compose/cli.md', 'Reference', 'Compose command line'] -- ['compose/yml.md', 'Reference', 'Compose yml'] -- ['compose/env.md', 'Reference', 'Compose ENV variables'] -- ['compose/completion.md', 'Reference', 'Compose commandline completion'] -- ['compose/django.md', 'Examples', 'Getting started with Compose and Django'] -- ['compose/rails.md', 'Examples', 'Getting started with Compose and Rails'] -- ['compose/wordpress.md', 'Examples', 'Getting started with Compose and Wordpress'] diff --git a/docs/production.md b/docs/production.md index 60a6873da..294f3c4e8 100644 --- a/docs/production.md +++ b/docs/production.md @@ -1,6 +1,13 @@ -page_title: Using Compose in production -page_description: Guide to using Docker Compose in production -page_keywords: documentation, docs, docker, compose, orchestration, containers, production + ## Using Compose in production diff --git a/docs/rails.md b/docs/rails.md index aedb4c6e7..2ff6f1752 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -1,10 +1,15 @@ -page_title: Quickstart Guide: Compose and Rails -page_description: Getting started with Docker Compose and Rails -page_keywords: documentation, docs, docker, compose, orchestration, containers, -rails + - -## Getting started with Compose and Rails +## Quickstart Guide: Compose and Rails This Quickstart guide will show you how to use Compose to set up and run a Rails/PostgreSQL app. Before starting, you'll need to have [Compose installed](install.md). @@ -119,7 +124,7 @@ you're using Boot2docker, `boot2docker ip` will tell you its address). ## More Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index b40d1a9f0..ad0e62966 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -1,14 +1,21 @@ -page_title: Quickstart Guide: Compose and Wordpress -page_description: Getting started with Docker Compose and Rails -page_keywords: documentation, docs, docker, compose, orchestration, containers, -wordpress + -## Getting started with Compose and Wordpress + +# Quickstart Guide: Compose and Wordpress You can use Compose to easily run Wordpress in an isolated environment built with Docker containers. -### Define the project +## Define the project First, [Install Compose](install.md) and then download Wordpress into the current directory: @@ -114,7 +121,7 @@ address). ## More Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/yml.md b/docs/yml.md index df791bc98..80d6d719f 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -1,10 +1,13 @@ ---- -layout: default -title: docker-compose.yml reference -page_title: docker-compose.yml reference -page_description: docker-compose.yml reference -page_keywords: fig, composition, compose, docker ---- + + # docker-compose.yml reference @@ -390,7 +393,7 @@ read_only: true ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) From 09018855cebceac34525122feb76c1885a5f4057 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 15 Jun 2015 13:43:57 -0400 Subject: [PATCH 113/118] Merge pull request #1550 from aanand/update-docker-py Update setup.py with new docker-py minimum (cherry picked from commit b3b44b8e4c7ee7463136bb13cf6c3d759e6d87e9) Signed-off-by: Aanand Prasad --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9364f57f3..a94d87374 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.7', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', - 'docker-py >= 1.2.2, < 1.3', + 'docker-py >= 1.2.3-rc1, < 1.3', 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', ] From f353d9fbc0df6835ef373c72342feae856e0276d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 15 Jun 2015 10:58:44 -0700 Subject: [PATCH 114/118] Merge pull request #1406 from vdemeester/667-compose-port-scale Fixing docker-compose port with scale (#667) (cherry picked from commit 5b2a0cc73d104340964b299c11723e465ea7c112) Signed-off-by: Aanand Prasad --- compose/cli/main.py | 7 +++--- .../docker-compose.yml | 6 +++++ tests/integration/cli_test.py | 22 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/ports-composefile-scale/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 0c3b85e5c..4f3f11e4e 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -169,13 +169,14 @@ class TopLevelCommand(Command): Usage: port [options] SERVICE PRIVATE_PORT Options: - --protocol=proto tcp or udp (defaults to tcp) + --protocol=proto tcp or udp [default: tcp] --index=index index of the container if there are multiple - instances of a service (defaults to 1) + instances of a service [default: 1] """ + index = int(options.get('--index')) service = project.get_service(options['SERVICE']) try: - container = service.get_container(number=options.get('--index') or 1) + container = service.get_container(number=index) except ValueError as e: raise UserError(str(e)) print(container.get_local_port( diff --git a/tests/fixtures/ports-composefile-scale/docker-compose.yml b/tests/fixtures/ports-composefile-scale/docker-compose.yml new file mode 100644 index 000000000..1a2bb485b --- /dev/null +++ b/tests/fixtures/ports-composefile-scale/docker-compose.yml @@ -0,0 +1,6 @@ + +simple: + image: busybox:latest + command: /bin/sleep 300 + ports: + - '3000' diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index cb7bc17fc..2d1f1f76e 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from operator import attrgetter import sys import os import shlex @@ -436,6 +437,27 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(get_port(3001), "0.0.0.0:49152") self.assertEqual(get_port(3002), "") + def test_port_with_scale(self): + + self.command.base_dir = 'tests/fixtures/ports-composefile-scale' + self.command.dispatch(['scale', 'simple=2'], None) + containers = sorted( + self.project.containers(service_names=['simple']), + key=attrgetter('name')) + + @patch('sys.stdout', new_callable=StringIO) + def get_port(number, mock_stdout, index=None): + if index is None: + self.command.dispatch(['port', 'simple', str(number)], None) + else: + self.command.dispatch(['port', '--index=' + str(index), 'simple', str(number)], None) + return mock_stdout.getvalue().rstrip() + + self.assertEqual(get_port(3000), containers[0].get_local_port(3000)) + self.assertEqual(get_port(3000, index=1), containers[0].get_local_port(3000)) + self.assertEqual(get_port(3000, index=2), containers[1].get_local_port(3000)) + self.assertEqual(get_port(3002), "") + def test_env_file_relative_to_compose_file(self): config_path = os.path.abspath('tests/fixtures/env-file/docker-compose.yml') self.command.dispatch(['-f', config_path, 'up', '-d'], None) From 7fa4cd1214deea8f61ce5195ecbe377f70d1e311 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 16 Jun 2015 16:26:40 -0700 Subject: [PATCH 115/118] Merge pull request #1552 from aanand/add-upgrade-instructions Add upgrading instructions to install docs (cherry picked from commit bc7161b475f7032bfc36e177935e9d7b13354718) --- docs/install.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/install.md b/docs/install.md index ec0e6e4d5..c1abd4fd6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -43,6 +43,18 @@ Compose can also be installed as a Python package: No further steps are required; Compose should now be successfully installed. You can test the installation by running `docker-compose --version`. +### Upgrading + +If you're coming from Compose 1.2 or earlier, you'll need to remove or migrate your existing containers after upgrading Compose. This is because, as of version 1.3, Compose uses Docker labels to keep track of containers, and so they need to be recreated with labels added. + +If Compose detects containers that were created without labels, it will refuse to run so that you don't end up with two sets of them. If you want to keep using your existing containers (for example, because they have data volumes you want to preserve) you can migrate them with the following command: + + docker-compose migrate-to-labels + +Alternatively, if you're not worried about keeping them, you can remove them - Compose will just create new ones. + + docker rm -f myapp_web_1 myapp_db_1 ... + ## Compose documentation - [User guide](compose-overview.md) From c3c5d91c47f00d607b68f345e367ed1b828852f8 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 18 Jun 2015 10:20:10 -0700 Subject: [PATCH 116/118] Merge pull request #1563 from moxiegirl/hugo-test-fixes Hugo final 1.7 Documentation PR -- please read carefully (cherry picked from commit 4e73e86d9480de0be87fa5390d915346a435ac26) Signed-off-by: Aanand Prasad --- docs/Dockerfile | 14 +++++++------- docs/cli.md | 2 +- docs/completion.md | 2 +- docs/django.md | 2 +- docs/env.md | 2 +- docs/extends.md | 4 ++-- docs/{compose-overview.md => index.md} | 0 docs/install.md | 2 +- docs/rails.md | 2 +- docs/wordpress.md | 2 +- docs/yml.md | 2 +- 11 files changed, 17 insertions(+), 17 deletions(-) rename docs/{compose-overview.md => index.md} (100%) diff --git a/docs/Dockerfile b/docs/Dockerfile index 55e7ce700..a49c1e7f3 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -8,17 +8,17 @@ COPY . /docs/content/compose/ # Sed to process GitHub Markdown # 1-2 Remove comment code from metadata block -# 3 Remove .md extension from link text -# 4 Change ](/ to ](/project/ in links -# 5 Change ](word) to ](/project/word) -# 6 Change ](../../ to ](/project/ -# 7 Change ](../ to ](/project/word) +# 3 Change ](/word to ](/project/ in links +# 4 Change ](word.md) to ](/project/word) +# 5 Remove .md extension from link text +# 6 Change ](../ to ](/project/word) +# 7 Change ](../../ to ](/project/ --> not implemented # # RUN find /docs/content/compose -type f -name "*.md" -exec sed -i.old \ -e '/^/g' \ -e '/^/g' \ - -e 's/\([(]\)\(.*\)\(\.md\)/\1\2/g' \ -e 's/\(\]\)\([(]\)\(\/\)/\1\2\/compose\//g' \ - -e 's/\(\][(]\)\([A-z]*[)]\)/\]\(\/compose\/\2/g' \ + -e 's/\(\][(]\)\([A-z].*\)\(\.md\)/\1\/compose\/\2/g' \ + -e 's/\([(]\)\(.*\)\(\.md\)/\1\2/g' \ -e 's/\(\][(]\)\(\.\.\/\)/\1\/compose\//g' {} \; diff --git a/docs/cli.md b/docs/cli.md index a2167d9c3..61a6aa6dd 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -192,7 +192,7 @@ Configures the path to the `ca.pem`, `cert.pem`, and `key.pem` files used for TL ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/completion.md b/docs/completion.md index 7fb696d80..3856d2701 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -59,7 +59,7 @@ Enjoy working with Compose faster and with less typos! ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/django.md b/docs/django.md index c44329e1c..84fdcbfe5 100644 --- a/docs/django.md +++ b/docs/django.md @@ -125,7 +125,7 @@ example, run `docker-compose up` and in another terminal run: ## More Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/env.md b/docs/env.md index 73496f32f..e38e6d50c 100644 --- a/docs/env.md +++ b/docs/env.md @@ -40,7 +40,7 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/extends.md b/docs/extends.md index 8527c81b3..054462b89 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -86,7 +86,7 @@ For full details on how to use `extends`, refer to the [reference](#reference). ### Example use case In this example, you’ll repurpose the example app from the [quick start -guide](compose-overview.md). (If you're not familiar with Compose, it's recommended that +guide](index.md). (If you're not familiar with Compose, it's recommended that you go through the quick start first.) This example assumes you want to use Compose both to develop an application locally and then deploy it to a production environment. @@ -371,7 +371,7 @@ volumes: ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/compose-overview.md b/docs/index.md similarity index 100% rename from docs/compose-overview.md rename to docs/index.md diff --git a/docs/install.md b/docs/install.md index c1abd4fd6..ac35c8d9f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -57,7 +57,7 @@ Alternatively, if you're not worried about keeping them, you can remove them - C ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) diff --git a/docs/rails.md b/docs/rails.md index 2ff6f1752..cb8078647 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -124,7 +124,7 @@ you're using Boot2docker, `boot2docker ip` will tell you its address). ## More Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index ad0e62966..aa62e4e4e 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -121,7 +121,7 @@ address). ## More Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/yml.md b/docs/yml.md index 80d6d719f..087f8ac74 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -393,7 +393,7 @@ read_only: true ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) From c21d6706b6df54f3304b6fc59cb307c42f0c54c5 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 18 Jun 2015 11:23:40 -0700 Subject: [PATCH 117/118] Merge pull request #1565 from aanand/use-docker-1.7.0 Use docker 1.7.0 and docker-py 1.2.3 (cherry picked from commit 8ffeaf2a54828014834f49e9a20d9486a6d6d335) Signed-off-by: Aanand Prasad Conflicts: Dockerfile --- Dockerfile | 6 +++--- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1ff2d3825..98dc59c55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,13 +48,13 @@ RUN set -ex; \ rm -rf pip-7.0.1; \ rm pip-7.0.1.tar.gz -ENV ALL_DOCKER_VERSIONS 1.6.0 1.7.0-rc2 +ENV ALL_DOCKER_VERSIONS 1.6.0 1.7.0 RUN set -ex; \ curl https://get.docker.com/builds/Linux/x86_64/docker-1.6.0 -o /usr/local/bin/docker-1.6.0; \ chmod +x /usr/local/bin/docker-1.6.0; \ - curl https://test.docker.com/builds/Linux/x86_64/docker-1.7.0-rc2 -o /usr/local/bin/docker-1.7.0-rc2; \ - chmod +x /usr/local/bin/docker-1.7.0-rc2 + curl https://test.docker.com/builds/Linux/x86_64/docker-1.7.0 -o /usr/local/bin/docker-1.7.0; \ + chmod +x /usr/local/bin/docker-1.7.0 # Set the default Docker to be run RUN ln -s /usr/local/bin/docker-1.6.0 /usr/local/bin/docker diff --git a/requirements.txt b/requirements.txt index 47fa1e05b..69bd4c5f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.10 -docker-py==1.2.3-rc1 +docker-py==1.2.3 dockerpty==0.3.4 docopt==0.6.1 requests==2.6.1 diff --git a/setup.py b/setup.py index a94d87374..d2e81e175 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.7', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', - 'docker-py >= 1.2.3-rc1, < 1.3', + 'docker-py >= 1.2.3, < 1.3', 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', ] From 00f61196a44ee140f389a51e50d39d1b846ba180 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 May 2015 12:43:23 +0100 Subject: [PATCH 118/118] Bump 1.3.0 Signed-off-by: Aanand Prasad --- CHANGES.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- docs/install.md | 2 +- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 277a188a3..78e629b89 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,51 @@ Change log ========== +1.3.0 (2015-06-18) +------------------ + +Firstly, two important notes: + +- **This release contains breaking changes, and you will need to either remove or migrate your existing containers before running your app** - see the [upgrading section of the install docs](https://github.com/docker/compose/blob/1.3.0rc1/docs/install.md#upgrading) for details. + +- Compose now requires Docker 1.6.0 or later. + +We've done a lot of work in this release to remove hacks and make Compose more stable: + +- Compose now uses container labels, rather than names, to keep track of containers. This makes Compose both faster and easier to integrate with your own tools. + +- Compose no longer uses "intermediate containers" when recreating containers for a service. This makes `docker-compose up` less complex and more resilient to failure. + +There are some new features: + +- `docker-compose up` has an **experimental** new behaviour: it will only recreate containers for services whose configuration has changed in `docker-compose.yml`. This will eventually become the default, but for now you can take it for a spin: + + $ docker-compose up --x-smart-recreate + +- When invoked in a subdirectory of a project, `docker-compose` will now climb up through parent directories until it finds a `docker-compose.yml`. + +Several new configuration keys have been added to `docker-compose.yml`: + +- `dockerfile`, like `docker build --file`, lets you specify an alternate Dockerfile to use with `build`. +- `labels`, like `docker run --labels`, lets you add custom metadata to containers. +- `extra_hosts`, like `docker run --add-host`, lets you add entries to a container's `/etc/hosts` file. +- `pid: host`, like `docker run --pid=host`, lets you reuse the same PID namespace as the host machine. +- `cpuset`, like `docker run --cpuset-cpus`, lets you specify which CPUs to allow execution in. +- `read_only`, like `docker run --read-only`, lets you mount a container's filesystem as read-only. +- `security_opt`, like `docker run --security-opt`, lets you specify [security options](https://docs.docker.com/reference/run/#security-configuration). +- `log_driver`, like `docker run --log-driver`, lets you specify a [log driver](https://docs.docker.com/reference/run/#logging-drivers-log-driver). + +Many bugs have been fixed, including the following: + +- The output of `docker-compose run` was sometimes truncated, especially when running under Jenkins. +- A service's volumes would sometimes not update after volume configuration was changed in `docker-compose.yml`. +- Authenticating against third-party registries would sometimes fail. +- `docker-compose run --rm` would fail to remove the container if the service had a `restart` policy in place. +- `docker-compose scale` would refuse to scale a service beyond 1 container if it exposed a specific port number on the host. +- Compose would refuse to create multiple volume entries with the same host path. + +Thanks @ahromis, @albers, @aleksandr-vin, @antoineco, @ccverak, @chernjie, @dnephin, @edmorley, @fordhurley, @josephpage, @KyleJamesWalker, @lsowen, @mchasal, @noironetworks, @sdake, @sdurrheimer, @sherter, @stephenlawrence, @thaJeztah, @thieman, @turtlemonvh, @twhiteman, @vdemeester, @xuxinkun and @zwily! + 1.2.0 (2015-04-16) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index 045e79144..9e4c3fdb2 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.3.0dev' +__version__ = '1.3.0' diff --git a/docs/install.md b/docs/install.md index ac35c8d9f..a608d8fe7 100644 --- a/docs/install.md +++ b/docs/install.md @@ -27,7 +27,7 @@ First, install Docker version 1.6 or greater: To install Compose, run the following commands: - curl -L https://github.com/docker/compose/releases/download/1.2.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.3.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands above, then `exit`.