From 461b600068465fb9260f3918ba12e8d501986515 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 31 Mar 2015 15:23:34 -0400 Subject: [PATCH 01/53] Merge pull request #1225 from aanand/fix-1222 When extending, `build` replaces `image` and vice versa (cherry picked from commit 6dbe321a45dfd7539234f889825b54e1a026e46f) Signed-off-by: Aanand Prasad --- compose/config.py | 6 ++++++ tests/unit/config_test.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/compose/config.py b/compose/config.py index 0cd7c1ae6..8d6ffea7a 100644 --- a/compose/config.py +++ b/compose/config.py @@ -189,6 +189,12 @@ def merge_service_dicts(base, override): override.get('volumes'), ) + if 'image' in override and 'build' in d: + del d['build'] + + if 'build' in override and 'image' in d: + del d['image'] + for k in ALLOWED_KEYS: if k not in ['environment', 'volumes']: if k in override: diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 8deb457af..67f24a92d 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -79,6 +79,39 @@ class MergeTest(unittest.TestCase): ) self.assertEqual(set(service_dict['volumes']), set(['/bar:/code', '/data'])) + def test_merge_build_or_image_no_override(self): + self.assertEqual( + config.merge_service_dicts({'build': '.'}, {}), + {'build': '.'}, + ) + + self.assertEqual( + config.merge_service_dicts({'image': 'redis'}, {}), + {'image': 'redis'}, + ) + + def test_merge_build_or_image_override_with_same(self): + self.assertEqual( + config.merge_service_dicts({'build': '.'}, {'build': './web'}), + {'build': './web'}, + ) + + self.assertEqual( + config.merge_service_dicts({'image': 'redis'}, {'image': 'postgres'}), + {'image': 'postgres'}, + ) + + def test_merge_build_or_image_override_with_other(self): + self.assertEqual( + config.merge_service_dicts({'build': '.'}, {'image': 'redis'}), + {'image': 'redis'} + ) + + self.assertEqual( + config.merge_service_dicts({'image': 'redis'}, {'build': '.'}), + {'build': '.'} + ) + class EnvTest(unittest.TestCase): def test_parse_environment_as_list(self): From b24a60ba9fdbfc9d3bdade6efbc9b629cdd4872b Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 31 Mar 2015 16:01:22 -0400 Subject: [PATCH 02/53] Merge pull request #1226 from aanand/merge-multi-value-options Merge multi-value options when extending (cherry picked from commit e708f4f59dcb417e90a5bbdcadcee37e8c6b7802) Signed-off-by: Aanand Prasad --- compose/config.py | 30 ++++++++++++++--- tests/unit/config_test.py | 71 +++++++++++++++++++++++++++++++++++---- 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/compose/config.py b/compose/config.py index 8d6ffea7a..7f2e302b3 100644 --- a/compose/config.py +++ b/compose/config.py @@ -195,10 +195,23 @@ def merge_service_dicts(base, override): if 'build' in override and 'image' in d: del d['image'] - for k in ALLOWED_KEYS: - if k not in ['environment', 'volumes']: - if k in override: - d[k] = override[k] + list_keys = ['ports', 'expose', 'external_links'] + + for key in list_keys: + if key in base or key in override: + d[key] = base.get(key, []) + override.get(key, []) + + list_or_string_keys = ['dns', 'dns_search'] + + for key in list_or_string_keys: + if key in base or key in override: + d[key] = to_list(base.get(key)) + to_list(override.get(key)) + + already_merged_keys = ['environment', 'volumes'] + list_keys + list_or_string_keys + + for k in set(ALLOWED_KEYS) - set(already_merged_keys): + if k in override: + d[k] = override[k] return d @@ -354,6 +367,15 @@ def expand_path(working_dir, path): return os.path.abspath(os.path.join(working_dir, path)) +def to_list(value): + if value is None: + return [] + elif isinstance(value, six.string_types): + return [value] + else: + return value + + def get_service_name_from_net(net_config): if not net_config: return diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 67f24a92d..d95ba7838 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -39,40 +39,40 @@ class ConfigTest(unittest.TestCase): config.make_service_dict('foo', {'ports': ['8000']}) -class MergeTest(unittest.TestCase): - def test_merge_volumes_empty(self): +class MergeVolumesTest(unittest.TestCase): + def test_empty(self): service_dict = config.merge_service_dicts({}, {}) self.assertNotIn('volumes', service_dict) - def test_merge_volumes_no_override(self): + def test_no_override(self): service_dict = config.merge_service_dicts( {'volumes': ['/foo:/code', '/data']}, {}, ) self.assertEqual(set(service_dict['volumes']), set(['/foo:/code', '/data'])) - def test_merge_volumes_no_base(self): + def test_no_base(self): service_dict = config.merge_service_dicts( {}, {'volumes': ['/bar:/code']}, ) self.assertEqual(set(service_dict['volumes']), set(['/bar:/code'])) - def test_merge_volumes_override_explicit_path(self): + def test_override_explicit_path(self): service_dict = config.merge_service_dicts( {'volumes': ['/foo:/code', '/data']}, {'volumes': ['/bar:/code']}, ) self.assertEqual(set(service_dict['volumes']), set(['/bar:/code', '/data'])) - def test_merge_volumes_add_explicit_path(self): + def test_add_explicit_path(self): service_dict = config.merge_service_dicts( {'volumes': ['/foo:/code', '/data']}, {'volumes': ['/bar:/code', '/quux:/data']}, ) self.assertEqual(set(service_dict['volumes']), set(['/bar:/code', '/quux:/data'])) - def test_merge_volumes_remove_explicit_path(self): + def test_remove_explicit_path(self): service_dict = config.merge_service_dicts( {'volumes': ['/foo:/code', '/quux:/data']}, {'volumes': ['/bar:/code', '/data']}, @@ -113,6 +113,63 @@ class MergeTest(unittest.TestCase): ) +class MergeListsTest(unittest.TestCase): + def test_empty(self): + service_dict = config.merge_service_dicts({}, {}) + self.assertNotIn('ports', service_dict) + + def test_no_override(self): + service_dict = config.merge_service_dicts( + {'ports': ['10:8000', '9000']}, + {}, + ) + self.assertEqual(set(service_dict['ports']), set(['10:8000', '9000'])) + + def test_no_base(self): + service_dict = config.merge_service_dicts( + {}, + {'ports': ['10:8000', '9000']}, + ) + self.assertEqual(set(service_dict['ports']), set(['10:8000', '9000'])) + + def test_add_item(self): + service_dict = config.merge_service_dicts( + {'ports': ['10:8000', '9000']}, + {'ports': ['20:8000']}, + ) + self.assertEqual(set(service_dict['ports']), set(['10:8000', '9000', '20:8000'])) + + +class MergeStringsOrListsTest(unittest.TestCase): + def test_no_override(self): + service_dict = config.merge_service_dicts( + {'dns': '8.8.8.8'}, + {}, + ) + self.assertEqual(set(service_dict['dns']), set(['8.8.8.8'])) + + def test_no_base(self): + service_dict = config.merge_service_dicts( + {}, + {'dns': '8.8.8.8'}, + ) + self.assertEqual(set(service_dict['dns']), set(['8.8.8.8'])) + + def test_add_string(self): + service_dict = config.merge_service_dicts( + {'dns': ['8.8.8.8']}, + {'dns': '9.9.9.9'}, + ) + self.assertEqual(set(service_dict['dns']), set(['8.8.8.8', '9.9.9.9'])) + + def test_add_list(self): + service_dict = config.merge_service_dicts( + {'dns': '8.8.8.8'}, + {'dns': ['9.9.9.9']}, + ) + self.assertEqual(set(service_dict['dns']), set(['8.8.8.8', '9.9.9.9'])) + + class EnvTest(unittest.TestCase): def test_parse_environment_as_list(self): environment =[ From e4e802d1f86ceb16d45d0176f94906e799f90fc9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 31 Mar 2015 21:20:02 -0400 Subject: [PATCH 03/53] Merge pull request #1213 from moysesb/relative_build Make value of 'build:' relative to the yml file. (cherry picked from commit 0f70b8638ff7167e9755d24dc8dab1579662f72d) Signed-off-by: Aanand Prasad --- compose/config.py | 14 ++++++++ docs/yml.md | 5 +-- tests/fixtures/build-ctx/Dockerfile | 2 ++ tests/fixtures/build-path/docker-compose.yml | 2 ++ .../docker-compose.yml | 2 +- .../simple-dockerfile/docker-compose.yml | 2 +- tests/unit/config_test.py | 33 +++++++++++++++++++ 7 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 tests/fixtures/build-ctx/Dockerfile create mode 100644 tests/fixtures/build-path/docker-compose.yml diff --git a/compose/config.py b/compose/config.py index 7f2e302b3..1dc64af25 100644 --- a/compose/config.py +++ b/compose/config.py @@ -171,6 +171,9 @@ def process_container_options(service_dict, working_dir=None): if 'volumes' in service_dict: service_dict['volumes'] = resolve_host_paths(service_dict['volumes'], working_dir=working_dir) + if 'build' in service_dict: + service_dict['build'] = resolve_build_path(service_dict['build'], working_dir=working_dir) + return service_dict @@ -330,6 +333,17 @@ def resolve_host_path(volume, working_dir): return container_path +def resolve_build_path(build_path, working_dir=None): + if working_dir is None: + raise Exception("No working_dir passed to resolve_build_path") + + _path = expand_path(working_dir, build_path) + if not os.path.exists(_path) or not os.access(_path, os.R_OK): + raise ConfigurationError("build path %s either does not exist or is not accessible." % _path) + else: + return _path + + def merge_volumes(base, override): d = dict_from_volumes(base) d.update(dict_from_volumes(override)) diff --git a/docs/yml.md b/docs/yml.md index 157ba4e67..a9909e816 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -29,8 +29,9 @@ image: a4bc65fd ### build -Path to a directory containing a Dockerfile. This directory is also the -build context that is sent to the Docker daemon. +Path to a directory containing a Dockerfile. When the value supplied is a +relative path, it is interpreted as relative to the location of the yml file +itself. This directory is also the build context that is sent to the Docker daemon. Compose will build and tag it with a generated name, and use that image thereafter. diff --git a/tests/fixtures/build-ctx/Dockerfile b/tests/fixtures/build-ctx/Dockerfile new file mode 100644 index 000000000..d1ceac6b7 --- /dev/null +++ b/tests/fixtures/build-ctx/Dockerfile @@ -0,0 +1,2 @@ +FROM busybox:latest +CMD echo "success" diff --git a/tests/fixtures/build-path/docker-compose.yml b/tests/fixtures/build-path/docker-compose.yml new file mode 100644 index 000000000..66e8916e9 --- /dev/null +++ b/tests/fixtures/build-path/docker-compose.yml @@ -0,0 +1,2 @@ +foo: + build: ../build-ctx/ diff --git a/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml b/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml index a10381187..786315020 100644 --- a/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml +++ b/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml @@ -1,2 +1,2 @@ service: - build: tests/fixtures/dockerfile_with_entrypoint + build: . diff --git a/tests/fixtures/simple-dockerfile/docker-compose.yml b/tests/fixtures/simple-dockerfile/docker-compose.yml index a3f56d46f..b0357541e 100644 --- a/tests/fixtures/simple-dockerfile/docker-compose.yml +++ b/tests/fixtures/simple-dockerfile/docker-compose.yml @@ -1,2 +1,2 @@ simple: - build: tests/fixtures/simple-dockerfile + build: . diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index d95ba7838..f25f3a9d6 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -381,3 +381,36 @@ class ExtendsTest(unittest.TestCase): ] self.assertEqual(set(dicts[0]['volumes']), set(paths)) + + +class BuildPathTest(unittest.TestCase): + def setUp(self): + self.abs_context_path = os.path.join(os.getcwd(), 'tests/fixtures/build-ctx') + + def test_nonexistent_path(self): + options = {'build': 'nonexistent.path'} + self.assertRaises( + config.ConfigurationError, + lambda: config.make_service_dict('foo', options, 'tests/fixtures/build-path'), + ) + + def test_relative_path(self): + relative_build_path = '../build-ctx/' + service_dict = config.make_service_dict( + 'relpath', + {'build': relative_build_path}, + working_dir='tests/fixtures/build-path' + ) + self.assertEquals(service_dict['build'], self.abs_context_path) + + def test_absolute_path(self): + service_dict = config.make_service_dict( + 'abspath', + {'build': self.abs_context_path}, + working_dir='tests/fixtures/build-path' + ) + self.assertEquals(service_dict['build'], self.abs_context_path) + + def test_from_file(self): + service_dict = config.load('tests/fixtures/build-path/docker-compose.yml') + self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}]) From 78227c3c068a3ca7be47d3104fceb8c1e065e078 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 27 Mar 2015 14:59:49 -0700 Subject: [PATCH 04/53] Merge pull request #1202 from aanand/jenkins-script WIP: Jenkins script (cherry picked from commit 853ce255eac5375562e399d3e105dc5a456dbb99) Signed-off-by: Aanand Prasad --- Dockerfile | 3 +++ script/build-linux | 18 +++++++++++------- script/build-linux-inner | 10 ++++++++++ script/ci | 18 ++++++++++++++++++ script/test-versions | 5 +---- script/wrapdocker | 2 +- 6 files changed, 44 insertions(+), 12 deletions(-) create mode 100755 script/build-linux-inner create mode 100755 script/ci diff --git a/Dockerfile b/Dockerfile index d7a6019aa..8ec05cc9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,9 @@ RUN set -ex; \ chmod +x /usr/local/bin/docker-$v; \ done +# Set the default Docker to be run +RUN ln -s /usr/local/bin/docker-1.3.3 /usr/local/bin/docker + RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ diff --git a/script/build-linux b/script/build-linux index 07c9d7ec6..5e4a9470e 100755 --- a/script/build-linux +++ b/script/build-linux @@ -1,8 +1,12 @@ -#!/bin/sh +#!/bin/bash + set -ex -mkdir -p `pwd`/dist -chmod 777 `pwd`/dist -docker build -t docker-compose . -docker run -u user -v `pwd`/dist:/code/dist --rm --entrypoint pyinstaller docker-compose -F bin/docker-compose -mv dist/docker-compose dist/docker-compose-Linux-x86_64 -docker run -u user -v `pwd`/dist:/code/dist --rm --entrypoint dist/docker-compose-Linux-x86_64 docker-compose --version + +TAG="docker-compose" +docker build -t "$TAG" . +docker run \ + --rm \ + --user=user \ + --volume="$(pwd):/code" \ + --entrypoint="script/build-linux-inner" \ + "$TAG" diff --git a/script/build-linux-inner b/script/build-linux-inner new file mode 100755 index 000000000..34b0c06fd --- /dev/null +++ b/script/build-linux-inner @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +mkdir -p `pwd`/dist +chmod 777 `pwd`/dist + +pyinstaller -F bin/docker-compose +mv dist/docker-compose dist/docker-compose-Linux-x86_64 +dist/docker-compose-Linux-x86_64 --version diff --git a/script/ci b/script/ci new file mode 100755 index 000000000..a1391c627 --- /dev/null +++ b/script/ci @@ -0,0 +1,18 @@ +#!/bin/bash +# This should be run inside a container built from the Dockerfile +# at the root of the repo: +# +# $ TAG="docker-compose:$(git rev-parse --short HEAD)" +# $ docker build -t "$TAG" . +# $ docker run --rm --volume="/var/run/docker.sock:/var/run/docker.sock" --volume="$(pwd)/.git:/code/.git" -e "TAG=$TAG" --entrypoint="script/ci" "$TAG" + +set -e + +>&2 echo "Validating DCO" +script/validate-dco + +export DOCKER_VERSIONS=all +. script/test-versions + +>&2 echo "Building Linux binary" +su -c script/build-linux-inner user diff --git a/script/test-versions b/script/test-versions index a9e3bc4c7..a172b9a33 100755 --- a/script/test-versions +++ b/script/test-versions @@ -4,9 +4,6 @@ set -e ->&2 echo "Validating DCO" -script/validate-dco - >&2 echo "Running lint checks" flake8 compose @@ -18,7 +15,7 @@ fi for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" - docker-1.5.0 run \ + docker run \ --rm \ --privileged \ --volume="/var/lib/docker" \ diff --git a/script/wrapdocker b/script/wrapdocker index 20dc9e3ce..7b699688a 100755 --- a/script/wrapdocker +++ b/script/wrapdocker @@ -4,7 +4,7 @@ if [ "$DOCKER_VERSION" == "" ]; then DOCKER_VERSION="1.5.0" fi -ln -s "/usr/local/bin/docker-$DOCKER_VERSION" "/usr/local/bin/docker" +ln -fs "/usr/local/bin/docker-$DOCKER_VERSION" "/usr/local/bin/docker" # If a pidfile is still around (for example after a container restart), # delete it so that docker can start. From a467a8a09486e9770a4d7de4f982aeb17d8439b2 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 9 Apr 2015 14:44:07 +0100 Subject: [PATCH 05/53] Merge pull request #1261 from aanand/fix-vars-in-volume-paths Fix vars in volume paths (cherry picked from commit 4926f8aef629631032327542a56ae35099807005) Signed-off-by: Aanand Prasad Conflicts: tests/unit/service_test.py --- compose/config.py | 2 ++ compose/service.py | 4 +--- tests/integration/service_test.py | 18 ++++++++++++++++++ tests/unit/config_test.py | 14 ++++++++++++++ tests/unit/service_test.py | 16 ---------------- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/compose/config.py b/compose/config.py index 1dc64af25..2dc59d231 100644 --- a/compose/config.py +++ b/compose/config.py @@ -328,6 +328,8 @@ def resolve_host_paths(volumes, working_dir=None): def resolve_host_path(volume, working_dir): container_path, host_path = split_volume(volume) if host_path is not None: + host_path = os.path.expanduser(host_path) + host_path = os.path.expandvars(host_path) return "%s:%s" % (expand_path(working_dir, host_path), container_path) else: return container_path diff --git a/compose/service.py b/compose/service.py index 936e3f9d0..86427a1ea 100644 --- a/compose/service.py +++ b/compose/service.py @@ -3,7 +3,6 @@ from __future__ import absolute_import from collections import namedtuple import logging import re -import os from operator import attrgetter import sys import six @@ -586,8 +585,7 @@ def parse_repository_tag(s): def build_volume_binding(volume_spec): internal = {'bind': volume_spec.internal, 'ro': volume_spec.mode == 'ro'} - external = os.path.expanduser(volume_spec.external) - return os.path.abspath(os.path.expandvars(external)), internal + return volume_spec.external, internal def build_port_bindings(ports): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index f0fb771d9..a89fde97b 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -123,6 +123,24 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) + @mock.patch.dict(os.environ) + def test_create_container_with_home_and_env_var_in_volume_path(self): + os.environ['VOLUME_NAME'] = 'my-volume' + os.environ['HOME'] = '/tmp/home-dir' + expected_host_path = os.path.join(os.environ['HOME'], os.environ['VOLUME_NAME']) + + host_path = '~/${VOLUME_NAME}' + container_path = '/container-path' + + service = self.create_service('db', volumes=['%s:%s' % (host_path, container_path)]) + container = service.create_container() + service.start_container(container) + + actual_host_path = container.get('Volumes')[container_path] + components = actual_host_path.split('/') + self.assertTrue(components[-2:] == ['home-dir', 'my-volume'], + msg="Last two components differ: %s, %s" % (actual_host_path, expected_host_path)) + def test_create_container_with_volumes_from(self): volume_service = self.create_service('data') volume_container_1 = volume_service.create_container() diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index f25f3a9d6..aa14a2a5e 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -39,6 +39,20 @@ class ConfigTest(unittest.TestCase): config.make_service_dict('foo', {'ports': ['8000']}) +class VolumePathTest(unittest.TestCase): + @mock.patch.dict(os.environ) + def test_volume_binding_with_environ(self): + os.environ['VOLUME_PATH'] = '/host/path' + d = config.make_service_dict('foo', {'volumes': ['${VOLUME_PATH}:/container/path']}, working_dir='.') + self.assertEqual(d['volumes'], ['/host/path:/container/path']) + + @mock.patch.dict(os.environ) + def test_volume_binding_with_home(self): + os.environ['HOME'] = '/home/user' + d = config.make_service_dict('foo', {'volumes': ['~:/container/path']}, working_dir='.') + self.assertEqual(d['volumes'], ['/home/user:/container/path']) + + class MergeVolumesTest(unittest.TestCase): def test_empty(self): service_dict = config.merge_service_dicts({}, {}) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c70c30bfa..24222dfe9 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals from __future__ import absolute_import -import os from .. import unittest import mock @@ -301,18 +300,3 @@ class ServiceVolumesTest(unittest.TestCase): self.assertEqual( binding, ('/outside', dict(bind='/inside', ro=False))) - - @mock.patch.dict(os.environ) - def test_build_volume_binding_with_environ(self): - os.environ['VOLUME_PATH'] = '/opt' - binding = build_volume_binding(parse_volume_spec('${VOLUME_PATH}:/opt')) - self.assertEqual(binding, ('/opt', dict(bind='/opt', ro=False))) - - @mock.patch.dict(os.environ) - def test_building_volume_binding_with_home(self): - os.environ['HOME'] = '/home/user' - binding = build_volume_binding(parse_volume_spec('~:/home/user')) - self.assertEqual( - binding, - ('/home/user', dict(bind='/home/user', ro=False))) - From b6acb3cd8cec598504a4f25a2f91383e71d61701 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 14 Apr 2015 11:04:03 -0400 Subject: [PATCH 06/53] Merge pull request #1278 from albers/completion-run-user Add bash completion for docker-compose run --user (cherry picked from commit 3cd116b99d71f0e0da84e77797392e12070734e1) Signed-off-by: Aanand Prasad --- contrib/completion/bash/docker-compose | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index af3368036..548773d61 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -232,14 +232,14 @@ _docker-compose_run() { compopt -o nospace return ;; - --entrypoint) + --entrypoint|--user|-u) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "--allow-insecure-ssl -d --entrypoint -e --no-deps --rm --service-ports -T" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--allow-insecure-ssl -d --entrypoint -e --no-deps --rm --service-ports -T --user -u" -- "$cur" ) ) ;; *) __docker-compose_services_all From 39ae91c81c2dd8cddd2cbb3601dee8349a596340 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 23 Mar 2015 10:40:23 -0700 Subject: [PATCH 07/53] Bump 1.2.0 Signed-off-by: Aanand Prasad --- CHANGES.md | 23 +++++++++++++++++++++++ compose/__init__.py | 2 +- docs/completion.md | 2 +- docs/install.md | 2 +- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 75c130906..277a188a3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,29 @@ Change log ========== +1.2.0 (2015-04-16) +------------------ + +- `docker-compose.yml` now supports an `extends` option, which enables a service to inherit configuration from another service in another configuration file. This is really good for sharing common configuration between apps, or for configuring the same app for different environments. Here's the [documentation](https://github.com/docker/compose/blob/master/docs/yml.md#extends). + +- When using Compose with a Swarm cluster, containers that depend on one another will be co-scheduled on the same node. This means that most Compose apps will now work out of the box, as long as they don't use `build`. + +- Repeated invocations of `docker-compose up` when using Compose with a Swarm cluster now work reliably. + +- Directories passed to `build`, filenames passed to `env_file` and volume host paths passed to `volumes` are now treated as relative to the *directory of the configuration file*, not the directory that `docker-compose` is being run in. In the majority of cases, those are the same, but if you use the `-f|--file` argument to specify a configuration file in another directory, **this is a breaking change**. + +- A service can now share another service's network namespace with `net: container:`. + +- `volumes_from` and `net: container:` entries are taken into account when resolving dependencies, so `docker-compose up ` will correctly start all dependencies of ``. + +- `docker-compose run` now accepts a `--user` argument to specify a user to run the command as, just like `docker run`. + +- The `up`, `stop` and `restart` commands now accept a `--timeout` (or `-t`) argument to specify how long to wait when attempting to gracefully stop containers, just like `docker stop`. + +- `docker-compose rm` now accepts `-f` as a shorthand for `--force`, just like `docker rm`. + +Thanks, @abesto, @albers, @alunduil, @dnephin, @funkyfuture, @gilclark, @IanVS, @KingsleyKelly, @knutwalker, @thaJeztah and @vmalloc! + 1.1.0 (2015-02-25) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index c770b3950..2c426c781 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import unicode_literals from .service import Service # noqa:flake8 -__version__ = '1.1.0' +__version__ = '1.2.0' diff --git a/docs/completion.md b/docs/completion.md index d9b94f6cf..6ac95c2ef 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -17,7 +17,7 @@ On a Mac, install with `brew install bash-completion` Place the completion script in `/etc/bash_completion.d/` (`/usr/local/etc/bash_completion.d/` on a Mac), using e.g. - curl -L https://raw.githubusercontent.com/docker/compose/1.1.0/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose + curl -L https://raw.githubusercontent.com/docker/compose/1.2.0/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose Completion will be available upon next login. diff --git a/docs/install.md b/docs/install.md index 0e60e1f18..064ddc5f1 100644 --- a/docs/install.md +++ b/docs/install.md @@ -20,7 +20,7 @@ First, install Docker version 1.3 or greater: To install Compose, run the following commands: - curl -L https://github.com/docker/compose/releases/download/1.1.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.2.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose Optionally, you can also install [command completion](completion.md) for the From 43af1684c143ad8d07a5132fb376d4812497a6d3 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 17 Apr 2015 16:02:57 +0100 Subject: [PATCH 08/53] Update docs for 1.2.0 Signed-off-by: Aanand Prasad --- docs/extends.md | 364 +++++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 30 ++++ docs/install.md | 6 +- docs/mkdocs.yml | 2 + docs/production.md | 77 ++++++++++ docs/yml.md | 54 +++---- 6 files changed, 495 insertions(+), 38 deletions(-) create mode 100644 docs/extends.md create mode 100644 docs/production.md diff --git a/docs/extends.md b/docs/extends.md new file mode 100644 index 000000000..2393ca6ae --- /dev/null +++ b/docs/extends.md @@ -0,0 +1,364 @@ +page_title: Extending services in Compose +page_description: How to use Docker Compose's "extends" keyword to share configuration between files and projects +page_keywords: fig, composition, compose, docker, orchestration, documentation, docs + + +## Extending services in Compose + +Docker Compose's `extends` keyword enables sharing of common configurations +among different files, or even different projects entirely. Extending services +is useful if you have several applications that reuse commonly-defined services. +Using `extends` you can define a service in one place and refer to it from +anywhere. + +Alternatively, you can deploy the same application to multiple environments with +a slightly different set of services in each case (or with changes to the +configuration of some services). Moreover, you can do so without copy-pasting +the configuration around. + +### Understand the extends configuration + +When defining any service in `docker-compose.yml`, you can declare that you are +extending another service like this: + +```yaml +web: + extends: + file: common-services.yml + service: webapp +``` + +This instructs Compose to re-use the configuration for the `webapp` service +defined in the `common-services.yml` file. Suppose that `common-services.yml` +looks like this: + +```yaml +webapp: + build: . + ports: + - "8000:8000" + volumes: + - "/data" +``` + +In this case, you'll get exactly the same result as if you wrote +`docker-compose.yml` with that `build`, `ports` and `volumes` configuration +defined directly under `web`. + +You can go further and define (or re-define) configuration locally in +`docker-compose.yml`: + +```yaml +web: + extends: + file: common-services.yml + service: webapp + environment: + - DEBUG=1 + cpu_shares: 5 +``` + +You can also write other services and link your `web` service to them: + +```yaml +web: + extends: + file: common-services.yml + service: webapp + environment: + - DEBUG=1 + cpu_shares: 5 + links: + - db +db: + image: postgres +``` + +For full details on how to use `extends`, refer to the [reference](#reference). + +### Example use case + +In this example, you’ll repurpose the example app from the [quick start +guide](index.md). (If you're not familiar with Compose, it's recommended that +you go through the quick start first.) This example assumes you want to use +Compose both to develop an application locally and then deploy it to a +production environment. + +The local and production environments are similar, but there are some +differences. In development, you mount the application code as a volume so that +it can pick up changes; in production, the code should be immutable from the +outside. This ensures it’s not accidentally changed. The development environment +uses a local Redis container, but in production another team manages the Redis +service, which is listening at `redis-production.example.com`. + +To configure with `extends` for this sample, you must: + +1. Define the web application as a Docker image in `Dockerfile` and a Compose + service in `common.yml`. + +2. Define the development environment in the standard Compose file, + `docker-compose.yml`. + + - Use `extends` to pull in the web service. + - Configure a volume to enable code reloading. + - Create an additional Redis service for the application to use locally. + +3. Define the production environment in a third Compose file, `production.yml`. + + - Use `extends` to pull in the web service. + - Configure the web service to talk to the external, production Redis service. + +#### Define the web app + +Defining the web application requires the following: + +1. Create an `app.py` file. + + This file contains a simple Python application that uses Flask to serve HTTP + and increments a counter in Redis: + + from flask import Flask + from redis import Redis + import os + + app = Flask(__name__) + redis = Redis(host=os.environ['REDIS_HOST'], port=6379) + + @app.route('/') + def hello(): + redis.incr('hits') + return 'Hello World! I have been seen %s times.\n' % redis.get('hits') + + if __name__ == "__main__": + app.run(host="0.0.0.0", debug=True) + + This code uses a `REDIS_HOST` environment variable to determine where to + find Redis. + +2. Define the Python dependencies in a `requirements.txt` file: + + flask + redis + +3. Create a `Dockerfile` to build an image containing the app: + + FROM python:2.7 + ADD . /code + WORKDIR /code + RUN pip install -r + requirements.txt + CMD python app.py + +4. Create a Compose configuration file called `common.yml`: + + This configuration defines how to run the app. + + web: + build: . + ports: + - "5000:5000" + + Typically, you would have dropped this configuration into + `docker-compose.yml` file, but in order to pull it into multiple files with + `extends`, it needs to be in a separate file. + +#### Define the development environment + +1. Create a `docker-compose.yml` file. + + The `extends` option pulls in the `web` service from the `common.yml` file + you created in the previous section. + + web: + extends: + file: common.yml + service: web + volumes: + - .:/code + links: + - redis + environment: + - REDIS_HOST=redis + redis: + image: redis + + The new addition defines a `web` service that: + + - Fetches the base configuration for `web` out of `common.yml`. + - Adds `volumes` and `links` configuration to the base (`common.yml`) + configuration. + - Sets the `REDIS_HOST` environment variable to point to the linked redis + container. This environment uses a stock `redis` image from the Docker Hub. + +2. Run `docker-compose up`. + + Compose creates, links, and starts a web and redis container linked together. + It mounts your application code inside the web container. + +3. Verify that the code is mounted by changing the message in + `app.py`—say, from `Hello world!` to `Hello from Compose!`. + + Don't forget to refresh your browser to see the change! + +#### Define the production environment + +You are almost done. Now, define your production environment: + +1. Create a `production.yml` file. + + As with `docker-compose.yml`, the `extends` option pulls in the `web` service + from `common.yml`. + + web: + extends: + file: common.yml + service: web + environment: + - REDIS_HOST=redis-production.example.com + +2. Run `docker-compose -f production.yml up`. + + Compose creates *just* a web container and configures the Redis connection via + the `REDIS_HOST` environment variable. This variable points to the production + Redis instance. + + > **Note**: If you try to load up the webapp in your browser you'll get an + > error—`redis-production.example.com` isn't actually a Redis server. + +You've now done a basic `extends` configuration. As your application develops, +you can make any necessary changes to the web service in `common.yml`. Compose +picks up both the development and production environments when you next run +`docker-compose`. You don't have to do any copy-and-paste, and you don't have to +manually keep both environments in sync. + + +### Reference + +You can use `extends` on any service together with other configuration keys. It +always expects a dictionary that should always contain two keys: `file` and +`service`. + +The `file` key specifies which file to look in. It can be an absolute path or a +relative one—if relative, it's treated as relative to the current file. + +The `service` key specifies the name of the service to extend, for example `web` +or `database`. + +You can extend a service that itself extends another. You can extend +indefinitely. Compose does not support circular references and `docker-compose` +returns an error if it encounters them. + +#### Adding and overriding configuration + +Compose copies configurations from the original service over to the local one, +**except** for `links` and `volumes_from`. These exceptions exist to avoid +implicit dependencies—you always define `links` and `volumes_from` +locally. This ensures dependencies between services are clearly visible when +reading the current file. Defining these locally also ensures changes to the +referenced file don't result in breakage. + +If a configuration option is defined in both the original service and the local +service, the local value either *override*s or *extend*s the definition of the +original service. This works differently for other configuration options. + +For single-value options like `image`, `command` or `mem_limit`, the new value +replaces the old value. **This is the default behaviour - all exceptions are +listed below.** + +```yaml +# original service +command: python app.py + +# local service +command: python otherapp.py + +# result +command: python otherapp.py +``` + +In the case of `build` and `image`, using one in the local service causes +Compose to discard the other, if it was defined in the original service. + +```yaml +# original service +build: . + +# local service +image: redis + +# result +image: redis +``` + +```yaml +# original service +image: redis + +# local service +build: . + +# result +build: . +``` + +For the **multi-value options** `ports`, `expose`, `external_links`, `dns` and +`dns_search`, Compose concatenates both sets of values: + +```yaml +# original service +expose: + - "3000" + +# local service +expose: + - "4000" + - "5000" + +# result +expose: + - "3000" + - "4000" + - "5000" +``` + +In the case of `environment`, Compose "merges" entries together with +locally-defined values taking precedence: + +```yaml +# original service +environment: + - FOO=original + - BAR=original + +# local service +environment: + - BAR=local + - BAZ=local + +# result +environment: + - FOO=original + - BAR=local + - BAZ=local +``` + +Finally, for `volumes`, Compose "merges" entries together with locally-defined +bindings taking precedence: + +```yaml +# original service +volumes: + - /original-dir/foo:/foo + - /original-dir/bar:/bar + +# local service +volumes: + - /local-dir/bar:/bar + - /local-dir/baz/:baz + +# result +volumes: + - /original-dir/foo:/foo + - /local-dir/bar:/bar + - /local-dir/baz/:baz +``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index a75e7285a..78d9de281 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,6 +5,8 @@ page_keywords: documentation, docs, docker, compose, orchestration, containers # Docker Compose +## Overview + Compose is a tool for defining and running complex applications with Docker. With Compose, you define a multi-container application in a single file, then spin your application up in a single command which does everything that needs to @@ -191,3 +193,31 @@ At this point, you have seen the basics of how Compose works. [Rails](rails.md), or [Wordpress](wordpress.md). - See the reference guides for complete details on the [commands](cli.md), the [configuration file](yml.md) and [environment variables](env.md). + +## Release Notes + +### Version 1.2.0 (April 7, 2015) + +For complete information on this release, see the [1.2.0 Milestone project page](https://github.com/docker/compose/wiki/1.2.0-Milestone-Project-Page). +In addition to bug fixes and refinements, this release adds the following: + +* The `extends` keyword, which adds the ability to extend services by sharing common configurations. For details, see +[PR #972](https://github.com/docker/compose/pull/1088). + +* Better integration with Swarm. Swarm will now schedule inter-dependent +containers on the same host. For details, see +[PR #972](https://github.com/docker/compose/pull/972). + +## Getting help + +Docker Compose is still in its infancy and under active development. If you need +help, would like to contribute, or simply want to talk about the project with +like-minded individuals, we have a number of open channels for communication. + +* To report bugs or file feature requests: please use the [issue tracker on Github](https://github.com/docker/compose/issues). + +* To talk about the project with people in real time: please join the `#docker-compose` channel on IRC. + +* To contribute code or documentation changes: please submit a [pull request on Github](https://github.com/docker/compose/pulls). + +For more information and resources, please visit the [Getting Help project page](https://docs.docker.com/project/get-help/). diff --git a/docs/install.md b/docs/install.md index 064ddc5f1..24928d74c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,5 +1,5 @@ page_title: Installing Compose -page_description: How to intall Docker Compose +page_description: How to install Docker Compose page_keywords: compose, orchestration, install, installation, docker, documentation @@ -23,6 +23,8 @@ To install Compose, run the following commands: curl -L https://github.com/docker/compose/releases/download/1.2.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose +> Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands above, then `exit`. + Optionally, you can also install [command completion](completion.md) for the bash shell. @@ -31,7 +33,7 @@ Compose can also be installed as a Python package: $ sudo pip install -U docker-compose -No further steps are required; Compose should now be successfully installed. +No further steps are required; Compose should now be successfully installed. You can test the installation by running `docker-compose --version`. ## Compose documentation diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 14335873d..428439bc4 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,5 +1,7 @@ - ['compose/index.md', 'User Guide', 'Docker Compose' ] +- ['compose/production.md', 'User Guide', 'Using Compose in production' ] +- ['compose/extends.md', 'User Guide', 'Extending services in Compose'] - ['compose/install.md', 'Installation', 'Docker Compose'] - ['compose/cli.md', 'Reference', 'Compose command line'] - ['compose/yml.md', 'Reference', 'Compose yml'] diff --git a/docs/production.md b/docs/production.md new file mode 100644 index 000000000..8524c99b8 --- /dev/null +++ b/docs/production.md @@ -0,0 +1,77 @@ +page_title: Using Compose in production +page_description: Guide to using Docker Compose in production +page_keywords: documentation, docs, docker, compose, orchestration, containers, production + + +## Using Compose in production + +While **Compose is not yet considered production-ready**, if you'd like to experiment and learn more about using it in production deployments, this guide +can help. +The project is actively working towards becoming +production-ready; to learn more about the progress being made, check out the +[roadmap](https://github.com/docker/compose/blob/master/ROADMAP.md) for details +on how it's coming along and what still needs to be done. + +When deploying to production, you'll almost certainly want to make changes to +your app configuration that are more appropriate to a live environment. These +changes may include: + +- Removing any volume bindings for application code, so that code stays inside + the container and can't be changed from outside +- Binding to different ports on the host +- Setting environment variables differently (e.g., to decrease the verbosity of + logging, or to enable email sending) +- Specifying a restart policy (e.g., `restart: always`) to avoid downtime +- Adding extra services (e.g., a log aggregator) + +For this reason, you'll probably want to define a separate Compose file, say +`production.yml`, which specifies production-appropriate configuration. + +> **Note:** The [extends](extends.md) keyword is useful for maintaining multiple +> Compose files which re-use common services without having to manually copy and +> paste. + +Once you've got an alternate configuration file, make Compose use it +by setting the `COMPOSE_FILE` environment variable: + + $ COMPOSE_FILE=production.yml + $ docker-compose up -d + +> **Note:** You can also use the file for a one-off command without setting +> an environment variable. You do this by passing the `-f` flag, e.g., +> `docker-compose -f production.yml up -d`. + +### Deploying changes + +When you make changes to your app code, you'll need to rebuild your image and +recreate your app's containers. To redeploy a service called +`web`, you would use: + + $ docker-compose build web + $ docker-compose up --no-deps -d web + +This will first rebuild the image for `web` and then stop, destroy, and recreate +*just* the `web` service. The `--no-deps` flag prevents Compose from also +recreating any services which `web` depends on. + +### Running Compose on a single server + +You can use Compose to deploy an app to a remote Docker host by setting the +`DOCKER_HOST`, `DOCKER_TLS_VERIFY`, and `DOCKER_CERT_PATH` environment variables +appropriately. For tasks like this, +[Docker Machine](https://docs.docker.com/machine) makes managing local and +remote Docker hosts very easy, and is recommended even if you're not deploying +remotely. + +Once you've set up your environment variables, all the normal `docker-compose` +commands will work with no further configuration. + +### Running Compose on a Swarm cluster + +[Docker Swarm](https://docs.docker.com/swarm), a Docker-native clustering +system, exposes the same API as a single Docker host, which means you can use +Compose against a Swarm instance and run your apps across multiple hosts. + +Compose/Swarm integration is still in the experimental stage, and Swarm is still +in beta, but if you'd like to explore and experiment, check out the +[integration guide](https://github.com/docker/compose/blob/master/SWARM.md). diff --git a/docs/yml.md b/docs/yml.md index a9909e816..c375648df 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -173,8 +173,12 @@ env_file: - /opt/secrets.env ``` +Compose expects each line in an env file to be in `VAR=VAL` format. Lines +beginning with `#` (i.e. comments) are ignored, as are blank lines. + ``` -RACK_ENV: development +# Set Rails/Rack environment +RACK_ENV=development ``` ### extends @@ -217,42 +221,10 @@ Here, the `web` service in **development.yml** inherits the configuration of the `webapp` service in **common.yml** - the `build` and `environment` keys - and adds `ports` and `links` configuration. It overrides one of the defined environment variables (DEBUG) with a new value, and the other one -(SEND_EMAILS) is left untouched. It's exactly as if you defined `web` like -this: +(SEND_EMAILS) is left untouched. -```yaml -web: - build: ./webapp - ports: - - "8000:8000" - links: - - db - environment: - - DEBUG=true - - SEND_EMAILS=false -``` - -The `extends` option is great for sharing configuration between different -apps, or for configuring the same app differently for different environments. -You could write a new file for a staging environment, **staging.yml**, which -binds to a different port and doesn't turn on debugging: - -``` -web: - extends: - file: common.yml - service: webapp - ports: - - "80:8000" - links: - - db -db: - image: postgres -``` - -> **Note:** When you extend a service, `links` and `volumes_from` -> configuration options are **not** inherited - you will have to define -> those manually each time you extend it. +For more on `extends`, see the [tutorial](extends.md#example) and +[reference](extends.md#reference). ### net @@ -264,6 +236,16 @@ net: "none" net: "container:[name or id]" net: "host" ``` +### pid + +``` +pid: "host" +``` + +Sets the PID mode to the host PID mode. This turns on sharing between +container and the host operating system the PID address space. Containers +launched with this flag will be able to access and manipulate other +containers in the bare-metal machine's namespace and vise-versa. ### dns From 686c25d50ff3822e0f1515cf6aa0c13de97a4368 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 27 May 2015 15:13:12 +0100 Subject: [PATCH 09/53] Script to prepare OSX build environment Signed-off-by: Aanand Prasad --- CONTRIBUTING.md | 12 ++++++++---- script/build-osx | 2 +- script/prepare-osx | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) create mode 100755 script/prepare-osx diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 373c8dc6f..fddf888dc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,16 +53,20 @@ you can specify a test directory, file, module, class or method: ## Building binaries -Linux: +`script/build-linux` will build the Linux binary inside a Docker container: $ script/build-linux -OS X: +`script/build-osx` will build the Mac OS X binary inside a virtualenv: $ script/build-osx -Note that this only works on Mountain Lion, not Mavericks, due to a -[bug in PyInstaller](http://www.pyinstaller.org/ticket/807). +For official releases, you should build inside a Mountain Lion VM for proper +compatibility. Run the this script first to prepare the environment before +building - it will use Homebrew to make sure Python is installed and +up-to-date. + + $ script/prepare-osx ## Release process diff --git a/script/build-osx b/script/build-osx index 26309744a..6ad00bcdb 100755 --- a/script/build-osx +++ b/script/build-osx @@ -1,7 +1,7 @@ #!/bin/bash set -ex rm -rf venv -virtualenv venv +virtualenv -p /usr/local/bin/python venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-dev.txt venv/bin/pip install . diff --git a/script/prepare-osx b/script/prepare-osx new file mode 100755 index 000000000..69ac56f1c --- /dev/null +++ b/script/prepare-osx @@ -0,0 +1,22 @@ +#!/bin/bash + +set -ex + +if !(which brew); then + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +fi + +brew update + +if [ ! -f /usr/local/bin/python ]; then + brew install python +fi + +if [ -n "$(brew outdated | grep python)" ]; then + brew upgrade python +fi + +if !(which virtualenv); then + pip install virtualenv +fi + From 93a846db318bbf7e332db39f0ed7a764053948d6 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 28 May 2015 17:18:04 +0100 Subject: [PATCH 10/53] Report Python and OpenSSL versions in --version output Signed-off-by: Aanand Prasad Conflicts: compose/cli/utils.py --- compose/cli/main.py | 5 ++--- compose/cli/utils.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index a558e8359..61f3ec3f9 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -10,7 +10,6 @@ import sys from docker.errors import APIError import dockerpty -from .. import __version__ from .. import legacy from ..project import NoSuchService, ConfigurationError from ..service import BuildError, CannotBeScaledError, NeedsBuildError @@ -20,7 +19,7 @@ from .docopt_command import NoSuchCommand from .errors import UserError from .formatter import Formatter from .log_printer import LogPrinter -from .utils import yesno +from .utils import get_version_info, yesno log = logging.getLogger(__name__) @@ -104,7 +103,7 @@ class TopLevelCommand(Command): """ def docopt_options(self): options = super(TopLevelCommand, self).docopt_options() - options['version'] = "docker-compose %s" % __version__ + options['version'] = get_version_info() return options def build(self, project, options): diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 5f5fed64e..93b991038 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -5,6 +5,9 @@ import datetime import os import subprocess import platform +import ssl + +from .. import __version__ def yesno(prompt, default=None): @@ -120,3 +123,11 @@ def is_mac(): def is_ubuntu(): return platform.system() == 'Linux' and platform.linux_distribution()[0] == 'Ubuntu' + + +def get_version_info(): + return '\n'.join([ + 'docker-compose version: %s' % __version__, + "%s version: %s" % (platform.python_implementation(), platform.python_version()), + "OpenSSL version: %s" % ssl.OPENSSL_VERSION, + ]) From f3d0c63db2621a7bbe77164a23d11d3530bd5d19 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 28 May 2015 17:24:03 +0100 Subject: [PATCH 11/53] Make sure we use Python 2.7.9 and OpenSSL 1.0.1 when building OSX binary Signed-off-by: Aanand Prasad --- script/build-osx | 3 +++ script/prepare-osx | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/script/build-osx b/script/build-osx index 6ad00bcdb..d6561aeea 100755 --- a/script/build-osx +++ b/script/build-osx @@ -1,5 +1,8 @@ #!/bin/bash set -ex + +PATH="/usr/local/bin:$PATH" + rm -rf venv virtualenv -p /usr/local/bin/python venv venv/bin/pip install -r requirements.txt diff --git a/script/prepare-osx b/script/prepare-osx index 69ac56f1c..ca2776b64 100755 --- a/script/prepare-osx +++ b/script/prepare-osx @@ -2,20 +2,51 @@ set -ex +python_version() { + python -V 2>&1 +} + +openssl_version() { + python -c "import ssl; print ssl.OPENSSL_VERSION" +} + +desired_python_version="2.7.9" +desired_python_brew_version="2.7.9" +python_formula="https://raw.githubusercontent.com/Homebrew/homebrew/1681e193e4d91c9620c4901efd4458d9b6fcda8e/Library/Formula/python.rb" + +desired_openssl_version="1.0.1j" +desired_openssl_brew_version="1.0.1j_1" +openssl_formula="https://raw.githubusercontent.com/Homebrew/homebrew/62fc2a1a65e83ba9dbb30b2e0a2b7355831c714b/Library/Formula/openssl.rb" + +PATH="/usr/local/bin:$PATH" + if !(which brew); then ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi brew update -if [ ! -f /usr/local/bin/python ]; then - brew install python +if !(python_version | grep "$desired_python_version"); then + if brew list | grep python; then + brew unlink python + fi + + brew install "$python_formula" + brew switch python "$desired_python_brew_version" fi -if [ -n "$(brew outdated | grep python)" ]; then - brew upgrade python +if !(openssl_version | grep "$desired_openssl_version"); then + if brew list | grep openssl; then + brew unlink openssl + fi + + brew install "$openssl_formula" + brew switch openssl "$desired_openssl_brew_version" fi +echo "*** Using $(python_version)" +echo "*** Using $(openssl_version)" + if !(which virtualenv); then pip install virtualenv fi From 8749bc08443ddb344ecf683e796cdb2d814b7f68 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 1 Jun 2015 14:01:30 +0100 Subject: [PATCH 12/53] Build Python 2.7.9 in Docker image Signed-off-by: Aanand Prasad --- Dockerfile | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b2ae0063c..fca5f9803 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,11 @@ FROM debian:wheezy RUN set -ex; \ apt-get update -qq; \ apt-get install -y \ - python \ - python-pip \ - python-dev \ + gcc \ + make \ + zlib1g \ + zlib1g-dev \ + libssl-dev \ git \ apt-transport-https \ ca-certificates \ @@ -15,6 +17,37 @@ RUN set -ex; \ ; \ rm -rf /var/lib/apt/lists/* +# Build Python 2.7.9 from source +RUN set -ex; \ + curl -LO https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz; \ + tar -xzf Python-2.7.9.tgz; \ + cd Python-2.7.9; \ + ./configure --enable-shared; \ + make; \ + make install; \ + cd ..; \ + rm -rf /Python-2.7.9; \ + rm Python-2.7.9.tgz + +# Make libpython findable +ENV LD_LIBRARY_PATH /usr/local/lib + +# Install setuptools +RUN set -ex; \ + curl -LO https://bootstrap.pypa.io/ez_setup.py; \ + python ez_setup.py; \ + rm ez_setup.py + +# Install pip +RUN set -ex; \ + curl -LO https://pypi.python.org/packages/source/p/pip/pip-7.0.1.tar.gz; \ + tar -xzf pip-7.0.1.tar.gz; \ + cd pip-7.0.1; \ + python setup.py install; \ + cd ..; \ + rm -rf pip-7.0.1; \ + rm pip-7.0.1.tar.gz + ENV ALL_DOCKER_VERSIONS 1.6.0 RUN set -ex; \ From 5a5bffebd178670e602e2e9ea8c177bc32ef62b5 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 27 May 2015 12:49:58 +0100 Subject: [PATCH 13/53] Merge pull request #1464 from twhiteman/bug1461 Possible division by zero error when pulling an image - fixes #1463 (cherry picked from commit d0e87929a1f39b4e98c2c8497f3f0ffc09fb9e43) Signed-off-by: Aanand Prasad --- compose/progress_stream.py | 5 +++-- tests/unit/progress_stream_test.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 39aab5ff7..317c6e815 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -74,8 +74,9 @@ def print_output_event(event, stream, is_terminal): stream.write("%s %s%s" % (status, event['progress'], terminator)) elif 'progressDetail' in event: detail = event['progressDetail'] - if 'current' in detail: - percentage = float(detail['current']) / float(detail['total']) * 100 + total = detail.get('total') + if 'current' in detail and total: + percentage = float(detail['current']) / float(total) * 100 stream.write('%s (%.1f%%)%s' % (status, percentage, terminator)) else: stream.write('%s%s' % (status, terminator)) diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index 142560681..317b77e9f 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -17,3 +17,21 @@ class ProgressStreamTestCase(unittest.TestCase): ] events = progress_stream.stream_output(output, StringIO()) self.assertEqual(len(events), 1) + + def test_stream_output_div_zero(self): + output = [ + '{"status": "Downloading", "progressDetail": {"current": ' + '0, "start": 1413653874, "total": 0}, ' + '"progress": "..."}', + ] + events = progress_stream.stream_output(output, StringIO()) + self.assertEqual(len(events), 1) + + def test_stream_output_null_total(self): + output = [ + '{"status": "Downloading", "progressDetail": {"current": ' + '0, "start": 1413653874, "total": null}, ' + '"progress": "..."}', + ] + events = progress_stream.stream_output(output, StringIO()) + self.assertEqual(len(events), 1) From 4f4ea2a402a42c29c9867b02287dd7ded2d5b0d0 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Fri, 29 May 2015 14:45:21 +0100 Subject: [PATCH 14/53] Merge pull request #1325 from sdurrheimer/master Zsh completion for docker-compose (cherry picked from commit b638728d6ca21982e321b4069ef92f8367f069f4) Signed-off-by: Aanand Prasad Conflicts: docs/completion.md --- CONTRIBUTING.md | 1 - contrib/completion/zsh/_docker-compose | 304 +++++++++++++++++++++++++ docs/completion.md | 37 ++- 3 files changed, 333 insertions(+), 9 deletions(-) create mode 100644 contrib/completion/zsh/_docker-compose diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fddf888dc..6914e2159 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,6 @@ up-to-date. 1. Open pull request that: - Updates the version in `compose/__init__.py` - Updates the binary URL in `docs/install.md` - - Updates the script URL in `docs/completion.md` - Adds release notes to `CHANGES.md` 2. Create unpublished GitHub release with release notes 3. Build Linux version on any Docker host with `script/build-linux` and attach diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose new file mode 100644 index 000000000..31052e1e0 --- /dev/null +++ b/contrib/completion/zsh/_docker-compose @@ -0,0 +1,304 @@ +#compdef docker-compose + +# Description +# ----------- +# zsh completion for docker-compose +# https://github.com/sdurrheimer/docker-compose-zsh-completion +# ------------------------------------------------------------------------- +# Version +# ------- +# 0.1.0 +# ------------------------------------------------------------------------- +# Authors +# ------- +# * Steve Durrheimer +# ------------------------------------------------------------------------- +# Inspiration +# ----------- +# * @albers docker-compose bash completion script +# * @felixr docker zsh completion script : https://github.com/felixr/docker-zsh-completion +# ------------------------------------------------------------------------- + +# For compatibility reasons, Compose and therefore its completion supports several +# stack compositon files as listed here, in descending priority. +# Support for these filenames might be dropped in some future version. +__docker-compose_compose_file() { + local file + for file in docker-compose.y{,a}ml fig.y{,a}ml ; do + [ -e $file ] && { + echo $file + return + } + done + echo docker-compose.yml +} + +# Extracts all service names from docker-compose.yml. +___docker-compose_all_services_in_compose_file() { + local already_selected + local -a services + already_selected=$(echo ${words[@]} | tr " " "|") + awk -F: '/^[a-zA-Z0-9]/{print $1}' "${compose_file:-$(__docker-compose_compose_file)}" 2>/dev/null | grep -Ev "$already_selected" +} + +# All services, even those without an existing container +__docker-compose_services_all() { + services=$(___docker-compose_all_services_in_compose_file) + _alternative "args:services:($services)" +} + +# All services that have an entry with the given key in their docker-compose.yml section +___docker-compose_services_with_key() { + local already_selected + local -a buildable + already_selected=$(echo ${words[@]} | tr " " "|") + # flatten sections to one line, then filter lines containing the key and return section name. + awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' "${compose_file:-$(__docker-compose_compose_file)}" 2>/dev/null | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' 2>/dev/null | grep -Ev "$already_selected" +} + +# All services that are defined by a Dockerfile reference +__docker-compose_services_from_build() { + buildable=$(___docker-compose_services_with_key build) + _alternative "args:buildable services:($buildable)" +} + +# All services that are defined by an image +__docker-compose_services_from_image() { + pullable=$(___docker-compose_services_with_key image) + _alternative "args:pullable services:($pullable)" +} + +__docker-compose_get_services() { + local kind expl + declare -a running stopped lines args services + + docker_status=$(docker ps > /dev/null 2>&1) + if [ $? -ne 0 ]; then + _message "Error! Docker is not running." + return 1 + fi + + kind=$1 + shift + [[ $kind = (stopped|all) ]] && args=($args -a) + + lines=(${(f)"$(_call_program commands docker ps ${args})"}) + services=(${(f)"$(_call_program commands docker-compose 2>/dev/null ${compose_file:+-f $compose_file} ${compose_project:+-p $compose_project} ps -q)"}) + + # Parse header line to find columns + local i=1 j=1 k header=${lines[1]} + declare -A begin end + while (( $j < ${#header} - 1 )) { + i=$(( $j + ${${header[$j,-1]}[(i)[^ ]]} - 1)) + j=$(( $i + ${${header[$i,-1]}[(i) ]} - 1)) + k=$(( $j + ${${header[$j,-1]}[(i)[^ ]]} - 2)) + begin[${header[$i,$(($j-1))]}]=$i + end[${header[$i,$(($j-1))]}]=$k + } + lines=(${lines[2,-1]}) + + # Container ID + local line s name + local -a names + for line in $lines; do + if [[ $services == *"${line[${begin[CONTAINER ID]},${end[CONTAINER ID]}]%% ##}"* ]]; then + names=(${(ps:,:)${${line[${begin[NAMES]},-1]}%% *}}) + for name in $names; do + s="${${name%_*}#*_}:${(l:15:: :::)${${line[${begin[CREATED]},${end[CREATED]}]/ ago/}%% ##}}" + s="$s, ${line[${begin[CONTAINER ID]},${end[CONTAINER ID]}]%% ##}" + s="$s, ${${${line[$begin[IMAGE],$end[IMAGE]]}/:/\\:}%% ##}" + if [[ ${line[${begin[STATUS]},${end[STATUS]}]} = Exit* ]]; then + stopped=($stopped $s) + else + running=($running $s) + fi + done + fi + done + + [[ $kind = (running|all) ]] && _describe -t services-running "running services" running + [[ $kind = (stopped|all) ]] && _describe -t services-stopped "stopped services" stopped +} + +__docker-compose_stoppedservices() { + __docker-compose_get_services stopped "$@" +} + +__docker-compose_runningservices() { + __docker-compose_get_services running "$@" +} + +__docker-compose_services () { + __docker-compose_get_services all "$@" +} + +__docker-compose_caching_policy() { + oldp=( "$1"(Nmh+1) ) # 1 hour + (( $#oldp )) +} + +__docker-compose_commands () { + local cache_policy + + zstyle -s ":completion:${curcontext}:" cache-policy cache_policy + if [[ -z "$cache_policy" ]]; then + zstyle ":completion:${curcontext}:" cache-policy __docker-compose_caching_policy + fi + + if ( [[ ${+_docker_compose_subcommands} -eq 0 ]] || _cache_invalid docker_compose_subcommands) \ + && ! _retrieve_cache docker_compose_subcommands; + then + local -a lines + lines=(${(f)"$(_call_program commands docker-compose 2>&1)"}) + _docker_compose_subcommands=(${${${lines[$((${lines[(i)Commands:]} + 1)),${lines[(I) *]}]}## #}/ ##/:}) + _store_cache docker_compose_subcommands _docker_compose_subcommands + fi + _describe -t docker-compose-commands "docker-compose command" _docker_compose_subcommands +} + +__docker-compose_subcommand () { + local -a _command_args + integer ret=1 + case "$words[1]" in + (build) + _arguments \ + '--no-cache[Do not use cache when building the image]' \ + '*:services:__docker-compose_services_from_build' && ret=0 + ;; + (help) + _arguments ':subcommand:__docker-compose_commands' && ret=0 + ;; + (kill) + _arguments \ + '-s[SIGNAL to send to the container. Default signal is SIGKILL.]:signal:_signals' \ + '*:running services:__docker-compose_runningservices' && ret=0 + ;; + (logs) + _arguments \ + '--no-color[Produce monochrome output.]' \ + '*:services:__docker-compose_services_all' && ret=0 + ;; + (migrate-to-labels) + _arguments \ + '(-):Recreate containers to add labels' && ret=0 + ;; + (port) + _arguments \ + '--protocol=-[tcp or udap (defaults to tcp)]:protocol:(tcp udp)' \ + '--index=-[index of the container if there are mutiple instances of a service (defaults to 1)]:index: ' \ + '1:running services:__docker-compose_runningservices' \ + '2:port:_ports' && ret=0 + ;; + (ps) + _arguments \ + '-q[Only display IDs]' \ + '*:services:__docker-compose_services_all' && ret=0 + ;; + (pull) + _arguments \ + '--allow-insecure-ssl[Allow insecure connections to the docker registry]' \ + '*:services:__docker-compose_services_from_image' && ret=0 + ;; + (rm) + _arguments \ + '(-f --force)'{-f,--force}"[Don't ask to confirm removal]" \ + '-v[Remove volumes associated with containers]' \ + '*:stopped services:__docker-compose_stoppedservices' && ret=0 + ;; + (run) + _arguments \ + '--allow-insecure-ssl[Allow insecure connections to the docker registry]' \ + '-d[Detached mode: Run container in the background, print new container name.]' \ + '--entrypoint[Overwrite the entrypoint of the image.]:entry point: ' \ + '*-e[KEY=VAL Set an environment variable (can be used multiple times)]:environment variable KEY=VAL: ' \ + '(-u --user)'{-u,--user=-}'[Run as specified username or uid]:username or uid:_users' \ + "--no-deps[Don't start linked services.]" \ + '--rm[Remove container after run. Ignored in detached mode.]' \ + "--service-ports[Run command with the service's ports enabled and mapped to the host.]" \ + '-T[Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY.]' \ + '(-):services:__docker-compose_services' \ + '(-):command: _command_names -e' \ + '*::arguments: _normal' && ret=0 + ;; + (scale) + _arguments '*:running services:__docker-compose_runningservices' && ret=0 + ;; + (start) + _arguments '*:stopped services:__docker-compose_stoppedservices' && ret=0 + ;; + (stop|restart) + _arguments \ + '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ + '*:running services:__docker-compose_runningservices' && ret=0 + ;; + (up) + _arguments \ + '--allow-insecure-ssl[Allow insecure connections to the docker registry]' \ + '-d[Detached mode: Run containers in the background, print new container names.]' \ + '--no-color[Produce monochrome output.]' \ + "--no-deps[Don't start linked services.]" \ + "--no-recreate[If containers already exist, don't recreate them.]" \ + "--no-build[Don't build an image, even if it's missing]" \ + '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ + "--x-smart-recreate[Only recreate containers whose configuration or image needs to be updated. (EXPERIMENTAL)]" \ + '*:services:__docker-compose_services_all' && ret=0 + ;; + (*) + _message 'Unknown sub command' + esac + + return ret +} + +_docker-compose () { + # Support for subservices, which allows for `compdef _docker docker-shell=_docker_containers`. + # Based on /usr/share/zsh/functions/Completion/Unix/_git without support for `ret`. + if [[ $service != docker-compose ]]; then + _call_function - _$service + return + fi + + local curcontext="$curcontext" state line ret=1 + typeset -A opt_args + + _arguments -C \ + '(- :)'{-h,--help}'[Get help]' \ + '--verbose[Show more output]' \ + '(- :)'{-v,--version}'[Print version and exit]' \ + '(-f --file)'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ + '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ + '(-): :->command' \ + '(-)*:: :->option-or-argument' && ret=0 + + local counter=1 + #local compose_file compose_project + while [ $counter -lt ${#words[@]} ]; do + case "${words[$counter]}" in + -f|--file) + (( counter++ )) + compose_file="${words[$counter]}" + ;; + -p|--project-name) + (( counter++ )) + compose_project="${words[$counter]}" + ;; + *) + ;; + esac + (( counter++ )) + done + + case $state in + (command) + __docker-compose_commands && ret=0 + ;; + (option-or-argument) + curcontext=${curcontext%:*:*}:docker-compose-$words[1]: + __docker-compose_subcommand && ret=0 + ;; + esac + + return ret +} + +_docker-compose "$@" diff --git a/docs/completion.md b/docs/completion.md index 35c53b55f..5168971f8 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -3,23 +3,44 @@ layout: default title: Command Completion --- -#Command Completion +# Command Completion Compose comes with [command completion](http://en.wikipedia.org/wiki/Command-line_completion) -for the bash shell. +for the bash and zsh shell. -##Installing Command Completion +## Installing Command Completion + +### Bash Make sure bash completion is installed. If you use a current Linux in a non-minimal installation, bash completion should be available. On a Mac, install with `brew install bash-completion` - -Place the completion script in `/etc/bash_completion.d/` (`/usr/local/etc/bash_completion.d/` on a Mac), using e.g. - curl -L https://raw.githubusercontent.com/docker/compose/1.2.0/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose - +Place the completion script in `/etc/bash_completion.d/` (`/usr/local/etc/bash_completion.d/` on a Mac), using e.g. + + curl -L https://raw.githubusercontent.com/docker/compose/$(docker-compose --version | awk '{print $2}')/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose + Completion will be available upon next login. -##Available completions +### Zsh + +Place the completion script in your `/path/to/zsh/completion`, using e.g. `~/.zsh/completion/` + + mkdir -p ~/.zsh/completion + curl -L https://raw.githubusercontent.com/docker/compose/$(docker-compose --version | awk '{print $2}')/contrib/completion/zsh/_docker-compose > ~/.zsh/completion/_docker-compose + +Include the directory in your `$fpath`, e.g. by adding in `~/.zshrc` + + fpath=(~/.zsh/completion $fpath) + +Make sure `compinit` is loaded or do it by adding in `~/.zshrc` + + autoload -Uz compinit && compinit -i + +Then reload your shell + + exec $SHELL -l + +## Available completions Depending on what you typed on the command line so far, it will complete From 631f5be02fdc087420989bf820345406e6bc0c7b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sat, 30 May 2015 09:01:39 -0500 Subject: [PATCH 15/53] Merge pull request #1481 from albers/completion-smart-recreate Support --x-smart-recreate in bash completion (cherry picked from commit 9a0bb325f2d1203b7aac915c3bfca4347cc93489) Signed-off-by: Aanand Prasad --- contrib/completion/bash/docker-compose | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index e62b1d8fc..ba3dff352 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -104,7 +104,7 @@ _docker-compose_docker-compose() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help -h --verbose --version --file -f --project-name -p" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help -h --verbose --version -v --file -f --project-name -p" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -293,7 +293,7 @@ _docker-compose_up() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--allow-insecure-ssl -d --no-build --no-color --no-deps --no-recreate -t --timeout" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--allow-insecure-ssl -d --no-build --no-color --no-deps --no-recreate -t --timeout --x-smart-recreate" -- "$cur" ) ) ;; *) __docker-compose_services_all From 8ed7dfef6fb8d3f6eeeb4c515315e9ae43baee29 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 8 Jun 2015 12:48:46 -0400 Subject: [PATCH 16/53] Merge pull request #1525 from aanand/fix-duplicate-logging Fix duplicate logging on up/run (cherry picked from commit e2b790f7328482591863e496de14c825fd3f8a23) Signed-off-by: Aanand Prasad --- compose/cli/main.py | 1 + compose/service.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 61f3ec3f9..fa4013161 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -335,6 +335,7 @@ class TopLevelCommand(Command): container_options['ports'] = [] container = service.create_container( + quiet=True, one_off=True, insecure_registry=insecure_registry, **container_options diff --git a/compose/service.py b/compose/service.py index ccfb38511..dd931beee 100644 --- a/compose/service.py +++ b/compose/service.py @@ -199,6 +199,7 @@ class Service(object): do_build=True, previous_container=None, number=None, + quiet=False, **override_options): """ Create a container for this service. If the image doesn't exist, attempt to pull @@ -216,7 +217,7 @@ class Service(object): previous_container=previous_container, ) - if 'name' in container_options: + if 'name' in container_options and not quiet: log.info("Creating %s..." % container_options['name']) return Container.create(self.client, **container_options) @@ -378,6 +379,7 @@ class Service(object): do_build=False, previous_container=container, number=container.labels.get(LABEL_CONTAINER_NUMBER), + quiet=True, ) self.start_container(new_container) container.remove() From dca3bbdea3eb9991d804cc8b9ac9de34a367b866 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 8 Jun 2015 16:21:02 -0400 Subject: [PATCH 17/53] Merge pull request #1527 from aanand/remove-logging-on-run-rm Remove logging on run --rm (cherry picked from commit 5578ccbb0113e285a20aeeee820c03766ef1ae6e) Signed-off-by: Aanand Prasad --- compose/cli/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index fa4013161..7fde4ebaa 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -348,7 +348,6 @@ class TopLevelCommand(Command): dockerpty.start(project.client, container.id, interactive=not options['-T']) exit_code = container.wait() if options['--rm']: - log.info("Removing %s..." % container.name) project.client.remove_container(container.id) sys.exit(exit_code) From 8212f1bd45ab36d41895fbbd45490cbb68170187 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 9 Jun 2015 18:21:14 -0400 Subject: [PATCH 18/53] Merge pull request #1529 from aanand/update-dockerpty Update dockerpty to 0.3.4 (cherry picked from commit 95b2eaac042bb761b4f94c35a1af539467714098) Signed-off-by: Aanand Prasad --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b93988480..d3909b766 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PyYAML==3.10 docker-py==1.2.2 -dockerpty==0.3.3 +dockerpty==0.3.4 docopt==0.6.1 requests==2.6.1 six==1.7.3 diff --git a/setup.py b/setup.py index 153275f69..9364f57f3 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ install_requires = [ 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', 'docker-py >= 1.2.2, < 1.3', - 'dockerpty >= 0.3.3, < 0.4', + 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', ] From 71514cb380c157bbd7c34ad26697dc0638783d79 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 9 Jun 2015 22:22:56 -0400 Subject: [PATCH 19/53] Merge pull request #1531 from aanand/test-crash-resilience Test that data volumes now survive a crash when recreating (cherry picked from commit 87c30ae6e48c2341593b03770089e3ff86108881) Signed-off-by: Aanand Prasad --- tests/integration/resilience_test.py | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/integration/resilience_test.py diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py new file mode 100644 index 000000000..8229e9d3c --- /dev/null +++ b/tests/integration/resilience_test.py @@ -0,0 +1,37 @@ +from __future__ import unicode_literals +from __future__ import absolute_import + +import mock + +from compose.project import Project +from .testcases import DockerClientTestCase + + +class ResilienceTest(DockerClientTestCase): + def test_recreate_fails(self): + db = self.create_service('db', volumes=['/var/db'], command='top') + project = Project('composetest', [db], self.client) + + container = db.create_container() + db.start_container(container) + host_path = container.get('Volumes')['/var/db'] + + project.up() + container = db.containers()[0] + self.assertEqual(container.get('Volumes')['/var/db'], host_path) + + with mock.patch('compose.service.Service.create_container', crash): + with self.assertRaises(Crash): + project.up() + + project.up() + container = db.containers()[0] + self.assertEqual(container.get('Volumes')['/var/db'], host_path) + + +class Crash(Exception): + pass + + +def crash(*args, **kwargs): + raise Crash() From ca14ed68f7060ffc6e7856a66e7d6f4d3e245a74 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 10 Jun 2015 13:03:55 -0400 Subject: [PATCH 20/53] Merge pull request #1533 from edmorley/update-b2d-shellinit-example Docs: Update boot2docker shellinit example to use 'eval' (cherry picked from commit 17e03b29f9381a10f08e551f0c88899b7961664f) Signed-off-by: Aanand Prasad --- docs/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.md b/docs/cli.md index e5594871d..162189481 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -155,7 +155,7 @@ By default, if there are existing containers for a service, `docker-compose up` Several environment variables are available for you to configure Compose's behaviour. Variables starting with `DOCKER_` are the same as those used to configure the -Docker command-line client. If you're using boot2docker, `$(boot2docker shellinit)` +Docker command-line client. If you're using boot2docker, `eval "$(boot2docker shellinit)"` will set them to their correct values. ### COMPOSE\_PROJECT\_NAME From ad4cc5d6dfc0718d44bbcb6497f3483fc05b09f4 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 10 Jun 2015 17:19:24 -0400 Subject: [PATCH 21/53] Merge pull request #1497 from aanand/use-1.7-rc1 Run tests against Docker 1.7 RC2 (cherry picked from commit 0e9ccd36f3c672902a5241f557ed81df19255ccc) Signed-off-by: Aanand Prasad --- Dockerfile | 6 ++++-- tests/integration/cli_test.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index fca5f9803..1ff2d3825 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,11 +48,13 @@ RUN set -ex; \ rm -rf pip-7.0.1; \ rm pip-7.0.1.tar.gz -ENV ALL_DOCKER_VERSIONS 1.6.0 +ENV ALL_DOCKER_VERSIONS 1.6.0 1.7.0-rc2 RUN set -ex; \ curl https://get.docker.com/builds/Linux/x86_64/docker-1.6.0 -o /usr/local/bin/docker-1.6.0; \ - chmod +x /usr/local/bin/docker-1.6.0 + chmod +x /usr/local/bin/docker-1.6.0; \ + curl https://test.docker.com/builds/Linux/x86_64/docker-1.7.0-rc2 -o /usr/local/bin/docker-1.7.0-rc2; \ + chmod +x /usr/local/bin/docker-1.7.0-rc2 # Set the default Docker to be run RUN ln -s /usr/local/bin/docker-1.6.0 /usr/local/bin/docker diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 92789363e..4d33808cd 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import sys import os +import shlex from six import StringIO from mock import patch @@ -240,8 +241,8 @@ class CLITestCase(DockerClientTestCase): service = self.project.get_service(name) container = service.containers(stopped=True, one_off=True)[0] self.assertEqual( - container.human_readable_command, - u'/bin/echo helloworld' + shlex.split(container.human_readable_command), + [u'/bin/echo', u'helloworld'], ) @patch('dockerpty.start') From b7e8770c4fe8f67c0f50fcb0d39094f5db7e8d3d Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 11 Jun 2015 21:54:53 +0100 Subject: [PATCH 22/53] Merge pull request #1538 from thieman/tnt-serivce-misspelled Correct misspelling of "Service" in an error message (cherry picked from commit bd246fb011aa6805d57eb31d641e3c072c072d63) Signed-off-by: Aanand Prasad --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index d3deeeaf9..bc093628c 100644 --- a/compose/project.py +++ b/compose/project.py @@ -171,7 +171,7 @@ class Project(object): try: net = Container.from_id(self.client, net_name) except APIError: - raise ConfigurationError('Serivce "%s" is trying to use the network of "%s", which is not the name of a service or container.' % (service_dict['name'], net_name)) + raise ConfigurationError('Service "%s" is trying to use the network of "%s", which is not the name of a service or container.' % (service_dict['name'], net_name)) else: net = service_dict['net'] From cd7f67018e9fcb4ff423e344dba55fd96289ce48 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 4 Jun 2015 16:21:01 +0100 Subject: [PATCH 23/53] Merge pull request #1466 from noironetworks/changing-scale-to-warning Modified scale awareness from exception to warning (cherry picked from commit 7d2a89427c59774a8cbf503a57cb9f3b0d47d1fe) Signed-off-by: Aanand Prasad --- compose/cli/main.py | 12 ++---------- compose/service.py | 8 +++----- tests/integration/service_test.py | 5 ----- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 7fde4ebaa..0c3b85e5c 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -12,7 +12,7 @@ import dockerpty from .. import legacy from ..project import NoSuchService, ConfigurationError -from ..service import BuildError, CannotBeScaledError, NeedsBuildError +from ..service import BuildError, NeedsBuildError from ..config import parse_environment from .command import Command from .docopt_command import NoSuchCommand @@ -371,15 +371,7 @@ class TopLevelCommand(Command): except ValueError: raise UserError('Number of containers for service "%s" is not a ' 'number' % service_name) - try: - project.get_service(service_name).scale(num) - except CannotBeScaledError: - raise UserError( - 'Service "%s" cannot be scaled because it specifies a port ' - 'on the host. If multiple containers for this service were ' - 'created, the port would clash.\n\nRemove the ":" from the ' - 'port definition in docker-compose.yml so Docker can choose a random ' - 'port for each container.' % service_name) + project.get_service(service_name).scale(num) def start(self, project, options): """ diff --git a/compose/service.py b/compose/service.py index dd931beee..5d0d171d8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -55,10 +55,6 @@ class BuildError(Exception): self.reason = reason -class CannotBeScaledError(Exception): - pass - - class ConfigError(ValueError): pass @@ -154,7 +150,9 @@ class Service(object): - removes all stopped containers """ if not self.can_be_scaled(): - raise CannotBeScaledError() + log.warn('Service %s specifies a port on the host. If multiple containers ' + 'for this service are created on a single host, the port will clash.' + % self.name) # Create enough containers containers = self.containers(stopped=True) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 8fd8212ce..7e88557f9 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -17,7 +17,6 @@ from compose.const import ( LABEL_VERSION, ) from compose.service import ( - CannotBeScaledError, ConfigError, Service, build_extra_hosts, @@ -526,10 +525,6 @@ class ServiceTest(DockerClientTestCase): service.scale(0) self.assertEqual(len(service.containers()), 0) - def test_scale_on_service_that_cannot_be_scaled(self): - service = self.create_service('web', ports=['8000:8000']) - self.assertRaises(CannotBeScaledError, lambda: service.scale(1)) - def test_scale_sets_ports(self): service = self.create_service('web', ports=['8000']) service.scale(2) From 59d6af73fa964b2e1ce65964561d21b409194724 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 12 Jun 2015 11:56:02 -0400 Subject: [PATCH 24/53] Merge pull request #1539 from bfirsh/add-image-affinity-to-test Add image affinity to test script (cherry picked from commit 4c2112dbfd4da219f2585569b716b59f7562b034) Signed-off-by: Aanand Prasad --- script/test | 1 + 1 file changed, 1 insertion(+) diff --git a/script/test b/script/test index ab0645fdc..700de7779 100755 --- a/script/test +++ b/script/test @@ -12,6 +12,7 @@ docker run \ --volume="$(pwd):/code" \ -e DOCKER_VERSIONS \ -e "TAG=$TAG" \ + -e "affinity:image==$TAG" \ --entrypoint="script/test-versions" \ "$TAG" \ "$@" From 363a6563c7ed80731908658e8cc9cf431885bb1b Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Thu, 11 Jun 2015 21:55:33 +0100 Subject: [PATCH 25/53] Merge pull request #1537 from aanand/reorder-service-utils Reorder service.py utility methods (cherry picked from commit e3525d64b55ba6b95adab54ac0b5baf22d7740e0) Signed-off-by: Aanand Prasad --- compose/service.py | 135 ++++++++++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 57 deletions(-) diff --git a/compose/service.py b/compose/service.py index 5d0d171d8..8b4115173 100644 --- a/compose/service.py +++ b/compose/service.py @@ -708,6 +708,47 @@ class Service(object): stream_output(output, sys.stdout) +# Names + + +def build_container_name(project, service, number, one_off=False): + bits = [project, service] + if one_off: + bits.append('run') + return '_'.join(bits + [str(number)]) + + +# Images + + +def parse_repository_tag(s): + if ":" not in s: + return s, "" + repo, tag = s.rsplit(":", 1) + if "/" in tag: + return s, "" + return repo, tag + + +# Volumes + + +def merge_volume_bindings(volumes_option, previous_container): + """Return a list of volume bindings for a container. Container data volumes + are replaced by those from the previous container. + """ + volume_bindings = dict( + build_volume_binding(parse_volume_spec(volume)) + for volume in volumes_option or [] + if ':' in volume) + + if previous_container: + volume_bindings.update( + get_container_data_volumes(previous_container, volumes_option)) + + return volume_bindings + + def get_container_data_volumes(container, volumes_option): """Find the container data volumes that are in `volumes_option`, and return a mapping of volume bindings for those volumes. @@ -736,51 +777,9 @@ def get_container_data_volumes(container, volumes_option): return dict(volumes) -def merge_volume_bindings(volumes_option, previous_container): - """Return a list of volume bindings for a container. Container data volumes - are replaced by those from the previous container. - """ - volume_bindings = dict( - build_volume_binding(parse_volume_spec(volume)) - for volume in volumes_option or [] - if ':' in volume) - - if previous_container: - volume_bindings.update( - get_container_data_volumes(previous_container, volumes_option)) - - return volume_bindings - - -def build_container_name(project, service, number, one_off=False): - bits = [project, service] - if one_off: - bits.append('run') - return '_'.join(bits + [str(number)]) - - -def build_container_labels(label_options, service_labels, number, one_off=False): - labels = label_options or {} - labels.update(label.split('=', 1) for label in service_labels) - labels[LABEL_CONTAINER_NUMBER] = str(number) - labels[LABEL_VERSION] = __version__ - return labels - - -def parse_restart_spec(restart_config): - if not restart_config: - return None - parts = restart_config.split(':') - if len(parts) > 2: - raise ConfigError("Restart %s has incorrect format, should be " - "mode[:max_retry]" % restart_config) - if len(parts) == 2: - name, max_retry_count = parts - else: - name, = parts - max_retry_count = 0 - - return {'Name': name, 'MaximumRetryCount': int(max_retry_count)} +def build_volume_binding(volume_spec): + internal = {'bind': volume_spec.internal, 'ro': volume_spec.mode == 'ro'} + return volume_spec.external, internal def parse_volume_spec(volume_config): @@ -803,18 +802,7 @@ def parse_volume_spec(volume_config): return VolumeSpec(external, internal, mode) -def parse_repository_tag(s): - if ":" not in s: - return s, "" - repo, tag = s.rsplit(":", 1) - if "/" in tag: - return s, "" - return repo, tag - - -def build_volume_binding(volume_spec): - internal = {'bind': volume_spec.internal, 'ro': volume_spec.mode == 'ro'} - return volume_spec.external, internal +# Ports def build_port_bindings(ports): @@ -845,6 +833,39 @@ def split_port(port): return internal_port, (external_ip, external_port or None) +# Labels + + +def build_container_labels(label_options, service_labels, number, one_off=False): + labels = label_options or {} + labels.update(label.split('=', 1) for label in service_labels) + labels[LABEL_CONTAINER_NUMBER] = str(number) + labels[LABEL_VERSION] = __version__ + return labels + + +# Restart policy + + +def parse_restart_spec(restart_config): + if not restart_config: + return None + parts = restart_config.split(':') + if len(parts) > 2: + raise ConfigError("Restart %s has incorrect format, should be " + "mode[:max_retry]" % restart_config) + if len(parts) == 2: + name, max_retry_count = parts + else: + name, = parts + max_retry_count = 0 + + return {'Name': name, 'MaximumRetryCount': int(max_retry_count)} + + +# Extra hosts + + def build_extra_hosts(extra_hosts_config): if not extra_hosts_config: return {} From 8f8693e13ed6c6a3fe518bc1928efa1d536e19e0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 29 May 2015 12:38:40 +0100 Subject: [PATCH 26/53] Merge pull request #1480 from bfirsh/change-sigint-test-to-use-sigstop Change kill SIGINT test to use SIGSTOP (cherry picked from commit a15f996744b4005441b289f6b3fb4eef551b5214) Signed-off-by: Aanand Prasad --- tests/integration/cli_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 4d33808cd..cb7bc17fc 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -361,22 +361,22 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(service.containers(stopped=True)), 1) self.assertFalse(service.containers(stopped=True)[0].is_running) - def test_kill_signal_sigint(self): + def test_kill_signal_sigstop(self): self.command.dispatch(['up', '-d'], None) service = self.project.get_service('simple') self.assertEqual(len(service.containers()), 1) self.assertTrue(service.containers()[0].is_running) - self.command.dispatch(['kill', '-s', 'SIGINT'], None) + self.command.dispatch(['kill', '-s', 'SIGSTOP'], None) self.assertEqual(len(service.containers()), 1) - # The container is still running. It has been only interrupted + # The container is still running. It has only been paused self.assertTrue(service.containers()[0].is_running) - def test_kill_interrupted_service(self): + def test_kill_stopped_service(self): self.command.dispatch(['up', '-d'], None) service = self.project.get_service('simple') - self.command.dispatch(['kill', '-s', 'SIGINT'], None) + self.command.dispatch(['kill', '-s', 'SIGSTOP'], None) self.assertTrue(service.containers()[0].is_running) self.command.dispatch(['kill', '-s', 'SIGKILL'], None) From 4353f7b9f92bc0e6bebd1fa8cf647407890851b1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sat, 30 May 2015 08:43:48 -0500 Subject: [PATCH 27/53] Merge pull request #1475 from fordhurley/patch-1 Fix markdown formatting for `--service-ports` example (cherry picked from commit d64bf88e26f7b1ce097a6b475799364720bcb6cb) Signed-off-by: Aanand Prasad --- docs/cli.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/cli.md b/docs/cli.md index 162189481..1fbd4cb28 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -95,7 +95,9 @@ specify the `--no-deps` flag: Similarly, if you do want the service's ports to be created and mapped to the host, specify the `--service-ports` flag: - $ docker-compose run --service-ports web python manage.py shell + + $ docker-compose run --service-ports web python manage.py shell + ### scale From 58a7844129c9d3797fc11d1680b77b9e9b31577f Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 29 May 2015 17:12:57 +0100 Subject: [PATCH 28/53] Merge pull request #1482 from bfirsh/add-build-and-dist-to-dockerignore Make it possible to run tests remotely (cherry picked from commit c8e096e0895cb3589c4699daa44c299ea23f790c) Signed-off-by: Aanand Prasad --- .dockerignore | 2 ++ script/test | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index f1b636b3e..a03616e53 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,4 @@ .git +build +dist venv diff --git a/script/test b/script/test index 700de7779..625af09b3 100755 --- a/script/test +++ b/script/test @@ -9,7 +9,6 @@ docker build -t "$TAG" . docker run \ --rm \ --volume="/var/run/docker.sock:/var/run/docker.sock" \ - --volume="$(pwd):/code" \ -e DOCKER_VERSIONS \ -e "TAG=$TAG" \ -e "affinity:image==$TAG" \ From 87b4545b44350e7fe5164071ea975d4e4b5a4d91 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 4 Jun 2015 11:18:23 -0500 Subject: [PATCH 29/53] Merge pull request #1508 from thaJeztah/update-dockerproject-links Update dockerproject.com links (cherry picked from commit 417e6ce0c9f67cc719d5f3bfa9e3adbfb16a34eb) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index acd3cbe7a..4b18fc9dc 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,6 @@ Installation and documentation Contributing ------------ -[![Build Status](http://jenkins.dockerproject.com/buildStatus/icon?job=Compose Master)](http://jenkins.dockerproject.com/job/Compose%20Master/) +[![Build Status](http://jenkins.dockerproject.org/buildStatus/icon?job=Compose%20Master)](http://jenkins.dockerproject.org/job/Compose%20Master/) Want to help build Compose? Check out our [contributing documentation](https://github.com/docker/compose/blob/master/CONTRIBUTING.md). From e724a346c7c26b5b1c824ae4760bd414144c56e3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 8 Jun 2015 12:49:32 -0400 Subject: [PATCH 30/53] Merge pull request #1526 from aanand/remove-start-or-create-containers Remove Service.start_or_create_containers() (cherry picked from commit 38a11c4c28b1af644448d519544b876132ae89a8) Signed-off-by: Aanand Prasad --- compose/service.py | 15 --------------- tests/integration/service_test.py | 4 ++-- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/compose/service.py b/compose/service.py index 8b4115173..71edd5e5e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -394,21 +394,6 @@ class Service(object): container.start() return container - def start_or_create_containers( - self, - insecure_registry=False, - do_build=True): - containers = self.containers(stopped=True) - - if not containers: - new_container = self.create_container( - insecure_registry=insecure_registry, - do_build=do_build, - ) - return [self.start_container(new_container)] - else: - return [self.start_container_if_stopped(c) for c in containers] - def config_hash(self): return json_hash(self.config_dict()) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 7e88557f9..32de5fa47 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -501,10 +501,10 @@ class ServiceTest(DockerClientTestCase): ], }) - def test_start_with_image_id(self): + def test_create_with_image_id(self): # Image id for the current busybox:latest service = self.create_service('foo', image='8c2e06607696') - self.assertTrue(service.start_or_create_containers()) + service.create_container() def test_scale(self): service = self.create_service('web') From 67bc3fabe4d045dff44774a1d9681748d8f990e0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sun, 14 Jun 2015 13:28:14 -0400 Subject: [PATCH 31/53] Merge pull request #1544 from aanand/fix-volume-deduping Fix volume binds de-duplication (cherry picked from commit 77e594dc9405707ef8787728ae63ca091593f3ba) Signed-off-by: Aanand Prasad --- compose/service.py | 5 +-- requirements.txt | 2 +- tests/unit/service_test.py | 87 +++++++++++++++++++++++++++++++++----- 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/compose/service.py b/compose/service.py index 71edd5e5e..1e91a9f23 100644 --- a/compose/service.py +++ b/compose/service.py @@ -731,7 +731,7 @@ def merge_volume_bindings(volumes_option, previous_container): volume_bindings.update( get_container_data_volumes(previous_container, volumes_option)) - return volume_bindings + return volume_bindings.values() def get_container_data_volumes(container, volumes_option): @@ -763,8 +763,7 @@ def get_container_data_volumes(container, volumes_option): def build_volume_binding(volume_spec): - internal = {'bind': volume_spec.internal, 'ro': volume_spec.mode == 'ro'} - return volume_spec.external, internal + return volume_spec.internal, "{}:{}:{}".format(*volume_spec) def parse_volume_spec(volume_config): diff --git a/requirements.txt b/requirements.txt index d3909b766..47fa1e05b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.10 -docker-py==1.2.2 +docker-py==1.2.3-rc1 dockerpty==0.3.4 docopt==0.6.1 requests==2.6.1 diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index add48086d..fb3a7fcbb 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -331,9 +331,7 @@ class ServiceVolumesTest(unittest.TestCase): def test_build_volume_binding(self): binding = build_volume_binding(parse_volume_spec('/outside:/inside')) - self.assertEqual( - binding, - ('/outside', dict(bind='/inside', ro=False))) + self.assertEqual(binding, ('/inside', '/outside:/inside:rw')) def test_get_container_data_volumes(self): options = [ @@ -360,8 +358,8 @@ class ServiceVolumesTest(unittest.TestCase): }, has_been_inspected=True) expected = { - '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False}, - '/var/lib/docker/cccccccc': {'bind': '/mnt/image/data', 'ro': False}, + '/existing/volume': '/var/lib/docker/aaaaaaaa:/existing/volume:rw', + '/mnt/image/data': '/var/lib/docker/cccccccc:/mnt/image/data:rw', } binds = get_container_data_volumes(container, options) @@ -384,11 +382,78 @@ class ServiceVolumesTest(unittest.TestCase): 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'}, }, has_been_inspected=True) - expected = { - '/host/volume': {'bind': '/host/volume', 'ro': True}, - '/host/rw/volume': {'bind': '/host/rw/volume', 'ro': False}, - '/var/lib/docker/aaaaaaaa': {'bind': '/existing/volume', 'ro': False}, - } + expected = [ + '/host/volume:/host/volume:ro', + '/host/rw/volume:/host/rw/volume:rw', + '/var/lib/docker/aaaaaaaa:/existing/volume:rw', + ] binds = merge_volume_bindings(options, intermediate_container) - self.assertEqual(binds, expected) + self.assertEqual(set(binds), set(expected)) + + def test_mount_same_host_path_to_two_volumes(self): + service = Service( + 'web', + image='busybox', + volumes=[ + '/host/path:/data1', + '/host/path:/data2', + ], + client=self.mock_client, + ) + + self.mock_client.inspect_image.return_value = { + 'Id': 'ababab', + 'ContainerConfig': { + 'Volumes': {} + } + } + + create_options = service._get_container_create_options( + override_options={}, + number=1, + ) + + self.assertEqual( + set(create_options['host_config']['Binds']), + set([ + '/host/path:/data1:rw', + '/host/path:/data2:rw', + ]), + ) + + def test_different_host_path_in_container_json(self): + service = Service( + 'web', + image='busybox', + volumes=['/host/path:/data'], + client=self.mock_client, + ) + + self.mock_client.inspect_image.return_value = { + 'Id': 'ababab', + 'ContainerConfig': { + 'Volumes': { + '/data': {}, + } + } + } + + self.mock_client.inspect_container.return_value = { + 'Id': '123123123', + 'Image': 'ababab', + 'Volumes': { + '/data': '/mnt/sda1/host/path', + }, + } + + create_options = service._get_container_create_options( + override_options={}, + number=1, + previous_container=Container(self.mock_client, {'Id': '123123123'}), + ) + + self.assertEqual( + create_options['host_config']['Binds'], + ['/mnt/sda1/host/path:/data:rw'], + ) From 719954b02f8d6f03e20bde2409b074295dc5da98 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 15 Jun 2015 10:36:37 -0700 Subject: [PATCH 32/53] Merge pull request #1545 from moxiegirl/test-tooling Updated for new documentation tooling (cherry picked from commit aaccd12d3df2ab64f44db5c6cd8bae282a314419) Signed-off-by: Aanand Prasad --- docs/Dockerfile | 31 +++++++---- docs/Makefile | 55 ++++++++++++++++++ docs/README.md | 77 ++++++++++++++++++++++++++ docs/cli.md | 17 ++++-- docs/completion.md | 18 ++++-- docs/{index.md => compose-overview.md} | 16 ++++-- docs/django.md | 18 ++++-- docs/env.md | 18 ++++-- docs/extends.md | 17 ++++-- docs/install.md | 21 ++++--- docs/mkdocs.yml | 12 ---- docs/production.md | 13 ++++- docs/rails.md | 19 ++++--- docs/wordpress.md | 21 ++++--- docs/yml.md | 19 ++++--- 15 files changed, 283 insertions(+), 89 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/README.md rename docs/{index.md => compose-overview.md} (96%) delete mode 100644 docs/mkdocs.yml diff --git a/docs/Dockerfile b/docs/Dockerfile index 59ef66cdd..55e7ce700 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,15 +1,24 @@ -FROM docs/base:latest -MAINTAINER Sven Dowideit (@SvenDowideit) +FROM docs/base:hugo +MAINTAINER Mary Anthony (@moxiegirl) -# to get the git info for this repo +# To get the git info for this repo COPY . /src -# Reset the /docs dir so we can replace the theme meta with the new repo's git info -RUN git reset --hard +COPY . /docs/content/compose/ -RUN grep "__version" /src/compose/__init__.py | sed "s/.*'\(.*\)'/\1/" > /docs/VERSION -COPY docs/* /docs/sources/compose/ -COPY docs/mkdocs.yml /docs/mkdocs-compose.yml - -# Then build everything together, ready for mkdocs -RUN /docs/build.sh +# Sed to process GitHub Markdown +# 1-2 Remove comment code from metadata block +# 3 Remove .md extension from link text +# 4 Change ](/ to ](/project/ in links +# 5 Change ](word) to ](/project/word) +# 6 Change ](../../ to ](/project/ +# 7 Change ](../ to ](/project/word) +# +# +RUN find /docs/content/compose -type f -name "*.md" -exec sed -i.old \ + -e '/^/g' \ + -e '/^/g' \ + -e 's/\([(]\)\(.*\)\(\.md\)/\1\2/g' \ + -e 's/\(\]\)\([(]\)\(\/\)/\1\2\/compose\//g' \ + -e 's/\(\][(]\)\([A-z]*[)]\)/\]\(\/compose\/\2/g' \ + -e 's/\(\][(]\)\(\.\.\/\)/\1\/compose\//g' {} \; diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..021e8f6e5 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,55 @@ +.PHONY: all binary build cross default docs docs-build docs-shell shell test test-unit test-integration test-integration-cli test-docker-py validate + +# env vars passed through directly to Docker's build scripts +# to allow things like `make DOCKER_CLIENTONLY=1 binary` easily +# `docs/sources/contributing/devenvironment.md ` and `project/PACKAGERS.md` have some limited documentation of some of these +DOCKER_ENVS := \ + -e BUILDFLAGS \ + -e DOCKER_CLIENTONLY \ + -e DOCKER_EXECDRIVER \ + -e DOCKER_GRAPHDRIVER \ + -e TESTDIRS \ + -e TESTFLAGS \ + -e TIMEOUT +# note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds + +# to allow `make DOCSDIR=docs docs-shell` (to create a bind mount in docs) +DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR)/$(DOCSDIR):/$(DOCSDIR)) + +# to allow `make DOCSPORT=9000 docs` +DOCSPORT := 8000 + +# Get the IP ADDRESS +DOCKER_IP=$(shell python -c "import urlparse ; print urlparse.urlparse('$(DOCKER_HOST)').hostname or ''") +HUGO_BASE_URL=$(shell test -z "$(DOCKER_IP)" && echo localhost || echo "$(DOCKER_IP)") +HUGO_BIND_IP=0.0.0.0 + +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) +DOCKER_IMAGE := docker$(if $(GIT_BRANCH),:$(GIT_BRANCH)) +DOCKER_DOCS_IMAGE := docs-base$(if $(GIT_BRANCH),:$(GIT_BRANCH)) + + +DOCKER_RUN_DOCS := docker run --rm -it $(DOCS_MOUNT) -e AWS_S3_BUCKET -e NOCACHE + +# for some docs workarounds (see below in "docs-build" target) +GITCOMMIT := $(shell git rev-parse --short HEAD 2>/dev/null) + +default: docs + +docs: docs-build + $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) + +docs-draft: docs-build + $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --buildDrafts="true" --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) + + +docs-shell: docs-build + $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 "$(DOCKER_DOCS_IMAGE)" bash + + +docs-build: +# ( git remote | grep -v upstream ) || git diff --name-status upstream/release..upstream/docs ./ > ./changed-files +# echo "$(GIT_BRANCH)" > GIT_BRANCH +# echo "$(AWS_S3_BUCKET)" > AWS_S3_BUCKET +# echo "$(GITCOMMIT)" > GITCOMMIT + docker build -t "$(DOCKER_DOCS_IMAGE)" . diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..00736e476 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,77 @@ +# Contributing to the Docker Compose documentation + +The documentation in this directory is part of the [https://docs.docker.com](https://docs.docker.com) website. Docker uses [the Hugo static generator](http://gohugo.io/overview/introduction/) to convert project Markdown files to a static HTML site. + +You don't need to be a Hugo expert to contribute to the compose documentation. If you are familiar with Markdown, you can modify the content in the `docs` files. + +If you want to add a new file or change the location of the document in the menu, you do need to know a little more. + +## Documentation contributing workflow + +1. Edit a Markdown file in the tree. + +2. Save your changes. + +3. Make sure you in your `docs` subdirectory. + +4. Build the documentation. + + $ make docs + ---> ffcf3f6c4e97 + Removing intermediate container a676414185e8 + Successfully built ffcf3f6c4e97 + docker run --rm -it -e AWS_S3_BUCKET -e NOCACHE -p 8000:8000 -e DOCKERHOST "docs-base:test-tooling" hugo server --port=8000 --baseUrl=192.168.59.103 --bind=0.0.0.0 + ERROR: 2015/06/13 MenuEntry's .Url is deprecated and will be removed in Hugo 0.15. Use .URL instead. + 0 of 4 drafts rendered + 0 future content + 12 pages created + 0 paginator pages created + 0 tags created + 0 categories created + in 55 ms + Serving pages from /docs/public + Web Server is available at http://0.0.0.0:8000/ + Press Ctrl+C to stop + +5. Open the available server in your browser. + + The documentation server has the complete menu but only the Docker Compose + documentation resolves. You can't access the other project docs from this + localized build. + +## Tips on Hugo metadata and menu positioning + +The top of each Docker Compose documentation file contains TOML metadata. The metadata is commented out to prevent it from appears in GitHub. + + + +The metadata alone has this structure: + + +++ + title = "Extending services in Compose" + description = "How to use Docker Compose's extends keyword to share configuration between files and projects" + keywords = ["fig, composition, compose, docker, orchestration, documentation, docs"] + [menu.main] + parent="smn_workw_compose" + weight=2 + +++ + +The `[menu.main]` section refers to navigation defined [in the main Docker menu](https://github.com/docker/docs-base/blob/hugo/config.toml). This metadata says *add a menu item called* Extending services in Compose *to the menu with the* `smn_workdw_compose` *identifier*. If you locate the menu in the configuration, you'll find *Create multi-container applications* is the menu title. + +You can move an article in the tree by specifying a new parent. You can shift the location of the item by changing its weight. Higher numbers are heavier and shift the item to the bottom of menu. Low or no numbers shift it up. + + +## Other key documentation repositories + +The `docker/docs-base` repository contains [the Hugo theme and menu configuration](https://github.com/docker/docs-base). If you open the `Dockerfile` you'll see the `make docs` relies on this as a base image for building the Compose documentation. + +The `docker/docs.docker.com` repository contains [build system for building the Docker documentation site](https://github.com/docker/docs.docker.com). Fork this repository to build the entire documentation site. diff --git a/docs/cli.md b/docs/cli.md index 1fbd4cb28..a2167d9c3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,9 +1,16 @@ -page_title: Compose CLI reference -page_description: Compose CLI reference -page_keywords: fig, composition, compose, docker, orchestration, cli, reference + -# CLI reference +# Compose CLI reference Most Docker Compose commands are run against one or more services. If the service is not specified, the command will apply to all services. @@ -185,7 +192,7 @@ Configures the path to the `ca.pem`, `cert.pem`, and `key.pem` files used for TL ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/completion.md b/docs/completion.md index 5168971f8..7fb696d80 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -1,7 +1,13 @@ ---- -layout: default -title: Command Completion ---- + # Command Completion @@ -53,11 +59,11 @@ Enjoy working with Compose faster and with less typos! ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) - [Command line reference](cli.md) - [Yaml file reference](yml.md) -- [Compose environment variables](env.md) +- [Compose environment variables](env.md) \ No newline at end of file diff --git a/docs/index.md b/docs/compose-overview.md similarity index 96% rename from docs/index.md rename to docs/compose-overview.md index 981a02702..33629957a 100644 --- a/docs/index.md +++ b/docs/compose-overview.md @@ -1,11 +1,15 @@ -page_title: Compose: Multi-container orchestration for Docker -page_description: Introduction and Overview of Compose -page_keywords: documentation, docs, docker, compose, orchestration, containers + -# Docker Compose - -## Overview +# Overview of Docker Compose Compose is a tool for defining and running multi-container applications with Docker. With Compose, you define a multi-container application in a single diff --git a/docs/django.md b/docs/django.md index 4cbebe041..c44329e1c 100644 --- a/docs/django.md +++ b/docs/django.md @@ -1,10 +1,16 @@ -page_title: Quickstart Guide: Compose and Django -page_description: Getting started with Docker Compose and Django -page_keywords: documentation, docs, docker, compose, orchestration, containers, -django + -## Getting started with Compose and Django +## Quickstart Guide: Compose and Django This Quick-start Guide will demonstrate how to use Compose to set up and run a @@ -119,7 +125,7 @@ example, run `docker-compose up` and in another terminal run: ## More Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/env.md b/docs/env.md index a4b543ae3..73496f32f 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1,9 +1,15 @@ ---- -layout: default -title: Compose environment variables reference ---- + -Environment variables reference +# Compose environment variables reference =============================== **Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](yml.md#links) for details. @@ -34,7 +40,7 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/extends.md b/docs/extends.md index fd372ce2d..8527c81b3 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -1,6 +1,13 @@ -page_title: Extending services in Compose -page_description: How to use Docker Compose's "extends" keyword to share configuration between files and projects -page_keywords: fig, composition, compose, docker, orchestration, documentation, docs + ## Extending services in Compose @@ -79,7 +86,7 @@ For full details on how to use `extends`, refer to the [reference](#reference). ### Example use case In this example, you’ll repurpose the example app from the [quick start -guide](index.md). (If you're not familiar with Compose, it's recommended that +guide](compose-overview.md). (If you're not familiar with Compose, it's recommended that you go through the quick start first.) This example assumes you want to use Compose both to develop an application locally and then deploy it to a production environment. @@ -364,7 +371,7 @@ volumes: ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/install.md b/docs/install.md index a521ec06c..ec0e6e4d5 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,14 +1,21 @@ -page_title: Installing Compose -page_description: How to install Docker Compose -page_keywords: compose, orchestration, install, installation, docker, documentation + -## Installing Compose +# Install Docker Compose To install Compose, you'll need to install Docker first. You'll then install Compose with a `curl` command. -### Install Docker +## Install Docker First, install Docker version 1.6 or greater: @@ -16,7 +23,7 @@ First, install Docker version 1.6 or greater: - [Instructions for Ubuntu](http://docs.docker.com/installation/ubuntulinux/) - [Instructions for other systems](http://docs.docker.com/installation/) -### Install Compose +## Install Compose To install Compose, run the following commands: @@ -38,7 +45,7 @@ You can test the installation by running `docker-compose --version`. ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml deleted file mode 100644 index 428439bc4..000000000 --- a/docs/mkdocs.yml +++ /dev/null @@ -1,12 +0,0 @@ - -- ['compose/index.md', 'User Guide', 'Docker Compose' ] -- ['compose/production.md', 'User Guide', 'Using Compose in production' ] -- ['compose/extends.md', 'User Guide', 'Extending services in Compose'] -- ['compose/install.md', 'Installation', 'Docker Compose'] -- ['compose/cli.md', 'Reference', 'Compose command line'] -- ['compose/yml.md', 'Reference', 'Compose yml'] -- ['compose/env.md', 'Reference', 'Compose ENV variables'] -- ['compose/completion.md', 'Reference', 'Compose commandline completion'] -- ['compose/django.md', 'Examples', 'Getting started with Compose and Django'] -- ['compose/rails.md', 'Examples', 'Getting started with Compose and Rails'] -- ['compose/wordpress.md', 'Examples', 'Getting started with Compose and Wordpress'] diff --git a/docs/production.md b/docs/production.md index 60a6873da..294f3c4e8 100644 --- a/docs/production.md +++ b/docs/production.md @@ -1,6 +1,13 @@ -page_title: Using Compose in production -page_description: Guide to using Docker Compose in production -page_keywords: documentation, docs, docker, compose, orchestration, containers, production + ## Using Compose in production diff --git a/docs/rails.md b/docs/rails.md index aedb4c6e7..2ff6f1752 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -1,10 +1,15 @@ -page_title: Quickstart Guide: Compose and Rails -page_description: Getting started with Docker Compose and Rails -page_keywords: documentation, docs, docker, compose, orchestration, containers, -rails + - -## Getting started with Compose and Rails +## Quickstart Guide: Compose and Rails This Quickstart guide will show you how to use Compose to set up and run a Rails/PostgreSQL app. Before starting, you'll need to have [Compose installed](install.md). @@ -119,7 +124,7 @@ you're using Boot2docker, `boot2docker ip` will tell you its address). ## More Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index b40d1a9f0..ad0e62966 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -1,14 +1,21 @@ -page_title: Quickstart Guide: Compose and Wordpress -page_description: Getting started with Docker Compose and Rails -page_keywords: documentation, docs, docker, compose, orchestration, containers, -wordpress + -## Getting started with Compose and Wordpress + +# Quickstart Guide: Compose and Wordpress You can use Compose to easily run Wordpress in an isolated environment built with Docker containers. -### Define the project +## Define the project First, [Install Compose](install.md) and then download Wordpress into the current directory: @@ -114,7 +121,7 @@ address). ## More Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/yml.md b/docs/yml.md index df791bc98..80d6d719f 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -1,10 +1,13 @@ ---- -layout: default -title: docker-compose.yml reference -page_title: docker-compose.yml reference -page_description: docker-compose.yml reference -page_keywords: fig, composition, compose, docker ---- + + # docker-compose.yml reference @@ -390,7 +393,7 @@ read_only: true ## Compose documentation -- [User guide](index.md) +- [User guide](compose-overview.md) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) From 09018855cebceac34525122feb76c1885a5f4057 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 15 Jun 2015 13:43:57 -0400 Subject: [PATCH 33/53] Merge pull request #1550 from aanand/update-docker-py Update setup.py with new docker-py minimum (cherry picked from commit b3b44b8e4c7ee7463136bb13cf6c3d759e6d87e9) Signed-off-by: Aanand Prasad --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9364f57f3..a94d87374 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.7', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', - 'docker-py >= 1.2.2, < 1.3', + 'docker-py >= 1.2.3-rc1, < 1.3', 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', ] From f353d9fbc0df6835ef373c72342feae856e0276d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 15 Jun 2015 10:58:44 -0700 Subject: [PATCH 34/53] Merge pull request #1406 from vdemeester/667-compose-port-scale Fixing docker-compose port with scale (#667) (cherry picked from commit 5b2a0cc73d104340964b299c11723e465ea7c112) Signed-off-by: Aanand Prasad --- compose/cli/main.py | 7 +++--- .../docker-compose.yml | 6 +++++ tests/integration/cli_test.py | 22 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/ports-composefile-scale/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 0c3b85e5c..4f3f11e4e 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -169,13 +169,14 @@ class TopLevelCommand(Command): Usage: port [options] SERVICE PRIVATE_PORT Options: - --protocol=proto tcp or udp (defaults to tcp) + --protocol=proto tcp or udp [default: tcp] --index=index index of the container if there are multiple - instances of a service (defaults to 1) + instances of a service [default: 1] """ + index = int(options.get('--index')) service = project.get_service(options['SERVICE']) try: - container = service.get_container(number=options.get('--index') or 1) + container = service.get_container(number=index) except ValueError as e: raise UserError(str(e)) print(container.get_local_port( diff --git a/tests/fixtures/ports-composefile-scale/docker-compose.yml b/tests/fixtures/ports-composefile-scale/docker-compose.yml new file mode 100644 index 000000000..1a2bb485b --- /dev/null +++ b/tests/fixtures/ports-composefile-scale/docker-compose.yml @@ -0,0 +1,6 @@ + +simple: + image: busybox:latest + command: /bin/sleep 300 + ports: + - '3000' diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index cb7bc17fc..2d1f1f76e 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from operator import attrgetter import sys import os import shlex @@ -436,6 +437,27 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(get_port(3001), "0.0.0.0:49152") self.assertEqual(get_port(3002), "") + def test_port_with_scale(self): + + self.command.base_dir = 'tests/fixtures/ports-composefile-scale' + self.command.dispatch(['scale', 'simple=2'], None) + containers = sorted( + self.project.containers(service_names=['simple']), + key=attrgetter('name')) + + @patch('sys.stdout', new_callable=StringIO) + def get_port(number, mock_stdout, index=None): + if index is None: + self.command.dispatch(['port', 'simple', str(number)], None) + else: + self.command.dispatch(['port', '--index=' + str(index), 'simple', str(number)], None) + return mock_stdout.getvalue().rstrip() + + self.assertEqual(get_port(3000), containers[0].get_local_port(3000)) + self.assertEqual(get_port(3000, index=1), containers[0].get_local_port(3000)) + self.assertEqual(get_port(3000, index=2), containers[1].get_local_port(3000)) + self.assertEqual(get_port(3002), "") + def test_env_file_relative_to_compose_file(self): config_path = os.path.abspath('tests/fixtures/env-file/docker-compose.yml') self.command.dispatch(['-f', config_path, 'up', '-d'], None) From 7fa4cd1214deea8f61ce5195ecbe377f70d1e311 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 16 Jun 2015 16:26:40 -0700 Subject: [PATCH 35/53] Merge pull request #1552 from aanand/add-upgrade-instructions Add upgrading instructions to install docs (cherry picked from commit bc7161b475f7032bfc36e177935e9d7b13354718) --- docs/install.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/install.md b/docs/install.md index ec0e6e4d5..c1abd4fd6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -43,6 +43,18 @@ Compose can also be installed as a Python package: No further steps are required; Compose should now be successfully installed. You can test the installation by running `docker-compose --version`. +### Upgrading + +If you're coming from Compose 1.2 or earlier, you'll need to remove or migrate your existing containers after upgrading Compose. This is because, as of version 1.3, Compose uses Docker labels to keep track of containers, and so they need to be recreated with labels added. + +If Compose detects containers that were created without labels, it will refuse to run so that you don't end up with two sets of them. If you want to keep using your existing containers (for example, because they have data volumes you want to preserve) you can migrate them with the following command: + + docker-compose migrate-to-labels + +Alternatively, if you're not worried about keeping them, you can remove them - Compose will just create new ones. + + docker rm -f myapp_web_1 myapp_db_1 ... + ## Compose documentation - [User guide](compose-overview.md) From c3c5d91c47f00d607b68f345e367ed1b828852f8 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 18 Jun 2015 10:20:10 -0700 Subject: [PATCH 36/53] Merge pull request #1563 from moxiegirl/hugo-test-fixes Hugo final 1.7 Documentation PR -- please read carefully (cherry picked from commit 4e73e86d9480de0be87fa5390d915346a435ac26) Signed-off-by: Aanand Prasad --- docs/Dockerfile | 14 +++++++------- docs/cli.md | 2 +- docs/completion.md | 2 +- docs/django.md | 2 +- docs/env.md | 2 +- docs/extends.md | 4 ++-- docs/{compose-overview.md => index.md} | 0 docs/install.md | 2 +- docs/rails.md | 2 +- docs/wordpress.md | 2 +- docs/yml.md | 2 +- 11 files changed, 17 insertions(+), 17 deletions(-) rename docs/{compose-overview.md => index.md} (100%) diff --git a/docs/Dockerfile b/docs/Dockerfile index 55e7ce700..a49c1e7f3 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -8,17 +8,17 @@ COPY . /docs/content/compose/ # Sed to process GitHub Markdown # 1-2 Remove comment code from metadata block -# 3 Remove .md extension from link text -# 4 Change ](/ to ](/project/ in links -# 5 Change ](word) to ](/project/word) -# 6 Change ](../../ to ](/project/ -# 7 Change ](../ to ](/project/word) +# 3 Change ](/word to ](/project/ in links +# 4 Change ](word.md) to ](/project/word) +# 5 Remove .md extension from link text +# 6 Change ](../ to ](/project/word) +# 7 Change ](../../ to ](/project/ --> not implemented # # RUN find /docs/content/compose -type f -name "*.md" -exec sed -i.old \ -e '/^/g' \ -e '/^/g' \ - -e 's/\([(]\)\(.*\)\(\.md\)/\1\2/g' \ -e 's/\(\]\)\([(]\)\(\/\)/\1\2\/compose\//g' \ - -e 's/\(\][(]\)\([A-z]*[)]\)/\]\(\/compose\/\2/g' \ + -e 's/\(\][(]\)\([A-z].*\)\(\.md\)/\1\/compose\/\2/g' \ + -e 's/\([(]\)\(.*\)\(\.md\)/\1\2/g' \ -e 's/\(\][(]\)\(\.\.\/\)/\1\/compose\//g' {} \; diff --git a/docs/cli.md b/docs/cli.md index a2167d9c3..61a6aa6dd 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -192,7 +192,7 @@ Configures the path to the `ca.pem`, `cert.pem`, and `key.pem` files used for TL ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/completion.md b/docs/completion.md index 7fb696d80..3856d2701 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -59,7 +59,7 @@ Enjoy working with Compose faster and with less typos! ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/django.md b/docs/django.md index c44329e1c..84fdcbfe5 100644 --- a/docs/django.md +++ b/docs/django.md @@ -125,7 +125,7 @@ example, run `docker-compose up` and in another terminal run: ## More Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/env.md b/docs/env.md index 73496f32f..e38e6d50c 100644 --- a/docs/env.md +++ b/docs/env.md @@ -40,7 +40,7 @@ Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/extends.md b/docs/extends.md index 8527c81b3..054462b89 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -86,7 +86,7 @@ For full details on how to use `extends`, refer to the [reference](#reference). ### Example use case In this example, you’ll repurpose the example app from the [quick start -guide](compose-overview.md). (If you're not familiar with Compose, it's recommended that +guide](index.md). (If you're not familiar with Compose, it's recommended that you go through the quick start first.) This example assumes you want to use Compose both to develop an application locally and then deploy it to a production environment. @@ -371,7 +371,7 @@ volumes: ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/compose-overview.md b/docs/index.md similarity index 100% rename from docs/compose-overview.md rename to docs/index.md diff --git a/docs/install.md b/docs/install.md index c1abd4fd6..ac35c8d9f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -57,7 +57,7 @@ Alternatively, if you're not worried about keeping them, you can remove them - C ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with Wordpress](wordpress.md) diff --git a/docs/rails.md b/docs/rails.md index 2ff6f1752..cb8078647 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -124,7 +124,7 @@ you're using Boot2docker, `boot2docker ip` will tell you its address). ## More Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index ad0e62966..aa62e4e4e 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -121,7 +121,7 @@ address). ## More Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/yml.md b/docs/yml.md index 80d6d719f..087f8ac74 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -393,7 +393,7 @@ read_only: true ## Compose documentation -- [User guide](compose-overview.md) +- [User guide](/) - [Installing Compose](install.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) From c21d6706b6df54f3304b6fc59cb307c42f0c54c5 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 18 Jun 2015 11:23:40 -0700 Subject: [PATCH 37/53] Merge pull request #1565 from aanand/use-docker-1.7.0 Use docker 1.7.0 and docker-py 1.2.3 (cherry picked from commit 8ffeaf2a54828014834f49e9a20d9486a6d6d335) Signed-off-by: Aanand Prasad Conflicts: Dockerfile --- Dockerfile | 6 +++--- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1ff2d3825..98dc59c55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,13 +48,13 @@ RUN set -ex; \ rm -rf pip-7.0.1; \ rm pip-7.0.1.tar.gz -ENV ALL_DOCKER_VERSIONS 1.6.0 1.7.0-rc2 +ENV ALL_DOCKER_VERSIONS 1.6.0 1.7.0 RUN set -ex; \ curl https://get.docker.com/builds/Linux/x86_64/docker-1.6.0 -o /usr/local/bin/docker-1.6.0; \ chmod +x /usr/local/bin/docker-1.6.0; \ - curl https://test.docker.com/builds/Linux/x86_64/docker-1.7.0-rc2 -o /usr/local/bin/docker-1.7.0-rc2; \ - chmod +x /usr/local/bin/docker-1.7.0-rc2 + curl https://test.docker.com/builds/Linux/x86_64/docker-1.7.0 -o /usr/local/bin/docker-1.7.0; \ + chmod +x /usr/local/bin/docker-1.7.0 # Set the default Docker to be run RUN ln -s /usr/local/bin/docker-1.6.0 /usr/local/bin/docker diff --git a/requirements.txt b/requirements.txt index 47fa1e05b..69bd4c5f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.10 -docker-py==1.2.3-rc1 +docker-py==1.2.3 dockerpty==0.3.4 docopt==0.6.1 requests==2.6.1 diff --git a/setup.py b/setup.py index a94d87374..d2e81e175 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.7', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', - 'docker-py >= 1.2.3-rc1, < 1.3', + 'docker-py >= 1.2.3, < 1.3', 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', ] From 00f61196a44ee140f389a51e50d39d1b846ba180 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 May 2015 12:43:23 +0100 Subject: [PATCH 38/53] Bump 1.3.0 Signed-off-by: Aanand Prasad --- CHANGES.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- docs/install.md | 2 +- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 277a188a3..78e629b89 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,51 @@ Change log ========== +1.3.0 (2015-06-18) +------------------ + +Firstly, two important notes: + +- **This release contains breaking changes, and you will need to either remove or migrate your existing containers before running your app** - see the [upgrading section of the install docs](https://github.com/docker/compose/blob/1.3.0rc1/docs/install.md#upgrading) for details. + +- Compose now requires Docker 1.6.0 or later. + +We've done a lot of work in this release to remove hacks and make Compose more stable: + +- Compose now uses container labels, rather than names, to keep track of containers. This makes Compose both faster and easier to integrate with your own tools. + +- Compose no longer uses "intermediate containers" when recreating containers for a service. This makes `docker-compose up` less complex and more resilient to failure. + +There are some new features: + +- `docker-compose up` has an **experimental** new behaviour: it will only recreate containers for services whose configuration has changed in `docker-compose.yml`. This will eventually become the default, but for now you can take it for a spin: + + $ docker-compose up --x-smart-recreate + +- When invoked in a subdirectory of a project, `docker-compose` will now climb up through parent directories until it finds a `docker-compose.yml`. + +Several new configuration keys have been added to `docker-compose.yml`: + +- `dockerfile`, like `docker build --file`, lets you specify an alternate Dockerfile to use with `build`. +- `labels`, like `docker run --labels`, lets you add custom metadata to containers. +- `extra_hosts`, like `docker run --add-host`, lets you add entries to a container's `/etc/hosts` file. +- `pid: host`, like `docker run --pid=host`, lets you reuse the same PID namespace as the host machine. +- `cpuset`, like `docker run --cpuset-cpus`, lets you specify which CPUs to allow execution in. +- `read_only`, like `docker run --read-only`, lets you mount a container's filesystem as read-only. +- `security_opt`, like `docker run --security-opt`, lets you specify [security options](https://docs.docker.com/reference/run/#security-configuration). +- `log_driver`, like `docker run --log-driver`, lets you specify a [log driver](https://docs.docker.com/reference/run/#logging-drivers-log-driver). + +Many bugs have been fixed, including the following: + +- The output of `docker-compose run` was sometimes truncated, especially when running under Jenkins. +- A service's volumes would sometimes not update after volume configuration was changed in `docker-compose.yml`. +- Authenticating against third-party registries would sometimes fail. +- `docker-compose run --rm` would fail to remove the container if the service had a `restart` policy in place. +- `docker-compose scale` would refuse to scale a service beyond 1 container if it exposed a specific port number on the host. +- Compose would refuse to create multiple volume entries with the same host path. + +Thanks @ahromis, @albers, @aleksandr-vin, @antoineco, @ccverak, @chernjie, @dnephin, @edmorley, @fordhurley, @josephpage, @KyleJamesWalker, @lsowen, @mchasal, @noironetworks, @sdake, @sdurrheimer, @sherter, @stephenlawrence, @thaJeztah, @thieman, @turtlemonvh, @twhiteman, @vdemeester, @xuxinkun and @zwily! + 1.2.0 (2015-04-16) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index 045e79144..9e4c3fdb2 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.3.0dev' +__version__ = '1.3.0' diff --git a/docs/install.md b/docs/install.md index ac35c8d9f..a608d8fe7 100644 --- a/docs/install.md +++ b/docs/install.md @@ -27,7 +27,7 @@ First, install Docker version 1.6 or greater: To install Compose, run the following commands: - curl -L https://github.com/docker/compose/releases/download/1.2.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.3.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands above, then `exit`. From bd0be2cdc7d24cbb0bc8ef80a1d3d756f6099fde Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Fri, 19 Jun 2015 16:01:04 -0700 Subject: [PATCH 39/53] Merge pull request #1580 from aanand/dont-set-network-mode-when-none-is-specified Don't set network mode when none is specified (cherry picked from commit 911cd60360ceef2a4c4c4e53b661679a4f1bc48a) Signed-off-by: Aanand Prasad --- compose/project.py | 2 +- compose/service.py | 2 +- tests/unit/project_test.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/compose/project.py b/compose/project.py index bc093628c..6446a6d33 100644 --- a/compose/project.py +++ b/compose/project.py @@ -178,7 +178,7 @@ class Project(object): del service_dict['net'] else: - net = 'bridge' + net = None return net diff --git a/compose/service.py b/compose/service.py index 1e91a9f23..12a021bfb 100644 --- a/compose/service.py +++ b/compose/service.py @@ -473,7 +473,7 @@ class Service(object): def _get_net(self): if not self.net: - return "bridge" + return None if isinstance(self.net, Service): containers = self.net.containers() diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index fc49e9b88..9ee6f28c3 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -209,6 +209,18 @@ class ProjectTest(unittest.TestCase): ], None) self.assertEqual(project.get_service('test')._get_volumes_from(), container_ids) + def test_net_unset(self): + mock_client = mock.create_autospec(docker.Client) + project = Project.from_dicts('test', [ + { + 'name': 'test', + 'image': 'busybox:latest', + } + ], mock_client) + service = project.get_service('test') + self.assertEqual(service._get_net(), None) + self.assertNotIn('NetworkMode', service._get_container_host_config({})) + def test_use_net_from_container(self): container_id = 'aabbccddee' container_dict = dict(Name='aaa', Id=container_id) From d6cd76c3c153b79fa0ceec5c84c984a65a4c41f3 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Sun, 21 Jun 2015 17:25:46 -0700 Subject: [PATCH 40/53] Merge pull request #1570 from aanand/fix-build-pull Explicitly set pull=False when building (cherry picked from commit 4f83a1891259bd821efb6c8f2332f06405e88732) Signed-off-by: Aanand Prasad --- compose/service.py | 1 + tests/unit/service_test.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/compose/service.py b/compose/service.py index 12a021bfb..6c2cc4da5 100644 --- a/compose/service.py +++ b/compose/service.py @@ -628,6 +628,7 @@ class Service(object): tag=self.image_name, stream=True, rm=True, + pull=False, nocache=no_cache, dockerfile=self.options.get('dockerfile', None), ) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index fb3a7fcbb..88d301470 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -303,6 +303,17 @@ class ServiceTest(unittest.TestCase): with self.assertRaises(NeedsBuildError): service.create_container(do_build=False) + def test_build_does_not_pull(self): + self.mock_client.build.return_value = [ + '{"stream": "Successfully built 12345"}', + ] + + service = Service('foo', client=self.mock_client, build='.') + service.build() + + self.assertEqual(self.mock_client.build.call_count, 1) + self.assertFalse(self.mock_client.build.call_args[1]['pull']) + class ServiceVolumesTest(unittest.TestCase): From 882ef2ccd881b0ebb9fb422c02be6802106c0234 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Sun, 21 Jun 2015 17:25:52 -0700 Subject: [PATCH 41/53] Merge pull request #1578 from aanand/fix-migrate-help Fix 'docker-compose help migrate-to-labels' (cherry picked from commit c8751980f9011b22ce9d661bd051e4fee44d4adf) Signed-off-by: Aanand Prasad --- compose/cli/docopt_command.py | 15 +++++++++------ compose/cli/main.py | 24 ++++++++++++++++++++---- tests/unit/cli_test.py | 17 +++++++++++++++++ 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/compose/cli/docopt_command.py b/compose/cli/docopt_command.py index ee6947012..6eeb33a31 100644 --- a/compose/cli/docopt_command.py +++ b/compose/cli/docopt_command.py @@ -33,12 +33,7 @@ class DocoptCommand(object): if command is None: raise SystemExit(getdoc(self)) - command = command.replace('-', '_') - - if not hasattr(self, command): - raise NoSuchCommand(command, self) - - handler = getattr(self, command) + handler = self.get_handler(command) docstring = getdoc(handler) if docstring is None: @@ -47,6 +42,14 @@ class DocoptCommand(object): command_options = docopt_full_help(docstring, options['ARGS'], options_first=True) return options, handler, command_options + def get_handler(self, command): + command = command.replace('-', '_') + + if not hasattr(self, command): + raise NoSuchCommand(command, self) + + return getattr(self, command) + class NoSuchCommand(Exception): def __init__(self, command, supercommand): diff --git a/compose/cli/main.py b/compose/cli/main.py index 4f3f11e4e..8aeb0459d 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -128,10 +128,8 @@ class TopLevelCommand(Command): Usage: help COMMAND """ - command = options['COMMAND'] - if not hasattr(self, command): - raise NoSuchCommand(command, self) - raise SystemExit(getdoc(getattr(self, command))) + handler = self.get_handler(options['COMMAND']) + raise SystemExit(getdoc(handler)) def kill(self, project, options): """ @@ -485,6 +483,24 @@ class TopLevelCommand(Command): """ Recreate containers to add labels + If you're coming from Compose 1.2 or earlier, you'll need to remove or + migrate your existing containers after upgrading Compose. This is + because, as of version 1.3, Compose uses Docker labels to keep track + of containers, and so they need to be recreated with labels added. + + If Compose detects containers that were created without labels, it + will refuse to run so that you don't end up with two sets of them. If + you want to keep using your existing containers (for example, because + they have data volumes you want to preserve) you can migrate them with + the following command: + + docker-compose migrate-to-labels + + Alternatively, if you're not worried about keeping them, you can + remove them - Compose will just create new ones. + + docker rm -f myapp_web_1 myapp_db_1 ... + Usage: migrate-to-labels """ legacy.migrate_project_to_labels(project) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 3173a274d..d10cb9b30 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -11,6 +11,7 @@ import mock from compose.cli import main from compose.cli.main import TopLevelCommand +from compose.cli.docopt_command import NoSuchCommand from compose.cli.errors import ComposeFileNotFound from compose.service import Service @@ -101,6 +102,22 @@ class CLITestCase(unittest.TestCase): with self.assertRaises(SystemExit): command.dispatch(['-h'], None) + def test_command_help(self): + with self.assertRaises(SystemExit) as ctx: + TopLevelCommand().dispatch(['help', 'up'], None) + + self.assertIn('Usage: up', str(ctx.exception)) + + def test_command_help_dashes(self): + with self.assertRaises(SystemExit) as ctx: + TopLevelCommand().dispatch(['help', 'migrate-to-labels'], None) + + self.assertIn('Usage: migrate-to-labels', str(ctx.exception)) + + def test_command_help_nonexistent(self): + with self.assertRaises(NoSuchCommand): + TopLevelCommand().dispatch(['help', 'nonexistent'], None) + def test_setup_logging(self): main.setup_logging() self.assertEqual(logging.getLogger().level, logging.DEBUG) From 4d4ef4e0b3744944fc2b68fca52a785bc481f12a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Sun, 21 Jun 2015 17:32:36 -0700 Subject: [PATCH 42/53] Bump 1.3.1 Signed-off-by: Aanand Prasad --- CHANGES.md | 9 +++++++++ compose/__init__.py | 2 +- docs/install.md | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 78e629b89..1f43d88d4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,15 @@ Change log ========== +1.3.1 (2015-06-21) +------------------ + +The following bugs have been fixed: + +- `docker-compose build` would always attempt to pull the base image before building. +- `docker-compose help migrate-to-labels` failed with an error. +- If no network mode was specified, Compose would set it to "bridge", rather than allowing the Docker daemon to use its configured default network mode. + 1.3.0 (2015-06-18) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index 9e4c3fdb2..f3ec6acb0 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.3.0' +__version__ = '1.3.1' diff --git a/docs/install.md b/docs/install.md index a608d8fe7..96a4a2376 100644 --- a/docs/install.md +++ b/docs/install.md @@ -27,7 +27,7 @@ First, install Docker version 1.6 or greater: To install Compose, run the following commands: - curl -L https://github.com/docker/compose/releases/download/1.3.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.3.1/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands above, then `exit`. From b12c29479ee7283f24af6025e2d3de0c15faf510 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 16 Jun 2015 10:45:49 -0700 Subject: [PATCH 43/53] Merge pull request #1521 from dano/validate-service-names Validate that service names passed to Project.containers aren't bogus. (cherry picked from commit bc14c473c97af14ed150160fe84d23fcb05fe4e2) Signed-off-by: Aanand Prasad --- compose/project.py | 12 ++++++++++++ tests/integration/cli_test.py | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/compose/project.py b/compose/project.py index 6446a6d33..907632276 100644 --- a/compose/project.py +++ b/compose/project.py @@ -99,6 +99,16 @@ class Project(object): raise NoSuchService(name) + def validate_service_names(self, service_names): + """ + Validate that the given list of service names only contains valid + services. Raises NoSuchService if one of the names is invalid. + """ + valid_names = self.service_names + for name in service_names: + if name not in valid_names: + raise NoSuchService(name) + def get_services(self, service_names=None, include_deps=False): """ Returns a list of this project's services filtered @@ -274,6 +284,8 @@ class Project(object): service.remove_stopped(**options) def containers(self, service_names=None, stopped=False, one_off=False): + if service_names: + self.validate_service_names(service_names) containers = [ Container.from_ps(self.client, container) for container in self.client.containers( diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 2d1f1f76e..dab9d4a2b 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -9,6 +9,7 @@ from mock import patch from .testcases import DockerClientTestCase from compose.cli.main import TopLevelCommand +from compose.project import NoSuchService class CLITestCase(DockerClientTestCase): @@ -351,6 +352,10 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(service.containers(stopped=True)), 1) self.assertFalse(service.containers(stopped=True)[0].is_running) + def test_logs_invalid_service_name(self): + with self.assertRaises(NoSuchService): + self.command.dispatch(['logs', 'madeupname'], None) + def test_kill(self): self.command.dispatch(['up', '-d'], None) service = self.project.get_service('simple') From c8295d36cc840c788dd9dad249b49f7808b2b7d9 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 3 Jul 2015 16:22:48 +0100 Subject: [PATCH 44/53] Merge pull request #1644 from aanand/fix-rm-bug Stop 'rm' and 'ps' listing services not defined in the current file (cherry picked from commit d85688892cb64128093db98d6f03d97ff8bd0e40) Signed-off-by: Aanand Prasad --- compose/project.py | 5 +++-- tests/integration/project_test.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/compose/project.py b/compose/project.py index 907632276..7c78401e4 100644 --- a/compose/project.py +++ b/compose/project.py @@ -286,6 +286,9 @@ class Project(object): def containers(self, service_names=None, stopped=False, one_off=False): if service_names: self.validate_service_names(service_names) + else: + service_names = self.service_names + containers = [ Container.from_ps(self.client, container) for container in self.client.containers( @@ -293,8 +296,6 @@ class Project(object): filters={'label': self.labels(one_off=one_off)})] def matches_service_names(container): - if not service_names: - return True return container.labels.get(LABEL_SERVICE) in service_names if not containers: diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 2976af823..314daf718 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -29,6 +29,21 @@ class ProjectTest(DockerClientTestCase): [c.name for c in containers], ['composetest_web_1']) + def test_containers_with_extra_service(self): + web = self.create_service('web') + web_1 = web.create_container() + + db = self.create_service('db') + db_1 = db.create_container() + + self.create_service('extra').create_container() + + project = Project('composetest', [web, db], self.client) + self.assertEqual( + set(project.containers(stopped=True)), + set([web_1, db_1]), + ) + def test_volumes_from_service(self): service_dicts = config.from_dictionary({ 'data': { From c31e25af722fe2ee9bee1644515cd2d0d47c4864 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 3 Jul 2015 16:25:53 +0100 Subject: [PATCH 45/53] Merge pull request #1642 from aanand/fix-1573 Fix bug where duplicate container is leftover after 'up' fails (cherry picked from commit f42fd6a3ad17cf9688e709bfed1639196777a342) Signed-off-by: Aanand Prasad --- compose/project.py | 3 +++ compose/service.py | 20 +++++++++++++++ tests/integration/resilience_test.py | 37 ++++++++++++++++++---------- tests/integration/service_test.py | 15 +++++++++++ 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/compose/project.py b/compose/project.py index 7c78401e4..a9df73f19 100644 --- a/compose/project.py +++ b/compose/project.py @@ -225,6 +225,9 @@ class Project(object): services = self.get_services(service_names, include_deps=start_deps) + for service in services: + service.remove_duplicate_containers() + plans = self._get_convergence_plans( services, allow_recreate=allow_recreate, diff --git a/compose/service.py b/compose/service.py index 6c2cc4da5..0db63e64f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -394,6 +394,26 @@ class Service(object): container.start() return container + def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT): + for c in self.duplicate_containers(): + log.info('Removing %s...' % c.name) + c.stop(timeout=timeout) + c.remove() + + def duplicate_containers(self): + containers = sorted( + self.containers(stopped=True), + key=lambda c: c.get('Created'), + ) + + numbers = set() + + for c in containers: + if c.number in numbers: + yield c + else: + numbers.add(c.number) + def config_hash(self): return json_hash(self.config_dict()) diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index 8229e9d3c..392490902 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -8,25 +8,36 @@ from .testcases import DockerClientTestCase class ResilienceTest(DockerClientTestCase): - def test_recreate_fails(self): - db = self.create_service('db', volumes=['/var/db'], command='top') - project = Project('composetest', [db], self.client) + def setUp(self): + self.db = self.create_service('db', volumes=['/var/db'], command='top') + self.project = Project('composetest', [self.db], self.client) - container = db.create_container() - db.start_container(container) - host_path = container.get('Volumes')['/var/db'] + container = self.db.create_container() + self.db.start_container(container) + self.host_path = container.get('Volumes')['/var/db'] - project.up() - container = db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], host_path) + def test_successful_recreate(self): + self.project.up() + container = self.db.containers()[0] + self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + def test_create_failure(self): with mock.patch('compose.service.Service.create_container', crash): with self.assertRaises(Crash): - project.up() + self.project.up() - project.up() - container = db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], host_path) + self.project.up() + container = self.db.containers()[0] + self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + + def test_start_failure(self): + with mock.patch('compose.service.Service.start_container', crash): + with self.assertRaises(Crash): + self.project.up() + + self.project.up() + container = self.db.containers()[0] + self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) class Crash(Exception): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 32de5fa47..98fa4213c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -705,3 +705,18 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(1, len(device_config)) self.assertDictEqual(device_dict, device_config[0]) + + def test_duplicate_containers(self): + service = self.create_service('web') + + options = service._get_container_create_options({}, 1) + original = Container.create(service.client, **options) + + self.assertEqual(set(service.containers(stopped=True)), set([original])) + self.assertEqual(set(service.duplicate_containers()), set()) + + options['name'] = 'temporary_container_name' + duplicate = Container.create(service.client, **options) + + self.assertEqual(set(service.containers(stopped=True)), set([original, duplicate])) + self.assertEqual(set(service.duplicate_containers()), set([duplicate])) From c1b9a76a54bf8bf4f8ab717768243410ee7d5169 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 7 Jul 2015 16:12:44 +0100 Subject: [PATCH 46/53] Merge pull request #1658 from aanand/fix-smart-recreate-nonexistent-image Fix smart recreate when 'image' is changed to something nonexistent (cherry picked from commit 2bc10db5451ec8e69119997061b0ba5c692feb90) Signed-off-by: Aanand Prasad --- compose/service.py | 23 ++++++++++++++++++++--- tests/integration/state_test.py | 7 +++++++ tests/unit/service_test.py | 16 ++++++++++++---- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/compose/service.py b/compose/service.py index 0db63e64f..f1a006c32 100644 --- a/compose/service.py +++ b/compose/service.py @@ -64,6 +64,10 @@ class NeedsBuildError(Exception): self.service = service +class NoSuchImageError(Exception): + pass + + VolumeSpec = namedtuple('VolumeSpec', 'external internal mode') @@ -224,8 +228,11 @@ class Service(object): do_build=True, insecure_registry=False): - if self.image(): + try: + self.image() return + except NoSuchImageError: + pass if self.can_be_built(): if do_build: @@ -240,7 +247,7 @@ class Service(object): return self.client.inspect_image(self.image_name) except APIError as e: if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation): - return None + raise NoSuchImageError("Image '{}' not found".format(self.image_name)) else: raise @@ -294,7 +301,17 @@ class Service(object): return ConvergencePlan('recreate', containers) def _containers_have_diverged(self, containers): - config_hash = self.config_hash() + config_hash = None + + try: + config_hash = self.config_hash() + except NoSuchImageError as e: + log.debug( + 'Service %s has diverged: %s', + self.name, six.text_type(e), + ) + return True + has_diverged = False for c in containers: diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 7a7d2b58f..6e1ba025b 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -191,6 +191,13 @@ class ServiceStateTest(DockerClientTestCase): web = self.create_service('web', command=["top", "-d", "1"]) self.assertEqual(('recreate', [container]), web.convergence_plan(smart_recreate=True)) + def test_trigger_recreate_with_nonexistent_image_tag(self): + web = self.create_service('web', image="busybox:latest") + container = web.create_container() + + web = self.create_service('web', image="nonexistent-image") + self.assertEqual(('recreate', [container]), web.convergence_plan(smart_recreate=True)) + def test_trigger_recreate_with_image_change(self): repo = 'composetest_myimage' tag = 'latest' diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 88d301470..dc1f7df34 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -12,6 +12,7 @@ from compose.const import LABEL_SERVICE, LABEL_PROJECT, LABEL_ONE_OFF from compose.service import ( ConfigError, NeedsBuildError, + NoSuchImageError, build_port_bindings, build_volume_binding, get_container_data_volumes, @@ -233,7 +234,7 @@ class ServiceTest(unittest.TestCase): images.append({'Id': 'abc123'}) return [] - service.image = lambda: images[0] if images else None + service.image = lambda *args, **kwargs: mock_get_image(images) self.mock_client.pull = pull service.create_container(insecure_registry=True) @@ -273,7 +274,7 @@ class ServiceTest(unittest.TestCase): images.append({'Id': 'abc123'}) return [] - service.image = lambda: images[0] if images else None + service.image = lambda *args, **kwargs: mock_get_image(images) self.mock_client.pull = pull service.create_container() @@ -283,7 +284,7 @@ class ServiceTest(unittest.TestCase): service = Service('foo', client=self.mock_client, build='.') images = [] - service.image = lambda *args, **kwargs: images[0] if images else None + service.image = lambda *args, **kwargs: mock_get_image(images) service.build = lambda: images.append({'Id': 'abc123'}) service.create_container(do_build=True) @@ -298,7 +299,7 @@ class ServiceTest(unittest.TestCase): def test_create_container_no_build_but_needs_build(self): service = Service('foo', client=self.mock_client, build='.') - service.image = lambda: None + service.image = lambda *args, **kwargs: mock_get_image([]) with self.assertRaises(NeedsBuildError): service.create_container(do_build=False) @@ -315,6 +316,13 @@ class ServiceTest(unittest.TestCase): self.assertFalse(self.mock_client.build.call_args[1]['pull']) +def mock_get_image(images): + if images: + return images[0] + else: + raise NoSuchImageError() + + class ServiceVolumesTest(unittest.TestCase): def setUp(self): From 4911c77134a38698fdcec2ebfbb2c76d77a4587f Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 15 Jun 2015 10:04:45 -0700 Subject: [PATCH 47/53] Merge pull request #1489 from dnephin/faster_integration_tests Faster integration tests (cherry picked from commit 5231288b4e44d15a9ca553504faeac3b618f49d6) Signed-off-by: Aanand Prasad Conflicts: compose/cli/main.py --- compose/cli/main.py | 23 +++++----- compose/const.py | 1 + compose/project.py | 6 ++- compose/service.py | 30 ++++--------- tests/fixtures/build-ctx/Dockerfile | 1 + .../dockerfile-with-volume/Dockerfile | 3 +- .../dockerfile_with_entrypoint/Dockerfile | 1 + tests/fixtures/simple-dockerfile/Dockerfile | 1 + tests/integration/cli_test.py | 19 +++++++-- tests/integration/legacy_test.py | 24 +++++++++-- tests/integration/project_test.py | 41 +++--------------- tests/integration/service_test.py | 42 +++++++++++++------ tests/integration/state_test.py | 39 +++++++++++++---- tests/integration/testcases.py | 20 +++++---- tests/unit/service_test.py | 11 ++++- 15 files changed, 151 insertions(+), 111 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 8aeb0459d..6c7ccf0da 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -11,6 +11,7 @@ from docker.errors import APIError import dockerpty from .. import legacy +from ..const import DEFAULT_TIMEOUT from ..project import NoSuchService, ConfigurationError from ..service import BuildError, NeedsBuildError from ..config import parse_environment @@ -392,9 +393,8 @@ class TopLevelCommand(Command): -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) """ - timeout = options.get('--timeout') - params = {} if timeout is None else {'timeout': int(timeout)} - project.stop(service_names=options['SERVICE'], **params) + timeout = float(options.get('--timeout') or DEFAULT_TIMEOUT) + project.stop(service_names=options['SERVICE'], timeout=timeout) def restart(self, project, options): """ @@ -406,9 +406,8 @@ class TopLevelCommand(Command): -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) """ - timeout = options.get('--timeout') - params = {} if timeout is None else {'timeout': int(timeout)} - project.restart(service_names=options['SERVICE'], **params) + timeout = float(options.get('--timeout') or DEFAULT_TIMEOUT) + project.restart(service_names=options['SERVICE'], timeout=timeout) def up(self, project, options): """ @@ -437,9 +436,9 @@ class TopLevelCommand(Command): image needs to be updated. (EXPERIMENTAL) --no-recreate If containers already exist, don't recreate them. --no-build Don't build an image, even if it's missing - -t, --timeout TIMEOUT When attached, use this timeout in seconds - for the shutdown. (default: 10) - + -t, --timeout TIMEOUT Use this timeout in seconds for container shutdown + when attached or when containers are already + running. (default: 10) """ insecure_registry = options['--allow-insecure-ssl'] detached = options['-d'] @@ -450,6 +449,7 @@ class TopLevelCommand(Command): allow_recreate = not options['--no-recreate'] smart_recreate = options['--x-smart-recreate'] service_names = options['SERVICE'] + timeout = float(options.get('--timeout') or DEFAULT_TIMEOUT) project.up( service_names=service_names, @@ -458,6 +458,7 @@ class TopLevelCommand(Command): smart_recreate=smart_recreate, insecure_registry=insecure_registry, do_build=not options['--no-build'], + timeout=timeout ) to_attach = [c for s in project.get_services(service_names) for c in s.containers()] @@ -475,9 +476,7 @@ class TopLevelCommand(Command): signal.signal(signal.SIGINT, handler) print("Gracefully stopping... (press Ctrl+C again to force)") - timeout = options.get('--timeout') - params = {} if timeout is None else {'timeout': int(timeout)} - project.stop(service_names=service_names, **params) + project.stop(service_names=service_names, timeout=timeout) def migrate_to_labels(self, project, _options): """ diff --git a/compose/const.py b/compose/const.py index f76fb572c..709c3a10d 100644 --- a/compose/const.py +++ b/compose/const.py @@ -1,4 +1,5 @@ +DEFAULT_TIMEOUT = 10 LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' LABEL_ONE_OFF = 'com.docker.compose.oneoff' LABEL_PROJECT = 'com.docker.compose.project' diff --git a/compose/project.py b/compose/project.py index a9df73f19..288afc5f5 100644 --- a/compose/project.py +++ b/compose/project.py @@ -6,7 +6,7 @@ from functools import reduce from docker.errors import APIError from .config import get_service_name_from_net, ConfigurationError -from .const import LABEL_PROJECT, LABEL_SERVICE, LABEL_ONE_OFF +from .const import LABEL_PROJECT, LABEL_SERVICE, LABEL_ONE_OFF, DEFAULT_TIMEOUT from .service import Service from .container import Container from .legacy import check_for_legacy_containers @@ -221,7 +221,8 @@ class Project(object): allow_recreate=True, smart_recreate=False, insecure_registry=False, - do_build=True): + do_build=True, + timeout=DEFAULT_TIMEOUT): services = self.get_services(service_names, include_deps=start_deps) @@ -241,6 +242,7 @@ class Project(object): plans[service.name], insecure_registry=insecure_registry, do_build=do_build, + timeout=timeout ) ] diff --git a/compose/service.py b/compose/service.py index f1a006c32..ddabbfc4a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -13,6 +13,7 @@ from docker.utils import create_host_config, LogConfig from . import __version__ from .config import DOCKER_CONFIG_KEYS, merge_environment from .const import ( + DEFAULT_TIMEOUT, LABEL_CONTAINER_NUMBER, LABEL_ONE_OFF, LABEL_PROJECT, @@ -258,26 +259,6 @@ class Service(object): else: return self.options['image'] - def converge(self, - allow_recreate=True, - smart_recreate=False, - insecure_registry=False, - do_build=True): - """ - If a container for this service doesn't exist, create and start one. If there are - any, stop them, create+start new ones, and remove the old containers. - """ - plan = self.convergence_plan( - allow_recreate=allow_recreate, - smart_recreate=smart_recreate, - ) - - return self.execute_convergence_plan( - plan, - insecure_registry=insecure_registry, - do_build=do_build, - ) - def convergence_plan(self, allow_recreate=True, smart_recreate=False): @@ -328,7 +309,8 @@ class Service(object): def execute_convergence_plan(self, plan, insecure_registry=False, - do_build=True): + do_build=True, + timeout=DEFAULT_TIMEOUT): (action, containers) = plan if action == 'create': @@ -345,6 +327,7 @@ class Service(object): self.recreate_container( c, insecure_registry=insecure_registry, + timeout=timeout ) for c in containers ] @@ -366,7 +349,8 @@ class Service(object): def recreate_container(self, container, - insecure_registry=False): + insecure_registry=False, + timeout=DEFAULT_TIMEOUT): """Recreate a container. The original container is renamed to a temporary name so that data @@ -375,7 +359,7 @@ class Service(object): """ log.info("Recreating %s..." % container.name) try: - container.stop() + container.stop(timeout=timeout) except APIError as e: if (e.response.status_code == 500 and e.explanation diff --git a/tests/fixtures/build-ctx/Dockerfile b/tests/fixtures/build-ctx/Dockerfile index d1ceac6b7..dd864b838 100644 --- a/tests/fixtures/build-ctx/Dockerfile +++ b/tests/fixtures/build-ctx/Dockerfile @@ -1,2 +1,3 @@ FROM busybox:latest +LABEL com.docker.compose.test_image=true CMD echo "success" diff --git a/tests/fixtures/dockerfile-with-volume/Dockerfile b/tests/fixtures/dockerfile-with-volume/Dockerfile index 6e5d0a55e..0d376ec48 100644 --- a/tests/fixtures/dockerfile-with-volume/Dockerfile +++ b/tests/fixtures/dockerfile-with-volume/Dockerfile @@ -1,3 +1,4 @@ -FROM busybox +FROM busybox:latest +LABEL com.docker.compose.test_image=true VOLUME /data CMD top diff --git a/tests/fixtures/dockerfile_with_entrypoint/Dockerfile b/tests/fixtures/dockerfile_with_entrypoint/Dockerfile index 7d28d2933..e7454e59b 100644 --- a/tests/fixtures/dockerfile_with_entrypoint/Dockerfile +++ b/tests/fixtures/dockerfile_with_entrypoint/Dockerfile @@ -1,2 +1,3 @@ FROM busybox:latest +LABEL com.docker.compose.test_image=true ENTRYPOINT echo "From prebuilt entrypoint" diff --git a/tests/fixtures/simple-dockerfile/Dockerfile b/tests/fixtures/simple-dockerfile/Dockerfile index d1ceac6b7..dd864b838 100644 --- a/tests/fixtures/simple-dockerfile/Dockerfile +++ b/tests/fixtures/simple-dockerfile/Dockerfile @@ -1,2 +1,3 @@ FROM busybox:latest +LABEL com.docker.compose.test_image=true CMD echo "success" diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index dab9d4a2b..421d59857 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -26,6 +26,7 @@ class CLITestCase(DockerClientTestCase): self.project.remove_stopped() for container in self.project.containers(stopped=True, one_off=True): container.remove(force=True) + super(CLITestCase, self).tearDown() @property def project(self): @@ -163,6 +164,19 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(old_ids, new_ids) + def test_up_with_timeout(self): + self.command.dispatch(['up', '-d', '-t', '1'], None) + service = self.project.get_service('simple') + another = self.project.get_service('another') + self.assertEqual(len(service.containers()), 1) + self.assertEqual(len(another.containers()), 1) + + # Ensure containers don't have stdin and stdout connected in -d mode + config = service.containers()[0].inspect()['Config'] + self.assertFalse(config['AttachStderr']) + self.assertFalse(config['AttachStdout']) + self.assertFalse(config['AttachStdin']) + @patch('dockerpty.start') def test_run_service_without_links(self, mock_stdout): self.command.base_dir = 'tests/fixtures/links-composefile' @@ -209,13 +223,10 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(old_ids, new_ids) @patch('dockerpty.start') - def test_run_without_command(self, __): + def test_run_without_command(self, _): self.command.base_dir = 'tests/fixtures/commands-composefile' self.check_build('tests/fixtures/simple-dockerfile', tag='composetest_test') - for c in self.project.containers(stopped=True, one_off=True): - c.remove() - self.command.dispatch(['run', 'implicit'], None) service = self.project.get_service('implicit') containers = service.containers(stopped=True, one_off=True) diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index 6c52b68d3..346c84f2e 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -1,12 +1,15 @@ +from docker.errors import APIError + from compose import legacy from compose.project import Project from .testcases import DockerClientTestCase -class ProjectTest(DockerClientTestCase): +class LegacyTestCase(DockerClientTestCase): def setUp(self): - super(ProjectTest, self).setUp() + super(LegacyTestCase, self).setUp() + self.containers = [] db = self.create_service('db') web = self.create_service('web', links=[(db, 'db')]) @@ -23,12 +26,25 @@ class ProjectTest(DockerClientTestCase): **service.options ) self.client.start(container) + self.containers.append(container) # Create a single one-off legacy container - self.client.create_container( + self.containers.append(self.client.create_container( name='{}_{}_run_1'.format(self.project.name, self.services[0].name), **self.services[0].options - ) + )) + + def tearDown(self): + super(LegacyTestCase, self).tearDown() + for container in self.containers: + try: + self.client.kill(container) + except APIError: + pass + try: + self.client.remove_container(container) + except APIError: + pass def get_legacy_containers(self, **kwargs): return list(legacy.get_legacy_containers( diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 314daf718..505f509ea 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals + from compose import config +from compose.const import LABEL_PROJECT from compose.project import Project from compose.container import Container from .testcases import DockerClientTestCase @@ -70,6 +72,7 @@ class ProjectTest(DockerClientTestCase): image='busybox:latest', volumes=['/var/data'], name='composetest_data_container', + labels={LABEL_PROJECT: 'composetest'}, ) project = Project.from_dicts( name='composetest', @@ -84,9 +87,6 @@ class ProjectTest(DockerClientTestCase): db = project.get_service('db') self.assertEqual(db.volumes_from, [data_container]) - project.kill() - project.remove_stopped() - def test_net_from_service(self): project = Project.from_dicts( name='composetest', @@ -110,15 +110,13 @@ class ProjectTest(DockerClientTestCase): net = project.get_service('net') self.assertEqual(web._get_net(), 'container:' + net.containers()[0].id) - project.kill() - project.remove_stopped() - def test_net_from_container(self): net_container = Container.create( self.client, image='busybox:latest', name='composetest_net_container', - command='top' + command='top', + labels={LABEL_PROJECT: 'composetest'}, ) net_container.start() @@ -138,9 +136,6 @@ class ProjectTest(DockerClientTestCase): web = project.get_service('web') self.assertEqual(web._get_net(), 'container:' + net_container.id) - project.kill() - project.remove_stopped() - def test_start_stop_kill_remove(self): web = self.create_service('web') db = self.create_service('db') @@ -186,9 +181,6 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(db.containers()), 1) self.assertEqual(len(web.containers()), 0) - project.kill() - project.remove_stopped() - def test_project_up_starts_uncreated_services(self): db = self.create_service('db') web = self.create_service('web', links=[(db, 'db')]) @@ -220,9 +212,6 @@ class ProjectTest(DockerClientTestCase): self.assertNotEqual(db_container.id, old_db_id) self.assertEqual(db_container.get('Volumes./etc'), db_volume_path) - project.kill() - project.remove_stopped() - def test_project_up_with_no_recreate_running(self): web = self.create_service('web') db = self.create_service('db', volumes=['/var/db']) @@ -243,9 +232,6 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(db_container.inspect()['Volumes']['/var/db'], db_volume_path) - project.kill() - project.remove_stopped() - def test_project_up_with_no_recreate_stopped(self): web = self.create_service('web') db = self.create_service('db', volumes=['/var/db']) @@ -273,9 +259,6 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(db_container.inspect()['Volumes']['/var/db'], db_volume_path) - project.kill() - project.remove_stopped() - def test_project_up_without_all_services(self): console = self.create_service('console') db = self.create_service('db') @@ -288,9 +271,6 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(db.containers()), 1) self.assertEqual(len(console.containers()), 1) - project.kill() - project.remove_stopped() - def test_project_up_starts_links(self): console = self.create_service('console') db = self.create_service('db', volumes=['/var/db']) @@ -306,9 +286,6 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(db.containers()), 1) self.assertEqual(len(console.containers()), 0) - project.kill() - project.remove_stopped() - def test_project_up_starts_depends(self): project = Project.from_dicts( name='composetest', @@ -344,9 +321,6 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(project.get_service('data').containers()), 1) self.assertEqual(len(project.get_service('console').containers()), 0) - project.kill() - project.remove_stopped() - def test_project_up_with_no_deps(self): project = Project.from_dicts( name='composetest', @@ -383,9 +357,6 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(project.get_service('data').containers(stopped=True)), 1) self.assertEqual(len(project.get_service('console').containers()), 0) - project.kill() - project.remove_stopped() - def test_unscale_after_restart(self): web = self.create_service('web') project = Project('composetest', [web], self.client) @@ -410,5 +381,3 @@ class ProjectTest(DockerClientTestCase): project.up() service = project.get_service('web') self.assertEqual(len(service.containers()), 1) - project.kill() - project.remove_stopped() diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 98fa4213c..bf12be7f2 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -2,8 +2,9 @@ from __future__ import unicode_literals from __future__ import absolute_import import os from os import path -import mock +from docker.errors import APIError +import mock import tempfile import shutil import six @@ -18,11 +19,11 @@ from compose.const import ( ) from compose.service import ( ConfigError, + ConvergencePlan, Service, build_extra_hosts, ) from compose.container import Container -from docker.errors import APIError from .testcases import DockerClientTestCase @@ -235,7 +236,12 @@ class ServiceTest(DockerClientTestCase): def test_create_container_with_volumes_from(self): volume_service = self.create_service('data') volume_container_1 = volume_service.create_container() - volume_container_2 = Container.create(self.client, image='busybox:latest', command=["top"]) + volume_container_2 = Container.create( + self.client, + image='busybox:latest', + command=["top"], + labels={LABEL_PROJECT: 'composetest'}, + ) host_service = self.create_service('host', volumes_from=[volume_service, volume_container_2]) host_container = host_service.create_container() host_service.start_container(host_container) @@ -244,7 +250,7 @@ class ServiceTest(DockerClientTestCase): self.assertIn(volume_container_2.id, host_container.get('HostConfig.VolumesFrom')) - def test_converge(self): + def test_execute_convergence_plan_recreate(self): service = self.create_service( 'db', environment={'FOO': '1'}, @@ -264,7 +270,8 @@ class ServiceTest(DockerClientTestCase): num_containers_before = len(self.client.containers(all=True)) service.options['environment']['FOO'] = '2' - new_container = service.converge()[0] + new_container, = service.execute_convergence_plan( + ConvergencePlan('recreate', [old_container])) self.assertEqual(new_container.get('Config.Entrypoint'), ['top']) self.assertEqual(new_container.get('Config.Cmd'), ['-d', '1']) @@ -281,7 +288,7 @@ class ServiceTest(DockerClientTestCase): self.client.inspect_container, old_container.id) - def test_converge_when_containers_are_stopped(self): + def test_execute_convergence_plan_when_containers_are_stopped(self): service = self.create_service( 'db', environment={'FOO': '1'}, @@ -290,11 +297,21 @@ class ServiceTest(DockerClientTestCase): command=['-d', '1'] ) service.create_container() - self.assertEqual(len(service.containers(stopped=True)), 1) - service.converge() - self.assertEqual(len(service.containers(stopped=True)), 1) - def test_converge_with_image_declared_volume(self): + containers = service.containers(stopped=True) + self.assertEqual(len(containers), 1) + container, = containers + self.assertFalse(container.is_running) + + service.execute_convergence_plan(ConvergencePlan('start', [container])) + + containers = service.containers() + self.assertEqual(len(containers), 1) + container.inspect() + self.assertEqual(container, containers[0]) + self.assertTrue(container.is_running) + + def test_execute_convergence_plan_with_image_declared_volume(self): service = Service( project='composetest', name='db', @@ -306,7 +323,8 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(old_container.get('Volumes').keys(), ['/data']) volume_path = old_container.get('Volumes')['/data'] - new_container = service.converge()[0] + new_container, = service.execute_convergence_plan( + ConvergencePlan('recreate', [old_container])) self.assertEqual(new_container.get('Volumes').keys(), ['/data']) self.assertEqual(new_container.get('Volumes')['/data'], volume_path) @@ -408,7 +426,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(len(self.client.images(name='composetest_test')), 1) def test_start_container_uses_tagged_image_if_it_exists(self): - self.client.build('tests/fixtures/simple-dockerfile', tag='composetest_test') + self.check_build('tests/fixtures/simple-dockerfile', tag='composetest_test') service = Service( name='test', client=self.client, diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 6e1ba025b..95fcc49de 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -12,8 +12,8 @@ from .testcases import DockerClientTestCase class ProjectTestCase(DockerClientTestCase): def run_up(self, cfg, **kwargs): - if 'smart_recreate' not in kwargs: - kwargs['smart_recreate'] = True + kwargs.setdefault('smart_recreate', True) + kwargs.setdefault('timeout', 0.1) project = self.make_project(cfg) project.up(**kwargs) @@ -153,7 +153,31 @@ class ProjectWithDependenciesTest(ProjectTestCase): self.assertEqual(new_containers - old_containers, set()) +def converge(service, + allow_recreate=True, + smart_recreate=False, + insecure_registry=False, + do_build=True): + """ + If a container for this service doesn't exist, create and start one. If there are + any, stop them, create+start new ones, and remove the old containers. + """ + plan = service.convergence_plan( + allow_recreate=allow_recreate, + smart_recreate=smart_recreate, + ) + + return service.execute_convergence_plan( + plan, + insecure_registry=insecure_registry, + do_build=do_build, + timeout=0.1, + ) + + class ServiceStateTest(DockerClientTestCase): + """Test cases for Service.convergence_plan.""" + def test_trigger_create(self): web = self.create_service('web') self.assertEqual(('create', []), web.convergence_plan(smart_recreate=True)) @@ -223,18 +247,19 @@ class ServiceStateTest(DockerClientTestCase): def test_trigger_recreate_with_build(self): context = tempfile.mkdtemp() + base_image = "FROM busybox\nLABEL com.docker.compose.test_image=true\n" try: dockerfile = os.path.join(context, 'Dockerfile') with open(dockerfile, 'w') as f: - f.write('FROM busybox\n') + f.write(base_image) web = self.create_service('web', build=context) container = web.create_container() with open(dockerfile, 'w') as f: - f.write('FROM busybox\nCMD echo hello world\n') + f.write(base_image + 'CMD echo hello world\n') web.build() web = self.create_service('web', build=context) @@ -256,15 +281,15 @@ class ConfigHashTest(DockerClientTestCase): def test_config_hash_with_custom_labels(self): web = self.create_service('web', labels={'foo': '1'}) - container = web.converge()[0] + container = converge(web)[0] self.assertIn(LABEL_CONFIG_HASH, container.labels) self.assertIn('foo', container.labels) def test_config_hash_sticks_around(self): web = self.create_service('web', command=["top"]) - container = web.converge()[0] + container = converge(web)[0] self.assertIn(LABEL_CONFIG_HASH, container.labels) web = self.create_service('web', command=["top", "-d", "1"]) - container = web.converge()[0] + container = converge(web)[0] self.assertIn(LABEL_CONFIG_HASH, container.labels) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 48fcf3ef2..98c5876eb 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from __future__ import absolute_import from compose.service import Service from compose.config import make_service_dict +from compose.const import LABEL_PROJECT from compose.cli.docker_client import docker_client from compose.progress_stream import stream_output from .. import unittest @@ -12,15 +13,15 @@ class DockerClientTestCase(unittest.TestCase): def setUpClass(cls): cls.client = docker_client() - # TODO: update to use labels in #652 - def setUp(self): - for c in self.client.containers(all=True): - if c['Names'] and 'composetest' in c['Names'][0]: - self.client.kill(c['Id']) - self.client.remove_container(c['Id']) - for i in self.client.images(): - if isinstance(i.get('Tag'), basestring) and 'composetest' in i['Tag']: - self.client.remove_image(i) + def tearDown(self): + for c in self.client.containers( + all=True, + filters={'label': '%s=composetest' % LABEL_PROJECT}): + self.client.kill(c['Id']) + self.client.remove_container(c['Id']) + for i in self.client.images( + filters={'label': 'com.docker.compose.test_image'}): + self.client.remove_image(i) def create_service(self, name, **kwargs): if 'image' not in kwargs and 'build' not in kwargs: @@ -36,5 +37,6 @@ class DockerClientTestCase(unittest.TestCase): ) def check_build(self, *args, **kwargs): + kwargs.setdefault('rm', True) build_output = self.client.build(*args, **kwargs) stream_output(build_output, open('/dev/null', 'w')) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index dc1f7df34..a63e39c65 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -247,7 +247,7 @@ class ServiceTest(unittest.TestCase): service.image = lambda: {'Id': 'abc123'} new_container = service.recreate_container(mock_container) - mock_container.stop.assert_called_once_with() + mock_container.stop.assert_called_once_with(timeout=10) self.mock_client.rename.assert_called_once_with( mock_container.id, '%s_%s' % (mock_container.short_id, mock_container.name)) @@ -255,6 +255,15 @@ class ServiceTest(unittest.TestCase): new_container.start.assert_called_once_with() mock_container.remove.assert_called_once_with() + @mock.patch('compose.service.Container', autospec=True) + def test_recreate_container_with_timeout(self, _): + mock_container = mock.create_autospec(Container) + self.mock_client.inspect_image.return_value = {'Id': 'abc123'} + service = Service('foo', client=self.mock_client, image='someimage') + service.recreate_container(mock_container, timeout=1) + + mock_container.stop.assert_called_once_with(timeout=1) + def test_parse_repository_tag(self): self.assertEqual(parse_repository_tag("root"), ("root", "")) self.assertEqual(parse_repository_tag("root:tag"), ("root", "tag")) From 4bc4d273ace15229853e1da41b71c104df33dce9 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 8 Jul 2015 14:48:03 +0100 Subject: [PATCH 48/53] Merge pull request #1643 from aanand/warn-about-legacy-one-off-containers Show an error on 'run' when there are legacy one-off containers (cherry picked from commit 81707ef1ad94403789166d2fe042c8a718a4c748) Signed-off-by: Aanand Prasad --- compose/cli/main.py | 24 ++++-- compose/legacy.py | 94 +++++++++++++++++----- compose/project.py | 3 +- compose/service.py | 3 +- tests/integration/legacy_test.py | 129 +++++++++++++++++++++++++++++-- tests/unit/cli_test.py | 6 +- 6 files changed, 221 insertions(+), 38 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 6c7ccf0da..d5d15177c 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -33,7 +33,7 @@ def main(): except KeyboardInterrupt: log.error("\nAborting.") sys.exit(1) - except (UserError, NoSuchService, ConfigurationError, legacy.LegacyContainersError) as e: + except (UserError, NoSuchService, ConfigurationError, legacy.LegacyError) as e: log.error(e.msg) sys.exit(1) except NoSuchCommand as e: @@ -334,12 +334,22 @@ class TopLevelCommand(Command): if not options['--service-ports']: container_options['ports'] = [] - container = service.create_container( - quiet=True, - one_off=True, - insecure_registry=insecure_registry, - **container_options - ) + try: + container = service.create_container( + quiet=True, + one_off=True, + insecure_registry=insecure_registry, + **container_options + ) + except APIError as e: + legacy.check_for_legacy_containers( + project.client, + project.name, + [service.name], + allow_one_off=False, + ) + + raise e if options['-d']: service.start_container(container) diff --git a/compose/legacy.py b/compose/legacy.py index 340511a76..c9ec65817 100644 --- a/compose/legacy.py +++ b/compose/legacy.py @@ -1,6 +1,7 @@ import logging import re +from .const import LABEL_VERSION from .container import get_container_name, Container @@ -24,41 +25,82 @@ Alternatively, remove them: $ docker rm -f {rm_args} """ +ONE_OFF_ADDENDUM_FORMAT = """ +You should also remove your one-off containers: + + $ docker rm -f {rm_args} +""" + +ONE_OFF_ERROR_MESSAGE_FORMAT = """ +Compose found the following containers without labels: + +{names_list} + +As of Compose 1.3.0, containers are identified with labels instead of naming convention. + +Remove them before continuing: + + $ docker rm -f {rm_args} +""" + def check_for_legacy_containers( client, project, services, - stopped=False, - one_off=False): + allow_one_off=True): """Check if there are containers named using the old naming convention and warn the user that those containers may need to be migrated to using labels, so that compose can find them. """ - containers = list(get_legacy_containers( - client, - project, - services, - stopped=stopped, - one_off=one_off)) + containers = get_legacy_containers(client, project, services, one_off=False) if containers: - raise LegacyContainersError([c.name for c in containers]) + one_off_containers = get_legacy_containers(client, project, services, one_off=True) + + raise LegacyContainersError( + [c.name for c in containers], + [c.name for c in one_off_containers], + ) + + if not allow_one_off: + one_off_containers = get_legacy_containers(client, project, services, one_off=True) + + if one_off_containers: + raise LegacyOneOffContainersError( + [c.name for c in one_off_containers], + ) -class LegacyContainersError(Exception): - def __init__(self, names): +class LegacyError(Exception): + def __unicode__(self): + return self.msg + + __str__ = __unicode__ + + +class LegacyContainersError(LegacyError): + def __init__(self, names, one_off_names): self.names = names + self.one_off_names = one_off_names self.msg = ERROR_MESSAGE_FORMAT.format( names_list="\n".join(" {}".format(name) for name in names), rm_args=" ".join(names), ) - def __unicode__(self): - return self.msg + if one_off_names: + self.msg += ONE_OFF_ADDENDUM_FORMAT.format(rm_args=" ".join(one_off_names)) - __str__ = __unicode__ + +class LegacyOneOffContainersError(LegacyError): + def __init__(self, one_off_names): + self.one_off_names = one_off_names + + self.msg = ONE_OFF_ERROR_MESSAGE_FORMAT.format( + names_list="\n".join(" {}".format(name) for name in one_off_names), + rm_args=" ".join(one_off_names), + ) def add_labels(project, container): @@ -76,8 +118,8 @@ def migrate_project_to_labels(project): project.client, project.name, project.service_names, - stopped=True, - one_off=False) + one_off=False, + ) for container in containers: add_labels(project, container) @@ -87,13 +129,29 @@ def get_legacy_containers( client, project, services, - stopped=False, one_off=False): - containers = client.containers(all=stopped) + return list(_get_legacy_containers_iter( + client, + project, + services, + one_off=one_off, + )) + + +def _get_legacy_containers_iter( + client, + project, + services, + one_off=False): + + containers = client.containers(all=True) for service in services: for container in containers: + if LABEL_VERSION in container['Labels']: + continue + name = get_container_name(container) if has_container(project, service, name, one_off=one_off): yield Container.from_ps(client, container) diff --git a/compose/project.py b/compose/project.py index 288afc5f5..11c1e1ce9 100644 --- a/compose/project.py +++ b/compose/project.py @@ -308,8 +308,7 @@ class Project(object): self.client, self.name, self.service_names, - stopped=stopped, - one_off=one_off) + ) return filter(matches_service_names, containers) diff --git a/compose/service.py b/compose/service.py index ddabbfc4a..2bf22eac3 100644 --- a/compose/service.py +++ b/compose/service.py @@ -110,8 +110,7 @@ class Service(object): self.client, self.project, [self.name], - stopped=stopped, - one_off=one_off) + ) return containers diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index 346c84f2e..806b9a457 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -1,3 +1,5 @@ +import unittest + from docker.errors import APIError from compose import legacy @@ -5,6 +7,64 @@ from compose.project import Project from .testcases import DockerClientTestCase +class UtilitiesTestCase(unittest.TestCase): + def test_has_container(self): + self.assertTrue( + legacy.has_container("composetest", "web", "composetest_web_1", one_off=False), + ) + self.assertFalse( + legacy.has_container("composetest", "web", "composetest_web_run_1", one_off=False), + ) + + def test_has_container_one_off(self): + self.assertFalse( + legacy.has_container("composetest", "web", "composetest_web_1", one_off=True), + ) + self.assertTrue( + legacy.has_container("composetest", "web", "composetest_web_run_1", one_off=True), + ) + + def test_has_container_different_project(self): + self.assertFalse( + legacy.has_container("composetest", "web", "otherapp_web_1", one_off=False), + ) + self.assertFalse( + legacy.has_container("composetest", "web", "otherapp_web_run_1", one_off=True), + ) + + def test_has_container_different_service(self): + self.assertFalse( + legacy.has_container("composetest", "web", "composetest_db_1", one_off=False), + ) + self.assertFalse( + legacy.has_container("composetest", "web", "composetest_db_run_1", one_off=True), + ) + + def test_is_valid_name(self): + self.assertTrue( + legacy.is_valid_name("composetest_web_1", one_off=False), + ) + self.assertFalse( + legacy.is_valid_name("composetest_web_run_1", one_off=False), + ) + + def test_is_valid_name_one_off(self): + self.assertFalse( + legacy.is_valid_name("composetest_web_1", one_off=True), + ) + self.assertTrue( + legacy.is_valid_name("composetest_web_run_1", one_off=True), + ) + + def test_is_valid_name_invalid(self): + self.assertFalse( + legacy.is_valid_name("foo"), + ) + self.assertFalse( + legacy.is_valid_name("composetest_web_lol_1", one_off=True), + ) + + class LegacyTestCase(DockerClientTestCase): def setUp(self): @@ -30,7 +90,7 @@ class LegacyTestCase(DockerClientTestCase): # Create a single one-off legacy container self.containers.append(self.client.create_container( - name='{}_{}_run_1'.format(self.project.name, self.services[0].name), + name='{}_{}_run_1'.format(self.project.name, db.name), **self.services[0].options )) @@ -47,27 +107,84 @@ class LegacyTestCase(DockerClientTestCase): pass def get_legacy_containers(self, **kwargs): - return list(legacy.get_legacy_containers( + return legacy.get_legacy_containers( self.client, self.project.name, [s.name for s in self.services], **kwargs - )) + ) def test_get_legacy_container_names(self): self.assertEqual(len(self.get_legacy_containers()), len(self.services)) def test_get_legacy_container_names_one_off(self): - self.assertEqual(len(self.get_legacy_containers(stopped=True, one_off=True)), 1) + self.assertEqual(len(self.get_legacy_containers(one_off=True)), 1) def test_migration_to_labels(self): + # Trying to get the container list raises an exception + with self.assertRaises(legacy.LegacyContainersError) as cm: - self.assertEqual(self.project.containers(stopped=True), []) + self.project.containers(stopped=True) self.assertEqual( set(cm.exception.names), set(['composetest_db_1', 'composetest_web_1', 'composetest_nginx_1']), ) + self.assertEqual( + set(cm.exception.one_off_names), + set(['composetest_db_run_1']), + ) + + # Migrate the containers + legacy.migrate_project_to_labels(self.project) - self.assertEqual(len(self.project.containers(stopped=True)), len(self.services)) + + # Getting the list no longer raises an exception + + containers = self.project.containers(stopped=True) + self.assertEqual(len(containers), len(self.services)) + + def test_migration_one_off(self): + # We've already migrated + + legacy.migrate_project_to_labels(self.project) + + # Trying to create a one-off container results in a Docker API error + + with self.assertRaises(APIError) as cm: + self.project.get_service('db').create_container(one_off=True) + + # Checking for legacy one-off containers raises an exception + + with self.assertRaises(legacy.LegacyOneOffContainersError) as cm: + legacy.check_for_legacy_containers( + self.client, + self.project.name, + ['db'], + allow_one_off=False, + ) + + self.assertEqual( + set(cm.exception.one_off_names), + set(['composetest_db_run_1']), + ) + + # Remove the old one-off container + + c = self.client.inspect_container('composetest_db_run_1') + self.client.remove_container(c) + + # Checking no longer raises an exception + + legacy.check_for_legacy_containers( + self.client, + self.project.name, + ['db'], + allow_one_off=False, + ) + + # Creating a one-off container no longer results in an API error + + self.project.get_service('db').create_container(one_off=True) + self.assertIsInstance(self.client.inspect_container('composetest_db_run_1'), dict) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index d10cb9b30..ab3ea56a9 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -127,7 +127,7 @@ class CLITestCase(unittest.TestCase): def test_run_with_environment_merged_with_options_list(self, mock_dockerpty): command = TopLevelCommand() mock_client = mock.create_autospec(docker.Client) - mock_project = mock.Mock() + mock_project = mock.Mock(client=mock_client) mock_project.get_service.return_value = Service( 'service', client=mock_client, @@ -156,7 +156,7 @@ class CLITestCase(unittest.TestCase): def test_run_service_with_restart_always(self): command = TopLevelCommand() mock_client = mock.create_autospec(docker.Client) - mock_project = mock.Mock() + mock_project = mock.Mock(client=mock_client) mock_project.get_service.return_value = Service( 'service', client=mock_client, @@ -180,7 +180,7 @@ class CLITestCase(unittest.TestCase): command = TopLevelCommand() mock_client = mock.create_autospec(docker.Client) - mock_project = mock.Mock() + mock_project = mock.Mock(client=mock_client) mock_project.get_service.return_value = Service( 'service', client=mock_client, From a80afd67abcddf425017b5b3853ab931d5a8d3d1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 14 Jul 2015 16:55:44 +0100 Subject: [PATCH 49/53] Merge pull request #1688 from aanand/use-docker-py-1.3.0 Use docker-py 1.3.0 (cherry picked from commit 1e71eebc7475ac5307eace3a3b9e97dc0d3873cd) Signed-off-by: Aanand Prasad --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 69bd4c5f9..fc5b68489 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ PyYAML==3.10 -docker-py==1.2.3 +docker-py==1.3.0 dockerpty==0.3.4 docopt==0.6.1 requests==2.6.1 six==1.7.3 texttable==0.8.2 -websocket-client==0.11.0 +websocket-client==0.32.0 diff --git a/setup.py b/setup.py index d2e81e175..d0ec10679 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.7', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.11.0, < 1.0', - 'docker-py >= 1.2.3, < 1.3', + 'docker-py >= 1.3.0, < 1.4', 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', ] From 95cf195dbd6509d7292fefd01d70f284fe0d4dd0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 14 Jul 2015 17:40:43 +0100 Subject: [PATCH 50/53] Bump 1.3.2 Signed-off-by: Aanand Prasad --- CHANGES.md | 15 +++++++++++++++ compose/__init__.py | 2 +- docs/install.md | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1f43d88d4..b87a2e7de 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,21 @@ Change log ========== +1.3.2 (2015-07-14) +------------------ + +The following bugs have been fixed: + +- When there were one-off containers created by running `docker-compose run` on an older version of Compose, `docker-compose run` would fail with a name collision. Compose now shows an error if you have leftover containers of this type lying around, and tells you how to remove them. +- Compose was not reading Docker authentication config files created in the new location, `~/docker/config.json`, and authentication against private registries would therefore fail. +- When a container had a pseudo-TTY attached, its output in `docker-compose up` would be truncated. +- `docker-compose up --x-smart-recreate` would sometimes fail when an image tag was updated. +- `docker-compose up` would sometimes create two containers with the same numeric suffix. +- `docker-compose rm` and `docker-compose ps` would sometimes list services that aren't part of the current project (though no containers were erroneously removed). +- Some `docker-compose` commands would not show an error if invalid service names were passed in. + +Thanks @dano, @josephpage, @kevinsimper, @lieryan, @phemmer, @soulrebel and @sschepens! + 1.3.1 (2015-06-21) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index f3ec6acb0..1f6957495 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.3.1' +__version__ = '1.3.2' diff --git a/docs/install.md b/docs/install.md index 96a4a2376..cdaac34f3 100644 --- a/docs/install.md +++ b/docs/install.md @@ -27,7 +27,7 @@ First, install Docker version 1.6 or greater: To install Compose, run the following commands: - curl -L https://github.com/docker/compose/releases/download/1.3.1/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.3.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands above, then `exit`. From cd441793052a4c216bf2942c3b8e4687f965c672 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 15 Jul 2015 17:30:31 +0100 Subject: [PATCH 51/53] Merge pull request #1705 from aanand/fix-labels-null Handle case where /containers/json returns "Labels": null (cherry picked from commit 7b9664be8e82b03f316d88c928953c62e897c2cd) Signed-off-by: Aanand Prasad --- compose/legacy.py | 2 +- tests/integration/legacy_test.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/compose/legacy.py b/compose/legacy.py index c9ec65817..6fbf74d69 100644 --- a/compose/legacy.py +++ b/compose/legacy.py @@ -149,7 +149,7 @@ def _get_legacy_containers_iter( for service in services: for container in containers: - if LABEL_VERSION in container['Labels']: + if LABEL_VERSION in (container.get('Labels') or {}): continue name = get_container_name(container) diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index 806b9a457..f79089b20 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -1,4 +1,5 @@ import unittest +from mock import Mock from docker.errors import APIError @@ -64,6 +65,22 @@ class UtilitiesTestCase(unittest.TestCase): legacy.is_valid_name("composetest_web_lol_1", one_off=True), ) + def test_get_legacy_containers_no_labels(self): + client = Mock() + client.containers.return_value = [ + { + "Id": "abc123", + "Image": "def456", + "Name": "composetest_web_1", + "Labels": None, + }, + ] + + containers = list(legacy.get_legacy_containers( + client, "composetest", ["web"])) + + self.assertEqual(len(containers), 1) + class LegacyTestCase(DockerClientTestCase): From e5f6ae767d78a2213eac2d52bade3dfdfe620730 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 16 Jul 2015 10:50:06 +0100 Subject: [PATCH 52/53] Merge pull request #1704 from aanand/fix-timeout-type Make sure up/restart/stop timeout is an int (cherry picked from commit c7dccccd1fa4dc2fe6f65d4a839a16567adbee9d) Signed-off-by: Aanand Prasad --- compose/cli/main.py | 6 +++--- tests/integration/state_test.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index d5d15177c..ba4445735 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -403,7 +403,7 @@ class TopLevelCommand(Command): -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) """ - timeout = float(options.get('--timeout') or DEFAULT_TIMEOUT) + timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) project.stop(service_names=options['SERVICE'], timeout=timeout) def restart(self, project, options): @@ -416,7 +416,7 @@ class TopLevelCommand(Command): -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) """ - timeout = float(options.get('--timeout') or DEFAULT_TIMEOUT) + timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) project.restart(service_names=options['SERVICE'], timeout=timeout) def up(self, project, options): @@ -459,7 +459,7 @@ class TopLevelCommand(Command): allow_recreate = not options['--no-recreate'] smart_recreate = options['--x-smart-recreate'] service_names = options['SERVICE'] - timeout = float(options.get('--timeout') or DEFAULT_TIMEOUT) + timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) project.up( service_names=service_names, diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 95fcc49de..fb91bc249 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -13,7 +13,7 @@ from .testcases import DockerClientTestCase class ProjectTestCase(DockerClientTestCase): def run_up(self, cfg, **kwargs): kwargs.setdefault('smart_recreate', True) - kwargs.setdefault('timeout', 0.1) + kwargs.setdefault('timeout', 1) project = self.make_project(cfg) project.up(**kwargs) @@ -171,7 +171,7 @@ def converge(service, plan, insecure_registry=insecure_registry, do_build=do_build, - timeout=0.1, + timeout=1, ) From 8cff440800a6098eba36cbce25bfef34e939b139 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 15 Jul 2015 17:36:45 +0100 Subject: [PATCH 53/53] Bump 1.3.3 Signed-off-by: Aanand Prasad --- CHANGES.md | 8 ++++++++ compose/__init__.py | 2 +- docs/install.md | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b87a2e7de..38a543249 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,14 @@ Change log ========== +1.3.3 (2015-07-15) +------------------ + +Two regressions have been fixed: + +- When stopping containers gracefully, Compose was setting the timeout to 0, effectively forcing a SIGKILL every time. +- Compose would sometimes crash depending on the formatting of container data returned from the Docker API. + 1.3.2 (2015-07-14) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index 1f6957495..97d9c11d9 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.3.2' +__version__ = '1.3.3' diff --git a/docs/install.md b/docs/install.md index cdaac34f3..c025469b6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -27,7 +27,7 @@ First, install Docker version 1.6 or greater: To install Compose, run the following commands: - curl -L https://github.com/docker/compose/releases/download/1.3.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.3.3/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands above, then `exit`.