From 000eaee16ab19608ee4d96f2ceb48cf24a763d76 Mon Sep 17 00:00:00 2001 From: wenchma Date: Tue, 8 Mar 2016 17:09:28 +0800 Subject: [PATCH 01/95] Update image format for service conf reference Signed-off-by: Wen Cheng Ma --- docs/compose-file.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 24e451603..d8e98fbc8 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -59,13 +59,13 @@ optionally [dockerfile](#dockerfile) and [args](#args). args: buildno: 1 -If you specify `image` as well as `build`, then Compose tags the built image -with the tag specified in `image`: +If you specify `image` as well as `build`, then Compose names the built image +with the `webapp` and optional `tag` specified in `image`: build: ./dir - image: webapp + image: webapp:tag -This will result in an image tagged `webapp`, built from `./dir`. +This will result in an image named `webapp` and tagged `tag`, built from `./dir`. > **Note**: In the [version 1 file format](#version-1), `build` is different in > two ways: From 7fc40dd7ccb5b839340c666a0902eb7bc47c80a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kalle=20M=C3=B8ller?= Date: Mon, 14 Mar 2016 01:54:15 +0100 Subject: [PATCH 02/95] Adding ssl_version to docker_clients kwargs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Select tls version based of COMPOSE_TLS_VERSION Changed from SSL to TLS Also did docs - missing default value Using getattr and raises AttributeError in case of unsupported version Signed-off-by: Kalle Møller --- compose/cli/command.py | 15 ++++++++++++--- compose/cli/docker_client.py | 4 ++-- docs/reference/envvars.md | 4 ++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 55f6df01a..7e219e110 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import logging import os import re +import ssl import six @@ -37,8 +38,8 @@ def get_config_path_from_options(options): return None -def get_client(verbose=False, version=None): - client = docker_client(version=version) +def get_client(verbose=False, version=None, tls_version=None): + client = docker_client(version=version, tls_version=tls_version) if verbose: version_info = six.iteritems(client.version()) log.info(get_version_info('full')) @@ -57,7 +58,15 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False) api_version = os.environ.get( 'COMPOSE_API_VERSION', API_VERSIONS[config_data.version]) - client = get_client(verbose=verbose, version=api_version) + compose_tls_version = os.environ.get( + 'COMPOSE_TLS_VERSION', + None) + + tls_version = None + if compose_tls_version: + tls_version = ssl.getattr("PROTOCOL_{}".format(compose_tls_version)) + + client = get_client(verbose=verbose, version=api_version, tls_version=tls_version) return Project.from_config(project_name, config_data, client) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 9e79fe777..5663a57c9 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -14,7 +14,7 @@ from .errors import UserError log = logging.getLogger(__name__) -def docker_client(version=None): +def docker_client(version=None, tls_version=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. @@ -24,7 +24,7 @@ def docker_client(version=None): "Please use COMPOSE_HTTP_TIMEOUT instead.") try: - kwargs = kwargs_from_env(assert_hostname=False) + kwargs = kwargs_from_env(assert_hostname=False, ssl_version=tls_version) except TLSParameterError: raise UserError( "TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY " diff --git a/docs/reference/envvars.md b/docs/reference/envvars.md index e1170be90..ca88276e7 100644 --- a/docs/reference/envvars.md +++ b/docs/reference/envvars.md @@ -75,6 +75,10 @@ Configures the path to the `ca.pem`, `cert.pem`, and `key.pem` files used for TL Configures the time (in seconds) a request to the Docker daemon is allowed to hang before Compose considers it failed. Defaults to 60 seconds. +## COMPOSE\_TLS\_VERSION + +Configure which TLS version is used for TLS communication with the `docker` daemon, defaults to `TBD` +Can be `TLSv1`, `TLSv1_1`, `TLSv1_2`. ## Related Information From a53b29467a681405e51318561ac0167ab2665504 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 22 Mar 2016 17:17:06 -0700 Subject: [PATCH 03/95] Update wordpress example to use official images The orchardup images are very old and not maintained. Signed-off-by: Ben Firshman --- docs/wordpress.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/wordpress.md b/docs/wordpress.md index 62f50c249..fcfaef191 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -36,8 +36,10 @@ with Docker containers. This quick-start guide demonstrates how to use Compose t In this case, your Dockerfile should include these two lines: - FROM orchardup/php5 + FROM php:5.6-fpm + RUN docker-php-ext-install mysql ADD . /code + CMD php -S 0.0.0.0:8000 -t /code/wordpress/ This tells the Docker Engine daemon how to build an image defining a container that contains PHP and WordPress. @@ -47,7 +49,6 @@ with Docker containers. This quick-start guide demonstrates how to use Compose t services: web: build: . - command: php -S 0.0.0.0:8000 -t /code/wordpress/ ports: - "8000:8000" depends_on: @@ -55,9 +56,12 @@ with Docker containers. This quick-start guide demonstrates how to use Compose t volumes: - .:/code db: - image: orchardup/mysql + image: mysql environment: + MYSQL_ROOT_PASSWORD: wordpress MYSQL_DATABASE: wordpress + MYSQL_USER: wordpress + MYSQL_PASSWORD: wordpress 5. Download WordPress into the current directory: @@ -71,8 +75,8 @@ with Docker containers. This quick-start guide demonstrates how to use Compose t Date: Fri, 25 Mar 2016 18:52:28 +0100 Subject: [PATCH 04/95] Add zsh completion for 'docker-compose rm -a --all' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index f67bc9f64..64e794286 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -266,6 +266,7 @@ __docker-compose_subcommand() { (rm) _arguments \ $opts_help \ + '(-a --all)'{-a,--all}"[Also remove one-off containers]" \ '(-f --force)'{-f,--force}"[Don't ask to confirm removal]" \ '-v[Remove volumes associated with containers]' \ '*:stopped services:__docker-compose_stoppedservices' && ret=0 From 63b448120a960194d3d0f23f751ec5e5534e397e Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 25 Mar 2016 19:03:36 +0100 Subject: [PATCH 05/95] Add zsh completion for 'docker-compose exec' command Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index f67bc9f64..a40f10100 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -223,6 +223,18 @@ __docker-compose_subcommand() { '--json[Output events as a stream of json objects.]' \ '*:services:__docker-compose_services_all' && ret=0 ;; + (exec) + _arguments \ + $opts_help \ + '-d[Detached mode: Run command in the background.]' \ + '--privileged[Give extended privileges to the process.]' \ + '--user=[Run the command as this user.]:username:_users' \ + '-T[Disable pseudo-tty allocation. By default `docker-compose exec` allocates a TTY.]' \ + '--index=[Index of the container if there are multiple instances of a service (default: 1)]:index: ' \ + '(-):running services:__docker-compose_runningservices' \ + '(-):command: _command_names -e' \ + '*::arguments: _normal' && ret=0 + ;; (help) _arguments ':subcommand:__docker-compose_commands' && ret=0 ;; From 9d58b19ecc21c56c5b6763361265fe66c2652601 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 25 Mar 2016 19:09:53 +0100 Subject: [PATCH 06/95] Add zsh completion for 'docker-compose logs -f --follow --tail -t --timestamps' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index f67bc9f64..ecd8db939 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -235,7 +235,10 @@ __docker-compose_subcommand() { (logs) _arguments \ $opts_help \ + '(-f --follow)'{-f,--follow}'[Follow log output]' \ '--no-color[Produce monochrome output.]' \ + '--tail=[Number of lines to show from the end of the logs for each container.]:number of lines: ' \ + '(-t --timestamps)'{-t,--timestamps}'[Show timestamps]' \ '*:services:__docker-compose_services_all' && ret=0 ;; (pause) From 9729c0d3c72f0c16932efd9dd2574d08f3d5a3a7 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 25 Mar 2016 19:15:34 +0100 Subject: [PATCH 07/95] Add zsh completion for 'docker-compose up --build' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index f67bc9f64..d837e61e4 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -313,6 +313,7 @@ __docker-compose_subcommand() { _arguments \ $opts_help \ '(--abort-on-container-exit)-d[Detached mode: Run containers in the background, print new container names.]' \ + '--build[Build images before starting containers.]' \ '--no-color[Produce monochrome output.]' \ "--no-deps[Don't start linked services.]" \ "--force-recreate[Recreate containers even if their configuration and image haven't changed. Incompatible with --no-recreate.]" \ From 8ae8f7ed4befe40578eddb005907f49943a063cb Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Fri, 25 Mar 2016 19:25:33 +0100 Subject: [PATCH 08/95] Add zsh completion for 'docker-compose run -w --workdir' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index f67bc9f64..3e3f24d0c 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -274,15 +274,16 @@ __docker-compose_subcommand() { _arguments \ $opts_help \ '-d[Detached mode: Run container in the background, print new container name.]' \ - '--name[Assign a name to the container]:name: ' \ - '--entrypoint[Overwrite the entrypoint of the image.]:entry point: ' \ '*-e[KEY=VAL Set an environment variable (can be used multiple times)]:environment variable KEY=VAL: ' \ - '(-u --user)'{-u,--user=-}'[Run as specified username or uid]:username or uid:_users' \ + '--entrypoint[Overwrite the entrypoint of the image.]:entry point: ' \ + '--name[Assign a name to the container]:name: ' \ "--no-deps[Don't start linked services.]" \ + '(-p --publish)'{-p,--publish=-}"[Run command with manually mapped container's port(s) to the host.]" \ '--rm[Remove container after run. Ignored in detached mode.]' \ "--service-ports[Run command with the service's ports enabled and mapped to the host.]" \ - '(-p --publish)'{-p,--publish=-}"[Run command with manually mapped container's port(s) to the host.]" \ '-T[Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY.]' \ + '(-u --user)'{-u,--user=-}'[Run as specified username or uid]:username or uid:_users' \ + '(-w --workdir)'{-w=,--workdir=}'[Working directory inside the container]:workdir: ' \ '(-):services:__docker-compose_services' \ '(-):command: _command_names -e' \ '*::arguments: _normal' && ret=0 From 93901ec4805b0a72ba71ae910d3214e4856cd876 Mon Sep 17 00:00:00 2001 From: Jon Lemmon Date: Mon, 28 Mar 2016 13:29:01 +1300 Subject: [PATCH 09/95] Rails Docs: Add nodejs to apt-get install command When using the latest version of Rails, the tutorial currently errors when running `docker-compose up` with the following error: ``` /usr/local/lib/ruby/gems/2.3.0/gems/bundler-1.11.2/lib/bundler/runtime.rb:80:in `rescue in block (2 levels) in require': There was an error while trying to load the gem 'uglifier'. (Bundler::GemRequireError) ``` Installing nodejs in the build fixes the issue. Signed-off-by: Jon Lemmon --- docs/rails.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rails.md b/docs/rails.md index a8fc383e7..eef6b2f4b 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -22,7 +22,7 @@ container. This is done using a file called `Dockerfile`. To begin with, the Dockerfile consists of: FROM ruby:2.2.0 - RUN apt-get update -qq && apt-get install -y build-essential libpq-dev + RUN apt-get update -qq && apt-get install -y build-essential libpq-dev nodejs RUN mkdir /myapp WORKDIR /myapp ADD Gemfile /myapp/Gemfile From 7116aefe4310c77a6d8f80a9f928ce6437e8bb49 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 28 Mar 2016 17:39:20 -0700 Subject: [PATCH 10/95] Fix assert_hostname logic in tls_config_from_options Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 27 ++++++++++++++++++++------- tests/unit/cli/docker_client_test.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index f782a1ae6..83cd8626c 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -21,24 +21,37 @@ def tls_config_from_options(options): cert = options.get('--tlscert') key = options.get('--tlskey') verify = options.get('--tlsverify') - hostname = urlparse(options.get('--host') or '').hostname + host = options.get('--host') + skip_hostname_check = options.get('--skip-hostname-check', False) + + if not skip_hostname_check: + hostname = urlparse(host).hostname if host else None + # If the protocol is omitted, urlparse fails to extract the hostname. + # Make another attempt by appending a protocol. + if not hostname and host: + hostname = urlparse('tcp://{0}'.format(host)).hostname advanced_opts = any([ca_cert, cert, key, verify]) if tls is True and not advanced_opts: return True - elif advanced_opts: + elif advanced_opts: # --tls is a noop client_cert = None if cert or key: client_cert = (cert, key) + + assert_hostname = None + if skip_hostname_check: + assert_hostname = False + elif hostname: + assert_hostname = hostname + return TLSConfig( client_cert=client_cert, verify=verify, ca_cert=ca_cert, - assert_hostname=( - hostname or not options.get('--skip-hostname-check', False) - ) + assert_hostname=assert_hostname ) - else: - return None + + return None def docker_client(environment, version=None, tls_config=None, host=None): diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index 56bab19c3..f4476ad3b 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -103,3 +103,31 @@ class TLSConfigTestCase(unittest.TestCase): options = {'--tlskey': self.key} with pytest.raises(docker.errors.TLSParameterError): tls_config_from_options(options) + + def test_assert_hostname_explicit_host(self): + options = { + '--tlscacert': self.ca_cert, '--host': 'tcp://foobar.co.uk:1254' + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.assert_hostname == 'foobar.co.uk' + + def test_assert_hostname_explicit_host_no_proto(self): + options = { + '--tlscacert': self.ca_cert, '--host': 'foobar.co.uk:1254' + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.assert_hostname == 'foobar.co.uk' + + def test_assert_hostname_implicit_host(self): + options = {'--tlscacert': self.ca_cert} + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.assert_hostname is None + + def test_assert_hostname_explicit_skip(self): + options = {'--tlscacert': self.ca_cert, '--skip-hostname-check': True} + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.assert_hostname is False From 71c86acaa4af0af5dec9baf7f1f4d7b236f249a3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 29 Mar 2016 18:01:27 -0700 Subject: [PATCH 11/95] Update docker-py version to include match_hostname fix Removed unnecessary assert_hostname computation in tls_config_from_options Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 17 +---------------- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 83cd8626c..e9f39d010 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -7,7 +7,6 @@ from docker import Client from docker.errors import TLSParameterError from docker.tls import TLSConfig from docker.utils import kwargs_from_env -from requests.utils import urlparse from ..const import HTTP_TIMEOUT from .errors import UserError @@ -21,16 +20,8 @@ def tls_config_from_options(options): cert = options.get('--tlscert') key = options.get('--tlskey') verify = options.get('--tlsverify') - host = options.get('--host') skip_hostname_check = options.get('--skip-hostname-check', False) - if not skip_hostname_check: - hostname = urlparse(host).hostname if host else None - # If the protocol is omitted, urlparse fails to extract the hostname. - # Make another attempt by appending a protocol. - if not hostname and host: - hostname = urlparse('tcp://{0}'.format(host)).hostname - advanced_opts = any([ca_cert, cert, key, verify]) if tls is True and not advanced_opts: @@ -40,15 +31,9 @@ def tls_config_from_options(options): if cert or key: client_cert = (cert, key) - assert_hostname = None - if skip_hostname_check: - assert_hostname = False - elif hostname: - assert_hostname = hostname - return TLSConfig( client_cert=client_cert, verify=verify, ca_cert=ca_cert, - assert_hostname=assert_hostname + assert_hostname=False if skip_hostname_check else None ) return None diff --git a/requirements.txt b/requirements.txt index 91d0487cd..4bee21ef4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ PyYAML==3.11 cached-property==1.2.0 -docker-py==1.8.0rc2 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4 +git+https://github.com/docker/docker-py.git@ac3d4aae2c525b052e661f42307223676ca1b313#egg=docker-py jsonschema==2.5.1 requests==2.7.0 six==1.7.3 From d27b82207cc0ef4364b56a3d1e823b47791836ba Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 29 Mar 2016 18:05:37 -0700 Subject: [PATCH 12/95] Remove obsolete assert_hostname tests Signed-off-by: Joffrey F --- requirements.txt | 2 +- tests/unit/cli/docker_client_test.py | 22 ---------------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4bee21ef4..898df3732 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ PyYAML==3.11 cached-property==1.2.0 +docker-py==1.8.0rc3 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4 -git+https://github.com/docker/docker-py.git@ac3d4aae2c525b052e661f42307223676ca1b313#egg=docker-py jsonschema==2.5.1 requests==2.7.0 six==1.7.3 diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index f4476ad3b..5334a9440 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -104,28 +104,6 @@ class TLSConfigTestCase(unittest.TestCase): with pytest.raises(docker.errors.TLSParameterError): tls_config_from_options(options) - def test_assert_hostname_explicit_host(self): - options = { - '--tlscacert': self.ca_cert, '--host': 'tcp://foobar.co.uk:1254' - } - result = tls_config_from_options(options) - assert isinstance(result, docker.tls.TLSConfig) - assert result.assert_hostname == 'foobar.co.uk' - - def test_assert_hostname_explicit_host_no_proto(self): - options = { - '--tlscacert': self.ca_cert, '--host': 'foobar.co.uk:1254' - } - result = tls_config_from_options(options) - assert isinstance(result, docker.tls.TLSConfig) - assert result.assert_hostname == 'foobar.co.uk' - - def test_assert_hostname_implicit_host(self): - options = {'--tlscacert': self.ca_cert} - result = tls_config_from_options(options) - assert isinstance(result, docker.tls.TLSConfig) - assert result.assert_hostname is None - def test_assert_hostname_explicit_skip(self): options = {'--tlscacert': self.ca_cert, '--skip-hostname-check': True} result = tls_config_from_options(options) From 78a8be07adc0f83ec627d6865eb17da5c69093fa Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 30 Mar 2016 11:11:19 -0700 Subject: [PATCH 13/95] Re-enabling assert_hostname when instantiating docker_client from the environment. Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index e9f39d010..0c0113bb7 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -49,7 +49,7 @@ def docker_client(environment, version=None, tls_config=None, host=None): "Please use COMPOSE_HTTP_TIMEOUT instead.") try: - kwargs = kwargs_from_env(assert_hostname=False, environment=environment) + kwargs = kwargs_from_env(environment=environment) except TLSParameterError: raise UserError( "TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY " From 3034803258612e66bff99cdcc718253633da6bb3 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 31 Mar 2016 15:45:14 +0100 Subject: [PATCH 14/95] Better variable substitution example Signed-off-by: Aanand Prasad --- docs/compose-file.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index e9ec0a2de..5aef5aca9 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -1089,21 +1089,24 @@ It's more complicated if you're using particular configuration features: ## Variable substitution Your configuration options can contain environment variables. Compose uses the -variable values from the shell environment in which `docker-compose` is run. For -example, suppose the shell contains `POSTGRES_VERSION=9.3` and you supply this -configuration: +variable values from the shell environment in which `docker-compose` is run. +For example, suppose the shell contains `EXTERNAL_PORT=8000` and you supply +this configuration: - db: - image: "postgres:${POSTGRES_VERSION}" + web: + build: . + ports: + - "${EXTERNAL_PORT}:5000" -When you run `docker-compose up` with this configuration, Compose looks for the -`POSTGRES_VERSION` environment variable in the shell and substitutes its value -in. For this example, Compose resolves the `image` to `postgres:9.3` before -running the configuration. +When you run `docker-compose up` with this configuration, Compose looks for +the `EXTERNAL_PORT` environment variable in the shell and substitutes its +value in. In this example, Compose resolves the port mapping to `"8000:5000"` +before creating the `web` container. If an environment variable is not set, Compose substitutes with an empty -string. In the example above, if `POSTGRES_VERSION` is not set, the value for -the `image` option is `postgres:`. +string. In the example above, if `EXTERNAL_PORT` is not set, the value for the +port mapping is `:5000` (which is of course an invalid port mapping, and will +result in an error when attempting to create the container). Both `$VARIABLE` and `${VARIABLE}` syntax are supported. Extended shell-style features, such as `${VARIABLE-default}` and `${VARIABLE/foo/bar}`, are not From 1a7a65f84da129cb3491c2dec3f37367444ce807 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 30 Mar 2016 11:58:28 -0700 Subject: [PATCH 15/95] Include docker-py requirements fix Signed-off-by: Joffrey F --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 898df3732..76f224fbe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PyYAML==3.11 cached-property==1.2.0 -docker-py==1.8.0rc3 +docker-py==1.8.0rc5 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4 From c1026e815a114b1210070d2daa56599d62d9a76e Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Sun, 10 Jan 2016 12:50:38 +0000 Subject: [PATCH 16/95] Update roadmap Bring it inline with current plans: - Use in production is not necessarily about the command-line tool, but also improving file format, integrations, new tools, etc. Signed-off-by: Ben Firshman --- ROADMAP.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 67903492e..c57397bd0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,13 +1,21 @@ # Roadmap +## An even better tool for development environments + +Compose is a great tool for development environments, but it could be even better. For example: + +- It should be possible to define hostnames for containers which work from the host machine, e.g. “mywebcontainer.local”. This is needed by apps comprising multiple web services which generate links to one another (e.g. a frontend website and a separate admin webapp) + ## More than just development environments -Over time we will extend Compose's remit to cover test, staging and production environments. This is not a simple task, and will take many incremental improvements such as: +Compose currently works really well in development, but we want to make the Compose file format better for test, staging, and production environments. To support these use cases, there will need to be improvements to the file format, improvements to the command-line tool, integrations with other tools, and perhaps new tools altogether. + +Some specific things we are considering: - Compose currently will attempt to get your application into the correct state when running `up`, but it has a number of shortcomings: - It should roll back to a known good state if it fails. - It should allow a user to check the actions it is about to perform before running them. -- It should be possible to partially modify the config file for different environments (dev/test/staging/prod), passing in e.g. custom ports or volume mount paths. ([#1377](https://github.com/docker/compose/issues/1377)) +- It should be possible to partially modify the config file for different environments (dev/test/staging/prod), passing in e.g. custom ports, volume mount paths, or volume drivers. ([#1377](https://github.com/docker/compose/issues/1377)) - Compose should recommend a technique for zero-downtime deploys. - It should be possible to continuously attempt to keep an application in the correct state, instead of just performing `up` a single time. @@ -23,9 +31,3 @@ Compose works well for applications that are in a single repository and depend o There are several ideas about how this could work, such as [including external files](https://github.com/docker/fig/issues/318). -## An even better tool for development environments - -Compose is a great tool for development environments, but it could be even better. For example: - -- [Compose could watch your code and automatically kick off builds when something changes.](https://github.com/docker/fig/issues/184) -- It should be possible to define hostnames for containers which work from the host machine, e.g. “mywebcontainer.local”. This is needed by apps comprising multiple web services which generate links to one another (e.g. a frontend website and a separate admin webapp) From 129fb5b356eddbb9d4939bd04e6944559f905672 Mon Sep 17 00:00:00 2001 From: Tony Witherspoon Date: Mon, 4 Apr 2016 13:15:28 -0400 Subject: [PATCH 17/95] Added code to output the top level command options if docker-compose help with no command options provided Signed-off-by: Tony Witherspoon --- compose/cli/main.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 6eada097f..cf92a57b0 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -355,10 +355,14 @@ class TopLevelCommand(object): """ Get help on a command. - Usage: help COMMAND + Usage: help [COMMAND] """ - handler = get_handler(cls, options['COMMAND']) - raise SystemExit(getdoc(handler)) + if options['COMMAND']: + subject = get_handler(cls, options['COMMAND']) + else: + subject = cls + + print(getdoc(subject)) def kill(self, options): """ From b33d7b3dd88dbadbcd4230e38dc0a5504f9a6297 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 5 Apr 2016 11:26:23 -0400 Subject: [PATCH 18/95] Prevent unnecessary inspection of containers when created from an inspect. Signed-off-by: Daniel Nephin --- ROADMAP.md | 1 - compose/container.py | 2 +- tests/integration/service_test.py | 12 ++++++------ tests/unit/project_test.py | 9 +++++++++ 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index c57397bd0..287e54680 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -30,4 +30,3 @@ The current state of integration is documented in [SWARM.md](SWARM.md). Compose works well for applications that are in a single repository and depend on services that are hosted on Docker Hub. If your application depends on another application within your organisation, Compose doesn't work as well. There are several ideas about how this could work, such as [including external files](https://github.com/docker/fig/issues/318). - diff --git a/compose/container.py b/compose/container.py index 6dac94999..2c16863df 100644 --- a/compose/container.py +++ b/compose/container.py @@ -39,7 +39,7 @@ class Container(object): @classmethod def from_id(cls, client, id): - return cls(client, client.inspect_container(id)) + return cls(client, client.inspect_container(id), has_been_inspected=True) @classmethod def create(cls, client, **options): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index e2ef1161d..0a109ada3 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -769,17 +769,17 @@ class ServiceTest(DockerClientTestCase): container = service.create_container(number=next_number, quiet=True) container.start() - self.assertTrue(container.is_running) - self.assertEqual(len(service.containers()), 1) + container.inspect() + assert container.is_running + assert len(service.containers()) == 1 service.scale(1) - - self.assertEqual(len(service.containers()), 1) + assert len(service.containers()) == 1 container.inspect() - self.assertTrue(container.is_running) + assert container.is_running captured_output = mock_log.info.call_args[0] - self.assertIn('Desired container number already achieved', captured_output) + assert 'Desired container number already achieved' in captured_output @mock.patch('compose.service.log') def test_scale_with_custom_container_name_outputs_warning(self, mock_log): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 0d381951c..b6a52e08d 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -270,12 +270,21 @@ class ProjectTest(unittest.TestCase): 'time': 1420092061, 'timeNano': 14200920610000004000, }, + { + 'status': 'destroy', + 'from': 'example/db', + 'id': 'eeeee', + 'time': 1420092061, + 'timeNano': 14200920610000004000, + }, ]) def dt_with_microseconds(dt, us): return datetime.datetime.fromtimestamp(dt).replace(microsecond=us) def get_container(cid): + if cid == 'eeeee': + raise NotFound(None, None, "oops") if cid == 'abcde': name = 'web' labels = {LABEL_SERVICE: name} From 3ef6b17bfc1d6aeb97b5ef2ac77c3659cd28ac4e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 6 Apr 2016 13:28:45 -0700 Subject: [PATCH 19/95] Use docker-py 1.8.0 Signed-off-by: Joffrey F --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 76f224fbe..b9b0f4036 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PyYAML==3.11 cached-property==1.2.0 -docker-py==1.8.0rc5 +docker-py==1.8.0 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4 From 5d0aab4a8e3a231f6fd548be6f9881ddefc60cfc Mon Sep 17 00:00:00 2001 From: Tony Witherspoon Date: Thu, 7 Apr 2016 12:42:14 -0400 Subject: [PATCH 20/95] updated cli_test.py to no longer expect raised SystemExit exceptions Signed-off-by: Tony Witherspoon --- tests/unit/cli_test.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index e0ada460d..b1475f841 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -64,12 +64,6 @@ class CLITestCase(unittest.TestCase): self.assertTrue(project.client) self.assertTrue(project.services) - def test_command_help(self): - with pytest.raises(SystemExit) as exc: - TopLevelCommand.help({'COMMAND': 'up'}) - - assert 'Usage: up' in exc.exconly() - def test_command_help_nonexistent(self): with pytest.raises(NoSuchCommand): TopLevelCommand.help({'COMMAND': 'nonexistent'}) From bcdf541c8c6ccc0070ab011a909f244f501676d6 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 8 Apr 2016 12:58:19 +0100 Subject: [PATCH 21/95] Refactor setup_queue() - Stop sharing set objects across threads - Use a second queue to signal when producer threads are done - Use a single consumer thread to check dependencies and kick off new producers Signed-off-by: Aanand Prasad --- compose/parallel.py | 64 ++++++++++++++++++++++++++++++--------------- compose/service.py | 3 +++ 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index c629a1abf..79699236d 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import logging import operator import sys from threading import Thread @@ -14,6 +15,9 @@ from compose.cli.signals import ShutdownException from compose.utils import get_output_stream +log = logging.getLogger(__name__) + + def parallel_execute(objects, func, get_name, msg, get_deps=None): """Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. @@ -73,35 +77,53 @@ def setup_queue(objects, func, get_deps, get_name): get_deps = _no_deps results = Queue() - started = set() # objects being processed - finished = set() # objects which have been processed + output = Queue() - def do_op(obj): + def consumer(): + started = set() # objects being processed + finished = set() # objects which have been processed + + def ready(obj): + """ + Returns true if obj is ready to be processed: + - all dependencies have been processed + - obj is not already being processed + """ + return obj not in started and all( + dep not in objects or dep in finished + for dep in get_deps(obj) + ) + + while len(finished) < len(objects): + for obj in filter(ready, objects): + log.debug('Starting producer thread for {}'.format(obj)) + t = Thread(target=producer, args=(obj,)) + t.daemon = True + t.start() + started.add(obj) + + try: + event = results.get(timeout=1) + except Empty: + continue + + obj = event[0] + log.debug('Finished processing: {}'.format(obj)) + finished.add(obj) + output.put(event) + + def producer(obj): try: result = func(obj) results.put((obj, result, None)) except Exception as e: results.put((obj, None, e)) - finished.add(obj) - feed() + t = Thread(target=consumer) + t.daemon = True + t.start() - def ready(obj): - # Is object ready for performing operation - return obj not in started and all( - dep not in objects or dep in finished - for dep in get_deps(obj) - ) - - def feed(): - for obj in filter(ready, objects): - started.add(obj) - t = Thread(target=do_op, args=(obj,)) - t.daemon = True - t.start() - - feed() - return results + return output class ParallelStreamWriter(object): diff --git a/compose/service.py b/compose/service.py index ed45f0781..05cfc7c61 100644 --- a/compose/service.py +++ b/compose/service.py @@ -135,6 +135,9 @@ class Service(object): self.networks = networks or {} self.options = options + def __repr__(self): + return ''.format(self.name) + def containers(self, stopped=False, one_off=False, filters={}): filters.update({'label': self.labels(one_off=one_off)}) From 141b96bb312d85753de2189227941512bd42f33e Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 8 Apr 2016 17:46:13 +0100 Subject: [PATCH 22/95] Abort operations if their dependencies fail Signed-off-by: Aanand Prasad --- compose/parallel.py | 102 +++++++++++++++++++++--------------- tests/unit/parallel_test.py | 73 ++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 43 deletions(-) create mode 100644 tests/unit/parallel_test.py diff --git a/compose/parallel.py b/compose/parallel.py index 79699236d..745d46351 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -32,7 +32,7 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): for obj in objects: writer.initialize(get_name(obj)) - q = setup_queue(objects, func, get_deps, get_name) + q = setup_queue(objects, func, get_deps) done = 0 errors = {} @@ -54,6 +54,8 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): elif isinstance(exception, APIError): errors[get_name(obj)] = exception.explanation writer.write(get_name(obj), 'error') + elif isinstance(exception, UpstreamError): + writer.write(get_name(obj), 'error') else: errors[get_name(obj)] = exception error_to_reraise = exception @@ -72,60 +74,74 @@ def _no_deps(x): return [] -def setup_queue(objects, func, get_deps, get_name): +def setup_queue(objects, func, get_deps): if get_deps is None: get_deps = _no_deps results = Queue() output = Queue() - def consumer(): - started = set() # objects being processed - finished = set() # objects which have been processed - - def ready(obj): - """ - Returns true if obj is ready to be processed: - - all dependencies have been processed - - obj is not already being processed - """ - return obj not in started and all( - dep not in objects or dep in finished - for dep in get_deps(obj) - ) - - while len(finished) < len(objects): - for obj in filter(ready, objects): - log.debug('Starting producer thread for {}'.format(obj)) - t = Thread(target=producer, args=(obj,)) - t.daemon = True - t.start() - started.add(obj) - - try: - event = results.get(timeout=1) - except Empty: - continue - - obj = event[0] - log.debug('Finished processing: {}'.format(obj)) - finished.add(obj) - output.put(event) - - def producer(obj): - try: - result = func(obj) - results.put((obj, result, None)) - except Exception as e: - results.put((obj, None, e)) - - t = Thread(target=consumer) + t = Thread(target=queue_consumer, args=(objects, func, get_deps, results, output)) t.daemon = True t.start() return output +def queue_producer(obj, func, results): + try: + result = func(obj) + results.put((obj, result, None)) + except Exception as e: + results.put((obj, None, e)) + + +def queue_consumer(objects, func, get_deps, results, output): + started = set() # objects being processed + finished = set() # objects which have been processed + failed = set() # objects which either failed or whose dependencies failed + + while len(finished) + len(failed) < len(objects): + pending = set(objects) - started - finished - failed + log.debug('Pending: {}'.format(pending)) + + for obj in pending: + deps = get_deps(obj) + + if any(dep in failed for dep in deps): + log.debug('{} has upstream errors - not processing'.format(obj)) + output.put((obj, None, UpstreamError())) + failed.add(obj) + elif all( + dep not in objects or dep in finished + for dep in deps + ): + log.debug('Starting producer thread for {}'.format(obj)) + t = Thread(target=queue_producer, args=(obj, func, results)) + t.daemon = True + t.start() + started.add(obj) + + try: + event = results.get(timeout=1) + except Empty: + continue + + obj, _, exception = event + if exception is None: + log.debug('Finished processing: {}'.format(obj)) + finished.add(obj) + else: + log.debug('Failed: {}'.format(obj)) + failed.add(obj) + + output.put(event) + + +class UpstreamError(Exception): + pass + + class ParallelStreamWriter(object): """Write out messages for operations happening in parallel. diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py new file mode 100644 index 000000000..6be560152 --- /dev/null +++ b/tests/unit/parallel_test.py @@ -0,0 +1,73 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import six +from docker.errors import APIError + +from compose.parallel import parallel_execute + + +web = 'web' +db = 'db' +data_volume = 'data_volume' +cache = 'cache' + +objects = [web, db, data_volume, cache] + +deps = { + web: [db, cache], + db: [data_volume], + data_volume: [], + cache: [], +} + + +def test_parallel_execute(): + results = parallel_execute( + objects=[1, 2, 3, 4, 5], + func=lambda x: x * 2, + get_name=six.text_type, + msg="Doubling", + ) + + assert sorted(results) == [2, 4, 6, 8, 10] + + +def test_parallel_execute_with_deps(): + log = [] + + def process(x): + log.append(x) + + parallel_execute( + objects=objects, + func=process, + get_name=lambda obj: obj, + msg="Processing", + get_deps=lambda obj: deps[obj], + ) + + assert sorted(log) == sorted(objects) + + assert log.index(data_volume) < log.index(db) + assert log.index(db) < log.index(web) + assert log.index(cache) < log.index(web) + + +def test_parallel_execute_with_upstream_errors(): + log = [] + + def process(x): + if x is data_volume: + raise APIError(None, None, "Something went wrong") + log.append(x) + + parallel_execute( + objects=objects, + func=process, + get_name=lambda obj: obj, + msg="Processing", + get_deps=lambda obj: deps[obj], + ) + + assert log == [cache] From af9526fb820f40a8b7eafb16d29f990b1696f4fe Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 8 Apr 2016 18:30:28 +0100 Subject: [PATCH 23/95] Move queue logic out of parallel_execute() Signed-off-by: Aanand Prasad --- compose/parallel.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index 745d46351..8172d8ead 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -32,22 +32,13 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): for obj in objects: writer.initialize(get_name(obj)) - q = setup_queue(objects, func, get_deps) + events = parallel_execute_stream(objects, func, get_deps) - done = 0 errors = {} results = [] error_to_reraise = None - while done < len(objects): - try: - obj, result, exception = q.get(timeout=1) - except Empty: - continue - # See https://github.com/docker/compose/issues/189 - except thread.error: - raise ShutdownException() - + for obj, result, exception in events: if exception is None: writer.write(get_name(obj), 'done') results.append(result) @@ -59,7 +50,6 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): else: errors[get_name(obj)] = exception error_to_reraise = exception - done += 1 for obj_name, error in errors.items(): stream.write("\nERROR: for {} {}\n".format(obj_name, error)) @@ -74,7 +64,7 @@ def _no_deps(x): return [] -def setup_queue(objects, func, get_deps): +def parallel_execute_stream(objects, func, get_deps): if get_deps is None: get_deps = _no_deps @@ -85,7 +75,17 @@ def setup_queue(objects, func, get_deps): t.daemon = True t.start() - return output + done = 0 + + while done < len(objects): + try: + yield output.get(timeout=1) + done += 1 + except Empty: + continue + # See https://github.com/docker/compose/issues/189 + except thread.error: + raise ShutdownException() def queue_producer(obj, func, results): From 3720b50c3b8c5534c0b139962f7f6d95dd32a066 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 8 Apr 2016 18:48:07 +0100 Subject: [PATCH 24/95] Extract get_deps test helper Signed-off-by: Aanand Prasad --- tests/unit/parallel_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 6be560152..889af4e2f 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -22,6 +22,10 @@ deps = { } +def get_deps(obj): + return deps[obj] + + def test_parallel_execute(): results = parallel_execute( objects=[1, 2, 3, 4, 5], @@ -44,7 +48,7 @@ def test_parallel_execute_with_deps(): func=process, get_name=lambda obj: obj, msg="Processing", - get_deps=lambda obj: deps[obj], + get_deps=get_deps, ) assert sorted(log) == sorted(objects) @@ -67,7 +71,7 @@ def test_parallel_execute_with_upstream_errors(): func=process, get_name=lambda obj: obj, msg="Processing", - get_deps=lambda obj: deps[obj], + get_deps=get_deps, ) assert log == [cache] From ffab27c0496769fade7f2aa32bd86f66a3c9c0e5 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 8 Apr 2016 18:53:16 +0100 Subject: [PATCH 25/95] Test events coming out of parallel_execute_stream in error case Signed-off-by: Aanand Prasad --- tests/unit/parallel_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 889af4e2f..9ed1b3623 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -5,6 +5,8 @@ import six from docker.errors import APIError from compose.parallel import parallel_execute +from compose.parallel import parallel_execute_stream +from compose.parallel import UpstreamError web = 'web' @@ -75,3 +77,14 @@ def test_parallel_execute_with_upstream_errors(): ) assert log == [cache] + + events = [ + (obj, result, type(exception)) + for obj, result, exception + in parallel_execute_stream(objects, process, get_deps) + ] + + assert (cache, None, type(None)) in events + assert (data_volume, None, APIError) in events + assert (db, None, UpstreamError) in events + assert (web, None, UpstreamError) in events From 54b6fc42195da8f7ca1b45828e49ce5e378baee0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 8 Apr 2016 18:54:02 +0100 Subject: [PATCH 26/95] Refactor so there's only one queue Signed-off-by: Aanand Prasad --- compose/parallel.py | 79 +++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index 8172d8ead..b3ca01530 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -69,24 +69,33 @@ def parallel_execute_stream(objects, func, get_deps): get_deps = _no_deps results = Queue() - output = Queue() - t = Thread(target=queue_consumer, args=(objects, func, get_deps, results, output)) - t.daemon = True - t.start() + started = set() # objects being processed + finished = set() # objects which have been processed + failed = set() # objects which either failed or whose dependencies failed - done = 0 + while len(finished) + len(failed) < len(objects): + for event in feed_queue(objects, func, get_deps, results, started, finished, failed): + yield event - while done < len(objects): try: - yield output.get(timeout=1) - done += 1 + event = results.get(timeout=1) except Empty: continue # See https://github.com/docker/compose/issues/189 except thread.error: raise ShutdownException() + obj, _, exception = event + if exception is None: + log.debug('Finished processing: {}'.format(obj)) + finished.add(obj) + else: + log.debug('Failed: {}'.format(obj)) + failed.add(obj) + + yield event + def queue_producer(obj, func, results): try: @@ -96,46 +105,26 @@ def queue_producer(obj, func, results): results.put((obj, None, e)) -def queue_consumer(objects, func, get_deps, results, output): - started = set() # objects being processed - finished = set() # objects which have been processed - failed = set() # objects which either failed or whose dependencies failed +def feed_queue(objects, func, get_deps, results, started, finished, failed): + pending = set(objects) - started - finished - failed + log.debug('Pending: {}'.format(pending)) - while len(finished) + len(failed) < len(objects): - pending = set(objects) - started - finished - failed - log.debug('Pending: {}'.format(pending)) + for obj in pending: + deps = get_deps(obj) - for obj in pending: - deps = get_deps(obj) - - if any(dep in failed for dep in deps): - log.debug('{} has upstream errors - not processing'.format(obj)) - output.put((obj, None, UpstreamError())) - failed.add(obj) - elif all( - dep not in objects or dep in finished - for dep in deps - ): - log.debug('Starting producer thread for {}'.format(obj)) - t = Thread(target=queue_producer, args=(obj, func, results)) - t.daemon = True - t.start() - started.add(obj) - - try: - event = results.get(timeout=1) - except Empty: - continue - - obj, _, exception = event - if exception is None: - log.debug('Finished processing: {}'.format(obj)) - finished.add(obj) - else: - log.debug('Failed: {}'.format(obj)) + if any(dep in failed for dep in deps): + log.debug('{} has upstream errors - not processing'.format(obj)) + yield (obj, None, UpstreamError()) failed.add(obj) - - output.put(event) + elif all( + dep not in objects or dep in finished + for dep in deps + ): + log.debug('Starting producer thread for {}'.format(obj)) + t = Thread(target=queue_producer, args=(obj, func, results)) + t.daemon = True + t.start() + started.add(obj) class UpstreamError(Exception): From 5450a67c2d75192b962c3c36cf73a417af4386b3 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 8 Apr 2016 19:06:07 +0100 Subject: [PATCH 27/95] Hold state in an object Signed-off-by: Aanand Prasad --- compose/parallel.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index b3ca01530..f400b2235 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -64,18 +64,30 @@ def _no_deps(x): return [] +class State(object): + def __init__(self, objects): + self.objects = objects + + self.started = set() # objects being processed + self.finished = set() # objects which have been processed + self.failed = set() # objects which either failed or whose dependencies failed + + def is_done(self): + return len(self.finished) + len(self.failed) >= len(self.objects) + + def pending(self): + return set(self.objects) - self.started - self.finished - self.failed + + def parallel_execute_stream(objects, func, get_deps): if get_deps is None: get_deps = _no_deps results = Queue() + state = State(objects) - started = set() # objects being processed - finished = set() # objects which have been processed - failed = set() # objects which either failed or whose dependencies failed - - while len(finished) + len(failed) < len(objects): - for event in feed_queue(objects, func, get_deps, results, started, finished, failed): + while not state.is_done(): + for event in feed_queue(objects, func, get_deps, results, state): yield event try: @@ -89,10 +101,10 @@ def parallel_execute_stream(objects, func, get_deps): obj, _, exception = event if exception is None: log.debug('Finished processing: {}'.format(obj)) - finished.add(obj) + state.finished.add(obj) else: log.debug('Failed: {}'.format(obj)) - failed.add(obj) + state.failed.add(obj) yield event @@ -105,26 +117,26 @@ def queue_producer(obj, func, results): results.put((obj, None, e)) -def feed_queue(objects, func, get_deps, results, started, finished, failed): - pending = set(objects) - started - finished - failed +def feed_queue(objects, func, get_deps, results, state): + pending = state.pending() log.debug('Pending: {}'.format(pending)) for obj in pending: deps = get_deps(obj) - if any(dep in failed for dep in deps): + if any(dep in state.failed for dep in deps): log.debug('{} has upstream errors - not processing'.format(obj)) yield (obj, None, UpstreamError()) - failed.add(obj) + state.failed.add(obj) elif all( - dep not in objects or dep in finished + dep not in objects or dep in state.finished for dep in deps ): log.debug('Starting producer thread for {}'.format(obj)) t = Thread(target=queue_producer, args=(obj, func, results)) t.daemon = True t.start() - started.add(obj) + state.started.add(obj) class UpstreamError(Exception): From be27e266da9bbb252e74d71ab22044628c6839d2 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 8 Apr 2016 19:07:40 +0100 Subject: [PATCH 28/95] Reduce queue timeout Signed-off-by: Aanand Prasad --- compose/parallel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/parallel.py b/compose/parallel.py index f400b2235..e360ca357 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -91,7 +91,7 @@ def parallel_execute_stream(objects, func, get_deps): yield event try: - event = results.get(timeout=1) + event = results.get(timeout=0.1) except Empty: continue # See https://github.com/docker/compose/issues/189 From 83df95d5118a340fca71ca912825b3e9ba89ff96 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 8 Apr 2016 11:59:06 -0400 Subject: [PATCH 29/95] Remove extra ensure_image_exists() which causes duplicate builds. Signed-off-by: Daniel Nephin --- compose/project.py | 11 +++++------ compose/service.py | 11 ++++------- tests/integration/service_test.py | 6 ++---- tests/unit/service_test.py | 18 +++++++++--------- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/compose/project.py b/compose/project.py index 8aa487319..0d891e455 100644 --- a/compose/project.py +++ b/compose/project.py @@ -309,12 +309,13 @@ class Project(object): ): services = self.get_services_without_duplicate(service_names, include_deps=True) + for svc in services: + svc.ensure_image_exists(do_build=do_build) plans = self._get_convergence_plans(services, strategy) for service in services: service.execute_convergence_plan( plans[service.name], - do_build, detached=True, start=False) @@ -366,21 +367,19 @@ class Project(object): remove_orphans=False): self.initialize() + self.find_orphan_containers(remove_orphans) + services = self.get_services_without_duplicate( service_names, include_deps=start_deps) - plans = self._get_convergence_plans(services, strategy) - for svc in services: svc.ensure_image_exists(do_build=do_build) - - self.find_orphan_containers(remove_orphans) + plans = self._get_convergence_plans(services, strategy) def do(service): return service.execute_convergence_plan( plans[service.name], - do_build=do_build, timeout=timeout, detached=detached ) diff --git a/compose/service.py b/compose/service.py index 05cfc7c61..e0f238882 100644 --- a/compose/service.py +++ b/compose/service.py @@ -254,7 +254,6 @@ class Service(object): def create_container(self, one_off=False, - do_build=BuildAction.none, previous_container=None, number=None, quiet=False, @@ -263,7 +262,9 @@ class Service(object): Create a container for this service. If the image doesn't exist, attempt to pull it. """ - self.ensure_image_exists(do_build=do_build) + # This is only necessary for `scale` and `volumes_from` + # auto-creating containers to satisfy the dependency. + self.ensure_image_exists() container_options = self._get_container_create_options( override_options, @@ -363,7 +364,6 @@ class Service(object): def execute_convergence_plan(self, plan, - do_build=BuildAction.none, timeout=DEFAULT_TIMEOUT, detached=False, start=True): @@ -371,7 +371,7 @@ class Service(object): should_attach_logs = not detached if action == 'create': - container = self.create_container(do_build=do_build) + container = self.create_container() if should_attach_logs: container.attach_log_stream() @@ -385,7 +385,6 @@ class Service(object): return [ self.recreate_container( container, - do_build=do_build, timeout=timeout, attach_logs=should_attach_logs, start_new_container=start @@ -412,7 +411,6 @@ class Service(object): def recreate_container( self, container, - do_build=BuildAction.none, timeout=DEFAULT_TIMEOUT, attach_logs=False, start_new_container=True): @@ -427,7 +425,6 @@ class Service(object): container.stop(timeout=timeout) container.rename_to_tmp_name() new_container = self.create_container( - do_build=do_build, previous_container=container, number=container.labels.get(LABEL_CONTAINER_NUMBER), quiet=True, diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 0a109ada3..df50d513a 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -1037,12 +1037,10 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(set(service.duplicate_containers()), set([duplicate])) -def converge(service, - strategy=ConvergenceStrategy.changed, - do_build=True): +def converge(service, strategy=ConvergenceStrategy.changed): """Create a converge plan from a strategy and execute the plan.""" plan = service.convergence_plan(strategy) - return service.execute_convergence_plan(plan, do_build=do_build, timeout=1) + return service.execute_convergence_plan(plan, timeout=1) class ConfigHashTest(DockerClientTestCase): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 5231237ab..fe3794daf 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -420,7 +420,7 @@ class ServiceTest(unittest.TestCase): parse_repository_tag("url:5000/repo@sha256:digest"), ("url:5000/repo", "sha256:digest", "@")) - def test_create_container_with_build(self): + def test_create_container(self): service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.side_effect = [ NoSuchImageError, @@ -431,7 +431,7 @@ class ServiceTest(unittest.TestCase): ] with mock.patch('compose.service.log', autospec=True) as mock_log: - service.create_container(do_build=BuildAction.none) + service.create_container() assert mock_log.warn.called _, args, _ = mock_log.warn.mock_calls[0] assert 'was built because it did not already exist' in args[0] @@ -448,20 +448,20 @@ class ServiceTest(unittest.TestCase): buildargs=None, ) - def test_create_container_no_build(self): + def test_ensure_image_exists_no_build(self): service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.return_value = {'Id': 'abc123'} - service.create_container(do_build=BuildAction.skip) - self.assertFalse(self.mock_client.build.called) + service.ensure_image_exists(do_build=BuildAction.skip) + assert not self.mock_client.build.called - def test_create_container_no_build_but_needs_build(self): + def test_ensure_image_exists_no_build_but_needs_build(self): service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.side_effect = NoSuchImageError with pytest.raises(NeedsBuildError): - service.create_container(do_build=BuildAction.skip) + service.ensure_image_exists(do_build=BuildAction.skip) - def test_create_container_force_build(self): + def test_ensure_image_exists_force_build(self): service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.return_value = {'Id': 'abc123'} self.mock_client.build.return_value = [ @@ -469,7 +469,7 @@ class ServiceTest(unittest.TestCase): ] with mock.patch('compose.service.log', autospec=True) as mock_log: - service.create_container(do_build=BuildAction.force) + service.ensure_image_exists(do_build=BuildAction.force) assert not mock_log.warn.called self.mock_client.build.assert_called_once_with( From d4e9a3b6b144d2dd126dee2369c10284ec52cdbc Mon Sep 17 00:00:00 2001 From: Sanyam Kapoor Date: Wed, 6 Apr 2016 23:05:40 +0530 Subject: [PATCH 30/95] Updated Wordpress tutorial The new tutorial now uses official Wordpress Docker Image. Signed-off-by: Sanyam Kapoor <1sanyamkapoor@gmail.com> --- docs/wordpress.md | 115 +++++++++++----------------------------------- 1 file changed, 26 insertions(+), 89 deletions(-) diff --git a/docs/wordpress.md b/docs/wordpress.md index fcfaef191..c257ad1a1 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -22,7 +22,7 @@ with Docker containers. This quick-start guide demonstrates how to use Compose t You can name the directory something easy for you to remember. This directory is the context for your application image. The directory should only contain resources to build that image. - This project directory will contain a `Dockerfile`, a `docker-compose.yaml` file, along with a downloaded `wordpress` directory and a custom `wp-config.php`, all of which you will create in the following steps. + This project directory will contain a `docker-compose.yaml` file which will be complete in itself for a good starter wordpress project. 2. Change directories into your project directory. @@ -30,113 +30,50 @@ with Docker containers. This quick-start guide demonstrates how to use Compose t $ cd my-wordpress/ -3. Create a `Dockerfile`, a file that defines the environment in which your application will run. - - For more information on how to write Dockerfiles, see the [Docker Engine user guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](https://docs.docker.com/engine/reference/builder/). - - In this case, your Dockerfile should include these two lines: - - FROM php:5.6-fpm - RUN docker-php-ext-install mysql - ADD . /code - CMD php -S 0.0.0.0:8000 -t /code/wordpress/ - - This tells the Docker Engine daemon how to build an image defining a container that contains PHP and WordPress. - -4. Create a `docker-compose.yml` file that will start your web service and a separate MySQL instance: +3. Create a `docker-compose.yml` file that will start your `Wordpress` blog and a separate `MySQL` instance with a volume mount for data persistence: version: '2' services: - web: - build: . - ports: - - "8000:8000" - depends_on: - - db - volumes: - - .:/code db: - image: mysql + image: mysql:5.7 + volumes: + - "./.data/db:/var/lib/mysql" + restart: always environment: MYSQL_ROOT_PASSWORD: wordpress MYSQL_DATABASE: wordpress MYSQL_USER: wordpress MYSQL_PASSWORD: wordpress -5. Download WordPress into the current directory: + wordpress: + depends_on: + - db + image: wordpress:latest + links: + - db + ports: + - "8000:80" + restart: always + environment: + WORDPRESS_DB_HOST: db:3306 + WORDPRESS_DB_PASSWORD: wordpress - $ curl https://wordpress.org/latest.tar.gz | tar -xvzf - - - This creates a directory called `wordpress` in your project directory. - -6. Create a `wp-config.php` file within the `wordpress` directory. - - A supporting file is needed to get this working. At the top level of the wordpress directory, add a new file called `wp-config.php` as shown. This is the standard WordPress config file with a single change to point the database configuration at the `db` container: - - - -7. Verify the contents and structure of your project directory. - - - ![WordPress files](images/wordpress-files.png) + **NOTE**: The folder `./.data/db` will be automatically created in the project directory + alongside the `docker-compose.yml` which will persist any updates made by wordpress to the + database. ### Build the project -With those four new files in place, run `docker-compose up` from your project directory. This will pull and build the needed images, and then start the web and database containers. +Now, run `docker-compose up -d` from your project directory. This will pull the needed images, and then start the wordpress and database containers. If you're using [Docker Machine](https://docs.docker.com/machine/), then `docker-machine ip MACHINE_VM` gives you the machine address and you can open `http://MACHINE_VM_IP:8000` in a browser. At this point, WordPress should be running on port `8000` of your Docker Host, and you can complete the "famous five-minute installation" as a WordPress administrator. +**NOTE**: The Wordpress site will not be immediately available on port `8000` because +the containers are still being initialized and may take a couple of minutes before the +first load. + ![Choose language for WordPress install](images/wordpress-lang.png) ![WordPress Welcome](images/wordpress-welcome.png) From 4192a009da5cbae5c811b3b965e4ecb4572c95f6 Mon Sep 17 00:00:00 2001 From: Victoria Bialas Date: Fri, 8 Apr 2016 16:40:07 -0700 Subject: [PATCH 31/95] added some formatting on the Wordress steps, and made heading levels in these sample app topics consistent Signed-off-by: Victoria Bialas --- docs/django.md | 6 +++--- docs/wordpress.md | 32 +++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/docs/django.md b/docs/django.md index fb1fa2141..6a222697e 100644 --- a/docs/django.md +++ b/docs/django.md @@ -15,7 +15,7 @@ weight=4 This quick-start guide demonstrates how to use Docker Compose to set up and run a simple Django/PostgreSQL app. Before starting, you'll need to have [Compose installed](install.md). -## Define the project components +### Define the project components For this project, you need to create a Dockerfile, a Python dependencies file, and a `docker-compose.yml` file. @@ -89,7 +89,7 @@ and a `docker-compose.yml` file. 10. Save and close the `docker-compose.yml` file. -## Create a Django project +### Create a Django project In this step, you create a Django started project by building the image from the build context defined in the previous procedure. @@ -137,7 +137,7 @@ In this step, you create a Django started project by building the image from the -rw-r--r-- 1 user staff 16 Feb 13 23:01 requirements.txt -## Connect the database +### Connect the database In this section, you set up the database connection for Django. diff --git a/docs/wordpress.md b/docs/wordpress.md index c257ad1a1..b39a8bbbe 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -16,7 +16,7 @@ You can use Docker Compose to easily run WordPress in an isolated environment bu with Docker containers. This quick-start guide demonstrates how to use Compose to set up and run WordPress. Before starting, you'll need to have [Compose installed](install.md). -## Define the project +### Define the project 1. Create an empty project directory. @@ -64,15 +64,37 @@ with Docker containers. This quick-start guide demonstrates how to use Compose t ### Build the project -Now, run `docker-compose up -d` from your project directory. This will pull the needed images, and then start the wordpress and database containers. +Now, run `docker-compose up -d` from your project directory. + +This pulls the needed images, and starts the wordpress and database containers, as shown in the example below. + + $ docker-compose up -d + Creating network "my_wordpress_default" with the default driver + Pulling db (mysql:5.7)... + 5.7: Pulling from library/mysql + efd26ecc9548: Pull complete + a3ed95caeb02: Pull complete + ... + Digest: sha256:34a0aca88e85f2efa5edff1cea77cf5d3147ad93545dbec99cfe705b03c520de + Status: Downloaded newer image for mysql:5.7 + Pulling wordpress (wordpress:latest)... + latest: Pulling from library/wordpress + efd26ecc9548: Already exists + a3ed95caeb02: Pull complete + 589a9d9a7c64: Pull complete + ... + Digest: sha256:ed28506ae44d5def89075fd5c01456610cd6c64006addfe5210b8c675881aff6 + Status: Downloaded newer image for wordpress:latest + Creating my_wordpress_db_1 + Creating my_wordpress_wordpress_1 + +### Bring up WordPress in a web browser If you're using [Docker Machine](https://docs.docker.com/machine/), then `docker-machine ip MACHINE_VM` gives you the machine address and you can open `http://MACHINE_VM_IP:8000` in a browser. At this point, WordPress should be running on port `8000` of your Docker Host, and you can complete the "famous five-minute installation" as a WordPress administrator. -**NOTE**: The Wordpress site will not be immediately available on port `8000` because -the containers are still being initialized and may take a couple of minutes before the -first load. +**NOTE**: The Wordpress site will not be immediately available on port `8000` because the containers are still being initialized and may take a couple of minutes before the first load. ![Choose language for WordPress install](images/wordpress-lang.png) From 0e3db185cf79e6638c2660be8e052af113ed7337 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 11 Apr 2016 12:37:00 +0100 Subject: [PATCH 32/95] Small refactor to feed_queue() Put the event tuple into the results queue rather than yielding it from the function. Signed-off-by: Aanand Prasad --- compose/parallel.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index e360ca357..ace1f029c 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -87,8 +87,7 @@ def parallel_execute_stream(objects, func, get_deps): state = State(objects) while not state.is_done(): - for event in feed_queue(objects, func, get_deps, results, state): - yield event + feed_queue(objects, func, get_deps, results, state) try: event = results.get(timeout=0.1) @@ -126,7 +125,7 @@ def feed_queue(objects, func, get_deps, results, state): if any(dep in state.failed for dep in deps): log.debug('{} has upstream errors - not processing'.format(obj)) - yield (obj, None, UpstreamError()) + results.put((obj, None, UpstreamError())) state.failed.add(obj) elif all( dep not in objects or dep in state.finished From 0671b8b8c3ce1873db87c4233f88e64876d43c6a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 11 Apr 2016 12:49:04 +0100 Subject: [PATCH 33/95] Document parallel helper functions Signed-off-by: Aanand Prasad --- compose/parallel.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index ace1f029c..d9c24ab66 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -65,12 +65,19 @@ def _no_deps(x): class State(object): + """ + Holds the state of a partially-complete parallel operation. + + state.started: objects being processed + state.finished: objects which have been processed + state.failed: objects which either failed or whose dependencies failed + """ def __init__(self, objects): self.objects = objects - self.started = set() # objects being processed - self.finished = set() # objects which have been processed - self.failed = set() # objects which either failed or whose dependencies failed + self.started = set() + self.finished = set() + self.failed = set() def is_done(self): return len(self.finished) + len(self.failed) >= len(self.objects) @@ -80,6 +87,21 @@ class State(object): def parallel_execute_stream(objects, func, get_deps): + """ + Runs func on objects in parallel while ensuring that func is + ran on object only after it is ran on all its dependencies. + + Returns an iterator of tuples which look like: + + # if func returned normally when run on object + (object, result, None) + + # if func raised an exception when run on object + (object, None, exception) + + # if func raised an exception when run on one of object's dependencies + (object, None, UpstreamError()) + """ if get_deps is None: get_deps = _no_deps @@ -109,6 +131,10 @@ def parallel_execute_stream(objects, func, get_deps): def queue_producer(obj, func, results): + """ + The entry point for a producer thread which runs func on a single object. + Places a tuple on the results queue once func has either returned or raised. + """ try: result = func(obj) results.put((obj, result, None)) @@ -117,6 +143,13 @@ def queue_producer(obj, func, results): def feed_queue(objects, func, get_deps, results, state): + """ + Starts producer threads for any objects which are ready to be processed + (i.e. they have no dependencies which haven't been successfully processed). + + Shortcuts any objects whose dependencies have failed and places an + (object, None, UpstreamError()) tuple on the results queue. + """ pending = state.pending() log.debug('Pending: {}'.format(pending)) From 15c5bc2e6c79cdb2edac4f8cab10d7bcbfc175d1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 11 Apr 2016 13:03:35 +0100 Subject: [PATCH 34/95] Rename a couple of functions in parallel.py Signed-off-by: Aanand Prasad --- compose/parallel.py | 8 ++++---- tests/unit/parallel_test.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index d9c24ab66..ee3d5777b 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -32,7 +32,7 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): for obj in objects: writer.initialize(get_name(obj)) - events = parallel_execute_stream(objects, func, get_deps) + events = parallel_execute_iter(objects, func, get_deps) errors = {} results = [] @@ -86,7 +86,7 @@ class State(object): return set(self.objects) - self.started - self.finished - self.failed -def parallel_execute_stream(objects, func, get_deps): +def parallel_execute_iter(objects, func, get_deps): """ Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. @@ -130,7 +130,7 @@ def parallel_execute_stream(objects, func, get_deps): yield event -def queue_producer(obj, func, results): +def producer(obj, func, results): """ The entry point for a producer thread which runs func on a single object. Places a tuple on the results queue once func has either returned or raised. @@ -165,7 +165,7 @@ def feed_queue(objects, func, get_deps, results, state): for dep in deps ): log.debug('Starting producer thread for {}'.format(obj)) - t = Thread(target=queue_producer, args=(obj, func, results)) + t = Thread(target=producer, args=(obj, func, results)) t.daemon = True t.start() state.started.add(obj) diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 9ed1b3623..45b0db1db 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -5,7 +5,7 @@ import six from docker.errors import APIError from compose.parallel import parallel_execute -from compose.parallel import parallel_execute_stream +from compose.parallel import parallel_execute_iter from compose.parallel import UpstreamError @@ -81,7 +81,7 @@ def test_parallel_execute_with_upstream_errors(): events = [ (obj, result, type(exception)) for obj, result, exception - in parallel_execute_stream(objects, process, get_deps) + in parallel_execute_iter(objects, process, get_deps) ] assert (cache, None, type(None)) in events From 3722bb38c66b3c3500e86295a43aafe14a050b50 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 11 Apr 2016 14:26:45 +0100 Subject: [PATCH 35/95] Clarify behaviour of rm and down Signed-off-by: Aanand Prasad --- compose/cli/main.py | 35 +++++++++++++++++++++++------------ docs/reference/down.md | 26 ++++++++++++++++++-------- docs/reference/rm.md | 11 ++++++----- 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 8348b8c37..839d97e8a 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -264,18 +264,29 @@ class TopLevelCommand(object): def down(self, options): """ - Stop containers and remove containers, networks, volumes, and images - created by `up`. Only containers and networks are removed by default. + Stops containers and removes containers, networks, volumes, and images + created by `up`. + + By default, the only things removed are: + + - Containers for services defined in the Compose file + - Networks defined in the `networks` section of the Compose file + - The default network, if one is used + + Networks and volumes defined as `external` are never removed. Usage: down [options] Options: - --rmi type Remove images, type may be one of: 'all' to remove - all images, or 'local' to remove only images that - don't have an custom name set by the `image` field - -v, --volumes Remove data volumes - --remove-orphans Remove containers for services not defined in - the Compose file + --rmi type Remove images. Type must be one of: + 'all': Remove all images used by any service. + 'local': Remove only images that don't have a custom tag + set by the `image` field. + -v, --volumes Remove named volumes declared in the `volumes` section + of the Compose file and anonymous volumes + attached to containers. + --remove-orphans Remove containers for services not defined in the + Compose file """ image_type = image_type_from_opt('--rmi', options['--rmi']) self.project.down(image_type, options['--volumes'], options['--remove-orphans']) @@ -496,10 +507,10 @@ class TopLevelCommand(object): def rm(self, options): """ - Remove stopped service containers. + Removes stopped service containers. - By default, volumes attached to containers will not be removed. You can see all - volumes with `docker volume ls`. + By default, anonymous volumes attached to containers will not be removed. You + can override this with `-v`. To list all volumes, use `docker volume ls`. Any data which is not in a volume will be lost. @@ -507,7 +518,7 @@ class TopLevelCommand(object): Options: -f, --force Don't ask to confirm removal - -v Remove volumes associated with containers + -v Remove any anonymous volumes attached to containers -a, --all Also remove one-off containers created by docker-compose run """ diff --git a/docs/reference/down.md b/docs/reference/down.md index e8b1db597..ffe88b4e0 100644 --- a/docs/reference/down.md +++ b/docs/reference/down.md @@ -12,17 +12,27 @@ parent = "smn_compose_cli" # down ``` -Stop containers and remove containers, networks, volumes, and images -created by `up`. Only containers and networks are removed by default. - Usage: down [options] Options: - --rmi type Remove images, type may be one of: 'all' to remove - all images, or 'local' to remove only images that - don't have an custom name set by the `image` field - -v, --volumes Remove data volumes - + --rmi type Remove images. Type must be one of: + 'all': Remove all images used by any service. + 'local': Remove only images that don't have a custom tag + set by the `image` field. + -v, --volumes Remove named volumes declared in the `volumes` section + of the Compose file and anonymous volumes + attached to containers. --remove-orphans Remove containers for services not defined in the Compose file ``` + +Stops containers and removes containers, networks, volumes, and images +created by `up`. + +By default, the only things removed are: + +- Containers for services defined in the Compose file +- Networks defined in the `networks` section of the Compose file +- The default network, if one is used + +Networks and volumes defined as `external` are never removed. diff --git a/docs/reference/rm.md b/docs/reference/rm.md index 97698b58b..8285a4ae5 100644 --- a/docs/reference/rm.md +++ b/docs/reference/rm.md @@ -15,14 +15,15 @@ parent = "smn_compose_cli" Usage: rm [options] [SERVICE...] Options: --f, --force Don't ask to confirm removal --v Remove volumes associated with containers --a, --all Also remove one-off containers + -f, --force Don't ask to confirm removal + -v Remove any anonymous volumes attached to containers + -a, --all Also remove one-off containers created by + docker-compose run ``` Removes stopped service containers. -By default, volumes attached to containers will not be removed. You can see all -volumes with `docker volume ls`. +By default, anonymous volumes attached to containers will not be removed. You +can override this with `-v`. To list all volumes, use `docker volume ls`. Any data which is not in a volume will be lost. From 7cfb5e7bc9fb93549de0915f378d6cd831835d52 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 11 Apr 2016 17:05:52 +0100 Subject: [PATCH 36/95] Fix race condition If processing of all objects finishes before the queue is drained, parallel_execute_iter() returns prematurely. Signed-off-by: Aanand Prasad --- compose/parallel.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/compose/parallel.py b/compose/parallel.py index ee3d5777b..63417dcb0 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -17,6 +17,8 @@ from compose.utils import get_output_stream log = logging.getLogger(__name__) +STOP = object() + def parallel_execute(objects, func, get_name, msg, get_deps=None): """Runs func on objects in parallel while ensuring that func is @@ -108,7 +110,7 @@ def parallel_execute_iter(objects, func, get_deps): results = Queue() state = State(objects) - while not state.is_done(): + while True: feed_queue(objects, func, get_deps, results, state) try: @@ -119,6 +121,9 @@ def parallel_execute_iter(objects, func, get_deps): except thread.error: raise ShutdownException() + if event is STOP: + break + obj, _, exception = event if exception is None: log.debug('Finished processing: {}'.format(obj)) @@ -170,6 +175,9 @@ def feed_queue(objects, func, get_deps, results, state): t.start() state.started.add(obj) + if state.is_done(): + results.put(STOP) + class UpstreamError(Exception): pass From 7781f62ddf54fa635890c1772e1729ff5461fd55 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 Apr 2016 12:03:16 +0100 Subject: [PATCH 37/95] Attempt to fix flaky logs test Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 13 +++++++------ tests/fixtures/logs-composefile/docker-compose.yml | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 707c24926..53ff66bbb 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1257,13 +1257,14 @@ class CLITestCase(DockerClientTestCase): 'logscomposefile_another_1', 'exited')) - # sleep for a short period to allow the tailing thread to receive the - # event. This is not great, but there isn't an easy way to do this - # without being able to stream stdout from the process. - time.sleep(0.5) - os.kill(proc.pid, signal.SIGINT) - result = wait_on_process(proc, returncode=1) + self.dispatch(['kill', 'simple']) + + result = wait_on_process(proc) + + assert 'hello' in result.stdout assert 'test' in result.stdout + assert 'logscomposefile_another_1 exited with code 0' in result.stdout + assert 'logscomposefile_simple_1 exited with code 137' in result.stdout def test_logs_default(self): self.base_dir = 'tests/fixtures/logs-composefile' diff --git a/tests/fixtures/logs-composefile/docker-compose.yml b/tests/fixtures/logs-composefile/docker-compose.yml index 0af9d805c..b719c91e0 100644 --- a/tests/fixtures/logs-composefile/docker-compose.yml +++ b/tests/fixtures/logs-composefile/docker-compose.yml @@ -1,6 +1,6 @@ simple: image: busybox:latest - command: sh -c "echo hello && sleep 200" + command: sh -c "echo hello && tail -f /dev/null" another: image: busybox:latest command: sh -c "echo test" From 276738f733c3512b939168c1475a6085a9482c6a Mon Sep 17 00:00:00 2001 From: Tony Witherspoon Date: Tue, 12 Apr 2016 11:47:15 -0400 Subject: [PATCH 38/95] Updated cli_test.py to validate against the updated help command conditions Signed-off-by: Tony Witherspoon --- tests/unit/cli_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 182e79ed5..9700d5927 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -5,6 +5,7 @@ from __future__ import unicode_literals import os import shutil import tempfile +from StringIO import StringIO import docker import py @@ -82,6 +83,12 @@ class CLITestCase(unittest.TestCase): self.assertTrue(project.client) self.assertTrue(project.services) + def test_command_help(self): + with mock.patch('sys.stdout', new=StringIO()) as fake_stdout: + TopLevelCommand.help({'COMMAND': 'up'}) + + assert "Usage: up" in fake_stdout.getvalue() + def test_command_help_nonexistent(self): with pytest.raises(NoSuchCommand): TopLevelCommand.help({'COMMAND': 'nonexistent'}) From ae46bf8907aec818a07167598efef26a778dadaa Mon Sep 17 00:00:00 2001 From: Tony Witherspoon Date: Tue, 12 Apr 2016 12:29:59 -0400 Subject: [PATCH 39/95] Updated StringIO import to support io module Signed-off-by: Tony Witherspoon --- tests/unit/cli_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 9700d5927..2c90b29b7 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals import os import shutil import tempfile -from StringIO import StringIO +from io import StringIO import docker import py From 339ebc0483cfc2ec72efba884c0de84088c2f905 Mon Sep 17 00:00:00 2001 From: Danyal Prout Date: Sun, 10 Apr 2016 15:53:42 +0100 Subject: [PATCH 40/95] Fixes #2096: Only show multiple port clash warning if multiple containers are about to be started. Signed-off-by: Danyal Prout --- compose/service.py | 2 +- tests/unit/service_test.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index e0f238882..054082fc9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -179,7 +179,7 @@ class Service(object): 'Remove the custom name to scale the service.' % (self.name, self.custom_container_name)) - if self.specifies_host_port(): + if self.specifies_host_port() and desired_num > 1: log.warn('The "%s" service specifies a port on the host. If multiple containers ' 'for this service are created on a single host, the port will clash.' % self.name) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index fe3794daf..d3fcb49a7 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -642,6 +642,26 @@ class ServiceTest(unittest.TestCase): service = Service('foo', project='testing') assert service.image_name == 'testing_foo' + @mock.patch('compose.service.log', autospec=True) + def test_only_log_warning_when_host_ports_clash(self, mock_log): + self.mock_client.inspect_image.return_value = {'Id': 'abcd'} + name = 'foo' + service = Service( + name, + client=self.mock_client, + ports=["8080:80"]) + + service.scale(0) + self.assertFalse(mock_log.warn.called) + + service.scale(1) + self.assertFalse(mock_log.warn.called) + + service.scale(2) + mock_log.warn.assert_called_once_with( + 'The "{}" service specifies a port on the host. If multiple containers ' + 'for this service are created on a single host, the port will clash.'.format(name)) + def sort_by_name(dictionary_list): return sorted(dictionary_list, key=lambda k: k['name']) From 50287722f2dd9df322122395e76e7778e185cdec Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 13 Apr 2016 12:57:22 -0400 Subject: [PATCH 41/95] Update release notes and set version to 1.8.0dev Signed-off-by: Daniel Nephin --- CHANGELOG.md | 88 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b93087f0..8ee45386a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,94 @@ Change log ========== +1.7.0 (2016-04-13) +------------------ + +**Breaking Changes** + +- `docker-compose logs` no longer follows log output by default. It now + matches the behaviour of `docker logs` and exits after the current logs + are printed. Use `-f` to get the old default behaviour. + +- Booleans are no longer allows as values for mappings in the Compose file + (for keys `environment`, `labels` and `extra_hosts`). Previously this + was a warning. Boolean values should be quoted so they become string values. + +New Features + +- Compose now looks for a `.env` file in the directory where it's run and + reads any environment variables defined inside, if they're not already + set in the shell environment. This lets you easily set defaults for + variables used in the Compose file, or for any of the `COMPOSE_*` or + `DOCKER_*` variables. + +- Added a `--remove-orphans` flag to both `docker-compose up` and + `docker-compose down` to remove containers for services that were removed + from the Compose file. + +- Added a `--all` flag to `docker-compose rm` to include containers created + by `docker-compose run`. This will become the default behavior in the next + version of Compose. + +- Added support for all the same TLS configuration flags used by the `docker` + client: `--tls`, `--tlscert`, `--tlskey`, etc. + +- Compose files now support the `tmpfs` and `shm_size` options. + +- Added the `--workdir` flag to `docker-compose run` + +- `docker-compose logs` now shows logs for new containers that are created + after it starts. + +- The `COMPOSE_FILE` environment variable can now contain multiple files, + separated by the host system's standard path separator (`:` on Mac/Linux, + `;` on Windows). + +- You can now specify a static IP address when connecting a service to a + network with the `ipv4_address` and `ipv6_address` options. + +- Added `--follow`, `--timestamp`, and `--tail` flags to the + `docker-compose logs` command. + +- `docker-compose up`, and `docker-compose start` will now start containers + in parallel where possible. + +- `docker-compose stop` now stops containers in reverse dependency order + instead of all at once. + +- Added the `--build` flag to `docker-compose up` to force it to build a new + image. It now shows a warning if an image is automatically built when the + flag is not used. + +- Added the `docker-compose exec` command for executing a process in a running + container. + + +Bug Fixes + +- `docker-compose down` now removes containers created by + `docker-compose run`. + +- A more appropriate error is shown when a timeout is hit during `up` when + using a tty. + +- Fixed a bug in `docker-compose down` where it would abort if some resources + had already been removed. + +- Fixed a bug where changes to network aliases would not trigger a service + to be recreated. + +- Fix a bug where a log message was printed about creating a new volume + when it already existed. + +- Fixed a bug where interrupting `up` would not always shut down containers. + +- Fixed a bug where `log_opt` and `log_driver` were not properly carried over + when extending services in the v1 Compose file format. + +- Fixed a bug where empty values for build args would cause file validation + to fail. + 1.6.2 (2016-02-23) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index fedc90ff8..1052c0670 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.7.0dev' +__version__ = '1.8.0dev' diff --git a/script/run/run.sh b/script/run/run.sh index 212f9b977..98d32c5f8 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.6.2" +VERSION="1.7.0" IMAGE="docker/compose:$VERSION" From e71c62b8d1ce9202b3df6f156528c403e60efafe Mon Sep 17 00:00:00 2001 From: Callum Rogers Date: Thu, 14 Apr 2016 10:49:10 +0100 Subject: [PATCH 42/95] Readme should use new docker compose format instead of the old one Signed-off-by: Callum Rogers --- README.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f88221519..93550f5ac 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,17 @@ they can be run together in an isolated environment: A `docker-compose.yml` looks like this: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - links: - - redis - redis: - image: redis + version: '2' + + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + redis: + image: redis For more information about the Compose file, see the [Compose file reference](https://github.com/docker/compose/blob/release/docs/compose-file.md) From abb5ae7fe4e3693b6099e52d43cf39e57c8e3e42 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 8 Apr 2016 15:45:03 -0400 Subject: [PATCH 43/95] Only disconnect if we don't already have the short id alias. Signed-off-by: Daniel Nephin --- compose/service.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/compose/service.py b/compose/service.py index 054082fc9..49eee1041 100644 --- a/compose/service.py +++ b/compose/service.py @@ -453,20 +453,21 @@ class Service(object): connected_networks = container.get('NetworkSettings.Networks') for network, netdefs in self.networks.items(): - aliases = netdefs.get('aliases', []) - ipv4_address = netdefs.get('ipv4_address', None) - ipv6_address = netdefs.get('ipv6_address', None) if network in connected_networks: - self.client.disconnect_container_from_network( - container.id, network) + if short_id_alias_exists(container, network): + continue + self.client.disconnect_container_from_network( + container.id, + network) + + aliases = netdefs.get('aliases', []) self.client.connect_container_to_network( container.id, network, aliases=list(self._get_aliases(container).union(aliases)), - ipv4_address=ipv4_address, - ipv6_address=ipv6_address, - links=self._get_links(False) - ) + ipv4_address=netdefs.get('ipv4_address', None), + ipv6_address=netdefs.get('ipv6_address', None), + links=self._get_links(False)) def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT): for c in self.duplicate_containers(): @@ -796,6 +797,12 @@ class Service(object): log.error(six.text_type(e)) +def short_id_alias_exists(container, network): + aliases = container.get( + 'NetworkSettings.Networks.{net}.Aliases'.format(net=network)) or () + return container.short_id in aliases + + class NetworkMode(object): """A `standard` network mode (ex: host, bridge)""" From e1356e1f6f6240a935c37617f787bded136a2049 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 11 Apr 2016 13:22:37 -0400 Subject: [PATCH 44/95] Set networking_config when creating a container. Signed-off-by: Daniel Nephin --- compose/service.py | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index 49eee1041..e6ea92331 100644 --- a/compose/service.py +++ b/compose/service.py @@ -461,10 +461,9 @@ class Service(object): container.id, network) - aliases = netdefs.get('aliases', []) self.client.connect_container_to_network( container.id, network, - aliases=list(self._get_aliases(container).union(aliases)), + aliases=self._get_aliases(netdefs, container), ipv4_address=netdefs.get('ipv4_address', None), ipv6_address=netdefs.get('ipv6_address', None), links=self._get_links(False)) @@ -534,11 +533,32 @@ class Service(object): numbers = [c.number for c in containers] return 1 if not numbers else max(numbers) + 1 - def _get_aliases(self, container): - if container.labels.get(LABEL_ONE_OFF) == "True": + def _get_aliases(self, network, container=None): + if container and container.labels.get(LABEL_ONE_OFF) == "True": return set() - return {self.name, container.short_id} + return list( + {self.name} | + ({container.short_id} if container else set()) | + set(network.get('aliases', ())) + ) + + def build_default_networking_config(self): + if not self.networks: + return {} + + network = self.networks[self.network_mode.id] + endpoint = { + 'Aliases': self._get_aliases(network), + 'IPAMConfig': {}, + } + + if network.get('ipv4_address'): + endpoint['IPAMConfig']['IPv4Address'] = network.get('ipv4_address') + if network.get('ipv6_address'): + endpoint['IPAMConfig']['IPv6Address'] = network.get('ipv6_address') + + return {"EndpointsConfig": {self.network_mode.id: endpoint}} def _get_links(self, link_to_self): links = {} @@ -634,6 +654,10 @@ class Service(object): override_options, one_off=one_off) + networking_config = self.build_default_networking_config() + if networking_config: + container_options['networking_config'] = networking_config + container_options['environment'] = format_environment( container_options['environment']) return container_options From ad306f047969a24ab9fd2d0cf1bdc5ccd01d1bc1 Mon Sep 17 00:00:00 2001 From: johnharris85 Date: Fri, 15 Apr 2016 13:30:13 +0100 Subject: [PATCH 45/95] Fix CLI docstring to reflect Docopt behaviour. Signed-off-by: John Harris --- compose/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 839d97e8a..29d808ce3 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -142,7 +142,7 @@ class TopLevelCommand(object): """Define and run multi-container applications with Docker. Usage: - docker-compose [-f=...] [options] [COMMAND] [ARGS...] + docker-compose [-f ...] [options] [COMMAND] [ARGS...] docker-compose -h|--help Options: From 4702703615ad1876b7f20e577dbb9cde59d1e329 Mon Sep 17 00:00:00 2001 From: Vladimir Lagunov Date: Fri, 15 Apr 2016 15:11:50 +0300 Subject: [PATCH 46/95] Fix #3248: Accidental config_hash change Signed-off-by: Vladimir Lagunov --- compose/config/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index dc3f56ea9..bd6e54fa2 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -726,7 +726,7 @@ class MergeDict(dict): merged = parse_sequence_func(self.base.get(field, [])) merged.update(parse_sequence_func(self.override.get(field, []))) - self[field] = [item.repr() for item in merged.values()] + self[field] = [item.repr() for item in sorted(merged.values())] def merge_scalar(self, field): if self.needs_merge(field): @@ -928,7 +928,7 @@ def dict_from_path_mappings(path_mappings): def path_mappings_from_dict(d): - return [join_path_mapping(v) for v in d.items()] + return [join_path_mapping(v) for v in sorted(d.items())] def split_path_mapping(volume_path): From 56c6e298199552f630432d6fefd770e35e5d7562 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 15 Apr 2016 15:42:36 -0400 Subject: [PATCH 47/95] Unit test for skipping network disconnect. Signed-off-by: Daniel Nephin --- compose/service.py | 2 +- tests/integration/project_test.py | 20 ++++++++++++++++---- tests/unit/service_test.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index e6ea92331..8b9f64f0f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -535,7 +535,7 @@ class Service(object): def _get_aliases(self, network, container=None): if container and container.labels.get(LABEL_ONE_OFF) == "True": - return set() + return [] return list( {self.name} | diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index d1732d1e4..c413b9aa0 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -565,7 +565,11 @@ class ProjectTest(DockerClientTestCase): 'name': 'web', 'image': 'busybox:latest', 'command': 'top', - 'networks': {'foo': None, 'bar': None, 'baz': None}, + 'networks': { + 'foo': None, + 'bar': None, + 'baz': {'aliases': ['extra']}, + }, }], volumes={}, networks={ @@ -581,15 +585,23 @@ class ProjectTest(DockerClientTestCase): config_data=config_data, ) project.up() - self.assertEqual(len(project.containers()), 1) + + containers = project.containers() + assert len(containers) == 1 + container, = containers for net_name in ['foo', 'bar', 'baz']: full_net_name = 'composetest_{}'.format(net_name) network_data = self.client.inspect_network(full_net_name) - self.assertEqual(network_data['Name'], full_net_name) + assert network_data['Name'] == full_net_name + + aliases_key = 'NetworkSettings.Networks.{net}.Aliases' + assert 'web' in container.get(aliases_key.format(net='composetest_foo')) + assert 'web' in container.get(aliases_key.format(net='composetest_baz')) + assert 'extra' in container.get(aliases_key.format(net='composetest_baz')) foo_data = self.client.inspect_network('composetest_foo') - self.assertEqual(foo_data['Driver'], 'bridge') + assert foo_data['Driver'] == 'bridge' @v2_only() def test_up_with_ipam_config(self): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index d3fcb49a7..a259c476f 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -663,6 +663,35 @@ class ServiceTest(unittest.TestCase): 'for this service are created on a single host, the port will clash.'.format(name)) +class TestServiceNetwork(object): + + def test_connect_container_to_networks_short_aliase_exists(self): + mock_client = mock.create_autospec(docker.Client) + service = Service( + 'db', + mock_client, + 'myproject', + image='foo', + networks={'project_default': {}}) + container = Container( + None, + { + 'Id': 'abcdef', + 'NetworkSettings': { + 'Networks': { + 'project_default': { + 'Aliases': ['analias', 'abcdef'], + }, + }, + }, + }, + True) + service.connect_container_to_networks(container) + + assert not mock_client.disconnect_container_from_network.call_count + assert not mock_client.connect_container_to_network.call_count + + def sort_by_name(dictionary_list): return sorted(dictionary_list, key=lambda k: k['name']) From 68272b021639490929b0cdcca970ebd902ff5f09 Mon Sep 17 00:00:00 2001 From: johnharris85 Date: Sun, 17 Apr 2016 14:00:07 -0400 Subject: [PATCH 48/95] Config now catches undefined service links Fixes issue #2922 Signed-off-by: John Harris --- compose/config/config.py | 2 ++ compose/config/validation.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index dc3f56ea9..3f76277cb 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -37,6 +37,7 @@ from .validation import validate_against_config_schema from .validation import validate_config_section from .validation import validate_depends_on from .validation import validate_extends_file_path +from .validation import validate_links from .validation import validate_network_mode from .validation import validate_service_constraints from .validation import validate_top_level_object @@ -580,6 +581,7 @@ def validate_service(service_config, service_names, version): validate_ulimits(service_config) validate_network_mode(service_config, service_names) validate_depends_on(service_config, service_names) + validate_links(service_config, service_names) if not service_dict.get('image') and has_uppercase(service_name): raise ConfigurationError( diff --git a/compose/config/validation.py b/compose/config/validation.py index 088bec3fc..e4b3a2530 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -171,6 +171,14 @@ def validate_network_mode(service_config, service_names): "is undefined.".format(s=service_config, dep=dependency)) +def validate_links(service_config, service_names): + for dependency in service_config.config.get('links', []): + if dependency not in service_names: + raise ConfigurationError( + "Service '{s.name}' has a link to service '{dep}' which is " + "undefined.".format(s=service_config, dep=dependency)) + + def validate_depends_on(service_config, service_names): for dependency in service_config.config.get('depends_on', []): if dependency not in service_names: From 377be5aa1f097166df91c95670f871959654be3a Mon Sep 17 00:00:00 2001 From: johnharris85 Date: Sun, 17 Apr 2016 14:01:06 -0400 Subject: [PATCH 49/95] Adding tests Signed-off-by: John Harris --- tests/unit/config/config_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 2bbbe6145..8bf416326 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1360,6 +1360,17 @@ class ConfigTest(unittest.TestCase): config.load(config_details) assert "Service 'one' depends on service 'three'" in exc.exconly() + def test_linked_service_is_undefined(self): + with self.assertRaises(ConfigurationError): + config.load( + build_config_details({ + 'version': '2', + 'services': { + 'web': {'image': 'busybox', 'links': ['db']}, + }, + }) + ) + def test_load_dockerfile_without_context(self): config_details = build_config_details({ 'version': '2', From 6d2805917c8e3f90b20781dfe35513d12e819533 Mon Sep 17 00:00:00 2001 From: johnharris85 Date: Sun, 17 Apr 2016 15:25:06 -0400 Subject: [PATCH 50/95] Account for aliased links Fix failing tests Signed-off-by: John Harris --- compose/config/validation.py | 8 ++++---- tests/fixtures/extends/invalid-links.yml | 2 ++ tests/unit/config/config_test.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index e4b3a2530..8c89cdf2b 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -172,11 +172,11 @@ def validate_network_mode(service_config, service_names): def validate_links(service_config, service_names): - for dependency in service_config.config.get('links', []): - if dependency not in service_names: + for link in service_config.config.get('links', []): + if link.split(':')[0] not in service_names: raise ConfigurationError( - "Service '{s.name}' has a link to service '{dep}' which is " - "undefined.".format(s=service_config, dep=dependency)) + "Service '{s.name}' has a link to service '{link}' which is " + "undefined.".format(s=service_config, link=link)) def validate_depends_on(service_config, service_names): diff --git a/tests/fixtures/extends/invalid-links.yml b/tests/fixtures/extends/invalid-links.yml index edfeb8b23..cea740cb7 100644 --- a/tests/fixtures/extends/invalid-links.yml +++ b/tests/fixtures/extends/invalid-links.yml @@ -1,3 +1,5 @@ +mydb: + build: '.' myweb: build: '.' extends: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 8bf416326..488305586 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1366,7 +1366,7 @@ class ConfigTest(unittest.TestCase): build_config_details({ 'version': '2', 'services': { - 'web': {'image': 'busybox', 'links': ['db']}, + 'web': {'image': 'busybox', 'links': ['db:db']}, }, }) ) From ba10f1cd55adfbcd228df1b6e1044b5c87ac06c8 Mon Sep 17 00:00:00 2001 From: Patrice FERLET Date: Wed, 20 Apr 2016 13:23:37 +0200 Subject: [PATCH 51/95] Fix the tests from jenkins Acceptance tests didn't set "help" command to return "0" EXIT_CODE. close #3354 related #3263 Signed-off-by: Patrice Ferlet --- tests/acceptance/cli_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 53ff66bbb..0b49efa01 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -140,8 +140,8 @@ class CLITestCase(DockerClientTestCase): def test_help(self): self.base_dir = 'tests/fixtures/no-composefile' - result = self.dispatch(['help', 'up'], returncode=1) - assert 'Usage: up [options] [SERVICE...]' in result.stderr + result = self.dispatch(['help', 'up'], returncode=0) + assert 'Usage: up [options] [SERVICE...]' in result.stdout # Prevent tearDown from trying to create a project self.base_dir = None From 55fcd1c3e32ccbd71caa14462a6239d4bf7a1685 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 20 Apr 2016 15:58:12 -0700 Subject: [PATCH 52/95] Clarify service networks documentation When jumping straight to this bit of the docs, it's not clear that these are options under a service rather than the top-level `networks` key. Added a service to make this super clear. Signed-off-by: Ben Firshman --- docs/compose-file.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 5aef5aca9..fc806a290 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -502,9 +502,11 @@ the special form `service:[service name]`. Networks to join, referencing entries under the [top-level `networks` key](#network-configuration-reference). - networks: - - some-network - - other-network + services: + some-service: + networks: + - some-network + - other-network #### aliases @@ -516,14 +518,16 @@ Since `aliases` is network-scoped, the same service can have different aliases o The general format is shown here. - networks: - some-network: - aliases: - - alias1 - - alias3 - other-network: - aliases: - - alias2 + services: + some-service: + networks: + some-network: + aliases: + - alias1 + - alias3 + other-network: + aliases: + - alias2 In the example below, three services are provided (`web`, `worker`, and `db`), along with two networks (`new` and `legacy`). The `db` service is reachable at the hostname `db` or `database` on the `new` network, and at `db` or `mysql` on the `legacy` network. From 27628f8655824a0ba96ef552c1b182aa8f48fa7f Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 20 Apr 2016 16:22:24 -0700 Subject: [PATCH 53/95] Make validation error less robotic "ERROR: Validation failed in file './docker-compose.yml', reason(s):" is now: "ERROR: The Compose file './docker-compose.yml' is invalid because:" Signed-off-by: Ben Firshman --- compose/config/validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 8c89cdf2b..726750a3d 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -416,6 +416,6 @@ def handle_errors(errors, format_error_func, filename): error_msg = '\n'.join(format_error_func(error) for error in errors) raise ConfigurationError( - "Validation failed{file_msg}, reason(s):\n{error_msg}".format( - file_msg=" in file '{}'".format(filename) if filename else "", + "The Compose file{file_msg} is invalid because:\n{error_msg}".format( + file_msg=" '{}'".format(filename) if filename else "", error_msg=error_msg)) From b67f110620bba758ae9b375b9f9743da317cfc45 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 20 Apr 2016 16:35:22 -0700 Subject: [PATCH 54/95] Explain the explanation about file versions This explanation looked like it was part of the error. Added an extra new line and a bit of copy to explain the explanation. Signed-off-by: Ben Firshman --- compose/config/errors.py | 9 +++++---- compose/config/validation.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/compose/config/errors.py b/compose/config/errors.py index d5df7ae55..d14cbbdd0 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -3,10 +3,11 @@ from __future__ import unicode_literals VERSION_EXPLANATION = ( - 'Either specify a version of "2" (or "2.0") and place your service ' - 'definitions under the `services` key, or omit the `version` key and place ' - 'your service definitions at the root of the file to use version 1.\n' - 'For more on the Compose file format versions, see ' + 'You might be seeing this error because you\'re using the wrong Compose ' + 'file version. Either specify a version of "2" (or "2.0") and place your ' + 'service definitions under the `services` key, or omit the `version` key ' + 'and place your service definitions at the root of the file to use ' + 'version 1.\nFor more on the Compose file format versions, see ' 'https://docs.docker.com/compose/compose-file/') diff --git a/compose/config/validation.py b/compose/config/validation.py index 726750a3d..7452e9849 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -219,7 +219,7 @@ def handle_error_for_schema_with_id(error, path): return get_unsupported_config_msg(path, invalid_config_key) if not error.path: - return '{}\n{}'.format(error.message, VERSION_EXPLANATION) + return '{}\n\n{}'.format(error.message, VERSION_EXPLANATION) def handle_generic_error(error, path): From 75bcc382d9965208ecb1e8b7e6caa5cc08916cf6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 19 Apr 2016 17:39:29 -0700 Subject: [PATCH 55/95] Force docker-py 1.8.0 or above Signed-off-by: Joffrey F --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7caae97d2..de009146d 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.8', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py > 1.7.2, < 2', + 'docker-py >= 1.8.0, < 2', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 26fe8213aa3edcb6bfb8ec287538f9d8674ae124 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 26 Apr 2016 11:58:41 -0400 Subject: [PATCH 56/95] Upgade pip to latest Hopefully fixes our builds. Signed-off-by: Daniel Nephin --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index acf9b6aeb..63fac3eb3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,11 +49,11 @@ RUN set -ex; \ # Install pip RUN set -ex; \ - curl -L https://pypi.python.org/packages/source/p/pip/pip-7.0.1.tar.gz | tar -xz; \ - cd pip-7.0.1; \ + curl -L https://pypi.python.org/packages/source/p/pip/pip-8.1.1.tar.gz | tar -xz; \ + cd pip-8.1.1; \ python setup.py install; \ cd ..; \ - rm -rf pip-7.0.1 + rm -rf pip-8.1.1 # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen From a4d3dd6197b9e15cf993823d93d321778d2fdcd8 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 18 Mar 2016 13:00:55 +0000 Subject: [PATCH 57/95] Remove v2_only decorators on config tests Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 0b49efa01..4d1990be1 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -145,15 +145,11 @@ class CLITestCase(DockerClientTestCase): # Prevent tearDown from trying to create a project self.base_dir = None - # TODO: this shouldn't be v2-dependent - @v2_only() def test_config_list_services(self): self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config', '--services']) assert set(result.stdout.rstrip().split('\n')) == {'web', 'other'} - # TODO: this shouldn't be v2-dependent - @v2_only() def test_config_quiet_with_error(self): self.base_dir = None result = self.dispatch([ @@ -162,14 +158,10 @@ class CLITestCase(DockerClientTestCase): ], returncode=1) assert "'notaservice' must be a mapping" in result.stderr - # TODO: this shouldn't be v2-dependent - @v2_only() def test_config_quiet(self): self.base_dir = 'tests/fixtures/v2-full' assert self.dispatch(['config', '-q']).stdout == '' - # TODO: this shouldn't be v2-dependent - @v2_only() def test_config_default(self): self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config']) From 84a3e2fe79552ca94172bd3958776b01eed0e31e Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 18 Mar 2016 13:01:35 +0000 Subject: [PATCH 58/95] Check full error message in test_up_with_net_is_invalid Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 4d1990be1..2a5a86044 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -675,9 +675,7 @@ class CLITestCase(DockerClientTestCase): ['-f', 'v2-invalid.yml', 'up', '-d'], returncode=1) - # TODO: fix validation error messages for v2 files - # assert "Unsupported config option for service 'web': 'net'" in exc.exconly() - assert "Unsupported config option" in result.stderr + assert "Unsupported config option for services.bar: 'net'" in result.stderr def test_up_with_net_v1(self): self.base_dir = 'tests/fixtures/net-container' From 6064d200f946c8d9738e1b73a03b1f78f947e2ef Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 18 Mar 2016 13:14:33 +0000 Subject: [PATCH 59/95] Fix output of 'config' for v1 files Signed-off-by: Aanand Prasad --- compose/config/serialize.py | 14 ++++++++++-- tests/acceptance/cli_test.py | 25 +++++++++++++++++++++ tests/fixtures/v1-config/docker-compose.yml | 10 +++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/v1-config/docker-compose.yml diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 06e0a027b..be6ba7204 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -5,6 +5,8 @@ import six import yaml from compose.config import types +from compose.config.config import V1 +from compose.config.config import V2_0 def serialize_config_type(dumper, data): @@ -17,12 +19,20 @@ yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) def serialize_config(config): + services = {service.pop('name'): service for service in config.services} + + if config.version == V1: + for service_dict in services.values(): + if 'network_mode' not in service_dict: + service_dict['network_mode'] = 'bridge' + output = { - 'version': config.version, - 'services': {service.pop('name'): service for service in config.services}, + 'version': V2_0, + 'services': services, 'networks': config.networks, 'volumes': config.volumes, } + return yaml.safe_dump( output, default_flow_style=False, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 2a5a86044..f7c958dd8 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -190,6 +190,31 @@ class CLITestCase(DockerClientTestCase): } assert output == expected + def test_config_v1(self): + self.base_dir = 'tests/fixtures/v1-config' + result = self.dispatch(['config']) + assert yaml.load(result.stdout) == { + 'version': '2.0', + 'services': { + 'net': { + 'image': 'busybox', + 'network_mode': 'bridge', + }, + 'volume': { + 'image': 'busybox', + 'volumes': ['/data:rw'], + 'network_mode': 'bridge', + }, + 'app': { + 'image': 'busybox', + 'volumes_from': ['service:volume:rw'], + 'network_mode': 'service:net', + }, + }, + 'networks': {}, + 'volumes': {}, + } + def test_ps(self): self.project.get_service('simple').create_container() result = self.dispatch(['ps']) diff --git a/tests/fixtures/v1-config/docker-compose.yml b/tests/fixtures/v1-config/docker-compose.yml new file mode 100644 index 000000000..8646c4edb --- /dev/null +++ b/tests/fixtures/v1-config/docker-compose.yml @@ -0,0 +1,10 @@ +net: + image: busybox +volume: + image: busybox + volumes: + - /data +app: + image: busybox + net: "container:net" + volumes_from: ["volume"] From 756ef14edc824ce2c52a2eb636c4884c95652e1e Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Apr 2016 17:30:04 +0100 Subject: [PATCH 60/95] Fix format of 'restart' option in 'config' output Signed-off-by: Aanand Prasad --- compose/config/serialize.py | 26 +++++++++++++++++----- compose/config/types.py | 9 ++++++++ tests/acceptance/cli_test.py | 27 +++++++++++++++++++++++ tests/fixtures/restart/docker-compose.yml | 14 ++++++++++++ 4 files changed, 70 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/restart/docker-compose.yml diff --git a/compose/config/serialize.py b/compose/config/serialize.py index be6ba7204..1b498c016 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -19,12 +19,14 @@ yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) def serialize_config(config): - services = {service.pop('name'): service for service in config.services} - - if config.version == V1: - for service_dict in services.values(): - if 'network_mode' not in service_dict: - service_dict['network_mode'] = 'bridge' + denormalized_services = [ + denormalize_service_dict(service_dict, config.version) + for service_dict in config.services + ] + services = { + service_dict.pop('name'): service_dict + for service_dict in denormalized_services + } output = { 'version': V2_0, @@ -38,3 +40,15 @@ def serialize_config(config): default_flow_style=False, indent=2, width=80) + + +def denormalize_service_dict(service_dict, version): + service_dict = service_dict.copy() + + if 'restart' in service_dict: + service_dict['restart'] = types.serialize_restart_spec(service_dict['restart']) + + if version == V1 and 'network_mode' not in service_dict: + service_dict['network_mode'] = 'bridge' + + return service_dict diff --git a/compose/config/types.py b/compose/config/types.py index fc3347c86..e6a3dea05 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -7,6 +7,8 @@ from __future__ import unicode_literals import os from collections import namedtuple +import six + from compose.config.config import V1 from compose.config.errors import ConfigurationError from compose.const import IS_WINDOWS_PLATFORM @@ -89,6 +91,13 @@ def parse_restart_spec(restart_config): return {'Name': name, 'MaximumRetryCount': int(max_retry_count)} +def serialize_restart_spec(restart_spec): + parts = [restart_spec['Name']] + if restart_spec['MaximumRetryCount']: + parts.append(six.text_type(restart_spec['MaximumRetryCount'])) + return ':'.join(parts) + + def parse_extra_hosts(extra_hosts_config): if not extra_hosts_config: return {} diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index f7c958dd8..515acb042 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -190,6 +190,33 @@ class CLITestCase(DockerClientTestCase): } assert output == expected + def test_config_restart(self): + self.base_dir = 'tests/fixtures/restart' + result = self.dispatch(['config']) + assert yaml.load(result.stdout) == { + 'version': '2.0', + 'services': { + 'never': { + 'image': 'busybox', + 'restart': 'no', + }, + 'always': { + 'image': 'busybox', + 'restart': 'always', + }, + 'on-failure': { + 'image': 'busybox', + 'restart': 'on-failure', + }, + 'on-failure-5': { + 'image': 'busybox', + 'restart': 'on-failure:5', + }, + }, + 'networks': {}, + 'volumes': {}, + } + def test_config_v1(self): self.base_dir = 'tests/fixtures/v1-config' result = self.dispatch(['config']) diff --git a/tests/fixtures/restart/docker-compose.yml b/tests/fixtures/restart/docker-compose.yml new file mode 100644 index 000000000..2d10aa397 --- /dev/null +++ b/tests/fixtures/restart/docker-compose.yml @@ -0,0 +1,14 @@ +version: "2" +services: + never: + image: busybox + restart: "no" + always: + image: busybox + restart: always + on-failure: + image: busybox + restart: on-failure + on-failure-5: + image: busybox + restart: "on-failure:5" From d3e645488a87840d1fab9660b98c09d2a8ec676f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 25 Apr 2016 17:58:20 -0700 Subject: [PATCH 61/95] Define WindowsError on non-win32 platforms Signed-off-by: Joffrey F --- compose/cli/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index dd859edc4..fff4a543f 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -12,6 +12,13 @@ from six.moves import input import compose +# WindowsError is not defined on non-win32 platforms. Avoid runtime errors by +# defining it as OSError (its parent class) if missing. +try: + WindowsError +except NameError: + WindowsError = OSError + def yesno(prompt, default=None): """ From 87ee38ed2c8da3fdee816737b55d7c7eb6e36a26 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 28 Apr 2016 12:57:02 +0000 Subject: [PATCH 62/95] convert docs Dockerfiles to use docs/base:oss Signed-off-by: Sven Dowideit --- docs/Dockerfile | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index b16d0d2c3..86ed32bc8 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,18 +1,8 @@ -FROM docs/base:latest +FROM docs/base:oss MAINTAINER Mary Anthony (@moxiegirl) -RUN svn checkout https://github.com/docker/docker/trunk/docs /docs/content/engine -RUN svn checkout https://github.com/docker/swarm/trunk/docs /docs/content/swarm -RUN svn checkout https://github.com/docker/machine/trunk/docs /docs/content/machine -RUN svn checkout https://github.com/docker/distribution/trunk/docs /docs/content/registry -RUN svn checkout https://github.com/docker/notary/trunk/docs /docs/content/notary -RUN svn checkout https://github.com/docker/kitematic/trunk/docs /docs/content/kitematic -RUN svn checkout https://github.com/docker/toolbox/trunk/docs /docs/content/toolbox -RUN svn checkout https://github.com/docker/opensource/trunk/docs /docs/content/project - - ENV PROJECT=compose # To get the git info for this repo COPY . /src - +RUN rm -r /docs/content/$PROJECT/ COPY . /docs/content/$PROJECT/ From 2efcec776c430c527d61069f16bea298d9e4fb37 Mon Sep 17 00:00:00 2001 From: Aaron Nall Date: Wed, 27 Apr 2016 22:44:28 +0000 Subject: [PATCH 63/95] Add missing log event filter when using docker-compose logs. Signed-off-by: Aaron Nall --- compose/cli/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index ae787c9b7..b86c34f86 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -426,7 +426,8 @@ class TopLevelCommand(object): self.project, containers, options['--no-color'], - log_args).run() + log_args, + event_stream=self.project.events(service_names=options['SERVICE'])).run() def pause(self, options): """ From 0b24883cef6ad5737b949815e107a968e96c2a55 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Apr 2016 12:21:47 -0700 Subject: [PATCH 64/95] Support combination of shorthand flag and equal sign for host option Signed-off-by: Joffrey F --- compose/cli/command.py | 5 ++++- tests/acceptance/cli_test.py | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index b7160deec..8ac3aff4f 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -21,12 +21,15 @@ log = logging.getLogger(__name__) def project_from_options(project_dir, options): environment = Environment.from_env_file(project_dir) + host = options.get('--host') + if host is not None: + host = host.lstrip('=') return get_project( project_dir, get_config_path_from_options(project_dir, options, environment), project_name=options.get('--project-name'), verbose=options.get('--verbose'), - host=options.get('--host'), + host=host, tls_config=tls_config_from_options(options), environment=environment ) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 515acb042..a02d0e99e 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -145,6 +145,13 @@ class CLITestCase(DockerClientTestCase): # Prevent tearDown from trying to create a project self.base_dir = None + def test_shorthand_host_opt(self): + self.dispatch( + ['-H={0}'.format(os.environ.get('DOCKER_HOST', 'unix://')), + 'up', '-d'], + returncode=0 + ) + def test_config_list_services(self): self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config', '--services']) From 84aa39e978c16877a64f1b097875667ff6eeef95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20R?= Date: Wed, 27 Apr 2016 13:45:59 +0200 Subject: [PATCH 65/95] Clarify env-file doc that .env is read from cwd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #3381 Signed-off-by: André R --- docs/env-file.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/env-file.md b/docs/env-file.md index a285a7908..be2625f88 100644 --- a/docs/env-file.md +++ b/docs/env-file.md @@ -13,8 +13,8 @@ weight=10 # Environment file Compose supports declaring default environment variables in an environment -file named `.env` and placed in the same folder as your -[compose file](compose-file.md). +file named `.env` placed in the folder `docker-compose` command is executed from +*(current working directory)*. Compose expects each line in an env file to be in `VAR=VAL` format. Lines beginning with `#` (i.e. comments) are ignored, as are blank lines. From e4bb678875adf1a5aa5fdc1fe542f00c4e279060 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 29 Apr 2016 16:37:26 -0700 Subject: [PATCH 66/95] Require latest docker-py version Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b9b0f4036..eb5275f4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PyYAML==3.11 cached-property==1.2.0 -docker-py==1.8.0 +docker-py==1.8.1 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4 diff --git a/setup.py b/setup.py index de009146d..0b37c1dd4 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ install_requires = [ 'requests >= 2.6.1, < 2.8', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py >= 1.8.0, < 2', + 'docker-py >= 1.8.1, < 2', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From fe17e0f94835aab59f71f33e055f1c52847ce673 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Apr 2016 15:52:25 -0700 Subject: [PATCH 67/95] Skip event objects that don't contain a status field Signed-off-by: Joffrey F --- compose/project.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 0d891e455..64ca7be76 100644 --- a/compose/project.py +++ b/compose/project.py @@ -342,7 +342,10 @@ class Project(object): filters={'label': self.labels()}, decode=True ): - if event['status'] in IMAGE_EVENTS: + # The first part of this condition is a guard against some events + # broadcasted by swarm that don't have a status field. + # See https://github.com/docker/compose/issues/3316 + if 'status' not in event or event['status'] in IMAGE_EVENTS: # We don't receive any image events because labels aren't applied # to images continue From 28fb91b34459dae8e0531370aa005d95321803f1 Mon Sep 17 00:00:00 2001 From: Thom Linton Date: Fri, 29 Apr 2016 16:31:19 -0700 Subject: [PATCH 68/95] Adds additional validation to 'env_vars_from_file'. The 'env_file' directive and feature precludes the use of the name '.env' in the path shared with 'docker-config.yml', regardless of whether or not it is enabled. This change adds an additional validation to allow the use of this path provided it is not a file. Signed-off-by: Thom Linton --- compose/config/environment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/config/environment.py b/compose/config/environment.py index ad5c0b3da..ff08b7714 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -28,6 +28,8 @@ def env_vars_from_file(filename): """ if not os.path.exists(filename): raise ConfigurationError("Couldn't find env file: %s" % filename) + elif not os.path.isfile(filename): + raise ConfigurationError("%s is not a file." % (filename)) env = {} for line in codecs.open(filename, 'r', 'utf-8'): line = line.strip() From 310b3d9441c8a63dc7f2685a1eb2d3e83e1584dc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 29 Apr 2016 19:42:07 -0700 Subject: [PATCH 69/95] Properly handle APIError failures in Project.up Signed-off-by: Joffrey F --- compose/cli/main.py | 3 ++- compose/parallel.py | 2 +- compose/project.py | 11 ++++++++++- tests/integration/project_test.py | 4 +++- tests/unit/parallel_test.py | 3 ++- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index b86c34f86..34e7f35c7 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -24,6 +24,7 @@ from ..const import IS_WINDOWS_PLATFORM from ..progress_stream import StreamOutputError from ..project import NoSuchService from ..project import OneOffFilter +from ..project import ProjectError from ..service import BuildAction from ..service import BuildError from ..service import ConvergenceStrategy @@ -58,7 +59,7 @@ def main(): except (KeyboardInterrupt, signals.ShutdownException): log.error("Aborting.") sys.exit(1) - except (UserError, NoSuchService, ConfigurationError) as e: + except (UserError, NoSuchService, ConfigurationError, ProjectError) as e: log.error(e.msg) sys.exit(1) except BuildError as e: diff --git a/compose/parallel.py b/compose/parallel.py index 63417dcb0..50b2dbeaf 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -59,7 +59,7 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): if error_to_reraise: raise error_to_reraise - return results + return results, errors def _no_deps(x): diff --git a/compose/project.py b/compose/project.py index 64ca7be76..d965c4a39 100644 --- a/compose/project.py +++ b/compose/project.py @@ -390,13 +390,18 @@ class Project(object): def get_deps(service): return {self.get_service(dep) for dep in service.get_dependency_names()} - results = parallel.parallel_execute( + results, errors = parallel.parallel_execute( services, do, operator.attrgetter('name'), None, get_deps ) + if errors: + raise ProjectError( + 'Encountered errors while bringing up the project.' + ) + return [ container for svc_containers in results @@ -531,3 +536,7 @@ class NoSuchService(Exception): def __str__(self): return self.msg + + +class ProjectError(Exception): + pass diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index c413b9aa0..7ef492a56 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -19,6 +19,7 @@ from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.container import Container from compose.project import Project +from compose.project import ProjectError from compose.service import ConvergenceStrategy from tests.integration.testcases import v2_only @@ -752,7 +753,8 @@ class ProjectTest(DockerClientTestCase): config_data=config_data, ) - assert len(project.up()) == 0 + with self.assertRaises(ProjectError): + project.up() @v2_only() def test_project_up_volumes(self): diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 45b0db1db..479c0f1d3 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -29,7 +29,7 @@ def get_deps(obj): def test_parallel_execute(): - results = parallel_execute( + results, errors = parallel_execute( objects=[1, 2, 3, 4, 5], func=lambda x: x * 2, get_name=six.text_type, @@ -37,6 +37,7 @@ def test_parallel_execute(): ) assert sorted(results) == [2, 4, 6, 8, 10] + assert errors == {} def test_parallel_execute_with_deps(): From 3b7191f246b5f7cd6b2fbdefa86547492861f025 Mon Sep 17 00:00:00 2001 From: Garrett Seward Date: Wed, 4 May 2016 10:45:04 -0700 Subject: [PATCH 70/95] Small typo Signed-off-by: spectralsun --- docs/compose-file.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index fc806a290..4902e8ddf 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -1083,7 +1083,7 @@ It's more complicated if you're using particular configuration features: data: {} By default, Compose creates a volume whose name is prefixed with your - project name. If you want it to just be called `data`, declared it as + project name. If you want it to just be called `data`, declare it as external: volumes: From 4b01f6dcd657636a6d05f453dfd58a6d3826ca5e Mon Sep 17 00:00:00 2001 From: Anton Simernia Date: Mon, 9 May 2016 18:15:32 +0700 Subject: [PATCH 71/95] add msg attribute to ProjectError class Signed-off-by: Anton Simernia --- compose/project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index d965c4a39..1b7fde23b 100644 --- a/compose/project.py +++ b/compose/project.py @@ -539,4 +539,5 @@ class NoSuchService(Exception): class ProjectError(Exception): - pass + def __init__(self, msg): + self.msg = msg From 4bf5271ae2d53f8c6467642b6bd4c3372ed52da8 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 12 May 2016 14:41:40 -0400 Subject: [PATCH 72/95] Skip invalid git tags in versions.py Signed-off-by: Daniel Nephin --- script/test/versions.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/script/test/versions.py b/script/test/versions.py index 98f97ef32..45ead1438 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -28,6 +28,7 @@ from __future__ import unicode_literals import argparse import itertools import operator +import sys from collections import namedtuple import requests @@ -103,6 +104,14 @@ def get_default(versions): return version +def get_versions(tags): + for tag in tags: + try: + yield Version.parse(tag['name']) + except ValueError: + print("Skipping invalid tag: {name}".format(**tag), file=sys.stderr) + + def get_github_releases(project): """Query the Github API for a list of version tags and return them in sorted order. @@ -112,7 +121,7 @@ def get_github_releases(project): url = '{}/{}/tags'.format(GITHUB_API, project) response = requests.get(url) response.raise_for_status() - versions = [Version.parse(tag['name']) for tag in response.json()] + versions = get_versions(response.json()) return sorted(versions, reverse=True, key=operator.attrgetter('order')) From e5645595e3057f7b6eadcde922dd9ae7e0ff9363 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 May 2016 16:29:27 -0700 Subject: [PATCH 73/95] Fail gracefully when -d is not provided for exec command on Win32 Signed-off-by: Joffrey F --- compose/cli/main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 34e7f35c7..3ab2f9654 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -334,6 +334,13 @@ class TopLevelCommand(object): """ index = int(options.get('--index')) service = self.project.get_service(options['SERVICE']) + detach = options['-d'] + + if IS_WINDOWS_PLATFORM and not detach: + raise UserError( + "Interactive mode is not yet supported on Windows.\n" + "Please pass the -d flag when using `docker-compose exec`." + ) try: container = service.get_container(number=index) except ValueError as e: @@ -350,7 +357,7 @@ class TopLevelCommand(object): exec_id = container.create_exec(command, **create_exec_options) - if options['-d']: + if detach: container.start_exec(exec_id, tty=tty) return From 844b7d463f63b4bd3915648b32432c9b3d0243c8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 May 2016 14:59:33 -0700 Subject: [PATCH 74/95] Update rm command to always remove one-off containers. Signed-off-by: Joffrey F --- compose/cli/main.py | 12 +++++------- tests/acceptance/cli_test.py | 2 -- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 3ab2f9654..afde71506 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -532,17 +532,15 @@ class TopLevelCommand(object): Options: -f, --force Don't ask to confirm removal -v Remove any anonymous volumes attached to containers - -a, --all Also remove one-off containers created by + -a, --all Obsolete. Also remove one-off containers created by docker-compose run """ if options.get('--all'): - one_off = OneOffFilter.include - else: log.warn( - 'Not including one-off containers created by `docker-compose run`.\n' - 'To include them, use `docker-compose rm --all`.\n' - 'This will be the default behavior in the next version of Compose.\n') - one_off = OneOffFilter.exclude + '--all flag is obsolete. This is now the default behavior ' + 'of `docker-compose rm`' + ) + one_off = OneOffFilter.include all_containers = self.project.containers( service_names=options['SERVICE'], stopped=True, one_off=one_off diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a02d0e99e..dfd75625c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1192,8 +1192,6 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 1) self.dispatch(['rm', '-f'], None) self.assertEqual(len(service.containers(stopped=True)), 0) - self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 1) - self.dispatch(['rm', '-f', '-a'], None) self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 0) service.create_container(one_off=False) From db0a6cf2bbcd7a5a673833b9558f0d142a0f304c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 12 May 2016 17:38:41 -0700 Subject: [PATCH 75/95] Always use the Windows version of splitdrive when parsing volume mappings Signed-off-by: Joffrey F --- compose/config/config.py | 3 ++- tests/unit/config/config_test.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index e52de4bf8..6cfce5da4 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import functools import logging +import ntpath import operator import os import string @@ -944,7 +945,7 @@ def split_path_mapping(volume_path): if volume_path.startswith('.') or volume_path.startswith('~'): drive, volume_config = '', volume_path else: - drive, volume_config = os.path.splitdrive(volume_path) + drive, volume_config = ntpath.splitdrive(volume_path) if ':' in volume_config: (host, container) = volume_config.split(':', 1) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 488305586..26a1e08a6 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2658,8 +2658,6 @@ class ExpandPathTest(unittest.TestCase): class VolumePathTest(unittest.TestCase): - - @pytest.mark.xfail((not IS_WINDOWS_PLATFORM), reason='does not have a drive') def test_split_path_mapping_with_windows_path(self): host_path = "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config" windows_volume_path = host_path + ":/opt/connect/config:ro" From 0c8aeb9e056caaa1fa2d1cc1133f6bd41505ec3c Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 19 May 2016 07:41:02 -0700 Subject: [PATCH 76/95] Fix bug where confirmation prompt doesn't show due to line buffering Signed-off-by: Aanand Prasad --- compose/cli/utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index fff4a543f..b58b50ef9 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -6,9 +6,9 @@ import os import platform import ssl import subprocess +import sys import docker -from six.moves import input import compose @@ -42,6 +42,16 @@ def yesno(prompt, default=None): return None +def input(prompt): + """ + Version of input (raw_input in Python 2) which forces a flush of sys.stdout + to avoid problems where the prompt fails to appear due to line buffering + """ + sys.stdout.write(prompt) + sys.stdout.flush() + return sys.stdin.readline().rstrip(b'\n') + + def call_silently(*args, **kwargs): """ Like subprocess.call(), but redirects stdout and stderr to /dev/null. From 2b5b665d3ab47ab7d1bbe0049b02af126f2eaa63 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 May 2016 14:53:37 -0700 Subject: [PATCH 77/95] Add test for path mapping with Windows containers Signed-off-by: Joffrey F --- tests/unit/config/config_test.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 26a1e08a6..ccb3bcfe1 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2664,7 +2664,15 @@ class VolumePathTest(unittest.TestCase): expected_mapping = ("/opt/connect/config:ro", host_path) mapping = config.split_path_mapping(windows_volume_path) - self.assertEqual(mapping, expected_mapping) + assert mapping == expected_mapping + + def test_split_path_mapping_with_windows_path_in_container(self): + host_path = 'c:\\Users\\remilia\\data' + container_path = 'c:\\scarletdevil\\data' + expected_mapping = (container_path, host_path) + + mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) + assert mapping == expected_mapping @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') From 33bed5c7066e53cb147afcbef2e9ab78cb0ab1f0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 20 May 2016 12:02:45 +0100 Subject: [PATCH 78/95] Use latest OpenSSL version (1.0.2h) when building Mac binary on Travis Signed-off-by: Aanand Prasad --- script/setup/osx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/setup/osx b/script/setup/osx index 10bbbecc3..39941de27 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -14,9 +14,9 @@ desired_python_version="2.7.9" desired_python_brew_version="2.7.9" python_formula="https://raw.githubusercontent.com/Homebrew/homebrew/1681e193e4d91c9620c4901efd4458d9b6fcda8e/Library/Formula/python.rb" -desired_openssl_version="1.0.1j" -desired_openssl_brew_version="1.0.1j_1" -openssl_formula="https://raw.githubusercontent.com/Homebrew/homebrew/62fc2a1a65e83ba9dbb30b2e0a2b7355831c714b/Library/Formula/openssl.rb" +desired_openssl_version="1.0.2h" +desired_openssl_brew_version="1.0.2h" +openssl_formula="https://raw.githubusercontent.com/Homebrew/homebrew-core/30d3766453347f6e22b3ed6c74bb926d6def2eb5/Formula/openssl.rb" PATH="/usr/local/bin:$PATH" From 842e372258809b0be035a7857f8577e33850cca7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 17 May 2016 15:02:29 -0700 Subject: [PATCH 79/95] Eliminate duplicates when merging port mappings from config files Signed-off-by: Joffrey F --- compose/config/config.py | 6 +++++- tests/integration/project_test.py | 36 +++++++++++++++++++++++++++++++ tests/unit/config/config_test.py | 8 +++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 6cfce5da4..e1466f060 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -744,6 +744,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('ulimits', parse_ulimits) md.merge_mapping('networks', parse_networks) md.merge_sequence('links', ServiceLink.parse) + md.merge_field('ports', merge_port_mappings, default=[]) for field in ['volumes', 'devices']: md.merge_field(field, merge_path_mappings) @@ -752,7 +753,6 @@ def merge_service_dicts(base, override, version): 'depends_on', 'expose', 'external_links', - 'ports', 'volumes_from', ]: md.merge_field(field, operator.add, default=[]) @@ -771,6 +771,10 @@ def merge_service_dicts(base, override, version): return dict(md) +def merge_port_mappings(base, override): + return list(set().union(base, override)) + + def merge_build(output, base, override): def to_dict(service): build_config = service.get('build', {}) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 7ef492a56..6e82e931f 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -834,6 +834,42 @@ class ProjectTest(DockerClientTestCase): self.assertTrue(log_config) self.assertEqual(log_config.get('Type'), 'none') + @v2_only() + def test_project_up_port_mappings_with_multiple_files(self): + base_file = config.ConfigFile( + 'base.yml', + { + 'version': V2_0, + 'services': { + 'simple': { + 'image': 'busybox:latest', + 'command': 'top', + 'ports': ['1234:1234'] + }, + }, + + }) + override_file = config.ConfigFile( + 'override.yml', + { + 'version': V2_0, + 'services': { + 'simple': { + 'ports': ['1234:1234'] + } + } + + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + config_data = config.load(details) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + project.up() + containers = project.containers() + self.assertEqual(len(containers), 1) + @v2_only() def test_initialize_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ccb3bcfe1..24ece4994 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1912,6 +1912,14 @@ class MergePortsTest(unittest.TestCase, MergeListsTest): base_config = ['10:8000', '9000'] override_config = ['20:8000'] + def test_duplicate_port_mappings(self): + service_dict = config.merge_service_dicts( + {self.config_name: self.base_config}, + {self.config_name: self.base_config}, + DEFAULT_VERSION + ) + assert set(service_dict[self.config_name]) == set(self.base_config) + class MergeNetworksTest(unittest.TestCase, MergeListsTest): config_name = 'networks' From c4229b469a7fdf37b84fdd7b911508936f442363 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 May 2016 15:42:37 -0700 Subject: [PATCH 80/95] Improve merging for several service config attributes All uniqueItems lists in the config now receive the same treatment removing duplicates. Signed-off-by: Joffrey F --- compose/config/config.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index e1466f060..97c427b91 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals import functools import logging import ntpath -import operator import os import string import sys @@ -744,18 +743,15 @@ def merge_service_dicts(base, override, version): md.merge_mapping('ulimits', parse_ulimits) md.merge_mapping('networks', parse_networks) md.merge_sequence('links', ServiceLink.parse) - md.merge_field('ports', merge_port_mappings, default=[]) for field in ['volumes', 'devices']: md.merge_field(field, merge_path_mappings) for field in [ - 'depends_on', - 'expose', - 'external_links', - 'volumes_from', + 'ports', 'cap_add', 'cap_drop', 'expose', 'external_links', + 'security_opt', 'volumes_from', 'depends_on', ]: - md.merge_field(field, operator.add, default=[]) + md.merge_field(field, merge_unique_items_lists, default=[]) for field in ['dns', 'dns_search', 'env_file', 'tmpfs']: md.merge_field(field, merge_list_or_string) @@ -771,8 +767,8 @@ def merge_service_dicts(base, override, version): return dict(md) -def merge_port_mappings(base, override): - return list(set().union(base, override)) +def merge_unique_items_lists(base, override): + return sorted(set().union(base, override)) def merge_build(output, base, override): From a34cd5ed543cbc98b703e83c41e13ea1757ad482 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 19 May 2016 17:26:42 +0100 Subject: [PATCH 81/95] Add "disambiguation" page for environment variables Signed-off-by: Aanand Prasad --- docs/environment-variables.md | 107 ++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 docs/environment-variables.md diff --git a/docs/environment-variables.md b/docs/environment-variables.md new file mode 100644 index 000000000..a2e74f0a9 --- /dev/null +++ b/docs/environment-variables.md @@ -0,0 +1,107 @@ + + +# Environment variables in Compose + +There are multiple parts of Compose that deal with environment variables in one sense or another. This page should help you find the information you need. + + +## Substituting environment variables in Compose files + +It's possible to use environment variables in your shell to populate values inside a Compose file: + + web: + image: "webapp:${TAG}" + +For more information, see the [Variable substitution](compose-file.md#variable-substitution) section in the Compose file reference. + + +## Setting environment variables in containers + +You can set environment variables in a service's containers with the ['environment' key](compose-file.md#environment), just like with `docker run -e VARIABLE=VALUE ...`: + + web: + environment: + - DEBUG=1 + + +## Passing environment variables through to containers + +You can pass environment variables from your shell straight through to a service's containers with the ['environment' key](compose-file.md#environment) by not giving them a value, just like with `docker run -e VARIABLE ...`: + + web: + environment: + - DEBUG + +The value of the `DEBUG` variable in the container will be taken from the value for the same variable in the shell in which Compose is run. + + +## The “env_file” configuration option + +You can pass multiple environment variables from an external file through to a service's containers with the ['env_file' option](compose-file.md#env-file), just like with `docker run --env-file=FILE ...`: + + web: + env_file: + - web-variables.env + + +## Setting environment variables with 'docker-compose run' + +Just like with `docker run -e`, you can set environment variables on a one-off container with `docker-compose run -e`: + + $ docker-compose run -e DEBUG=1 web python console.py + +You can also pass a variable through from the shell by not giving it a value: + + $ docker-compose run -e DEBUG web python console.py + +The value of the `DEBUG` variable in the container will be taken from the value for the same variable in the shell in which Compose is run. + + +## The “.env” file + +You can set default values for any environment variables referenced in the Compose file, or used to configure Compose, in an [environment file](env-file.md) named `.env`: + + $ cat .env + TAG=v1.5 + + $ cat docker-compose.yml + version: '2.0' + services: + web: + image: "webapp:${TAG}" + +When you run `docker-compose up`, the `web` service defined above uses the image `webapp:v1.5`. You can verify this with the [config command](reference/config.md), which prints your resolved application config to the terminal: + + $ docker-compose config + version: '2.0' + services: + web: + image: 'webapp:v1.5' + +Values in the shell take precedence over those specified in the `.env` file. If you set `TAG` to a different value in your shell, the substitution in `image` uses that instead: + + $ export TAG=v2.0 + + $ docker-compose config + version: '2.0' + services: + web: + image: 'webapp:v2.0' + +## Configuring Compose using environment variables + +Several environment variables are available for you to configure the Docker Compose command-line behaviour. They begin with `COMPOSE_` or `DOCKER_`, and are documented in [CLI Environment Variables](reference/envvars.md). + + +## Environment variables created by links + +When using the ['links' option](compose-file.md#links) in a [v1 Compose file](compose-file.md#version-1), environment variables will be created for each link. They are documented in the [Link environment variables reference](link-env-deprecated.md). Please note, however, that these variables are deprecated - you should just use the link alias as a hostname instead. From c46737ed026055411e1249efc96053ee6acfe37a Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 26 May 2016 12:44:53 -0700 Subject: [PATCH 82/95] remove command completion for `docker-compose rm --a` As `--all|-a` is deprecated, there's no use to suggest it any more in command completion. Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- contrib/completion/zsh/_docker-compose | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 66747fbd5..763cafc4f 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -325,7 +325,7 @@ _docker_compose_restart() { _docker_compose_rm() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--all -a --force -f --help -v" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--force -f --help -v" -- "$cur" ) ) ;; *) __docker_compose_services_stopped diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index ec9cb682f..0da217dcb 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -281,7 +281,6 @@ __docker-compose_subcommand() { (rm) _arguments \ $opts_help \ - '(-a --all)'{-a,--all}"[Also remove one-off containers]" \ '(-f --force)'{-f,--force}"[Don't ask to confirm removal]" \ '-v[Remove volumes associated with containers]' \ '*:stopped services:__docker-compose_stoppedservices' && ret=0 From e3e8a619cce64a127df7d7962a2694116914b566 Mon Sep 17 00:00:00 2001 From: Andrey Devyatkin Date: Fri, 27 May 2016 07:48:13 +0200 Subject: [PATCH 83/95] Fix #3281: Unexpected result when using build args with default values Fix the issue when build arg is set to None instead of empty string. Usecase: cat docker-compose.yml .... args: - http_proxy - https_proxy - no_proxy If http_proxy, https_proxy, no_proxy environment variables are not defined then http_proxy, https_proxy, no_proxy build args will be set to string None which breaks all downloads With this change build args will not passed to docker engine if they are equal to string None Signed-off-by: Andrey Devyatkin --- compose/service.py | 8 +++++++- tests/unit/service_test.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 8b9f64f0f..64a464536 100644 --- a/compose/service.py +++ b/compose/service.py @@ -701,6 +701,12 @@ class Service(object): build_opts = self.options.get('build', {}) path = build_opts.get('context') + # If build argument is not defined and there is no environment variable + # with the same name then build argument value will be None + # Moreover it will be sent to the docker engine as None and then + # interpreted as string None which in many cases will fail the build + # That is why we filter out all pairs with value equal to None + buildargs = {k: v for k, v in build_opts.get('args', {}).items() if v != 'None'} # python2 os.path() doesn't support unicode, so we need to encode it to # a byte string if not six.PY3: @@ -715,7 +721,7 @@ class Service(object): pull=pull, nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), - buildargs=build_opts.get('args', None), + buildargs=buildargs, ) try: diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index a259c476f..ae2cab208 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -445,7 +445,7 @@ class ServiceTest(unittest.TestCase): forcerm=False, nocache=False, rm=True, - buildargs=None, + buildargs={}, ) def test_ensure_image_exists_no_build(self): @@ -481,7 +481,33 @@ class ServiceTest(unittest.TestCase): forcerm=False, nocache=False, rm=True, - buildargs=None, + buildargs={}, + ) + + def test_ensure_filter_out_empty_build_args(self): + args = {u'no_proxy': 'None', u'https_proxy': 'something'} + service = Service('foo', + client=self.mock_client, + build={'context': '.', 'args': args}) + self.mock_client.inspect_image.return_value = {'Id': 'abc123'} + self.mock_client.build.return_value = [ + '{"stream": "Successfully built abcd"}', + ] + + with mock.patch('compose.service.log', autospec=True) as mock_log: + service.ensure_image_exists(do_build=BuildAction.force) + + assert not mock_log.warn.called + self.mock_client.build.assert_called_once_with( + tag='default_foo', + dockerfile=None, + stream=True, + path='.', + pull=False, + forcerm=False, + nocache=False, + rm=True, + buildargs={u'https_proxy': 'something'}, ) def test_build_does_not_pull(self): From 1298b9aa5d8d9f7b99c2f1130a3d3661bbda2c16 Mon Sep 17 00:00:00 2001 From: Denis Makogon Date: Tue, 24 May 2016 15:16:36 +0300 Subject: [PATCH 84/95] Issue-3503: Improve timestamp validation in tests CLITestCase.test_events_human_readable fails due to wrong assumption that host where tests were launched will have the same date time as Docker daemon. This fix introduces internal method for validating timestamp in Docker logs Signed-off-by: Denys Makogon --- tests/acceptance/cli_test.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index dfd75625c..4efaf0cf4 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1473,6 +1473,17 @@ class CLITestCase(DockerClientTestCase): assert Counter(e['action'] for e in lines) == {'create': 2, 'start': 2} def test_events_human_readable(self): + + def has_timestamp(string): + str_iso_date, str_iso_time, container_info = string.split(' ', 2) + try: + return isinstance(datetime.datetime.strptime( + '%s %s' % (str_iso_date, str_iso_time), + '%Y-%m-%d %H:%M:%S.%f'), + datetime.datetime) + except ValueError: + return False + events_proc = start_process(self.base_dir, ['events']) self.dispatch(['up', '-d', 'simple']) wait_on_condition(ContainerCountCondition(self.project, 1)) @@ -1489,7 +1500,8 @@ class CLITestCase(DockerClientTestCase): assert expected_template.format('create', container.id) in lines[0] assert expected_template.format('start', container.id) in lines[1] - assert lines[0].startswith(datetime.date.today().isoformat()) + + assert has_timestamp(lines[0]) def test_env_file_relative_to_compose_file(self): config_path = os.path.abspath('tests/fixtures/env-file/docker-compose.yml') From c148849f0e219ff61a7a29164fd88c113faf7ef3 Mon Sep 17 00:00:00 2001 From: Andrey Devyatkin Date: Fri, 27 May 2016 19:59:27 +0200 Subject: [PATCH 85/95] Fix #3281: Unexpected result when using build args with default values Fix the issue when build arg is set to None instead of empty string. Usecase: cat docker-compose.yml .... args: - http_proxy - https_proxy - no_proxy If http_proxy, https_proxy, no_proxy environment variables are not defined then http_proxy, https_proxy, no_proxy build args will be set to string None which breaks all downloads With this change undefined build args will be set to empty string instead of string None Signed-off-by: Andrey Devyatkin --- compose/service.py | 8 +------- compose/utils.py | 2 +- tests/unit/config/config_test.py | 2 +- tests/unit/service_test.py | 30 ++---------------------------- 4 files changed, 5 insertions(+), 37 deletions(-) diff --git a/compose/service.py b/compose/service.py index 64a464536..8b9f64f0f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -701,12 +701,6 @@ class Service(object): build_opts = self.options.get('build', {}) path = build_opts.get('context') - # If build argument is not defined and there is no environment variable - # with the same name then build argument value will be None - # Moreover it will be sent to the docker engine as None and then - # interpreted as string None which in many cases will fail the build - # That is why we filter out all pairs with value equal to None - buildargs = {k: v for k, v in build_opts.get('args', {}).items() if v != 'None'} # python2 os.path() doesn't support unicode, so we need to encode it to # a byte string if not six.PY3: @@ -721,7 +715,7 @@ class Service(object): pull=pull, nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), - buildargs=buildargs, + buildargs=build_opts.get('args', None), ) try: diff --git a/compose/utils.py b/compose/utils.py index 494beea34..1e01fcb62 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -95,4 +95,4 @@ def microseconds_from_time_nano(time_nano): def build_string_dict(source_dict): - return dict((k, str(v)) for k, v in source_dict.items()) + return dict((k, str(v if v else '')) for k, v in source_dict.items()) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 24ece4994..3e5a7face 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -715,7 +715,7 @@ class ConfigTest(unittest.TestCase): ).services[0] assert 'args' in service['build'] assert 'foo' in service['build']['args'] - assert service['build']['args']['foo'] == 'None' + assert service['build']['args']['foo'] == '' def test_load_with_multiple_files_mismatched_networks_format(self): base_file = config.ConfigFile( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index ae2cab208..a259c476f 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -445,7 +445,7 @@ class ServiceTest(unittest.TestCase): forcerm=False, nocache=False, rm=True, - buildargs={}, + buildargs=None, ) def test_ensure_image_exists_no_build(self): @@ -481,33 +481,7 @@ class ServiceTest(unittest.TestCase): forcerm=False, nocache=False, rm=True, - buildargs={}, - ) - - def test_ensure_filter_out_empty_build_args(self): - args = {u'no_proxy': 'None', u'https_proxy': 'something'} - service = Service('foo', - client=self.mock_client, - build={'context': '.', 'args': args}) - self.mock_client.inspect_image.return_value = {'Id': 'abc123'} - self.mock_client.build.return_value = [ - '{"stream": "Successfully built abcd"}', - ] - - with mock.patch('compose.service.log', autospec=True) as mock_log: - service.ensure_image_exists(do_build=BuildAction.force) - - assert not mock_log.warn.called - self.mock_client.build.assert_called_once_with( - tag='default_foo', - dockerfile=None, - stream=True, - path='.', - pull=False, - forcerm=False, - nocache=False, - rm=True, - buildargs={u'https_proxy': 'something'}, + buildargs=None, ) def test_build_does_not_pull(self): From 90fba58df9caf98b3d1573dbeba34e8d7858d188 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 27 May 2016 21:29:47 +0000 Subject: [PATCH 86/95] Fix links Signed-off-by: Sven Dowideit --- docs/gettingstarted.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index 60482bce5..ff944177b 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -137,8 +137,8 @@ The `redis` service uses the latest public [Redis](https://registry.hub.docker.c 2. Enter `http://0.0.0.0:5000/` in a browser to see the application running. If you're using Docker on Linux natively, then the web app should now be - listening on port 5000 on your Docker daemon host. If http://0.0.0.0:5000 - doesn't resolve, you can also try http://localhost:5000. + listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` + doesn't resolve, you can also try `http://localhost:5000`. If you're using Docker Machine on a Mac, use `docker-machine ip MACHINE_VM` to get the IP address of your Docker host. Then, `open http://MACHINE_VM_IP:5000` in a From a67ba5536db72203b22fc989b91f54f598e1d1f9 Mon Sep 17 00:00:00 2001 From: Andrey Devyatkin Date: Sat, 28 May 2016 11:39:41 +0200 Subject: [PATCH 87/95] Fix #3281: Unexpected result when using build args with default values Fix the issue when build arg is set to None instead of empty string. Usecase: cat docker-compose.yml .... args: - http_proxy - https_proxy - no_proxy If http_proxy, https_proxy, no_proxy environment variables are not defined then http_proxy, https_proxy, no_proxy build args will be set to string None which breaks all downloads With this change undefined build args will be set to empty string instead of string None Signed-off-by: Andrey Devyatkin --- compose/utils.py | 2 +- tests/unit/config/config_test.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/compose/utils.py b/compose/utils.py index 1e01fcb62..925a8e791 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -95,4 +95,4 @@ def microseconds_from_time_nano(time_nano): def build_string_dict(source_dict): - return dict((k, str(v if v else '')) for k, v in source_dict.items()) + return dict((k, str(v if v is not None else '')) for k, v in source_dict.items()) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 3e5a7face..0abb8daea 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -717,6 +717,34 @@ class ConfigTest(unittest.TestCase): assert 'foo' in service['build']['args'] assert service['build']['args']['foo'] == '' + # If build argument is None then it will be converted to the empty + # string. Make sure that int zero kept as it is, i.e. not converted to + # the empty string + def test_build_args_check_zero_preserved(self): + service = config.load( + build_config_details( + { + 'version': '2', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'dockerfile': 'Dockerfile-alt', + 'args': { + 'foo': 0 + } + } + } + } + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services[0] + assert 'args' in service['build'] + assert 'foo' in service['build']['args'] + assert service['build']['args']['foo'] == '0' + def test_load_with_multiple_files_mismatched_networks_format(self): base_file = config.ConfigFile( 'base.yaml', From dd3590180da36f5359d6463003b49ea2fca90315 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Tue, 31 May 2016 21:18:42 +0000 Subject: [PATCH 88/95] more fixes Signed-off-by: Sven Dowideit --- docs/Dockerfile | 4 ++-- docs/Makefile | 27 +++++---------------------- docs/django.md | 4 ++-- docs/gettingstarted.md | 2 +- docs/link-env-deprecated.md | 6 +++--- docs/overview.md | 4 ++-- docs/production.md | 4 ++-- docs/rails.md | 4 ++-- docs/swarm.md | 4 ++-- 9 files changed, 21 insertions(+), 38 deletions(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index 86ed32bc8..7b5a3b246 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,8 +1,8 @@ FROM docs/base:oss -MAINTAINER Mary Anthony (@moxiegirl) +MAINTAINER Docker Docs ENV PROJECT=compose # To get the git info for this repo COPY . /src -RUN rm -r /docs/content/$PROJECT/ +RUN rm -rf /docs/content/$PROJECT/ COPY . /docs/content/$PROJECT/ diff --git a/docs/Makefile b/docs/Makefile index b9ef05482..e6629289b 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,17 +1,4 @@ -.PHONY: all binary build cross default docs docs-build docs-shell shell test test-unit test-integration test-integration-cli test-docker-py validate - -# env vars passed through directly to Docker's build scripts -# to allow things like `make DOCKER_CLIENTONLY=1 binary` easily -# `docs/sources/contributing/devenvironment.md ` and `project/PACKAGERS.md` have some limited documentation of some of these -DOCKER_ENVS := \ - -e BUILDFLAGS \ - -e DOCKER_CLIENTONLY \ - -e DOCKER_EXECDRIVER \ - -e DOCKER_GRAPHDRIVER \ - -e TESTDIRS \ - -e TESTFLAGS \ - -e TIMEOUT -# note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds +.PHONY: all default docs docs-build docs-shell shell test # to allow `make DOCSDIR=1 docs-shell` (to create a bind mount in docs) DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR):/docs/content/compose) @@ -25,9 +12,8 @@ HUGO_BASE_URL=$(shell test -z "$(DOCKER_IP)" && echo localhost || echo "$(DOCKER HUGO_BIND_IP=0.0.0.0 GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) -DOCKER_IMAGE := docker$(if $(GIT_BRANCH),:$(GIT_BRANCH)) -DOCKER_DOCS_IMAGE := docs-base$(if $(GIT_BRANCH),:$(GIT_BRANCH)) - +GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g") +DOCKER_DOCS_IMAGE := docker-docs$(if $(GIT_BRANCH_CLEAN),:$(GIT_BRANCH_CLEAN)) DOCKER_RUN_DOCS := docker run --rm -it $(DOCS_MOUNT) -e AWS_S3_BUCKET -e NOCACHE @@ -42,14 +28,11 @@ docs: docs-build docs-draft: docs-build $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --buildDrafts="true" --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) - docs-shell: docs-build $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 "$(DOCKER_DOCS_IMAGE)" bash +test: docs-build + $(DOCKER_RUN_DOCS) "$(DOCKER_DOCS_IMAGE)" docs-build: -# ( git remote | grep -v upstream ) || git diff --name-status upstream/release..upstream/docs ./ > ./changed-files -# echo "$(GIT_BRANCH)" > GIT_BRANCH -# echo "$(AWS_S3_BUCKET)" > AWS_S3_BUCKET -# echo "$(GITCOMMIT)" > GITCOMMIT docker build -t "$(DOCKER_DOCS_IMAGE)" . diff --git a/docs/django.md b/docs/django.md index 6a222697e..b4bcee97e 100644 --- a/docs/django.md +++ b/docs/django.md @@ -29,8 +29,8 @@ and a `docker-compose.yml` file. The Dockerfile defines an application's image content via one or more build commands that configure that image. Once built, you can run the image in a container. For more information on `Dockerfiles`, see the [Docker user - guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) - and the [Dockerfile reference](https://docs.docker.com/engine/reference/builder/). + guide](/engine/userguide/containers/dockerimages.md#building-an-image-from-a-dockerfile) + and the [Dockerfile reference](/engine/reference/builder.md). 3. Add the following content to the `Dockerfile`. diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index ff944177b..8c706e4f0 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -77,7 +77,7 @@ dependencies the Python application requires, including Python itself. * Install the Python dependencies. * Set the default command for the container to `python app.py` - For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + For more information on how to write Dockerfiles, see the [Docker user guide](/engine/userguide/containers/dockerimages.md#building-an-image-from-a-dockerfile) and the [Dockerfile reference](/engine/reference/builder.md). 2. Build the image. diff --git a/docs/link-env-deprecated.md b/docs/link-env-deprecated.md index 55ba5f2d1..b1f01b3b6 100644 --- a/docs/link-env-deprecated.md +++ b/docs/link-env-deprecated.md @@ -16,7 +16,9 @@ weight=89 > > Environment variables will only be populated if you're using the [legacy version 1 Compose file format](compose-file.md#versioning). -Compose uses [Docker links] to expose services' containers to one another. Each linked container injects a set of environment variables, each of which begins with the uppercase name of the container. +Compose uses [Docker links](/engine/userguide/networking/default_network/dockerlinks.md) +to expose services' containers to one another. Each linked container injects a set of +environment variables, each of which begins with the uppercase name of the container. To see what environment variables are available to a service, run `docker-compose run SERVICE env`. @@ -38,8 +40,6 @@ Protocol (tcp or udp), e.g. `DB_PORT_5432_TCP_PROTO=tcp` name\_NAME
Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` -[Docker links]: https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/ - ## Related Information - [User guide](index.md) diff --git a/docs/overview.md b/docs/overview.md index 03ade3566..ef07a45be 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -159,8 +159,8 @@ and destroy isolated testing environments for your test suite. By defining the f Compose has traditionally been focused on development and testing workflows, but with each release we're making progress on more production-oriented features. You can use Compose to deploy to a remote Docker Engine. The Docker Engine may be a single instance provisioned with -[Docker Machine](https://docs.docker.com/machine/) or an entire -[Docker Swarm](https://docs.docker.com/swarm/) cluster. +[Docker Machine](/machine/overview.md) or an entire +[Docker Swarm](/swarm/overview.md) cluster. For details on using production-oriented features, see [compose in production](production.md) in this documentation. diff --git a/docs/production.md b/docs/production.md index 9acf64e56..cfb872936 100644 --- a/docs/production.md +++ b/docs/production.md @@ -65,7 +65,7 @@ recreating any services which `web` depends on. You can use Compose to deploy an app to a remote Docker host by setting the `DOCKER_HOST`, `DOCKER_TLS_VERIFY`, and `DOCKER_CERT_PATH` environment variables appropriately. For tasks like this, -[Docker Machine](/machine/overview) makes managing local and +[Docker Machine](/machine/overview.md) makes managing local and remote Docker hosts very easy, and is recommended even if you're not deploying remotely. @@ -74,7 +74,7 @@ commands will work with no further configuration. ### Running Compose on a Swarm cluster -[Docker Swarm](/swarm/overview), a Docker-native clustering +[Docker Swarm](/swarm/overview.md), a Docker-native clustering system, exposes the same API as a single Docker host, which means you can use Compose against a Swarm instance and run your apps across multiple hosts. diff --git a/docs/rails.md b/docs/rails.md index eef6b2f4b..f54d8286a 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -32,7 +32,7 @@ Dockerfile consists of: That'll put your application code inside an image that will build a container with Ruby, Bundler and all your dependencies inside it. For more information on -how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](https://docs.docker.com/engine/reference/builder/). +how to write Dockerfiles, see the [Docker user guide](/engine/userguide/containers/dockerimages.md#building-an-image-from-a-dockerfile) and the [Dockerfile reference](/engine/reference/builder.md). Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten in a moment by `rails new`. @@ -152,7 +152,7 @@ Finally, you need to create the database. In another terminal, run: $ docker-compose run web rake db:create -That's it. Your app should now be running on port 3000 on your Docker daemon. If you're using [Docker Machine](https://docs.docker.com/machine/), then `docker-machine ip MACHINE_VM` returns the Docker host IP address. +That's it. Your app should now be running on port 3000 on your Docker daemon. If you're using [Docker Machine](/machine/overview.md), then `docker-machine ip MACHINE_VM` returns the Docker host IP address. ![Rails example](images/rails-welcome.png) diff --git a/docs/swarm.md b/docs/swarm.md index ece721939..bbab69087 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -11,7 +11,7 @@ parent="workw_compose" # Using Compose with Swarm -Docker Compose and [Docker Swarm](/swarm/overview) aim to have full integration, meaning +Docker Compose and [Docker Swarm](/swarm/overview.md) aim to have full integration, meaning you can point a Compose app at a Swarm cluster and have it all just work as if you were using a single Docker host. @@ -30,7 +30,7 @@ format](compose-file.md#versioning) you are using: or a custom driver which supports multi-host networking. Read [Get started with multi-host networking](https://docs.docker.com/engine/userguide/networking/get-started-overlay/) to see how to -set up a Swarm cluster with [Docker Machine](/machine/overview) and the overlay driver. Once you've got it running, deploying your app to it should be as simple as: +set up a Swarm cluster with [Docker Machine](/machine/overview.md) and the overlay driver. Once you've got it running, deploying your app to it should be as simple as: $ eval "$(docker-machine env --swarm )" $ docker-compose up From ea640f38217e5d3796bbca49a5a1870582139d8d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 17 May 2016 16:32:54 -0700 Subject: [PATCH 89/95] Remove external_name from serialized config output Signed-off-by: Joffrey F --- compose/config/serialize.py | 6 +++++- tests/acceptance/cli_test.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 1b498c016..52de77b83 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -27,11 +27,15 @@ def serialize_config(config): service_dict.pop('name'): service_dict for service_dict in denormalized_services } + networks = config.networks.copy() + for net_name, net_conf in networks.items(): + if 'external_name' in net_conf: + del net_conf['external_name'] output = { 'version': V2_0, 'services': services, - 'networks': config.networks, + 'networks': networks, 'volumes': config.volumes, } diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a02d0e99e..6bb111ef9 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -224,6 +224,20 @@ class CLITestCase(DockerClientTestCase): 'volumes': {}, } + def test_config_external_network(self): + self.base_dir = 'tests/fixtures/networks' + result = self.dispatch(['-f', 'external-networks.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'networks' in json_result + assert json_result['networks'] == { + 'networks_foo': { + 'external': True # {'name': 'networks_foo'} + }, + 'bar': { + 'external': {'name': 'networks_bar'} + } + } + def test_config_v1(self): self.base_dir = 'tests/fixtures/v1-config' result = self.dispatch(['config']) From 61324ef30839bdcf99e20e0de2a4bb029e189166 Mon Sep 17 00:00:00 2001 From: Sander Maijers Date: Fri, 10 Jun 2016 16:30:46 +0200 Subject: [PATCH 90/95] Fix byte/str typing error Signed-off-by: Sander Maijers --- compose/cli/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index b58b50ef9..cc2b680de 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -49,7 +49,7 @@ def input(prompt): """ sys.stdout.write(prompt) sys.stdout.flush() - return sys.stdin.readline().rstrip(b'\n') + return sys.stdin.readline().rstrip('\n') def call_silently(*args, **kwargs): From 60f7e021ada69b4bdfce397eb2153c6c35eb2428 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 13 Jun 2016 15:32:10 -0700 Subject: [PATCH 91/95] Fix split_path_mapping behavior when mounting "/" Signed-off-by: Joffrey F --- compose/config/config.py | 7 ++++--- tests/unit/config/config_test.py | 7 +++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 97c427b91..7a2b3d366 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -940,9 +940,10 @@ def split_path_mapping(volume_path): path. Using splitdrive so windows absolute paths won't cause issues with splitting on ':'. """ - # splitdrive has limitations when it comes to relative paths, so when it's - # relative, handle special case to set the drive to '' - if volume_path.startswith('.') or volume_path.startswith('~'): + # splitdrive is very naive, so handle special cases where we can be sure + # the first character is not a drive. + if (volume_path.startswith('.') or volume_path.startswith('~') or + volume_path.startswith('/')): drive, volume_config = '', volume_path else: drive, volume_config = ntpath.splitdrive(volume_path) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 24ece4994..89c424a4c 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2682,6 +2682,13 @@ class VolumePathTest(unittest.TestCase): mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) assert mapping == expected_mapping + def test_split_path_mapping_with_root_mount(self): + host_path = '/' + container_path = '/var/hostroot' + expected_mapping = (container_path, host_path) + mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) + assert mapping == expected_mapping + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class BuildPathTest(unittest.TestCase): From 1ea9dda1d3b1db1d2bcb248b4e4eb57a26a06fd4 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 10 May 2016 16:14:54 +0100 Subject: [PATCH 92/95] Implement 'docker-compose push' and 'docker-compose bundle' Signed-off-by: Aanand Prasad --- .dockerignore | 1 + compose/bundle.py | 186 ++++++++++++++++++++++++++++++++++++ compose/cli/command.py | 10 ++ compose/cli/main.py | 60 +++++++++--- compose/config/serialize.py | 8 +- compose/progress_stream.py | 19 ++++ compose/project.py | 4 + compose/service.py | 28 ++++-- 8 files changed, 296 insertions(+), 20 deletions(-) create mode 100644 compose/bundle.py diff --git a/.dockerignore b/.dockerignore index 055ae7ed1..e79da8620 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,4 @@ coverage-html docs/_site venv .tox +dist diff --git a/compose/bundle.py b/compose/bundle.py new file mode 100644 index 000000000..a6d0d2d13 --- /dev/null +++ b/compose/bundle.py @@ -0,0 +1,186 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import json +import logging + +import six +from docker.utils.ports import split_port + +from .cli.errors import UserError +from .config.serialize import denormalize_config +from .network import get_network_defs_for_service +from .service import NoSuchImageError +from .service import parse_repository_tag + + +log = logging.getLogger(__name__) + + +SERVICE_KEYS = { + 'command': 'Command', + 'environment': 'Env', + 'working_dir': 'WorkingDir', +} + + +VERSION = '0.1' + + +def serialize_bundle(config, image_digests): + if config.networks: + log.warn("Unsupported top level key 'networks' - ignoring") + + if config.volumes: + log.warn("Unsupported top level key 'volumes' - ignoring") + + return json.dumps( + to_bundle(config, image_digests), + indent=2, + sort_keys=True, + ) + + +def get_image_digests(project): + return { + service.name: get_image_digest(service) + for service in project.services + } + + +def get_image_digest(service): + if 'image' not in service.options: + raise UserError( + "Service '{s.name}' doesn't define an image tag. An image name is " + "required to generate a proper image digest for the bundle. Specify " + "an image repo and tag with the 'image' option.".format(s=service)) + + repo, tag, separator = parse_repository_tag(service.options['image']) + # Compose file already uses a digest, no lookup required + if separator == '@': + return service.options['image'] + + try: + image = service.image() + except NoSuchImageError: + action = 'build' if 'build' in service.options else 'pull' + raise UserError( + "Image not found for service '{service}'. " + "You might need to run `docker-compose {action} {service}`." + .format(service=service.name, action=action)) + + if image['RepoDigests']: + # TODO: pick a digest based on the image tag if there are multiple + # digests + return image['RepoDigests'][0] + + if 'build' not in service.options: + log.warn( + "Compose needs to pull the image for '{s.name}' in order to create " + "a bundle. This may result in a more recent image being used. " + "It is recommended that you use an image tagged with a " + "specific version to minimize the potential " + "differences.".format(s=service)) + digest = service.pull() + else: + try: + digest = service.push() + except: + log.error( + "Failed to push image for service '{s.name}'. Please use an " + "image tag that can be pushed to a Docker " + "registry.".format(s=service)) + raise + + if not digest: + raise ValueError("Failed to get digest for %s" % service.name) + + identifier = '{repo}@{digest}'.format(repo=repo, digest=digest) + + # Pull by digest so that image['RepoDigests'] is populated for next time + # and we don't have to pull/push again + service.client.pull(identifier) + + return identifier + + +def to_bundle(config, image_digests): + config = denormalize_config(config) + + return { + 'version': VERSION, + 'services': { + name: convert_service_to_bundle( + name, + service_dict, + image_digests[name], + ) + for name, service_dict in config['services'].items() + }, + } + + +def convert_service_to_bundle(name, service_dict, image_id): + container_config = {'Image': image_id} + + for key, value in service_dict.items(): + if key in ('build', 'image', 'ports', 'expose', 'networks'): + pass + elif key == 'environment': + container_config['env'] = { + envkey: envvalue for envkey, envvalue in value.items() + if envvalue + } + elif key in SERVICE_KEYS: + container_config[SERVICE_KEYS[key]] = value + else: + log.warn("Unsupported key '{}' in services.{} - ignoring".format(key, name)) + + container_config['Networks'] = make_service_networks(name, service_dict) + + ports = make_port_specs(service_dict) + if ports: + container_config['Ports'] = ports + + return container_config + + +def make_service_networks(name, service_dict): + networks = [] + + for network_name, network_def in get_network_defs_for_service(service_dict).items(): + for key in network_def.keys(): + log.warn( + "Unsupported key '{}' in services.{}.networks.{} - ignoring" + .format(key, name, network_name)) + + networks.append(network_name) + + return networks + + +def make_port_specs(service_dict): + ports = [] + + internal_ports = [ + internal_port + for port_def in service_dict.get('ports', []) + for internal_port in split_port(port_def)[0] + ] + + internal_ports += service_dict.get('expose', []) + + for internal_port in internal_ports: + spec = make_port_spec(internal_port) + if spec not in ports: + ports.append(spec) + + return ports + + +def make_port_spec(value): + components = six.text_type(value).partition('/') + return { + 'Protocol': components[2] or 'tcp', + 'Port': int(components[0]), + } diff --git a/compose/cli/command.py b/compose/cli/command.py index 8ac3aff4f..44112fce6 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -35,6 +35,16 @@ def project_from_options(project_dir, options): ) +def get_config_from_options(base_dir, options): + environment = Environment.from_env_file(base_dir) + config_path = get_config_path_from_options( + base_dir, options, environment + ) + return config.load( + config.find(base_dir, config_path, environment) + ) + + def get_config_path_from_options(base_dir, options, environment): file_option = options.get('--file') if file_option: diff --git a/compose/cli/main.py b/compose/cli/main.py index afde71506..3e4404630 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -14,10 +14,10 @@ from operator import attrgetter from . import errors from . import signals from .. import __version__ -from ..config import config +from ..bundle import get_image_digests +from ..bundle import serialize_bundle from ..config import ConfigurationError from ..config import parse_environment -from ..config.environment import Environment from ..config.serialize import serialize_config from ..const import DEFAULT_TIMEOUT from ..const import IS_WINDOWS_PLATFORM @@ -30,7 +30,7 @@ from ..service import BuildError from ..service import ConvergenceStrategy from ..service import ImageType from ..service import NeedsBuildError -from .command import get_config_path_from_options +from .command import get_config_from_options from .command import project_from_options from .docopt_command import DocoptDispatcher from .docopt_command import get_handler @@ -98,7 +98,7 @@ def perform_command(options, handler, command_options): handler(command_options) return - if options['COMMAND'] == 'config': + if options['COMMAND'] in ('config', 'bundle'): command = TopLevelCommand(None) handler(command, options, command_options) return @@ -164,6 +164,7 @@ class TopLevelCommand(object): Commands: build Build or rebuild services + bundle Generate a Docker bundle from the Compose file config Validate and view the compose file create Create services down Stop and remove containers, networks, images, and volumes @@ -176,6 +177,7 @@ class TopLevelCommand(object): port Print the public port for a port binding ps List containers pull Pulls service images + push Push service images restart Restart services rm Remove stopped containers run Run a one-off command @@ -212,6 +214,34 @@ class TopLevelCommand(object): pull=bool(options.get('--pull', False)), force_rm=bool(options.get('--force-rm', False))) + def bundle(self, config_options, options): + """ + Generate a Docker bundle from the Compose file. + + Local images will be pushed to a Docker registry, and remote images + will be pulled to fetch an image digest. + + Usage: bundle [options] + + Options: + -o, --output PATH Path to write the bundle file to. + Defaults to ".dsb". + """ + self.project = project_from_options('.', config_options) + compose_config = get_config_from_options(self.project_dir, config_options) + + output = options["--output"] + if not output: + output = "{}.dsb".format(self.project.name) + + with errors.handle_connection_errors(self.project.client): + image_digests = get_image_digests(self.project) + + with open(output, 'w') as f: + f.write(serialize_bundle(compose_config, image_digests)) + + log.info("Wrote bundle to {}".format(output)) + def config(self, config_options, options): """ Validate and view the compose file. @@ -224,13 +254,7 @@ class TopLevelCommand(object): --services Print the service names, one per line. """ - environment = Environment.from_env_file(self.project_dir) - config_path = get_config_path_from_options( - self.project_dir, config_options, environment - ) - compose_config = config.load( - config.find(self.project_dir, config_path, environment) - ) + compose_config = get_config_from_options(self.project_dir, config_options) if options['--quiet']: return @@ -518,6 +542,20 @@ class TopLevelCommand(object): ignore_pull_failures=options.get('--ignore-pull-failures') ) + def push(self, options): + """ + Pushes images for services. + + Usage: push [options] [SERVICE...] + + Options: + --ignore-push-failures Push what it can and ignores images with push failures. + """ + self.project.push( + service_names=options['SERVICE'], + ignore_push_failures=options.get('--ignore-push-failures') + ) + def rm(self, options): """ Removes stopped service containers. diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 52de77b83..b788a55de 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -18,7 +18,7 @@ yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) -def serialize_config(config): +def denormalize_config(config): denormalized_services = [ denormalize_service_dict(service_dict, config.version) for service_dict in config.services @@ -32,15 +32,17 @@ def serialize_config(config): if 'external_name' in net_conf: del net_conf['external_name'] - output = { + return { 'version': V2_0, 'services': services, 'networks': networks, 'volumes': config.volumes, } + +def serialize_config(config): return yaml.safe_dump( - output, + denormalize_config(config), default_flow_style=False, indent=2, width=80) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 1f873d1d9..a0f5601f1 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -91,3 +91,22 @@ def print_output_event(event, stream, is_terminal): stream.write("%s%s" % (event['stream'], terminator)) else: stream.write("%s%s\n" % (status, terminator)) + + +def get_digest_from_pull(events): + for event in events: + status = event.get('status') + if not status or 'Digest' not in status: + continue + + _, digest = status.split(':', 1) + return digest.strip() + return None + + +def get_digest_from_push(events): + for event in events: + digest = event.get('aux', {}).get('Digest') + if digest: + return digest + return None diff --git a/compose/project.py b/compose/project.py index 1b7fde23b..676b6ae8c 100644 --- a/compose/project.py +++ b/compose/project.py @@ -440,6 +440,10 @@ class Project(object): for service in self.get_services(service_names, include_deps=False): service.pull(ignore_pull_failures) + def push(self, service_names=None, ignore_push_failures=False): + for service in self.get_services(service_names, include_deps=False): + service.push(ignore_push_failures) + def _labeled_containers(self, stopped=False, one_off=OneOffFilter.exclude): return list(filter(None, [ Container.from_ps(self.client, container) diff --git a/compose/service.py b/compose/service.py index 8b9f64f0f..af572e5b5 100644 --- a/compose/service.py +++ b/compose/service.py @@ -15,6 +15,7 @@ from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port from . import __version__ +from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment from .config.types import VolumeSpec @@ -806,20 +807,35 @@ class Service(object): repo, tag, separator = parse_repository_tag(self.options['image']) tag = tag or 'latest' log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag)) - output = self.client.pull( - repo, - tag=tag, - stream=True, - ) + output = self.client.pull(repo, tag=tag, stream=True) try: - stream_output(output, sys.stdout) + return progress_stream.get_digest_from_pull( + stream_output(output, sys.stdout)) except StreamOutputError as e: if not ignore_pull_failures: raise else: log.error(six.text_type(e)) + def push(self, ignore_push_failures=False): + if 'image' not in self.options or 'build' not in self.options: + return + + repo, tag, separator = parse_repository_tag(self.options['image']) + tag = tag or 'latest' + log.info('Pushing %s (%s%s%s)...' % (self.name, repo, separator, tag)) + output = self.client.push(repo, tag=tag, stream=True) + + try: + return progress_stream.get_digest_from_push( + stream_output(output, sys.stdout)) + except StreamOutputError as e: + if not ignore_push_failures: + raise + else: + log.error(six.text_type(e)) + def short_id_alias_exists(container, network): aliases = container.get( From 9b7bd69cfca3f957f11a8f309ca816f13b52c436 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 14 Jun 2016 12:55:29 -0400 Subject: [PATCH 93/95] Support entrypoint, labels, and user in the bundle. Signed-off-by: Daniel Nephin --- .dockerignore | 1 - compose/bundle.py | 64 +++++++++++++++++++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/.dockerignore b/.dockerignore index e79da8620..055ae7ed1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,4 +7,3 @@ coverage-html docs/_site venv .tox -dist diff --git a/compose/bundle.py b/compose/bundle.py index a6d0d2d13..e93c5bd9c 100644 --- a/compose/bundle.py +++ b/compose/bundle.py @@ -5,11 +5,13 @@ import json import logging import six +from docker.utils import split_command from docker.utils.ports import split_port from .cli.errors import UserError from .config.serialize import denormalize_config from .network import get_network_defs_for_service +from .service import format_environment from .service import NoSuchImageError from .service import parse_repository_tag @@ -18,11 +20,22 @@ log = logging.getLogger(__name__) SERVICE_KEYS = { - 'command': 'Command', - 'environment': 'Env', 'working_dir': 'WorkingDir', + 'user': 'User', + 'labels': 'Labels', } +IGNORED_KEYS = {'build'} + +SUPPORTED_KEYS = { + 'image', + 'ports', + 'expose', + 'networks', + 'command', + 'environment', + 'entrypoint', +} | set(SERVICE_KEYS) VERSION = '0.1' @@ -120,22 +133,32 @@ def to_bundle(config, image_digests): } -def convert_service_to_bundle(name, service_dict, image_id): - container_config = {'Image': image_id} +def convert_service_to_bundle(name, service_dict, image_digest): + container_config = {'Image': image_digest} for key, value in service_dict.items(): - if key in ('build', 'image', 'ports', 'expose', 'networks'): - pass - elif key == 'environment': - container_config['env'] = { + if key in IGNORED_KEYS: + continue + + if key not in SUPPORTED_KEYS: + log.warn("Unsupported key '{}' in services.{} - ignoring".format(key, name)) + continue + + if key == 'environment': + container_config['Env'] = format_environment({ envkey: envvalue for envkey, envvalue in value.items() if envvalue - } - elif key in SERVICE_KEYS: - container_config[SERVICE_KEYS[key]] = value - else: - log.warn("Unsupported key '{}' in services.{} - ignoring".format(key, name)) + }) + continue + if key in SERVICE_KEYS: + container_config[SERVICE_KEYS[key]] = value + continue + + set_command_and_args( + container_config, + service_dict.get('entrypoint', []), + service_dict.get('command', [])) container_config['Networks'] = make_service_networks(name, service_dict) ports = make_port_specs(service_dict) @@ -145,6 +168,21 @@ def convert_service_to_bundle(name, service_dict, image_id): return container_config +# See https://github.com/docker/swarmkit/blob//agent/exec/container/container.go#L95 +def set_command_and_args(config, entrypoint, command): + if isinstance(entrypoint, six.string_types): + entrypoint = split_command(entrypoint) + if isinstance(command, six.string_types): + command = split_command(command) + + if entrypoint: + config['Command'] = entrypoint + command + return + + if command: + config['Args'] = command + + def make_service_networks(name, service_dict): networks = [] From 80afbd3961800532d184a5b04b57a2f411bdb8c3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 14 Jun 2016 12:23:04 -0700 Subject: [PATCH 94/95] Skip TLS version test if TLSv1_2 is not available on platform Signed-off-by: Joffrey F --- tests/unit/cli/command_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/cli/command_test.py b/tests/unit/cli/command_test.py index 28adff3f3..50fc84e17 100644 --- a/tests/unit/cli/command_test.py +++ b/tests/unit/cli/command_test.py @@ -55,6 +55,7 @@ class TestGetTlsVersion(object): environment = {} assert get_tls_version(environment) is None + @pytest.mark.skipif(not hasattr(ssl, 'PROTOCOL_TLSv1_2'), reason='TLS v1.2 unsupported') def test_get_tls_version_upgrade(self): environment = {'COMPOSE_TLS_VERSION': 'TLSv1_2'} assert get_tls_version(environment) == ssl.PROTOCOL_TLSv1_2 From 9bf6bc6dbdb42c61defe97b98ec83f5f6ba63b98 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 14 Jun 2016 11:33:45 -0700 Subject: [PATCH 95/95] Bump 1.8.0-rc1 Signed-off-by: Joffrey F --- CHANGELOG.md | 91 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- docs/install.md | 6 +-- script/run/run.sh | 2 +- 4 files changed, 96 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ee45386a..39ac86982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,97 @@ Change log ========== +1.8.0 (2016-06-14) +----------------- + +New Features + +- Added `docker-compose bundle`, a command that builds a bundle file + to be consumed by the new *Docker Stack* commands in Docker 1.12. + This command automatically pushes and pulls images as needed. + +- Added `docker-compose push`, a command that pushes service images + to a registry. + +- As announced in 1.7.0, `docker-compose rm` now removes containers + created by `docker-compose run` by default. + +- Compose now supports specifying a custom TLS version for + interaction with the Docker Engine using the `COMPOSE_TLS_VERSION` + environment variable. + +Bug Fixes + +- Fixed a bug where Compose would erroneously try to read `.env` + at the project's root when it is a directory. + +- Improved config merging when multiple compose files are involved + for several service sub-keys. + +- Fixed a bug where volume mappings containing Windows drives would + sometimes be parsed incorrectly. + +- Fixed a bug in Windows environment where volume mappings of the + host's root directory would be parsed incorrectly. + +- Fixed a bug where `docker-compose config` would ouput an invalid + Compose file if external networks were specified. + +- Fixed an issue where unset buildargs would be assigned a string + containing `'None'` instead of the expected empty value. + +- Fixed a bug where yes/no prompts on Windows would not show before + receiving input. + +- Fixed a bug where trying to `docker-compose exec` on Windows + without the `-d` option would exit with a stacktrace. This will + still fail for the time being, but should do so gracefully. + +- Fixed a bug where errors during `docker-compose up` would show + an unrelated stacktrace at the end of the process. + + +1.7.1 (2016-05-04) +----------------- + +Bug Fixes + +- Fixed a bug where the output of `docker-compose config` for v1 files + would be an invalid configuration file. + +- Fixed a bug where `docker-compose config` would not check the validity + of links. + +- Fixed an issue where `docker-compose help` would not output a list of + available commands and generic options as expected. + +- Fixed an issue where filtering by service when using `docker-compose logs` + would not apply for newly created services. + +- Fixed a bug where unchanged services would sometimes be recreated in + in the up phase when using Compose with Python 3. + +- Fixed an issue where API errors encountered during the up phase would + not be recognized as a failure state by Compose. + +- Fixed a bug where Compose would raise a NameError because of an undefined + exception name on non-Windows platforms. + +- Fixed a bug where the wrong version of `docker-py` would sometimes be + installed alongside Compose. + +- Fixed a bug where the host value output by `docker-machine config default` + would not be recognized as valid options by the `docker-compose` + command line. + +- Fixed an issue where Compose would sometimes exit unexpectedly while + reading events broadcasted by a Swarm cluster. + +- Corrected a statement in the docs about the location of the `.env` file, + which is indeed read from the current directory, instead of in the same + location as the Compose file. + + 1.7.0 (2016-04-13) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index 1052c0670..1dd11e791 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.8.0dev' +__version__ = '1.8.0-rc1' diff --git a/docs/install.md b/docs/install.md index 95416e7ae..5191a4b58 100644 --- a/docs/install.md +++ b/docs/install.md @@ -39,7 +39,7 @@ which the release page specifies, in your terminal. The following is an example command illustrating the format: - curl -L https://github.com/docker/compose/releases/download/1.6.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.8.0-rc1/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose If you have problems installing with `curl`, see [Alternative Install Options](#alternative-install-options). @@ -54,7 +54,7 @@ which the release page specifies, in your terminal. 7. Test the installation. $ docker-compose --version - docker-compose version: 1.6.2 + docker-compose version: 1.8.0-rc1 ## Alternative install options @@ -77,7 +77,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.6.2/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.8.0-rc1/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/script/run/run.sh b/script/run/run.sh index 98d32c5f8..f9199ce15 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.7.0" +VERSION="1.8.0-rc1" IMAGE="docker/compose:$VERSION"