Merge pull request #4368 from dnephin/secrets-using-bind-mounts

Secrets using bind mounts

compose/config/config.py

@@ -12,10 +12,12 @@ import six
 import yaml
 from cached_property import cached_property
 
+from . import types
 from ..const import COMPOSEFILE_V1 as V1
 from ..const import COMPOSEFILE_V2_0 as V2_0
 from ..const import COMPOSEFILE_V2_1 as V2_1
 from ..const import COMPOSEFILE_V3_0 as V3_0
+from ..const import COMPOSEFILE_V3_1 as V3_1
 from ..utils import build_string_dict
 from ..utils import parse_nanoseconds_int
 from ..utils import splitdrive
@@ -82,6 +84,7 @@ DOCKER_CONFIG_KEYS = [
     'privileged',
     'read_only',
     'restart',
+    'secrets',
     'security_opt',
     'shm_size',
     'stdin_open',
@@ -202,8 +205,11 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
     def get_networks(self):
         return {} if self.version == V1 else self.config.get('networks', {})
 
+    def get_secrets(self):
+        return {} if self.version < V3_1 else self.config.get('secrets', {})
 
-class Config(namedtuple('_Config', 'version services volumes networks')):
+
+class Config(namedtuple('_Config', 'version services volumes networks secrets')):
     """
     :param version: configuration version
     :type  version: int
@@ -328,6 +334,7 @@ def load(config_details):
     networks = load_mapping(
         config_details.config_files, 'get_networks', 'Network'
     )
+    secrets = load_secrets(config_details.config_files, config_details.working_dir)
     service_dicts = load_services(config_details, main_file)
 
     if main_file.version != V1:
@@ -342,7 +349,7 @@ def load(config_details):
             "`docker stack deploy` to deploy to a swarm."
             .format(", ".join(sorted(s['name'] for s in services_using_deploy))))
 
-    return Config(main_file.version, service_dicts, volumes, networks)
+    return Config(main_file.version, service_dicts, volumes, networks, secrets)
 
 
 def load_mapping(config_files, get_func, entity_type):
@@ -356,22 +363,12 @@ def load_mapping(config_files, get_func, entity_type):
 
             external = config.get('external')
             if external:
-                if len(config.keys()) > 1:
+                validate_external(entity_type, name, config)
-                    raise ConfigurationError(
-                        '{} {} declared as external but specifies'
-                        ' additional attributes ({}). '.format(
-                            entity_type,
-                            name,
-                            ', '.join([k for k in config.keys() if k != 'external'])
-                        )
-                    )
                 if isinstance(external, dict):
                     config['external_name'] = external.get('name')
                 else:
                     config['external_name'] = name
 
-            mapping[name] = config
-
             if 'driver_opts' in config:
                 config['driver_opts'] = build_string_dict(
                     config['driver_opts']
@@ -383,6 +380,39 @@ def load_mapping(config_files, get_func, entity_type):
     return mapping
 
 
+def validate_external(entity_type, name, config):
+    if len(config.keys()) <= 1:
+        return
+
+    raise ConfigurationError(
+        "{} {} declared as external but specifies additional attributes "
+        "({}).".format(
+            entity_type, name, ', '.join(k for k in config if k != 'external')))
+
+
+def load_secrets(config_files, working_dir):
+    mapping = {}
+
+    for config_file in config_files:
+        for name, config in config_file.get_secrets().items():
+            mapping[name] = config or {}
+            if not config:
+                continue
+
+            external = config.get('external')
+            if external:
+                validate_external('Secret', name, config)
+                if isinstance(external, dict):
+                    config['external_name'] = external.get('name')
+                else:
+                    config['external_name'] = name
+
+            if 'file' in config:
+                config['file'] = expand_path(working_dir, config['file'])
+
+    return mapping
+
+
 def load_services(config_details, config_file):
     def build_service(service_name, service_dict, service_names):
         service_config = ServiceConfig.with_abs_paths(
@@ -820,6 +850,7 @@ def merge_service_dicts(base, override, version):
     md.merge_mapping('sysctls', parse_sysctls)
     md.merge_mapping('depends_on', parse_depends_on)
     md.merge_sequence('links', ServiceLink.parse)
+    md.merge_sequence('secrets', types.ServiceSecret.parse)
 
     for field in ['volumes', 'devices']:
         md.merge_field(field, merge_path_mappings)

compose/config/config_schema_v3.1.json

@@ -0,0 +1,428 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "id": "config_schema_v3.1.json",
+  "type": "object",
+  "required": ["version"],
+
+  "properties": {
+    "version": {
+      "type": "string"
+    },
+
+    "services": {
+      "id": "#/properties/services",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/service"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "networks": {
+      "id": "#/properties/networks",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/network"
+        }
+      }
+    },
+
+    "volumes": {
+      "id": "#/properties/volumes",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/volume"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "secrets": {
+      "id": "#/properties/secrets",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/secret"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+
+  "additionalProperties": false,
+
+  "definitions": {
+
+    "service": {
+      "id": "#/definitions/service",
+      "type": "object",
+
+      "properties": {
+        "deploy": {"$ref": "#/definitions/deployment"},
+        "build": {
+          "oneOf": [
+            {"type": "string"},
+            {
+              "type": "object",
+              "properties": {
+                "context": {"type": "string"},
+                "dockerfile": {"type": "string"},
+                "args": {"$ref": "#/definitions/list_or_dict"}
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "cgroup_parent": {"type": "string"},
+        "command": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "container_name": {"type": "string"},
+        "depends_on": {"$ref": "#/definitions/list_of_strings"},
+        "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "dns": {"$ref": "#/definitions/string_or_list"},
+        "dns_search": {"$ref": "#/definitions/string_or_list"},
+        "domainname": {"type": "string"},
+        "entrypoint": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "env_file": {"$ref": "#/definitions/string_or_list"},
+        "environment": {"$ref": "#/definitions/list_or_dict"},
+
+        "expose": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"],
+            "format": "expose"
+          },
+          "uniqueItems": true
+        },
+
+        "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "extra_hosts": {"$ref": "#/definitions/list_or_dict"},
+        "healthcheck": {"$ref": "#/definitions/healthcheck"},
+        "hostname": {"type": "string"},
+        "image": {"type": "string"},
+        "ipc": {"type": "string"},
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+
+        "logging": {
+            "type": "object",
+
+            "properties": {
+                "driver": {"type": "string"},
+                "options": {
+                  "type": "object",
+                  "patternProperties": {
+                    "^.+$": {"type": ["string", "number", "null"]}
+                  }
+                }
+            },
+            "additionalProperties": false
+        },
+
+        "mac_address": {"type": "string"},
+        "network_mode": {"type": "string"},
+
+        "networks": {
+          "oneOf": [
+            {"$ref": "#/definitions/list_of_strings"},
+            {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9._-]+$": {
+                  "oneOf": [
+                    {
+                      "type": "object",
+                      "properties": {
+                        "aliases": {"$ref": "#/definitions/list_of_strings"},
+                        "ipv4_address": {"type": "string"},
+                        "ipv6_address": {"type": "string"}
+                      },
+                      "additionalProperties": false
+                    },
+                    {"type": "null"}
+                  ]
+                }
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "pid": {"type": ["string", "null"]},
+
+        "ports": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"],
+            "format": "ports"
+          },
+          "uniqueItems": true
+        },
+
+        "privileged": {"type": "boolean"},
+        "read_only": {"type": "boolean"},
+        "restart": {"type": "string"},
+        "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "shm_size": {"type": ["number", "string"]},
+        "secrets": {
+          "type": "array",
+          "items": {
+            "oneOf": [
+              {"type": "string"},
+              {
+                "type": "object",
+                "properties": {
+                  "source": {"type": "string"},
+                  "target": {"type": "string"},
+                  "uid": {"type": "string"},
+                  "gid": {"type": "string"},
+                  "mode": {"type": "number"}
+                }
+              }
+            ]
+          }
+        },
+        "sysctls": {"$ref": "#/definitions/list_or_dict"},
+        "stdin_open": {"type": "boolean"},
+        "stop_grace_period": {"type": "string", "format": "duration"},
+        "stop_signal": {"type": "string"},
+        "tmpfs": {"$ref": "#/definitions/string_or_list"},
+        "tty": {"type": "boolean"},
+        "ulimits": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-z]+$": {
+              "oneOf": [
+                {"type": "integer"},
+                {
+                  "type":"object",
+                  "properties": {
+                    "hard": {"type": "integer"},
+                    "soft": {"type": "integer"}
+                  },
+                  "required": ["soft", "hard"],
+                  "additionalProperties": false
+                }
+              ]
+            }
+          }
+        },
+        "user": {"type": "string"},
+        "userns_mode": {"type": "string"},
+        "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "working_dir": {"type": "string"}
+      },
+      "additionalProperties": false
+    },
+
+    "healthcheck": {
+      "id": "#/definitions/healthcheck",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "disable": {"type": "boolean"},
+        "interval": {"type": "string"},
+        "retries": {"type": "number"},
+        "test": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "timeout": {"type": "string"}
+      }
+    },
+    "deployment": {
+      "id": "#/definitions/deployment",
+      "type": ["object", "null"],
+      "properties": {
+        "mode": {"type": "string"},
+        "replicas": {"type": "integer"},
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "update_config": {
+          "type": "object",
+          "properties": {
+            "parallelism": {"type": "integer"},
+            "delay": {"type": "string", "format": "duration"},
+            "failure_action": {"type": "string"},
+            "monitor": {"type": "string", "format": "duration"},
+            "max_failure_ratio": {"type": "number"}
+          },
+          "additionalProperties": false
+        },
+        "resources": {
+          "type": "object",
+          "properties": {
+            "limits": {"$ref": "#/definitions/resource"},
+            "reservations": {"$ref": "#/definitions/resource"}
+          }
+        },
+        "restart_policy": {
+          "type": "object",
+          "properties": {
+            "condition": {"type": "string"},
+            "delay": {"type": "string", "format": "duration"},
+            "max_attempts": {"type": "integer"},
+            "window": {"type": "string", "format": "duration"}
+          },
+          "additionalProperties": false
+        },
+        "placement": {
+          "type": "object",
+          "properties": {
+            "constraints": {"type": "array", "items": {"type": "string"}}
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "resource": {
+      "id": "#/definitions/resource",
+      "type": "object",
+      "properties": {
+        "cpus": {"type": "string"},
+        "memory": {"type": "string"}
+      },
+      "additionalProperties": false
+    },
+
+    "network": {
+      "id": "#/definitions/network",
+      "type": ["object", "null"],
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "ipam": {
+          "type": "object",
+          "properties": {
+            "driver": {"type": "string"},
+            "config": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "subnet": {"type": "string"}
+                },
+                "additionalProperties": false
+              }
+            }
+          },
+          "additionalProperties": false
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          },
+          "additionalProperties": false
+        },
+        "internal": {"type": "boolean"},
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false
+    },
+
+    "volume": {
+      "id": "#/definitions/volume",
+      "type": ["object", "null"],
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          },
+          "additionalProperties": false
+        },
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false
+    },
+
+    "secret": {
+      "id": "#/definitions/secret",
+      "type": "object",
+      "properties": {
+        "file": {"type": "string"},
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          }
+        },
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false
+    },
+
+    "string_or_list": {
+      "oneOf": [
+        {"type": "string"},
+        {"$ref": "#/definitions/list_of_strings"}
+      ]
+    },
+
+    "list_of_strings": {
+      "type": "array",
+      "items": {"type": "string"},
+      "uniqueItems": true
+    },
+
+    "list_or_dict": {
+      "oneOf": [
+        {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": ["string", "number", "null"]
+            }
+          },
+          "additionalProperties": false
+        },
+        {"type": "array", "items": {"type": "string"}, "uniqueItems": true}
+      ]
+    },
+
+    "constraints": {
+      "service": {
+        "id": "#/definitions/constraints/service",
+        "anyOf": [
+          {"required": ["build"]},
+          {"required": ["image"]}
+        ],
+        "properties": {
+          "build": {
+            "required": ["context"]
+          }
+        }
+      }
+    }
+  }
+}

compose/config/types.py

@@ -10,8 +10,8 @@ from collections import namedtuple
 
 import six
 
-from compose.config.config import V1
+from ..const import COMPOSEFILE_V1 as V1
-from compose.config.errors import ConfigurationError
+from .errors import ConfigurationError
 from compose.const import IS_WINDOWS_PLATFORM
 from compose.utils import splitdrive
 
@@ -234,3 +234,22 @@ class ServiceLink(namedtuple('_ServiceLink', 'target alias')):
     @property
     def merge_field(self):
         return self.alias
+
+
+class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')):
+
+    @classmethod
+    def parse(cls, spec):
+        if isinstance(spec, six.string_types):
+            return cls(spec, None, None, None, None)
+        return cls(
+            spec.get('source'),
+            spec.get('target'),
+            spec.get('uid'),
+            spec.get('gid'),
+            spec.get('mode'),
+        )
+
+    @property
+    def merge_field(self):
+        return self.source

compose/const.py

@@ -16,16 +16,20 @@ LABEL_VERSION = 'com.docker.compose.version'
 LABEL_VOLUME = 'com.docker.compose.volume'
 LABEL_CONFIG_HASH = 'com.docker.compose.config-hash'
 
+SECRETS_PATH = '/run/secrets'
+
 COMPOSEFILE_V1 = '1'
 COMPOSEFILE_V2_0 = '2.0'
 COMPOSEFILE_V2_1 = '2.1'
 COMPOSEFILE_V3_0 = '3.0'
+COMPOSEFILE_V3_1 = '3.1'
 
 API_VERSIONS = {
     COMPOSEFILE_V1: '1.21',
     COMPOSEFILE_V2_0: '1.22',
     COMPOSEFILE_V2_1: '1.24',
     COMPOSEFILE_V3_0: '1.25',
+    COMPOSEFILE_V3_1: '1.25',
 }
 
 API_VERSION_TO_ENGINE_VERSION = {
@@ -33,4 +37,5 @@ API_VERSION_TO_ENGINE_VERSION = {
     API_VERSIONS[COMPOSEFILE_V2_0]: '1.10.0',
     API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0',
     API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0',
+    API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0',
 }

compose/project.py

@@ -104,6 +104,11 @@ class Project(object):
                     for volume_spec in service_dict.get('volumes', [])
                 ]
 
+            secrets = get_secrets(
+                service_dict['name'],
+                service_dict.pop('secrets', None) or [],
+                config_data.secrets)
+
             project.services.append(
                 Service(
                     service_dict.pop('name'),
@@ -114,6 +119,7 @@ class Project(object):
                     links=links,
                     network_mode=network_mode,
                     volumes_from=volumes_from,
+                    secrets=secrets,
                     **service_dict)
             )
 
@@ -553,6 +559,33 @@ def get_volumes_from(project, service_dict):
     return [build_volume_from(vf) for vf in volumes_from]
 
 
+def get_secrets(service, service_secrets, secret_defs):
+    secrets = []
+
+    for secret in service_secrets:
+        secret_def = secret_defs.get(secret.source)
+        if not secret_def:
+            raise ConfigurationError(
+                "Service \"{service}\" uses an undefined secret \"{secret}\" "
+                .format(service=service, secret=secret.source))
+
+        if secret_def.get('external_name'):
+            log.warn("Service \"{service}\" uses secret \"{secret}\" which is external. "
+                     "External secrets are not available to containers created by "
+                     "docker-compose.".format(service=service, secret=secret.source))
+            continue
+
+        if secret.uid or secret.gid or secret.mode:
+            log.warn("Service \"{service}\" uses secret \"{secret}\" with uid, "
+                     "gid, or mode. These fields are not supported by this "
+                     "implementation of the Compose file".format(
+                        service=service, secret=secret.source))
+
+        secrets.append({'secret': secret, 'file': secret_def.get('file')})
+
+    return secrets
+
+
 def warn_for_swarm_mode(client):
     info = client.info()
     if info.get('Swarm', {}).get('LocalNodeState') == 'active':

compose/service.py

@@ -17,6 +17,7 @@ from docker.utils.ports import build_port_bindings
 from docker.utils.ports import split_port
 
 from . import __version__
+from . import const
 from . import progress_stream
 from .config import DOCKER_CONFIG_KEYS
 from .config import merge_environment
@@ -139,6 +140,7 @@ class Service(object):
         volumes_from=None,
         network_mode=None,
         networks=None,
+        secrets=None,
         **options
     ):
         self.name = name
@@ -149,6 +151,7 @@ class Service(object):
         self.volumes_from = volumes_from or []
         self.network_mode = network_mode or NetworkMode(None)
         self.networks = networks or {}
+        self.secrets = secrets or []
         self.options = options
 
     def __repr__(self):
@@ -692,9 +695,14 @@ class Service(object):
         override_options['binds'] = binds
         container_options['environment'].update(affinity)
 
-        if 'volumes' in container_options:
+        container_options['volumes'] = dict(
-            container_options['volumes'] = dict(
+            (v.internal, {}) for v in container_options.get('volumes') or {})
-                (v.internal, {}) for v in container_options['volumes'])
+
+        secret_volumes = self.get_secret_volumes()
+        if secret_volumes:
+            override_options['binds'].extend(v.repr() for v in secret_volumes)
+            container_options['volumes'].update(
+                (v.internal, {}) for v in secret_volumes)
 
         container_options['image'] = self.image_name
 
@@ -765,6 +773,15 @@ class Service(object):
 
         return host_config
 
+    def get_secret_volumes(self):
+        def build_spec(secret):
+            target = '{}/{}'.format(
+                const.SECRETS_PATH,
+                secret['secret'].target or secret['secret'].source)
+            return VolumeSpec(secret['file'], target, 'ro')
+
+        return [build_spec(secret) for secret in self.secrets]
+
     def build(self, no_cache=False, pull=False, force_rm=False):
         log.info('Building %s' % self.name)
 

tests/fixtures/secrets/default

@@ -0,0 +1 @@
+This is the secret

tests/integration/project_test.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import os.path
 import random
 
 import py
@@ -8,12 +9,14 @@ import pytest
 from docker.errors import NotFound
 
 from .. import mock
-from ..helpers import build_config
+from ..helpers import build_config as load_config
 from .testcases import DockerClientTestCase
 from compose.config import config
 from compose.config import ConfigurationError
+from compose.config import types
 from compose.config.config import V2_0
 from compose.config.config import V2_1
+from compose.config.config import V3_1
 from compose.config.types import VolumeFromSpec
 from compose.config.types import VolumeSpec
 from compose.const import LABEL_PROJECT
@@ -26,6 +29,16 @@ from compose.project import ProjectError
 from compose.service import ConvergenceStrategy
 from tests.integration.testcases import v2_1_only
 from tests.integration.testcases import v2_only
+from tests.integration.testcases import v3_only
+
+
+def build_config(**kwargs):
+    return config.Config(
+        version=kwargs.get('version'),
+        services=kwargs.get('services'),
+        volumes=kwargs.get('volumes'),
+        networks=kwargs.get('networks'),
+        secrets=kwargs.get('secrets'))
 
 
 class ProjectTest(DockerClientTestCase):
@@ -70,7 +83,7 @@ class ProjectTest(DockerClientTestCase):
     def test_volumes_from_service(self):
         project = Project.from_config(
             name='composetest',
-            config_data=build_config({
+            config_data=load_config({
                 'data': {
                     'image': 'busybox:latest',
                     'volumes': ['/var/data'],
@@ -96,7 +109,7 @@ class ProjectTest(DockerClientTestCase):
         )
         project = Project.from_config(
             name='composetest',
-            config_data=build_config({
+            config_data=load_config({
                 'db': {
                     'image': 'busybox:latest',
                     'volumes_from': ['composetest_data_container'],
@@ -112,7 +125,7 @@ class ProjectTest(DockerClientTestCase):
         project = Project.from_config(
             name='composetest',
             client=self.client,
-            config_data=build_config({
+            config_data=load_config({
                 'version': V2_0,
                 'services': {
                     'net': {
@@ -139,7 +152,7 @@ class ProjectTest(DockerClientTestCase):
         def get_project():
             return Project.from_config(
                 name='composetest',
-                config_data=build_config({
+                config_data=load_config({
                     'version': V2_0,
                     'services': {
                         'web': {
@@ -174,7 +187,7 @@ class ProjectTest(DockerClientTestCase):
     def test_net_from_service_v1(self):
         project = Project.from_config(
             name='composetest',
-            config_data=build_config({
+            config_data=load_config({
                 'net': {
                     'image': 'busybox:latest',
                     'command': ["top"]
@@ -198,7 +211,7 @@ class ProjectTest(DockerClientTestCase):
         def get_project():
             return Project.from_config(
                 name='composetest',
-                config_data=build_config({
+                config_data=load_config({
                     'web': {
                         'image': 'busybox:latest',
                         'net': 'container:composetest_net_container'
@@ -469,7 +482,7 @@ class ProjectTest(DockerClientTestCase):
     def test_project_up_starts_depends(self):
         project = Project.from_config(
             name='composetest',
-            config_data=build_config({
+            config_data=load_config({
                 'console': {
                     'image': 'busybox:latest',
                     'command': ["top"],
@@ -504,7 +517,7 @@ class ProjectTest(DockerClientTestCase):
     def test_project_up_with_no_deps(self):
         project = Project.from_config(
             name='composetest',
-            config_data=build_config({
+            config_data=load_config({
                 'console': {
                     'image': 'busybox:latest',
                     'command': ["top"],
@@ -564,7 +577,7 @@ class ProjectTest(DockerClientTestCase):
 
     @v2_only()
     def test_project_up_networks(self):
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -576,7 +589,6 @@ class ProjectTest(DockerClientTestCase):
                     'baz': {'aliases': ['extra']},
                 },
             }],
-            volumes={},
             networks={
                 'foo': {'driver': 'bridge'},
                 'bar': {'driver': None},
@@ -610,14 +622,13 @@ class ProjectTest(DockerClientTestCase):
 
     @v2_only()
     def test_up_with_ipam_config(self):
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'networks': {'front': None},
             }],
-            volumes={},
             networks={
                 'front': {
                     'driver': 'bridge',
@@ -671,7 +682,7 @@ class ProjectTest(DockerClientTestCase):
 
     @v2_only()
     def test_up_with_network_static_addresses(self):
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -684,7 +695,6 @@ class ProjectTest(DockerClientTestCase):
                     }
                 },
             }],
-            volumes={},
             networks={
                 'static_test': {
                     'driver': 'bridge',
@@ -726,7 +736,7 @@ class ProjectTest(DockerClientTestCase):
     @v2_1_only()
     def test_up_with_enable_ipv6(self):
         self.require_api_version('1.23')
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -738,7 +748,6 @@ class ProjectTest(DockerClientTestCase):
                     }
                 },
             }],
-            volumes={},
             networks={
                 'static_test': {
                     'driver': 'bridge',
@@ -770,7 +779,7 @@ class ProjectTest(DockerClientTestCase):
 
     @v2_only()
     def test_up_with_network_static_addresses_missing_subnet(self):
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -782,7 +791,6 @@ class ProjectTest(DockerClientTestCase):
                     }
                 },
             }],
-            volumes={},
             networks={
                 'static_test': {
                     'driver': 'bridge',
@@ -807,7 +815,7 @@ class ProjectTest(DockerClientTestCase):
 
     @v2_1_only()
     def test_up_with_network_link_local_ips(self):
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_1,
             services=[{
                 'name': 'web',
@@ -818,7 +826,6 @@ class ProjectTest(DockerClientTestCase):
                     }
                 }
             }],
-            volumes={},
             networks={
                 'linklocaltest': {'driver': 'bridge'}
             }
@@ -844,15 +851,13 @@ class ProjectTest(DockerClientTestCase):
     @v2_1_only()
     def test_up_with_isolation(self):
         self.require_api_version('1.24')
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_1,
             services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'isolation': 'default'
             }],
-            volumes={},
-            networks={}
         )
         project = Project.from_config(
             client=self.client,
@@ -866,15 +871,13 @@ class ProjectTest(DockerClientTestCase):
     @v2_1_only()
     def test_up_with_invalid_isolation(self):
         self.require_api_version('1.24')
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_1,
             services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'isolation': 'foobar'
             }],
-            volumes={},
-            networks={}
         )
         project = Project.from_config(
             client=self.client,
@@ -887,14 +890,13 @@ class ProjectTest(DockerClientTestCase):
     @v2_only()
     def test_project_up_with_network_internal(self):
         self.require_api_version('1.23')
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'networks': {'internal': None},
             }],
-            volumes={},
             networks={
                 'internal': {'driver': 'bridge', 'internal': True},
             },
@@ -917,14 +919,13 @@ class ProjectTest(DockerClientTestCase):
 
         network_name = 'network_with_label'
 
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
                 'image': 'busybox:latest',
                 'networks': {network_name: None}
             }],
-            volumes={},
             networks={
                 network_name: {'labels': {'label_key': 'label_val'}}
             }
@@ -951,7 +952,7 @@ class ProjectTest(DockerClientTestCase):
     def test_project_up_volumes(self):
         vol_name = '{0:x}'.format(random.getrandbits(32))
         full_vol_name = 'composetest_{0}'.format(vol_name)
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -959,7 +960,6 @@ class ProjectTest(DockerClientTestCase):
                 'command': 'top'
             }],
             volumes={vol_name: {'driver': 'local'}},
-            networks={},
         )
 
         project = Project.from_config(
@@ -979,7 +979,7 @@ class ProjectTest(DockerClientTestCase):
 
         volume_name = 'volume_with_label'
 
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -993,7 +993,6 @@ class ProjectTest(DockerClientTestCase):
                     }
                 }
             },
-            networks={},
         )
 
         project = Project.from_config(
@@ -1106,7 +1105,7 @@ class ProjectTest(DockerClientTestCase):
     def test_initialize_volumes(self):
         vol_name = '{0:x}'.format(random.getrandbits(32))
         full_vol_name = 'composetest_{0}'.format(vol_name)
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -1114,7 +1113,6 @@ class ProjectTest(DockerClientTestCase):
                 'command': 'top'
             }],
             volumes={vol_name: {}},
-            networks={},
         )
 
         project = Project.from_config(
@@ -1124,14 +1122,14 @@ class ProjectTest(DockerClientTestCase):
         project.volumes.initialize()
 
         volume_data = self.client.inspect_volume(full_vol_name)
-        self.assertEqual(volume_data['Name'], full_vol_name)
+        assert volume_data['Name'] == full_vol_name
-        self.assertEqual(volume_data['Driver'], 'local')
+        assert volume_data['Driver'] == 'local'
 
     @v2_only()
     def test_project_up_implicit_volume_driver(self):
         vol_name = '{0:x}'.format(random.getrandbits(32))
         full_vol_name = 'composetest_{0}'.format(vol_name)
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -1139,7 +1137,6 @@ class ProjectTest(DockerClientTestCase):
                 'command': 'top'
             }],
             volumes={vol_name: {}},
-            networks={},
         )
 
         project = Project.from_config(
@@ -1152,11 +1149,47 @@ class ProjectTest(DockerClientTestCase):
         self.assertEqual(volume_data['Name'], full_vol_name)
         self.assertEqual(volume_data['Driver'], 'local')
 
+    @v3_only()
+    def test_project_up_with_secrets(self):
+        create_host_file(self.client, os.path.abspath('tests/fixtures/secrets/default'))
+
+        config_data = build_config(
+            version=V3_1,
+            services=[{
+                'name': 'web',
+                'image': 'busybox:latest',
+                'command': 'cat /run/secrets/special',
+                'secrets': [
+                    types.ServiceSecret.parse({'source': 'super', 'target': 'special'}),
+                ],
+            }],
+            secrets={
+                'super': {
+                    'file': os.path.abspath('tests/fixtures/secrets/default'),
+                },
+            },
+        )
+
+        project = Project.from_config(
+            client=self.client,
+            name='composetest',
+            config_data=config_data,
+        )
+        project.up()
+        project.stop()
+
+        containers = project.containers(stopped=True)
+        assert len(containers) == 1
+        container, = containers
+
+        output = container.logs()
+        assert output == b"This is the secret\n"
+
     @v2_only()
     def test_initialize_volumes_invalid_volume_driver(self):
         vol_name = '{0:x}'.format(random.getrandbits(32))
 
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -1164,7 +1197,6 @@ class ProjectTest(DockerClientTestCase):
                 'command': 'top'
             }],
             volumes={vol_name: {'driver': 'foobar'}},
-            networks={},
         )
 
         project = Project.from_config(
@@ -1179,7 +1211,7 @@ class ProjectTest(DockerClientTestCase):
         vol_name = '{0:x}'.format(random.getrandbits(32))
         full_vol_name = 'composetest_{0}'.format(vol_name)
 
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -1187,7 +1219,6 @@ class ProjectTest(DockerClientTestCase):
                 'command': 'top'
             }],
             volumes={vol_name: {'driver': 'local'}},
-            networks={},
         )
         project = Project.from_config(
             name='composetest',
@@ -1218,7 +1249,7 @@ class ProjectTest(DockerClientTestCase):
         vol_name = '{0:x}'.format(random.getrandbits(32))
         full_vol_name = 'composetest_{0}'.format(vol_name)
 
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -1226,7 +1257,6 @@ class ProjectTest(DockerClientTestCase):
                 'command': 'top'
             }],
             volumes={vol_name: {'driver': 'local'}},
-            networks={},
         )
         project = Project.from_config(
             name='composetest',
@@ -1257,7 +1287,7 @@ class ProjectTest(DockerClientTestCase):
         vol_name = 'composetest_{0:x}'.format(random.getrandbits(32))
         full_vol_name = 'composetest_{0}'.format(vol_name)
         self.client.create_volume(vol_name)
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -1267,7 +1297,6 @@ class ProjectTest(DockerClientTestCase):
             volumes={
                 vol_name: {'external': True, 'external_name': vol_name}
             },
-            networks=None,
         )
         project = Project.from_config(
             name='composetest',
@@ -1282,7 +1311,7 @@ class ProjectTest(DockerClientTestCase):
     def test_initialize_volumes_inexistent_external_volume(self):
         vol_name = '{0:x}'.format(random.getrandbits(32))
 
-        config_data = config.Config(
+        config_data = build_config(
             version=V2_0,
             services=[{
                 'name': 'web',
@@ -1292,7 +1321,6 @@ class ProjectTest(DockerClientTestCase):
             volumes={
                 vol_name: {'external': True, 'external_name': vol_name}
             },
-            networks=None,
         )
         project = Project.from_config(
             name='composetest',
@@ -1349,7 +1377,7 @@ class ProjectTest(DockerClientTestCase):
             }
         }
 
-        config_data = build_config(config_dict)
+        config_data = load_config(config_dict)
         project = Project.from_config(
             name='composetest', config_data=config_data, client=self.client
         )
@@ -1357,7 +1385,7 @@ class ProjectTest(DockerClientTestCase):
         config_dict['service2'] = config_dict['service1']
         del config_dict['service1']
 
-        config_data = build_config(config_dict)
+        config_data = load_config(config_dict)
         project = Project.from_config(
             name='composetest', config_data=config_data, client=self.client
         )
@@ -1402,7 +1430,7 @@ class ProjectTest(DockerClientTestCase):
                 }
             }
         }
-        config_data = build_config(config_dict)
+        config_data = load_config(config_dict)
         project = Project.from_config(
             name='composetest', config_data=config_data, client=self.client
         )
@@ -1439,7 +1467,7 @@ class ProjectTest(DockerClientTestCase):
                 }
             }
         }
-        config_data = build_config(config_dict)
+        config_data = load_config(config_dict)
         project = Project.from_config(
             name='composetest', config_data=config_data, client=self.client
         )
@@ -1475,7 +1503,7 @@ class ProjectTest(DockerClientTestCase):
                 }
             }
         }
-        config_data = build_config(config_dict)
+        config_data = load_config(config_dict)
         project = Project.from_config(
             name='composetest', config_data=config_data, client=self.client
         )
@@ -1489,3 +1517,30 @@ class ProjectTest(DockerClientTestCase):
         assert 'svc1' in svc2.get_dependency_names()
         with pytest.raises(NoHealthCheckConfigured):
             svc1.is_healthy()
+
+
+def create_host_file(client, filename):
+    dirname = os.path.dirname(filename)
+
+    with open(filename, 'r') as fh:
+        content = fh.read()
+
+    container = client.create_container(
+        'busybox:latest',
+        ['sh', '-c', 'echo -n "{}" > {}'.format(content, filename)],
+        volumes={dirname: {}},
+        host_config=client.create_host_config(
+            binds={dirname: {'bind': dirname, 'ro': False}},
+            network_mode='none',
+        ),
+    )
+    try:
+        client.start(container)
+        exitcode = client.wait(container)
+
+        if exitcode != 0:
+            output = client.logs(container)
+            raise Exception(
+                "Container exited with code {}:\n{}".format(exitcode, output))
+    finally:
+        client.remove_container(container, force=True)

tests/integration/testcases.py

@@ -41,9 +41,9 @@ def engine_max_version():
     version = os.environ['DOCKER_VERSION'].partition('-')[0]
     if version_lt(version, '1.10'):
         return V1
-    elif version_lt(version, '1.12'):
+    if version_lt(version, '1.12'):
         return V2_0
-    elif version_lt(version, '1.13'):
+    if version_lt(version, '1.13'):
         return V2_1
     return V3_0
 
@@ -52,8 +52,9 @@ def build_version_required_decorator(ignored_versions):
     def decorator(f):
         @functools.wraps(f)
         def wrapper(self, *args, **kwargs):
-            if engine_max_version() in ignored_versions:
+            max_version = engine_max_version()
-                skip("Engine version is too low")
+            if max_version in ignored_versions:
+                skip("Engine version %s is too low" % max_version)
                 return
             return f(self, *args, **kwargs)
         return wrapper

tests/unit/bundle_test.py

@@ -77,7 +77,8 @@ def test_to_bundle():
         version=2,
         services=services,
         volumes={'special': {}},
-        networks={'extra': {}})
+        networks={'extra': {}},
+        secrets={})
 
     with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log:
         output = bundle.to_bundle(config, image_digests)

tests/unit/project_test.py

@@ -36,6 +36,7 @@ class ProjectTest(unittest.TestCase):
             ],
             networks=None,
             volumes=None,
+            secrets=None,
         )
         project = Project.from_config(
             name='composetest',
@@ -64,6 +65,7 @@ class ProjectTest(unittest.TestCase):
             ],
             networks=None,
             volumes=None,
+            secrets=None,
         )
         project = Project.from_config('composetest', config, None)
         self.assertEqual(len(project.services), 2)
@@ -170,6 +172,7 @@ class ProjectTest(unittest.TestCase):
                 }],
                 networks=None,
                 volumes=None,
+                secrets=None,
             ),
         )
         assert project.get_service('test')._get_volumes_from() == [container_id + ":rw"]
@@ -202,6 +205,7 @@ class ProjectTest(unittest.TestCase):
                 ],
                 networks=None,
                 volumes=None,
+                secrets=None,
             ),
         )
         assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"]
@@ -227,6 +231,7 @@ class ProjectTest(unittest.TestCase):
                 ],
                 networks=None,
                 volumes=None,
+                secrets=None,
             ),
         )
         with mock.patch.object(Service, 'containers') as mock_return:
@@ -360,6 +365,7 @@ class ProjectTest(unittest.TestCase):
                 ],
                 networks=None,
                 volumes=None,
+                secrets=None,
             ),
         )
         service = project.get_service('test')
@@ -384,6 +390,7 @@ class ProjectTest(unittest.TestCase):
                 ],
                 networks=None,
                 volumes=None,
+                secrets=None,
             ),
         )
         service = project.get_service('test')
@@ -417,6 +424,7 @@ class ProjectTest(unittest.TestCase):
                 ],
                 networks=None,
                 volumes=None,
+                secrets=None,
             ),
         )
 
@@ -437,6 +445,7 @@ class ProjectTest(unittest.TestCase):
                 ],
                 networks=None,
                 volumes=None,
+                secrets=None,
             ),
         )
 
@@ -457,6 +466,7 @@ class ProjectTest(unittest.TestCase):
                 ],
                 networks={'custom': {}},
                 volumes=None,
+                secrets=None,
             ),
         )
 
@@ -487,6 +497,7 @@ class ProjectTest(unittest.TestCase):
                 }],
                 networks=None,
                 volumes=None,
+                secrets=None,
             ),
         )
         self.assertEqual([c.id for c in project.containers()], ['1'])
@@ -503,6 +514,7 @@ class ProjectTest(unittest.TestCase):
                 }],
                 networks={'default': {}},
                 volumes={'data': {}},
+                secrets=None,
             ),
         )
         self.mock_client.remove_network.side_effect = NotFound(None, None, 'oops')