Update config resolution to always use explicit version numbers

Also includes several bugfixes for resolution and validation.

Signed-off-by: Joffrey F <joffrey@docker.com>

compose/config/config.py

@@ -10,6 +10,7 @@ from collections import namedtuple
 import six
 import yaml
 
+from ..const import COMPOSEFILE_VERSIONS
 from .errors import CircularReference
 from .errors import ComposeFileNotFound
 from .errors import ConfigurationError
@@ -24,6 +25,7 @@ from .validation import validate_against_fields_schema
 from .validation import validate_against_service_schema
 from .validation import validate_extends_file_path
 from .validation import validate_top_level_object
+from .validation import validate_top_level_service_objects
 
 
 DOCKER_CONFIG_KEYS = [
@@ -161,13 +163,24 @@ def find(base_dir, filenames):
 
 def get_config_version(config_details):
     def get_version(config):
-        validate_top_level_object(config)
-        return config.config.get('version')
+        if config.config is None:
+            return None
+        version = config.config.get('version', 1)
+        if isinstance(version, dict):
+            version = 1
+        return version
+
     main_file = config_details.config_files[0]
+    validate_top_level_object(main_file)
     version = get_version(main_file)
     for next_file in config_details.config_files[1:]:
+        validate_top_level_object(next_file)
         next_file_version = get_version(next_file)
-        if version != next_file_version:
+        if version is None:
+            version = next_file_version
+            continue
+
+        if version != next_file_version and next_file_version is not None:
             raise ConfigurationError(
                 "Version mismatch: main file {0} specifies version {1} but "
                 "extension file {2} uses version {3}".format(
@@ -224,6 +237,9 @@ def load(config_details):
     Return a fully interpolated, extended and validated configuration.
     """
     version = get_config_version(config_details)
+    if version not in COMPOSEFILE_VERSIONS:
+        raise ConfigurationError('Invalid config version provided: {0}'.format(version))
+
     processed_files = []
     for config_file in config_details.config_files:
         processed_files.append(
@@ -231,9 +247,10 @@ def load(config_details):
         )
     config_details = config_details._replace(config_files=processed_files)
 
-    if not version or isinstance(version, dict):
+    if version == 1:
         service_dicts = load_services(
-            config_details.working_dir, config_details.config_files
+            config_details.working_dir, config_details.config_files,
+            version
         )
         volumes = {}
     elif version == 2:
@@ -242,11 +259,9 @@ def load(config_details):
             for f in config_details.config_files
         ]
         service_dicts = load_services(
-            config_details.working_dir, config_files
+            config_details.working_dir, config_files, version
         )
         volumes = load_volumes(config_details.config_files)
-    else:
-        raise ConfigurationError('Invalid config version provided: {0}'.format(version))
 
     return Config(version, service_dicts, volumes)
 
@@ -259,14 +274,14 @@ def load_volumes(config_files):
     return volumes
 
 
-def load_services(working_dir, config_files):
+def load_services(working_dir, config_files, version):
     def build_service(filename, service_name, service_dict):
         service_config = ServiceConfig.with_abs_paths(
             working_dir,
             filename,
             service_name,
             service_dict)
-        resolver = ServiceExtendsResolver(service_config)
+        resolver = ServiceExtendsResolver(service_config, version)
         service_dict = process_service(resolver.run())
 
         # TODO: move to validate_service()
@@ -301,8 +316,8 @@ def load_services(working_dir, config_files):
 
 
 def process_config_file(config_file, service_name=None, version=None):
-    validate_top_level_object(config_file)
-    processed_config = interpolate_environment_variables(config_file.config)
+    validate_top_level_service_objects(config_file, version)
+    processed_config = interpolate_environment_variables(config_file.config, version)
     validate_against_fields_schema(
         processed_config, config_file.filename, version
     )
@@ -316,10 +331,11 @@ def process_config_file(config_file, service_name=None, version=None):
 
 
 class ServiceExtendsResolver(object):
-    def __init__(self, service_config, already_seen=None):
+    def __init__(self, service_config, version, already_seen=None):
         self.service_config = service_config
         self.working_dir = service_config.working_dir
         self.already_seen = already_seen or []
+        self.version = version
 
     @property
     def signature(self):
@@ -348,7 +364,8 @@ class ServiceExtendsResolver(object):
 
         extended_file = process_config_file(
             ConfigFile.from_filename(config_path),
-            service_name=service_name)
+            service_name=service_name, version=self.version
+        )
         service_config = extended_file.config[service_name]
         return config_path, service_config, service_name
 
@@ -359,6 +376,7 @@ class ServiceExtendsResolver(object):
                 extended_config_path,
                 service_name,
                 service_dict),
+            self.version,
             already_seen=self.already_seen + [self.signature])
 
         service_config = resolver.run()

compose/config/fields_schema_v1.json

@@ -2,7 +2,7 @@
   "$schema": "http://json-schema.org/draft-04/schema#",
 
   "type": "object",
-  "id": "fields_schema.json",
+  "id": "fields_schema_v1.json",
 
   "patternProperties": {
     "^[a-zA-Z0-9._-]+$": {

compose/config/fields_schema_v2.json

@@ -1,38 +1,44 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-
   "type": "object",
+  "id": "fields_schema_v2.json",
+
   "properties": {
     "version": {
       "enum": [2]
     },
     "services": {
+      "id": "#/properties/services",
       "type": "object",
       "patternProperties": {
         "^[a-zA-Z0-9._-]+$": {
-          "$ref": "fields_schema.json#/definitions/service"
+          "$ref": "fields_schema_v1.json#/definitions/service"
         }
-      }
+      },
+      "additionalProperties": false
     },
     "volumes": {
+      "id": "#/properties/volumes",
       "type": "object",
       "patternProperties": {
         "^[a-zA-Z0-9._-]+$": {
           "$ref": "#/definitions/volume"
         }
-      }
+      },
+      "additionalProperties": false
     }
   },
 
   "definitions": {
     "volume": {
+      "id": "#/definitions/volume",
       "type": "object",
       "properties": {
         "driver": {"type": "string"},
         "driver_opts": {
           "type": "object",
           "patternProperties": {
-            "^.+$": {"type": ["boolean", "string", "number"]}
+            "^.+$": {"type": ["string", "number"]}
           },
           "additionalProperties": false
         }

compose/config/interpolation.py

@@ -8,13 +8,19 @@ from .errors import ConfigurationError
 log = logging.getLogger(__name__)
 
 
-def interpolate_environment_variables(config):
+def interpolate_environment_variables(config, version):
     mapping = BlankDefaultDict(os.environ)
+    service_dicts = config if version == 1 else config.get('services', {})
 
-    return dict(
+    interpolated = dict(
         (service_name, process_service(service_name, service_dict, mapping))
-        for (service_name, service_dict) in config.items()
+        for (service_name, service_dict) in service_dicts.items()
     )
+    if version == 1:
+        return interpolated
+    result = dict(config)
+    result.update({'services': interpolated})
+    return result
 
 
 def process_service(service_name, service_dict, mapping):

compose/config/service_schema.json

@@ -5,7 +5,7 @@
   "type": "object",
 
   "allOf": [
-    {"$ref": "fields_schema.json#/definitions/service"},
+    {"$ref": "fields_schema_v1.json#/definitions/service"},
     {"$ref": "#/definitions/constraints"}
   ],
 

compose/config/validation.py

@@ -74,14 +74,15 @@ def format_boolean_in_environment(instance):
     return True
 
 
-def validate_top_level_service_objects(config_file):
+def validate_top_level_service_objects(config_file, version):
     """Perform some high level validation of the service name and value.
 
     This validation must happen before interpolation, which must happen
     before the rest of validation, which is why it's separate from the
     rest of the service validation.
     """
-    for service_name, service_dict in config_file.config.items():
+    service_dicts = config_file.config if version == 1 else config_file.config.get('services', {})
+    for service_name, service_dict in service_dicts.items():
         if not isinstance(service_name, six.string_types):
             raise ConfigurationError(
                 "In file '{}' service name: {} needs to be a string, eg '{}'".format(
@@ -105,7 +106,6 @@ def validate_top_level_object(config_file):
             "that you have defined a service at the top level.".format(
                 config_file.filename,
                 type(config_file.config)))
-    validate_top_level_service_objects(config_file)
 
 
 def validate_extends_file_path(service_name, extends_options, filename):
@@ -134,10 +134,14 @@ def anglicize_validator(validator):
     return 'a ' + validator
 
 
+def is_service_dict_schema(schema_id):
+    return schema_id == 'fields_schema_v1.json' or schema_id == '#/properties/services'
+
+
 def handle_error_for_schema_with_id(error, service_name):
     schema_id = error.schema['id']
 
-    if schema_id == 'fields_schema.json' and error.validator == 'additionalProperties':
+    if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties':
         return "Invalid service name '{}' - only {} characters are allowed".format(
             # The service_name is the key to the json object
             list(error.instance)[0],
@@ -281,10 +285,8 @@ def process_errors(errors, service_name=None):
     return '\n'.join(format_error_message(error, service_name) for error in errors)
 
 
-def validate_against_fields_schema(config, filename, version=None):
-    schema_filename = "fields_schema.json"
-    if version:
-        schema_filename = "fields_schema_v{0}.json".format(version)
+def validate_against_fields_schema(config, filename, version):
+    schema_filename = "fields_schema_v{0}.json".format(version)
     _validate_against_schema(
         config,
         schema_filename,

compose/const.py

@@ -10,3 +10,4 @@ LABEL_PROJECT = 'com.docker.compose.project'
 LABEL_SERVICE = 'com.docker.compose.service'
 LABEL_VERSION = 'com.docker.compose.version'
 LABEL_CONFIG_HASH = 'com.docker.compose.config-hash'
+COMPOSEFILE_VERSIONS = (1, 2)

docker-compose.spec

@@ -18,8 +18,13 @@ exe = EXE(pyz,
           a.datas,
           [
             (
-                'compose/config/fields_schema.json',
-                'compose/config/fields_schema.json',
+                'compose/config/fields_schema_v1.json',
+                'compose/config/fields_schema_v1.json',
+                'DATA'
+            ),
+            (
+                'compose/config/fields_schema_v2.json',
+                'compose/config/fields_schema_v2.json',
                 'DATA'
             ),
             (
@@ -33,6 +38,7 @@ exe = EXE(pyz,
                 'DATA'
             )
           ],
+
           name='docker-compose',
           debug=False,
           strip=None,