Merge pull request #3653 from shin-/3637-link-local-ips

Add support for link-local IPs in service.networks definition

compose/config/config.py

@@ -14,6 +14,7 @@ from cached_property import cached_property
 
 from ..const import COMPOSEFILE_V1 as V1
 from ..const import COMPOSEFILE_V2_0 as V2_0
+from ..const import COMPOSEFILE_V2_1 as V2_1
 from ..utils import build_string_dict
 from ..utils import splitdrive
 from .environment import env_vars_from_file
@@ -174,7 +175,7 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
         if version == '2':
             version = V2_0
 
-        if version != V2_0:
+        if version not in (V2_0, V2_1):
             raise ConfigurationError(
                 'Version in "{}" is unsupported. {}'
                 .format(self.filename, VERSION_EXPLANATION))
@@ -424,7 +425,7 @@ def process_config_file(config_file, environment, service_name=None):
         'service',
         environment,)
 
-    if config_file.version == V2_0:
+    if config_file.version in (V2_0, V2_1):
         processed_config = dict(config_file.config)
         processed_config['services'] = services
         processed_config['volumes'] = interpolate_config_section(

compose/config/config_schema_v2.1.json

@@ -0,0 +1,319 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "id": "config_schema_v2.1.json",
+  "type": "object",
+
+  "properties": {
+    "version": {
+      "type": "string"
+    },
+
+    "services": {
+      "id": "#/properties/services",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/service"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "networks": {
+      "id": "#/properties/networks",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/network"
+        }
+      }
+    },
+
+    "volumes": {
+      "id": "#/properties/volumes",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/volume"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+
+  "additionalProperties": false,
+
+  "definitions": {
+
+    "service": {
+      "id": "#/definitions/service",
+      "type": "object",
+
+      "properties": {
+        "build": {
+          "oneOf": [
+            {"type": "string"},
+            {
+              "type": "object",
+              "properties": {
+                "context": {"type": "string"},
+                "dockerfile": {"type": "string"},
+                "args": {"$ref": "#/definitions/list_or_dict"}
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "cgroup_parent": {"type": "string"},
+        "command": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "container_name": {"type": "string"},
+        "cpu_shares": {"type": ["number", "string"]},
+        "cpu_quota": {"type": ["number", "string"]},
+        "cpuset": {"type": "string"},
+        "depends_on": {"$ref": "#/definitions/list_of_strings"},
+        "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "dns": {"$ref": "#/definitions/string_or_list"},
+        "dns_search": {"$ref": "#/definitions/string_or_list"},
+        "domainname": {"type": "string"},
+        "entrypoint": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "env_file": {"$ref": "#/definitions/string_or_list"},
+        "environment": {"$ref": "#/definitions/list_or_dict"},
+
+        "expose": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"],
+            "format": "expose"
+          },
+          "uniqueItems": true
+        },
+
+        "extends": {
+          "oneOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "object",
+
+              "properties": {
+                "service": {"type": "string"},
+                "file": {"type": "string"}
+              },
+              "required": ["service"],
+              "additionalProperties": false
+            }
+          ]
+        },
+
+        "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "extra_hosts": {"$ref": "#/definitions/list_or_dict"},
+        "hostname": {"type": "string"},
+        "image": {"type": "string"},
+        "ipc": {"type": "string"},
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+
+        "logging": {
+            "type": "object",
+
+            "properties": {
+                "driver": {"type": "string"},
+                "options": {"type": "object"}
+            },
+            "additionalProperties": false
+        },
+
+        "mac_address": {"type": "string"},
+        "mem_limit": {"type": ["number", "string"]},
+        "memswap_limit": {"type": ["number", "string"]},
+        "network_mode": {"type": "string"},
+
+        "networks": {
+          "oneOf": [
+            {"$ref": "#/definitions/list_of_strings"},
+            {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9._-]+$": {
+                  "oneOf": [
+                    {
+                      "type": "object",
+                      "properties": {
+                        "aliases": {"$ref": "#/definitions/list_of_strings"},
+                        "ipv4_address": {"type": "string"},
+                        "ipv6_address": {"type": "string"},
+                        "link_local_ips": {"$ref": "#/definitions/list_of_strings"}
+                      },
+                      "additionalProperties": false
+                    },
+                    {"type": "null"}
+                  ]
+                }
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "pid": {"type": ["string", "null"]},
+
+        "ports": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"],
+            "format": "ports"
+          },
+          "uniqueItems": true
+        },
+
+        "privileged": {"type": "boolean"},
+        "read_only": {"type": "boolean"},
+        "restart": {"type": "string"},
+        "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "shm_size": {"type": ["number", "string"]},
+        "stdin_open": {"type": "boolean"},
+        "stop_signal": {"type": "string"},
+        "tmpfs": {"$ref": "#/definitions/string_or_list"},
+        "tty": {"type": "boolean"},
+        "ulimits": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-z]+$": {
+              "oneOf": [
+                {"type": "integer"},
+                {
+                  "type":"object",
+                  "properties": {
+                    "hard": {"type": "integer"},
+                    "soft": {"type": "integer"}
+                  },
+                  "required": ["soft", "hard"],
+                  "additionalProperties": false
+                }
+              ]
+            }
+          }
+        },
+        "user": {"type": "string"},
+        "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "volume_driver": {"type": "string"},
+        "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "working_dir": {"type": "string"}
+      },
+
+      "dependencies": {
+        "memswap_limit": ["mem_limit"]
+      },
+      "additionalProperties": false
+    },
+
+    "network": {
+      "id": "#/definitions/network",
+      "type": "object",
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "ipam": {
+            "type": "object",
+            "properties": {
+                "driver": {"type": "string"},
+                "config": {
+                    "type": "array"
+                }
+            },
+            "additionalProperties": false
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "volume": {
+      "id": "#/definitions/volume",
+      "type": ["object", "null"],
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          }
+        },
+        "additionalProperties": false
+      },
+      "additionalProperties": false
+    },
+
+    "string_or_list": {
+      "oneOf": [
+        {"type": "string"},
+        {"$ref": "#/definitions/list_of_strings"}
+      ]
+    },
+
+    "list_of_strings": {
+      "type": "array",
+      "items": {"type": "string"},
+      "uniqueItems": true
+    },
+
+    "list_or_dict": {
+      "oneOf": [
+        {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": ["string", "number", "null"]
+            }
+          },
+          "additionalProperties": false
+        },
+        {"type": "array", "items": {"type": "string"}, "uniqueItems": true}
+      ]
+    },
+
+    "constraints": {
+      "service": {
+        "id": "#/definitions/constraints/service",
+        "anyOf": [
+          {"required": ["build"]},
+          {"required": ["image"]}
+        ],
+        "properties": {
+          "build": {
+            "required": ["context"]
+          }
+        }
+      }
+    }
+  }
+}

compose/config/serialize.py

@@ -7,6 +7,7 @@ import yaml
 from compose.config import types
 from compose.config.config import V1
 from compose.config.config import V2_0
+from compose.config.config import V2_1
 
 
 def serialize_config_type(dumper, data):
@@ -32,8 +33,12 @@ def denormalize_config(config):
         if 'external_name' in net_conf:
             del net_conf['external_name']
 
+    version = config.version
+    if version not in (V2_0, V2_1):
+        version = V2_1
+
     return {
-        'version': V2_0,
+        'version': version,
         'services': services,
         'networks': networks,
         'volumes': config.volumes,

compose/const.py

@@ -16,13 +16,16 @@ LABEL_CONFIG_HASH = 'com.docker.compose.config-hash'
 
 COMPOSEFILE_V1 = '1'
 COMPOSEFILE_V2_0 = '2.0'
+COMPOSEFILE_V2_1 = '2.1'
 
 API_VERSIONS = {
     COMPOSEFILE_V1: '1.21',
     COMPOSEFILE_V2_0: '1.22',
+    COMPOSEFILE_V2_1: '1.24',
 }
 
 API_VERSION_TO_ENGINE_VERSION = {
     API_VERSIONS[COMPOSEFILE_V1]: '1.9.0',
-    API_VERSIONS[COMPOSEFILE_V2_0]: '1.10.0'
+    API_VERSIONS[COMPOSEFILE_V2_0]: '1.10.0',
+    API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0',
 }

compose/service.py

@@ -478,7 +478,9 @@ class Service(object):
                 aliases=self._get_aliases(netdefs, container),
                 ipv4_address=netdefs.get('ipv4_address', None),
                 ipv6_address=netdefs.get('ipv6_address', None),
-                links=self._get_links(False))
+                links=self._get_links(False),
+                link_local_ips=netdefs.get('link_local_ips', None),
+            )
 
     def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT):
         for c in self.duplicate_containers():

docs/compose-file.md

@@ -621,6 +621,31 @@ An example:
           - subnet: 2001:3984:3989::/64
             gateway: 2001:3984:3989::1
 
+#### link_local_ips
+
+> [Added in version 2.1 file format](#version-21).
+
+Specify a list of link-local IPs. Link-local IPs are special IPs which belong
+to a well known subnet and are purely managed by the operator, usually
+dependent on the architecture where they are deployed. Therefore they are not
+managed by docker (IPAM driver).
+
+Example usage:
+
+    version: '2.1'
+    services:
+      app:
+        image: busybox
+        command: top
+        networks:
+          app_net:
+            link_local_ips:
+              - 57.123.22.11
+              - 57.123.22.13
+    networks:
+      app_net:
+        driver: bridge
+
 ### pid
 
     pid: "host"
@@ -1054,6 +1079,15 @@ A more extended example, defining volumes and networks:
       back-tier:
         driver: bridge
 
+### Version 2.1
+
+An upgrade of [version 2](#version-2) that introduces new parameters only
+available with Docker Engine version **1.12.0+**
+
+Introduces:
+
+- [`link_local_ips`](#link_local_ips)
+- ...
 
 ### Upgrading
 

tests/acceptance/cli_test.py

@@ -257,7 +257,7 @@ class CLITestCase(DockerClientTestCase):
         self.base_dir = 'tests/fixtures/v1-config'
         result = self.dispatch(['config'])
         assert yaml.load(result.stdout) == {
-            'version': '2.0',
+            'version': '2.1',
             'services': {
                 'net': {
                     'image': 'busybox',

tests/integration/project_test.py

@@ -13,6 +13,7 @@ from .testcases import DockerClientTestCase
 from compose.config import config
 from compose.config import ConfigurationError
 from compose.config.config import V2_0
+from compose.config.config import V2_1
 from compose.config.types import VolumeFromSpec
 from compose.config.types import VolumeSpec
 from compose.const import LABEL_PROJECT
@@ -21,6 +22,7 @@ from compose.container import Container
 from compose.project import Project
 from compose.project import ProjectError
 from compose.service import ConvergenceStrategy
+from tests.integration.testcases import v2_1_only
 from tests.integration.testcases import v2_only
 
 
@@ -756,6 +758,42 @@ class ProjectTest(DockerClientTestCase):
         with self.assertRaises(ProjectError):
             project.up()
 
+    @v2_1_only()
+    def test_up_with_network_link_local_ips(self):
+        config_data = config.Config(
+            version=V2_1,
+            services=[{
+                'name': 'web',
+                'image': 'busybox:latest',
+                'networks': {
+                    'linklocaltest': {
+                        'link_local_ips': ['169.254.8.8']
+                    }
+                }
+            }],
+            volumes={},
+            networks={
+                'linklocaltest': {'driver': 'bridge'}
+            }
+        )
+        project = Project.from_config(
+            client=self.client,
+            name='composetest',
+            config_data=config_data
+        )
+        project.up()
+
+        service_container = project.get_service('web').containers()[0]
+        ipam_config = service_container.inspect().get(
+            'NetworkSettings', {}
+        ).get(
+            'Networks', {}
+        ).get(
+            'composetest_linklocaltest', {}
+        ).get('IPAMConfig', {})
+        assert 'LinkLocalIPs' in ipam_config
+        assert ipam_config['LinkLocalIPs'] == ['169.254.8.8']
+
     @v2_only()
     def test_project_up_with_network_internal(self):
         self.require_api_version('1.23')

tests/integration/testcases.py

@@ -12,6 +12,7 @@ from compose.cli.docker_client import docker_client
 from compose.config.config import resolve_environment
 from compose.config.config import V1
 from compose.config.config import V2_0
+from compose.config.config import V2_1
 from compose.config.environment import Environment
 from compose.const import API_VERSIONS
 from compose.const import LABEL_PROJECT
@@ -33,18 +34,22 @@ def get_links(container):
     return [format_link(link) for link in links]
 
 
-def engine_version_too_low_for_v2():
+def engine_max_version():
     if 'DOCKER_VERSION' not in os.environ:
-        return False
+        return V2_1
     version = os.environ['DOCKER_VERSION'].partition('-')[0]
-    return version_lt(version, '1.10')
+    if version_lt(version, '1.10'):
+        return V1
+    elif version_lt(version, '1.12'):
+        return V2_0
+    return V2_1
 
 
 def v2_only():
     def decorator(f):
         @functools.wraps(f)
         def wrapper(self, *args, **kwargs):
-            if engine_version_too_low_for_v2():
+            if engine_max_version() == V1:
                 skip("Engine version is too low")
                 return
             return f(self, *args, **kwargs)
@@ -53,14 +58,23 @@ def v2_only():
     return decorator
 
 
+def v2_1_only():
+    def decorator(f):
+        @functools.wraps(f)
+        def wrapper(self, *args, **kwargs):
+            if engine_max_version() in (V1, V2_0):
+                skip('Engine version is too low')
+                return
+            return f(self, *args, **kwargs)
+        return wrapper
+
+    return decorator
+
+
 class DockerClientTestCase(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        if engine_version_too_low_for_v2():
+        version = API_VERSIONS[engine_max_version()]
-            version = API_VERSIONS[V1]
-        else:
-            version = API_VERSIONS[V2_0]
-
         cls.client = docker_client(Environment(), version)
 
     @classmethod

tests/unit/config/config_test.py

@@ -17,6 +17,7 @@ from compose.config.config import resolve_build_args
 from compose.config.config import resolve_environment
 from compose.config.config import V1
 from compose.config.config import V2_0
+from compose.config.config import V2_1
 from compose.config.environment import Environment
 from compose.config.errors import ConfigurationError
 from compose.config.errors import VERSION_EXPLANATION
@@ -155,6 +156,8 @@ class ConfigTest(unittest.TestCase):
         for version in ['2', '2.0']:
             cfg = config.load(build_config_details({'version': version}))
             assert cfg.version == V2_0
+        cfg = config.load(build_config_details({'version': '2.1'}))
+        assert cfg.version == V2_1
 
     def test_v1_file_version(self):
         cfg = config.load(build_config_details({'web': {'image': 'busybox'}}))
@@ -182,7 +185,7 @@ class ConfigTest(unittest.TestCase):
         with pytest.raises(ConfigurationError) as excinfo:
             config.load(
                 build_config_details(
-                    {'version': '2.1'},
+                    {'version': '2.18'},
                     filename='filename.yml',
                 )
             )
@@ -344,6 +347,35 @@ class ConfigTest(unittest.TestCase):
                 }, 'working_dir', 'filename.yml')
             )
 
+    def test_load_config_link_local_ips_network(self):
+        base_file = config.ConfigFile(
+            'base.yaml',
+            {
+                'version': '2.1',
+                'services': {
+                    'web': {
+                        'image': 'example/web',
+                        'networks': {
+                            'foobar': {
+                                'aliases': ['foo', 'bar'],
+                                'link_local_ips': ['169.254.8.8']
+                            }
+                        }
+                    }
+                },
+                'networks': {'foobar': {}}
+            }
+        )
+
+        details = config.ConfigDetails('.', [base_file])
+        web_service = config.load(details).services[0]
+        assert web_service['networks'] == {
+            'foobar': {
+                'aliases': ['foo', 'bar'],
+                'link_local_ips': ['169.254.8.8']
+            }
+        }
+
     def test_load_config_invalid_service_names(self):
         for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']:
             with pytest.raises(ConfigurationError) as exc: