Merge pull request #5767 from docker/5402-lcow-support

Add 2.4 file format with platform support (build/pull)

compose/cli/command.py

@@ -122,7 +122,9 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False,
     )
 
     with errors.handle_connection_errors(client):
-        return Project.from_config(project_name, config_data, client)
+        return Project.from_config(
+            project_name, config_data, client, environment.get('DOCKER_DEFAULT_PLATFORM')
+        )
 
 
 def get_project_name(working_dir, project_name=None, environment=None):

compose/config/config.py

@@ -129,11 +129,12 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [
     'container_name',
     'credential_spec',
     'dockerfile',
+    'init',
     'log_driver',
     'log_opt',
     'logging',
     'network_mode',
-    'init',
+    'platform',
     'scale',
     'stop_grace_period',
 ]

compose/config/config_schema_v2.4.json

@@ -0,0 +1,513 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "id": "config_schema_v2.4.json",
+  "type": "object",
+
+  "properties": {
+    "version": {
+      "type": "string"
+    },
+
+    "services": {
+      "id": "#/properties/services",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/service"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "networks": {
+      "id": "#/properties/networks",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/network"
+        }
+      }
+    },
+
+    "volumes": {
+      "id": "#/properties/volumes",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/volume"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+
+  "patternProperties": {"^x-": {}},
+  "additionalProperties": false,
+
+  "definitions": {
+
+    "service": {
+      "id": "#/definitions/service",
+      "type": "object",
+
+      "properties": {
+        "blkio_config": {
+          "type": "object",
+          "properties": {
+            "device_read_bps": {
+              "type": "array",
+              "items": {"$ref": "#/definitions/blkio_limit"}
+            },
+            "device_read_iops": {
+              "type": "array",
+              "items": {"$ref": "#/definitions/blkio_limit"}
+            },
+            "device_write_bps": {
+              "type": "array",
+              "items": {"$ref": "#/definitions/blkio_limit"}
+            },
+            "device_write_iops": {
+              "type": "array",
+              "items": {"$ref": "#/definitions/blkio_limit"}
+            },
+            "weight": {"type": "integer"},
+            "weight_device": {
+              "type": "array",
+              "items": {"$ref": "#/definitions/blkio_weight"}
+            }
+          },
+          "additionalProperties": false
+        },
+
+        "build": {
+          "oneOf": [
+            {"type": "string"},
+            {
+              "type": "object",
+              "properties": {
+                "context": {"type": "string"},
+                "dockerfile": {"type": "string"},
+                "args": {"$ref": "#/definitions/list_or_dict"},
+                "labels": {"$ref": "#/definitions/labels"},
+                "cache_from": {"$ref": "#/definitions/list_of_strings"},
+                "network": {"type": "string"},
+                "target": {"type": "string"},
+                "shm_size": {"type": ["integer", "string"]},
+                "extra_hosts": {"$ref": "#/definitions/list_or_dict"},
+                "isolation": {"type": "string"}
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "cap_add": {"$ref": "#/definitions/list_of_strings"},
+        "cap_drop": {"$ref": "#/definitions/list_of_strings"},
+        "cgroup_parent": {"type": "string"},
+        "command": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "container_name": {"type": "string"},
+        "cpu_count": {"type": "integer", "minimum": 0},
+        "cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100},
+        "cpu_shares": {"type": ["number", "string"]},
+        "cpu_quota": {"type": ["number", "string"]},
+        "cpu_period": {"type": ["number", "string"]},
+        "cpu_rt_period": {"type": ["number", "string"]},
+        "cpu_rt_runtime": {"type": ["number", "string"]},
+        "cpus": {"type": "number", "minimum": 0},
+        "cpuset": {"type": "string"},
+        "depends_on": {
+          "oneOf": [
+            {"$ref": "#/definitions/list_of_strings"},
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "patternProperties": {
+                "^[a-zA-Z0-9._-]+$": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "properties": {
+                    "condition": {
+                      "type": "string",
+                      "enum": ["service_started", "service_healthy"]
+                    }
+                  },
+                  "required": ["condition"]
+                }
+              }
+            }
+          ]
+        },
+        "device_cgroup_rules": {"$ref": "#/definitions/list_of_strings"},
+        "devices": {"$ref": "#/definitions/list_of_strings"},
+        "dns_opt": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "uniqueItems": true
+        },
+        "dns": {"$ref": "#/definitions/string_or_list"},
+        "dns_search": {"$ref": "#/definitions/string_or_list"},
+        "domainname": {"type": "string"},
+        "entrypoint": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "env_file": {"$ref": "#/definitions/string_or_list"},
+        "environment": {"$ref": "#/definitions/list_or_dict"},
+
+        "expose": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"],
+            "format": "expose"
+          },
+          "uniqueItems": true
+        },
+
+        "extends": {
+          "oneOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "object",
+
+              "properties": {
+                "service": {"type": "string"},
+                "file": {"type": "string"}
+              },
+              "required": ["service"],
+              "additionalProperties": false
+            }
+          ]
+        },
+
+        "external_links": {"$ref": "#/definitions/list_of_strings"},
+        "extra_hosts": {"$ref": "#/definitions/list_or_dict"},
+        "group_add": {
+            "type": "array",
+            "items": {
+                "type": ["string", "number"]
+            },
+            "uniqueItems": true
+        },
+        "healthcheck": {"$ref": "#/definitions/healthcheck"},
+        "hostname": {"type": "string"},
+        "image": {"type": "string"},
+        "init": {"type": ["boolean", "string"]},
+        "ipc": {"type": "string"},
+        "isolation": {"type": "string"},
+        "labels": {"$ref": "#/definitions/labels"},
+        "links": {"$ref": "#/definitions/list_of_strings"},
+
+        "logging": {
+            "type": "object",
+
+            "properties": {
+                "driver": {"type": "string"},
+                "options": {"type": "object"}
+            },
+            "additionalProperties": false
+        },
+
+        "mac_address": {"type": "string"},
+        "mem_limit": {"type": ["number", "string"]},
+        "mem_reservation": {"type": ["string", "integer"]},
+        "mem_swappiness": {"type": "integer"},
+        "memswap_limit": {"type": ["number", "string"]},
+        "network_mode": {"type": "string"},
+
+        "networks": {
+          "oneOf": [
+            {"$ref": "#/definitions/list_of_strings"},
+            {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9._-]+$": {
+                  "oneOf": [
+                    {
+                      "type": "object",
+                      "properties": {
+                        "aliases": {"$ref": "#/definitions/list_of_strings"},
+                        "ipv4_address": {"type": "string"},
+                        "ipv6_address": {"type": "string"},
+                        "link_local_ips": {"$ref": "#/definitions/list_of_strings"},
+                        "priority": {"type": "number"}
+                      },
+                      "additionalProperties": false
+                    },
+                    {"type": "null"}
+                  ]
+                }
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "oom_kill_disable": {"type": "boolean"},
+        "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000},
+        "pid": {"type": ["string", "null"]},
+        "platform": {"type": "string"},
+        "ports": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"],
+            "format": "ports"
+          },
+          "uniqueItems": true
+        },
+        "privileged": {"type": "boolean"},
+        "read_only": {"type": "boolean"},
+        "restart": {"type": "string"},
+        "runtime": {"type": "string"},
+        "scale": {"type": "integer"},
+        "security_opt": {"$ref": "#/definitions/list_of_strings"},
+        "shm_size": {"type": ["number", "string"]},
+        "sysctls": {"$ref": "#/definitions/list_or_dict"},
+        "pids_limit": {"type": ["number", "string"]},
+        "stdin_open": {"type": "boolean"},
+        "stop_grace_period": {"type": "string", "format": "duration"},
+        "stop_signal": {"type": "string"},
+        "storage_opt": {"type": "object"},
+        "tmpfs": {"$ref": "#/definitions/string_or_list"},
+        "tty": {"type": "boolean"},
+        "ulimits": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-z]+$": {
+              "oneOf": [
+                {"type": "integer"},
+                {
+                  "type":"object",
+                  "properties": {
+                    "hard": {"type": "integer"},
+                    "soft": {"type": "integer"}
+                  },
+                  "required": ["soft", "hard"],
+                  "additionalProperties": false
+                }
+              ]
+            }
+          }
+        },
+        "user": {"type": "string"},
+        "userns_mode": {"type": "string"},
+        "volumes": {
+          "type": "array",
+          "items": {
+            "oneOf": [
+              {"type": "string"},
+              {
+                "type": "object",
+                "required": ["type"],
+                "additionalProperties": false,
+                "properties": {
+                  "type": {"type": "string"},
+                  "source": {"type": "string"},
+                  "target": {"type": "string"},
+                  "read_only": {"type": "boolean"},
+                  "consistency": {"type": "string"},
+                  "bind": {
+                    "type": "object",
+                    "properties": {
+                      "propagation": {"type": "string"}
+                    }
+                  },
+                  "volume": {
+                    "type": "object",
+                    "properties": {
+                      "nocopy": {"type": "boolean"}
+                    }
+                  },
+                  "tmpfs": {
+                    "type": "object",
+                    "properties": {
+                      "size": {"type": ["integer", "string"]}
+                    }
+                  }
+                }
+              }
+            ],
+            "uniqueItems": true
+          }
+        },
+        "volume_driver": {"type": "string"},
+        "volumes_from": {"$ref": "#/definitions/list_of_strings"},
+        "working_dir": {"type": "string"}
+      },
+
+      "dependencies": {
+        "memswap_limit": ["mem_limit"]
+      },
+      "additionalProperties": false
+    },
+
+    "healthcheck": {
+      "id": "#/definitions/healthcheck",
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "disable": {"type": "boolean"},
+        "interval": {"type": "string"},
+        "retries": {"type": "number"},
+        "start_period": {"type": "string"},
+        "test": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "timeout": {"type": "string"}
+      }
+    },
+
+    "network": {
+      "id": "#/definitions/network",
+      "type": "object",
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "ipam": {
+            "type": "object",
+            "properties": {
+                "driver": {"type": "string"},
+                "config": {
+                    "type": "array"
+                },
+                "options": {
+                  "type": "object",
+                  "patternProperties": {
+                    "^.+$": {"type": "string"}
+                  },
+                  "additionalProperties": false
+                }
+            },
+            "additionalProperties": false
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          },
+          "additionalProperties": false
+        },
+        "internal": {"type": "boolean"},
+        "enable_ipv6": {"type": "boolean"},
+        "labels": {"$ref": "#/definitions/labels"},
+        "name": {"type": "string"}
+      },
+      "additionalProperties": false
+    },
+
+    "volume": {
+      "id": "#/definitions/volume",
+      "type": ["object", "null"],
+      "properties": {
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          },
+          "additionalProperties": false
+        },
+        "labels": {"$ref": "#/definitions/labels"},
+        "name": {"type": "string"}
+      },
+      "additionalProperties": false
+    },
+
+    "string_or_list": {
+      "oneOf": [
+        {"type": "string"},
+        {"$ref": "#/definitions/list_of_strings"}
+      ]
+    },
+
+    "list_of_strings": {
+      "type": "array",
+      "items": {"type": "string"},
+      "uniqueItems": true
+    },
+
+    "list_or_dict": {
+      "oneOf": [
+        {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": ["string", "number", "null"]
+            }
+          },
+          "additionalProperties": false
+        },
+        {"type": "array", "items": {"type": "string"}, "uniqueItems": true}
+      ]
+    },
+
+    "labels": {
+      "oneOf": [
+        {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": "string"
+            }
+          },
+          "additionalProperties": false
+        },
+        {"type": "array", "items": {"type": "string"}, "uniqueItems": true}
+      ]
+    },
+
+    "blkio_limit": {
+      "type": "object",
+      "properties": {
+        "path": {"type": "string"},
+        "rate": {"type": ["integer", "string"]}
+      },
+      "additionalProperties": false
+    },
+    "blkio_weight": {
+      "type": "object",
+      "properties": {
+        "path": {"type": "string"},
+        "weight": {"type": "integer"}
+      },
+      "additionalProperties": false
+    },
+
+    "constraints": {
+      "service": {
+        "id": "#/definitions/constraints/service",
+        "anyOf": [
+          {"required": ["build"]},
+          {"required": ["image"]}
+        ],
+        "properties": {
+          "build": {
+            "required": ["context"]
+          }
+        }
+      }
+    }
+  }
+}

compose/const.py

@@ -27,6 +27,7 @@ COMPOSEFILE_V2_0 = ComposeVersion('2.0')
 COMPOSEFILE_V2_1 = ComposeVersion('2.1')
 COMPOSEFILE_V2_2 = ComposeVersion('2.2')
 COMPOSEFILE_V2_3 = ComposeVersion('2.3')
+COMPOSEFILE_V2_4 = ComposeVersion('2.4')
 
 COMPOSEFILE_V3_0 = ComposeVersion('3.0')
 COMPOSEFILE_V3_1 = ComposeVersion('3.1')
@@ -42,6 +43,7 @@ API_VERSIONS = {
     COMPOSEFILE_V2_1: '1.24',
     COMPOSEFILE_V2_2: '1.25',
     COMPOSEFILE_V2_3: '1.30',
+    COMPOSEFILE_V2_4: '1.35',
     COMPOSEFILE_V3_0: '1.25',
     COMPOSEFILE_V3_1: '1.25',
     COMPOSEFILE_V3_2: '1.25',
@@ -57,6 +59,7 @@ API_VERSION_TO_ENGINE_VERSION = {
     API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0',
     API_VERSIONS[COMPOSEFILE_V2_2]: '1.13.0',
     API_VERSIONS[COMPOSEFILE_V2_3]: '17.06.0',
+    API_VERSIONS[COMPOSEFILE_V2_4]: '17.12.0',
     API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0',
     API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0',
     API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0',

compose/project.py

@@ -77,7 +77,7 @@ class Project(object):
         return labels
 
     @classmethod
-    def from_config(cls, name, config_data, client):
+    def from_config(cls, name, config_data, client, default_platform=None):
         """
         Construct a Project from a config.Config object.
         """
@@ -128,6 +128,7 @@ class Project(object):
                     volumes_from=volumes_from,
                     secrets=secrets,
                     pid_mode=pid_mode,
+                    platform=service_dict.pop('platform', default_platform),
                     **service_dict)
             )
 

compose/service.py

@@ -998,6 +998,12 @@ class Service(object):
         if not six.PY3 and not IS_WINDOWS_PLATFORM:
             path = path.encode('utf8')
 
+        platform = self.options.get('platform')
+        if platform and version_lt(self.client.api_version, '1.35'):
+            raise OperationFailedError(
+                'Impossible to perform platform-targeted builds for API version < 1.35'
+            )
+
         build_output = self.client.build(
             path=path,
             tag=self.image_name,
@@ -1018,6 +1024,7 @@ class Service(object):
             },
             gzip=gzip,
             isolation=build_opts.get('isolation', self.options.get('isolation', None)),
+            platform=platform,
         )
 
         try:
@@ -1119,11 +1126,20 @@ class Service(object):
             return
 
         repo, tag, separator = parse_repository_tag(self.options['image'])
-        tag = tag or 'latest'
+        kwargs = {
+            'tag': tag or 'latest',
+            'stream': True,
+            'platform': self.options.get('platform'),
+        }
         if not silent:
             log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag))
+
+        if kwargs['platform'] and version_lt(self.client.api_version, '1.35'):
+            raise OperationFailedError(
+                'Impossible to perform platform-targeted builds for API version < 1.35'
+            )
         try:
-            output = self.client.pull(repo, tag=tag, stream=True)
+            output = self.client.pull(repo, **kwargs)
             if silent:
                 with open(os.devnull, 'w') as devnull:
                     return progress_stream.get_digest_from_pull(

docker-compose.spec

@@ -42,6 +42,11 @@ exe = EXE(pyz,
                 'compose/config/config_schema_v2.3.json',
                 'DATA'
             ),
+            (
+                'compose/config/config_schema_v2.4.json',
+                'compose/config/config_schema_v2.4.json',
+                'DATA'
+            ),
             (
                 'compose/config/config_schema_v3.0.json',
                 'compose/config/config_schema_v3.0.json',

tests/unit/project_test.py

@@ -13,6 +13,7 @@ from compose.config.config import Config
 from compose.config.types import VolumeFromSpec
 from compose.const import COMPOSEFILE_V1 as V1
 from compose.const import COMPOSEFILE_V2_0 as V2_0
+from compose.const import COMPOSEFILE_V2_4 as V2_4
 from compose.const import LABEL_SERVICE
 from compose.container import Container
 from compose.project import NoSuchService
@@ -561,3 +562,29 @@ class ProjectTest(unittest.TestCase):
     def test_no_such_service_unicode(self):
         assert NoSuchService('十六夜　咲夜'.encode('utf-8')).msg == 'No such service: 十六夜　咲夜'
         assert NoSuchService('十六夜　咲夜').msg == 'No such service: 十六夜　咲夜'
+
+    def test_project_platform_value(self):
+        service_config = {
+            'name': 'web',
+            'image': 'busybox:latest',
+        }
+        config_data = Config(
+            version=V2_4, services=[service_config], networks={}, volumes={}, secrets=None, configs=None
+        )
+
+        project = Project.from_config(name='test', client=self.mock_client, config_data=config_data)
+        assert project.get_service('web').options.get('platform') is None
+
+        project = Project.from_config(
+            name='test', client=self.mock_client, config_data=config_data, default_platform='windows'
+        )
+        assert project.get_service('web').options.get('platform') == 'windows'
+
+        service_config['platform'] = 'linux/s390x'
+        project = Project.from_config(name='test', client=self.mock_client, config_data=config_data)
+        assert project.get_service('web').options.get('platform') == 'linux/s390x'
+
+        project = Project.from_config(
+            name='test', client=self.mock_client, config_data=config_data, default_platform='windows'
+        )
+        assert project.get_service('web').options.get('platform') == 'linux/s390x'

tests/unit/service_test.py

@@ -21,6 +21,7 @@ from compose.const import LABEL_PROJECT
 from compose.const import LABEL_SERVICE
 from compose.const import SECRETS_PATH
 from compose.container import Container
+from compose.errors import OperationFailedError
 from compose.parallel import ParallelStreamWriter
 from compose.project import OneOffFilter
 from compose.service import build_ulimits
@@ -400,7 +401,8 @@ class ServiceTest(unittest.TestCase):
         self.mock_client.pull.assert_called_once_with(
             'someimage',
             tag='sometag',
-            stream=True)
+            stream=True,
+            platform=None)
         mock_log.info.assert_called_once_with('Pulling foo (someimage:sometag)...')
 
     def test_pull_image_no_tag(self):
@@ -409,7 +411,8 @@ class ServiceTest(unittest.TestCase):
         self.mock_client.pull.assert_called_once_with(
             'ababab',
             tag='latest',
-            stream=True)
+            stream=True,
+            platform=None)
 
     @mock.patch('compose.service.log', autospec=True)
     def test_pull_image_digest(self, mock_log):
@@ -418,9 +421,30 @@ class ServiceTest(unittest.TestCase):
         self.mock_client.pull.assert_called_once_with(
             'someimage',
             tag='sha256:1234',
-            stream=True)
+            stream=True,
+            platform=None)
         mock_log.info.assert_called_once_with('Pulling foo (someimage@sha256:1234)...')
 
+    @mock.patch('compose.service.log', autospec=True)
+    def test_pull_image_with_platform(self, mock_log):
+        self.mock_client.api_version = '1.35'
+        service = Service(
+            'foo', client=self.mock_client, image='someimage:sometag', platform='windows/x86_64'
+        )
+        service.pull()
+        assert self.mock_client.pull.call_count == 1
+        call_args = self.mock_client.pull.call_args
+        assert call_args[1]['platform'] == 'windows/x86_64'
+
+    @mock.patch('compose.service.log', autospec=True)
+    def test_pull_image_with_platform_unsupported_api(self, mock_log):
+        self.mock_client.api_version = '1.33'
+        service = Service(
+            'foo', client=self.mock_client, image='someimage:sometag', platform='linux/arm'
+        )
+        with pytest.raises(OperationFailedError):
+            service.pull()
+
     @mock.patch('compose.service.Container', autospec=True)
     def test_recreate_container(self, _):
         mock_container = mock.create_autospec(Container)
@@ -513,6 +537,19 @@ class ServiceTest(unittest.TestCase):
         assert self.mock_client.build.call_count == 1
         assert not self.mock_client.build.call_args[1]['pull']
 
+    def test_build_does_with_platform(self):
+        self.mock_client.api_version = '1.35'
+        self.mock_client.build.return_value = [
+            b'{"stream": "Successfully built 12345"}',
+        ]
+
+        service = Service('foo', client=self.mock_client, build={'context': '.'}, platform='linux')
+        service.build()
+
+        assert self.mock_client.build.call_count == 1
+        call_args = self.mock_client.build.call_args
+        assert call_args[1]['platform'] == 'linux'
+
     def test_build_with_override_build_args(self):
         self.mock_client.build.return_value = [
             b'{"stream": "Successfully built 12345"}',