From dc838067fd4ccdf381108acb03bac261924fd297 Mon Sep 17 00:00:00 2001 From: Marc van den Hoogen Date: Fri, 18 Aug 2017 13:40:11 +0200 Subject: [PATCH] Add shm_size to build-options (issue #3866) * Add shm_size to build configuration * Make it possible to enlarge/customize shm size during build * Value in bytes, or use string like "512M" or "1G" ... * Add to compose format 2.3 and (provisionally) >=3.5 format * Add automated test for shm_size in build-opts Signed-off-by: Marc van den Hoogen Made unit tests compatible with previously added shm_size build-option Signed-off-by: Marc van den Hoogen Also support shm_size build-opt when conf override Signed-off-by: Marc van den Hoogen Automated test for shm_size build-option Signed-off-by: Marc van den Hoogen Schema 3.4, add shm_size to schema 2.3, updated const.py Signed-off-by: Marc van den Hoogen Corrected typo in config_schema_v3.4 Signed-off-by: Marc van den Hoogen Add support for g/m/k units for shm_size in build-opts Signed-off-by: Marc van den Hoogen Reorder imports in service.py Signed-off-by: Marc van den Hoogen --- compose/config/config.py | 1 + compose/config/config_schema_v2.3.json | 3 +- compose/config/config_schema_v3.5.json | 542 ++++++++++++++++++ compose/const.py | 3 + compose/service.py | 2 + tests/acceptance/cli_test.py | 6 + tests/fixtures/build-shm-size/Dockerfile | 4 + .../build-shm-size/docker-compose.yml | 7 + tests/unit/service_test.py | 2 + 9 files changed, 569 insertions(+), 1 deletion(-) create mode 100644 compose/config/config_schema_v3.5.json create mode 100644 tests/fixtures/build-shm-size/Dockerfile create mode 100644 tests/fixtures/build-shm-size/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index b90ab0305..f16dd01b3 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1020,6 +1020,7 @@ def merge_build(output, base, override): md.merge_scalar('dockerfile') md.merge_scalar('network') md.merge_scalar('target') + md.merge_scalar('shm_size') md.merge_mapping('args', parse_build_arguments) md.merge_field('cache_from', merge_unique_items_lists, default=[]) md.merge_mapping('labels', parse_labels) diff --git a/compose/config/config_schema_v2.3.json b/compose/config/config_schema_v2.3.json index a790bb405..ceaf44954 100644 --- a/compose/config/config_schema_v2.3.json +++ b/compose/config/config_schema_v2.3.json @@ -91,7 +91,8 @@ "labels": {"$ref": "#/definitions/list_or_dict"}, "cache_from": {"$ref": "#/definitions/list_of_strings"}, "network": {"type": "string"}, - "target": {"type": "string"} + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]} }, "additionalProperties": false } diff --git a/compose/config/config_schema_v3.5.json b/compose/config/config_schema_v3.5.json new file mode 100644 index 000000000..fa95d6a24 --- /dev/null +++ b/compose/config/config_schema_v3.5.json @@ -0,0 +1,542 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.5.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secret" + } + }, + "additionalProperties": false + }, + + "configs": { + "id": "#/properties/configs", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/config" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "cache_from": {"$ref": "#/definitions/list_of_strings"}, + "network": {"type": "string"}, + "target": {"type": "string"}, + "shm_size": {"type": ["integer", "string"]} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "configs": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "container_name": {"type": "string"}, + "credential_spec": {"type": "object", "properties": { + "file": {"type": "string"}, + "registry": {"type": "string"} + }}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "oneOf": [ + {"type": "number", "format": "ports"}, + {"type": "string", "format": "ports"}, + { + "type": "object", + "properties": { + "mode": {"type": "string"}, + "target": {"type": "integer"}, + "published": {"type": "integer"}, + "protocol": {"type": "string"} + }, + "additionalProperties": false + } + ] + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": {"type": "string"}, + "source": {"type": "string"}, + "target": {"type": "string"}, + "read_only": {"type": "boolean"}, + "consistency": {"type": "string"}, + "bind": { + "type": "object", + "properties": { + "propagation": {"type": "string"} + } + }, + "volume": { + "type": "object", + "properties": { + "nocopy": {"type": "boolean"} + } + } + } + } + ], + "uniqueItems": true + } + }, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "endpoint_mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"}, + "order": {"type": "string", "enum": [ + "start-first", "stop-first" + ]} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + }, + "additionalProperties": false + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}}, + "preferences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "spread": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "attachable": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "name": {"type": "string"}, + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "config": { + "id": "#/definitions/config", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/const.py b/compose/const.py index b5970f82a..2ac08b89a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -32,6 +32,7 @@ COMPOSEFILE_V3_1 = ComposeVersion('3.1') COMPOSEFILE_V3_2 = ComposeVersion('3.2') COMPOSEFILE_V3_3 = ComposeVersion('3.3') COMPOSEFILE_V3_4 = ComposeVersion('3.4') +COMPOSEFILE_V3_5 = ComposeVersion('3.5') API_VERSIONS = { COMPOSEFILE_V1: '1.21', @@ -44,6 +45,7 @@ API_VERSIONS = { COMPOSEFILE_V3_2: '1.25', COMPOSEFILE_V3_3: '1.30', COMPOSEFILE_V3_4: '1.30', + COMPOSEFILE_V3_5: '1.30', } API_VERSION_TO_ENGINE_VERSION = { @@ -57,4 +59,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V3_2]: '1.13.0', API_VERSIONS[COMPOSEFILE_V3_3]: '17.06.0', API_VERSIONS[COMPOSEFILE_V3_4]: '17.06.0', + API_VERSIONS[COMPOSEFILE_V3_5]: '17.06.0', } diff --git a/compose/service.py b/compose/service.py index 2829240f2..28c032763 100644 --- a/compose/service.py +++ b/compose/service.py @@ -43,6 +43,7 @@ from .parallel import parallel_execute from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash +from .utils import parse_bytes from .utils import parse_seconds_float @@ -916,6 +917,7 @@ class Service(object): buildargs=build_args, network_mode=build_opts.get('network', None), target=build_opts.get('target', None), + shmsize=parse_bytes(build_opts.get('shm_size')) if build_opts.get('shm_size') else None, ) try: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b8cece495..d84c47153 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -527,6 +527,12 @@ class CLITestCase(DockerClientTestCase): ] assert not containers + def test_build_shm_size_build_option(self): + pull_busybox(self.client) + self.base_dir = 'tests/fixtures/build-shm-size' + result = self.dispatch(['build', '--no-cache'], None) + assert 'shm_size: 96' in result.stdout + def test_bundle_with_digests(self): self.base_dir = 'tests/fixtures/bundle-with-digests/' tmpdir = py.test.ensuretemp('cli_test_bundle') diff --git a/tests/fixtures/build-shm-size/Dockerfile b/tests/fixtures/build-shm-size/Dockerfile new file mode 100644 index 000000000..f91733d63 --- /dev/null +++ b/tests/fixtures/build-shm-size/Dockerfile @@ -0,0 +1,4 @@ +FROM busybox + +# Report the shm_size (through the size of /dev/shm) +RUN echo "shm_size:" $(df -h /dev/shm | tail -n 1 | awk '{print $2}') diff --git a/tests/fixtures/build-shm-size/docker-compose.yml b/tests/fixtures/build-shm-size/docker-compose.yml new file mode 100644 index 000000000..238a51322 --- /dev/null +++ b/tests/fixtures/build-shm-size/docker-compose.yml @@ -0,0 +1,7 @@ +version: '3.5' + +services: + custom_shm_size: + build: + context: . + shm_size: 100663296 # =96M diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0293695ab..43ccf081c 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -475,6 +475,7 @@ class ServiceTest(unittest.TestCase): cache_from=None, network_mode=None, target=None, + shmsize=None, ) def test_ensure_image_exists_no_build(self): @@ -515,6 +516,7 @@ class ServiceTest(unittest.TestCase): cache_from=None, network_mode=None, target=None, + shmsize=None ) def test_build_does_not_pull(self):