diff --git a/compose/config/config.py b/compose/config/config.py index c8d93faf6..8200900f8 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import codecs +import functools import logging import operator import os @@ -455,6 +456,12 @@ def resolve_environment(service_dict): return dict(resolve_env_var(k, v) for k, v in six.iteritems(env)) +def resolve_build_args(build): + args = {} + args.update(parse_build_arguments(build.get('args'))) + return dict(resolve_env_var(k, v) for k, v in six.iteritems(args)) + + def validate_extended_service_dict(service_dict, filename, service): error_prefix = "Cannot extend service '%s' in %s:" % (service, filename) @@ -492,12 +499,16 @@ def process_service(service_config): for path in to_list(service_dict['env_file']) ] + if 'build' in service_dict: + if isinstance(service_dict['build'], six.string_types): + service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) + elif isinstance(service_dict['build'], dict) and 'context' in service_dict['build']: + path = service_dict['build']['context'] + service_dict['build']['context'] = resolve_build_path(working_dir, path) + if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) - if 'build' in service_dict: - service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) - if 'labels' in service_dict: service_dict['labels'] = parse_labels(service_dict['labels']) @@ -535,6 +546,8 @@ def finalize_service(service_config, service_names, version): if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) + normalize_build(service_dict, service_config.working_dir) + return normalize_v1_service_format(service_dict) @@ -599,10 +612,31 @@ def merge_service_dicts(base, override, version): if version == 1: legacy_v1_merge_image_or_build(d, base, override) + else: + merge_build(d, base, override) return d +def merge_build(output, base, override): + build = {} + + if 'build' in base: + if isinstance(base['build'], six.string_types): + build['context'] = base['build'] + else: + build.update(base['build']) + + if 'build' in override: + if isinstance(override['build'], six.string_types): + build['context'] = override['build'] + else: + build.update(override['build']) + + if build: + output['build'] = build + + def legacy_v1_merge_image_or_build(output, base, override): output.pop('image', None) output.pop('build', None) @@ -622,22 +656,6 @@ def merge_environment(base, override): return env -def parse_environment(environment): - if not environment: - return {} - - if isinstance(environment, list): - return dict(split_env(e) for e in environment) - - if isinstance(environment, dict): - return dict(environment) - - raise ConfigurationError( - "environment \"%s\" must be a list or mapping," % - environment - ) - - def split_env(env): if isinstance(env, six.binary_type): env = env.decode('utf-8', 'replace') @@ -647,6 +665,34 @@ def split_env(env): return env, None +def split_label(label): + if '=' in label: + return label.split('=', 1) + else: + return label, '' + + +def parse_dict_or_list(split_func, type_name, arguments): + if not arguments: + return {} + + if isinstance(arguments, list): + return dict(split_func(e) for e in arguments) + + if isinstance(arguments, dict): + return dict(arguments) + + raise ConfigurationError( + "%s \"%s\" must be a list or mapping," % + (type_name, arguments) + ) + + +parse_build_arguments = functools.partial(parse_dict_or_list, split_env, 'build arguments') +parse_environment = functools.partial(parse_dict_or_list, split_env, 'environment') +parse_labels = functools.partial(parse_dict_or_list, split_label, 'labels') + + def resolve_env_var(key, val): if val is not None: return key, val @@ -690,6 +736,26 @@ def resolve_volume_path(working_dir, volume): return container_path +def normalize_build(service_dict, working_dir): + build = {} + + # supported in V1 only + if 'dockerfile' in service_dict: + build['dockerfile'] = service_dict.pop('dockerfile') + + if 'build' in service_dict: + # Shortcut where specifying a string is treated as the build context + if isinstance(service_dict['build'], six.string_types): + build['context'] = service_dict.pop('build') + else: + build.update(service_dict['build']) + if 'args' in build: + build['args'] = resolve_build_args(build) + + if build: + service_dict['build'] = build + + def resolve_build_path(working_dir, build_path): if is_url(build_path): return build_path @@ -702,7 +768,13 @@ def is_url(build_path): def validate_paths(service_dict): if 'build' in service_dict: - build_path = service_dict['build'] + build = service_dict.get('build', {}) + + if isinstance(build, six.string_types): + build_path = build + elif isinstance(build, dict) and 'context' in build: + build_path = build['context'] + if ( not is_url(build_path) and (not os.path.exists(build_path) or not os.access(build_path, os.R_OK)) diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index 5f4e04781..23d0381c7 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -15,7 +15,20 @@ "type": "object", "properties": { - "build": {"type": "string"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + } + ] + }, "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "cgroup_parent": {"type": "string"}, diff --git a/compose/config/validation.py b/compose/config/validation.py index 0bf756915..639e8bed2 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -150,18 +150,29 @@ def handle_error_for_schema_with_id(error, service_name): VALID_NAME_CHARS) if schema_id == '#/definitions/constraints': + # Build context could in 'build' or 'build.context' and dockerfile could be + # in 'dockerfile' or 'build.dockerfile' + context = False + dockerfile = 'dockerfile' in error.instance + if 'build' in error.instance: + if isinstance(error.instance['build'], six.string_types): + context = True + else: + context = 'context' in error.instance['build'] + dockerfile = dockerfile or 'dockerfile' in error.instance['build'] + # TODO: only applies to v1 - if 'image' in error.instance and 'build' in error.instance: + if 'image' in error.instance and context: return ( "Service '{}' has both an image and build path specified. " "A service can either be built to image or use an existing " "image, not both.".format(service_name)) - if 'image' not in error.instance and 'build' not in error.instance: + if 'image' not in error.instance and not context: return ( "Service '{}' has neither an image nor a build path " "specified. At least one must be provided.".format(service_name)) # TODO: only applies to v1 - if 'image' in error.instance and 'dockerfile' in error.instance: + if 'image' in error.instance and dockerfile: return ( "Service '{}' has both an image and alternate Dockerfile. " "A service can either be built to image or use an existing " diff --git a/compose/service.py b/compose/service.py index 0a7f0d8e8..c91c3a58c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -638,7 +638,8 @@ class Service(object): def build(self, no_cache=False, pull=False, force_rm=False): log.info('Building %s' % self.name) - path = self.options['build'] + build_opts = self.options.get('build', {}) + path = build_opts.get('context') # python2 os.path() doesn't support unicode, so we need to encode it to # a byte string if not six.PY3: @@ -652,7 +653,8 @@ class Service(object): forcerm=force_rm, pull=pull, nocache=no_cache, - dockerfile=self.options.get('dockerfile', None), + dockerfile=build_opts.get('dockerfile', None), + buildargs=build_opts.get('args', None), ) try: diff --git a/docs/compose-file.md b/docs/compose-file.md index 4759cde05..a9e540148 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -37,7 +37,8 @@ those files, all the [services](#service-configuration-reference) are declared at the root of the document. Version 1 files do not support the declaration of -named [volumes](#volume-configuration-reference) +named [volumes](#volume-configuration-reference) or +[build arguments](#args). Example: @@ -89,6 +90,30 @@ definition. ### build +Configuration options that are applied at build time. + +In version 1 this must be given as a string representing the context. + + build: . + +In version 2 this can alternatively be given as an object with extra options. + + version: 2 + services: + web: + build: . + + version: 2 + services: + web: + build: + context: . + dockerfile: Dockerfile-alternate + args: + buildno: 1 + +#### context + Either a path to a directory containing a Dockerfile, or a url to a git repository. When the value supplied is a relative path, it is interpreted as relative to the @@ -99,9 +124,46 @@ Compose will build and tag it with a generated name, and use that image thereaft build: /path/to/build/dir -Using `build` together with `image` is not allowed. Attempting to do so results in + build: + context: /path/to/build/dir + +Using `context` together with `image` is not allowed. Attempting to do so results in an error. +#### dockerfile + +Alternate Dockerfile. + +Compose will use an alternate file to build with. A build path must also be +specified using the `build` key. + + build: + context: /path/to/build/dir + dockerfile: Dockerfile-alternate + +Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. + +#### args + +Add build arguments. You can use either an array or a dictionary. Any +boolean values; true, false, yes, no, need to be enclosed in quotes to ensure +they are not converted to True or False by the YML parser. + +Build arguments with only a key are resolved to their environment value on the +machine Compose is running on. + +> **Note:** Introduced in version 2 of the compose file format. + + build: + args: + buildno: 1 + user: someuser + + build: + args: + - buildno=1 + - user=someuser + ### cap_add, cap_drop Add or drop container capabilities. @@ -194,6 +256,7 @@ The entrypoint can also be a list, in a manner similar to [dockerfile](https://d - memory_limit=-1 - vendor/bin/phpunit + ### env_file Add environment variables from a file. Can be a single value or a list. diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 37ceb65c4..0e91dcf7c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -294,7 +294,7 @@ class ServiceTest(DockerClientTestCase): project='composetest', name='db', client=self.client, - build='tests/fixtures/dockerfile-with-volume', + build={'context': 'tests/fixtures/dockerfile-with-volume'}, ) old_container = create_and_start_container(service) @@ -315,7 +315,7 @@ class ServiceTest(DockerClientTestCase): def test_execute_convergence_plan_when_image_volume_masks_config(self): service = self.create_service( 'db', - build='tests/fixtures/dockerfile-with-volume', + build={'context': 'tests/fixtures/dockerfile-with-volume'}, ) old_container = create_and_start_container(service) @@ -346,7 +346,7 @@ class ServiceTest(DockerClientTestCase): def test_execute_convergence_plan_without_start(self): service = self.create_service( 'db', - build='tests/fixtures/dockerfile-with-volume' + build={'context': 'tests/fixtures/dockerfile-with-volume'} ) containers = service.execute_convergence_plan(ConvergencePlan('create', []), start=False) @@ -450,7 +450,7 @@ class ServiceTest(DockerClientTestCase): service = Service( name='test', client=self.client, - build='tests/fixtures/simple-dockerfile', + build={'context': 'tests/fixtures/simple-dockerfile'}, project='composetest', ) container = create_and_start_container(service) @@ -463,7 +463,7 @@ class ServiceTest(DockerClientTestCase): service = Service( name='test', client=self.client, - build='this/does/not/exist/and/will/throw/error', + build={'context': 'this/does/not/exist/and/will/throw/error'}, project='composetest', ) container = create_and_start_container(service) @@ -483,7 +483,7 @@ class ServiceTest(DockerClientTestCase): with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: f.write("FROM busybox\n") - self.create_service('web', build=base_dir).build() + self.create_service('web', build={'context': base_dir}).build() assert self.client.inspect_image('composetest_web') def test_build_non_ascii_filename(self): @@ -496,7 +496,7 @@ class ServiceTest(DockerClientTestCase): with open(os.path.join(base_dir.encode('utf8'), b'foo\xE2bar'), 'w') as f: f.write("hello world\n") - self.create_service('web', build=text_type(base_dir)).build() + self.create_service('web', build={'context': text_type(base_dir)}).build() assert self.client.inspect_image('composetest_web') def test_build_with_image_name(self): @@ -508,16 +508,30 @@ class ServiceTest(DockerClientTestCase): image_name = 'examples/composetest:latest' self.addCleanup(self.client.remove_image, image_name) - self.create_service('web', build=base_dir, image=image_name).build() + self.create_service('web', build={'context': base_dir}, image=image_name).build() assert self.client.inspect_image(image_name) def test_build_with_git_url(self): build_url = "https://github.com/dnephin/docker-build-from-url.git" - service = self.create_service('buildwithurl', build=build_url) + service = self.create_service('buildwithurl', build={'context': build_url}) self.addCleanup(self.client.remove_image, service.image_name) service.build() assert service.image() + def test_build_with_build_args(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + f.write("ARG build_version\n") + + service = self.create_service('buildwithargs', + build={'context': text_type(base_dir), + 'args': {"build_version": "1"}}) + service.build() + assert service.image() + def test_start_container_stays_unpriviliged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 6e656c292..36099d2dd 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -266,13 +266,13 @@ class ServiceStateTest(DockerClientTestCase): dockerfile = context.join('Dockerfile') dockerfile.write(base_image) - web = self.create_service('web', build=str(context)) + web = self.create_service('web', build={'context': str(context)}) container = web.create_container() dockerfile.write(base_image + 'CMD echo hello world\n') web.build() - web = self.create_service('web', build=str(context)) + web = self.create_service('web', build={'context': str(context)}) self.assertEqual(('recreate', [container]), web.convergence_plan()) def test_image_changed_to_build(self): @@ -286,7 +286,7 @@ class ServiceStateTest(DockerClientTestCase): web = self.create_service('web', image='busybox') container = web.create_container() - web = self.create_service('web', build=str(context)) + web = self.create_service('web', build={'context': str(context)}) plan = web.convergence_plan() self.assertEqual(('recreate', [container]), plan) containers = web.execute_convergence_plan(plan) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index f0d432d58..ddb992fd5 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -12,6 +12,7 @@ import py import pytest from compose.config import config +from compose.config.config import resolve_build_args from compose.config.config import resolve_environment from compose.config.errors import ConfigurationError from compose.config.types import VolumeSpec @@ -284,7 +285,7 @@ class ConfigTest(unittest.TestCase): expected = [ { 'name': 'web', - 'build': os.path.abspath('/'), + 'build': {'context': os.path.abspath('/')}, 'volumes': [VolumeSpec.parse('/home/user/project:/code')], 'links': ['db'], }, @@ -414,6 +415,59 @@ class ConfigTest(unittest.TestCase): assert services[1]['name'] == 'db' assert services[2]['name'] == 'web' + def test_config_build_configuration(self): + service = config.load( + build_config_details( + {'web': { + 'build': '.', + 'dockerfile': 'Dockerfile-alt' + }}, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services + self.assertTrue('context' in service[0]['build']) + self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt') + + def test_config_build_configuration_v2(self): + service = config.load( + build_config_details( + { + 'version': 2, + 'services': { + 'web': { + 'build': '.', + 'dockerfile': 'Dockerfile-alt' + } + } + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services + self.assertTrue('context' in service[0]['build']) + self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt') + + service = config.load( + build_config_details( + { + 'version': 2, + 'services': { + 'web': { + 'build': { + 'context': '.', + 'dockerfile': 'Dockerfile-alt' + } + } + } + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services + self.assertTrue('context' in service[0]['build']) + self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt') + def test_load_with_multiple_files_v2(self): base_file = config.ConfigFile( 'base.yaml', @@ -445,7 +499,7 @@ class ConfigTest(unittest.TestCase): expected = [ { 'name': 'web', - 'build': os.path.abspath('/'), + 'build': {'context': os.path.abspath('/')}, 'image': 'example/web', 'volumes': [VolumeSpec.parse('/home/user/project:/code')], }, @@ -1157,7 +1211,7 @@ class BuildOrImageMergeTest(unittest.TestCase): self.assertEqual( config.merge_service_dicts({'image': 'redis'}, {'build': '.'}, V1), - {'build': '.'}, + {'build': '.'} ) @@ -1388,6 +1442,24 @@ class EnvTest(unittest.TestCase): }, ) + @mock.patch.dict(os.environ) + def test_resolve_build_args(self): + os.environ['env_arg'] = 'value2' + + build = { + 'context': '.', + 'args': { + 'arg1': 'value1', + 'empty_arg': '', + 'env_arg': None, + 'no_env': None + } + } + self.assertEqual( + resolve_build_args(build), + {'arg1': 'value1', 'empty_arg': '', 'env_arg': 'value2', 'no_env': ''}, + ) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @mock.patch.dict(os.environ) def test_resolve_path(self): @@ -1873,7 +1945,7 @@ class BuildPathTest(unittest.TestCase): def test_from_file(self): service_dict = load_from_filename('tests/fixtures/build-path/docker-compose.yml') - self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}]) + self.assertEquals(service_dict, [{'name': 'foo', 'build': {'context': self.abs_context_path}}]) def test_valid_url_in_build_path(self): valid_urls = [ @@ -1888,7 +1960,7 @@ class BuildPathTest(unittest.TestCase): service_dict = config.load(build_config_details({ 'validurl': {'build': valid_url}, }, '.', None)).services - assert service_dict[0]['build'] == valid_url + assert service_dict[0]['build'] == {'context': valid_url} def test_invalid_url_in_build_path(self): invalid_urls = [ diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 9a3e13b45..c9244a47d 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -355,7 +355,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual(parse_repository_tag("url:5000/repo@sha256:digest"), ("url:5000/repo", "sha256:digest", "@")) def test_create_container_with_build(self): - service = Service('foo', client=self.mock_client, build='.') + service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.side_effect = [ NoSuchImageError, {'Id': 'abc123'}, @@ -374,17 +374,18 @@ class ServiceTest(unittest.TestCase): forcerm=False, nocache=False, rm=True, + buildargs=None, ) def test_create_container_no_build(self): - service = Service('foo', client=self.mock_client, build='.') + service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.return_value = {'Id': 'abc123'} service.create_container(do_build=False) self.assertFalse(self.mock_client.build.called) def test_create_container_no_build_but_needs_build(self): - service = Service('foo', client=self.mock_client, build='.') + service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.side_effect = NoSuchImageError with self.assertRaises(NeedsBuildError): service.create_container(do_build=False) @@ -394,7 +395,7 @@ class ServiceTest(unittest.TestCase): b'{"stream": "Successfully built 12345"}', ] - service = Service('foo', client=self.mock_client, build='.') + service = Service('foo', client=self.mock_client, build={'context': '.'}) service.build() self.assertEqual(self.mock_client.build.call_count, 1)