Add support for build arguments

Allows 'build' configuration option to be specified as an
object and adds support for build args.

Signed-off-by: Garrett Heel <garrettheel@gmail.com>

compose/config/config.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import codecs
+import functools
 import logging
 import operator
 import os
@@ -455,6 +456,12 @@ def resolve_environment(service_dict):
     return dict(resolve_env_var(k, v) for k, v in six.iteritems(env))
 
 
+def resolve_build_args(build):
+    args = {}
+    args.update(parse_build_arguments(build.get('args')))
+    return dict(resolve_env_var(k, v) for k, v in six.iteritems(args))
+
+
 def validate_extended_service_dict(service_dict, filename, service):
     error_prefix = "Cannot extend service '%s' in %s:" % (service, filename)
 
@@ -492,12 +499,16 @@ def process_service(service_config):
             for path in to_list(service_dict['env_file'])
         ]
 
+    if 'build' in service_dict:
+        if isinstance(service_dict['build'], six.string_types):
+            service_dict['build'] = resolve_build_path(working_dir, service_dict['build'])
+        elif isinstance(service_dict['build'], dict) and 'context' in service_dict['build']:
+            path = service_dict['build']['context']
+            service_dict['build']['context'] = resolve_build_path(working_dir, path)
+
     if 'volumes' in service_dict and service_dict.get('volume_driver') is None:
         service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict)
 
-    if 'build' in service_dict:
-        service_dict['build'] = resolve_build_path(working_dir, service_dict['build'])
-
     if 'labels' in service_dict:
         service_dict['labels'] = parse_labels(service_dict['labels'])
 
@@ -535,6 +546,8 @@ def finalize_service(service_config, service_names, version):
     if 'restart' in service_dict:
         service_dict['restart'] = parse_restart_spec(service_dict['restart'])
 
+    normalize_build(service_dict, service_config.working_dir)
+
     return normalize_v1_service_format(service_dict)
 
 
@@ -599,10 +612,31 @@ def merge_service_dicts(base, override, version):
 
     if version == 1:
         legacy_v1_merge_image_or_build(d, base, override)
+    else:
+        merge_build(d, base, override)
 
     return d
 
 
+def merge_build(output, base, override):
+    build = {}
+
+    if 'build' in base:
+        if isinstance(base['build'], six.string_types):
+            build['context'] = base['build']
+        else:
+            build.update(base['build'])
+
+    if 'build' in override:
+        if isinstance(override['build'], six.string_types):
+            build['context'] = override['build']
+        else:
+            build.update(override['build'])
+
+    if build:
+        output['build'] = build
+
+
 def legacy_v1_merge_image_or_build(output, base, override):
     output.pop('image', None)
     output.pop('build', None)
@@ -622,22 +656,6 @@ def merge_environment(base, override):
     return env
 
 
-def parse_environment(environment):
-    if not environment:
-        return {}
-
-    if isinstance(environment, list):
-        return dict(split_env(e) for e in environment)
-
-    if isinstance(environment, dict):
-        return dict(environment)
-
-    raise ConfigurationError(
-        "environment \"%s\" must be a list or mapping," %
-        environment
-    )
-
-
 def split_env(env):
     if isinstance(env, six.binary_type):
         env = env.decode('utf-8', 'replace')
@@ -647,6 +665,34 @@ def split_env(env):
         return env, None
 
 
+def split_label(label):
+    if '=' in label:
+        return label.split('=', 1)
+    else:
+        return label, ''
+
+
+def parse_dict_or_list(split_func, type_name, arguments):
+    if not arguments:
+        return {}
+
+    if isinstance(arguments, list):
+        return dict(split_func(e) for e in arguments)
+
+    if isinstance(arguments, dict):
+        return dict(arguments)
+
+    raise ConfigurationError(
+        "%s \"%s\" must be a list or mapping," %
+        (type_name, arguments)
+    )
+
+
+parse_build_arguments = functools.partial(parse_dict_or_list, split_env, 'build arguments')
+parse_environment = functools.partial(parse_dict_or_list, split_env, 'environment')
+parse_labels = functools.partial(parse_dict_or_list, split_label, 'labels')
+
+
 def resolve_env_var(key, val):
     if val is not None:
         return key, val
@@ -690,6 +736,26 @@ def resolve_volume_path(working_dir, volume):
         return container_path
 
 
+def normalize_build(service_dict, working_dir):
+    build = {}
+
+    # supported in V1 only
+    if 'dockerfile' in service_dict:
+        build['dockerfile'] = service_dict.pop('dockerfile')
+
+    if 'build' in service_dict:
+        # Shortcut where specifying a string is treated as the build context
+        if isinstance(service_dict['build'], six.string_types):
+            build['context'] = service_dict.pop('build')
+        else:
+            build.update(service_dict['build'])
+            if 'args' in build:
+                build['args'] = resolve_build_args(build)
+
+    if build:
+        service_dict['build'] = build
+
+
 def resolve_build_path(working_dir, build_path):
     if is_url(build_path):
         return build_path
@@ -702,7 +768,13 @@ def is_url(build_path):
 
 def validate_paths(service_dict):
     if 'build' in service_dict:
-        build_path = service_dict['build']
+        build = service_dict.get('build', {})
+
+        if isinstance(build, six.string_types):
+            build_path = build
+        elif isinstance(build, dict) and 'context' in build:
+            build_path = build['context']
+
         if (
             not is_url(build_path) and
             (not os.path.exists(build_path) or not os.access(build_path, os.R_OK))

compose/config/service_schema_v2.json

@@ -15,7 +15,20 @@
       "type": "object",
 
       "properties": {
-        "build": {"type": "string"},
+        "build": {
+          "oneOf": [
+            {"type": "string"},
+            {
+              "type": "object",
+              "properties": {
+                "context": {"type": "string"},
+                "dockerfile": {"type": "string"},
+                "args": {"$ref": "#/definitions/list_or_dict"}
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
         "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
         "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
         "cgroup_parent": {"type": "string"},

compose/config/validation.py

@@ -150,18 +150,29 @@ def handle_error_for_schema_with_id(error, service_name):
             VALID_NAME_CHARS)
 
     if schema_id == '#/definitions/constraints':
+        # Build context could in 'build' or 'build.context' and dockerfile could be
+        # in 'dockerfile' or 'build.dockerfile'
+        context = False
+        dockerfile = 'dockerfile' in error.instance
+        if 'build' in error.instance:
+            if isinstance(error.instance['build'], six.string_types):
+                context = True
+            else:
+                context = 'context' in error.instance['build']
+                dockerfile = dockerfile or 'dockerfile' in error.instance['build']
+
         # TODO: only applies to v1
-        if 'image' in error.instance and 'build' in error.instance:
+        if 'image' in error.instance and context:
             return (
                 "Service '{}' has both an image and build path specified. "
                 "A service can either be built to image or use an existing "
                 "image, not both.".format(service_name))
-        if 'image' not in error.instance and 'build' not in error.instance:
+        if 'image' not in error.instance and not context:
             return (
                 "Service '{}' has neither an image nor a build path "
                 "specified. At least one must be provided.".format(service_name))
         # TODO: only applies to v1
-        if 'image' in error.instance and 'dockerfile' in error.instance:
+        if 'image' in error.instance and dockerfile:
             return (
                 "Service '{}' has both an image and alternate Dockerfile. "
                 "A service can either be built to image or use an existing "

compose/service.py

@@ -638,7 +638,8 @@ class Service(object):
     def build(self, no_cache=False, pull=False, force_rm=False):
         log.info('Building %s' % self.name)
 
-        path = self.options['build']
+        build_opts = self.options.get('build', {})
+        path = build_opts.get('context')
         # python2 os.path() doesn't support unicode, so we need to encode it to
         # a byte string
         if not six.PY3:
@@ -652,7 +653,8 @@ class Service(object):
             forcerm=force_rm,
             pull=pull,
             nocache=no_cache,
-            dockerfile=self.options.get('dockerfile', None),
+            dockerfile=build_opts.get('dockerfile', None),
+            buildargs=build_opts.get('args', None),
         )
 
         try:

docs/compose-file.md

@@ -37,7 +37,8 @@ those files, all the [services](#service-configuration-reference) are declared
 at the root of the document.
 
 Version 1 files do not support the declaration of
-named [volumes](#volume-configuration-reference)
+named [volumes](#volume-configuration-reference) or
+[build arguments](#args).
 
 Example:
 
@@ -89,6 +90,30 @@ definition.
 
 ### build
 
+Configuration options that are applied at build time.
+
+In version 1 this must be given as a string representing the context.
+
+  build: .
+
+In version 2 this can alternatively be given as an object with extra options.
+
+  version: 2
+  services:
+    web:
+      build: .
+
+    version: 2
+    services:
+      web:
+        build:
+          context: .
+          dockerfile: Dockerfile-alternate
+          args:
+            buildno: 1
+
+#### context
+
 Either a path to a directory containing a Dockerfile, or a url to a git repository.
 
 When the value supplied is a relative path, it is interpreted as relative to the
@@ -99,9 +124,46 @@ Compose will build and tag it with a generated name, and use that image thereaft
 
     build: /path/to/build/dir
 
-Using `build` together with `image` is not allowed. Attempting to do so results in
+    build:
+      context: /path/to/build/dir
+
+Using `context` together with `image` is not allowed. Attempting to do so results in
 an error.
 
+#### dockerfile
+
+Alternate Dockerfile.
+
+Compose will use an alternate file to build with. A build path must also be
+specified using the `build` key.
+
+    build:
+      context: /path/to/build/dir
+      dockerfile: Dockerfile-alternate
+
+Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error.
+
+#### args
+
+Add build arguments. You can use either an array or a dictionary. Any
+boolean values; true, false, yes, no, need to be enclosed in quotes to ensure
+they are not converted to True or False by the YML parser.
+
+Build arguments with only a key are resolved to their environment value on the
+machine Compose is running on.
+
+> **Note:** Introduced in version 2 of the compose file format.
+
+    build:
+      args:
+        buildno: 1
+        user: someuser
+
+    build:
+      args:
+        - buildno=1
+        - user=someuser
+
 ### cap_add, cap_drop
 
 Add or drop container capabilities.
@@ -194,6 +256,7 @@ The entrypoint can also be a list, in a manner similar to [dockerfile](https://d
         - memory_limit=-1
         - vendor/bin/phpunit
 
+
 ### env_file
 
 Add environment variables from a file. Can be a single value or a list.

tests/integration/service_test.py

@@ -294,7 +294,7 @@ class ServiceTest(DockerClientTestCase):
             project='composetest',
             name='db',
             client=self.client,
-            build='tests/fixtures/dockerfile-with-volume',
+            build={'context': 'tests/fixtures/dockerfile-with-volume'},
         )
 
         old_container = create_and_start_container(service)
@@ -315,7 +315,7 @@ class ServiceTest(DockerClientTestCase):
     def test_execute_convergence_plan_when_image_volume_masks_config(self):
         service = self.create_service(
             'db',
-            build='tests/fixtures/dockerfile-with-volume',
+            build={'context': 'tests/fixtures/dockerfile-with-volume'},
         )
 
         old_container = create_and_start_container(service)
@@ -346,7 +346,7 @@ class ServiceTest(DockerClientTestCase):
     def test_execute_convergence_plan_without_start(self):
         service = self.create_service(
             'db',
-            build='tests/fixtures/dockerfile-with-volume'
+            build={'context': 'tests/fixtures/dockerfile-with-volume'}
         )
 
         containers = service.execute_convergence_plan(ConvergencePlan('create', []), start=False)
@@ -450,7 +450,7 @@ class ServiceTest(DockerClientTestCase):
         service = Service(
             name='test',
             client=self.client,
-            build='tests/fixtures/simple-dockerfile',
+            build={'context': 'tests/fixtures/simple-dockerfile'},
             project='composetest',
         )
         container = create_and_start_container(service)
@@ -463,7 +463,7 @@ class ServiceTest(DockerClientTestCase):
         service = Service(
             name='test',
             client=self.client,
-            build='this/does/not/exist/and/will/throw/error',
+            build={'context': 'this/does/not/exist/and/will/throw/error'},
             project='composetest',
         )
         container = create_and_start_container(service)
@@ -483,7 +483,7 @@ class ServiceTest(DockerClientTestCase):
         with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
             f.write("FROM busybox\n")
 
-        self.create_service('web', build=base_dir).build()
+        self.create_service('web', build={'context': base_dir}).build()
         assert self.client.inspect_image('composetest_web')
 
     def test_build_non_ascii_filename(self):
@@ -496,7 +496,7 @@ class ServiceTest(DockerClientTestCase):
         with open(os.path.join(base_dir.encode('utf8'), b'foo\xE2bar'), 'w') as f:
             f.write("hello world\n")
 
-        self.create_service('web', build=text_type(base_dir)).build()
+        self.create_service('web', build={'context': text_type(base_dir)}).build()
         assert self.client.inspect_image('composetest_web')
 
     def test_build_with_image_name(self):
@@ -508,16 +508,30 @@ class ServiceTest(DockerClientTestCase):
 
         image_name = 'examples/composetest:latest'
         self.addCleanup(self.client.remove_image, image_name)
-        self.create_service('web', build=base_dir, image=image_name).build()
+        self.create_service('web', build={'context': base_dir}, image=image_name).build()
         assert self.client.inspect_image(image_name)
 
     def test_build_with_git_url(self):
         build_url = "https://github.com/dnephin/docker-build-from-url.git"
-        service = self.create_service('buildwithurl', build=build_url)
+        service = self.create_service('buildwithurl', build={'context': build_url})
         self.addCleanup(self.client.remove_image, service.image_name)
         service.build()
         assert service.image()
 
+    def test_build_with_build_args(self):
+        base_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base_dir)
+
+        with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
+            f.write("FROM busybox\n")
+            f.write("ARG build_version\n")
+
+        service = self.create_service('buildwithargs',
+                                      build={'context': text_type(base_dir),
+                                             'args': {"build_version": "1"}})
+        service.build()
+        assert service.image()
+
     def test_start_container_stays_unpriviliged(self):
         service = self.create_service('web')
         container = create_and_start_container(service).inspect()

tests/integration/state_test.py

@@ -266,13 +266,13 @@ class ServiceStateTest(DockerClientTestCase):
         dockerfile = context.join('Dockerfile')
         dockerfile.write(base_image)
 
-        web = self.create_service('web', build=str(context))
+        web = self.create_service('web', build={'context': str(context)})
         container = web.create_container()
 
         dockerfile.write(base_image + 'CMD echo hello world\n')
         web.build()
 
-        web = self.create_service('web', build=str(context))
+        web = self.create_service('web', build={'context': str(context)})
         self.assertEqual(('recreate', [container]), web.convergence_plan())
 
     def test_image_changed_to_build(self):
@@ -286,7 +286,7 @@ class ServiceStateTest(DockerClientTestCase):
         web = self.create_service('web', image='busybox')
         container = web.create_container()
 
-        web = self.create_service('web', build=str(context))
+        web = self.create_service('web', build={'context': str(context)})
         plan = web.convergence_plan()
         self.assertEqual(('recreate', [container]), plan)
         containers = web.execute_convergence_plan(plan)

tests/unit/config/config_test.py

@@ -12,6 +12,7 @@ import py
 import pytest
 
 from compose.config import config
+from compose.config.config import resolve_build_args
 from compose.config.config import resolve_environment
 from compose.config.errors import ConfigurationError
 from compose.config.types import VolumeSpec
@@ -284,7 +285,7 @@ class ConfigTest(unittest.TestCase):
         expected = [
             {
                 'name': 'web',
-                'build': os.path.abspath('/'),
+                'build': {'context': os.path.abspath('/')},
                 'volumes': [VolumeSpec.parse('/home/user/project:/code')],
                 'links': ['db'],
             },
@@ -414,6 +415,59 @@ class ConfigTest(unittest.TestCase):
         assert services[1]['name'] == 'db'
         assert services[2]['name'] == 'web'
 
+    def test_config_build_configuration(self):
+        service = config.load(
+            build_config_details(
+                {'web': {
+                    'build': '.',
+                    'dockerfile': 'Dockerfile-alt'
+                }},
+                'tests/fixtures/extends',
+                'filename.yml'
+            )
+        ).services
+        self.assertTrue('context' in service[0]['build'])
+        self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt')
+
+    def test_config_build_configuration_v2(self):
+        service = config.load(
+            build_config_details(
+                {
+                    'version': 2,
+                    'services': {
+                        'web': {
+                            'build': '.',
+                            'dockerfile': 'Dockerfile-alt'
+                        }
+                    }
+                },
+                'tests/fixtures/extends',
+                'filename.yml'
+            )
+        ).services
+        self.assertTrue('context' in service[0]['build'])
+        self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt')
+
+        service = config.load(
+            build_config_details(
+                {
+                    'version': 2,
+                    'services': {
+                        'web': {
+                            'build': {
+                                'context': '.',
+                                'dockerfile': 'Dockerfile-alt'
+                            }
+                        }
+                    }
+                },
+                'tests/fixtures/extends',
+                'filename.yml'
+            )
+        ).services
+        self.assertTrue('context' in service[0]['build'])
+        self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt')
+
     def test_load_with_multiple_files_v2(self):
         base_file = config.ConfigFile(
             'base.yaml',
@@ -445,7 +499,7 @@ class ConfigTest(unittest.TestCase):
         expected = [
             {
                 'name': 'web',
-                'build': os.path.abspath('/'),
+                'build': {'context': os.path.abspath('/')},
                 'image': 'example/web',
                 'volumes': [VolumeSpec.parse('/home/user/project:/code')],
             },
@@ -1157,7 +1211,7 @@ class BuildOrImageMergeTest(unittest.TestCase):
 
         self.assertEqual(
             config.merge_service_dicts({'image': 'redis'}, {'build': '.'}, V1),
-            {'build': '.'},
+            {'build': '.'}
         )
 
 
@@ -1388,6 +1442,24 @@ class EnvTest(unittest.TestCase):
             },
         )
 
+    @mock.patch.dict(os.environ)
+    def test_resolve_build_args(self):
+        os.environ['env_arg'] = 'value2'
+
+        build = {
+            'context': '.',
+            'args': {
+                'arg1': 'value1',
+                'empty_arg': '',
+                'env_arg': None,
+                'no_env': None
+            }
+        }
+        self.assertEqual(
+            resolve_build_args(build),
+            {'arg1': 'value1', 'empty_arg': '', 'env_arg': 'value2', 'no_env': ''},
+        )
+
     @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash')
     @mock.patch.dict(os.environ)
     def test_resolve_path(self):
@@ -1873,7 +1945,7 @@ class BuildPathTest(unittest.TestCase):
 
     def test_from_file(self):
         service_dict = load_from_filename('tests/fixtures/build-path/docker-compose.yml')
-        self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}])
+        self.assertEquals(service_dict, [{'name': 'foo', 'build': {'context': self.abs_context_path}}])
 
     def test_valid_url_in_build_path(self):
         valid_urls = [
@@ -1888,7 +1960,7 @@ class BuildPathTest(unittest.TestCase):
             service_dict = config.load(build_config_details({
                 'validurl': {'build': valid_url},
             }, '.', None)).services
-            assert service_dict[0]['build'] == valid_url
+            assert service_dict[0]['build'] == {'context': valid_url}
 
     def test_invalid_url_in_build_path(self):
         invalid_urls = [

tests/unit/service_test.py

@@ -355,7 +355,7 @@ class ServiceTest(unittest.TestCase):
         self.assertEqual(parse_repository_tag("url:5000/repo@sha256:digest"), ("url:5000/repo", "sha256:digest", "@"))
 
     def test_create_container_with_build(self):
-        service = Service('foo', client=self.mock_client, build='.')
+        service = Service('foo', client=self.mock_client, build={'context': '.'})
         self.mock_client.inspect_image.side_effect = [
             NoSuchImageError,
             {'Id': 'abc123'},
@@ -374,17 +374,18 @@ class ServiceTest(unittest.TestCase):
             forcerm=False,
             nocache=False,
             rm=True,
+            buildargs=None,
         )
 
     def test_create_container_no_build(self):
-        service = Service('foo', client=self.mock_client, build='.')
+        service = Service('foo', client=self.mock_client, build={'context': '.'})
         self.mock_client.inspect_image.return_value = {'Id': 'abc123'}
 
         service.create_container(do_build=False)
         self.assertFalse(self.mock_client.build.called)
 
     def test_create_container_no_build_but_needs_build(self):
-        service = Service('foo', client=self.mock_client, build='.')
+        service = Service('foo', client=self.mock_client, build={'context': '.'})
         self.mock_client.inspect_image.side_effect = NoSuchImageError
         with self.assertRaises(NeedsBuildError):
             service.create_container(do_build=False)
@@ -394,7 +395,7 @@ class ServiceTest(unittest.TestCase):
             b'{"stream": "Successfully built 12345"}',
         ]
 
-        service = Service('foo', client=self.mock_client, build='.')
+        service = Service('foo', client=self.mock_client, build={'context': '.'})
         service.build()
 
         self.assertEqual(self.mock_client.build.call_count, 1)