Merge pull request #2653 from shin-/Runscope-build-args

Support for build arguments
This commit is contained in:
Aanand Prasad 2016-01-14 23:13:54 +00:00
commit 47e53b49c2
9 changed files with 317 additions and 88 deletions

View File

@ -2,6 +2,7 @@ from __future__ import absolute_import
from __future__ import unicode_literals
import codecs
import functools
import logging
import operator
import os
@ -462,6 +463,11 @@ def resolve_environment(service_dict):
return dict(resolve_env_var(k, v) for k, v in six.iteritems(env))
def resolve_build_args(build):
args = parse_build_arguments(build.get('args'))
return dict(resolve_env_var(k, v) for k, v in six.iteritems(args))
def validate_extended_service_dict(service_dict, filename, service):
error_prefix = "Cannot extend service '%s' in %s:" % (service, filename)
@ -499,12 +505,16 @@ def process_service(service_config):
for path in to_list(service_dict['env_file'])
]
if 'build' in service_dict:
if isinstance(service_dict['build'], six.string_types):
service_dict['build'] = resolve_build_path(working_dir, service_dict['build'])
elif isinstance(service_dict['build'], dict) and 'context' in service_dict['build']:
path = service_dict['build']['context']
service_dict['build']['context'] = resolve_build_path(working_dir, path)
if 'volumes' in service_dict and service_dict.get('volume_driver') is None:
service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict)
if 'build' in service_dict:
service_dict['build'] = resolve_build_path(working_dir, service_dict['build'])
if 'labels' in service_dict:
service_dict['labels'] = parse_labels(service_dict['labels'])
@ -542,6 +552,8 @@ def finalize_service(service_config, service_names, version):
if 'restart' in service_dict:
service_dict['restart'] = parse_restart_spec(service_dict['restart'])
normalize_build(service_dict, service_config.working_dir)
return normalize_v1_service_format(service_dict)
@ -556,6 +568,12 @@ def normalize_v1_service_format(service_dict):
service_dict['logging']['options'] = service_dict['log_opt']
del service_dict['log_opt']
if 'dockerfile' in service_dict:
service_dict['build'] = service_dict.get('build', {})
service_dict['build'].update({
'dockerfile': service_dict.pop('dockerfile')
})
return service_dict
@ -606,10 +624,31 @@ def merge_service_dicts(base, override, version):
if version == 1:
legacy_v1_merge_image_or_build(d, base, override)
else:
merge_build(d, base, override)
return d
def merge_build(output, base, override):
build = {}
if 'build' in base:
if isinstance(base['build'], six.string_types):
build['context'] = base['build']
else:
build.update(base['build'])
if 'build' in override:
if isinstance(override['build'], six.string_types):
build['context'] = override['build']
else:
build.update(override['build'])
if build:
output['build'] = build
def legacy_v1_merge_image_or_build(output, base, override):
output.pop('image', None)
output.pop('build', None)
@ -629,22 +668,6 @@ def merge_environment(base, override):
return env
def parse_environment(environment):
if not environment:
return {}
if isinstance(environment, list):
return dict(split_env(e) for e in environment)
if isinstance(environment, dict):
return dict(environment)
raise ConfigurationError(
"environment \"%s\" must be a list or mapping," %
environment
)
def split_env(env):
if isinstance(env, six.binary_type):
env = env.decode('utf-8', 'replace')
@ -654,6 +677,42 @@ def split_env(env):
return env, None
def split_label(label):
if '=' in label:
return label.split('=', 1)
else:
return label, ''
def parse_dict_or_list(split_func, type_name, arguments):
if not arguments:
return {}
if isinstance(arguments, list):
return dict(split_func(e) for e in arguments)
if isinstance(arguments, dict):
return dict(arguments)
raise ConfigurationError(
"%s \"%s\" must be a list or mapping," %
(type_name, arguments)
)
parse_build_arguments = functools.partial(parse_dict_or_list, split_env, 'build arguments')
parse_environment = functools.partial(parse_dict_or_list, split_env, 'environment')
parse_labels = functools.partial(parse_dict_or_list, split_label, 'labels')
def parse_ulimits(ulimits):
if not ulimits:
return {}
if isinstance(ulimits, dict):
return dict(ulimits)
def resolve_env_var(key, val):
if val is not None:
return key, val
@ -697,6 +756,21 @@ def resolve_volume_path(working_dir, volume):
return container_path
def normalize_build(service_dict, working_dir):
if 'build' in service_dict:
build = {}
# Shortcut where specifying a string is treated as the build context
if isinstance(service_dict['build'], six.string_types):
build['context'] = service_dict.pop('build')
else:
build.update(service_dict['build'])
if 'args' in build:
build['args'] = resolve_build_args(build)
service_dict['build'] = build
def resolve_build_path(working_dir, build_path):
if is_url(build_path):
return build_path
@ -709,7 +783,13 @@ def is_url(build_path):
def validate_paths(service_dict):
if 'build' in service_dict:
build_path = service_dict['build']
build = service_dict.get('build', {})
if isinstance(build, six.string_types):
build_path = build
elif isinstance(build, dict) and 'context' in build:
build_path = build['context']
if (
not is_url(build_path) and
(not os.path.exists(build_path) or not os.access(build_path, os.R_OK))
@ -764,32 +844,6 @@ def join_path_mapping(pair):
return ":".join((host, container))
def parse_labels(labels):
if not labels:
return {}
if isinstance(labels, list):
return dict(split_label(e) for e in labels)
if isinstance(labels, dict):
return dict(labels)
def split_label(label):
if '=' in label:
return label.split('=', 1)
else:
return label, ''
def parse_ulimits(ulimits):
if not ulimits:
return {}
if isinstance(ulimits, dict):
return dict(ulimits)
def expand_path(working_dir, path):
return os.path.abspath(os.path.join(working_dir, os.path.expanduser(path)))

View File

@ -15,7 +15,20 @@
"type": "object",
"properties": {
"build": {"type": "string"},
"build": {
"oneOf": [
{"type": "string"},
{
"type": "object",
"properties": {
"context": {"type": "string"},
"dockerfile": {"type": "string"},
"args": {"$ref": "#/definitions/list_or_dict"}
},
"additionalProperties": false
}
]
},
"cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"cgroup_parent": {"type": "string"},
@ -32,7 +45,6 @@
"devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"dns": {"$ref": "#/definitions/string_or_list"},
"dns_search": {"$ref": "#/definitions/string_or_list"},
"dockerfile": {"type": "string"},
"domainname": {"type": "string"},
"entrypoint": {
"oneOf": [

View File

@ -150,18 +150,29 @@ def handle_error_for_schema_with_id(error, service_name):
VALID_NAME_CHARS)
if schema_id == '#/definitions/constraints':
# Build context could in 'build' or 'build.context' and dockerfile could be
# in 'dockerfile' or 'build.dockerfile'
context = False
dockerfile = 'dockerfile' in error.instance
if 'build' in error.instance:
if isinstance(error.instance['build'], six.string_types):
context = True
else:
context = 'context' in error.instance['build']
dockerfile = dockerfile or 'dockerfile' in error.instance['build']
# TODO: only applies to v1
if 'image' in error.instance and 'build' in error.instance:
if 'image' in error.instance and context:
return (
"Service '{}' has both an image and build path specified. "
"A service can either be built to image or use an existing "
"image, not both.".format(service_name))
if 'image' not in error.instance and 'build' not in error.instance:
if 'image' not in error.instance and not context:
return (
"Service '{}' has neither an image nor a build path "
"specified. At least one must be provided.".format(service_name))
# TODO: only applies to v1
if 'image' in error.instance and 'dockerfile' in error.instance:
if 'image' in error.instance and dockerfile:
return (
"Service '{}' has both an image and alternate Dockerfile. "
"A service can either be built to image or use an existing "

View File

@ -638,7 +638,8 @@ class Service(object):
def build(self, no_cache=False, pull=False, force_rm=False):
log.info('Building %s' % self.name)
path = self.options['build']
build_opts = self.options.get('build', {})
path = build_opts.get('context')
# python2 os.path() doesn't support unicode, so we need to encode it to
# a byte string
if not six.PY3:
@ -652,7 +653,8 @@ class Service(object):
forcerm=force_rm,
pull=pull,
nocache=no_cache,
dockerfile=self.options.get('dockerfile', None),
dockerfile=build_opts.get('dockerfile', None),
buildargs=build_opts.get('args', None),
)
try:

View File

@ -37,7 +37,8 @@ those files, all the [services](#service-configuration-reference) are declared
at the root of the document.
Version 1 files do not support the declaration of
named [volumes](#volume-configuration-reference)
named [volumes](#volume-configuration-reference) or
[build arguments](#args).
Example:
@ -89,6 +90,30 @@ definition.
### build
Configuration options that are applied at build time.
In version 1 this must be given as a string representing the context.
build: .
In version 2 this can alternatively be given as an object with extra options.
version: 2
services:
web:
build: .
version: 2
services:
web:
build:
context: .
dockerfile: Dockerfile-alternate
args:
buildno: 1
#### context
Either a path to a directory containing a Dockerfile, or a url to a git repository.
When the value supplied is a relative path, it is interpreted as relative to the
@ -99,9 +124,46 @@ Compose will build and tag it with a generated name, and use that image thereaft
build: /path/to/build/dir
Using `build` together with `image` is not allowed. Attempting to do so results in
build:
context: /path/to/build/dir
Using `context` together with `image` is not allowed. Attempting to do so results in
an error.
#### dockerfile
Alternate Dockerfile.
Compose will use an alternate file to build with. A build path must also be
specified using the `build` key.
build:
context: /path/to/build/dir
dockerfile: Dockerfile-alternate
Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error.
#### args
Add build arguments. You can use either an array or a dictionary. Any
boolean values; true, false, yes, no, need to be enclosed in quotes to ensure
they are not converted to True or False by the YML parser.
Build arguments with only a key are resolved to their environment value on the
machine Compose is running on.
> **Note:** Introduced in version 2 of the compose file format.
build:
args:
buildno: 1
user: someuser
build:
args:
- buildno=1
- user=someuser
### cap_add, cap_drop
Add or drop container capabilities.
@ -166,18 +228,6 @@ Custom DNS search domains. Can be a single value or a list.
- dc1.example.com
- dc2.example.com
### dockerfile
Alternate Dockerfile.
Compose will use an alternate file to build with. A build path must also be
specified using the `build` key.
build: /path/to/build/dir
dockerfile: Dockerfile-alternate
Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error.
### entrypoint
Override the default entrypoint.
@ -194,6 +244,7 @@ The entrypoint can also be a list, in a manner similar to [dockerfile](https://d
- memory_limit=-1
- vendor/bin/phpunit
### env_file
Add environment variables from a file. Can be a single value or a list.

View File

@ -294,7 +294,7 @@ class ServiceTest(DockerClientTestCase):
project='composetest',
name='db',
client=self.client,
build='tests/fixtures/dockerfile-with-volume',
build={'context': 'tests/fixtures/dockerfile-with-volume'},
)
old_container = create_and_start_container(service)
@ -315,7 +315,7 @@ class ServiceTest(DockerClientTestCase):
def test_execute_convergence_plan_when_image_volume_masks_config(self):
service = self.create_service(
'db',
build='tests/fixtures/dockerfile-with-volume',
build={'context': 'tests/fixtures/dockerfile-with-volume'},
)
old_container = create_and_start_container(service)
@ -346,7 +346,7 @@ class ServiceTest(DockerClientTestCase):
def test_execute_convergence_plan_without_start(self):
service = self.create_service(
'db',
build='tests/fixtures/dockerfile-with-volume'
build={'context': 'tests/fixtures/dockerfile-with-volume'}
)
containers = service.execute_convergence_plan(ConvergencePlan('create', []), start=False)
@ -450,7 +450,7 @@ class ServiceTest(DockerClientTestCase):
service = Service(
name='test',
client=self.client,
build='tests/fixtures/simple-dockerfile',
build={'context': 'tests/fixtures/simple-dockerfile'},
project='composetest',
)
container = create_and_start_container(service)
@ -463,7 +463,7 @@ class ServiceTest(DockerClientTestCase):
service = Service(
name='test',
client=self.client,
build='this/does/not/exist/and/will/throw/error',
build={'context': 'this/does/not/exist/and/will/throw/error'},
project='composetest',
)
container = create_and_start_container(service)
@ -483,7 +483,7 @@ class ServiceTest(DockerClientTestCase):
with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
f.write("FROM busybox\n")
self.create_service('web', build=base_dir).build()
self.create_service('web', build={'context': base_dir}).build()
assert self.client.inspect_image('composetest_web')
def test_build_non_ascii_filename(self):
@ -496,7 +496,7 @@ class ServiceTest(DockerClientTestCase):
with open(os.path.join(base_dir.encode('utf8'), b'foo\xE2bar'), 'w') as f:
f.write("hello world\n")
self.create_service('web', build=text_type(base_dir)).build()
self.create_service('web', build={'context': text_type(base_dir)}).build()
assert self.client.inspect_image('composetest_web')
def test_build_with_image_name(self):
@ -508,16 +508,30 @@ class ServiceTest(DockerClientTestCase):
image_name = 'examples/composetest:latest'
self.addCleanup(self.client.remove_image, image_name)
self.create_service('web', build=base_dir, image=image_name).build()
self.create_service('web', build={'context': base_dir}, image=image_name).build()
assert self.client.inspect_image(image_name)
def test_build_with_git_url(self):
build_url = "https://github.com/dnephin/docker-build-from-url.git"
service = self.create_service('buildwithurl', build=build_url)
service = self.create_service('buildwithurl', build={'context': build_url})
self.addCleanup(self.client.remove_image, service.image_name)
service.build()
assert service.image()
def test_build_with_build_args(self):
base_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base_dir)
with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
f.write("FROM busybox\n")
f.write("ARG build_version\n")
service = self.create_service('buildwithargs',
build={'context': text_type(base_dir),
'args': {"build_version": "1"}})
service.build()
assert service.image()
def test_start_container_stays_unpriviliged(self):
service = self.create_service('web')
container = create_and_start_container(service).inspect()

View File

@ -266,13 +266,13 @@ class ServiceStateTest(DockerClientTestCase):
dockerfile = context.join('Dockerfile')
dockerfile.write(base_image)
web = self.create_service('web', build=str(context))
web = self.create_service('web', build={'context': str(context)})
container = web.create_container()
dockerfile.write(base_image + 'CMD echo hello world\n')
web.build()
web = self.create_service('web', build=str(context))
web = self.create_service('web', build={'context': str(context)})
self.assertEqual(('recreate', [container]), web.convergence_plan())
def test_image_changed_to_build(self):
@ -286,7 +286,7 @@ class ServiceStateTest(DockerClientTestCase):
web = self.create_service('web', image='busybox')
container = web.create_container()
web = self.create_service('web', build=str(context))
web = self.create_service('web', build={'context': str(context)})
plan = web.convergence_plan()
self.assertEqual(('recreate', [container]), plan)
containers = web.execute_convergence_plan(plan)

View File

@ -12,6 +12,7 @@ import py
import pytest
from compose.config import config
from compose.config.config import resolve_build_args
from compose.config.config import resolve_environment
from compose.config.errors import ConfigurationError
from compose.config.types import VolumeSpec
@ -284,7 +285,7 @@ class ConfigTest(unittest.TestCase):
expected = [
{
'name': 'web',
'build': os.path.abspath('/'),
'build': {'context': os.path.abspath('/')},
'volumes': [VolumeSpec.parse('/home/user/project:/code')],
'links': ['db'],
},
@ -414,6 +415,71 @@ class ConfigTest(unittest.TestCase):
assert services[1]['name'] == 'db'
assert services[2]['name'] == 'web'
def test_config_build_configuration(self):
service = config.load(
build_config_details(
{'web': {
'build': '.',
'dockerfile': 'Dockerfile-alt'
}},
'tests/fixtures/extends',
'filename.yml'
)
).services
self.assertTrue('context' in service[0]['build'])
self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt')
def test_config_build_configuration_v2(self):
# service.dockerfile is invalid in v2
with self.assertRaises(ConfigurationError):
config.load(
build_config_details(
{
'version': 2,
'services': {
'web': {
'build': '.',
'dockerfile': 'Dockerfile-alt'
}
}
},
'tests/fixtures/extends',
'filename.yml'
)
)
service = config.load(
build_config_details({
'version': 2,
'services': {
'web': {
'build': '.'
}
}
}, 'tests/fixtures/extends', 'filename.yml')
).services[0]
self.assertTrue('context' in service['build'])
service = config.load(
build_config_details(
{
'version': 2,
'services': {
'web': {
'build': {
'context': '.',
'dockerfile': 'Dockerfile-alt'
}
}
}
},
'tests/fixtures/extends',
'filename.yml'
)
).services
self.assertTrue('context' in service[0]['build'])
self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt')
def test_load_with_multiple_files_v2(self):
base_file = config.ConfigFile(
'base.yaml',
@ -445,7 +511,7 @@ class ConfigTest(unittest.TestCase):
expected = [
{
'name': 'web',
'build': os.path.abspath('/'),
'build': {'context': os.path.abspath('/')},
'image': 'example/web',
'volumes': [VolumeSpec.parse('/home/user/project:/code')],
},
@ -1157,7 +1223,7 @@ class BuildOrImageMergeTest(unittest.TestCase):
self.assertEqual(
config.merge_service_dicts({'image': 'redis'}, {'build': '.'}, V1),
{'build': '.'},
{'build': '.'}
)
@ -1388,6 +1454,24 @@ class EnvTest(unittest.TestCase):
},
)
@mock.patch.dict(os.environ)
def test_resolve_build_args(self):
os.environ['env_arg'] = 'value2'
build = {
'context': '.',
'args': {
'arg1': 'value1',
'empty_arg': '',
'env_arg': None,
'no_env': None
}
}
self.assertEqual(
resolve_build_args(build),
{'arg1': 'value1', 'empty_arg': '', 'env_arg': 'value2', 'no_env': ''},
)
@pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash')
@mock.patch.dict(os.environ)
def test_resolve_path(self):
@ -1871,7 +1955,7 @@ class BuildPathTest(unittest.TestCase):
def test_from_file(self):
service_dict = load_from_filename('tests/fixtures/build-path/docker-compose.yml')
self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}])
self.assertEquals(service_dict, [{'name': 'foo', 'build': {'context': self.abs_context_path}}])
def test_valid_url_in_build_path(self):
valid_urls = [
@ -1886,7 +1970,7 @@ class BuildPathTest(unittest.TestCase):
service_dict = config.load(build_config_details({
'validurl': {'build': valid_url},
}, '.', None)).services
assert service_dict[0]['build'] == valid_url
assert service_dict[0]['build'] == {'context': valid_url}
def test_invalid_url_in_build_path(self):
invalid_urls = [

View File

@ -355,7 +355,7 @@ class ServiceTest(unittest.TestCase):
self.assertEqual(parse_repository_tag("url:5000/repo@sha256:digest"), ("url:5000/repo", "sha256:digest", "@"))
def test_create_container_with_build(self):
service = Service('foo', client=self.mock_client, build='.')
service = Service('foo', client=self.mock_client, build={'context': '.'})
self.mock_client.inspect_image.side_effect = [
NoSuchImageError,
{'Id': 'abc123'},
@ -374,17 +374,18 @@ class ServiceTest(unittest.TestCase):
forcerm=False,
nocache=False,
rm=True,
buildargs=None,
)
def test_create_container_no_build(self):
service = Service('foo', client=self.mock_client, build='.')
service = Service('foo', client=self.mock_client, build={'context': '.'})
self.mock_client.inspect_image.return_value = {'Id': 'abc123'}
service.create_container(do_build=False)
self.assertFalse(self.mock_client.build.called)
def test_create_container_no_build_but_needs_build(self):
service = Service('foo', client=self.mock_client, build='.')
service = Service('foo', client=self.mock_client, build={'context': '.'})
self.mock_client.inspect_image.side_effect = NoSuchImageError
with self.assertRaises(NeedsBuildError):
service.create_container(do_build=False)
@ -394,7 +395,7 @@ class ServiceTest(unittest.TestCase):
b'{"stream": "Successfully built 12345"}',
]
service = Service('foo', client=self.mock_client, build='.')
service = Service('foo', client=self.mock_client, build={'context': '.'})
service.build()
self.assertEqual(self.mock_client.build.call_count, 1)