tyler.durden
/
compose
mirror of https://github.com/docker/compose.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6865 from ulyssessouza/support-cli-build 

Add support to CLI build
This commit is contained in:
Ulysses Souza 2019-08-30 13:46:21 +02:00 committed by GitHub
parent 4cfa622de8 5add9192ac
commit 2199278b44
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 217 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
compose/cli/main.py
View File

@ -263,14 +263,17 @@ class TopLevelCommand(object):
Usage: build [options] [--build-arg key=val...] [SERVICE...]
Options:
--build-arg key=val Set build-time variables for services.
--compress Compress the build context using gzip.
--force-rm Always remove intermediate containers.
-m, --memory MEM Set memory limit for the build container.
--no-cache Do not use cache when building the image.
--no-rm Do not remove intermediate containers after a successful build.
--pull Always attempt to pull a newer version of the image.
-m, --memory MEM Sets memory limit for the build container.
--build-arg key=val Set build-time variables for services.
--parallel Build images in parallel.
--progress string Set type of progress output (auto, plain, tty).
EXPERIMENTAL flag for native builder.
To enable, run with COMPOSE_DOCKER_CLI_BUILD=1)
--pull Always attempt to pull a newer version of the image.
-q, --quiet Don't print anything to STDOUT
"""
service_names = options['SERVICE']
@ -283,6 +286,8 @@ class TopLevelCommand(object):
)
build_args = resolve_build_args(build_args, self.toplevel_environment)
native_builder = self.toplevel_environment.get_boolean('COMPOSE_DOCKER_CLI_BUILD')
self.project.build(
service_names=options['SERVICE'],
no_cache=bool(options.get('--no-cache', False)),
@ -293,7 +298,9 @@ class TopLevelCommand(object):
build_args=build_args,
gzip=options.get('--compress', False),
parallel_build=options.get('--parallel', False),
silent=options.get('--quiet', False)
silent=options.get('--quiet', False),
cli=native_builder,
progress=options.get('--progress'),
)
def bundle(self, options):
@ -1071,6 +1078,8 @@ class TopLevelCommand(object):
for excluded in [x for x in opts if options.get(x) and no_start]:
raise UserError('--no-start and {} cannot be combined.'.format(excluded))
native_builder = self.toplevel_environment.get_boolean('COMPOSE_DOCKER_CLI_BUILD')
with up_shutdown_context(self.project, service_names, timeout, detached):
warn_for_swarm_mode(self.project.client)
@ -1090,6 +1099,7 @@ class TopLevelCommand(object):
reset_container_image=rebuild,
renew_anonymous_volumes=options.get('--renew-anon-volumes'),
silent=options.get('--quiet-pull'),
cli=native_builder,
)
try:

20
compose/project.py
View File

@ -355,7 +355,8 @@ class Project(object):
return containers
def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None,
build_args=None, gzip=False, parallel_build=False, rm=True, silent=False):
build_args=None, gzip=False, parallel_build=False, rm=True, silent=False, cli=False,
progress=None):
services = []
for service in self.get_services(service_names):
@ -364,8 +365,17 @@ class Project(object):
elif not silent:
log.info('%s uses an image, skipping' % service.name)
if cli:
log.warning("Native build is an experimental feature and could change at any time")
if parallel_build:
log.warning("Flag '--parallel' is ignored when building with "
"COMPOSE_DOCKER_CLI_BUILD=1")
if gzip:
log.warning("Flag '--compress' is ignored when building with "
"COMPOSE_DOCKER_CLI_BUILD=1")
def build_service(service):
service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent)
service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent, cli, progress)
if parallel_build:
_, errors = parallel.parallel_execute(
services,
@ -509,8 +519,12 @@ class Project(object):
reset_container_image=False,
renew_anonymous_volumes=False,
silent=False,
cli=False,
):
if cli:
log.warning("Native build is an experimental feature and could change at any time")
self.initialize()
if not ignore_orphans:
self.find_orphan_containers(remove_orphans)
@ -523,7 +537,7 @@ class Project(object):
include_deps=start_deps)
for svc in services:
svc.ensure_image_exists(do_build=do_build, silent=silent)
svc.ensure_image_exists(do_build=do_build, silent=silent, cli=cli)
plans = self._get_convergence_plans(
services, strategy, always_recreate_deps=always_recreate_deps)

151
compose/service.py
View File

@ -2,10 +2,12 @@ from __future__ import absolute_import
from __future__ import unicode_literals
import itertools
import json
import logging
import os
import re
import sys
import tempfile
from collections import namedtuple
from collections import OrderedDict
from operator import attrgetter
@ -59,6 +61,11 @@ from .utils import parse_seconds_float
from .utils import truncate_id
from .utils import unique_everseen
if six.PY2:
import subprocess32 as subprocess
else:
import subprocess
log = logging.getLogger(__name__)
@ -338,9 +345,9 @@ class Service(object):
raise OperationFailedError("Cannot create container for service %s: %s" %
(self.name, ex.explanation))
def ensure_image_exists(self, do_build=BuildAction.none, silent=False):
def ensure_image_exists(self, do_build=BuildAction.none, silent=False, cli=False):
if self.can_be_built() and do_build == BuildAction.force:
self.build()
self.build(cli=cli)
return
try:
@ -356,7 +363,7 @@ class Service(object):
if do_build == BuildAction.skip:
raise NeedsBuildError(self)
self.build()
self.build(cli=cli)
log.warning(
"Image for service {} was built because it did not already exist. To "
"rebuild this image you must use `docker-compose build` or "
@ -1049,7 +1056,7 @@ class Service(object):
return [build_spec(secret) for secret in self.secrets]
def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None,
gzip=False, rm=True, silent=False):
gzip=False, rm=True, silent=False, cli=False, progress=None):
output_stream = open(os.devnull, 'w')
if not silent:
output_stream = sys.stdout
@ -1070,7 +1077,8 @@ class Service(object):
'Impossible to perform platform-targeted builds for API version < 1.35'
)
build_output = self.client.build(
builder = self.client if not cli else _CLIBuilder(progress)
build_output = builder.build(
path=path,
tag=self.image_name,
rm=rm,
@ -1701,3 +1709,136 @@ def rewrite_build_path(path):
path = WINDOWS_LONGPATH_PREFIX + os.path.normpath(path)
return path
class _CLIBuilder(object):
def __init__(self, progress):
self._progress = progress
def build(self, path, tag=None, quiet=False, fileobj=None,
nocache=False, rm=False, timeout=None,
custom_context=False, encoding=None, pull=False,
forcerm=False, dockerfile=None, container_limits=None,
decode=False, buildargs=None, gzip=False, shmsize=None,
labels=None, cache_from=None, target=None, network_mode=None,
squash=None, extra_hosts=None, platform=None, isolation=None,
use_config_proxy=True):
"""
Args:
path (str): Path to the directory containing the Dockerfile
buildargs (dict): A dictionary of build arguments
cache_from (:py:class:`list`): A list of images used for build
cache resolution
container_limits (dict): A dictionary of limits applied to each
container created by the build process. Valid keys:
- memory (int): set memory limit for build
- memswap (int): Total memory (memory + swap), -1 to disable
swap
- cpushares (int): CPU shares (relative weight)
- cpusetcpus (str): CPUs in which to allow execution, e.g.,
``"0-3"``, ``"0,1"``
custom_context (bool): Optional if using ``fileobj``
decode (bool): If set to ``True``, the returned stream will be
decoded into dicts on the fly. Default ``False``
dockerfile (str): path within the build context to the Dockerfile
encoding (str): The encoding for a stream. Set to ``gzip`` for
compressing
extra_hosts (dict): Extra hosts to add to /etc/hosts in building
containers, as a mapping of hostname to IP address.
fileobj: A file object to use as the Dockerfile. (Or a file-like
object)
forcerm (bool): Always remove intermediate containers, even after
unsuccessful builds
isolation (str): Isolation technology used during build.
Default: `None`.
labels (dict): A dictionary of labels to set on the image
network_mode (str): networking mode for the run commands during
build
nocache (bool): Don't use the cache when set to ``True``
platform (str): Platform in the format ``os[/arch[/variant]]``
pull (bool): Downloads any updates to the FROM image in Dockerfiles
quiet (bool): Whether to return the status
rm (bool): Remove intermediate containers. The ``docker build``
command now defaults to ``--rm=true``, but we have kept the old
default of `False` to preserve backward compatibility
shmsize (int): Size of `/dev/shm` in bytes. The size must be
greater than 0. If omitted the system uses 64MB
squash (bool): Squash the resulting images layers into a
single layer.
tag (str): A tag to add to the final image
target (str): Name of the build-stage to build in a multi-stage
Dockerfile
timeout (int): HTTP timeout
use_config_proxy (bool): If ``True``, and if the docker client
configuration file (``~/.docker/config.json`` by default)
contains a proxy configuration, the corresponding environment
variables will be set in the container being built.
Returns:
A generator for the build output.
"""
if dockerfile:
dockerfile = os.path.join(path, dockerfile)
iidfile = tempfile.mktemp()
command_builder = _CommandBuilder()
command_builder.add_params("--build-arg", buildargs)
command_builder.add_list("--cache-from", cache_from)
command_builder.add_arg("--file", dockerfile)
command_builder.add_flag("--force-rm", forcerm)
command_builder.add_arg("--memory", container_limits.get("memory"))
command_builder.add_flag("--no-cache", nocache)
command_builder.add_flag("--progress", self._progress)
command_builder.add_flag("--pull", pull)
command_builder.add_arg("--tag", tag)
command_builder.add_arg("--target", target)
command_builder.add_arg("--iidfile", iidfile)
args = command_builder.build([path])
magic_word = "Successfully built "
appear = False
with subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) as p:
while True:
line = p.stdout.readline()
if not line:
break
if line.startswith(magic_word):
appear = True
yield json.dumps({"stream": line})
with open(iidfile) as f:
line = f.readline()
image_id = line.split(":")[1].strip()
os.remove(iidfile)
# In case of `DOCKER_BUILDKIT=1`
# there is no success message already present in the output.
# Since that's the way `Service::build` gets the `image_id`
# it has to be added `manually`
if not appear:
yield json.dumps({"stream": "{}{}\n".format(magic_word, image_id)})
class _CommandBuilder(object):
def __init__(self):
self._args = ["docker", "build"]
def add_arg(self, name, value):
if value:
self._args.extend([name, str(value)])
def add_flag(self, name, flag):
if flag:
self._args.extend([name])
def add_params(self, name, params):
if params:
for key, val in params.items():
self._args.extend([name, "{}={}".format(key, val)])
def add_list(self, name, values):
if values:
for val in values:
self._args.extend([name, val])
def build(self, args):
return self._args + args

1
setup.py
View File

@ -52,6 +52,7 @@ if sys.version_info[:2] < (3, 4):
tests_require.append('mock >= 1.0.1, < 4')
extras_require = {
':python_version < "3.2"': ['subprocess32 >= 3.5.4, < 4'],
':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'],
':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5, < 4'],
':python_version < "3.3"': ['ipaddress >= 1.0.16, < 2'],

39
tests/integration/service_test.py
View File

@ -38,6 +38,8 @@ from compose.container import Container
from compose.errors import OperationFailedError
from compose.parallel import ParallelStreamWriter
from compose.project import OneOffFilter
from compose.project import Project
from compose.service import BuildAction
from compose.service import ConvergencePlan
from compose.service import ConvergenceStrategy
from compose.service import NetworkMode
@ -966,6 +968,43 @@ class ServiceTest(DockerClientTestCase):
assert self.client.inspect_image('composetest_web')
def test_build_cli(self):
base_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base_dir)
with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
f.write("FROM busybox\n")
service = self.create_service('web',
build={'context': base_dir},
environment={
'COMPOSE_DOCKER_CLI_BUILD': '1',
'DOCKER_BUILDKIT': '1',
})
service.build(cli=True)
self.addCleanup(self.client.remove_image, service.image_name)
assert self.client.inspect_image('composetest_web')
def test_up_build_cli(self):
base_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base_dir)
with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f:
f.write("FROM busybox\n")
web = self.create_service('web',
build={'context': base_dir},
environment={
'COMPOSE_DOCKER_CLI_BUILD': '1',
'DOCKER_BUILDKIT': '1',
})
project = Project('composetest', [web], self.client)
project.up(do_build=BuildAction.force)
containers = project.containers(['web'])
assert len(containers) == 1
assert containers[0].name.startswith('composetest_web_')
def test_build_non_ascii_filename(self):
base_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base_dir)