Merge pull request #5537 from docker/Rozelette-branch_limit_test

Add configurable parallel operations limit
This commit is contained in:
Joffrey F 2018-01-04 13:04:27 -08:00 committed by GitHub
commit b97b76d24f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 126 additions and 67 deletions

View File

@ -10,6 +10,7 @@ import six
from . import errors from . import errors
from . import verbose_proxy from . import verbose_proxy
from .. import config from .. import config
from .. import parallel
from ..config.environment import Environment from ..config.environment import Environment
from ..const import API_VERSIONS from ..const import API_VERSIONS
from ..project import Project from ..project import Project
@ -23,6 +24,8 @@ log = logging.getLogger(__name__)
def project_from_options(project_dir, options): def project_from_options(project_dir, options):
environment = Environment.from_env_file(project_dir) environment = Environment.from_env_file(project_dir)
set_parallel_limit(environment)
host = options.get('--host') host = options.get('--host')
if host is not None: if host is not None:
host = host.lstrip('=') host = host.lstrip('=')
@ -38,6 +41,22 @@ def project_from_options(project_dir, options):
) )
def set_parallel_limit(environment):
parallel_limit = environment.get('COMPOSE_PARALLEL_LIMIT')
if parallel_limit:
try:
parallel_limit = int(parallel_limit)
except ValueError:
raise errors.UserError(
'COMPOSE_PARALLEL_LIMIT must be an integer (found: "{}")'.format(
environment.get('COMPOSE_PARALLEL_LIMIT')
)
)
if parallel_limit <= 1:
raise errors.UserError('COMPOSE_PARALLEL_LIMIT can not be less than 2')
parallel.GlobalLimit.set_global_limit(parallel_limit)
def get_config_from_options(base_dir, options): def get_config_from_options(base_dir, options):
environment = Environment.from_env_file(base_dir) environment = Environment.from_env_file(base_dir)
config_path = get_config_path_from_options( config_path = get_config_path_from_options(

View File

@ -18,6 +18,7 @@ LABEL_VERSION = 'com.docker.compose.version'
LABEL_VOLUME = 'com.docker.compose.volume' LABEL_VOLUME = 'com.docker.compose.volume'
LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash'
NANOCPUS_SCALE = 1000000000 NANOCPUS_SCALE = 1000000000
PARALLEL_LIMIT = 64
SECRETS_PATH = '/run/secrets' SECRETS_PATH = '/run/secrets'

View File

@ -15,6 +15,7 @@ from six.moves.queue import Queue
from compose.cli.colors import green from compose.cli.colors import green
from compose.cli.colors import red from compose.cli.colors import red
from compose.cli.signals import ShutdownException from compose.cli.signals import ShutdownException
from compose.const import PARALLEL_LIMIT
from compose.errors import HealthCheckFailed from compose.errors import HealthCheckFailed
from compose.errors import NoHealthCheckConfigured from compose.errors import NoHealthCheckConfigured
from compose.errors import OperationFailedError from compose.errors import OperationFailedError
@ -26,6 +27,20 @@ log = logging.getLogger(__name__)
STOP = object() STOP = object()
class GlobalLimit(object):
"""Simple class to hold a global semaphore limiter for a project. This class
should be treated as a singleton that is instantiated when the project is.
"""
global_limiter = Semaphore(PARALLEL_LIMIT)
@classmethod
def set_global_limit(cls, value):
if value is None:
value = PARALLEL_LIMIT
cls.global_limiter = Semaphore(value)
def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, parent_objects=None): def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, parent_objects=None):
"""Runs func on objects in parallel while ensuring that func is """Runs func on objects in parallel while ensuring that func is
ran on object only after it is ran on all its dependencies. ran on object only after it is ran on all its dependencies.
@ -173,7 +188,7 @@ def producer(obj, func, results, limiter):
The entry point for a producer thread which runs func on a single object. The entry point for a producer thread which runs func on a single object.
Places a tuple on the results queue once func has either returned or raised. Places a tuple on the results queue once func has either returned or raised.
""" """
with limiter: with limiter, GlobalLimit.global_limiter:
try: try:
result = func(obj) result = func(obj)
results.put((obj, result, None)) results.put((obj, result, None))

View File

@ -1,11 +1,13 @@
from __future__ import absolute_import from __future__ import absolute_import
from __future__ import unicode_literals from __future__ import unicode_literals
import unittest
from threading import Lock from threading import Lock
import six import six
from docker.errors import APIError from docker.errors import APIError
from compose.parallel import GlobalLimit
from compose.parallel import parallel_execute from compose.parallel import parallel_execute
from compose.parallel import parallel_execute_iter from compose.parallel import parallel_execute_iter
from compose.parallel import ParallelStreamWriter from compose.parallel import ParallelStreamWriter
@ -31,91 +33,113 @@ def get_deps(obj):
return [(dep, None) for dep in deps[obj]] return [(dep, None) for dep in deps[obj]]
def test_parallel_execute(): class ParallelTest(unittest.TestCase):
results, errors = parallel_execute(
objects=[1, 2, 3, 4, 5],
func=lambda x: x * 2,
get_name=six.text_type,
msg="Doubling",
)
assert sorted(results) == [2, 4, 6, 8, 10] def test_parallel_execute(self):
assert errors == {} results, errors = parallel_execute(
objects=[1, 2, 3, 4, 5],
func=lambda x: x * 2,
get_name=six.text_type,
msg="Doubling",
)
assert sorted(results) == [2, 4, 6, 8, 10]
assert errors == {}
def test_parallel_execute_with_limit(): def test_parallel_execute_with_limit(self):
limit = 1 limit = 1
tasks = 20 tasks = 20
lock = Lock() lock = Lock()
def f(obj): def f(obj):
locked = lock.acquire(False) locked = lock.acquire(False)
# we should always get the lock because we're the only thread running # we should always get the lock because we're the only thread running
assert locked assert locked
lock.release() lock.release()
return None return None
results, errors = parallel_execute( results, errors = parallel_execute(
objects=list(range(tasks)), objects=list(range(tasks)),
func=f, func=f,
get_name=six.text_type, get_name=six.text_type,
msg="Testing", msg="Testing",
limit=limit, limit=limit,
) )
assert results == tasks * [None] assert results == tasks * [None]
assert errors == {} assert errors == {}
def test_parallel_execute_with_global_limit(self):
GlobalLimit.set_global_limit(1)
self.addCleanup(GlobalLimit.set_global_limit, None)
tasks = 20
lock = Lock()
def test_parallel_execute_with_deps(): def f(obj):
log = [] locked = lock.acquire(False)
# we should always get the lock because we're the only thread running
assert locked
lock.release()
return None
def process(x): results, errors = parallel_execute(
log.append(x) objects=list(range(tasks)),
func=f,
get_name=six.text_type,
msg="Testing",
)
parallel_execute( assert results == tasks * [None]
objects=objects, assert errors == {}
func=process,
get_name=lambda obj: obj,
msg="Processing",
get_deps=get_deps,
)
assert sorted(log) == sorted(objects) def test_parallel_execute_with_deps(self):
log = []
assert log.index(data_volume) < log.index(db) def process(x):
assert log.index(db) < log.index(web) log.append(x)
assert log.index(cache) < log.index(web)
parallel_execute(
objects=objects,
func=process,
get_name=lambda obj: obj,
msg="Processing",
get_deps=get_deps,
)
def test_parallel_execute_with_upstream_errors(): assert sorted(log) == sorted(objects)
log = []
def process(x): assert log.index(data_volume) < log.index(db)
if x is data_volume: assert log.index(db) < log.index(web)
raise APIError(None, None, "Something went wrong") assert log.index(cache) < log.index(web)
log.append(x)
parallel_execute( def test_parallel_execute_with_upstream_errors(self):
objects=objects, log = []
func=process,
get_name=lambda obj: obj,
msg="Processing",
get_deps=get_deps,
)
assert log == [cache] def process(x):
if x is data_volume:
raise APIError(None, None, "Something went wrong")
log.append(x)
events = [ parallel_execute(
(obj, result, type(exception)) objects=objects,
for obj, result, exception func=process,
in parallel_execute_iter(objects, process, get_deps, None) get_name=lambda obj: obj,
] msg="Processing",
get_deps=get_deps,
)
assert (cache, None, type(None)) in events assert log == [cache]
assert (data_volume, None, APIError) in events
assert (db, None, UpstreamError) in events events = [
assert (web, None, UpstreamError) in events (obj, result, type(exception))
for obj, result, exception
in parallel_execute_iter(objects, process, get_deps, None)
]
assert (cache, None, type(None)) in events
assert (data_volume, None, APIError) in events
assert (db, None, UpstreamError) in events
assert (web, None, UpstreamError) in events
def test_parallel_execute_alignment(capsys): def test_parallel_execute_alignment(capsys):