From 2b245bdf9e254746061a4ce9499bf517e488c8f5 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 10 Mar 2014 13:57:13 +0000 Subject: [PATCH 1/9] Update to docker-py 0.3.1 From https://github.com/dotcloud/docker-py/commit/7f55a101f813f3e96413d1b577e98d9467b0bffc This now requires Docker 0.9 or greater. --- .travis.yml | 8 +- fig/packages/docker/__init__.py | 3 + fig/packages/docker/auth/auth.py | 13 +- fig/packages/docker/client.py | 154 +++++++++++++---------- fig/packages/docker/unixconn/unixconn.py | 2 +- fig/packages/docker/utils/__init__.py | 2 +- fig/packages/docker/utils/utils.py | 36 +++++- tests/testcases.py | 2 +- 8 files changed, 139 insertions(+), 81 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d6cc863b..ea561955b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,12 +3,8 @@ python: - '2.6' - '2.7' env: -- DOCKER_VERSION=0.8.0 -- DOCKER_VERSION=0.8.1 -matrix: - allow_failures: - - python: '3.2' - - python: '3.3' +- DOCKER_VERSION=0.9.1 +- DOCKER_VERSION=0.10.0 install: script/travis-install script: - pwd diff --git a/fig/packages/docker/__init__.py b/fig/packages/docker/__init__.py index 5f642a855..5388e7286 100644 --- a/fig/packages/docker/__init__.py +++ b/fig/packages/docker/__init__.py @@ -12,4 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +__title__ = 'docker-py' +__version__ = '0.3.0' + from .client import Client, APIError # flake8: noqa diff --git a/fig/packages/docker/auth/auth.py b/fig/packages/docker/auth/auth.py index 8037dcbb6..69cfa89d3 100644 --- a/fig/packages/docker/auth/auth.py +++ b/fig/packages/docker/auth/auth.py @@ -48,7 +48,7 @@ def resolve_repository_name(repo_name): raise ValueError('Repository name cannot contain a ' 'scheme ({0})'.format(repo_name)) parts = repo_name.split('/', 1) - if not '.' in parts[0] and not ':' in parts[0] and parts[0] != 'localhost': + if '.' not in parts[0] and ':' not in parts[0] and parts[0] != 'localhost': # This is a docker index repo (ex: foo/bar or ubuntu) return INDEX_URL, repo_name if len(parts) < 2: @@ -87,6 +87,11 @@ def resolve_authconfig(authconfig, registry=None): return authconfig.get(swap_protocol(registry), None) +def encode_auth(auth_info): + return base64.b64encode(auth_info.get('username', '') + b':' + + auth_info.get('password', '')) + + def decode_auth(auth): if isinstance(auth, six.string_types): auth = auth.encode('ascii') @@ -100,6 +105,12 @@ def encode_header(auth): return base64.b64encode(auth_json) +def encode_full_header(auth): + """ Returns the given auth block encoded for the X-Registry-Config header. + """ + return encode_header({'configs': auth}) + + def load_config(root=None): """Loads authentication data from a Docker configuration file in the given root directory.""" diff --git a/fig/packages/docker/client.py b/fig/packages/docker/client.py index 948a3a67d..77bb962f8 100644 --- a/fig/packages/docker/client.py +++ b/fig/packages/docker/client.py @@ -28,13 +28,17 @@ from .utils import utils if not six.PY3: import websocket +DEFAULT_DOCKER_API_VERSION = '1.9' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 class APIError(requests.exceptions.HTTPError): def __init__(self, message, response, explanation=None): - super(APIError, self).__init__(message, response=response) + # requests 1.2 supports response as a keyword argument, but + # requests 1.1 doesn't + super(APIError, self).__init__(message) + self.response = response self.explanation = explanation @@ -65,7 +69,7 @@ class APIError(requests.exceptions.HTTPError): class Client(requests.Session): - def __init__(self, base_url=None, version="1.6", + def __init__(self, base_url=None, version=DEFAULT_DOCKER_API_VERSION, timeout=DEFAULT_TIMEOUT_SECONDS): super(Client, self).__init__() if base_url is None: @@ -125,7 +129,7 @@ class Client(requests.Session): mem_limit=0, ports=None, environment=None, dns=None, volumes=None, volumes_from=None, network_disabled=False, entrypoint=None, - cpu_shares=None, working_dir=None): + cpu_shares=None, working_dir=None, domainname=None): if isinstance(command, six.string_types): command = shlex.split(str(command)) if isinstance(environment, dict): @@ -133,7 +137,7 @@ class Client(requests.Session): '{0}={1}'.format(k, v) for k, v in environment.items() ] - if ports and isinstance(ports, list): + if isinstance(ports, list): exposed_ports = {} for port_definition in ports: port = port_definition @@ -145,12 +149,15 @@ class Client(requests.Session): exposed_ports['{0}/{1}'.format(port, proto)] = {} ports = exposed_ports - if volumes and isinstance(volumes, list): + if isinstance(volumes, list): volumes_dict = {} for vol in volumes: volumes_dict[vol] = {} volumes = volumes_dict + if volumes_from and not isinstance(volumes_from, six.string_types): + volumes_from = ','.join(volumes_from) + attach_stdin = False attach_stdout = False attach_stderr = False @@ -165,26 +172,27 @@ class Client(requests.Session): stdin_once = True return { - 'Hostname': hostname, + 'Hostname': hostname, + 'Domainname': domainname, 'ExposedPorts': ports, - 'User': user, - 'Tty': tty, - 'OpenStdin': stdin_open, - 'StdinOnce': stdin_once, - 'Memory': mem_limit, - 'AttachStdin': attach_stdin, + 'User': user, + 'Tty': tty, + 'OpenStdin': stdin_open, + 'StdinOnce': stdin_once, + 'Memory': mem_limit, + 'AttachStdin': attach_stdin, 'AttachStdout': attach_stdout, 'AttachStderr': attach_stderr, - 'Env': environment, - 'Cmd': command, - 'Dns': dns, - 'Image': image, - 'Volumes': volumes, - 'VolumesFrom': volumes_from, + 'Env': environment, + 'Cmd': command, + 'Dns': dns, + 'Image': image, + 'Volumes': volumes, + 'VolumesFrom': volumes_from, 'NetworkDisabled': network_disabled, - 'Entrypoint': entrypoint, - 'CpuShares': cpu_shares, - 'WorkingDir': working_dir + 'Entrypoint': entrypoint, + 'CpuShares': cpu_shares, + 'WorkingDir': working_dir } def _post_json(self, url, data, **kwargs): @@ -222,31 +230,18 @@ class Client(requests.Session): def _create_websocket_connection(self, url): return websocket.create_connection(url) - def _stream_result(self, response): - """Generator for straight-out, non chunked-encoded HTTP responses.""" + def _get_raw_response_socket(self, response): self._raise_for_status(response) - for line in response.iter_lines(chunk_size=1, decode_unicode=True): - # filter out keep-alive new lines - if line: - yield line + '\n' - - def _stream_result_socket(self, response): - self._raise_for_status(response) - return response.raw._fp.fp._sock + if six.PY3: + return response.raw._fp.fp.raw._sock + else: + return response.raw._fp.fp._sock def _stream_helper(self, response): """Generator for data coming from a chunked-encoded HTTP response.""" - socket_fp = self._stream_result_socket(response) - socket_fp.setblocking(1) - socket = socket_fp.makefile() - while True: - size = int(socket.readline(), 16) - if size <= 0: - break - data = socket.readline() - if not data: - break - yield data + for line in response.iter_lines(chunk_size=32): + if line: + yield line def _multiplexed_buffer_helper(self, response): """A generator of multiplexed data blocks read from a buffered @@ -265,17 +260,20 @@ class Client(requests.Session): def _multiplexed_socket_stream_helper(self, response): """A generator of multiplexed data blocks coming from a response socket.""" - socket = self._stream_result_socket(response) + socket = self._get_raw_response_socket(response) def recvall(socket, size): - data = '' + blocks = [] while size > 0: block = socket.recv(size) if not block: return None - data += block + blocks.append(block) size -= len(block) + + sep = bytes() if six.PY3 else str() + data = sep.join(blocks) return data while True: @@ -304,9 +302,18 @@ class Client(requests.Session): u = self._url("/containers/{0}/attach".format(container)) response = self._post(u, params=params, stream=stream) - # Stream multi-plexing was introduced in API v1.6. + # Stream multi-plexing was only introduced in API v1.6. Anything before + # that needs old-style streaming. if utils.compare_version('1.6', self._version) < 0: - return stream and self._stream_result(response) or \ + def stream_result(): + self._raise_for_status(response) + for line in response.iter_lines(chunk_size=1, + decode_unicode=True): + # filter out keep-alive new lines + if line: + yield line + + return stream_result() if stream else \ self._result(response, binary=True) return stream and self._multiplexed_socket_stream_helper(response) or \ @@ -319,13 +326,15 @@ class Client(requests.Session): 'stderr': 1, 'stream': 1 } + if ws: return self._attach_websocket(container, params) if isinstance(container, dict): container = container.get('Id') + u = self._url("/containers/{0}/attach".format(container)) - return self._stream_result_socket(self.post( + return self._get_raw_response_socket(self.post( u, None, params=self._attach_params(params), stream=True)) def build(self, path=None, tag=None, quiet=False, fileobj=None, @@ -341,6 +350,9 @@ class Client(requests.Session): else: context = utils.tar(path) + if utils.compare_version('1.8', self._version) >= 0: + stream = True + u = self._url('/build') params = { 't': tag, @@ -352,6 +364,19 @@ class Client(requests.Session): if context is not None: headers = {'Content-Type': 'application/tar'} + if utils.compare_version('1.9', self._version) >= 0: + # If we don't have any auth data so far, try reloading the config + # file one more time in case anything showed up in there. + if not self._auth_configs: + self._auth_configs = auth.load_config() + + # Send the full auth configuration (if any exists), since the build + # could use any (or all) of the registries. + if self._auth_configs: + headers['X-Registry-Config'] = auth.encode_full_header( + self._auth_configs + ) + response = self._post( u, data=context, @@ -363,8 +388,9 @@ class Client(requests.Session): if context is not None: context.close() + if stream: - return self._stream_result(response) + return self._stream_helper(response) else: output = self._result(response) srch = r'Successfully built ([0-9a-f]+)' @@ -403,6 +429,8 @@ class Client(requests.Session): return res def copy(self, container, resource): + if isinstance(container, dict): + container = container.get('Id') res = self._post_json( self._url("/containers/{0}/copy".format(container)), data={"Resource": resource}, @@ -416,12 +444,12 @@ class Client(requests.Session): mem_limit=0, ports=None, environment=None, dns=None, volumes=None, volumes_from=None, network_disabled=False, name=None, entrypoint=None, - cpu_shares=None, working_dir=None): + cpu_shares=None, working_dir=None, domainname=None): config = self._container_config( image, command, hostname, user, detach, stdin_open, tty, mem_limit, ports, environment, dns, volumes, volumes_from, network_disabled, - entrypoint, cpu_shares, working_dir + entrypoint, cpu_shares, working_dir, domainname ) return self.create_container_from_config(config, name) @@ -440,21 +468,7 @@ class Client(requests.Session): format(container))), True) def events(self): - u = self._url("/events") - - socket = self._stream_result_socket(self.get(u, stream=True)) - - while True: - chunk = socket.recv(4096) - if chunk: - # Messages come in the format of length, data, newline. - length, data = chunk.split("\n", 1) - length = int(length, 16) - if length > len(data): - data += socket.recv(length - len(data)) - yield json.loads(data) - else: - break + return self._stream_helper(self.get(self._url('/events'), stream=True)) def export(self, container): if isinstance(container, dict): @@ -471,6 +485,8 @@ class Client(requests.Session): def images(self, name=None, quiet=False, all=False, viz=False): if viz: + if utils.compare_version('1.7', self._version) >= 0: + raise Exception('Viz output is not supported in API >= 1.7!') return self._result(self._get(self._url("images/viz"))) params = { 'filter': name, @@ -618,7 +634,7 @@ class Client(requests.Session): self._auth_configs = auth.load_config() authcfg = auth.resolve_authconfig(self._auth_configs, registry) - # Do not fail here if no atuhentication exists for this specific + # Do not fail here if no authentication exists for this specific # registry as we can have a readonly pull. Just put the header if # we can. if authcfg: @@ -644,7 +660,7 @@ class Client(requests.Session): self._auth_configs = auth.load_config() authcfg = auth.resolve_authconfig(self._auth_configs, registry) - # Do not fail here if no atuhentication exists for this specific + # Do not fail here if no authentication exists for this specific # registry as we can have a readonly pull. Just put the header if # we can. if authcfg: @@ -652,7 +668,7 @@ class Client(requests.Session): response = self._post_json(u, None, headers=headers, stream=stream) else: - response = self._post_json(u, authcfg, stream=stream) + response = self._post_json(u, None, stream=stream) return stream and self._stream_helper(response) \ or self._result(response) diff --git a/fig/packages/docker/unixconn/unixconn.py b/fig/packages/docker/unixconn/unixconn.py index b5c65931b..176659e7c 100644 --- a/fig/packages/docker/unixconn/unixconn.py +++ b/fig/packages/docker/unixconn/unixconn.py @@ -40,7 +40,7 @@ class UnixHTTPConnection(httplib.HTTPConnection, object): self.sock = sock def _extract_path(self, url): - #remove the base_url entirely.. + # remove the base_url entirely.. return url.replace(self.base_url, "") def request(self, method, url, **kwargs): diff --git a/fig/packages/docker/utils/__init__.py b/fig/packages/docker/utils/__init__.py index 386a01af7..8a85975d7 100644 --- a/fig/packages/docker/utils/__init__.py +++ b/fig/packages/docker/utils/__init__.py @@ -1,3 +1,3 @@ from .utils import ( - compare_version, convert_port_bindings, mkbuildcontext, ping, tar + compare_version, convert_port_bindings, mkbuildcontext, ping, tar, parse_repository_tag ) # flake8: noqa diff --git a/fig/packages/docker/utils/utils.py b/fig/packages/docker/utils/utils.py index 1cb04f0cb..0e4c1c1fc 100644 --- a/fig/packages/docker/utils/utils.py +++ b/fig/packages/docker/utils/utils.py @@ -15,6 +15,7 @@ import io import tarfile import tempfile +from distutils.version import StrictVersion import requests from fig.packages import six @@ -51,15 +52,34 @@ def tar(path): def compare_version(v1, v2): - return float(v2) - float(v1) + """Compare docker versions + + >>> v1 = '1.9' + >>> v2 = '1.10' + >>> compare_version(v1, v2) + 1 + >>> compare_version(v2, v1) + -1 + >>> compare_version(v2, v2) + 0 + """ + s1 = StrictVersion(v1) + s2 = StrictVersion(v2) + if s1 == s2: + return 0 + elif s1 > s2: + return -1 + else: + return 1 def ping(url): try: res = requests.get(url) - return res.status >= 400 except Exception: return False + else: + return res.status_code < 400 def _convert_port_binding(binding): @@ -94,3 +114,15 @@ def convert_port_bindings(port_bindings): else: result[key] = [_convert_port_binding(v)] return result + + +def parse_repository_tag(repo): + column_index = repo.rfind(':') + if column_index < 0: + return repo, "" + tag = repo[column_index+1:] + slash_index = tag.find('/') + if slash_index < 0: + return repo[:column_index], tag + + return repo, "" diff --git a/tests/testcases.py b/tests/testcases.py index 6556311de..ac395040d 100644 --- a/tests/testcases.py +++ b/tests/testcases.py @@ -18,7 +18,7 @@ class DockerClientTestCase(unittest.TestCase): self.client.kill(c['Id']) self.client.remove_container(c['Id']) for i in self.client.images(): - if isinstance(i['Tag'], basestring) and 'figtest' in i['Tag']: + if isinstance(i.get('Tag'), basestring) and 'figtest' in i['Tag']: self.client.remove_image(i) def create_service(self, name, **kwargs): From f8ee52ca2a618b15208d3d6cd2c1c05b4fc5025d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 25 Mar 2014 17:12:59 +0000 Subject: [PATCH 2/9] Fix build output docker-py now streams us the raw JSON events, so we have to replicate the Docker client's progress logic. On the bright side, we now have well-behaved progress bars when pulling an image during `fig build` (no more ski slopes) and `fig up` (no more silence). --- fig/service.py | 91 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 5 deletions(-) diff --git a/fig/service.py b/fig/service.py index 4059ead6a..3e032fa8f 100644 --- a/fig/service.py +++ b/fig/service.py @@ -5,6 +5,7 @@ import logging import re import os import sys +import json from .container import Container log = logging.getLogger(__name__) @@ -146,7 +147,8 @@ class Service(object): except APIError as e: if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation): log.info('Pulling image %s...' % container_options['image']) - self.client.pull(container_options['image']) + output = self.client.pull(container_options['image'], stream=True) + stream_output(output, sys.stdout) return Container.create(self.client, **container_options) raise @@ -299,14 +301,15 @@ class Service(object): stream=True ) + all_events = stream_output(build_output, sys.stdout) + image_id = None - for line in build_output: - if line: - match = re.search(r'Successfully built ([0-9a-f]+)', line) + for event in all_events: + if 'stream' in event: + match = re.search(r'Successfully built ([0-9a-f]+)', event.get('stream', '')) if match: image_id = match.group(1) - sys.stdout.write(line.encode(sys.__stdout__.encoding or 'utf-8')) if image_id is None: raise BuildError(self) @@ -329,6 +332,84 @@ class Service(object): return True +def stream_output(output, stream): + is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno()) + all_events = [] + lines = {} + diff = 0 + + for chunk in output: + event = json.loads(chunk) + all_events.append(event) + + if 'progress' in event or 'progressDetail' in event: + image_id = event['id'] + + if image_id in lines: + diff = len(lines) - lines[image_id] + else: + lines[image_id] = len(lines) + stream.write("\n") + diff = 0 + + if is_terminal: + # move cursor up `diff` rows + stream.write("%c[%dA" % (27, diff)) + + try: + print_output_event(event, stream, is_terminal) + except Exception: + stream.write(repr(event) + "\n") + raise + + if 'id' in event and is_terminal: + # move cursor back down + stream.write("%c[%dB" % (27, diff)) + + stream.flush() + + return all_events + +def print_output_event(event, stream, is_terminal): + if 'errorDetail' in event: + raise Exception(event['errorDetail']['message']) + + terminator = '' + + if is_terminal and 'stream' not in event: + # erase current line + stream.write("%c[2K\r" % 27) + terminator = "\r" + pass + elif 'progressDetail' in event: + return + + if 'time' in event: + stream.write("[%s] " % event['time']) + + if 'id' in event: + stream.write("%s: " % event['id']) + + if 'from' in event: + stream.write("(from %s) " % event['from']) + + status = event.get('status', '') + + if 'progress' in event: + stream.write("%s %s%s" % (status, event['progress'], terminator)) + elif 'progressDetail' in event: + detail = event['progressDetail'] + if 'current' in detail: + percentage = float(detail['current']) / float(detail['total']) * 100 + stream.write('%s (%.1f%%)%s' % (status, percentage, terminator)) + else: + stream.write('%s%s' % (status, terminator)) + elif 'stream' in event: + stream.write("%s%s" % (event['stream'], terminator)) + else: + stream.write("%s%s\n" % (status, terminator)) + + NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$') From 80991f152124d2b951683f391fac74678b59b49f Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 23 Apr 2014 15:46:26 +0100 Subject: [PATCH 3/9] Set "VolumesFrom" when starting containers This is necessary when working with Docker 0.10.0 and up. Fortunately, we can set it both when creating and starting, and retain compatibility with 0.8.x and 0.9.x. recreate_containers() is now responsible for starting containers, as well as creating them. This greatly simplifies usage of the Service class. --- fig/cli/main.py | 3 +-- fig/packages/docker/client.py | 9 +++++++-- fig/project.py | 29 +++++----------------------- fig/service.py | 25 +++++++++++++----------- tests/project_test.py | 36 ++++++++++++----------------------- tests/service_test.py | 14 +++++++------- 6 files changed, 46 insertions(+), 70 deletions(-) diff --git a/fig/cli/main.py b/fig/cli/main.py index 9585371df..4700803d1 100644 --- a/fig/cli/main.py +++ b/fig/cli/main.py @@ -301,10 +301,9 @@ class TopLevelCommand(Command): """ detached = options['-d'] - new = self.project.up(service_names=options['SERVICE']) + to_attach = self.project.up(service_names=options['SERVICE']) if not detached: - to_attach = [c for (s, c) in new] print("Attaching to", list_containers(to_attach)) log_printer = LogPrinter(to_attach, attach_params={"logs": True}) diff --git a/fig/packages/docker/client.py b/fig/packages/docker/client.py index 77bb962f8..8b447d785 100644 --- a/fig/packages/docker/client.py +++ b/fig/packages/docker/client.py @@ -698,8 +698,8 @@ class Client(requests.Session): params={'term': term}), True) - def start(self, container, binds=None, port_bindings=None, lxc_conf=None, - publish_all_ports=False, links=None, privileged=False): + def start(self, container, binds=None, volumes_from=None, port_bindings=None, + lxc_conf=None, publish_all_ports=False, links=None, privileged=False): if isinstance(container, dict): container = container.get('Id') @@ -718,6 +718,11 @@ class Client(requests.Session): ] start_config['Binds'] = bind_pairs + if volumes_from and not isinstance(volumes_from, six.string_types): + volumes_from = ','.join(volumes_from) + + start_config['VolumesFrom'] = volumes_from + if port_bindings: start_config['PortBindings'] = utils.convert_port_bindings( port_bindings diff --git a/fig/project.py b/fig/project.py index 38bbba222..b271a810a 100644 --- a/fig/project.py +++ b/fig/project.py @@ -105,23 +105,6 @@ class Project(object): unsorted = [self.get_service(name) for name in service_names] return [s for s in self.services if s in unsorted] - def recreate_containers(self, service_names=None): - """ - For each service, create or recreate their containers. - Returns a tuple with two lists. The first is a list of - (service, old_container) tuples; the second is a list - of (service, new_container) tuples. - """ - old = [] - new = [] - - for service in self.get_services(service_names): - (s_old, s_new) = service.recreate_containers() - old += [(service, container) for container in s_old] - new += [(service, container) for container in s_new] - - return (old, new) - def start(self, service_names=None, **options): for service in self.get_services(service_names): service.start(**options) @@ -142,15 +125,13 @@ class Project(object): log.info('%s uses an image, skipping' % service.name) def up(self, service_names=None): - (old, new) = self.recreate_containers(service_names=service_names) + new_containers = [] - for (service, container) in new: - service.start_container(container) + for service in self.get_services(service_names): + for (_, new) in service.recreate_containers(): + new_containers.append(new) - for (service, container) in old: - container.remove() - - return new + return new_containers def remove_stopped(self, service_names=None, **options): for service in self.get_services(service_names): diff --git a/fig/service.py b/fig/service.py index 3e032fa8f..20c4e120f 100644 --- a/fig/service.py +++ b/fig/service.py @@ -154,25 +154,24 @@ class Service(object): def recreate_containers(self, **override_options): """ - If a container for this service doesn't exist, create one. If there are - any, stop them and create new ones. Does not remove the old containers. + If a container for this service doesn't exist, create and start one. If there are + any, stop them, create+start new ones, and remove the old containers. """ containers = self.containers(stopped=True) if len(containers) == 0: log.info("Creating %s..." % self.next_container_name()) - return ([], [self.create_container(**override_options)]) + container = self.create_container(**override_options) + self.start_container(container) + return [(None, container)] else: - old_containers = [] - new_containers = [] + tuples = [] for c in containers: log.info("Recreating %s..." % c.name) - (old_container, new_container) = self.recreate_container(c, **override_options) - old_containers.append(old_container) - new_containers.append(new_container) + tuples.append(self.recreate_container(c, **override_options)) - return (old_containers, new_containers) + return tuples def recreate_container(self, container, **override_options): if container.is_running: @@ -185,17 +184,20 @@ class Service(object): entrypoint=['echo'], command=[], ) - intermediate_container.start() + intermediate_container.start(volumes_from=container.id) intermediate_container.wait() container.remove() options = dict(override_options) options['volumes_from'] = intermediate_container.id new_container = self.create_container(**options) + self.start_container(new_container, volumes_from=intermediate_container.id) + + intermediate_container.remove() return (intermediate_container, new_container) - def start_container(self, container=None, **override_options): + def start_container(self, container=None, volumes_from=None, **override_options): if container is None: container = self.create_container(**override_options) @@ -228,6 +230,7 @@ class Service(object): links=self._get_links(link_to_self=override_options.get('one_off', False)), port_bindings=port_bindings, binds=volume_bindings, + volumes_from=volumes_from, privileged=privileged, ) return container diff --git a/tests/project_test.py b/tests/project_test.py index b8a5d6823..bde40e89b 100644 --- a/tests/project_test.py +++ b/tests/project_test.py @@ -63,29 +63,6 @@ class ProjectTest(DockerClientTestCase): project = Project('test', [web], self.client) self.assertEqual(project.get_service('web'), web) - def test_recreate_containers(self): - web = self.create_service('web') - db = self.create_service('db') - project = Project('test', [web, db], self.client) - - old_web_container = web.create_container() - self.assertEqual(len(web.containers(stopped=True)), 1) - self.assertEqual(len(db.containers(stopped=True)), 0) - - (old, new) = project.recreate_containers() - self.assertEqual(len(old), 1) - self.assertEqual(old[0][0], web) - self.assertEqual(len(new), 2) - self.assertEqual(new[0][0], web) - self.assertEqual(new[1][0], db) - - self.assertEqual(len(web.containers(stopped=True)), 1) - self.assertEqual(len(db.containers(stopped=True)), 1) - - # remove intermediate containers - for (service, container) in old: - container.remove() - def test_start_stop_kill_remove(self): web = self.create_service('web') db = self.create_service('db') @@ -121,12 +98,23 @@ class ProjectTest(DockerClientTestCase): def test_project_up(self): web = self.create_service('web') - db = self.create_service('db') + db = self.create_service('db', volumes=['/var/db']) project = Project('figtest', [web, db], self.client) project.start() self.assertEqual(len(project.containers()), 0) + + project.up(['db']) + self.assertEqual(len(project.containers()), 1) + old_db_id = project.containers()[0].id + db_volume_path = project.containers()[0].inspect()['Volumes']['/var/db'] + project.up() self.assertEqual(len(project.containers()), 2) + + db_container = [c for c in project.containers() if 'db' in c.name][0] + self.assertNotEqual(c.id, old_db_id) + self.assertEqual(c.inspect()['Volumes']['/var/db'], db_volume_path) + project.kill() project.remove_stopped() diff --git a/tests/service_test.py b/tests/service_test.py index 5e8fe3ba3..a8ea017a8 100644 --- a/tests/service_test.py +++ b/tests/service_test.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from __future__ import absolute_import from fig import Service from fig.service import CannotBeScaledError, ConfigError +from fig.packages.docker.client import APIError from .testcases import DockerClientTestCase @@ -132,23 +133,22 @@ class ServiceTest(DockerClientTestCase): num_containers_before = len(self.client.containers(all=True)) service.options['environment']['FOO'] = '2' - (intermediate, new) = service.recreate_containers() - self.assertEqual(len(intermediate), 1) - self.assertEqual(len(new), 1) + tuples = service.recreate_containers() + self.assertEqual(len(tuples), 1) - new_container = new[0] - intermediate_container = intermediate[0] + intermediate_container = tuples[0][0] + new_container = tuples[0][1] self.assertEqual(intermediate_container.dictionary['Config']['Entrypoint'], ['echo']) self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['ps']) self.assertEqual(new_container.dictionary['Config']['Cmd'], ['ax']) self.assertIn('FOO=2', new_container.dictionary['Config']['Env']) self.assertEqual(new_container.name, 'figtest_db_1') - service.start_container(new_container) self.assertEqual(new_container.inspect()['Volumes']['/var/db'], volume_path) - self.assertEqual(len(self.client.containers(all=True)), num_containers_before + 1) + self.assertEqual(len(self.client.containers(all=True)), num_containers_before) self.assertNotEqual(old_container.id, new_container.id) + self.assertRaises(APIError, lambda: self.client.inspect_container(intermediate_container.id)) def test_start_container_passes_through_options(self): db = self.create_service('db') From 5166b2c1a87aa4f09fecbe48d33bffc93b478134 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 23 Apr 2014 18:16:35 +0100 Subject: [PATCH 4/9] Update docker-py Using commit: https://github.com/aanand/docker-py/commit/b31bb4d879c8ecc37491edb9f56369c513577918 --- fig/packages/docker/__init__.py | 2 +- fig/packages/docker/auth/auth.py | 15 +++++--- fig/packages/docker/client.py | 66 ++++++++++++-------------------- fig/packages/docker/errors.py | 61 +++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 48 deletions(-) create mode 100644 fig/packages/docker/errors.py diff --git a/fig/packages/docker/__init__.py b/fig/packages/docker/__init__.py index 5388e7286..e10a57610 100644 --- a/fig/packages/docker/__init__.py +++ b/fig/packages/docker/__init__.py @@ -15,4 +15,4 @@ __title__ = 'docker-py' __version__ = '0.3.0' -from .client import Client, APIError # flake8: noqa +from .client import Client # flake8: noqa diff --git a/fig/packages/docker/auth/auth.py b/fig/packages/docker/auth/auth.py index 69cfa89d3..36f3e4c92 100644 --- a/fig/packages/docker/auth/auth.py +++ b/fig/packages/docker/auth/auth.py @@ -20,6 +20,7 @@ import os from fig.packages import six from ..utils import utils +from .. import errors INDEX_URL = 'https://index.docker.io/v1/' DOCKER_CONFIG_FILENAME = '.dockercfg' @@ -45,18 +46,19 @@ def expand_registry_url(hostname): def resolve_repository_name(repo_name): if '://' in repo_name: - raise ValueError('Repository name cannot contain a ' - 'scheme ({0})'.format(repo_name)) + raise errors.InvalidRepository( + 'Repository name cannot contain a scheme ({0})'.format(repo_name)) parts = repo_name.split('/', 1) if '.' not in parts[0] and ':' not in parts[0] and parts[0] != 'localhost': # This is a docker index repo (ex: foo/bar or ubuntu) return INDEX_URL, repo_name if len(parts) < 2: - raise ValueError('Invalid repository name ({0})'.format(repo_name)) + raise errors.InvalidRepository( + 'Invalid repository name ({0})'.format(repo_name)) if 'index.docker.io' in parts[0]: - raise ValueError('Invalid repository name,' - 'try "{0}" instead'.format(parts[1])) + raise errors.InvalidRepository( + 'Invalid repository name, try "{0}" instead'.format(parts[1])) return expand_registry_url(parts[0]), parts[1] @@ -147,7 +149,8 @@ def load_config(root=None): data.append(line.strip().split(' = ')[1]) if len(data) < 2: # Not enough data - raise Exception('Invalid or empty configuration file!') + raise errors.InvalidConfigFile( + 'Invalid or empty configuration file!') username, password = decode_auth(data[0]) conf[INDEX_URL] = { diff --git a/fig/packages/docker/client.py b/fig/packages/docker/client.py index 8b447d785..7f00b4c45 100644 --- a/fig/packages/docker/client.py +++ b/fig/packages/docker/client.py @@ -24,6 +24,7 @@ from fig.packages import six from .auth import auth from .unixconn import unixconn from .utils import utils +from . import errors if not six.PY3: import websocket @@ -33,41 +34,6 @@ DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 -class APIError(requests.exceptions.HTTPError): - def __init__(self, message, response, explanation=None): - # requests 1.2 supports response as a keyword argument, but - # requests 1.1 doesn't - super(APIError, self).__init__(message) - self.response = response - - self.explanation = explanation - - if self.explanation is None and response.content: - self.explanation = response.content.strip() - - def __str__(self): - message = super(APIError, self).__str__() - - if self.is_client_error(): - message = '%s Client Error: %s' % ( - self.response.status_code, self.response.reason) - - elif self.is_server_error(): - message = '%s Server Error: %s' % ( - self.response.status_code, self.response.reason) - - if self.explanation: - message = '%s ("%s")' % (message, self.explanation) - - return message - - def is_client_error(self): - return 400 <= self.response.status_code < 500 - - def is_server_error(self): - return 500 <= self.response.status_code < 600 - - class Client(requests.Session): def __init__(self, base_url=None, version=DEFAULT_DOCKER_API_VERSION, timeout=DEFAULT_TIMEOUT_SECONDS): @@ -112,7 +78,7 @@ class Client(requests.Session): try: response.raise_for_status() except requests.exceptions.HTTPError as e: - raise APIError(e, response, explanation=explanation) + raise errors.APIError(e, response, explanation=explanation) def _result(self, response, json=False, binary=False): assert not (json and binary) @@ -239,9 +205,23 @@ class Client(requests.Session): def _stream_helper(self, response): """Generator for data coming from a chunked-encoded HTTP response.""" - for line in response.iter_lines(chunk_size=32): - if line: - yield line + socket_fp = self._get_raw_response_socket(response) + socket_fp.setblocking(1) + socket = socket_fp.makefile() + while True: + # Because Docker introduced newlines at the end of chunks in v0.9, + # and only on some API endpoints, we have to cater for both cases. + size_line = socket.readline() + if size_line == '\r\n': + size_line = socket.readline() + + size = int(size_line, 16) + if size <= 0: + break + data = socket.readline() + if not data: + break + yield data def _multiplexed_buffer_helper(self, response): """A generator of multiplexed data blocks read from a buffered @@ -341,7 +321,7 @@ class Client(requests.Session): nocache=False, rm=False, stream=False, timeout=None): remote = context = headers = None if path is None and fileobj is None: - raise Exception("Either path or fileobj needs to be provided.") + raise TypeError("Either path or fileobj needs to be provided.") if fileobj is not None: context = utils.mkbuildcontext(fileobj) @@ -714,8 +694,12 @@ class Client(requests.Session): } if binds: bind_pairs = [ - '{0}:{1}'.format(host, dest) for host, dest in binds.items() + '%s:%s:%s' % ( + h, d['bind'], + 'ro' if 'ro' in d and d['ro'] else 'rw' + ) for h, d in binds.items() ] + start_config['Binds'] = bind_pairs if volumes_from and not isinstance(volumes_from, six.string_types): diff --git a/fig/packages/docker/errors.py b/fig/packages/docker/errors.py new file mode 100644 index 000000000..9aad700d9 --- /dev/null +++ b/fig/packages/docker/errors.py @@ -0,0 +1,61 @@ +# Copyright 2014 dotCloud inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests + + +class APIError(requests.exceptions.HTTPError): + def __init__(self, message, response, explanation=None): + # requests 1.2 supports response as a keyword argument, but + # requests 1.1 doesn't + super(APIError, self).__init__(message) + self.response = response + + self.explanation = explanation + + if self.explanation is None and response.content: + self.explanation = response.content.strip() + + def __str__(self): + message = super(APIError, self).__str__() + + if self.is_client_error(): + message = '%s Client Error: %s' % ( + self.response.status_code, self.response.reason) + + elif self.is_server_error(): + message = '%s Server Error: %s' % ( + self.response.status_code, self.response.reason) + + if self.explanation: + message = '%s ("%s")' % (message, self.explanation) + + return message + + def is_client_error(self): + return 400 <= self.response.status_code < 500 + + def is_server_error(self): + return 500 <= self.response.status_code < 600 + + +class DockerException(Exception): + pass + + +class InvalidRepository(DockerException): + pass + + +class InvalidConfigFile(DockerException): + pass From 9e1dfcfb37be6fc34e1b89d68521e408d6cb14d8 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 23 Apr 2014 18:20:27 +0100 Subject: [PATCH 5/9] Update docker-py APIError imports --- fig/cli/main.py | 2 +- fig/service.py | 2 +- tests/service_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fig/cli/main.py b/fig/cli/main.py index 4700803d1..39ed9f60b 100644 --- a/fig/cli/main.py +++ b/fig/cli/main.py @@ -15,7 +15,7 @@ from .formatter import Formatter from .log_printer import LogPrinter from .utils import yesno -from ..packages.docker.client import APIError +from ..packages.docker.errors import APIError from .errors import UserError from .docopt_command import NoSuchCommand from .socketclient import SocketClient diff --git a/fig/service.py b/fig/service.py index 20c4e120f..54c35d0c2 100644 --- a/fig/service.py +++ b/fig/service.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals from __future__ import absolute_import -from .packages.docker.client import APIError +from .packages.docker.errors import APIError import logging import re import os diff --git a/tests/service_test.py b/tests/service_test.py index a8ea017a8..f1b1f9de5 100644 --- a/tests/service_test.py +++ b/tests/service_test.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals from __future__ import absolute_import from fig import Service from fig.service import CannotBeScaledError, ConfigError -from fig.packages.docker.client import APIError +from fig.packages.docker.errors import APIError from .testcases import DockerClientTestCase From 6e932794f71b99ac6bd85274190d285a156fcdcc Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 25 Apr 2014 12:28:00 +0100 Subject: [PATCH 6/9] Fix regression when mounting volumes Caused by https://github.com/dotcloud/docker-py/commit/77fec67c608d0839a795f6eb807c9df2fd5bfd45 --- fig/service.py | 5 ++++- tests/service_test.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fig/service.py b/fig/service.py index 54c35d0c2..2c7cc1c89 100644 --- a/fig/service.py +++ b/fig/service.py @@ -222,7 +222,10 @@ class Service(object): for volume in options['volumes']: if ':' in volume: external_dir, internal_dir = volume.split(':') - volume_bindings[os.path.abspath(external_dir)] = internal_dir + volume_bindings[os.path.abspath(external_dir)] = { + 'bind': internal_dir, + 'ro': False, + } privileged = options.get('privileged', False) diff --git a/tests/service_test.py b/tests/service_test.py index f1b1f9de5..78947e1f6 100644 --- a/tests/service_test.py +++ b/tests/service_test.py @@ -114,6 +114,12 @@ class ServiceTest(DockerClientTestCase): service.start_container(container) self.assertIn('/var/db', container.inspect()['Volumes']) + def test_create_container_with_specified_volume(self): + service = self.create_service('db', volumes=['/tmp:/host-tmp']) + container = service.create_container() + service.start_container(container) + self.assertIn('/host-tmp', container.inspect()['Volumes']) + def test_recreate_containers(self): service = self.create_service( 'db', From ca7151aeb1848770e6c0df4d49bdc4b70bc57dfa Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Fri, 25 Apr 2014 22:58:21 +0100 Subject: [PATCH 7/9] Split tests into unit and integration --- tests/integration/__init__.py | 0 tests/{ => integration}/cli_test.py | 12 +---- tests/{ => integration}/project_test.py | 59 --------------------- tests/{ => integration}/service_test.py | 28 +--------- tests/{ => integration}/testcases.py | 2 +- tests/unit/__init__.py | 0 tests/unit/cli_test.py | 16 ++++++ tests/{ => unit}/container_test.py | 10 ++-- tests/unit/project_test.py | 69 +++++++++++++++++++++++++ tests/unit/service_test.py | 29 +++++++++++ tests/{ => unit}/sort_service_test.py | 2 +- tests/{ => unit}/split_buffer_test.py | 2 +- 12 files changed, 124 insertions(+), 105 deletions(-) create mode 100644 tests/integration/__init__.py rename tests/{ => integration}/cli_test.py (88%) rename tests/{ => integration}/project_test.py (61%) rename tests/{ => integration}/service_test.py (90%) rename tests/{ => integration}/testcases.py (97%) create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/cli_test.py rename tests/{ => unit}/container_test.py (84%) create mode 100644 tests/unit/project_test.py create mode 100644 tests/unit/service_test.py rename tests/{ => unit}/sort_service_test.py (99%) rename tests/{ => unit}/split_buffer_test.py (97%) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cli_test.py b/tests/integration/cli_test.py similarity index 88% rename from tests/cli_test.py rename to tests/integration/cli_test.py index 2b81e26b0..125b018ed 100644 --- a/tests/cli_test.py +++ b/tests/integration/cli_test.py @@ -2,8 +2,8 @@ from __future__ import unicode_literals from __future__ import absolute_import from .testcases import DockerClientTestCase from mock import patch -from fig.packages.six import StringIO from fig.cli.main import TopLevelCommand +from fig.packages.six import StringIO class CLITestCase(DockerClientTestCase): def setUp(self): @@ -15,16 +15,6 @@ class CLITestCase(DockerClientTestCase): self.command.project.kill() self.command.project.remove_stopped() - def test_yaml_filename_check(self): - self.command.base_dir = 'tests/fixtures/longer-filename-figfile' - - project = self.command.project - - self.assertTrue( project.get_service('definedinyamlnotyml'), "Service: definedinyamlnotyml should have been loaded from .yaml file" ) - - def test_help(self): - self.assertRaises(SystemExit, lambda: self.command.dispatch(['-h'], None)) - @patch('sys.stdout', new_callable=StringIO) def test_ps(self, mock_stdout): self.command.project.get_service('simple').create_container() diff --git a/tests/project_test.py b/tests/integration/project_test.py similarity index 61% rename from tests/project_test.py rename to tests/integration/project_test.py index bde40e89b..fa7e38586 100644 --- a/tests/project_test.py +++ b/tests/integration/project_test.py @@ -4,65 +4,6 @@ from .testcases import DockerClientTestCase class ProjectTest(DockerClientTestCase): - def test_from_dict(self): - project = Project.from_dicts('figtest', [ - { - 'name': 'web', - 'image': 'ubuntu' - }, - { - 'name': 'db', - 'image': 'ubuntu' - } - ], self.client) - self.assertEqual(len(project.services), 2) - self.assertEqual(project.get_service('web').name, 'web') - self.assertEqual(project.get_service('web').options['image'], 'ubuntu') - self.assertEqual(project.get_service('db').name, 'db') - self.assertEqual(project.get_service('db').options['image'], 'ubuntu') - - def test_from_dict_sorts_in_dependency_order(self): - project = Project.from_dicts('figtest', [ - { - 'name': 'web', - 'image': 'ubuntu', - 'links': ['db'], - }, - { - 'name': 'db', - 'image': 'ubuntu' - } - ], self.client) - - self.assertEqual(project.services[0].name, 'db') - self.assertEqual(project.services[1].name, 'web') - - def test_from_config(self): - project = Project.from_config('figtest', { - 'web': { - 'image': 'ubuntu', - }, - 'db': { - 'image': 'ubuntu', - }, - }, self.client) - self.assertEqual(len(project.services), 2) - self.assertEqual(project.get_service('web').name, 'web') - self.assertEqual(project.get_service('web').options['image'], 'ubuntu') - self.assertEqual(project.get_service('db').name, 'db') - self.assertEqual(project.get_service('db').options['image'], 'ubuntu') - - def test_from_config_throws_error_when_not_dict(self): - with self.assertRaises(ConfigurationError): - project = Project.from_config('figtest', { - 'web': 'ubuntu', - }, self.client) - - def test_get_service(self): - web = self.create_service('web') - project = Project('test', [web], self.client) - self.assertEqual(project.get_service('web'), web) - def test_start_stop_kill_remove(self): web = self.create_service('web') db = self.create_service('db') diff --git a/tests/service_test.py b/tests/integration/service_test.py similarity index 90% rename from tests/service_test.py rename to tests/integration/service_test.py index 78947e1f6..78ddbd850 100644 --- a/tests/service_test.py +++ b/tests/integration/service_test.py @@ -1,35 +1,11 @@ from __future__ import unicode_literals from __future__ import absolute_import from fig import Service -from fig.service import CannotBeScaledError, ConfigError +from fig.service import CannotBeScaledError from fig.packages.docker.errors import APIError from .testcases import DockerClientTestCase - class ServiceTest(DockerClientTestCase): - def test_name_validations(self): - self.assertRaises(ConfigError, lambda: Service(name='')) - - self.assertRaises(ConfigError, lambda: Service(name=' ')) - self.assertRaises(ConfigError, lambda: Service(name='/')) - self.assertRaises(ConfigError, lambda: Service(name='!')) - self.assertRaises(ConfigError, lambda: Service(name='\xe2')) - self.assertRaises(ConfigError, lambda: Service(name='_')) - self.assertRaises(ConfigError, lambda: Service(name='____')) - self.assertRaises(ConfigError, lambda: Service(name='foo_bar')) - self.assertRaises(ConfigError, lambda: Service(name='__foo_bar__')) - - Service('a') - Service('foo') - - def test_project_validation(self): - self.assertRaises(ConfigError, lambda: Service(name='foo', project='_')) - Service(name='foo', project='bar') - - def test_config_validation(self): - self.assertRaises(ConfigError, lambda: Service(name='foo', port=['8000'])) - Service(name='foo', ports=['8000']) - def test_containers(self): foo = self.create_service('foo') bar = self.create_service('bar') @@ -277,5 +253,3 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(len(containers), 2) for container in containers: self.assertEqual(list(container.inspect()['HostConfig']['PortBindings'].keys()), ['8000/tcp']) - - diff --git a/tests/testcases.py b/tests/integration/testcases.py similarity index 97% rename from tests/testcases.py rename to tests/integration/testcases.py index ac395040d..f913b7c3e 100644 --- a/tests/testcases.py +++ b/tests/integration/testcases.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from fig.packages.docker import Client from fig.service import Service from fig.cli.utils import docker_url -from . import unittest +from .. import unittest class DockerClientTestCase(unittest.TestCase): diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py new file mode 100644 index 000000000..a3f138048 --- /dev/null +++ b/tests/unit/cli_test.py @@ -0,0 +1,16 @@ +from __future__ import unicode_literals +from __future__ import absolute_import +from .. import unittest +from fig.cli.main import TopLevelCommand +from fig.packages.six import StringIO + +class CLITestCase(unittest.TestCase): + def test_yaml_filename_check(self): + command = TopLevelCommand() + command.base_dir = 'tests/fixtures/longer-filename-figfile' + self.assertTrue(command.project.get_service('definedinyamlnotyml')) + + def test_help(self): + command = TopLevelCommand() + with self.assertRaises(SystemExit): + command.dispatch(['-h'], None) diff --git a/tests/container_test.py b/tests/unit/container_test.py similarity index 84% rename from tests/container_test.py rename to tests/unit/container_test.py index 351a807a8..b1d87f7cf 100644 --- a/tests/container_test.py +++ b/tests/unit/container_test.py @@ -1,10 +1,10 @@ from __future__ import unicode_literals -from .testcases import DockerClientTestCase +from .. import unittest from fig.container import Container -class ContainerTest(DockerClientTestCase): +class ContainerTest(unittest.TestCase): def test_from_ps(self): - container = Container.from_ps(self.client, { + container = Container.from_ps(None, { "Id":"abc", "Image":"ubuntu:12.04", "Command":"sleep 300", @@ -22,7 +22,7 @@ class ContainerTest(DockerClientTestCase): }) def test_environment(self): - container = Container(self.client, { + container = Container(None, { 'ID': 'abc', 'Config': { 'Env': [ @@ -37,7 +37,7 @@ class ContainerTest(DockerClientTestCase): }) def test_number(self): - container = Container.from_ps(self.client, { + container = Container.from_ps(None, { "Id":"abc", "Image":"ubuntu:12.04", "Command":"sleep 300", diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py new file mode 100644 index 000000000..4a2ad1422 --- /dev/null +++ b/tests/unit/project_test.py @@ -0,0 +1,69 @@ +from __future__ import unicode_literals +from .. import unittest +from fig.service import Service +from fig.project import Project, ConfigurationError + +class ProjectTest(unittest.TestCase): + def test_from_dict(self): + project = Project.from_dicts('figtest', [ + { + 'name': 'web', + 'image': 'ubuntu' + }, + { + 'name': 'db', + 'image': 'ubuntu' + } + ], None) + self.assertEqual(len(project.services), 2) + self.assertEqual(project.get_service('web').name, 'web') + self.assertEqual(project.get_service('web').options['image'], 'ubuntu') + self.assertEqual(project.get_service('db').name, 'db') + self.assertEqual(project.get_service('db').options['image'], 'ubuntu') + + def test_from_dict_sorts_in_dependency_order(self): + project = Project.from_dicts('figtest', [ + { + 'name': 'web', + 'image': 'ubuntu', + 'links': ['db'], + }, + { + 'name': 'db', + 'image': 'ubuntu' + } + ], None) + + self.assertEqual(project.services[0].name, 'db') + self.assertEqual(project.services[1].name, 'web') + + def test_from_config(self): + project = Project.from_config('figtest', { + 'web': { + 'image': 'ubuntu', + }, + 'db': { + 'image': 'ubuntu', + }, + }, None) + self.assertEqual(len(project.services), 2) + self.assertEqual(project.get_service('web').name, 'web') + self.assertEqual(project.get_service('web').options['image'], 'ubuntu') + self.assertEqual(project.get_service('db').name, 'db') + self.assertEqual(project.get_service('db').options['image'], 'ubuntu') + + def test_from_config_throws_error_when_not_dict(self): + with self.assertRaises(ConfigurationError): + project = Project.from_config('figtest', { + 'web': 'ubuntu', + }, None) + + def test_get_service(self): + web = Service( + project='figtest', + name='web', + client=None, + image="ubuntu", + ) + project = Project('test', [web], None) + self.assertEqual(project.get_service('web'), web) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py new file mode 100644 index 000000000..490cb60d6 --- /dev/null +++ b/tests/unit/service_test.py @@ -0,0 +1,29 @@ +from __future__ import unicode_literals +from __future__ import absolute_import +from .. import unittest +from fig import Service +from fig.service import ConfigError + +class ServiceTest(unittest.TestCase): + def test_name_validations(self): + self.assertRaises(ConfigError, lambda: Service(name='')) + + self.assertRaises(ConfigError, lambda: Service(name=' ')) + self.assertRaises(ConfigError, lambda: Service(name='/')) + self.assertRaises(ConfigError, lambda: Service(name='!')) + self.assertRaises(ConfigError, lambda: Service(name='\xe2')) + self.assertRaises(ConfigError, lambda: Service(name='_')) + self.assertRaises(ConfigError, lambda: Service(name='____')) + self.assertRaises(ConfigError, lambda: Service(name='foo_bar')) + self.assertRaises(ConfigError, lambda: Service(name='__foo_bar__')) + + Service('a') + Service('foo') + + def test_project_validation(self): + self.assertRaises(ConfigError, lambda: Service(name='foo', project='_')) + Service(name='foo', project='bar') + + def test_config_validation(self): + self.assertRaises(ConfigError, lambda: Service(name='foo', port=['8000'])) + Service(name='foo', ports=['8000']) diff --git a/tests/sort_service_test.py b/tests/unit/sort_service_test.py similarity index 99% rename from tests/sort_service_test.py rename to tests/unit/sort_service_test.py index 13cff89d1..e2a7bdb38 100644 --- a/tests/sort_service_test.py +++ b/tests/unit/sort_service_test.py @@ -1,5 +1,5 @@ from fig.project import sort_service_dicts, DependencyError -from . import unittest +from .. import unittest class SortServiceTest(unittest.TestCase): diff --git a/tests/split_buffer_test.py b/tests/unit/split_buffer_test.py similarity index 97% rename from tests/split_buffer_test.py rename to tests/unit/split_buffer_test.py index b90463c07..a78e99a6e 100644 --- a/tests/split_buffer_test.py +++ b/tests/unit/split_buffer_test.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals from __future__ import absolute_import from fig.cli.utils import split_buffer -from . import unittest +from .. import unittest class SplitBufferTest(unittest.TestCase): def test_single_line_chunks(self): From 29c9763feb8eac648323911627f60624fdd11559 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 18 Feb 2014 16:21:30 +0000 Subject: [PATCH 8/9] Use Orchard to run integration tests --- .travis.yml | 21 +++++++++++++++------ requirements-dev.txt | 1 + script/travis | 23 ----------------------- script/travis-install | 18 ------------------ script/travis-integration | 10 ++++++++++ 5 files changed, 26 insertions(+), 47 deletions(-) delete mode 100755 script/travis delete mode 100755 script/travis-install create mode 100755 script/travis-integration diff --git a/.travis.yml b/.travis.yml index ea561955b..d1c437cd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,13 +3,22 @@ python: - '2.6' - '2.7' env: -- DOCKER_VERSION=0.9.1 -- DOCKER_VERSION=0.10.0 -install: script/travis-install + global: + - secure: exbot0LTV/0Wic6ElKCrOZmh2ZrieuGwEqfYKf5rVuwu1sLngYRihh+lBL/hTwc79NSu829pbwiWfsQZrXbk/yvaS7avGR0CLDoipyPxlYa2/rfs/o4OdTZqXv0LcFmmd54j5QBMpWU1S+CYOwNkwas57trrvIpPbzWjMtfYzOU= +install: +- pip install . +- pip install -r requirements.txt +- pip install -r requirements-dev.txt +- sudo curl -L -o /usr/local/bin/orchard https://github.com/orchardup/go-orchard/releases/download/2.0.5/linux +- sudo chmod +x /usr/local/bin/orchard +before_script: + - '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && orchard hosts rm -f $TRAVIS_JOB_ID' + - '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && orchard hosts create $TRAVIS_JOB_ID || false' script: -- pwd -- env -- sekexe/run "`pwd`/script/travis $TRAVIS_PYTHON_VERSION" + - nosetests tests/unit + - '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && script/travis-integration || false' +after_script: + - '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && orchard hosts rm -f $TRAVIS_JOB_ID || false' deploy: provider: pypi user: orchard diff --git a/requirements-dev.txt b/requirements-dev.txt index 46d6a8aa5..8d4190056 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ mock==1.0.1 nose==1.3.0 pyinstaller==2.1 +unittest2 diff --git a/script/travis b/script/travis deleted file mode 100755 index 878b86e04..000000000 --- a/script/travis +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Exit on first error -set -ex - -# Put Python eggs in a writeable directory -export PYTHON_EGG_CACHE="/tmp/.python-eggs" - -# Activate correct virtualenv -TRAVIS_PYTHON_VERSION=$1 -source /home/travis/virtualenv/python${TRAVIS_PYTHON_VERSION}/bin/activate - -env - -# Kill background processes on exit -trap 'kill -9 $(jobs -p)' SIGINT SIGTERM EXIT - -# Start docker daemon -docker -d -H unix:///var/run/docker.sock 2>> /dev/null >> /dev/null & -sleep 2 - -# $init is set by sekexe -cd $(dirname $init)/.. && nosetests -v diff --git a/script/travis-install b/script/travis-install deleted file mode 100755 index 44aa35325..000000000 --- a/script/travis-install +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -ex - -sudo sh -c "wget -qO- https://get.docker.io/gpg | apt-key add -" -sudo sh -c "echo deb http://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list" -sudo apt-get update -echo exit 101 | sudo tee /usr/sbin/policy-rc.d -sudo chmod +x /usr/sbin/policy-rc.d -sudo apt-get install -qy slirp lxc lxc-docker-$DOCKER_VERSION -git clone git://github.com/jpetazzo/sekexe -python setup.py install -pip install -r requirements-dev.txt - -if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then - pip install unittest2 -fi - diff --git a/script/travis-integration b/script/travis-integration new file mode 100755 index 000000000..c0ff5b4b9 --- /dev/null +++ b/script/travis-integration @@ -0,0 +1,10 @@ +#!/bin/bash +set -ex + +# Kill background processes on exit +trap 'kill -9 $(jobs -p)' SIGINT SIGTERM EXIT + +export DOCKER_HOST=tcp://localhost:4243 +orchard proxy -H $TRAVIS_JOB_ID $DOCKER_HOST & +sleep 2 +nosetests -v From fd85be2c9e248fee0e746df7e6e8455b652f20a5 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 28 Apr 2014 18:22:11 +0100 Subject: [PATCH 9/9] Update docker[-osx] version in install docs --- docs/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install.md b/docs/install.md index c5ad320cf..2269e76c9 100644 --- a/docs/install.md +++ b/docs/install.md @@ -6,9 +6,9 @@ title: Installing Fig Installing Fig ============== -First, install Docker (version 0.8 or higher). If you're on OS X, you can use [docker-osx](https://github.com/noplay/docker-osx): +First, install Docker version 0.10.0. If you're on OS X, you can use [docker-osx](https://github.com/noplay/docker-osx): - $ curl https://raw.github.com/noplay/docker-osx/0.8.0/docker-osx > /usr/local/bin/docker-osx + $ curl https://raw.github.com/noplay/docker-osx/0.10.0/docker-osx > /usr/local/bin/docker-osx $ chmod +x /usr/local/bin/docker-osx $ docker-osx shell