tyler.durden
/
compose
mirror of https://github.com/docker/compose.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #186 from orchardup/update-docker-py-7f55a101f813f3e96413d1b577e98d9467b0bffc 

WIP: Docker >=0.9 support, docker-py 0.3.1
This commit is contained in:
Aanand Prasad 2014-04-28 18:51:38 +01:00
parent 24959209cc fd85be2c9e
commit 8f1793dd06
29 changed files with 588 additions and 418 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.travis.yml
View File

@ -3,17 +3,22 @@ python:
- '2.6'
- '2.7'
env:
- DOCKER_VERSION=0.8.0
- DOCKER_VERSION=0.8.1
matrix:
allow_failures:
- python: '3.2'
- python: '3.3'
install: script/travis-install
global:
- secure: exbot0LTV/0Wic6ElKCrOZmh2ZrieuGwEqfYKf5rVuwu1sLngYRihh+lBL/hTwc79NSu829pbwiWfsQZrXbk/yvaS7avGR0CLDoipyPxlYa2/rfs/o4OdTZqXv0LcFmmd54j5QBMpWU1S+CYOwNkwas57trrvIpPbzWjMtfYzOU=
install:
- pip install .
- pip install -r requirements.txt
- pip install -r requirements-dev.txt
- sudo curl -L -o /usr/local/bin/orchard https://github.com/orchardup/go-orchard/releases/download/2.0.5/linux
- sudo chmod +x /usr/local/bin/orchard
before_script:
- '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && orchard hosts rm -f $TRAVIS_JOB_ID'
- '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && orchard hosts create $TRAVIS_JOB_ID || false'
script:
- pwd
- env
- sekexe/run "`pwd`/script/travis $TRAVIS_PYTHON_VERSION"
- nosetests tests/unit
- '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && script/travis-integration || false'
after_script:
- '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && orchard hosts rm -f $TRAVIS_JOB_ID || false'
deploy:
provider: pypi
user: orchard

4
docs/install.md
View File

@ -6,9 +6,9 @@ title: Installing Fig
Installing Fig
==============
First, install Docker (version 0.8 or higher). If you're on OS X, you can use [docker-osx](https://github.com/noplay/docker-osx):
First, install Docker version 0.10.0. If you're on OS X, you can use [docker-osx](https://github.com/noplay/docker-osx):
$ curl https://raw.github.com/noplay/docker-osx/0.8.0/docker-osx > /usr/local/bin/docker-osx
$ curl https://raw.github.com/noplay/docker-osx/0.10.0/docker-osx > /usr/local/bin/docker-osx
$ chmod +x /usr/local/bin/docker-osx
$ docker-osx shell

5
fig/cli/main.py
View File

@ -15,7 +15,7 @@ from .formatter import Formatter
from .log_printer import LogPrinter
from .utils import yesno
from ..packages.docker.client import APIError
from ..packages.docker.errors import APIError
from .errors import UserError
from .docopt_command import NoSuchCommand
from .socketclient import SocketClient
@ -301,10 +301,9 @@ class TopLevelCommand(Command):
"""
detached = options['-d']
new = self.project.up(service_names=options['SERVICE'])
to_attach = self.project.up(service_names=options['SERVICE'])
if not detached:
to_attach = [c for (s, c) in new]
print("Attaching to", list_containers(to_attach))
log_printer = LogPrinter(to_attach, attach_params={"logs": True})

5
fig/packages/docker/__init__.py
View File

@ -12,4 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from .client import Client, APIError # flake8: noqa
__title__ = 'docker-py'
__version__ = '0.3.0'
from .client import Client # flake8: noqa

28
fig/packages/docker/auth/auth.py
View File

@ -20,6 +20,7 @@ import os
from fig.packages import six
from ..utils import utils
from .. import errors
INDEX_URL = 'https://index.docker.io/v1/'
DOCKER_CONFIG_FILENAME = '.dockercfg'
@ -45,18 +46,19 @@ def expand_registry_url(hostname):
def resolve_repository_name(repo_name):
if '://' in repo_name:
raise ValueError('Repository name cannot contain a '
'scheme ({0})'.format(repo_name))
raise errors.InvalidRepository(
'Repository name cannot contain a scheme ({0})'.format(repo_name))
parts = repo_name.split('/', 1)
if not '.' in parts[0] and not ':' in parts[0] and parts[0] != 'localhost':
if '.' not in parts[0] and ':' not in parts[0] and parts[0] != 'localhost':
# This is a docker index repo (ex: foo/bar or ubuntu)
return INDEX_URL, repo_name
if len(parts) < 2:
raise ValueError('Invalid repository name ({0})'.format(repo_name))
raise errors.InvalidRepository(
'Invalid repository name ({0})'.format(repo_name))
if 'index.docker.io' in parts[0]:
raise ValueError('Invalid repository name,'
'try "{0}" instead'.format(parts[1]))
raise errors.InvalidRepository(
'Invalid repository name, try "{0}" instead'.format(parts[1]))
return expand_registry_url(parts[0]), parts[1]
@ -87,6 +89,11 @@ def resolve_authconfig(authconfig, registry=None):
return authconfig.get(swap_protocol(registry), None)
def encode_auth(auth_info):
return base64.b64encode(auth_info.get('username', '') + b':' +
auth_info.get('password', ''))
def decode_auth(auth):
if isinstance(auth, six.string_types):
auth = auth.encode('ascii')
@ -100,6 +107,12 @@ def encode_header(auth):
return base64.b64encode(auth_json)
def encode_full_header(auth):
""" Returns the given auth block encoded for the X-Registry-Config header.
"""
return encode_header({'configs': auth})
def load_config(root=None):
"""Loads authentication data from a Docker configuration file in the given
root directory."""
@ -136,7 +149,8 @@ def load_config(root=None):
data.append(line.strip().split(' = ')[1])
if len(data) < 2:
# Not enough data
raise Exception('Invalid or empty configuration file!')
raise errors.InvalidConfigFile(
'Invalid or empty configuration file!')
username, password = decode_auth(data[0])
conf[INDEX_URL] = {

197
fig/packages/docker/client.py
View File

@ -24,48 +24,18 @@ from fig.packages import six
from .auth import auth
from .unixconn import unixconn
from .utils import utils
from . import errors
if not six.PY3:
import websocket
DEFAULT_DOCKER_API_VERSION = '1.9'
DEFAULT_TIMEOUT_SECONDS = 60
STREAM_HEADER_SIZE_BYTES = 8
class APIError(requests.exceptions.HTTPError):
def __init__(self, message, response, explanation=None):
super(APIError, self).__init__(message, response=response)
self.explanation = explanation
if self.explanation is None and response.content:
self.explanation = response.content.strip()
def __str__(self):
message = super(APIError, self).__str__()
if self.is_client_error():
message = '%s Client Error: %s' % (
self.response.status_code, self.response.reason)
elif self.is_server_error():
message = '%s Server Error: %s' % (
self.response.status_code, self.response.reason)
if self.explanation:
message = '%s ("%s")' % (message, self.explanation)
return message
def is_client_error(self):
return 400 <= self.response.status_code < 500
def is_server_error(self):
return 500 <= self.response.status_code < 600
class Client(requests.Session):
def __init__(self, base_url=None, version="1.6",
def __init__(self, base_url=None, version=DEFAULT_DOCKER_API_VERSION,
timeout=DEFAULT_TIMEOUT_SECONDS):
super(Client, self).__init__()
if base_url is None:
@ -108,7 +78,7 @@ class Client(requests.Session):
try:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
raise APIError(e, response, explanation=explanation)
raise errors.APIError(e, response, explanation=explanation)
def _result(self, response, json=False, binary=False):
assert not (json and binary)
@ -125,7 +95,7 @@ class Client(requests.Session):
mem_limit=0, ports=None, environment=None, dns=None,
volumes=None, volumes_from=None,
network_disabled=False, entrypoint=None,
cpu_shares=None, working_dir=None):
cpu_shares=None, working_dir=None, domainname=None):
if isinstance(command, six.string_types):
command = shlex.split(str(command))
if isinstance(environment, dict):
@ -133,7 +103,7 @@ class Client(requests.Session):
'{0}={1}'.format(k, v) for k, v in environment.items()
]
if ports and isinstance(ports, list):
if isinstance(ports, list):
exposed_ports = {}
for port_definition in ports:
port = port_definition
@ -145,12 +115,15 @@ class Client(requests.Session):
exposed_ports['{0}/{1}'.format(port, proto)] = {}
ports = exposed_ports
if volumes and isinstance(volumes, list):
if isinstance(volumes, list):
volumes_dict = {}
for vol in volumes:
volumes_dict[vol] = {}
volumes = volumes_dict
if volumes_from and not isinstance(volumes_from, six.string_types):
volumes_from = ','.join(volumes_from)
attach_stdin = False
attach_stdout = False
attach_stderr = False
@ -165,26 +138,27 @@ class Client(requests.Session):
stdin_once = True
return {
'Hostname': hostname,
'Hostname': hostname,
'Domainname': domainname,
'ExposedPorts': ports,
'User': user,
'Tty': tty,
'OpenStdin': stdin_open,
'StdinOnce': stdin_once,
'Memory': mem_limit,
'AttachStdin': attach_stdin,
'User': user,
'Tty': tty,
'OpenStdin': stdin_open,
'StdinOnce': stdin_once,
'Memory': mem_limit,
'AttachStdin': attach_stdin,
'AttachStdout': attach_stdout,
'AttachStderr': attach_stderr,
'Env': environment,
'Cmd': command,
'Dns': dns,
'Image': image,
'Volumes': volumes,
'VolumesFrom': volumes_from,
'Env': environment,
'Cmd': command,
'Dns': dns,
'Image': image,
'Volumes': volumes,
'VolumesFrom': volumes_from,
'NetworkDisabled': network_disabled,
'Entrypoint': entrypoint,
'CpuShares': cpu_shares,
'WorkingDir': working_dir
'Entrypoint': entrypoint,
'CpuShares': cpu_shares,
'WorkingDir': working_dir
}
def _post_json(self, url, data, **kwargs):
@ -222,25 +196,26 @@ class Client(requests.Session):
def _create_websocket_connection(self, url):
return websocket.create_connection(url)
def _stream_result(self, response):
"""Generator for straight-out, non chunked-encoded HTTP responses."""
def _get_raw_response_socket(self, response):
self._raise_for_status(response)
for line in response.iter_lines(chunk_size=1, decode_unicode=True):
# filter out keep-alive new lines
if line:
yield line + '\n'
def _stream_result_socket(self, response):
self._raise_for_status(response)
return response.raw._fp.fp._sock
if six.PY3:
return response.raw._fp.fp.raw._sock
else:
return response.raw._fp.fp._sock
def _stream_helper(self, response):
"""Generator for data coming from a chunked-encoded HTTP response."""
socket_fp = self._stream_result_socket(response)
socket_fp = self._get_raw_response_socket(response)
socket_fp.setblocking(1)
socket = socket_fp.makefile()
while True:
size = int(socket.readline(), 16)
# Because Docker introduced newlines at the end of chunks in v0.9,
# and only on some API endpoints, we have to cater for both cases.
size_line = socket.readline()
if size_line == '\r\n':
size_line = socket.readline()
size = int(size_line, 16)
if size <= 0:
break
data = socket.readline()
@ -265,17 +240,20 @@ class Client(requests.Session):
def _multiplexed_socket_stream_helper(self, response):
"""A generator of multiplexed data blocks coming from a response
socket."""
socket = self._stream_result_socket(response)
socket = self._get_raw_response_socket(response)
def recvall(socket, size):
data = ''
blocks = []
while size > 0:
block = socket.recv(size)
if not block:
return None
data += block
blocks.append(block)
size -= len(block)
sep = bytes() if six.PY3 else str()
data = sep.join(blocks)
return data
while True:
@ -304,9 +282,18 @@ class Client(requests.Session):
u = self._url("/containers/{0}/attach".format(container))
response = self._post(u, params=params, stream=stream)
# Stream multi-plexing was introduced in API v1.6.
# Stream multi-plexing was only introduced in API v1.6. Anything before
# that needs old-style streaming.
if utils.compare_version('1.6', self._version) < 0:
return stream and self._stream_result(response) or \
def stream_result():
self._raise_for_status(response)
for line in response.iter_lines(chunk_size=1,
decode_unicode=True):
# filter out keep-alive new lines
if line:
yield line
return stream_result() if stream else \
self._result(response, binary=True)
return stream and self._multiplexed_socket_stream_helper(response) or \
@ -319,20 +306,22 @@ class Client(requests.Session):
'stderr': 1,
'stream': 1
}
if ws:
return self._attach_websocket(container, params)
if isinstance(container, dict):
container = container.get('Id')
u = self._url("/containers/{0}/attach".format(container))
return self._stream_result_socket(self.post(
return self._get_raw_response_socket(self.post(
u, None, params=self._attach_params(params), stream=True))
def build(self, path=None, tag=None, quiet=False, fileobj=None,
nocache=False, rm=False, stream=False, timeout=None):
remote = context = headers = None
if path is None and fileobj is None:
raise Exception("Either path or fileobj needs to be provided.")
raise TypeError("Either path or fileobj needs to be provided.")
if fileobj is not None:
context = utils.mkbuildcontext(fileobj)
@ -341,6 +330,9 @@ class Client(requests.Session):
else:
context = utils.tar(path)
if utils.compare_version('1.8', self._version) >= 0:
stream = True
u = self._url('/build')
params = {
't': tag,
@ -352,6 +344,19 @@ class Client(requests.Session):
if context is not None:
headers = {'Content-Type': 'application/tar'}
if utils.compare_version('1.9', self._version) >= 0:
# If we don't have any auth data so far, try reloading the config
# file one more time in case anything showed up in there.
if not self._auth_configs:
self._auth_configs = auth.load_config()
# Send the full auth configuration (if any exists), since the build
# could use any (or all) of the registries.
if self._auth_configs:
headers['X-Registry-Config'] = auth.encode_full_header(
self._auth_configs
)
response = self._post(
u,
data=context,
@ -363,8 +368,9 @@ class Client(requests.Session):
if context is not None:
context.close()
if stream:
return self._stream_result(response)
return self._stream_helper(response)
else:
output = self._result(response)
srch = r'Successfully built ([0-9a-f]+)'
@ -403,6 +409,8 @@ class Client(requests.Session):
return res
def copy(self, container, resource):
if isinstance(container, dict):
container = container.get('Id')
res = self._post_json(
self._url("/containers/{0}/copy".format(container)),
data={"Resource": resource},
@ -416,12 +424,12 @@ class Client(requests.Session):
mem_limit=0, ports=None, environment=None, dns=None,
volumes=None, volumes_from=None,
network_disabled=False, name=None, entrypoint=None,
cpu_shares=None, working_dir=None):
cpu_shares=None, working_dir=None, domainname=None):
config = self._container_config(
image, command, hostname, user, detach, stdin_open, tty, mem_limit,
ports, environment, dns, volumes, volumes_from, network_disabled,
entrypoint, cpu_shares, working_dir
entrypoint, cpu_shares, working_dir, domainname
)
return self.create_container_from_config(config, name)
@ -440,21 +448,7 @@ class Client(requests.Session):
format(container))), True)
def events(self):
u = self._url("/events")
socket = self._stream_result_socket(self.get(u, stream=True))
while True:
chunk = socket.recv(4096)
if chunk:
# Messages come in the format of length, data, newline.
length, data = chunk.split("\n", 1)
length = int(length, 16)
if length > len(data):
data += socket.recv(length - len(data))
yield json.loads(data)
else:
break
return self._stream_helper(self.get(self._url('/events'), stream=True))
def export(self, container):
if isinstance(container, dict):
@ -471,6 +465,8 @@ class Client(requests.Session):
def images(self, name=None, quiet=False, all=False, viz=False):
if viz:
if utils.compare_version('1.7', self._version) >= 0:
raise Exception('Viz output is not supported in API >= 1.7!')
return self._result(self._get(self._url("images/viz")))
params = {
'filter': name,
@ -618,7 +614,7 @@ class Client(requests.Session):
self._auth_configs = auth.load_config()
authcfg = auth.resolve_authconfig(self._auth_configs, registry)
# Do not fail here if no atuhentication exists for this specific
# Do not fail here if no authentication exists for this specific
# registry as we can have a readonly pull. Just put the header if
# we can.
if authcfg:
@ -644,7 +640,7 @@ class Client(requests.Session):
self._auth_configs = auth.load_config()
authcfg = auth.resolve_authconfig(self._auth_configs, registry)
# Do not fail here if no atuhentication exists for this specific
# Do not fail here if no authentication exists for this specific
# registry as we can have a readonly pull. Just put the header if
# we can.
if authcfg:
@ -652,7 +648,7 @@ class Client(requests.Session):
response = self._post_json(u, None, headers=headers, stream=stream)
else:
response = self._post_json(u, authcfg, stream=stream)
response = self._post_json(u, None, stream=stream)
return stream and self._stream_helper(response) \
or self._result(response)
@ -682,8 +678,8 @@ class Client(requests.Session):
params={'term': term}),
True)
def start(self, container, binds=None, port_bindings=None, lxc_conf=None,
publish_all_ports=False, links=None, privileged=False):
def start(self, container, binds=None, volumes_from=None, port_bindings=None,
lxc_conf=None, publish_all_ports=False, links=None, privileged=False):
if isinstance(container, dict):
container = container.get('Id')
@ -698,10 +694,19 @@ class Client(requests.Session):
}
if binds:
bind_pairs = [
'{0}:{1}'.format(host, dest) for host, dest in binds.items()
'%s:%s:%s' % (
h, d['bind'],
'ro' if 'ro' in d and d['ro'] else 'rw'
) for h, d in binds.items()
]
start_config['Binds'] = bind_pairs
if volumes_from and not isinstance(volumes_from, six.string_types):
volumes_from = ','.join(volumes_from)
start_config['VolumesFrom'] = volumes_from
if port_bindings:
start_config['PortBindings'] = utils.convert_port_bindings(
port_bindings

61
fig/packages/docker/errors.py Normal file
View File

@ -0,0 +1,61 @@
# Copyright 2014 dotCloud inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import requests
class APIError(requests.exceptions.HTTPError):
def __init__(self, message, response, explanation=None):
# requests 1.2 supports response as a keyword argument, but
# requests 1.1 doesn't
super(APIError, self).__init__(message)
self.response = response
self.explanation = explanation
if self.explanation is None and response.content:
self.explanation = response.content.strip()
def __str__(self):
message = super(APIError, self).__str__()
if self.is_client_error():
message = '%s Client Error: %s' % (
self.response.status_code, self.response.reason)
elif self.is_server_error():
message = '%s Server Error: %s' % (
self.response.status_code, self.response.reason)
if self.explanation:
message = '%s ("%s")' % (message, self.explanation)
return message
def is_client_error(self):
return 400 <= self.response.status_code < 500
def is_server_error(self):
return 500 <= self.response.status_code < 600
class DockerException(Exception):
pass
class InvalidRepository(DockerException):
pass
class InvalidConfigFile(DockerException):
pass

2
fig/packages/docker/unixconn/unixconn.py
View File

@ -40,7 +40,7 @@ class UnixHTTPConnection(httplib.HTTPConnection, object):
self.sock = sock
def _extract_path(self, url):
#remove the base_url entirely..
# remove the base_url entirely..
return url.replace(self.base_url, "")
def request(self, method, url, **kwargs):

2
fig/packages/docker/utils/__init__.py
View File

@ -1,3 +1,3 @@
from .utils import (
compare_version, convert_port_bindings, mkbuildcontext, ping, tar
compare_version, convert_port_bindings, mkbuildcontext, ping, tar, parse_repository_tag
) # flake8: noqa

36
fig/packages/docker/utils/utils.py
View File

@ -15,6 +15,7 @@
import io
import tarfile
import tempfile
from distutils.version import StrictVersion
import requests
from fig.packages import six
@ -51,15 +52,34 @@ def tar(path):
def compare_version(v1, v2):
return float(v2) - float(v1)
"""Compare docker versions
>>> v1 = '1.9'
>>> v2 = '1.10'
>>> compare_version(v1, v2)
1
>>> compare_version(v2, v1)
-1
>>> compare_version(v2, v2)
0
"""
s1 = StrictVersion(v1)
s2 = StrictVersion(v2)
if s1 == s2:
return 0
elif s1 > s2:
return -1
else:
return 1
def ping(url):
try:
res = requests.get(url)
return res.status >= 400
except Exception:
return False
else:
return res.status_code < 400
def _convert_port_binding(binding):
@ -94,3 +114,15 @@ def convert_port_bindings(port_bindings):
else:
result[key] = [_convert_port_binding(v)]
return result
def parse_repository_tag(repo):
column_index = repo.rfind(':')
if column_index < 0:
return repo, ""
tag = repo[column_index+1:]
slash_index = tag.find('/')
if slash_index < 0:
return repo[:column_index], tag
return repo, ""

29
fig/project.py
View File

@ -105,23 +105,6 @@ class Project(object):
unsorted = [self.get_service(name) for name in service_names]
return [s for s in self.services if s in unsorted]
def recreate_containers(self, service_names=None):
"""
For each service, create or recreate their containers.
Returns a tuple with two lists. The first is a list of
(service, old_container) tuples; the second is a list
of (service, new_container) tuples.
"""
old = []
new = []
for service in self.get_services(service_names):
(s_old, s_new) = service.recreate_containers()
old += [(service, container) for container in s_old]
new += [(service, container) for container in s_new]
return (old, new)
def start(self, service_names=None, **options):
for service in self.get_services(service_names):
service.start(**options)
@ -142,15 +125,13 @@ class Project(object):
log.info('%s uses an image, skipping' % service.name)
def up(self, service_names=None):
(old, new) = self.recreate_containers(service_names=service_names)
new_containers = []
for (service, container) in new:
service.start_container(container)
for service in self.get_services(service_names):
for (_, new) in service.recreate_containers():
new_containers.append(new)
for (service, container) in old:
container.remove()
return new
return new_containers
def remove_stopped(self, service_names=None, **options):
for service in self.get_services(service_names):

123
fig/service.py
View File

@ -1,10 +1,11 @@
from __future__ import unicode_literals
from __future__ import absolute_import
from .packages.docker.client import APIError
from .packages.docker.errors import APIError
import logging
import re
import os
import sys
import json
from .container import Container
log = logging.getLogger(__name__)
@ -146,31 +147,31 @@ class Service(object):
except APIError as e:
if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation):
log.info('Pulling image %s...' % container_options['image'])
self.client.pull(container_options['image'])
output = self.client.pull(container_options['image'], stream=True)
stream_output(output, sys.stdout)
return Container.create(self.client, **container_options)
raise
def recreate_containers(self, **override_options):
"""
If a container for this service doesn't exist, create one. If there are
any, stop them and create new ones. Does not remove the old containers.
If a container for this service doesn't exist, create and start one. If there are
any, stop them, create+start new ones, and remove the old containers.
"""
containers = self.containers(stopped=True)
if len(containers) == 0:
log.info("Creating %s..." % self.next_container_name())
return ([], [self.create_container(**override_options)])
container = self.create_container(**override_options)
self.start_container(container)
return [(None, container)]
else:
old_containers = []
new_containers = []
tuples = []
for c in containers:
log.info("Recreating %s..." % c.name)
(old_container, new_container) = self.recreate_container(c, **override_options)
old_containers.append(old_container)
new_containers.append(new_container)
tuples.append(self.recreate_container(c, **override_options))
return (old_containers, new_containers)
return tuples
def recreate_container(self, container, **override_options):
if container.is_running:
@ -183,17 +184,20 @@ class Service(object):
entrypoint=['echo'],
command=[],
)
intermediate_container.start()
intermediate_container.start(volumes_from=container.id)
intermediate_container.wait()
container.remove()
options = dict(override_options)
options['volumes_from'] = intermediate_container.id
new_container = self.create_container(**options)
self.start_container(new_container, volumes_from=intermediate_container.id)
intermediate_container.remove()
return (intermediate_container, new_container)
def start_container(self, container=None, **override_options):
def start_container(self, container=None, volumes_from=None, **override_options):
if container is None:
container = self.create_container(**override_options)
@ -218,7 +222,10 @@ class Service(object):
for volume in options['volumes']:
if ':' in volume:
external_dir, internal_dir = volume.split(':')
volume_bindings[os.path.abspath(external_dir)] = internal_dir
volume_bindings[os.path.abspath(external_dir)] = {
'bind': internal_dir,
'ro': False,
}
privileged = options.get('privileged', False)
@ -226,6 +233,7 @@ class Service(object):
links=self._get_links(link_to_self=override_options.get('one_off', False)),
port_bindings=port_bindings,
binds=volume_bindings,
volumes_from=volumes_from,
privileged=privileged,
)
return container
@ -299,14 +307,15 @@ class Service(object):
stream=True
)
all_events = stream_output(build_output, sys.stdout)
image_id = None
for line in build_output:
if line:
match = re.search(r'Successfully built ([0-9a-f]+)', line)
for event in all_events:
if 'stream' in event:
match = re.search(r'Successfully built ([0-9a-f]+)', event.get('stream', ''))
if match:
image_id = match.group(1)
sys.stdout.write(line.encode(sys.__stdout__.encoding or 'utf-8'))
if image_id is None:
raise BuildError(self)
@ -329,6 +338,84 @@ class Service(object):
return True
def stream_output(output, stream):
is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno())
all_events = []
lines = {}
diff = 0
for chunk in output:
event = json.loads(chunk)
all_events.append(event)
if 'progress' in event or 'progressDetail' in event:
image_id = event['id']
if image_id in lines:
diff = len(lines) - lines[image_id]
else:
lines[image_id] = len(lines)
stream.write("\n")
diff = 0
if is_terminal:
# move cursor up `diff` rows
stream.write("%c[%dA" % (27, diff))
try:
print_output_event(event, stream, is_terminal)
except Exception:
stream.write(repr(event) + "\n")
raise
if 'id' in event and is_terminal:
# move cursor back down
stream.write("%c[%dB" % (27, diff))
stream.flush()
return all_events
def print_output_event(event, stream, is_terminal):
if 'errorDetail' in event:
raise Exception(event['errorDetail']['message'])
terminator = ''
if is_terminal and 'stream' not in event:
# erase current line
stream.write("%c[2K\r" % 27)
terminator = "\r"
pass
elif 'progressDetail' in event:
return
if 'time' in event:
stream.write("[%s] " % event['time'])
if 'id' in event:
stream.write("%s: " % event['id'])
if 'from' in event:
stream.write("(from %s) " % event['from'])
status = event.get('status', '')
if 'progress' in event:
stream.write("%s %s%s" % (status, event['progress'], terminator))
elif 'progressDetail' in event:
detail = event['progressDetail']
if 'current' in detail:
percentage = float(detail['current']) / float(detail['total']) * 100
stream.write('%s (%.1f%%)%s' % (status, percentage, terminator))
else:
stream.write('%s%s' % (status, terminator))
elif 'stream' in event:
stream.write("%s%s" % (event['stream'], terminator))
else:
stream.write("%s%s\n" % (status, terminator))
NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')

1
requirements-dev.txt
View File

@ -1,3 +1,4 @@
mock==1.0.1
nose==1.3.0
pyinstaller==2.1
unittest2

23
script/travis
View File

@ -1,23 +0,0 @@
#!/bin/bash
# Exit on first error
set -ex
# Put Python eggs in a writeable directory
export PYTHON_EGG_CACHE="/tmp/.python-eggs"
# Activate correct virtualenv
TRAVIS_PYTHON_VERSION=$1
source /home/travis/virtualenv/python${TRAVIS_PYTHON_VERSION}/bin/activate
env
# Kill background processes on exit
trap 'kill -9 $(jobs -p)' SIGINT SIGTERM EXIT
# Start docker daemon
docker -d -H unix:///var/run/docker.sock 2>> /dev/null >> /dev/null &
sleep 2
# $init is set by sekexe
cd $(dirname $init)/.. && nosetests -v

18
script/travis-install
View File

@ -1,18 +0,0 @@
#!/bin/bash
set -ex
sudo sh -c "wget -qO- https://get.docker.io/gpg | apt-key add -"
sudo sh -c "echo deb http://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list"
sudo apt-get update
echo exit 101 | sudo tee /usr/sbin/policy-rc.d
sudo chmod +x /usr/sbin/policy-rc.d
sudo apt-get install -qy slirp lxc lxc-docker-$DOCKER_VERSION
git clone git://github.com/jpetazzo/sekexe
python setup.py install
pip install -r requirements-dev.txt
if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then
pip install unittest2
fi

10
script/travis-integration Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
set -ex
# Kill background processes on exit
trap 'kill -9 $(jobs -p)' SIGINT SIGTERM EXIT
export DOCKER_HOST=tcp://localhost:4243
orchard proxy -H $TRAVIS_JOB_ID $DOCKER_HOST &
sleep 2
nosetests -v

0
tests/integration/__init__.py Normal file
View File

12
tests/cli_test.py → tests/integration/cli_test.py
View File

@ -2,8 +2,8 @@ from __future__ import unicode_literals
from __future__ import absolute_import
from .testcases import DockerClientTestCase
from mock import patch
from fig.packages.six import StringIO
from fig.cli.main import TopLevelCommand
from fig.packages.six import StringIO
class CLITestCase(DockerClientTestCase):
def setUp(self):
@ -15,16 +15,6 @@ class CLITestCase(DockerClientTestCase):
self.command.project.kill()
self.command.project.remove_stopped()
def test_yaml_filename_check(self):
self.command.base_dir = 'tests/fixtures/longer-filename-figfile'
project = self.command.project
self.assertTrue( project.get_service('definedinyamlnotyml'), "Service: definedinyamlnotyml should have been loaded from .yaml file" )
def test_help(self):
self.assertRaises(SystemExit, lambda: self.command.dispatch(['-h'], None))
@patch('sys.stdout', new_callable=StringIO)
def test_ps(self, mock_stdout):
self.command.project.get_service('simple').create_container()

87
tests/integration/project_test.py Normal file
View File

@ -0,0 +1,87 @@
from __future__ import unicode_literals
from fig.project import Project, ConfigurationError
from .testcases import DockerClientTestCase
class ProjectTest(DockerClientTestCase):
def test_start_stop_kill_remove(self):
web = self.create_service('web')
db = self.create_service('db')
project = Project('figtest', [web, db], self.client)
project.start()
self.assertEqual(len(web.containers()), 0)
self.assertEqual(len(db.containers()), 0)
web_container_1 = web.create_container()
web_container_2 = web.create_container()
db_container = db.create_container()
project.start(service_names=['web'])
self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name]))
project.start()
self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name, db_container.name]))
project.stop(service_names=['web'], timeout=1)
self.assertEqual(set(c.name for c in project.containers()), set([db_container.name]))
project.kill(service_names=['db'])
self.assertEqual(len(project.containers()), 0)
self.assertEqual(len(project.containers(stopped=True)), 3)
project.remove_stopped(service_names=['web'])
self.assertEqual(len(project.containers(stopped=True)), 1)
project.remove_stopped()
self.assertEqual(len(project.containers(stopped=True)), 0)
def test_project_up(self):
web = self.create_service('web')
db = self.create_service('db', volumes=['/var/db'])
project = Project('figtest', [web, db], self.client)
project.start()
self.assertEqual(len(project.containers()), 0)
project.up(['db'])
self.assertEqual(len(project.containers()), 1)
old_db_id = project.containers()[0].id
db_volume_path = project.containers()[0].inspect()['Volumes']['/var/db']
project.up()
self.assertEqual(len(project.containers()), 2)
db_container = [c for c in project.containers() if 'db' in c.name][0]
self.assertNotEqual(c.id, old_db_id)
self.assertEqual(c.inspect()['Volumes']['/var/db'], db_volume_path)
project.kill()
project.remove_stopped()
def test_unscale_after_restart(self):
web = self.create_service('web')
project = Project('figtest', [web], self.client)
project.start()
service = project.get_service('web')
service.scale(1)
self.assertEqual(len(service.containers()), 1)
service.scale(3)
self.assertEqual(len(service.containers()), 3)
project.up()
service = project.get_service('web')
self.assertEqual(len(service.containers()), 3)
service.scale(1)
self.assertEqual(len(service.containers()), 1)
project.up()
service = project.get_service('web')
self.assertEqual(len(service.containers()), 1)
# does scale=0 ,makes any sense? after recreating at least 1 container is running
service.scale(0)
project.up()
service = project.get_service('web')
self.assertEqual(len(service.containers()), 1)
project.kill()
project.remove_stopped()

48
tests/service_test.py → tests/integration/service_test.py
View File

@ -1,34 +1,11 @@
from __future__ import unicode_literals
from __future__ import absolute_import
from fig import Service
from fig.service import CannotBeScaledError, ConfigError
from fig.service import CannotBeScaledError
from fig.packages.docker.errors import APIError
from .testcases import DockerClientTestCase
class ServiceTest(DockerClientTestCase):
def test_name_validations(self):
self.assertRaises(ConfigError, lambda: Service(name=''))
self.assertRaises(ConfigError, lambda: Service(name=' '))
self.assertRaises(ConfigError, lambda: Service(name='/'))
self.assertRaises(ConfigError, lambda: Service(name='!'))
self.assertRaises(ConfigError, lambda: Service(name='\xe2'))
self.assertRaises(ConfigError, lambda: Service(name='_'))
self.assertRaises(ConfigError, lambda: Service(name='____'))
self.assertRaises(ConfigError, lambda: Service(name='foo_bar'))
self.assertRaises(ConfigError, lambda: Service(name='__foo_bar__'))
Service('a')
Service('foo')
def test_project_validation(self):
self.assertRaises(ConfigError, lambda: Service(name='foo', project='_'))
Service(name='foo', project='bar')
def test_config_validation(self):
self.assertRaises(ConfigError, lambda: Service(name='foo', port=['8000']))
Service(name='foo', ports=['8000'])
def test_containers(self):
foo = self.create_service('foo')
bar = self.create_service('bar')
@ -113,6 +90,12 @@ class ServiceTest(DockerClientTestCase):
service.start_container(container)
self.assertIn('/var/db', container.inspect()['Volumes'])
def test_create_container_with_specified_volume(self):
service = self.create_service('db', volumes=['/tmp:/host-tmp'])
container = service.create_container()
service.start_container(container)
self.assertIn('/host-tmp', container.inspect()['Volumes'])
def test_recreate_containers(self):
service = self.create_service(
'db',
@ -132,23 +115,22 @@ class ServiceTest(DockerClientTestCase):
num_containers_before = len(self.client.containers(all=True))
service.options['environment']['FOO'] = '2'
(intermediate, new) = service.recreate_containers()
self.assertEqual(len(intermediate), 1)
self.assertEqual(len(new), 1)
tuples = service.recreate_containers()
self.assertEqual(len(tuples), 1)
new_container = new[0]
intermediate_container = intermediate[0]
intermediate_container = tuples[0][0]
new_container = tuples[0][1]
self.assertEqual(intermediate_container.dictionary['Config']['Entrypoint'], ['echo'])
self.assertEqual(new_container.dictionary['Config']['Entrypoint'], ['ps'])
self.assertEqual(new_container.dictionary['Config']['Cmd'], ['ax'])
self.assertIn('FOO=2', new_container.dictionary['Config']['Env'])
self.assertEqual(new_container.name, 'figtest_db_1')
service.start_container(new_container)
self.assertEqual(new_container.inspect()['Volumes']['/var/db'], volume_path)
self.assertEqual(len(self.client.containers(all=True)), num_containers_before + 1)
self.assertEqual(len(self.client.containers(all=True)), num_containers_before)
self.assertNotEqual(old_container.id, new_container.id)
self.assertRaises(APIError, lambda: self.client.inspect_container(intermediate_container.id))
def test_start_container_passes_through_options(self):
db = self.create_service('db')
@ -271,5 +253,3 @@ class ServiceTest(DockerClientTestCase):
self.assertEqual(len(containers), 2)
for container in containers:
self.assertEqual(list(container.inspect()['HostConfig']['PortBindings'].keys()), ['8000/tcp'])

4
tests/testcases.py → tests/integration/testcases.py
View File

@ -3,7 +3,7 @@ from __future__ import absolute_import
from fig.packages.docker import Client
from fig.service import Service
from fig.cli.utils import docker_url
from . import unittest
from .. import unittest
class DockerClientTestCase(unittest.TestCase):
@ -18,7 +18,7 @@ class DockerClientTestCase(unittest.TestCase):
self.client.kill(c['Id'])
self.client.remove_container(c['Id'])
for i in self.client.images():
if isinstance(i['Tag'], basestring) and 'figtest' in i['Tag']:
if isinstance(i.get('Tag'), basestring) and 'figtest' in i['Tag']:
self.client.remove_image(i)
def create_service(self, name, **kwargs):

158
tests/project_test.py
View File

@ -1,158 +0,0 @@
from __future__ import unicode_literals
from fig.project import Project, ConfigurationError
from .testcases import DockerClientTestCase
class ProjectTest(DockerClientTestCase):
def test_from_dict(self):
project = Project.from_dicts('figtest', [
{
'name': 'web',
'image': 'ubuntu'
},
{
'name': 'db',
'image': 'ubuntu'
}
], self.client)
self.assertEqual(len(project.services), 2)
self.assertEqual(project.get_service('web').name, 'web')
self.assertEqual(project.get_service('web').options['image'], 'ubuntu')
self.assertEqual(project.get_service('db').name, 'db')
self.assertEqual(project.get_service('db').options['image'], 'ubuntu')
def test_from_dict_sorts_in_dependency_order(self):
project = Project.from_dicts('figtest', [
{
'name': 'web',
'image': 'ubuntu',
'links': ['db'],
},
{
'name': 'db',
'image': 'ubuntu'
}
], self.client)
self.assertEqual(project.services[0].name, 'db')
self.assertEqual(project.services[1].name, 'web')
def test_from_config(self):
project = Project.from_config('figtest', {
'web': {
'image': 'ubuntu',
},
'db': {
'image': 'ubuntu',
},
}, self.client)
self.assertEqual(len(project.services), 2)
self.assertEqual(project.get_service('web').name, 'web')
self.assertEqual(project.get_service('web').options['image'], 'ubuntu')
self.assertEqual(project.get_service('db').name, 'db')
self.assertEqual(project.get_service('db').options['image'], 'ubuntu')
def test_from_config_throws_error_when_not_dict(self):
with self.assertRaises(ConfigurationError):
project = Project.from_config('figtest', {
'web': 'ubuntu',
}, self.client)
def test_get_service(self):
web = self.create_service('web')
project = Project('test', [web], self.client)
self.assertEqual(project.get_service('web'), web)
def test_recreate_containers(self):
web = self.create_service('web')
db = self.create_service('db')
project = Project('test', [web, db], self.client)
old_web_container = web.create_container()
self.assertEqual(len(web.containers(stopped=True)), 1)
self.assertEqual(len(db.containers(stopped=True)), 0)
(old, new) = project.recreate_containers()
self.assertEqual(len(old), 1)
self.assertEqual(old[0][0], web)
self.assertEqual(len(new), 2)
self.assertEqual(new[0][0], web)
self.assertEqual(new[1][0], db)
self.assertEqual(len(web.containers(stopped=True)), 1)
self.assertEqual(len(db.containers(stopped=True)), 1)
# remove intermediate containers
for (service, container) in old:
container.remove()
def test_start_stop_kill_remove(self):
web = self.create_service('web')
db = self.create_service('db')
project = Project('figtest', [web, db], self.client)
project.start()
self.assertEqual(len(web.containers()), 0)
self.assertEqual(len(db.containers()), 0)
web_container_1 = web.create_container()
web_container_2 = web.create_container()
db_container = db.create_container()
project.start(service_names=['web'])
self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name]))
project.start()
self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name, db_container.name]))
project.stop(service_names=['web'], timeout=1)
self.assertEqual(set(c.name for c in project.containers()), set([db_container.name]))
project.kill(service_names=['db'])
self.assertEqual(len(project.containers()), 0)
self.assertEqual(len(project.containers(stopped=True)), 3)
project.remove_stopped(service_names=['web'])
self.assertEqual(len(project.containers(stopped=True)), 1)
project.remove_stopped()
self.assertEqual(len(project.containers(stopped=True)), 0)
def test_project_up(self):
web = self.create_service('web')
db = self.create_service('db')
project = Project('figtest', [web, db], self.client)
project.start()
self.assertEqual(len(project.containers()), 0)
project.up()
self.assertEqual(len(project.containers()), 2)
project.kill()
project.remove_stopped()
def test_unscale_after_restart(self):
web = self.create_service('web')
project = Project('figtest', [web], self.client)
project.start()
service = project.get_service('web')
service.scale(1)
self.assertEqual(len(service.containers()), 1)
service.scale(3)
self.assertEqual(len(service.containers()), 3)
project.up()
service = project.get_service('web')
self.assertEqual(len(service.containers()), 3)
service.scale(1)
self.assertEqual(len(service.containers()), 1)
project.up()
service = project.get_service('web')
self.assertEqual(len(service.containers()), 1)
# does scale=0 ,makes any sense? after recreating at least 1 container is running
service.scale(0)
project.up()
service = project.get_service('web')
self.assertEqual(len(service.containers()), 1)
project.kill()
project.remove_stopped()

0
tests/unit/__init__.py Normal file
View File

16
tests/unit/cli_test.py Normal file
View File

@ -0,0 +1,16 @@
from __future__ import unicode_literals
from __future__ import absolute_import
from .. import unittest
from fig.cli.main import TopLevelCommand
from fig.packages.six import StringIO
class CLITestCase(unittest.TestCase):
def test_yaml_filename_check(self):
command = TopLevelCommand()
command.base_dir = 'tests/fixtures/longer-filename-figfile'
self.assertTrue(command.project.get_service('definedinyamlnotyml'))
def test_help(self):
command = TopLevelCommand()
with self.assertRaises(SystemExit):
command.dispatch(['-h'], None)

10
tests/container_test.py → tests/unit/container_test.py
View File

@ -1,10 +1,10 @@
from __future__ import unicode_literals
from .testcases import DockerClientTestCase
from .. import unittest
from fig.container import Container
class ContainerTest(DockerClientTestCase):
class ContainerTest(unittest.TestCase):
def test_from_ps(self):
container = Container.from_ps(self.client, {
container = Container.from_ps(None, {
"Id":"abc",
"Image":"ubuntu:12.04",
"Command":"sleep 300",
@ -22,7 +22,7 @@ class ContainerTest(DockerClientTestCase):
})
def test_environment(self):
container = Container(self.client, {
container = Container(None, {
'ID': 'abc',
'Config': {
'Env': [
@ -37,7 +37,7 @@ class ContainerTest(DockerClientTestCase):
})
def test_number(self):
container = Container.from_ps(self.client, {
container = Container.from_ps(None, {
"Id":"abc",
"Image":"ubuntu:12.04",
"Command":"sleep 300",

69
tests/unit/project_test.py Normal file
View File

@ -0,0 +1,69 @@
from __future__ import unicode_literals
from .. import unittest
from fig.service import Service
from fig.project import Project, ConfigurationError
class ProjectTest(unittest.TestCase):
def test_from_dict(self):
project = Project.from_dicts('figtest', [
{
'name': 'web',
'image': 'ubuntu'
},
{
'name': 'db',
'image': 'ubuntu'
}
], None)
self.assertEqual(len(project.services), 2)
self.assertEqual(project.get_service('web').name, 'web')
self.assertEqual(project.get_service('web').options['image'], 'ubuntu')
self.assertEqual(project.get_service('db').name, 'db')
self.assertEqual(project.get_service('db').options['image'], 'ubuntu')
def test_from_dict_sorts_in_dependency_order(self):
project = Project.from_dicts('figtest', [
{
'name': 'web',
'image': 'ubuntu',
'links': ['db'],
},
{
'name': 'db',
'image': 'ubuntu'
}
], None)
self.assertEqual(project.services[0].name, 'db')
self.assertEqual(project.services[1].name, 'web')
def test_from_config(self):
project = Project.from_config('figtest', {
'web': {
'image': 'ubuntu',
},
'db': {
'image': 'ubuntu',
},
}, None)
self.assertEqual(len(project.services), 2)
self.assertEqual(project.get_service('web').name, 'web')
self.assertEqual(project.get_service('web').options['image'], 'ubuntu')
self.assertEqual(project.get_service('db').name, 'db')
self.assertEqual(project.get_service('db').options['image'], 'ubuntu')
def test_from_config_throws_error_when_not_dict(self):
with self.assertRaises(ConfigurationError):
project = Project.from_config('figtest', {
'web': 'ubuntu',
}, None)
def test_get_service(self):
web = Service(
project='figtest',
name='web',
client=None,
image="ubuntu",
)
project = Project('test', [web], None)
self.assertEqual(project.get_service('web'), web)

29
tests/unit/service_test.py Normal file
View File

@ -0,0 +1,29 @@
from __future__ import unicode_literals
from __future__ import absolute_import
from .. import unittest
from fig import Service
from fig.service import ConfigError
class ServiceTest(unittest.TestCase):
def test_name_validations(self):
self.assertRaises(ConfigError, lambda: Service(name=''))
self.assertRaises(ConfigError, lambda: Service(name=' '))
self.assertRaises(ConfigError, lambda: Service(name='/'))
self.assertRaises(ConfigError, lambda: Service(name='!'))
self.assertRaises(ConfigError, lambda: Service(name='\xe2'))
self.assertRaises(ConfigError, lambda: Service(name='_'))
self.assertRaises(ConfigError, lambda: Service(name='____'))
self.assertRaises(ConfigError, lambda: Service(name='foo_bar'))
self.assertRaises(ConfigError, lambda: Service(name='__foo_bar__'))
Service('a')
Service('foo')
def test_project_validation(self):
self.assertRaises(ConfigError, lambda: Service(name='foo', project='_'))
Service(name='foo', project='bar')
def test_config_validation(self):
self.assertRaises(ConfigError, lambda: Service(name='foo', port=['8000']))
Service(name='foo', ports=['8000'])

2
tests/sort_service_test.py → tests/unit/sort_service_test.py
View File

@ -1,5 +1,5 @@
from fig.project import sort_service_dicts, DependencyError
from . import unittest
from .. import unittest
class SortServiceTest(unittest.TestCase):

2
tests/split_buffer_test.py → tests/unit/split_buffer_test.py
View File

@ -1,7 +1,7 @@
from __future__ import unicode_literals
from __future__ import absolute_import
from fig.cli.utils import split_buffer
from . import unittest
from .. import unittest
class SplitBufferTest(unittest.TestCase):
def test_single_line_chunks(self):