From 2b245bdf9e254746061a4ce9499bf517e488c8f5 Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Mon, 10 Mar 2014 13:57:13 +0000 Subject: [PATCH] Update to docker-py 0.3.1 From https://github.com/dotcloud/docker-py/commit/7f55a101f813f3e96413d1b577e98d9467b0bffc This now requires Docker 0.9 or greater. --- .travis.yml | 8 +- fig/packages/docker/__init__.py | 3 + fig/packages/docker/auth/auth.py | 13 +- fig/packages/docker/client.py | 154 +++++++++++++---------- fig/packages/docker/unixconn/unixconn.py | 2 +- fig/packages/docker/utils/__init__.py | 2 +- fig/packages/docker/utils/utils.py | 36 +++++- tests/testcases.py | 2 +- 8 files changed, 139 insertions(+), 81 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d6cc863b..ea561955b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,12 +3,8 @@ python: - '2.6' - '2.7' env: -- DOCKER_VERSION=0.8.0 -- DOCKER_VERSION=0.8.1 -matrix: - allow_failures: - - python: '3.2' - - python: '3.3' +- DOCKER_VERSION=0.9.1 +- DOCKER_VERSION=0.10.0 install: script/travis-install script: - pwd diff --git a/fig/packages/docker/__init__.py b/fig/packages/docker/__init__.py index 5f642a855..5388e7286 100644 --- a/fig/packages/docker/__init__.py +++ b/fig/packages/docker/__init__.py @@ -12,4 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +__title__ = 'docker-py' +__version__ = '0.3.0' + from .client import Client, APIError # flake8: noqa diff --git a/fig/packages/docker/auth/auth.py b/fig/packages/docker/auth/auth.py index 8037dcbb6..69cfa89d3 100644 --- a/fig/packages/docker/auth/auth.py +++ b/fig/packages/docker/auth/auth.py @@ -48,7 +48,7 @@ def resolve_repository_name(repo_name): raise ValueError('Repository name cannot contain a ' 'scheme ({0})'.format(repo_name)) parts = repo_name.split('/', 1) - if not '.' in parts[0] and not ':' in parts[0] and parts[0] != 'localhost': + if '.' not in parts[0] and ':' not in parts[0] and parts[0] != 'localhost': # This is a docker index repo (ex: foo/bar or ubuntu) return INDEX_URL, repo_name if len(parts) < 2: @@ -87,6 +87,11 @@ def resolve_authconfig(authconfig, registry=None): return authconfig.get(swap_protocol(registry), None) +def encode_auth(auth_info): + return base64.b64encode(auth_info.get('username', '') + b':' + + auth_info.get('password', '')) + + def decode_auth(auth): if isinstance(auth, six.string_types): auth = auth.encode('ascii') @@ -100,6 +105,12 @@ def encode_header(auth): return base64.b64encode(auth_json) +def encode_full_header(auth): + """ Returns the given auth block encoded for the X-Registry-Config header. + """ + return encode_header({'configs': auth}) + + def load_config(root=None): """Loads authentication data from a Docker configuration file in the given root directory.""" diff --git a/fig/packages/docker/client.py b/fig/packages/docker/client.py index 948a3a67d..77bb962f8 100644 --- a/fig/packages/docker/client.py +++ b/fig/packages/docker/client.py @@ -28,13 +28,17 @@ from .utils import utils if not six.PY3: import websocket +DEFAULT_DOCKER_API_VERSION = '1.9' DEFAULT_TIMEOUT_SECONDS = 60 STREAM_HEADER_SIZE_BYTES = 8 class APIError(requests.exceptions.HTTPError): def __init__(self, message, response, explanation=None): - super(APIError, self).__init__(message, response=response) + # requests 1.2 supports response as a keyword argument, but + # requests 1.1 doesn't + super(APIError, self).__init__(message) + self.response = response self.explanation = explanation @@ -65,7 +69,7 @@ class APIError(requests.exceptions.HTTPError): class Client(requests.Session): - def __init__(self, base_url=None, version="1.6", + def __init__(self, base_url=None, version=DEFAULT_DOCKER_API_VERSION, timeout=DEFAULT_TIMEOUT_SECONDS): super(Client, self).__init__() if base_url is None: @@ -125,7 +129,7 @@ class Client(requests.Session): mem_limit=0, ports=None, environment=None, dns=None, volumes=None, volumes_from=None, network_disabled=False, entrypoint=None, - cpu_shares=None, working_dir=None): + cpu_shares=None, working_dir=None, domainname=None): if isinstance(command, six.string_types): command = shlex.split(str(command)) if isinstance(environment, dict): @@ -133,7 +137,7 @@ class Client(requests.Session): '{0}={1}'.format(k, v) for k, v in environment.items() ] - if ports and isinstance(ports, list): + if isinstance(ports, list): exposed_ports = {} for port_definition in ports: port = port_definition @@ -145,12 +149,15 @@ class Client(requests.Session): exposed_ports['{0}/{1}'.format(port, proto)] = {} ports = exposed_ports - if volumes and isinstance(volumes, list): + if isinstance(volumes, list): volumes_dict = {} for vol in volumes: volumes_dict[vol] = {} volumes = volumes_dict + if volumes_from and not isinstance(volumes_from, six.string_types): + volumes_from = ','.join(volumes_from) + attach_stdin = False attach_stdout = False attach_stderr = False @@ -165,26 +172,27 @@ class Client(requests.Session): stdin_once = True return { - 'Hostname': hostname, + 'Hostname': hostname, + 'Domainname': domainname, 'ExposedPorts': ports, - 'User': user, - 'Tty': tty, - 'OpenStdin': stdin_open, - 'StdinOnce': stdin_once, - 'Memory': mem_limit, - 'AttachStdin': attach_stdin, + 'User': user, + 'Tty': tty, + 'OpenStdin': stdin_open, + 'StdinOnce': stdin_once, + 'Memory': mem_limit, + 'AttachStdin': attach_stdin, 'AttachStdout': attach_stdout, 'AttachStderr': attach_stderr, - 'Env': environment, - 'Cmd': command, - 'Dns': dns, - 'Image': image, - 'Volumes': volumes, - 'VolumesFrom': volumes_from, + 'Env': environment, + 'Cmd': command, + 'Dns': dns, + 'Image': image, + 'Volumes': volumes, + 'VolumesFrom': volumes_from, 'NetworkDisabled': network_disabled, - 'Entrypoint': entrypoint, - 'CpuShares': cpu_shares, - 'WorkingDir': working_dir + 'Entrypoint': entrypoint, + 'CpuShares': cpu_shares, + 'WorkingDir': working_dir } def _post_json(self, url, data, **kwargs): @@ -222,31 +230,18 @@ class Client(requests.Session): def _create_websocket_connection(self, url): return websocket.create_connection(url) - def _stream_result(self, response): - """Generator for straight-out, non chunked-encoded HTTP responses.""" + def _get_raw_response_socket(self, response): self._raise_for_status(response) - for line in response.iter_lines(chunk_size=1, decode_unicode=True): - # filter out keep-alive new lines - if line: - yield line + '\n' - - def _stream_result_socket(self, response): - self._raise_for_status(response) - return response.raw._fp.fp._sock + if six.PY3: + return response.raw._fp.fp.raw._sock + else: + return response.raw._fp.fp._sock def _stream_helper(self, response): """Generator for data coming from a chunked-encoded HTTP response.""" - socket_fp = self._stream_result_socket(response) - socket_fp.setblocking(1) - socket = socket_fp.makefile() - while True: - size = int(socket.readline(), 16) - if size <= 0: - break - data = socket.readline() - if not data: - break - yield data + for line in response.iter_lines(chunk_size=32): + if line: + yield line def _multiplexed_buffer_helper(self, response): """A generator of multiplexed data blocks read from a buffered @@ -265,17 +260,20 @@ class Client(requests.Session): def _multiplexed_socket_stream_helper(self, response): """A generator of multiplexed data blocks coming from a response socket.""" - socket = self._stream_result_socket(response) + socket = self._get_raw_response_socket(response) def recvall(socket, size): - data = '' + blocks = [] while size > 0: block = socket.recv(size) if not block: return None - data += block + blocks.append(block) size -= len(block) + + sep = bytes() if six.PY3 else str() + data = sep.join(blocks) return data while True: @@ -304,9 +302,18 @@ class Client(requests.Session): u = self._url("/containers/{0}/attach".format(container)) response = self._post(u, params=params, stream=stream) - # Stream multi-plexing was introduced in API v1.6. + # Stream multi-plexing was only introduced in API v1.6. Anything before + # that needs old-style streaming. if utils.compare_version('1.6', self._version) < 0: - return stream and self._stream_result(response) or \ + def stream_result(): + self._raise_for_status(response) + for line in response.iter_lines(chunk_size=1, + decode_unicode=True): + # filter out keep-alive new lines + if line: + yield line + + return stream_result() if stream else \ self._result(response, binary=True) return stream and self._multiplexed_socket_stream_helper(response) or \ @@ -319,13 +326,15 @@ class Client(requests.Session): 'stderr': 1, 'stream': 1 } + if ws: return self._attach_websocket(container, params) if isinstance(container, dict): container = container.get('Id') + u = self._url("/containers/{0}/attach".format(container)) - return self._stream_result_socket(self.post( + return self._get_raw_response_socket(self.post( u, None, params=self._attach_params(params), stream=True)) def build(self, path=None, tag=None, quiet=False, fileobj=None, @@ -341,6 +350,9 @@ class Client(requests.Session): else: context = utils.tar(path) + if utils.compare_version('1.8', self._version) >= 0: + stream = True + u = self._url('/build') params = { 't': tag, @@ -352,6 +364,19 @@ class Client(requests.Session): if context is not None: headers = {'Content-Type': 'application/tar'} + if utils.compare_version('1.9', self._version) >= 0: + # If we don't have any auth data so far, try reloading the config + # file one more time in case anything showed up in there. + if not self._auth_configs: + self._auth_configs = auth.load_config() + + # Send the full auth configuration (if any exists), since the build + # could use any (or all) of the registries. + if self._auth_configs: + headers['X-Registry-Config'] = auth.encode_full_header( + self._auth_configs + ) + response = self._post( u, data=context, @@ -363,8 +388,9 @@ class Client(requests.Session): if context is not None: context.close() + if stream: - return self._stream_result(response) + return self._stream_helper(response) else: output = self._result(response) srch = r'Successfully built ([0-9a-f]+)' @@ -403,6 +429,8 @@ class Client(requests.Session): return res def copy(self, container, resource): + if isinstance(container, dict): + container = container.get('Id') res = self._post_json( self._url("/containers/{0}/copy".format(container)), data={"Resource": resource}, @@ -416,12 +444,12 @@ class Client(requests.Session): mem_limit=0, ports=None, environment=None, dns=None, volumes=None, volumes_from=None, network_disabled=False, name=None, entrypoint=None, - cpu_shares=None, working_dir=None): + cpu_shares=None, working_dir=None, domainname=None): config = self._container_config( image, command, hostname, user, detach, stdin_open, tty, mem_limit, ports, environment, dns, volumes, volumes_from, network_disabled, - entrypoint, cpu_shares, working_dir + entrypoint, cpu_shares, working_dir, domainname ) return self.create_container_from_config(config, name) @@ -440,21 +468,7 @@ class Client(requests.Session): format(container))), True) def events(self): - u = self._url("/events") - - socket = self._stream_result_socket(self.get(u, stream=True)) - - while True: - chunk = socket.recv(4096) - if chunk: - # Messages come in the format of length, data, newline. - length, data = chunk.split("\n", 1) - length = int(length, 16) - if length > len(data): - data += socket.recv(length - len(data)) - yield json.loads(data) - else: - break + return self._stream_helper(self.get(self._url('/events'), stream=True)) def export(self, container): if isinstance(container, dict): @@ -471,6 +485,8 @@ class Client(requests.Session): def images(self, name=None, quiet=False, all=False, viz=False): if viz: + if utils.compare_version('1.7', self._version) >= 0: + raise Exception('Viz output is not supported in API >= 1.7!') return self._result(self._get(self._url("images/viz"))) params = { 'filter': name, @@ -618,7 +634,7 @@ class Client(requests.Session): self._auth_configs = auth.load_config() authcfg = auth.resolve_authconfig(self._auth_configs, registry) - # Do not fail here if no atuhentication exists for this specific + # Do not fail here if no authentication exists for this specific # registry as we can have a readonly pull. Just put the header if # we can. if authcfg: @@ -644,7 +660,7 @@ class Client(requests.Session): self._auth_configs = auth.load_config() authcfg = auth.resolve_authconfig(self._auth_configs, registry) - # Do not fail here if no atuhentication exists for this specific + # Do not fail here if no authentication exists for this specific # registry as we can have a readonly pull. Just put the header if # we can. if authcfg: @@ -652,7 +668,7 @@ class Client(requests.Session): response = self._post_json(u, None, headers=headers, stream=stream) else: - response = self._post_json(u, authcfg, stream=stream) + response = self._post_json(u, None, stream=stream) return stream and self._stream_helper(response) \ or self._result(response) diff --git a/fig/packages/docker/unixconn/unixconn.py b/fig/packages/docker/unixconn/unixconn.py index b5c65931b..176659e7c 100644 --- a/fig/packages/docker/unixconn/unixconn.py +++ b/fig/packages/docker/unixconn/unixconn.py @@ -40,7 +40,7 @@ class UnixHTTPConnection(httplib.HTTPConnection, object): self.sock = sock def _extract_path(self, url): - #remove the base_url entirely.. + # remove the base_url entirely.. return url.replace(self.base_url, "") def request(self, method, url, **kwargs): diff --git a/fig/packages/docker/utils/__init__.py b/fig/packages/docker/utils/__init__.py index 386a01af7..8a85975d7 100644 --- a/fig/packages/docker/utils/__init__.py +++ b/fig/packages/docker/utils/__init__.py @@ -1,3 +1,3 @@ from .utils import ( - compare_version, convert_port_bindings, mkbuildcontext, ping, tar + compare_version, convert_port_bindings, mkbuildcontext, ping, tar, parse_repository_tag ) # flake8: noqa diff --git a/fig/packages/docker/utils/utils.py b/fig/packages/docker/utils/utils.py index 1cb04f0cb..0e4c1c1fc 100644 --- a/fig/packages/docker/utils/utils.py +++ b/fig/packages/docker/utils/utils.py @@ -15,6 +15,7 @@ import io import tarfile import tempfile +from distutils.version import StrictVersion import requests from fig.packages import six @@ -51,15 +52,34 @@ def tar(path): def compare_version(v1, v2): - return float(v2) - float(v1) + """Compare docker versions + + >>> v1 = '1.9' + >>> v2 = '1.10' + >>> compare_version(v1, v2) + 1 + >>> compare_version(v2, v1) + -1 + >>> compare_version(v2, v2) + 0 + """ + s1 = StrictVersion(v1) + s2 = StrictVersion(v2) + if s1 == s2: + return 0 + elif s1 > s2: + return -1 + else: + return 1 def ping(url): try: res = requests.get(url) - return res.status >= 400 except Exception: return False + else: + return res.status_code < 400 def _convert_port_binding(binding): @@ -94,3 +114,15 @@ def convert_port_bindings(port_bindings): else: result[key] = [_convert_port_binding(v)] return result + + +def parse_repository_tag(repo): + column_index = repo.rfind(':') + if column_index < 0: + return repo, "" + tag = repo[column_index+1:] + slash_index = tag.find('/') + if slash_index < 0: + return repo[:column_index], tag + + return repo, "" diff --git a/tests/testcases.py b/tests/testcases.py index 6556311de..ac395040d 100644 --- a/tests/testcases.py +++ b/tests/testcases.py @@ -18,7 +18,7 @@ class DockerClientTestCase(unittest.TestCase): self.client.kill(c['Id']) self.client.remove_container(c['Id']) for i in self.client.images(): - if isinstance(i['Tag'], basestring) and 'figtest' in i['Tag']: + if isinstance(i.get('Tag'), basestring) and 'figtest' in i['Tag']: self.client.remove_image(i) def create_service(self, name, **kwargs):