tyler.durden
/
compose
mirror of https://github.com/docker/compose.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #255 from orchardup/fix-unicode 

Fix encoding errors
This commit is contained in:
Ben Firshman 2014-06-18 16:00:43 +01:00
parent feb8ad7b4c 2bd6e3d0a5
commit 5b10c4811f
8 changed files with 180 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,4 +3,5 @@
/build
/dist
/docs/_site
/venv
fig.spec

9
fig/cli/log_printer.py
View File

@ -10,16 +10,17 @@ from .utils import split_buffer
class LogPrinter(object):
def __init__(self, containers, attach_params=None):
def __init__(self, containers, attach_params=None, output=sys.stdout):
self.containers = containers
self.attach_params = attach_params or {}
self.prefix_width = self._calculate_prefix_width(containers)
self.generators = self._make_log_generators()
self.output = output
def run(self):
mux = Multiplexer(self.generators)
for line in mux.loop():
sys.stdout.write(line.encode(sys.__stdout__.encoding or 'utf-8'))
self.output.write(line)
def _calculate_prefix_width(self, containers):
"""
@ -45,12 +46,12 @@ class LogPrinter(object):
return generators
def _make_log_generator(self, container, color_fn):
prefix = color_fn(self._generate_prefix(container))
prefix = color_fn(self._generate_prefix(container)).encode('utf-8')
# Attach to container before log printer starts running
line_generator = split_buffer(self._attach(container), '\n')
for line in line_generator:
yield prefix + line.decode('utf-8')
yield prefix + line
exit_code = container.wait()
yield color_fn("%s exited with code %s\n" % (container.name, exit_code))

2
fig/cli/socketclient.py
View File

@ -81,7 +81,7 @@ class SocketClient:
chunk = socket.recv(4096)
if chunk:
stream.write(chunk.encode(stream.encoding or 'utf-8'))
stream.write(chunk)
stream.flush()
else:
break

83
fig/progress_stream.py Normal file
View File

@ -0,0 +1,83 @@
import json
import os
import codecs
class StreamOutputError(Exception):
pass
def stream_output(output, stream):
is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno())
stream = codecs.getwriter('utf-8')(stream)
all_events = []
lines = {}
diff = 0
for chunk in output:
event = json.loads(chunk)
all_events.append(event)
if 'progress' in event or 'progressDetail' in event:
image_id = event['id']
if image_id in lines:
diff = len(lines) - lines[image_id]
else:
lines[image_id] = len(lines)
stream.write("\n")
diff = 0
if is_terminal:
# move cursor up `diff` rows
stream.write("%c[%dA" % (27, diff))
print_output_event(event, stream, is_terminal)
if 'id' in event and is_terminal:
# move cursor back down
stream.write("%c[%dB" % (27, diff))
stream.flush()
return all_events
def print_output_event(event, stream, is_terminal):
if 'errorDetail' in event:
raise StreamOutputError(event['errorDetail']['message'])
terminator = ''
if is_terminal and 'stream' not in event:
# erase current line
stream.write("%c[2K\r" % 27)
terminator = "\r"
pass
elif 'progressDetail' in event:
return
if 'time' in event:
stream.write("[%s] " % event['time'])
if 'id' in event:
stream.write("%s: " % event['id'])
if 'from' in event:
stream.write("(from %s) " % event['from'])
status = event.get('status', '')
if 'progress' in event:
stream.write("%s %s%s" % (status, event['progress'], terminator))
elif 'progressDetail' in event:
detail = event['progressDetail']
if 'current' in detail:
percentage = float(detail['current']) / float(detail['total']) * 100
stream.write('%s (%.1f%%)%s' % (status, percentage, terminator))
else:
stream.write('%s%s' % (status, terminator))
elif 'stream' in event:
stream.write("%s%s" % (event['stream'], terminator))
else:
stream.write("%s%s\n" % (status, terminator))

80
fig/service.py
View File

@ -5,8 +5,8 @@ import logging
import re
import os
import sys
import json
from .container import Container
from .progress_stream import stream_output, StreamOutputError
log = logging.getLogger(__name__)
@ -343,84 +343,6 @@ class Service(object):
return True
class StreamOutputError(Exception):
pass
def stream_output(output, stream):
is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno())
all_events = []
lines = {}
diff = 0
for chunk in output:
event = json.loads(chunk)
all_events.append(event)
if 'progress' in event or 'progressDetail' in event:
image_id = event['id']
if image_id in lines:
diff = len(lines) - lines[image_id]
else:
lines[image_id] = len(lines)
stream.write("\n")
diff = 0
if is_terminal:
# move cursor up `diff` rows
stream.write("%c[%dA" % (27, diff))
print_output_event(event, stream, is_terminal)
if 'id' in event and is_terminal:
# move cursor back down
stream.write("%c[%dB" % (27, diff))
stream.flush()
return all_events
def print_output_event(event, stream, is_terminal):
if 'errorDetail' in event:
raise StreamOutputError(event['errorDetail']['message'])
terminator = ''
if is_terminal and 'stream' not in event:
# erase current line
stream.write("%c[2K\r" % 27)
terminator = "\r"
pass
elif 'progressDetail' in event:
return
if 'time' in event:
stream.write("[%s] " % event['time'])
if 'id' in event:
stream.write("%s: " % event['id'])
if 'from' in event:
stream.write("(from %s) " % event['from'])
status = event.get('status', '')
if 'progress' in event:
stream.write("%s %s%s" % (status, event['progress'], terminator))
elif 'progressDetail' in event:
detail = event['progressDetail']
if 'current' in detail:
percentage = float(detail['current']) / float(detail['total']) * 100
stream.write('%s (%.1f%%)%s' % (status, percentage, terminator))
else:
stream.write('%s%s' % (status, terminator))
elif 'stream' in event:
stream.write("%s%s" % (event['stream'], terminator))
else:
stream.write("%s%s\n" % (status, terminator))
NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')

2
script/test
View File

@ -1,2 +1,2 @@
#!/bin/sh
nosetests
PYTHONIOENCODING=ascii nosetests $@

57
tests/unit/log_printer_test.py Normal file
View File

@ -0,0 +1,57 @@
from __future__ import unicode_literals
from __future__ import absolute_import
import os
from fig.cli.log_printer import LogPrinter
from .. import unittest
class LogPrinterTest(unittest.TestCase):
def test_single_container(self):
def reader(*args, **kwargs):
yield "hello\nworld"
container = MockContainer(reader)
output = run_log_printer([container])
self.assertIn('hello', output)
self.assertIn('world', output)
def test_unicode(self):
glyph = u'\u2022'.encode('utf-8')
def reader(*args, **kwargs):
yield glyph + b'\n'
container = MockContainer(reader)
output = run_log_printer([container])
self.assertIn(glyph, output)
def run_log_printer(containers):
r, w = os.pipe()
reader, writer = os.fdopen(r, 'r'), os.fdopen(w, 'w')
printer = LogPrinter(containers, output=writer)
printer.run()
writer.close()
return reader.read()
class MockContainer(object):
def __init__(self, reader):
self._reader = reader
@property
def name(self):
return 'myapp_web_1'
@property
def name_without_project(self):
return 'web_1'
def attach(self, *args, **kwargs):
return self._reader()
def wait(self, *args, **kwargs):
return 0

47
tests/unit/split_buffer_test.py
View File

@ -6,32 +6,47 @@ from .. import unittest
class SplitBufferTest(unittest.TestCase):
def test_single_line_chunks(self):
def reader():
yield "abc\n"
yield "def\n"
yield "ghi\n"
yield b'abc\n'
yield b'def\n'
yield b'ghi\n'
self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi\n"])
self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi\n'])
def test_no_end_separator(self):
def reader():
yield "abc\n"
yield "def\n"
yield "ghi"
yield b'abc\n'
yield b'def\n'
yield b'ghi'
self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi"])
self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi'])
def test_multiple_line_chunk(self):
def reader():
yield "abc\ndef\nghi"
yield b'abc\ndef\nghi'
self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi"])
self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi'])
def test_chunked_line(self):
def reader():
yield "a"
yield "b"
yield "c"
yield "\n"
yield "d"
yield b'a'
yield b'b'
yield b'c'
yield b'\n'
yield b'd'
self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "d"])
self.assert_produces(reader, [b'abc\n', b'd'])
def test_preserves_unicode_sequences_within_lines(self):
string = u"a\u2022c\n".encode('utf-8')
def reader():
yield string
self.assert_produces(reader, [string])
def assert_produces(self, reader, expectations):
split = split_buffer(reader(), b'\n')
for (actual, expected) in zip(split, expectations):
self.assertEqual(type(actual), type(expected))
self.assertEqual(actual, expected)