From 26c7dd37126cca09b149d3de7f134d5c8a766fdd Mon Sep 17 00:00:00 2001
From: Daniel Nephin <dnephin@docker.com>
Date: Wed, 4 Nov 2015 15:54:59 -0500
Subject: [PATCH] Handle non-utf8 unicode without raising an error.
Signed-off-by: Daniel Nephin <dnephin@docker.com>
---
compose/config/config.py | 2 +-
compose/utils.py | 2 +-
tests/unit/utils_test.py | 16 ++++++++++++++++
3 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/compose/config/config.py b/compose/config/config.py
index 5bc534fe9..ff3ae780c 100644
--- a/compose/config/config.py
+++ b/compose/config/config.py
@@ -457,7 +457,7 @@ def parse_environment(environment):
def split_env(env):
if isinstance(env, six.binary_type):
- env = env.decode('utf-8')
+ env = env.decode('utf-8', 'replace')
if '=' in env:
return env.split('=', 1)
else:
diff --git a/compose/utils.py b/compose/utils.py
index c8fddc5f1..08f6034fc 100644
--- a/compose/utils.py
+++ b/compose/utils.py
@@ -95,7 +95,7 @@ def stream_as_text(stream):
"""
for data in stream:
if not isinstance(data, six.text_type):
- data = data.decode('utf-8')
+ data = data.decode('utf-8', 'replace')
yield data
diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py
index b272c7349..e3d0bc00b 100644
--- a/tests/unit/utils_test.py
+++ b/tests/unit/utils_test.py
@@ -1,3 +1,6 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
from .. import unittest
from compose import utils
@@ -14,3 +17,16 @@ class JsonSplitterTestCase(unittest.TestCase):
utils.json_splitter(data),
({'foo': 'bar'}, '{"next": "obj"}')
)
+
+
+class StreamAsTextTestCase(unittest.TestCase):
+
+ def test_stream_with_non_utf_unicode_character(self):
+ stream = [b'\xed\xf3\xf3']
+ output, = utils.stream_as_text(stream)
+ assert output == '���'
+
+ def test_stream_with_utf_character(self):
+ stream = ['ěĝ'.encode('utf-8')]
+ output, = utils.stream_as_text(stream)
+ assert output == 'ěĝ'