Merge pull request #3789 from shin-/3788-json-splitter-fix

json_splitter: Don't break when buffer contains leading whitespace.

compose/cli/main.py

@@ -23,6 +23,7 @@ from ..config.environment import Environment
 from ..config.serialize import serialize_config
 from ..const import DEFAULT_TIMEOUT
 from ..const import IS_WINDOWS_PLATFORM
+from ..errors import StreamParseError
 from ..progress_stream import StreamOutputError
 from ..project import NoSuchService
 from ..project import OneOffFilter
@@ -75,7 +76,7 @@ def main():
     except NeedsBuildError as e:
         log.error("Service '%s' needs to be built, but --no-build was passed." % e.service.name)
         sys.exit(1)
-    except errors.ConnectionError:
+    except (errors.ConnectionError, StreamParseError):
         sys.exit(1)
 
 

compose/errors.py

@@ -5,3 +5,8 @@ from __future__ import unicode_literals
 class OperationFailedError(Exception):
     def __init__(self, reason):
         self.msg = reason
+
+
+class StreamParseError(RuntimeError):
+    def __init__(self, reason):
+        self.msg = reason

compose/utils.py

@@ -5,11 +5,15 @@ import codecs
 import hashlib
 import json
 import json.decoder
+import logging
 
 import six
 
+from .errors import StreamParseError
+
 
 json_decoder = json.JSONDecoder()
+log = logging.getLogger(__name__)
 
 
 def get_output_stream(stream):
@@ -60,13 +64,21 @@ def split_buffer(stream, splitter=None, decoder=lambda a: a):
             yield item
 
     if buffered:
+        try:
             yield decoder(buffered)
+        except Exception as e:
+            log.error(
+                'Compose tried decoding the following data chunk, but failed:'
+                '\n%s' % repr(buffered)
+            )
+            raise StreamParseError(e)
 
 
 def json_splitter(buffer):
     """Attempt to parse a json object from a buffer. If there is at least one
     object, return it and the rest of the buffer, otherwise return None.
     """
+    buffer = buffer.strip()
     try:
         obj, index = json_decoder.raw_decode(buffer)
         rest = buffer[json.decoder.WHITESPACE.match(buffer, index).end():]

tests/unit/utils_test.py

@@ -15,6 +15,10 @@ class TestJsonSplitter(object):
         data = '{"foo": "bar"}\n  \n{"next": "obj"}'
         assert utils.json_splitter(data) == ({'foo': 'bar'}, '{"next": "obj"}')
 
+    def test_json_splitter_leading_whitespace(self):
+        data = '\n   \r{"foo": "bar"}\n\n   {"next": "obj"}'
+        assert utils.json_splitter(data) == ({'foo': 'bar'}, '{"next": "obj"}')
+
 
 class TestStreamAsText(object):
 
@@ -43,3 +47,16 @@ class TestJsonStream(object):
             [1, 2, 3],
             [],
         ]
+
+    def test_with_leading_whitespace(self):
+        stream = [
+            '\n  \r\n  {"one": "two"}{"x": 1}',
+            '  {"three": "four"}\t\t{"x": 2}'
+        ]
+        output = list(utils.json_stream(stream))
+        assert output == [
+            {'one': 'two'},
+            {'x': 1},
+            {'three': 'four'},
+            {'x': 2}
+        ]