Fix JSONDecodeError when deserializing streamed JSON documents

databroker/client.py

@@ -72,23 +72,23 @@ def documents(self, fill=False):
         link = self.item["links"]["self"].replace(
             "/metadata", "/documents", 1
         )
-        request = self.context.http_client.build_request(
-            "GET",
-            link,
-            params={"fill": fill},
-            headers={"Accept": "application/json-seq"},
-        )
-        response = self.context.http_client.send(request, stream=True)
-        try:
+        with self.context.http_client.stream(
+            "GET", link, params={"fill": fill},
+            headers={"Accept": "application/json-seq"}
+        ) as response:
             if response.is_error:
                 response.read()
                 handle_error(response)
+            tail = ""
             for chunk in response.iter_bytes():
                 for line in chunk.decode().splitlines():
-                    item = json.loads(line)
-                    yield (item["name"], _document_types[item["name"]](item["doc"]))
-        finally:
-            response.close()
+                    try:
+                        item = json.loads(tail + line)
+                    except json.JSONDecodeError:
+                        tail += line
+                    else:
+                        yield (item["name"], _document_types[item["name"]](item["doc"]))
+                        tail = ""
 
     def __getattr__(self, key):
         """

databroker/tests/test_broker.py

@@ -21,9 +21,11 @@
 import six
 import numpy as np
 import event_model
+import httpx
 
 from databroker._core import DOCT_NAMES
 from databroker.tests.utils import get_uids
+from tiled.client import from_uri
 
 from bluesky import __version__ as bluesky_version
 from bluesky.plans import count
@@ -1196,3 +1198,13 @@ def describe(self):
     signal = StringSignal(value="A", enum_strings=("A", "B"), name="signal")
     uid, = get_uids(RE(count([signal], 5)))
     data = db.v2[uid]["primary"]["data"]
+
+def test_large_document():
+    API_URL = "https://tiled-demo.blueskyproject.io/api/v1/"
+    try:
+        httpx.get(API_URL).raise_for_status()
+    except Exception:
+        raise pytest.skip(f"Could not connect to {API_URL}")
+    c = from_uri(API_URL)
+    run = c["csx"]["raw"]["ca658886-ee6b-4b3c-b47f-a58b08dbac8b"]
+    list(run.documents())