Merge pull request #63 from Leibniz-HBI/62-fix-backlog-never-walked-t…

…he-record-tree

Make the logger walk the record tree.

dabapush/Reader/Reader.py

@@ -94,6 +94,7 @@ def records(self) -> Iterator[Record]:
     def log(self, record: Record):
         """Log the record to the persistent record log file."""
         with self.log_path.open("a", encoding="utf8") as f:
-            ujson.dump(record.to_log(), f)
-            f.write("\n")
-            log.debug(f"Done with {record.uuid}")
+            for sub_record in record.walk_tree(only_leafs=True):
+                ujson.dump(sub_record.to_log(), f)
+                f.write("\n")
+                log.debug(f"Done with {record.uuid}")

dabapush/Record.py

@@ -200,10 +200,9 @@ def __eq__(self, other: Self) -> bool:
                 "Cannot compare Record with non-Record type"
                 f" Comparison was Record == {type(other)}"
             )
-        if not self.payload or not other.payload:
-            return self.uuid == other.uuid
-
-        return self.payload == other.payload
+        # if self.payload or other.payload:
+        #    return self.payload == other.payload
+        return self.uuid == other.uuid
 
     def __dispatch_event__(self, event: EventType):
         """Dispatch an event to the event handlers."""

tests/Reader/test_NDJSONReader.py

@@ -35,21 +35,30 @@ def test_read(isolated_test_dir: Path, data):  # pylint: disable=W0621
 def test_read_with_backlog(isolated_test_dir: Path, data):  # pylint: disable=W0621
     """Should only read the new data."""
     reader = NDJSONReaderConfiguration(
-        "test", read_path=str(isolated_test_dir.resolve())
+        "test", read_path=str(isolated_test_dir.resolve()), pattern="*.ndjson"
     ).get_instance()
     file_path = isolated_test_dir / "test.ndjson"
     with file_path.open("wt") as file:
         for line in data:
             json.dump(line, file)
             file.write("\n")
 
-    records = list(reader.read())
-    assert len(records) == 20
+    def wrapper():
+        n = None
+        for n, record in enumerate(reader.read()):
+            record.done()
+        return n or 0
+
+    n = wrapper()
+
+    assert n + 1 == 20
 
     reader2 = NDJSONReaderConfiguration(
         "test", read_path=str(isolated_test_dir.resolve())
     ).get_instance()
 
-    records = list(reader2.read())
-    assert len(records) == 0
+    records2 = list(reader2.read())
+    log_path = isolated_test_dir / ".dabapush/test.jsonl"
+    assert log_path.exists()
     assert len(reader2.back_log) == 20
+    assert len(records2) == 0

tests/Writer/test_ndjson_writer.py

@@ -3,7 +3,7 @@
 import pytest
 
 from dabapush.Record import Record
-from dabapush.Writer.NDJSONWriter import NDJSONWriterConfiguration
+from dabapush.Writer.ndjson_writer import NDJSONWriterConfiguration
 
 
 @pytest.mark.parametrize(