From 1bf9005b326d642138f7f21f2b88dae968bfff17 Mon Sep 17 00:00:00 2001
From: Philipp Kessling <p.kessling@leibniz-hbi.de>
Date: Mon, 2 Dec 2024 16:17:08 +0100
Subject: [PATCH] Make the logger walk the record tree.

---
 dabapush/Reader/Reader.py          |  7 ++++---
 dabapush/Record.py                 |  7 +++----
 tests/Reader/test_NDJSONReader.py  | 19 ++++++++++++++-----
 tests/Writer/test_ndjson_writer.py |  2 +-
 4 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/dabapush/Reader/Reader.py b/dabapush/Reader/Reader.py
index 2547c24..c49c627 100644
--- a/dabapush/Reader/Reader.py
+++ b/dabapush/Reader/Reader.py
@@ -94,6 +94,7 @@ def records(self) -> Iterator[Record]:
     def log(self, record: Record):
         """Log the record to the persistent record log file."""
         with self.log_path.open("a", encoding="utf8") as f:
-            ujson.dump(record.to_log(), f)
-            f.write("\n")
-            log.debug(f"Done with {record.uuid}")
+            for sub_record in record.walk_tree(only_leafs=True):
+                ujson.dump(sub_record.to_log(), f)
+                f.write("\n")
+                log.debug(f"Done with {record.uuid}")
diff --git a/dabapush/Record.py b/dabapush/Record.py
index 06471da..04a1801 100644
--- a/dabapush/Record.py
+++ b/dabapush/Record.py
@@ -200,10 +200,9 @@ def __eq__(self, other: Self) -> bool:
                 "Cannot compare Record with non-Record type"
                 f" Comparison was Record == {type(other)}"
             )
-        if not self.payload or not other.payload:
-            return self.uuid == other.uuid
-
-        return self.payload == other.payload
+        # if self.payload or other.payload:
+        #    return self.payload == other.payload
+        return self.uuid == other.uuid
 
     def __dispatch_event__(self, event: EventType):
         """Dispatch an event to the event handlers."""
diff --git a/tests/Reader/test_NDJSONReader.py b/tests/Reader/test_NDJSONReader.py
index 5a3676c..124af67 100644
--- a/tests/Reader/test_NDJSONReader.py
+++ b/tests/Reader/test_NDJSONReader.py
@@ -35,7 +35,7 @@ def test_read(isolated_test_dir: Path, data):  # pylint: disable=W0621
 def test_read_with_backlog(isolated_test_dir: Path, data):  # pylint: disable=W0621
     """Should only read the new data."""
     reader = NDJSONReaderConfiguration(
-        "test", read_path=str(isolated_test_dir.resolve())
+        "test", read_path=str(isolated_test_dir.resolve()), pattern="*.ndjson"
     ).get_instance()
     file_path = isolated_test_dir / "test.ndjson"
     with file_path.open("wt") as file:
@@ -43,13 +43,22 @@ def test_read_with_backlog(isolated_test_dir: Path, data):  # pylint: disable=W0
             json.dump(line, file)
             file.write("\n")
 
-    records = list(reader.read())
-    assert len(records) == 20
+    def wrapper():
+        n = None
+        for n, record in enumerate(reader.read()):
+            record.done()
+        return n or 0
+
+    n = wrapper()
+
+    assert n + 1 == 20
 
     reader2 = NDJSONReaderConfiguration(
         "test", read_path=str(isolated_test_dir.resolve())
     ).get_instance()
 
-    records = list(reader2.read())
-    assert len(records) == 0
+    records2 = list(reader2.read())
+    log_path = isolated_test_dir / ".dabapush/test.jsonl"
+    assert log_path.exists()
     assert len(reader2.back_log) == 20
+    assert len(records2) == 0
diff --git a/tests/Writer/test_ndjson_writer.py b/tests/Writer/test_ndjson_writer.py
index eb673eb..6dd2d7e 100644
--- a/tests/Writer/test_ndjson_writer.py
+++ b/tests/Writer/test_ndjson_writer.py
@@ -3,7 +3,7 @@
 import pytest
 
 from dabapush.Record import Record
-from dabapush.Writer.NDJSONWriter import NDJSONWriterConfiguration
+from dabapush.Writer.ndjson_writer import NDJSONWriterConfiguration
 
 
 @pytest.mark.parametrize(