From 518c4b381b8c5c6c43be9137fc452421d41de89f Mon Sep 17 00:00:00 2001
From: Sascha Brawer <sascha@brawer.ch>
Date: Wed, 15 May 2024 15:35:21 +0200
Subject: [PATCH] Make logging more verbose

https://github.com/brawer/wikidata-qrank/issues/40
---
 cmd/qrank-builder/itemsignals.go      | 1 +
 cmd/qrank-builder/linemerger.go       | 6 ++++++
 cmd/qrank-builder/pagesignals.go      | 3 +++
 cmd/qrank-builder/pagesignals_test.go | 1 +
 4 files changed, 11 insertions(+)

diff --git a/cmd/qrank-builder/itemsignals.go b/cmd/qrank-builder/itemsignals.go
index a5b3f6a..0d45604 100644
--- a/cmd/qrank-builder/itemsignals.go
+++ b/cmd/qrank-builder/itemsignals.go
@@ -189,6 +189,7 @@ func buildItemSignals(ctx context.Context, pageviews []string, sites *map[string
 	config.NumWorkers = runtime.NumCPU()
 	sorter, outChan, errChan := extsort.New(sigChan, ItemSignalsFromBytes, ItemSignalsLess, config)
 	merger := NewLineMerger(scanners)
+	logger.Printf("BuildItemSignals(): merging signals from %d files; #0=PageSignalsScanner; rest=pageviews", len(scanners))
 	group, groupCtx := errgroup.WithContext(ctx)
 	group.Go(func() error {
 		joiner := itemSignalsJoiner{out: sigChan}
diff --git a/cmd/qrank-builder/linemerger.go b/cmd/qrank-builder/linemerger.go
index 05c964d..7efcfc0 100644
--- a/cmd/qrank-builder/linemerger.go
+++ b/cmd/qrank-builder/linemerger.go
@@ -32,6 +32,9 @@ func NewLineMerger(r []LineScanner) *LineMerger {
 			m.heap = append(m.heap, item)
 		}
 		if err := item.scanner.Err(); err != nil {
+			if logger != nil {
+				logger.Printf("LineMerger: scanner #%d failed to scan first line, err=%v", item.index, err)
+			}
 			m.err = err
 			return m
 		}
@@ -59,6 +62,9 @@ func (m *LineMerger) Advance() bool {
 	}
 	if err := item.scanner.Err(); err != nil {
 		m.err = err
+		if logger != nil {
+			logger.Printf("LineMerger: scanner #%d failed, err=%v", item.index, err)
+		}
 		return false
 	}
 	return len(m.heap) > 0
diff --git a/cmd/qrank-builder/pagesignals.go b/cmd/qrank-builder/pagesignals.go
index a9a953f..493c277 100644
--- a/cmd/qrank-builder/pagesignals.go
+++ b/cmd/qrank-builder/pagesignals.go
@@ -372,14 +372,17 @@ func (s *pageSignalsScanner) Scan() bool {
 			}
 			s.err = s.scanner.Err()
 			if s.err != nil {
+				logger.Printf("PageSignalsScanner.Scan(): failed, domain=%s, err=%v", s.domains[s.curDomain], s.err)
 				break
 			}
 		}
 		s.curDomain += 1
 		if s.curDomain == len(s.domains) {
+			logger.Println("PageSignalsScanner.Scan(): finished last domain")
 			break
 		}
 
+		logger.Printf("PageSignalsScanner.Scan(): opening %s", s.paths[s.curDomain])
 		s.reader, s.err = NewS3Reader(context.Background(), "qrank", s.paths[s.curDomain], s.storage)
 		if s.err != nil {
 			break
diff --git a/cmd/qrank-builder/pagesignals_test.go b/cmd/qrank-builder/pagesignals_test.go
index 82c4ec2..acc47a9 100644
--- a/cmd/qrank-builder/pagesignals_test.go
+++ b/cmd/qrank-builder/pagesignals_test.go
@@ -118,6 +118,7 @@ func TestStoredPageSignals(t *testing.T) {
 }
 
 func TestPageSignalsScanner(t *testing.T) {
+	logger = log.New(&bytes.Buffer{}, "", log.Lshortfile)
 	s3 := NewFakeS3()
 	storeFakePageSignals("enwiki-20111231", "1,Q111|7,Q777", s3, t)
 	storeFakePageSignals("rmwiki-20110203", "1,Q11|2,Q22|3,Q33", s3, t)