Retry Dimensions pubs query

rialto_airflow/harvest/dimensions.py

@@ -22,6 +22,7 @@ def dois_from_orcid(orcid):
         """.format(orcid)
 
     # The Dimensions API can flake out sometimes, so try to catch & retry.
+    # TODO: Consider using retry param in query() instead
     try_count = 0
     while try_count < 20:
         try_count += 1
@@ -82,7 +83,7 @@ def publications_from_dois(dois: list, batch_size=200):
             limit 1000
             """
 
-        result = dsl().query(q)
+        result = dsl().query(q, retry=5)
 
         for pub in result["publications"]:
             yield normalize_publication(pub)

test/harvest/test_merge_pubs.py

@@ -138,7 +138,7 @@ def sul_pubs_csv(tmp_path):
 
 def test_dimensions_pubs_df(dimensions_pubs_csv):
     lazy_df = merge_pubs.dimensions_pubs_df(dimensions_pubs_csv)
-    assert type(lazy_df) == pl.lazyframe.frame.LazyFrame
+    assert isinstance(lazy_df, pl.lazyframe.frame.LazyFrame)
     df = lazy_df.collect()
     assert df.shape[0] == 2
     assert "bogus" not in df.columns, "Unneeded columns have been dropped"
@@ -147,7 +147,7 @@ def test_dimensions_pubs_df(dimensions_pubs_csv):
 
 def test_openalex_pubs_df(openalex_pubs_csv):
     lazy_df = merge_pubs.openalex_pubs_df(openalex_pubs_csv)
-    assert type(lazy_df) == pl.lazyframe.frame.LazyFrame
+    assert isinstance(lazy_df, pl.lazyframe.frame.LazyFrame)
     df = lazy_df.collect()
     assert df.shape[0] == 2
     assert "bogus" not in df.columns, "Unneeded columns have been dropped"
@@ -156,7 +156,7 @@ def test_openalex_pubs_df(openalex_pubs_csv):
 
 def test_sulpub_df(sul_pubs_csv):
     lazy_df = merge_pubs.sulpub_df(sul_pubs_csv)
-    assert type(lazy_df) == pl.lazyframe.frame.LazyFrame
+    assert isinstance(lazy_df, pl.lazyframe.frame.LazyFrame)
     df = lazy_df.collect()
     assert df.shape[0] == 2, "Row without a doi has been dropped"
     assert df.columns == [

test/harvest/test_openalex.py

@@ -121,6 +121,7 @@ def test_pyalex_urlencoding():
     ), "we handle url URL encoding DOIs until pyalex does"
 
 
+@pytest.mark.skip(reason="This record no longer exhibits the problem")
 def test_pyalex_varnish_bug():
     # it seems like this author has a few records that are so big they blow out
     # OpenAlex's Varnish index. See https://groups.google.com/u/1/g/openalex-community/c/hl09WRF3Naw