feat(re-index): recreate upload ranges each upload execution (#726)

Closes: MSEARCH-934
folio-org · Jan 6, 2025 · e6f3f90 · e6f3f90
1 parent ada7406
commit e6f3f90
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 32 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -23,6 +23,7 @@
 * Fix the "Invalid reference" appears after updating ownership ([MSEARCH-915](https://folio-org.atlassian.net/browse/MSEARCH-915))
 
 ### Tech Dept
+* Recreate upload ranges each upload execution ([MSEARCH-934](https://folio-org.atlassian.net/browse/MSEARCH-934))
 * Remove unnecessary LccnNormalizerStructureB ([MSEARCH-926](https://folio-org.atlassian.net/browse/MSEARCH-926))
 
 ### Dependencies

diff --git a/src/main/java/org/folio/search/service/reindex/ReindexUploadRangeIndexService.java b/src/main/java/org/folio/search/service/reindex/ReindexUploadRangeIndexService.java
@@ -40,7 +40,7 @@ public void prepareAndSendIndexRanges(ReindexEntityType entityType) {
     var repository = Optional.ofNullable(repositories.get(entityType))
       .orElseThrow(() -> new UnsupportedOperationException("No repository found for entity type: " + entityType));
 
-    var uploadRanges = repository.getUploadRanges(true);
+    var uploadRanges = repository.createUploadRanges();
     statusService.updateReindexUploadStarted(entityType, uploadRanges.size());
     indexRangeEventProducer.sendMessages(prepareEvents(uploadRanges));
   }

diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java
@@ -42,6 +42,7 @@ ON CONFLICT (id)
     """;
 
   private static final String SELECT_UPLOAD_RANGE_BY_ENTITY_TYPE_SQL = "SELECT * FROM %s WHERE entity_type = ?;";
+  private static final String DELETE_UPLOAD_RANGE_SQL = "DELETE FROM %s WHERE entity_type = ?;";
   private static final TypeReference<LinkedHashSet<InstanceSubResource>> VALUE_TYPE_REF = new TypeReference<>() { };
 
   protected final ReindexConfigurationProperties reindexConfig;
@@ -54,14 +55,19 @@ protected UploadRangeRepository(JdbcTemplate jdbcTemplate,
     this.reindexConfig = reindexConfig;
   }
 
-  public List<UploadRangeEntity> getUploadRanges(boolean populateIfNotExist) {
+  public List<UploadRangeEntity> getUploadRanges() {
     var fullTableName = getFullTableName(context, UPLOAD_RANGE_TABLE);
     var sql = SELECT_UPLOAD_RANGE_BY_ENTITY_TYPE_SQL.formatted(fullTableName);
-    var uploadRanges = jdbcTemplate.query(sql, uploadRangeRowMapper(), entityType().getType());
 
-    return populateIfNotExist && uploadRanges.isEmpty()
-           ? prepareAndSaveUploadRanges()
-           : uploadRanges;
+    return jdbcTemplate.query(sql, uploadRangeRowMapper(), entityType().getType());
+  }
+
+  public List<UploadRangeEntity> createUploadRanges() {
+    var fullTableName = getFullTableName(context, UPLOAD_RANGE_TABLE);
+    var deleteSql = DELETE_UPLOAD_RANGE_SQL.formatted(fullTableName);
+    jdbcTemplate.update(deleteSql, entityType().getType());
+
+    return prepareAndSaveUploadRanges();
   }
 
   public List<Map<String, Object>> fetchByIdRange(String lower, String upper) {

diff --git a/src/test/java/org/folio/search/service/reindex/ReindexUploadRangeIndexServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexUploadRangeIndexServiceTest.java
@@ -46,7 +46,7 @@ void setUp() {
   @Test
   void prepareAndSendIndexRanges_positive(@Random UploadRangeEntity uploadRange) {
     // arrange
-    when(repository.getUploadRanges(true)).thenReturn(List.of(uploadRange));
+    when(repository.createUploadRanges()).thenReturn(List.of(uploadRange));
 
     // act
     service.prepareAndSendIndexRanges(ReindexEntityType.INSTANCE);

diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepositoriesIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/ReindexJdbcRepositoriesIT.java
@@ -23,8 +23,8 @@
 import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase;
 import org.springframework.boot.test.autoconfigure.jdbc.JdbcTest;
 import org.springframework.boot.test.autoconfigure.json.AutoConfigureJson;
-import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.context.bean.override.mockito.MockitoBean;
 import org.springframework.test.context.jdbc.Sql;
 
 @IntegrationTest
@@ -35,9 +35,9 @@
 class ReindexJdbcRepositoriesIT {
 
   private @Autowired JdbcTemplate jdbcTemplate;
-  private @MockBean FolioExecutionContext context;
-  private @MockBean ReindexConfigurationProperties reindexConfig;
-  private @MockBean ConsortiumTenantProvider tenantProvider;
+  private @MockitoBean FolioExecutionContext context;
+  private @MockitoBean ReindexConfigurationProperties reindexConfig;
+  private @MockitoBean ConsortiumTenantProvider tenantProvider;
   private MergeInstanceRepository mergeRepository;
   private UploadInstanceRepository uploadRepository;
 
@@ -77,15 +77,15 @@ void updateRangeStatus() {
     var mergeRange = mergeRepository.getMergeRanges().stream()
       .filter(range -> range.getEntityType().equals(ReindexEntityType.INSTANCE))
       .findFirst();
-    var uploadRange = uploadRepository.getUploadRanges(false).stream()
+    var uploadRange = uploadRepository.getUploadRanges().stream()
       .filter(range -> range.getEntityType().equals(ReindexEntityType.INSTANCE))
       .findFirst();
 
     assertThat(mergeRange).isPresent().get()
       .matches(range -> timestamp.getTime() == range.getFinishedAt().getTime()
-        && ReindexRangeStatus.SUCCESS == range.getStatus() && failCause.equals(range.getFailCause()));
+                        && ReindexRangeStatus.SUCCESS == range.getStatus() && failCause.equals(range.getFailCause()));
     assertThat(uploadRange).isPresent().get()
       .matches(range -> timestamp.getTime() == range.getFinishedAt().getTime()
-        && ReindexRangeStatus.FAIL == range.getStatus() && failCause.equals(range.getFailCause()));
+                        && ReindexRangeStatus.FAIL == range.getStatus() && failCause.equals(range.getFailCause()));
   }
 }
diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/SubjectRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/SubjectRepositoryIT.java
@@ -33,10 +33,10 @@
 import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase;
 import org.springframework.boot.test.autoconfigure.jdbc.JdbcTest;
 import org.springframework.boot.test.autoconfigure.json.AutoConfigureJson;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.boot.test.mock.mockito.SpyBean;
 import org.springframework.dao.InvalidDataAccessResourceUsageException;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.context.bean.override.mockito.MockitoBean;
+import org.springframework.test.context.bean.override.mockito.MockitoSpyBean;
 import org.springframework.test.context.jdbc.Sql;
 
 @IntegrationTest
@@ -46,8 +46,8 @@
 @AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
 class SubjectRepositoryIT {
 
-  private @SpyBean JdbcTemplate jdbcTemplate;
-  private @MockBean FolioExecutionContext context;
+  private @MockitoSpyBean JdbcTemplate jdbcTemplate;
+  private @MockitoBean FolioExecutionContext context;
   private SubjectRepository repository;
   private ReindexConfigurationProperties properties;
 
@@ -71,22 +71,12 @@ public String getDBSchemaName(String tenantId) {
   }
 
   @Test
-  void getUploadRanges_returnEmptyList_whenNoUploadRangesAndNotPopulate() {
-    // act
-    var ranges = repository.getUploadRanges(false);
-
-    // assert
-    assertThat(ranges).isEmpty();
-  }
-
-  @Test
-  @Sql("/sql/populate-subjects.sql")
-  void getUploadRanges_returnList_whenNoUploadRangesAndNotPopulate() {
+  void getUploadRanges_returnList() {
     // arrange
     properties.setUploadRangeLevel(1);
 
     // act
-    var ranges = repository.getUploadRanges(true);
+    var ranges = repository.createUploadRanges();
 
     // assert
     assertThat(ranges)
@@ -139,7 +129,7 @@ void deleteByInstanceIds_oneSubjectRemovedAndOneInstanceCounterDecremented() {
       .contains(
         tuple("Sci-Fi", List.of(
           Map.of("count", 1, "shared", true, "tenantId", "consortium"),
-            Map.of("count", 1, "shared", false, "tenantId", "member_tenant"))));
+          Map.of("count", 1, "shared", false, "tenantId", "member_tenant"))));
   }
 
   @Test

diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/UploadRangeRepositoriesIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/UploadRangeRepositoriesIT.java
@@ -56,7 +56,7 @@ public String getDBSchemaName(String tenantId) {
   @Sql({"/sql/populate-instances.sql"})
   void getUploadRanges_shouldNotPopulateStatus() {
     // act
-    var uploadRanges = uploadRepository.getUploadRanges(true);
+    var uploadRanges = uploadRepository.createUploadRanges();
     System.out.println(uploadRanges.size());
 
     // assert