[ALS-6800] Detailed, paginated concepts fetch

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptController.java

@@ -11,6 +11,8 @@
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.*;
 
+import java.util.List;
+
 @Controller
 public class ConceptController {
 
@@ -43,6 +45,21 @@ public ResponseEntity<Page<Concept>> listConcepts(
         return ResponseEntity.ok(pageResp);
     }
 
+    @GetMapping(path = "/concepts/dump")
+    public ResponseEntity<Page<Concept>> dumpConcepts(
+        @RequestParam(name = "page_number", defaultValue = "0", required = false) int page,
+        @RequestParam(name = "page_size", defaultValue = "10", required = false) int size
+    ) {
+        PageRequest pagination = PageRequest.of(page, size);
+        PageImpl<Concept> pageResp = new PageImpl<>(
+            conceptService.listDetailedConcepts(new Filter(List.of(), ""), pagination),
+            pagination,
+            conceptService.countConcepts(new Filter(List.of(), ""))
+        );
+
+        return ResponseEntity.ok(pageResp);
+    }
+
     @PostMapping(path = "/concepts/detail/{dataset}")
     public ResponseEntity<Concept> conceptDetail(
         @PathVariable(name = "dataset") String dataset,

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptMetaExtractor.java

@@ -0,0 +1,29 @@
+package edu.harvard.dbmi.avillach.dictionary.concept;
+
+import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept;
+import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
+import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.stereotype.Component;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
+@Component
+public class ConceptMetaExtractor implements ResultSetExtractor<Map<Concept, Map<String, String>>> {
+
+    @Override
+    public Map<Concept, Map<String, String>> extractData(ResultSet rs) throws SQLException, DataAccessException {
+        Map<Concept, Map<String, String>> sets = new HashMap<>();
+        while (rs.next()) {
+            Concept key = new ConceptShell(rs.getString("concept_path"), rs.getString("dataset_name"));
+            Map<String, String> meta = sets.getOrDefault(key, new HashMap<>());
+            meta.put(rs.getString("KEY"), rs.getString("VALUE"));
+            sets.put(key, meta);
+        }
+        return sets;
+    }
+}

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepository.java

@@ -22,14 +22,18 @@ public class ConceptRepository {
     private final ConceptRowMapper mapper;
 
     private final ConceptFilterQueryGenerator filterGen;
+    private final ConceptMetaExtractor conceptMetaExtractor;
+
 
     @Autowired
     public ConceptRepository(
-        NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen
+        NamedParameterJdbcTemplate template, ConceptRowMapper mapper, ConceptFilterQueryGenerator filterGen,
+        ConceptMetaExtractor conceptMetaExtractor
     ) {
         this.template = template;
         this.mapper = mapper;
         this.filterGen = filterGen;
+        this.conceptMetaExtractor = conceptMetaExtractor;
     }
 
 
@@ -107,4 +111,26 @@ public Map<String, String> getConceptMeta(String dataset, String conceptPath) {
             .addValue("dataset", dataset);
         return template.query(sql, params, new MapExtractor("KEY", "VALUE"));
     }
+
+    public Map<Concept, Map<String, String>> getConceptMetaForConcepts(List<Concept> concepts) {
+        String sql = """
+            SELECT
+                concept_node_meta.KEY, concept_node_meta.VALUE,
+                concept_node.CONCEPT_PATH AS concept_path, dataset.REF AS dataset_name
+            FROM
+                concept_node
+                LEFT JOIN concept_node_meta ON concept_node.concept_node_id = concept_node_meta.concept_node_id
+                LEFT JOIN dataset ON concept_node.dataset_id = dataset.dataset_id
+            WHERE
+                (concept_node.CONCEPT_PATH, dataset.REF) IN (:pairs)
+            ORDER BY concept_node.CONCEPT_PATH, dataset.REF
+            """;
+        List<String[]> pairs = concepts.stream()
+            .map(c -> new String[]{c.conceptPath(), c.dataset()})
+            .toList();
+        MapSqlParameterSource params = new MapSqlParameterSource().addValue("pairs", pairs);
+
+        return template.query(sql, params, conceptMetaExtractor);
+
+    }
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptService.java

@@ -2,14 +2,18 @@
 
 import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept;
 import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
+import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell;
 import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept;
 import edu.harvard.dbmi.avillach.dictionary.filter.Filter;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Pageable;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 @Service
 public class ConceptService {
@@ -25,6 +29,16 @@ public List<Concept> listConcepts(Filter filter, Pageable page) {
         return conceptRepository.getConcepts(filter, page);
     }
 
+    public List<Concept> listDetailedConcepts(Filter filter, Pageable page) {
+        List<Concept> concepts = conceptRepository.getConcepts(filter, page);
+        Map<Concept, Map<String, String>> metas = conceptRepository.getConceptMetaForConcepts(concepts);
+        return concepts.stream().map(concept -> (Concept) switch (concept) {
+            case ContinuousConcept cont -> new ContinuousConcept(cont, metas.getOrDefault(cont, Map.of()));
+            case CategoricalConcept cat -> new CategoricalConcept(cat, metas.getOrDefault(cat, Map.of()));
+            case ConceptShell ignored -> throw new RuntimeException("Concept shell escaped to API");
+        }).toList();
+    }
+
     public long countConcepts(Filter filter) {
         return conceptRepository.countConcepts(filter);
     }
@@ -36,6 +50,7 @@ public Optional<Concept> conceptDetail(String dataset, String conceptPath) {
                 return switch (core) {
                     case ContinuousConcept cont -> new ContinuousConcept(cont, meta);
                     case CategoricalConcept cat -> new CategoricalConcept(cat, meta);
+                    case ConceptShell ignored -> throw new RuntimeException("Concept shell escaped to API");
                 };
             }
         );

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/CategoricalConcept.java

@@ -5,6 +5,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 public record CategoricalConcept(
     String conceptPath, String name, String display, String dataset, String description,
@@ -29,4 +30,14 @@ public CategoricalConcept(CategoricalConcept core, Map<String, String> meta) {
     public ConceptType type() {
         return ConceptType.Categorical;
     }
+
+    @Override
+    public boolean equals(Object object) {
+        return conceptEquals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(conceptPath, dataset);
+    }
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/Concept.java

@@ -5,6 +5,7 @@
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 
 import java.util.Map;
+import java.util.Objects;
 
 
 @JsonIgnoreProperties(ignoreUnknown = true)
@@ -18,7 +19,7 @@
     @JsonSubTypes.Type(value = CategoricalConcept.class, name = "Categorical"),
 })
 public sealed interface Concept
-    permits CategoricalConcept, ContinuousConcept {
+    permits CategoricalConcept, ConceptShell, ContinuousConcept {
 
     /**
      * @return The complete concept path for this concept (// delimited)
@@ -48,5 +49,10 @@ public sealed interface Concept
 
     Map<String, String> meta();
 
-
+    default boolean conceptEquals(Object object) {
+        if (this == object) return true;
+        if (!(object instanceof Concept)) return false;
+        Concept that = (Concept) object;
+        return Objects.equals(dataset(), that.dataset()) && Objects.equals(conceptPath(), that.conceptPath());
+    }
 }

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ConceptShell.java

@@ -0,0 +1,36 @@
+package edu.harvard.dbmi.avillach.dictionary.concept.model;
+
+import java.util.Map;
+import java.util.Objects;
+
+public record ConceptShell(String conceptPath, String dataset) implements Concept {
+    @Override
+    public String name() {
+        return "Shell. Not for external use.";
+    }
+
+    @Override
+    public String display() {
+        return "Shell. Not for external use.";
+    }
+
+    @Override
+    public ConceptType type() {
+        return ConceptType.Continuous;
+    }
+
+    @Override
+    public Map<String, String> meta() {
+        return Map.of();
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        return conceptEquals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(conceptPath, dataset);
+    }
+}

src/main/java/edu/harvard/dbmi/avillach/dictionary/concept/model/ContinuousConcept.java

@@ -4,6 +4,7 @@
 import jakarta.annotation.Nullable;
 
 import java.util.Map;
+import java.util.Objects;
 
 public record ContinuousConcept(
     String conceptPath, String name, String display, String dataset, String description,
@@ -21,4 +22,14 @@ public ContinuousConcept(ContinuousConcept core, Map<String, String> meta) {
     public ConceptType type() {
         return ConceptType.Continuous;
     }
+
+    @Override
+    public boolean equals(Object object) {
+        return conceptEquals(object);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(conceptPath, dataset);
+    }
 }

src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptControllerTest.java

@@ -12,6 +12,7 @@
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
@@ -143,4 +144,25 @@ void shouldNotGetConceptTreeWhenConceptDNE() {
 
         Assertions.assertEquals(HttpStatus.NOT_FOUND, actual.getStatusCode());
     }
+
+    @Test
+    void shouldDumpConcepts() {
+        Concept fooBar = new CategoricalConcept(
+            "/foo//bar", "bar", "Bar", "my_dataset", "foo!", List.of("a", "b"), List.of(),
+            Map.of("key", "value")
+        );
+        Concept fooBaz = new ContinuousConcept(
+            "/foo//baz", "baz", "Baz", "my_dataset", "foo!", 0, 100,
+            Map.of("key", "value")
+        );
+        List<Concept> concepts = List.of(fooBar, fooBaz);
+        PageRequest page = PageRequest.of(0, 10);
+        Mockito.when(conceptService.listDetailedConcepts(new Filter(List.of(), ""), page))
+            .thenReturn(concepts);
+
+        ResponseEntity<Page<Concept>> actual = subject.dumpConcepts(0, 10);
+
+        Assertions.assertEquals(concepts, actual.getBody().getContent());
+        Assertions.assertEquals(HttpStatus.OK, actual.getStatusCode());
+    }
 }

src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptRepositoryTest.java

@@ -2,6 +2,7 @@
 
 import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept;
 import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
+import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell;
 import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept;
 import edu.harvard.dbmi.avillach.dictionary.facet.Facet;
 import edu.harvard.dbmi.avillach.dictionary.filter.Filter;
@@ -145,7 +146,36 @@ void shouldNotGetConceptThatDNE() {
         Optional<Concept> actual = subject.getConcept("invalid.invalid", "fake");
         Assertions.assertEquals(Optional.empty(), actual);
 
-        actual = subject.getConcept("fake", "\\\\\\\\B\\\\\\\\2\\\\\\\\Z\\\\\\\\");
+        actual = subject.getConcept("fake", "\\\\B\\\\2\\\\Z\\\\");
         Assertions.assertEquals(Optional.empty(), actual);
     }
+
+    @Test
+    void shouldGetMetaForMultipleConcepts() {
+        List<Concept> concepts = List.of(
+            new ContinuousConcept("\\phs000007\\pht000022\\phv00004260\\FM219\\", "", "", "phs000007", "", null, null, Map.of()),
+            new ContinuousConcept("\\phs000007\\pht000033\\phv00008849\\D080\\", "", "", "phs000007", "", null, null, Map.of())
+        );
+
+        Map<Concept, Map<String, String>> actual = subject.getConceptMetaForConcepts(concepts);
+        Map<Concept, Map<String, String>> expected = Map.of(
+            new ConceptShell("\\phs000007\\pht000022\\phv00004260\\FM219\\", "phs000007"), Map.of(
+                "unique_identifier", "no",
+                "stigmatizing", "no",
+                "bdc_open_access", "yes",
+                "values", "[0, 1]",
+                "description", "# 12 OZ CUPS OF CAFFEINATED COLA / DAY",
+                "free_text", "no"
+            ),
+            new ConceptShell("\\phs000007\\pht000033\\phv00008849\\D080\\", "phs000007"), Map.of(
+                "unique_identifier", "no",
+                "stigmatizing", "no",
+                "bdc_open_access", "yes",
+                "values", "[0, 5]",
+                "description", "# 12 OZ CUPS OF CAFFEINATED COLA/DAY",
+                "free_text", "no"
+            )
+        );
+        Assertions.assertEquals(expected, actual);
+    }
 }

src/test/java/edu/harvard/dbmi/avillach/dictionary/concept/ConceptServiceTest.java

@@ -2,6 +2,7 @@
 
 import edu.harvard.dbmi.avillach.dictionary.concept.model.CategoricalConcept;
 import edu.harvard.dbmi.avillach.dictionary.concept.model.Concept;
+import edu.harvard.dbmi.avillach.dictionary.concept.model.ConceptShell;
 import edu.harvard.dbmi.avillach.dictionary.concept.model.ContinuousConcept;
 import edu.harvard.dbmi.avillach.dictionary.filter.Filter;
 import org.junit.jupiter.api.Assertions;
@@ -81,4 +82,33 @@ void shouldShowDetailForCategorical() {
 
         Assertions.assertEquals(expected, actual);
     }
+
+    @Test
+    void shouldShowDetailForMultiple() {
+        ConceptShell shellA = new ConceptShell("pathA", "dataset");
+        CategoricalConcept conceptA = new CategoricalConcept("pathA", "", "", "dataset", null, List.of("a"), List.of(), null);
+        Map<String, String> metaA = Map.of("VALUES", "a", "stigmatizing", "true");
+
+        ConceptShell shellB = new ConceptShell("pathB", "dataset");
+        ContinuousConcept conceptB = new ContinuousConcept("pathB", "", "", "dataset", null, 0, 1, null);
+        Map<String, String> metaB = Map.of("MIN", "0", "MAX", "1", "stigmatizing", "true");
+
+        Map<Concept, Map<String, String>> metas = Map.of(shellA, metaA, shellB, metaB);
+        List<Concept> concepts = List.of(conceptA, conceptB);
+        Filter emptyFilter = new Filter(List.of(), "");
+
+
+        Mockito.when(repository.getConceptMetaForConcepts(concepts))
+            .thenReturn(metas);
+        Mockito.when(repository.getConcepts(emptyFilter, Pageable.unpaged()))
+            .thenReturn(concepts);
+
+        List<Concept> actual = subject.listDetailedConcepts(emptyFilter, Pageable.unpaged());
+        List<Concept> expected = List.of(
+            new CategoricalConcept(conceptA, metaA),
+            new ContinuousConcept(conceptB, metaB)
+        );
+
+        Assertions.assertEquals(expected, actual);
+    }
 }