feat(export): allow fetch by bucket (#525)

data/config/va.json

@@ -224,7 +224,14 @@
       "projectId": "search",
       "graphqlField": "patients",
       "index": "",
-      "nodeCountField": "Chicago_ID"
+      "nodeCountField": "Chicago_ID",
+      "manifestMapping": {
+        "resourceIndexType": "file",
+        "resourceIdField": "object_id",
+        "referenceIdFieldInResourceIndex": "case_id",
+        "referenceIdFieldInDataIndex": "object_id",
+        "divideKey": "data_type"
+      }
     }
   }
 }

src/DataExplorer/actionHelper.js

@@ -4,6 +4,7 @@ import {
   queryDataByIds,
   queryCountByValues,
   queryDataByValues,
+  queryAggregations,
 } from './arrangerQueryHelper';
 import { hasKeyChain } from './utils';
 import { fetchWithCreds } from '../actions';
@@ -136,22 +137,37 @@ export const exportToWorkspace = async (
     || !hasKeyChain(arrangerConfig, 'manifestMapping.referenceIdFieldInResourceIndex')) {
     errorCallback(500, MSG_EXPORT_MANIFEST_FAIL);
   }
-
-  const manifestJSON = await queryDataByValues(
+  // Fetch docs by bucket for this key
+  const key = arrangerConfig.manifestMapping.divideKey
+  const rowFilter = {
+    name: arrangerConfig.manifestMapping.referenceIdFieldInResourceIndex,
+    values: selectedTableRows
+  }
+  let filtersByKey = [[rowFilter]]
+  if (key !== undefined ){
+    const keyBuckets = await queryAggregations(
+      apiFunc,
+      projectId,
+      arrangerConfig.manifestMapping.resourceIndexType,
+      key,
+    )
+    filtersByKey = keyBuckets.map(
+      bucket => [rowFilter, {name: key, values: [bucket.key]}])
+  }
+  const manifestJSON = await Promise.all(
+    filtersByKey.map(filters => queryDataByValues(
     apiFunc,
     projectId,
     arrangerConfig.manifestMapping.resourceIndexType,
-    arrangerConfig.manifestMapping.referenceIdFieldInResourceIndex,
-    selectedTableRows,
+    filters,
     [
       arrangerConfig.manifestMapping.resourceIdField,
       arrangerConfig.manifestMapping.referenceIdFieldInResourceIndex,
     ],
-  );
-
+  )));
   fetchWithCreds({
     path: `${manifestServiceApiPath}`,
-    body: JSON.stringify(manifestJSON),
+    body: JSON.stringify(manifestJSON.flat()),
     method: 'POST',
   })
     .then(
@@ -200,8 +216,10 @@ export const getManifestEntryCount = async (
     apiFunc,
     projectId,
     arrangerConfig.manifestMapping.resourceIndexType,
-    arrangerConfig.manifestMapping.referenceIdFieldInResourceIndex,
-    selectedTableRows,
+    [{
+      name: arrangerConfig.manifestMapping.referenceIdFieldInResourceIndex,
+      values:selectedTableRows
+    }],
   );
   return manifestEntryCount;
 };

src/DataExplorer/arrangerQueryHelper.js

@@ -75,10 +75,64 @@ export const constructGraphQLQueryWithSQON = (
   return gqlQuery;
 };
 
+/**
+ * Constructs graphql aggregation string for arranger to get data.
+ * @param {string} indexType - type name of index for query
+ * @param {string} key - key name to get aggregation
+ * @returns {object} graphql query object
+ */
+export const constructAggregationQuery = (
+  indexType,
+  key,
+) => {
+  const gqlQuery = {
+    query: `query {
+                ${indexType} {
+                  aggregations {
+                    ${key} {
+                      buckets {
+                        doc_count
+                        key
+                      }
+                    }
+                  }
+                }
+              }`
+  };
+  return gqlQuery;
+};
+
+/**
+ * Query arranger for aggregation by aggregation key
+ * @param {function} apiFunc - function created by arranger for fetching data
+ * @param {stirng} projectId - arranger project ID
+ * @param {string} indexType - index type for query
+ * @param {string} key - key name to get aggregation
+ * @returns {Object[]} List of objects, each has the same keys as in "fields"
+ */
+export const queryAggregations = async (
+  apiFunc,
+  projectId,
+  indexType,
+  key,
+) => {
+  const MSG_QUERY_BY_AGG_FAIL = 'Error while querying Arranger aggregation';
+  const responseData = await apiFunc({
+    endpoint: getEndpoint(projectId),
+    body: constructAggregationQuery(
+      indexType,
+      key,
+    ),
+  });
+  if (!responseData) {
+    throw MSG_QUERY_BY_AGG_FAIL;
+  }
+  return responseData.data[indexType].aggregations[key].buckets
+};
+
 /**
  * Constructs graphql string for arranger to get data.
- * @param {stirng} filterFieldName - field name for filetering
- * @param {string[]} filterFieldValues - list of values for filtering
+ * @param {Object[]} filters - a list of {name, values}
  * @param {string} indexType - type name of index for query
  * @param {string[]} nodeList - list of node for respone
  * @param {boolean} isGettingCount - if set true, only get count of total hits;
@@ -87,24 +141,23 @@ export const constructGraphQLQueryWithSQON = (
  * @returns {object} graphql query object
  */
 export const constructGraphQLQuery = (
-  filterFieldName,
-  filterFieldValues,
+  filters,
   indexType,
   nodeList,
   isGettingCount,
   count,
 ) => {
   const sqonObj = {
     op: 'and',
-    content: [
-      {
+    content: filters.map(filter => {
+      return {
         op: 'in',
         content: {
-          field: filterFieldName,
-          value: [...filterFieldValues],
+          field: filter.name,
+          value: [...filter.values],
         },
-      },
-    ],
+      }
+    }),
   };
   return constructGraphQLQueryWithSQON(indexType, sqonObj, nodeList, isGettingCount, count);
 };
@@ -177,8 +230,7 @@ export const queryDataByIds = async (
   const responseData = await apiFunc({
     endpoint: getEndpoint(projectId),
     body: constructGraphQLQuery(
-      graphqlIdField.toString(),
-      idList,
+      [{name: graphqlIdField.toString(), values: idList}],
       indexType,
       [...fields],
       false,
@@ -239,23 +291,20 @@ export const queryDataBySQON = async (
  * @param {function} apiFunc - function created by arranger for fetching data
  * @param {stirng} projectId - arranger project ID
  * @param {string} indexType - index type for query
- * @param {string} filterField - field name for query
- * @param {string[]} filterValues - list of value to match for query
+ * @param {Object[]} filters - a list of {name, values}
  * @returns {number} number of matched objects
  */
 export const queryCountByValues = async (
   apiFunc,
   projectId,
   indexType,
-  filterField,
-  filterValues,
+  filters
 ) => {
   const MSG_QUERY_COUNT_BY_VALUE_FAIL = 'Error while querying Arranger count by values';
   const countQueryResponse = await apiFunc({
     endpoint: getEndpoint(projectId),
     body: constructGraphQLQuery(
-      filterField,
-      filterValues,
+      filters,
       indexType,
       [],
       true),
@@ -273,26 +322,23 @@ export const queryCountByValues = async (
  * @param {function} apiFunc - function created by arranger for fetching data
  * @param {stirng} projectId - arranger project ID
  * @param {string} indexType - index type for query
- * @param {string} filterField - field name for query
- * @param {string[]} filterValues - list of value to match for query
+ * @param {Object[]} filters - a list of {name, values}
  * @param {string[]} returnFields - list of target fields for response
  * @returns {Object[]} List of objects, each has the same keys as in "returnFields"
  */
 export const queryDataByValues = async (
   apiFunc,
   projectId,
   indexType,
-  filterField,
-  filterValues,
+  filters,
   returnFields,
 ) => {
   const MSG_QUERY_BY_VALUE_FAIL = 'Error while querying Arranger data by values';
-  const count = await queryCountByValues(apiFunc, projectId, indexType, filterField, filterValues);
+  const count = await queryCountByValues(apiFunc, projectId, indexType, filters);
   const responseData = await apiFunc({
     endpoint: getEndpoint(projectId),
     body: constructGraphQLQuery(
-      filterField,
-      filterValues,
+      filters,
       indexType,
       [...returnFields],
       false,

src/DataExplorer/arrangerQueryHelper.test.js

@@ -6,8 +6,10 @@ describe('Arranger query helper', () => {
   const values = ['1', '2'];
   const targetFields = ['gender'];
   const queryForCount = constructGraphQLQuery(
-    fieldName,
-    values,
+    [{
+      name: fieldName,
+      values: values,
+    }],
     indexType,
     [...targetFields],
     true,
@@ -39,8 +41,10 @@ describe('Arranger query helper', () => {
   };
   const fakeReturnCount = 6;
   const queryForData = constructGraphQLQuery(
-    fieldName,
-    values,
+    [{
+      name: fieldName,
+      values: values,
+    }],
     indexType,
     [...targetFields],
     false,