From c789ee5fd1233dcff5631dce6a7c3f82df359ca5 Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Tue, 12 Dec 2023 16:18:09 -0600 Subject: [PATCH] Fix/total count (#250) * fix total count * fix * update * update tests --- src/server/__mocks__/mockDataFromES.js | 3 +++ src/server/__mocks__/mockESData/mockNestedAggs.js | 8 +++++++- .../mockESData/mockNestedTermsAndMissingAggs.js | 5 +++++ .../__mocks__/mockESData/mockNumericAggsGlobalStats.js | 4 ++++ .../mockESData/mockNumericHistogramFixBinCount.js | 4 ++++ .../__mocks__/mockESData/mockNumericHistogramFixWidth.js | 4 ++++ src/server/__mocks__/mockESData/mockTextAggs.js | 9 ++++++--- src/server/es/index.js | 5 +++++ 8 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/server/__mocks__/mockDataFromES.js b/src/server/__mocks__/mockDataFromES.js index 335bf9bb..51e36631 100644 --- a/src/server/__mocks__/mockDataFromES.js +++ b/src/server/__mocks__/mockDataFromES.js @@ -45,6 +45,7 @@ const mockResourcePath = () => { '*.analyzed': {}, }, }, + track_total_hits: true, }; const fakeResource = { aggregations: { @@ -120,6 +121,7 @@ const mockResourcePath = () => { '*.analyzed': {}, }, }, + track_total_hits: true, }; const fakeResourceWithFilter1 = { aggregations: { @@ -180,6 +182,7 @@ const mockResourcePath = () => { '*.analyzed': {}, }, }, + track_total_hits: true, }; const fakeResourceWithFilter2 = { aggregations: { diff --git a/src/server/__mocks__/mockESData/mockNestedAggs.js b/src/server/__mocks__/mockESData/mockNestedAggs.js index d5af2569..975c93e2 100644 --- a/src/server/__mocks__/mockESData/mockNestedAggs.js +++ b/src/server/__mocks__/mockESData/mockNestedAggs.js @@ -18,7 +18,7 @@ const mockNestedAggs = () => { terms: { field: 'visits.visit_label', missing_bucket: true, - order: "desc" + order: 'desc', }, }, }, @@ -29,6 +29,7 @@ const mockNestedAggs = () => { }, }, }, + track_total_hits: true, }; const fakeNestedAggs1 = { aggregations: { @@ -67,6 +68,7 @@ const mockNestedAggs = () => { }, }, }, + track_total_hits: true, }; mockSearchEndpoint(nestedAggsQuery1, fakeNestedAggs1); @@ -87,6 +89,7 @@ const mockNestedAggs = () => { }, }, }, + track_total_hits: true, }; const fakeNestedAggs2 = { aggregations: { @@ -101,6 +104,7 @@ const mockNestedAggs = () => { }, }, }, + track_total_hits: true, }; mockSearchEndpoint(nestedAggsQuery2, fakeNestedAggs2); @@ -134,6 +138,7 @@ const mockNestedAggs = () => { }, }, }, + track_total_hits: true, }; const fakeNestedAggs3 = { aggregations: { @@ -185,6 +190,7 @@ const mockNestedAggs = () => { }, }, }, + track_total_hits: true, }; mockSearchEndpoint(nestedAggsQuery3, fakeNestedAggs3); }; diff --git a/src/server/__mocks__/mockESData/mockNestedTermsAndMissingAggs.js b/src/server/__mocks__/mockESData/mockNestedTermsAndMissingAggs.js index 6195ab8b..93280669 100644 --- a/src/server/__mocks__/mockESData/mockNestedTermsAndMissingAggs.js +++ b/src/server/__mocks__/mockESData/mockNestedTermsAndMissingAggs.js @@ -32,6 +32,7 @@ const mockNestedTermsAndMissingAggs = () => { }, }, }, + track_total_hits: true, }; const fakeMissingAggs = { aggregations: { @@ -103,6 +104,7 @@ const mockNestedTermsAndMissingAggs = () => { }, }, }, + track_total_hits: true, }; const fakeTermsAggs = { aggregations: { @@ -167,6 +169,7 @@ const mockNestedTermsAndMissingAggs = () => { ], }, }, + track_total_hits: true, }; mockSearchEndpoint(termsAggsQuery, fakeTermsAggs); @@ -213,6 +216,7 @@ const mockNestedTermsAndMissingAggs = () => { }, }, }, + track_total_hits: true, }; const combinedTermsAggs = { aggregations: { @@ -286,6 +290,7 @@ const mockNestedTermsAndMissingAggs = () => { ], }, }, + track_total_hits: true, }; mockSearchEndpoint(combinedAggsQuery, combinedTermsAggs); }; diff --git a/src/server/__mocks__/mockESData/mockNumericAggsGlobalStats.js b/src/server/__mocks__/mockESData/mockNumericAggsGlobalStats.js index c329e380..816b67a8 100644 --- a/src/server/__mocks__/mockESData/mockNumericAggsGlobalStats.js +++ b/src/server/__mocks__/mockESData/mockNumericAggsGlobalStats.js @@ -11,6 +11,7 @@ const mockNumericAggsGlobalStats = () => { }, }, }, + track_total_hits: true, }; const fakeFileCountGlobalStatsAggs = { aggregations: { @@ -40,6 +41,7 @@ const mockNumericAggsGlobalStats = () => { }, }, }, + track_total_hits: true, }; const fakeFileCountGlobalStatsAggs1 = { aggregations: { @@ -84,6 +86,7 @@ const mockNumericAggsGlobalStats = () => { }, }, }, + track_total_hits: true, }; const fakeFileCountGlobalStatsAggs2 = { aggregations: { @@ -113,6 +116,7 @@ const mockNumericAggsGlobalStats = () => { }, }, }, + track_total_hits: true, }; const fakeFileCountGlobalStatsAggs3 = { aggregations: { diff --git a/src/server/__mocks__/mockESData/mockNumericHistogramFixBinCount.js b/src/server/__mocks__/mockESData/mockNumericHistogramFixBinCount.js index 30c7b373..741983d8 100644 --- a/src/server/__mocks__/mockESData/mockNumericHistogramFixBinCount.js +++ b/src/server/__mocks__/mockESData/mockNumericHistogramFixBinCount.js @@ -45,6 +45,7 @@ const mockHistogramFixBinCount = () => { }, }, }, + track_total_hits: true, }; const fileCountHistogramFixBinCountResult = { aggregations: { @@ -158,6 +159,7 @@ const mockHistogramFixBinCount = () => { }, }, }, + track_total_hits: true, }; const fileCountHistogramFixBinCountResult1 = { aggregations: { @@ -264,6 +266,7 @@ const mockHistogramFixBinCount = () => { }, }, }, + track_total_hits: true, }; const fileCountHistogramFixBinCountResult2 = { aggregations: { @@ -377,6 +380,7 @@ const mockHistogramFixBinCount = () => { }, }, }, + track_total_hits: true, }; const fileCountHistogramFixBinCountResult3 = { aggregations: { diff --git a/src/server/__mocks__/mockESData/mockNumericHistogramFixWidth.js b/src/server/__mocks__/mockESData/mockNumericHistogramFixWidth.js index 80cf3e8d..44028bba 100644 --- a/src/server/__mocks__/mockESData/mockNumericHistogramFixWidth.js +++ b/src/server/__mocks__/mockESData/mockNumericHistogramFixWidth.js @@ -24,6 +24,7 @@ const mockHistogramFixWidth = () => { }, }, }, + track_total_hits: true, }; const fileCountHistogramFixWidthResult = { aggregations: { @@ -107,6 +108,7 @@ const mockHistogramFixWidth = () => { }, }, }, + track_total_hits: true, }; const fileCountHistogramFixWidthResult1 = { aggregations: { @@ -206,6 +208,7 @@ const mockHistogramFixWidth = () => { }, }, }, + track_total_hits: true, }; const fileCountHistogramFixWidthResult2 = { aggregations: { @@ -256,6 +259,7 @@ const mockHistogramFixWidth = () => { }, }, }, + track_total_hits: true, }; const fileCountHistogramFixWidthResult3 = { aggregations: { diff --git a/src/server/__mocks__/mockESData/mockTextAggs.js b/src/server/__mocks__/mockESData/mockTextAggs.js index 1d892aaa..b1dadc9c 100644 --- a/src/server/__mocks__/mockESData/mockTextAggs.js +++ b/src/server/__mocks__/mockESData/mockTextAggs.js @@ -13,7 +13,7 @@ const mockTextAggs = () => { terms: { field: 'gender', missing_bucket: true, - order: "desc" + order: 'desc', }, }, }, @@ -22,6 +22,7 @@ const mockTextAggs = () => { }, }, }, + track_total_hits: true, }; const fakeGenderAggs = { aggregations: { @@ -80,7 +81,7 @@ const mockTextAggs = () => { terms: { field: 'gender', missing_bucket: true, - order: "desc" + order: 'desc', }, }, }, @@ -89,6 +90,7 @@ const mockTextAggs = () => { }, }, }, + track_total_hits: true, }; const fakeGenderAggs2 = { aggregations: { @@ -141,7 +143,7 @@ const mockTextAggs = () => { terms: { field: 'gender', missing_bucket: true, - order: "desc" + order: 'desc', }, }, }, @@ -150,6 +152,7 @@ const mockTextAggs = () => { }, }, }, + track_total_hits: true, }; const fakeGenderAggs3 = { aggregations: { diff --git a/src/server/es/index.js b/src/server/es/index.js index 6ec5c702..723e1e18 100644 --- a/src/server/es/index.js +++ b/src/server/es/index.js @@ -52,6 +52,7 @@ class ES { [`*${config.analyzedTextFieldSuffix}`]: {}, }, }; + validatedQueryBody.track_total_hits = true; log.info('[ES.query] index, type, query body: ', esIndex, esType, JSON.stringify(validatedQueryBody)); return this.client.search({ index: esIndex, @@ -425,6 +426,10 @@ class ES { { esInstance: this, esIndex, esType }, { filter, fields: false, size: 0 }, ); + // Really shouldn't be getting this, but just in case + if (result.hits.total.relation !== 'eq') { + log.error(`The returned total count might be inaccurate. See hits.total object: ${result.hits.total}`); + } return result.hits.total.value; }