diff --git a/bootstrap-app/src/cmr/bootstrap/system.clj b/bootstrap-app/src/cmr/bootstrap/system.clj index 6e20b41604..2d72c54984 100644 --- a/bootstrap-app/src/cmr/bootstrap/system.clj +++ b/bootstrap-app/src/cmr/bootstrap/system.clj @@ -78,7 +78,8 @@ :caches {acl/token-imp-cache-key (acl/create-token-imp-cache) kf/kms-cache-key (kf/create-kms-cache) common-health/health-cache-key (common-health/create-health-cache)} - :scheduler (jobs/create-scheduler `system-holder [jvm-info/log-jvm-statistics-job]) + :scheduler (jobs/create-scheduler `system-holder [jvm-info/log-jvm-statistics-job + (kf/refresh-kms-cache-job "bootstrap-kms-cache-refresh")]) :queue-broker queue-broker}] (transmit-config/system-with-connections sys [:metadata-db :echo-rest :kms :indexer :access-control]))) diff --git a/common-app-lib/src/cmr/common_app/services/kms_fetcher.clj b/common-app-lib/src/cmr/common_app/services/kms_fetcher.clj index b5dba45e76..45e2086c50 100644 --- a/common-app-lib/src/cmr/common_app/services/kms_fetcher.clj +++ b/common-app-lib/src/cmr/common_app/services/kms_fetcher.clj @@ -52,24 +52,12 @@ "The key used to store the KMS cache in the system cache map." :kms) -(defconfig kms-cache-consistent-timeout-seconds - "The number of seconds between when the KMS cache should check with redis for consistence" - {:default 3600 - :type Long}) - (defn create-kms-cache "Used to create the cache that will be used for caching KMS keywords. All applications caching KMS keywords should use the same fallback cache to ensure functionality even if GCMD KMS becomes unavailable." [] - (stl-cache/create-single-thread-lookup-cache - (fallback-cache/create-fallback-cache - (consistent-cache/create-consistent-cache - {:hash-timeout-seconds (kms-cache-consistent-timeout-seconds)}) - (deflating-cache/create-deflating-cache - (redis-cache/create-redis-cache) - kms-lookup/create-kms-index - kms-lookup/deflate)))) + (redis-cache/create-redis-cache)) (defn- fetch-gcmd-keywords-map "Calls GCMD KMS endpoints to retrieve the keywords. Response is a map structured in the same way @@ -112,9 +100,8 @@ (defn refresh-kms-cache-job "The singleton job that refreshes the KMS cache. The keywords are infrequently updated by the - GCMD team. They update the CSV file which we read from every 6 hours. I arbitrarily chose 2 hours - so that we are never more than 8 hours from the time a keyword is updated." + GCMD team, usually once a week." [job-key] {:job-type RefreshKmsCacheJob :job-key job-key - :interval 7200}) + :daily-at-hour-and-minute [02 00]}) diff --git a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj index d64ab38bf6..7851afb0ab 100644 --- a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj +++ b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj @@ -146,12 +146,6 @@ :locations-index location-lookup-map :measurement-index measurement-lookup-map}))) -(defn deflate - "Takes a KMS index and returns a minimal version to store more efficiently and in a way that - the index can be recreated in a way to reduce the memory usage." - [kms-index] - (dissoc kms-index :short-name-index :umm-c-index :locations-index :measurement-index)) - (defn lookup-by-short-name "Takes a kms-index, the keyword scheme, and a short name and returns the full KMS hierarchy for that short name. Comparison is made case insensitively." diff --git a/common-app-lib/test/cmr/common_app/test/services/kms_lookup.clj b/common-app-lib/test/cmr/common_app/test/services/kms_lookup.clj index 0fb53d7a1d..7d782294e6 100644 --- a/common-app-lib/test/cmr/common_app/test/services/kms_lookup.clj +++ b/common-app-lib/test/cmr/common_app/test/services/kms_lookup.clj @@ -43,9 +43,6 @@ "KMS index to use for all of the tests which creates the KMS index from the sample map." (kms-lookup/create-kms-index sample-map)) -(deftest roundtrip-inflate-and-deflate-index-test - (is (= sample-map (kms-lookup/deflate (kms-lookup/create-kms-index sample-map))))) - (deftest lookup-by-location-string-test (testing "Full location hierarchy is returned" (is (= {:category "CONTINENT" :type "AFRICA" :subregion-1 "CENTRAL AFRICA" diff --git a/dev-system/src/cmr/dev_system/system.clj b/dev-system/src/cmr/dev_system/system.clj index de1c4b82e0..6e6b8bafb0 100644 --- a/dev-system/src/cmr/dev_system/system.clj +++ b/dev-system/src/cmr/dev_system/system.clj @@ -276,7 +276,7 @@ {:mock-echo echo-component :access-control (create-access-control-app queue-broker) :metadata-db (create-metadata-db-app db-component queue-broker) - :bootstrap (when-not db-component (create-bootstrap-app queue-broker)) + :bootstrap (create-bootstrap-app queue-broker) :indexer (create-indexer-app queue-broker) :ingest (create-ingest-app db queue-broker) :search (create-search-app db-component queue-broker) diff --git a/indexer-app/src/cmr/indexer/system.clj b/indexer-app/src/cmr/indexer/system.clj index 6cb6033c47..16481a0267 100644 --- a/indexer-app/src/cmr/indexer/system.clj +++ b/indexer-app/src/cmr/indexer/system.clj @@ -72,7 +72,6 @@ :scheduler (jobs/create-scheduler `system-holder [(af/refresh-acl-cache-job "indexer-acl-cache-refresh") - (kf/refresh-kms-cache-job "indexer-kms-cache-refresh") jvm-info/log-jvm-statistics-job (cache-info/create-log-cache-info-job "indexer")]) :queue-broker (queue-broker/create-queue-broker (config/queue-config))}] diff --git a/search-app/src/cmr/search/system.clj b/search-app/src/cmr/search/system.clj index 1482d095eb..db5a2b12a3 100644 --- a/search-app/src/cmr/search/system.clj +++ b/search-app/src/cmr/search/system.clj @@ -121,9 +121,6 @@ metadata-transformer/xsl-transformer-cache-name (mem-cache/create-in-memory-cache) acl/token-imp-cache-key (acl/create-token-imp-cache) acl/token-pc-cache-key (acl/create-token-pc-cache) - ;; Note that search does not have a job to refresh the KMS cache. The indexer - ;; already refreshes the cache. Since we use a consistent cache, the search - ;; application will also pick up the updated KMS keywords. launchpad-user-cache/launchpad-user-cache-key (launchpad-user-cache/create-launchpad-user-cache) urs/urs-cache-key (urs/create-urs-cache) kf/kms-cache-key (kf/create-kms-cache) diff --git a/common-app-lib/test/cmr/common_app/services/kms_fetcher_test.clj b/system-int-test/test/cmr/system_int_test/misc/kms_fetcher_test.clj similarity index 79% rename from common-app-lib/test/cmr/common_app/services/kms_fetcher_test.clj rename to system-int-test/test/cmr/system_int_test/misc/kms_fetcher_test.clj index b92d979bfd..4c1c57d783 100644 --- a/common-app-lib/test/cmr/common_app/services/kms_fetcher_test.clj +++ b/system-int-test/test/cmr/system_int_test/misc/kms_fetcher_test.clj @@ -1,13 +1,17 @@ -(ns cmr.common-app.services.kms-fetcher-test +(ns cmr.system-int-test.misc.kms-fetcher-test (:require [clojure.test :refer :all] [cmr.common.cache :as cache] + [cmr.common.cache.in-memory-cache :as mem-cache] [cmr.common-app.services.kms-fetcher :as fetcher] - [cmr.indexer.system :as idx-sys] + [cmr.transmit.config :as transmit-config] [cmr.transmit.kms :as trans-kms])) (deftest validate-getting-kms-keywords-test - (let [context {:system (idx-sys/create-system)} + (let [sys (transmit-config/system-with-connections + {:caches {fetcher/kms-cache-key (mem-cache/create-in-memory-cache)}} + [:kms]) + context {:system sys} kms-cache (cache/context->cache context fetcher/kms-cache-key) _ (#'fetcher/refresh-kms-cache context) kms-map (cache/get-value kms-cache fetcher/kms-cache-key)]