CMR-9497 Moving KMS cache straight to redis

bootstrap-app/src/cmr/bootstrap/system.clj

@@ -78,7 +78,8 @@
              :caches {acl/token-imp-cache-key (acl/create-token-imp-cache)
                       kf/kms-cache-key (kf/create-kms-cache)
                       common-health/health-cache-key (common-health/create-health-cache)}
-             :scheduler (jobs/create-scheduler `system-holder [jvm-info/log-jvm-statistics-job])
+             :scheduler (jobs/create-scheduler `system-holder [jvm-info/log-jvm-statistics-job
+                                                               (kf/refresh-kms-cache-job "bootstrap-kms-cache-refresh")])
              :queue-broker queue-broker}]
     (transmit-config/system-with-connections sys [:metadata-db :echo-rest :kms
                                                   :indexer :access-control])))

common-app-lib/src/cmr/common_app/services/kms_fetcher.clj

@@ -52,24 +52,12 @@
   "The key used to store the KMS cache in the system cache map."
   :kms)
 
-(defconfig kms-cache-consistent-timeout-seconds
-  "The number of seconds between when the KMS cache should check with redis for consistence"
-  {:default 3600
-   :type Long})
-
 (defn create-kms-cache
   "Used to create the cache that will be used for caching KMS keywords. All applications caching
   KMS keywords should use the same fallback cache to ensure functionality even if GCMD KMS becomes
   unavailable."
   []
-  (stl-cache/create-single-thread-lookup-cache
-    (fallback-cache/create-fallback-cache
-      (consistent-cache/create-consistent-cache
-       {:hash-timeout-seconds (kms-cache-consistent-timeout-seconds)})
-      (deflating-cache/create-deflating-cache
-        (redis-cache/create-redis-cache)
-        kms-lookup/create-kms-index
-        kms-lookup/deflate))))
+  (redis-cache/create-redis-cache))
 
 (defn- fetch-gcmd-keywords-map
   "Calls GCMD KMS endpoints to retrieve the keywords. Response is a map structured in the same way
@@ -112,9 +100,8 @@
 
 (defn refresh-kms-cache-job
   "The singleton job that refreshes the KMS cache. The keywords are infrequently updated by the
-  GCMD team. They update the CSV file which we read from every 6 hours. I arbitrarily chose 2 hours
-  so that we are never more than 8 hours from the time a keyword is updated."
+  GCMD team, usually once a week."
   [job-key]
   {:job-type RefreshKmsCacheJob
    :job-key job-key
-   :interval 7200})
+   :daily-at-hour-and-minute [02 00]})

common-app-lib/src/cmr/common_app/services/kms_lookup.clj

@@ -146,12 +146,6 @@
             :locations-index location-lookup-map
             :measurement-index measurement-lookup-map})))
 
-(defn deflate
-  "Takes a KMS index and returns a minimal version to store more efficiently and in a way that
-  the index can be recreated in a way to reduce the memory usage."
-  [kms-index]
-  (dissoc kms-index :short-name-index :umm-c-index :locations-index :measurement-index))
-
 (defn lookup-by-short-name
   "Takes a kms-index, the keyword scheme, and a short name and returns the full KMS hierarchy for
   that short name. Comparison is made case insensitively."

common-app-lib/test/cmr/common_app/test/services/kms_lookup.clj

@@ -43,9 +43,6 @@
   "KMS index to use for all of the tests which creates the KMS index from the sample map."
   (kms-lookup/create-kms-index sample-map))
 
-(deftest roundtrip-inflate-and-deflate-index-test
-  (is (= sample-map (kms-lookup/deflate (kms-lookup/create-kms-index sample-map)))))
-
 (deftest lookup-by-location-string-test
   (testing "Full location hierarchy is returned"
     (is (= {:category "CONTINENT" :type "AFRICA" :subregion-1 "CENTRAL AFRICA"

dev-system/src/cmr/dev_system/system.clj

@@ -276,7 +276,7 @@
              {:mock-echo echo-component
               :access-control (create-access-control-app queue-broker)
               :metadata-db (create-metadata-db-app db-component queue-broker)
-              :bootstrap (when-not db-component (create-bootstrap-app queue-broker))
+              :bootstrap (create-bootstrap-app queue-broker)
               :indexer (create-indexer-app queue-broker)
               :ingest (create-ingest-app db queue-broker)
               :search (create-search-app db-component queue-broker)

indexer-app/src/cmr/indexer/system.clj

@@ -72,7 +72,6 @@
              :scheduler (jobs/create-scheduler
                          `system-holder
                          [(af/refresh-acl-cache-job "indexer-acl-cache-refresh")
-                          (kf/refresh-kms-cache-job "indexer-kms-cache-refresh")
                           jvm-info/log-jvm-statistics-job
                           (cache-info/create-log-cache-info-job "indexer")])
              :queue-broker (queue-broker/create-queue-broker (config/queue-config))}]

search-app/src/cmr/search/system.clj

@@ -121,9 +121,6 @@
                       metadata-transformer/xsl-transformer-cache-name (mem-cache/create-in-memory-cache)
                       acl/token-imp-cache-key (acl/create-token-imp-cache)
                       acl/token-pc-cache-key (acl/create-token-pc-cache)
-                      ;; Note that search does not have a job to refresh the KMS cache. The indexer
-                      ;; already refreshes the cache. Since we use a consistent cache, the search
-                      ;; application will also pick up the updated KMS keywords.
                       launchpad-user-cache/launchpad-user-cache-key (launchpad-user-cache/create-launchpad-user-cache)
                       urs/urs-cache-key (urs/create-urs-cache)
                       kf/kms-cache-key (kf/create-kms-cache)

common-app-lib/test/cmr/common_app/services/kms_fetcher_test.clj → system-int-test/test/cmr/system_int_test/misc/kms_fetcher_test.clj

@@ -1,13 +1,17 @@
-(ns cmr.common-app.services.kms-fetcher-test
+(ns cmr.system-int-test.misc.kms-fetcher-test
   (:require
    [clojure.test :refer :all]
    [cmr.common.cache :as cache]
+   [cmr.common.cache.in-memory-cache :as mem-cache]
    [cmr.common-app.services.kms-fetcher :as fetcher]
-   [cmr.indexer.system :as idx-sys]
+   [cmr.transmit.config :as transmit-config]
    [cmr.transmit.kms :as trans-kms]))
 
 (deftest validate-getting-kms-keywords-test
-  (let [context {:system (idx-sys/create-system)}
+  (let [sys (transmit-config/system-with-connections
+             {:caches {fetcher/kms-cache-key (mem-cache/create-in-memory-cache)}}
+             [:kms])
+        context {:system sys}
         kms-cache (cache/context->cache context fetcher/kms-cache-key)
         _  (#'fetcher/refresh-kms-cache context)
         kms-map (cache/get-value kms-cache fetcher/kms-cache-key)]