Add setting to disable the GeoIP database downloader (#14823)

This commit adds a new logstash.yml setting "xpack.geoip.downloader.enabled" to disable the GeoIP databases auto-update feature. When disabled, Logstash will fall back to the CC database license indefinitely and delete any previously downloaded EULA databases.

Closes #14724

config/logstash.yml

@@ -375,3 +375,4 @@
 # X-Pack GeoIP plugin
 # https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html#plugins-filters-geoip-manage_update
 #xpack.geoip.download.endpoint: "https://geoip.elastic.co/v1/database"
+#xpack.geoip.downloader.enabled: true

docker/data/logstash/env2yaml/env2yaml.go

@@ -145,6 +145,7 @@ func normalizeSetting(setting string) (string, error) {
 		"xpack.management.elasticsearch.ssl.keystore.path",
 		"xpack.management.elasticsearch.ssl.keystore.password",
 		"xpack.geoip.download.endpoint",
+		"xpack.geoip.downloader.enabled",
 		"cloud.id",
 		"cloud.auth",
 	}

x-pack/lib/filters/geoip/database_manager.rb

@@ -128,7 +128,7 @@ def execute_download_job
       logger.error(e.message, error_details(e, logger))
     ensure
       check_age
-      clean_up_database
+      clean_up_database(@metadata.dirnames)
       database_metric.update_download_stats(success_cnt)
 
       ThreadContext.put("pipeline.id", pipeline_id)
@@ -185,9 +185,9 @@ def check_age(database_types = DB_TYPES)
     end
   end
 
-  # Clean up directories which are not mentioned in metadata and not CC database
-  def clean_up_database
-    protected_dirnames = (@metadata.dirnames + [CC]).uniq
+  # Clean up directories which are not mentioned in the excluded_dirnames and not CC database
+  def clean_up_database(excluded_dirnames = [])
+    protected_dirnames = (excluded_dirnames + [CC]).uniq
     existing_dirnames = ::Dir.children(get_data_dir_path)
                              .select { |f| ::File.directory? ::File.join(get_data_dir_path, f) }
 
@@ -214,6 +214,42 @@ def trigger_download
     end
   end
 
+  def trigger_cc_database_fallback
+    return if @triggered
+
+    @trigger_lock.synchronize do
+      return if @triggered
+
+      logger.info "The MaxMind EULA requires users to update the GeoIP databases within 30 days following the release of the update. " \
+                  "By setting `xpack.geoip.downloader.enabled` value in logstash.yml to `false`, any previously downloaded version of the database " \
+                  "are destroyed and replaced by the MaxMind Creative Commons license database."
+
+      setup_cc_database
+      @triggered = true
+    end
+  end
+
+  def setup_cc_database
+    prepare_cc_db
+    delete_eula_databases
+    DatabaseMetadata.new.delete
+  end
+
+  def delete_eula_databases
+    begin
+      clean_up_database
+    rescue => e
+      details = error_details(e, logger)
+      details[:databases_path] = get_data_dir_path
+      logger.error "Failed to delete existing MaxMind EULA databases. To be compliant with the MaxMind EULA, you must "\
+                   "manually destroy any downloaded version of the EULA databases.", details
+    end
+  end
+
+  def database_auto_update?
+    LogStash::SETTINGS.get("xpack.geoip.downloader.enabled")
+  end
+
   public
 
   # @note this method is expected to execute on a separate thread
@@ -226,14 +262,19 @@ def database_update_check
 
   def subscribe_database_path(database_type, database_path, geoip_plugin)
     if database_path.nil?
-      trigger_download
+      if database_auto_update?
+        trigger_download
 
-      logger.info "By not manually configuring a database path with `database =>`, you accepted and agreed MaxMind EULA. "\
-                  "For more details please visit https://www.maxmind.com/en/geolite2/eula" if @states[database_type].is_eula
+        logger.info "By not manually configuring a database path with `database =>`, you accepted and agreed MaxMind EULA. "\
+                    "For more details please visit https://www.maxmind.com/en/geolite2/eula" if @states[database_type].is_eula
 
-      @states[database_type].plugins.push(geoip_plugin) unless @states[database_type].plugins.member?(geoip_plugin)
-      @trigger_lock.synchronize do
-        @states[database_type].database_path
+        @states[database_type].plugins.push(geoip_plugin) unless @states[database_type].plugins.member?(geoip_plugin)
+        @trigger_lock.synchronize do
+          @states[database_type].database_path
+        end
+      else
+        trigger_cc_database_fallback
+        get_db_path(database_type, CC)
       end
     else
       logger.info "GeoIP database path is configured manually so the plugin will not check for update. "\

x-pack/lib/filters/geoip/database_metadata.rb

@@ -94,6 +94,10 @@ def exist?
     file_exist?(@metadata_path)
   end
 
+  def delete
+    ::File.delete(@metadata_path) if exist?
+  end
+
   class Column
     DATABASE_TYPE = 0
     CHECK_AT      = 1

x-pack/lib/filters/geoip/extension.rb

@@ -12,6 +12,7 @@ def additionals_settings(settings)
       require "logstash/runner"
       logger.trace("Registering additional geoip settings")
       settings.register(LogStash::Setting::NullableString.new("xpack.geoip.download.endpoint"))
+      settings.register(LogStash::Setting::Boolean.new("xpack.geoip.downloader.enabled", true))
     rescue => e
       logger.error("Cannot register new settings", :message => e.message, :backtrace => e.backtrace)
       raise e

x-pack/spec/filters/geoip/database_manager_spec.rb

@@ -131,6 +131,7 @@ def expect_initial_download_metric(c)
           allow(mock_geoip_plugin).to receive_message_chain('execution_context.pipeline_id').and_return('pipeline_1', 'pipeline_2')
           expect(mock_geoip_plugin).to receive(:update_filter).with(:update, instance_of(String)).at_least(:twice)
           expect(mock_metadata).to receive(:update_timestamp).never
+          expect(mock_metadata).to receive(:dirnames)
           expect(db_manager).to receive(:check_age)
           expect(db_manager).to receive(:clean_up_database)
 
@@ -147,6 +148,7 @@ def expect_initial_download_metric(c)
         expect(mock_download_manager).to receive(:fetch_database).and_return([invalid_city_fetch, valid_asn_fetch])
         expect(mock_metadata).to receive(:save_metadata).with(ASN, second_dirname, true).at_least(:once)
         expect(mock_metadata).to receive(:update_timestamp).never
+        expect(mock_metadata).to receive(:dirnames)
         expect(db_manager).to receive(:check_age)
         expect(db_manager).to receive(:clean_up_database)
 
@@ -162,6 +164,7 @@ def expect_initial_download_metric(c)
         expect(mock_download_manager).to receive(:fetch_database).and_return([valid_asn_fetch])
         expect(mock_metadata).to receive(:save_metadata).with(ASN, second_dirname, true).at_least(:once)
         expect(mock_metadata).to receive(:update_timestamp).with(CITY).at_least(:once)
+        expect(mock_metadata).to receive(:dirnames)
         expect(db_manager).to receive(:check_age)
         expect(db_manager).to receive(:clean_up_database)
 
@@ -177,6 +180,7 @@ def expect_initial_download_metric(c)
         expect(mock_download_manager).to receive(:fetch_database).and_return([])
         expect(mock_metadata).to receive(:save_metadata).never
         expect(mock_metadata).to receive(:update_timestamp).at_least(:twice)
+        expect(mock_metadata).to receive(:dirnames)
         expect(db_manager).to receive(:check_age)
         expect(db_manager).to receive(:clean_up_database)
 
@@ -193,6 +197,7 @@ def expect_initial_download_metric(c)
         expect(db_manager).to receive(:check_age)
         expect(db_manager).to receive(:clean_up_database)
         expect(mock_metadata).to receive(:save_metadata).never
+        expect(mock_metadata).to receive(:dirnames)
 
         db_manager.send(:execute_download_job)
 
@@ -331,9 +336,8 @@ def expect_healthy_database_metric(c)
 
       it "should delete file which is not in metadata" do
         FileUtils.touch [asn00, city00, asn02, city02]
-        expect(mock_metadata).to receive(:dirnames).and_return([dirname])
 
-        db_manager.send(:clean_up_database)
+        db_manager.send(:clean_up_database, [dirname])
 
         [asn02, city02].each { |file_path| expect(::File.exist?(file_path)).to be_falsey }
         [get_dir_path(CC), asn00, city00].each { |file_path| expect(::File.exist?(file_path)).to be_truthy }
@@ -368,6 +372,54 @@ def expect_healthy_database_metric(c)
           expect(path).to be_nil
         end
       end
+
+      context "downloader setting" do
+        context "enabled" do
+          it "should trigger database download" do
+            allow(db_manager).to receive(:trigger_download)
+            db_manager.subscribe_database_path(CITY, nil, mock_geoip_plugin)
+            expect(db_manager).to have_received(:trigger_download)
+          end
+        end
+
+        context "disabled" do
+          it "should return cc database when database path is nil" do
+            allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.downloader.enabled").and_return(false)
+            allow(mock_metadata).to receive(:delete).once
+
+            path = db_manager.subscribe_database_path(CITY, nil, mock_geoip_plugin)
+
+            expect(path).to eq(default_city_db_path)
+          end
+
+          it "should delete eula databases and metadata when database path is nil" do
+            allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.downloader.enabled").and_return(false)
+            allow(mock_metadata).to receive(:delete).once
+
+            eula_db_dirname = get_dir_path("foo")
+            FileUtils.mkdir_p(eula_db_dirname)
+            rewrite_temp_metadata(metadata_path, [ ["City","1620246514","","foo",true],
+                                                   ["ASN","1620246514","","foo",true]])
+
+            path = db_manager.subscribe_database_path(CITY, nil, mock_geoip_plugin)
+
+            expect(path).to eq(default_city_db_path)
+            expect(File).not_to exist(eula_db_dirname)
+          end
+
+          it "should return user input database path" do
+            allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.downloader.enabled").and_return(false)
+            allow(db_manager).to receive(:trigger_download)
+            allow(db_manager).to receive(:trigger_cc_database_fallback)
+
+            path = db_manager.subscribe_database_path(CITY, "path/to/db", mock_geoip_plugin)
+
+            expect(db_manager).not_to have_received(:trigger_download)
+            expect(db_manager).not_to have_received(:trigger_cc_database_fallback)
+            expect(path).to eq("path/to/db")
+          end
+        end
+      end
     end
 
     context "unsubscribe" do