From e4dc82a9b319625f79582730e173c8736d5a1ba6 Mon Sep 17 00:00:00 2001 From: Edmo Vamerlatti Costa <11836452+edmocosta@users.noreply.github.com> Date: Thu, 5 Jan 2023 15:46:35 +0100 Subject: [PATCH] Add setting to disable the GeoIP database downloader (#14823) This commit adds a new logstash.yml setting "xpack.geoip.downloader.enabled" to disable the GeoIP databases auto-update feature. When disabled, Logstash will fall back to the CC database license indefinitely and delete any previously downloaded EULA databases. Closes #14724 --- config/logstash.yml | 1 + docker/data/logstash/env2yaml/env2yaml.go | 1 + x-pack/lib/filters/geoip/database_manager.rb | 61 ++++++++++++++++--- x-pack/lib/filters/geoip/database_metadata.rb | 4 ++ x-pack/lib/filters/geoip/extension.rb | 1 + .../filters/geoip/database_manager_spec.rb | 56 ++++++++++++++++- 6 files changed, 112 insertions(+), 12 deletions(-) diff --git a/config/logstash.yml b/config/logstash.yml index 2391f92cdbd..f7560e1400d 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -375,3 +375,4 @@ # X-Pack GeoIP plugin # https://www.elastic.co/guide/en/logstash/current/plugins-filters-geoip.html#plugins-filters-geoip-manage_update #xpack.geoip.download.endpoint: "https://geoip.elastic.co/v1/database" +#xpack.geoip.downloader.enabled: true diff --git a/docker/data/logstash/env2yaml/env2yaml.go b/docker/data/logstash/env2yaml/env2yaml.go index 7038a85c70e..09484a36502 100644 --- a/docker/data/logstash/env2yaml/env2yaml.go +++ b/docker/data/logstash/env2yaml/env2yaml.go @@ -145,6 +145,7 @@ func normalizeSetting(setting string) (string, error) { "xpack.management.elasticsearch.ssl.keystore.path", "xpack.management.elasticsearch.ssl.keystore.password", "xpack.geoip.download.endpoint", + "xpack.geoip.downloader.enabled", "cloud.id", "cloud.auth", } diff --git a/x-pack/lib/filters/geoip/database_manager.rb b/x-pack/lib/filters/geoip/database_manager.rb index 469dc32fe64..a09d76876f9 100644 --- a/x-pack/lib/filters/geoip/database_manager.rb +++ b/x-pack/lib/filters/geoip/database_manager.rb @@ -128,7 +128,7 @@ def execute_download_job logger.error(e.message, error_details(e, logger)) ensure check_age - clean_up_database + clean_up_database(@metadata.dirnames) database_metric.update_download_stats(success_cnt) ThreadContext.put("pipeline.id", pipeline_id) @@ -185,9 +185,9 @@ def check_age(database_types = DB_TYPES) end end - # Clean up directories which are not mentioned in metadata and not CC database - def clean_up_database - protected_dirnames = (@metadata.dirnames + [CC]).uniq + # Clean up directories which are not mentioned in the excluded_dirnames and not CC database + def clean_up_database(excluded_dirnames = []) + protected_dirnames = (excluded_dirnames + [CC]).uniq existing_dirnames = ::Dir.children(get_data_dir_path) .select { |f| ::File.directory? ::File.join(get_data_dir_path, f) } @@ -214,6 +214,42 @@ def trigger_download end end + def trigger_cc_database_fallback + return if @triggered + + @trigger_lock.synchronize do + return if @triggered + + logger.info "The MaxMind EULA requires users to update the GeoIP databases within 30 days following the release of the update. " \ + "By setting `xpack.geoip.downloader.enabled` value in logstash.yml to `false`, any previously downloaded version of the database " \ + "are destroyed and replaced by the MaxMind Creative Commons license database." + + setup_cc_database + @triggered = true + end + end + + def setup_cc_database + prepare_cc_db + delete_eula_databases + DatabaseMetadata.new.delete + end + + def delete_eula_databases + begin + clean_up_database + rescue => e + details = error_details(e, logger) + details[:databases_path] = get_data_dir_path + logger.error "Failed to delete existing MaxMind EULA databases. To be compliant with the MaxMind EULA, you must "\ + "manually destroy any downloaded version of the EULA databases.", details + end + end + + def database_auto_update? + LogStash::SETTINGS.get("xpack.geoip.downloader.enabled") + end + public # @note this method is expected to execute on a separate thread @@ -226,14 +262,19 @@ def database_update_check def subscribe_database_path(database_type, database_path, geoip_plugin) if database_path.nil? - trigger_download + if database_auto_update? + trigger_download - logger.info "By not manually configuring a database path with `database =>`, you accepted and agreed MaxMind EULA. "\ - "For more details please visit https://www.maxmind.com/en/geolite2/eula" if @states[database_type].is_eula + logger.info "By not manually configuring a database path with `database =>`, you accepted and agreed MaxMind EULA. "\ + "For more details please visit https://www.maxmind.com/en/geolite2/eula" if @states[database_type].is_eula - @states[database_type].plugins.push(geoip_plugin) unless @states[database_type].plugins.member?(geoip_plugin) - @trigger_lock.synchronize do - @states[database_type].database_path + @states[database_type].plugins.push(geoip_plugin) unless @states[database_type].plugins.member?(geoip_plugin) + @trigger_lock.synchronize do + @states[database_type].database_path + end + else + trigger_cc_database_fallback + get_db_path(database_type, CC) end else logger.info "GeoIP database path is configured manually so the plugin will not check for update. "\ diff --git a/x-pack/lib/filters/geoip/database_metadata.rb b/x-pack/lib/filters/geoip/database_metadata.rb index caa6013b453..a2f70d4ef52 100644 --- a/x-pack/lib/filters/geoip/database_metadata.rb +++ b/x-pack/lib/filters/geoip/database_metadata.rb @@ -94,6 +94,10 @@ def exist? file_exist?(@metadata_path) end + def delete + ::File.delete(@metadata_path) if exist? + end + class Column DATABASE_TYPE = 0 CHECK_AT = 1 diff --git a/x-pack/lib/filters/geoip/extension.rb b/x-pack/lib/filters/geoip/extension.rb index f2f7f5d4540..778195f77c6 100644 --- a/x-pack/lib/filters/geoip/extension.rb +++ b/x-pack/lib/filters/geoip/extension.rb @@ -12,6 +12,7 @@ def additionals_settings(settings) require "logstash/runner" logger.trace("Registering additional geoip settings") settings.register(LogStash::Setting::NullableString.new("xpack.geoip.download.endpoint")) + settings.register(LogStash::Setting::Boolean.new("xpack.geoip.downloader.enabled", true)) rescue => e logger.error("Cannot register new settings", :message => e.message, :backtrace => e.backtrace) raise e diff --git a/x-pack/spec/filters/geoip/database_manager_spec.rb b/x-pack/spec/filters/geoip/database_manager_spec.rb index 6130c7387c9..019a138799f 100644 --- a/x-pack/spec/filters/geoip/database_manager_spec.rb +++ b/x-pack/spec/filters/geoip/database_manager_spec.rb @@ -131,6 +131,7 @@ def expect_initial_download_metric(c) allow(mock_geoip_plugin).to receive_message_chain('execution_context.pipeline_id').and_return('pipeline_1', 'pipeline_2') expect(mock_geoip_plugin).to receive(:update_filter).with(:update, instance_of(String)).at_least(:twice) expect(mock_metadata).to receive(:update_timestamp).never + expect(mock_metadata).to receive(:dirnames) expect(db_manager).to receive(:check_age) expect(db_manager).to receive(:clean_up_database) @@ -147,6 +148,7 @@ def expect_initial_download_metric(c) expect(mock_download_manager).to receive(:fetch_database).and_return([invalid_city_fetch, valid_asn_fetch]) expect(mock_metadata).to receive(:save_metadata).with(ASN, second_dirname, true).at_least(:once) expect(mock_metadata).to receive(:update_timestamp).never + expect(mock_metadata).to receive(:dirnames) expect(db_manager).to receive(:check_age) expect(db_manager).to receive(:clean_up_database) @@ -162,6 +164,7 @@ def expect_initial_download_metric(c) expect(mock_download_manager).to receive(:fetch_database).and_return([valid_asn_fetch]) expect(mock_metadata).to receive(:save_metadata).with(ASN, second_dirname, true).at_least(:once) expect(mock_metadata).to receive(:update_timestamp).with(CITY).at_least(:once) + expect(mock_metadata).to receive(:dirnames) expect(db_manager).to receive(:check_age) expect(db_manager).to receive(:clean_up_database) @@ -177,6 +180,7 @@ def expect_initial_download_metric(c) expect(mock_download_manager).to receive(:fetch_database).and_return([]) expect(mock_metadata).to receive(:save_metadata).never expect(mock_metadata).to receive(:update_timestamp).at_least(:twice) + expect(mock_metadata).to receive(:dirnames) expect(db_manager).to receive(:check_age) expect(db_manager).to receive(:clean_up_database) @@ -193,6 +197,7 @@ def expect_initial_download_metric(c) expect(db_manager).to receive(:check_age) expect(db_manager).to receive(:clean_up_database) expect(mock_metadata).to receive(:save_metadata).never + expect(mock_metadata).to receive(:dirnames) db_manager.send(:execute_download_job) @@ -331,9 +336,8 @@ def expect_healthy_database_metric(c) it "should delete file which is not in metadata" do FileUtils.touch [asn00, city00, asn02, city02] - expect(mock_metadata).to receive(:dirnames).and_return([dirname]) - db_manager.send(:clean_up_database) + db_manager.send(:clean_up_database, [dirname]) [asn02, city02].each { |file_path| expect(::File.exist?(file_path)).to be_falsey } [get_dir_path(CC), asn00, city00].each { |file_path| expect(::File.exist?(file_path)).to be_truthy } @@ -368,6 +372,54 @@ def expect_healthy_database_metric(c) expect(path).to be_nil end end + + context "downloader setting" do + context "enabled" do + it "should trigger database download" do + allow(db_manager).to receive(:trigger_download) + db_manager.subscribe_database_path(CITY, nil, mock_geoip_plugin) + expect(db_manager).to have_received(:trigger_download) + end + end + + context "disabled" do + it "should return cc database when database path is nil" do + allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.downloader.enabled").and_return(false) + allow(mock_metadata).to receive(:delete).once + + path = db_manager.subscribe_database_path(CITY, nil, mock_geoip_plugin) + + expect(path).to eq(default_city_db_path) + end + + it "should delete eula databases and metadata when database path is nil" do + allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.downloader.enabled").and_return(false) + allow(mock_metadata).to receive(:delete).once + + eula_db_dirname = get_dir_path("foo") + FileUtils.mkdir_p(eula_db_dirname) + rewrite_temp_metadata(metadata_path, [ ["City","1620246514","","foo",true], + ["ASN","1620246514","","foo",true]]) + + path = db_manager.subscribe_database_path(CITY, nil, mock_geoip_plugin) + + expect(path).to eq(default_city_db_path) + expect(File).not_to exist(eula_db_dirname) + end + + it "should return user input database path" do + allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.downloader.enabled").and_return(false) + allow(db_manager).to receive(:trigger_download) + allow(db_manager).to receive(:trigger_cc_database_fallback) + + path = db_manager.subscribe_database_path(CITY, "path/to/db", mock_geoip_plugin) + + expect(db_manager).not_to have_received(:trigger_download) + expect(db_manager).not_to have_received(:trigger_cc_database_fallback) + expect(path).to eq("path/to/db") + end + end + end end context "unsubscribe" do