From 81ae9f83a24727dccb75711e6a70ec5745b47d40 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Mon, 16 Dec 2024 19:57:54 +0100 Subject: [PATCH] Rewrite ENC to use classes This rewrite uses classes to coherently group functionality. It attempts to separate out the actual client logic into a ForemanClient class. Then around that it wraps methods to push and watch as well as the ENC. Uploading is rewritten to always use the same logic, regardless of whether it's single threaded or parallel. Single threaded just spawns a single thread. This also has the advantage of being able to reuse HTTP connection instances, possibly leading to higher throughput since it doesn't have to establish fresh HTTP(S) connections for every request. --- files/enc.rb | 622 ++++++++++++++++++++++++--------------------------- 1 file changed, 295 insertions(+), 327 deletions(-) diff --git a/files/enc.rb b/files/enc.rb index 4b16178..c4951f1 100644 --- a/files/enc.rb +++ b/files/enc.rb @@ -7,349 +7,368 @@ # If --push-facts is given as the only arg, it uploads facts for all hosts and then exits. # Useful in scenarios where the ENC isn't used. +require 'etc' +require 'fileutils' +require 'json' +require 'net/http' +require 'net/https' require 'rbconfig' +require 'timeout' require 'yaml' -if RbConfig::CONFIG['host_os'] =~ /freebsd|dragonfly/i - $settings_file ||= '/usr/local/etc/puppet/foreman.yaml' -else - $settings_file ||= File.exist?('/etc/puppetlabs/puppet/foreman.yaml') ? '/etc/puppetlabs/puppet/foreman.yaml' : '/etc/puppet/foreman.yaml' -end +# TODO: still needed? +Encoding.default_external = Encoding::UTF_8 -SETTINGS = YAML.load_file($settings_file) +class FactUploadError < StandardError; end +class NodeRetrievalError < StandardError; end -# Default external encoding -if defined?(Encoding) - Encoding.default_external = Encoding::UTF_8 -end +class Settings + class << self + def settings_file + if RbConfig::CONFIG['host_os'] =~ /freebsd|dragonfly/i + '/usr/local/etc/puppet/foreman.yaml' + elsif File.exist?('/etc/puppetlabs/puppet/foreman.yaml') + '/etc/puppetlabs/puppet/foreman.yaml' + else + '/etc/puppet/foreman.yaml' + end + end -def url - SETTINGS[:url] || raise("Must provide URL in #{$settings_file}") -end + SETTINGS = new(settings_file) + end -def puppetdir - SETTINGS[:puppetdir] || raise("Must provide puppet base directory in #{$settings_file}") -end + def initialize(path) + @settings = YAML.load_file(path) + @settings.delete_if! { |_key, value| value.respond_to(:empty?) && value.empty? } + end -def puppetuser - SETTINGS[:puppetuser] || 'puppet' -end + def url + @settings[:url] || raise("Must provide URL in #{settings_file}") + end -def fact_extension - SETTINGS[:fact_extension] || 'yaml' -end + def puppet_dir + @settings[:puppetdir] || raise("Must provide puppet base directory in #{settings_file}") + end -def fact_directory - data_dir = fact_extension == 'yaml' ? 'yaml' : 'server_data' - File.join(puppetdir, data_dir, 'facts') -end + def puppet_user + @settings.fetch(:puppetuser, 'puppet') + end -def fact_file(certname) - File.join(fact_directory, "#{certname}.#{fact_extension}") -end + def fact_extension + @settings.fetch(:fact_extension, 'yaml') + end -def fact_files - Dir[File.join(fact_directory, "*.#{fact_extension}")] -end + def facts? + @settings[:facts] + end -def certname_from_filename(filename) - File.basename(filename, ".#{fact_extension}") -end + def ssl_ca + @settings[:ssl_ca] + end -def stat_file(certname) - FileUtils.mkdir_p "#{puppetdir}/yaml/foreman/" - "#{puppetdir}/yaml/foreman/#{certname}.yaml" -end + def ssl_cert + @settings[:ssl_cert] + end + + def ssl_key + @settings[:ssl_key] + end + + def stat_dir + File.join(Settings.puppet_dir, 'yaml', 'foreman') + end -def tsecs - SETTINGS[:timeout] || 10 + def timeout + @settings[:timeout] + end + + def threads + if @settings[:threads].to_i > 0 + @settings[:threads].to_i + else + require 'facter' + max(Facter.value(:processorcount).to_i, 1) + end + end end -def thread_count - return SETTINGS[:threads].to_i if not SETTINGS[:threads].nil? and SETTINGS[:threads].to_i > 0 - require 'facter' - processors = Facter.value(:processorcount).to_i - processors > 0 ? processors : 1 +class Facts + class << self + EXTENSION = Settings.fact_extension + + def directory + data_dir = EXTENSION == 'yaml' ? 'yaml' : 'server_data' + File.join(Settings.puppet_dir, data_dir, 'facts') + end + + def file(certname) + File.join(directory, "#{certname}.#{EXTENSION}") + end + + def files + Dir[File.join(directory, "*.#{EXTENSION}")] + end + + def certname_from_filename(filename) + File.basename(filename, ".#{EXTENSION}") + end + end end -class Http_Fact_Requests - include Enumerable +class Cache + INSTANCE = new(Settings.stat_dir) - def initialize - @results_array = [] + def initialize(directory) + @directory = directory end - def <<(val) - @results_array << val + def write(key, result) + FileUtils.mkdir_p(@directory) + File.write(path(key), result) end - def each(&block) - @results_array.each(&block) + def read(key) + File.read(path(key)) + rescue StandardError => e + raise "Unable to read from Cache file: #{e}" end - def pop - @results_array.pop + def fresh?(key, mtime) + File.stat(path(key)).mtime.utc >= mtime.utc + rescue Errno::ENOENT + false end -end -class FactUploadError < StandardError; end -class NodeRetrievalError < StandardError; end + private -require 'etc' -require 'net/http' -require 'net/https' -require 'fileutils' -require 'timeout' -begin - require 'json' -rescue LoadError - # Debian packaging guidelines state to avoid needing rubygems, so - # we only try to load it if the first require fails (for RPMs) - begin - require 'rubygems' rescue nil - require 'json' - rescue LoadError => e - puts "You need the `json` gem to use the Foreman ENC script" - # code 1 is already used below - exit 2 + def path(key) + File.join(@directory, "#{key}.yaml") end end -def parse_file(filename, mac_address_workaround = false) - case File.extname(filename) - when '.yaml' - data = File.read(filename) - quote_macs!(data) if mac_address_workaround && YAML.safe_load('22:22:22:22:22:22').is_a?(Integer) - YAML.safe_load(data.gsub(/\!ruby\/object.*$/,''), permitted_classes: [Symbol, Time]) - when '.json' - JSON.parse(File.read(filename)) - else - raise "Unknown extension for file '#{filename}'" +class ForemanClient + attr_reader :base_url + + def initialize(base_url) + @base_url = base_url end -end -def empty_values_hash?(facts_file) - puppet_facts = parse_file(facts_file) - puppet_facts['values'].empty? -end + def upload_facts(certname, filename = nil) + filename ||= Fact.file(certname) + begin + stat = File.stat(filename) + rescue Errno::ENOENT + warn "Fact file #{filename} does not exist" + end -def process_host_facts(certname) - f = fact_file(certname) - if File.size(f) != 0 - if empty_values_hash?(f) - puts "Empty values hash in fact file #{f}, not uploading" - return 0 + unless stat.size? + warn "Fact file #{filename} does not contain any facts" + return end - req = generate_fact_request(certname, f) - begin - upload_facts(certname, req) if req - return 0 - rescue => e - $stderr.puts "During fact upload occurred an exception: #{e}" - return 1 + cache_name = cache_key(certname) + return if Cache::INSTANCE.fresh?(cache_name, stat) + + unless (body = build_fact_body(certname, filename)) + warn "Empty values hash in fact file #{filename}, not uploading" + return end - else - $stderr.puts "Fact file #{f} does not contain any facts" - return 2 - end -end -def process_all_facts(http_requests) - fact_files.each do |f| - # Skip empty host fact files - if File.size(f) != 0 - if empty_values_hash?(f) - puts "Empty values hash in fact file #{f}, not uploading" - next - end + uri = URI.parse("#{base_url}/api/hosts/facts") + req = Net::HTTP::Post.new(uri.request_uri) + req.add_field('Accept', 'application/json,version=2') + req.content_type = 'application/json' + req.body = body.to_json - certname = certname_from_filename(f) - req = generate_fact_request(certname, f) - if http_requests - http_requests << [certname, req] - elsif req - upload_facts(certname, req) - end - else - $stderr.puts "Fact file #{f} does not contain any fact" + response = connection.request(req) + unless response.code.start_with?('2') + raise FactUploadError("#{certname}: During the fact upload the server responded with: #{response.code} #{response.message}. Error is ignored and the execution continues.") end + + Cache::INSTANCE.write(cache_name, "Facts from this host were last pushed to #{uri} at #{Time.now}\n") + rescue FactUploadError + raise + rescue StandardError => e + raise FactUploadError, "Could not send facts to Foreman: #{e}" end -end -def quote_macs! facts - # Adds single quotes to all unquoted mac addresses in the raw yaml fact string - # if they might otherwise be interpreted as base60 ints - facts.gsub!(/: ([0-5][0-9](:[0-5][0-9]){5})$/,": '\\1'") -end + def enc(certname) + uri = URI.parse("#{base_url}/node/#{certname}?format=yml") + req = Net::HTTP::Get.new(uri.request_uri) -def build_body(certname,filename) - puppet_facts = parse_file(filename, true) - hostname = puppet_facts['values']['fqdn'] || certname + response = connection.request(req) + unless response.code == '200' + raise NodeRetrievalError, "Error retrieving node #{certname}: #{response.class}\nCheck Foreman's /var/log/foreman/production.log for more information." + end + response.body + end - # if there is no environment in facts - # get it from node file ({puppetdir}/yaml/node/ - unless puppet_facts['values'].key?('environment') || puppet_facts['values'].key?('agent_specified_environment') - node_filename = filename.sub('/facts/', '/node/') - if File.exist?(node_filename) - node_data = parse_file(node_filename) + private - if node_data.key?('environment') - puppet_facts['values']['environment'] = node_data['environment'] - end - end + def cache_key(certname) + "#{certname}-push-facts" end - begin - require 'facter' - puppet_facts['values']['puppetmaster_fqdn'] = Facter.value('networking.fqdn').to_s - rescue LoadError - puppet_facts['values']['puppetmaster_fqdn'] = `hostname -f`.strip + def connection + @connection ||= initialize_http end - # filter any non-printable char from the value, if it is a String - puppet_facts['values'].each do |key, val| - if val.is_a? String - puppet_facts['values'][key] = val.scan(/[[:print:]]/).join + def initialize_http + uri = URI(base_url) + res = Net::HTTP.new(uri.host, uri.port) + res.open_timeout = Settings.timeout + res.read_timeout = Settings.timeout + if uri.scheme == 'https' + res.use_ssl = true + if (ssl_ca = Settings.ssl_ca) + res.ca_file = ssl_ca + res.verify_mode = OpenSSL::SSL::VERIFY_PEER + end + if (ssl_cert = Settings.ssl_cert) && (ssl_key = Settings.ssl_key) + res.cert = OpenSSL::X509::Certificate.new(File.read(ssl_cert)) + res.key = OpenSSL::PKey.new(File.read(ssl_key)) + end end + res end - {'facts' => puppet_facts['values'], 'name' => hostname, 'certname' => certname} -end + def build_fact_body(certname, filename) + puppet_facts = parse_file(filename, mac_address_workaround: true) + return if puppet_facts['values'].empty? -def initialize_http(uri) - res = Net::HTTP.new(uri.host, uri.port) - res.open_timeout = SETTINGS[:timeout] - res.read_timeout = SETTINGS[:timeout] - res.use_ssl = uri.scheme == 'https' - if res.use_ssl? - if SETTINGS[:ssl_ca] && !SETTINGS[:ssl_ca].empty? - res.ca_file = SETTINGS[:ssl_ca] - res.verify_mode = OpenSSL::SSL::VERIFY_PEER - else - res.verify_mode = OpenSSL::SSL::VERIFY_NONE - end - if SETTINGS[:ssl_cert] && !SETTINGS[:ssl_cert].empty? && SETTINGS[:ssl_key] && !SETTINGS[:ssl_key].empty? - res.cert = OpenSSL::X509::Certificate.new(File.read(SETTINGS[:ssl_cert])) - res.key = OpenSSL::PKey::RSA.new(File.read(SETTINGS[:ssl_key]), nil) + # if there is no environment in facts + # get it from node file ({Settings.puppet_dir}/yaml/node/ + unless puppet_facts['values'].key?('environment') || puppet_facts['values'].key?('agent_specified_environment') + node_filename = filename.sub('/facts/', '/node/') + if File.exist?(node_filename) + node_data = parse_file(node_filename) + + if node_data.key?('environment') + puppet_facts['values']['environment'] = node_data['environment'] + end + end end - end - res -end -def generate_fact_request(certname, filename) - # Temp file keeping the last run time - stat = stat_file("#{certname}-push-facts") - last_run = File.exist?(stat) ? File.stat(stat).mtime.utc : Time.now - 365*24*60*60 - last_fact = File.exist?(filename) ? File.stat(filename).mtime.utc : Time.at(0) - if last_fact > last_run begin - uri = URI.parse("#{url}/api/hosts/facts") - req = Net::HTTP::Post.new(uri.request_uri) - req.add_field('Accept', 'application/json,version=2' ) - req.content_type = 'application/json' - req.body = build_body(certname, filename).to_json - req - rescue => e - raise "Could not generate facts for Foreman: #{e}" + require 'facter' + puppet_facts['values']['puppetmaster_fqdn'] = Facter.value(:fqdn).to_s + rescue LoadError + puppet_facts['values']['puppetmaster_fqdn'] = `hostname -f`.strip end - end -end -def cache(certname, result) - File.open(stat_file(certname), 'w') {|f| f.write(result) } -end + # filter any non-printable char from the value + puppet_facts['values'].transform_values! { |val| val.is_a?(String) ? val.scan(/[[:print:]]/).join : val } -def read_cache(certname) - File.read(stat_file(certname)) -rescue => e - raise "Unable to read from Cache file: #{e}" -end + facts = puppet_facts['values'] -def enc(certname) - uri = URI.parse("#{url}/node/#{certname}?format=yml") - req = Net::HTTP::Get.new(uri.request_uri) - initialize_http(uri).start do |http| - response = http.request(req) + { + 'facts' => facts, + 'name' => facts.dig('networking', 'fqdn') || certname, + 'certname' => certname, + } + end - unless response.code == "200" - raise NodeRetrievalError, "Error retrieving node #{certname}: #{response.class}\nCheck Foreman's /var/log/foreman/production.log for more information." + def parse_file(filename, mac_address_workaround: false) + case File.extname(filename) + when '.yaml' + data = File.read(filename) + quote_macs!(data) if mac_address_workaround && YAML.safe_load('22:22:22:22:22:22').is_a?(Integer) + YAML.safe_load(data.gsub(/\!ruby\/object.*$/,''), permitted_classes: [Symbol, Time]) + when '.json' + JSON.parse(File.read(filename)) + else + raise "Unknown extension for file '#{filename}'" end - response.body + end + + def quote_macs!(facts) + # Adds single quotes to all unquoted mac addresses in the raw yaml fact string + # if they might otherwise be interpreted as base60 ints + facts.gsub!(/: ([0-5][0-9](:[0-5][0-9]){5})$/, ": '\\1'") end end -def upload_facts(certname, req) - return nil if req.nil? - uri = URI.parse("#{url}/api/hosts/facts") - begin - initialize_http(uri).start do |http| - response = http.request(req) - if response.code.start_with?('2') - cache("#{certname}-push-facts", "Facts from this host were last pushed to #{uri} at #{Time.now}\n") - else - $stderr.puts "#{certname}: During the fact upload the server responded with: #{response.code} #{response.message}. Error is ignored and the execution continues." - $stderr.puts response.body - end +def enc(certname, strip_environment:) + Timeout.timeout(Settings.timeout || 10) do + # send facts to Foreman - enable 'facts' setting to activate + # if you use this option below, make sure that you don't send facts to foreman via the rake task or push facts alternatives. + client = ForemanClient.new(url) + client.upload_facts(certname) if Settings.facts? + result = client.enc(certname) + if strip_environment + require 'yaml' + yaml = YAML.safe_load(result) + yaml.delete('environment') + # Always reset the result to back to clean yaml on our end + result = yaml.to_yaml end - rescue => e - $stderr.puts "During fact upload occured an exception: #{e}" - raise FactUploadError, "Could not send facts to Foreman: #{e}" + Cache::INSTANCE.write(certname, result) + result end +rescue Timeout::Error, SocketError, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, NodeRetrievalError, FactUploadError => e + warn "Serving cached ENC: #{e}" + Cache::INSTANCE.read(certname) end -def upload_facts_parallel(http_fact_requests, wait = true) - t = thread_count.times.map { - Thread.new(http_fact_requests) do |fact_requests| - while factref = fact_requests.pop - certname = factref[0] - httpobj = factref[1] - if httpobj - upload_facts(certname, httpobj) +def push_facts(certname:, threads:, watch:) + pending = Queue.new + + watchers = threads.times do + Thread.new do + client = ForemanClient.new(url) + + while (certname, filename = pending.pop) + client.upload_facts(certname, filename) end end + end + + # TODO: watch and single hostname is inconsistent + if certname + pending << [certname, nil] + else + Facts.files.each do |filename| + certname = Facts.certname_from_filename(filename) + pending << [certname, filename] end - } - if wait - t.each(&:join) end + + watch_facts(pending) if watch +ensure + pending&.close + watchers&.map(&:join) end -def watch_and_send_facts(parallel) +def watch_facts(queue) begin require 'inotify' rescue LoadError - puts "You need the `ruby-inotify` (not inotify!) gem to watch for fact updates" + warn 'You need the `ruby-inotify` (not inotify!) gem to watch for fact updates' exit 2 end watch_descriptors = [] - pending = [] - threads = thread_count - last_send = Time.now inotify_limit = `sysctl fs.inotify.max_user_watches`.gsub(/[^\d]/, '').to_i inotify = Inotify.new - fact_dir = fact_directory + fact_dir = Facts.directory # actually we need only MOVED_TO events because puppet uses File.rename after tmp file created and flushed. # see lib/puppet/util.rb near line 469 - inotify.add_watch(fact_dir, Inotify::CREATE | Inotify::MOVED_TO ) + inotify.add_watch(fact_dir, Inotify::CREATE | Inotify::MOVED_TO) - files = fact_files + files = Facts.files if files.length > inotify_limit - puts "Looks like your inotify watch limit is #{inotify_limit} but you are asking to watch at least #{files.length} fact files." - puts "Increase the watch limit via the system tunable fs.inotify.max_user_watches, exiting." + warn "Looks like your inotify watch limit is #{inotify_limit} but you are asking to watch at least #{files.length} fact files." + warn 'Increase the watch limit via the system tunable fs.inotify.max_user_watches, exiting.' exit 2 end files.each do |f| - begin - watch_descriptors[inotify.add_watch(f, Inotify::CLOSE_WRITE)] = f - end + watch_descriptors[inotify.add_watch(f, Inotify::CLOSE_WRITE)] = f end inotify.each_event do |ev| @@ -363,108 +382,57 @@ def watch_and_send_facts(parallel) add_watch = true end - if File.extname(fn) != ".#{fact_extension}" - next - end + next if File.extname(fn) != ".#{Settings.fact_extension}" if add_watch || (ev.mask & Inotify::ONESHOT) watch_descriptors[inotify.add_watch(fn, Inotify::CLOSE_WRITE)] = fn end - if fn - certname = certname_from_filename(fn) - req = generate_fact_request certname, fn - if parallel - pending << [certname,req] - else - upload_facts(certname,req) - end - end - if parallel && (pending.length >= threads || ((last_send + 5) < Time.now)) - if pending.length > 0 - upload_facts_parallel(pending, false) - pending = [] - end - last_send = Time.now - end + queue << [Facts.certname_from_filename(fn), fn] end end +def run_as_user(username) + Process::GID.change_privilege(Etc.getgrnam(username).gid) unless Etc.getpwuid.name == username + Process::UID.change_privilege(Etc.getpwnam(username).uid) unless Etc.getpwuid.name == username + # Facter (in Settings.threads) tries to read from $HOME, which is still /root after the UID change + ENV['HOME'] = Etc.getpwnam(username).dir + # Change CWD to the determined home directory before continuing to make + # sure we don't reside in /root or anywhere else we don't have access + # permissions + Dir.chdir ENV['HOME'] +end + # Actual code starts here -if __FILE__ == $0 then +if __FILE__ == $0 # Setuid to puppet user if we can begin - Process::GID.change_privilege(Etc.getgrnam(puppetuser).gid) unless Etc.getpwuid.name == puppetuser - Process::UID.change_privilege(Etc.getpwnam(puppetuser).uid) unless Etc.getpwuid.name == puppetuser - # Facter (in thread_count) tries to read from $HOME, which is still /root after the UID change - ENV['HOME'] = Etc.getpwnam(puppetuser).dir - # Change CWD to the determined home directory before continuing to make - # sure we don't reside in /root or anywhere else we don't have access - # permissions - Dir.chdir ENV['HOME'] - rescue - $stderr.puts "cannot switch to user #{puppetuser}, continuing as '#{Etc.getpwuid.name}'" + run_as_user(Settings.puppet_user) + rescue StandardError + warn "cannot switch to user #{puppet_user}, continuing as '#{Etc.getpwuid.name}'" end begin - no_env = ARGV.delete("--no-environment") - watch = ARGV.delete("--watch-facts") - push_facts_parallel = ARGV.delete("--push-facts-parallel") - push_facts = ARGV.delete("--push-facts") - if watch && ! ( push_facts || push_facts_parallel ) - raise "Cannot watch for facts without specifying --push-facts or --push-facts-parallel" - end - if push_facts + no_env = ARGV.delete('--no-environment') + watch = ARGV.delete('--watch-facts') + push_facts_parallel = ARGV.delete('--push-facts-parallel') + push_facts = ARGV.delete('--push-facts') + certname = ARGV[0] + + if push_facts || push_facts_parallel # push all facts files to Foreman and don't act as an ENC - if ARGV.empty? - process_all_facts(false) - else - process_host_facts(ARGV[0]) - end - elsif push_facts_parallel - http_fact_requests = Http_Fact_Requests.new - process_all_facts(http_fact_requests) - upload_facts_parallel(http_fact_requests) + threads = push_facts_parallel ? Settings.threads : 1 + push_facts(certname: certname, threads: threads, watch: watch) + elsif watch + raise 'Cannot watch for facts without specifying --push-facts or --push-facts-parallel' else - certname = ARGV[0] || raise("Must provide certname as an argument") - # - # query External node - begin - result = "" - Timeout.timeout(tsecs) do - # send facts to Foreman - enable 'facts' setting to activate - # if you use this option below, make sure that you don't send facts to foreman via the rake task or push facts alternatives. - # - if SETTINGS[:facts] - req = generate_fact_request(certname, fact_file(certname)) - upload_facts(certname, req) - end - - result = enc(certname) - cache(certname, result) - end - rescue Timeout::Error, SocketError, Errno::EHOSTUNREACH, Errno::ECONNREFUSED, NodeRetrievalError, FactUploadError => e - $stderr.puts "Serving cached ENC: #{e}" - # Read from cache, we got some sort of an error. - result = read_cache(certname) - end + raise 'Must provide certname as an argument' unless certname - if no_env - require 'yaml' - yaml = YAML.safe_load(result) - yaml.delete('environment') - # Always reset the result to back to clean yaml on our end - puts yaml.to_yaml - else - puts result - end + puts enc(certname, strip_environment: no_env) end - rescue => e + rescue StandardError => e warn e exit 1 end - if watch - watch_and_send_facts(push_facts_parallel) - end end