From 79eb661409d697caab5374a8c61a7e3c6c9dc38e Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Sat, 23 Dec 2023 15:32:21 +0100 Subject: [PATCH] Collect data from the Icinga 2 API (#109) Collect data from Icinga 2 API --- .golangci.yml | 1 + README.md | 71 +++++----- internal/collection/collection.go | 17 ++- main.go | 14 +- modules/icinga2/api.go | 128 ++++++++++++++++++ modules/icinga2/collector.go | 6 + .../icinga2/testdata/api/status-example.txt | 1 + 7 files changed, 201 insertions(+), 37 deletions(-) create mode 100644 modules/icinga2/api.go create mode 100644 modules/icinga2/testdata/api/status-example.txt diff --git a/.golangci.yml b/.golangci.yml index bafe1f9..b724f41 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,6 +4,7 @@ run: linters: enable-all: true disable: + - goimports - cyclop - depguard - exhaustivestruct diff --git a/README.md b/README.md index 7b91cd7..ad4052b 100644 --- a/README.md +++ b/README.md @@ -26,16 +26,23 @@ By default, we collect all we can find. You can control this by only enabling ce If you want to see what is collected, add `--verbose` -| Short | Long | Description | -|:-----:|:------------------|------------------------------------------------------------------------------------------------------------------------| -| -o | --output | Output file for the zip content (default: current directory and named like '$HOSTNAME'-netways-support-$TIMESTAMP.zip) | -| | --nodetails | Disable detailed collection including logs and more | -| | --enable | List of enabled modules (default: all) | -| | --disable | List of disabled modules (default: none) | -| | --hide | List of keywords to obfuscate. Can be used multiple times | -| | --command-timeout | Timeout for command execution in modules (default: 1m0s) | -| -v | --verbose | Enable verbose logging | -| -V | --version | Print version and exit | +To collect advanced data for module `Icinga 2`, you can use the Icinga 2 API to collect data from all endpoints provided. +The API requests are performed with a global API user you have to create yourself. Just create that user in a global zone like 'director-global' + + +| Short | Long | Description | +|:-----:|:------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| -o | --output | Output file for the zip content (default: current directory and named like '$HOSTNAME'-netways-support-$TIMESTAMP.zip) | +| | --nodetails | Disable detailed collection including logs and more | +| | --enable | List of enabled modules (default: all) | +| | --disable | List of disabled modules (default: none) | +| | --hide | List of keywords to obfuscate. Can be used multiple times | +| | --command-timeout | Timeout for command execution in modules (default: 1m0s) | +| | --icinga2-api-user | Username of global Icinga 2 API user to collect data about Icinga 2 Infrastructure | +| | --icinga2-api-pass | Password for global Icinga 2 API user to collect data about Icinga 2 Infrastructure | +| | --icinga2-api-endpoints | List of Icinga 2 API Endpoints (including port) to collect data from. FQDN or IP address must be reachable. (Example: i2-master01.local:5665) | +| -v | --verbose | Enable verbose logging | +| -V | --version | Print version and exit | ## Modules @@ -45,28 +52,28 @@ collected. Most modules check if the component is installed before trying to collect data. If the module is not detected, it will not be collected. -| Module name | Description | -|----------------|------------------------------------------------------------------------------------------------------------------------| -| ansible | Configuration and packages | -| base | Basic information about the system (operating system, kernel, memory, cpu, processes, repositories, firewalls, etc.) | -| corosync | Includes corosync and pacemaker. Collects configuration, logs, packages and service status | -| elastic | Includes elasticsearch, logstash and kibana. Collects configuration, packages and service status | -| grafana | Configuration, logs, plugins, packages and service status | -| graphite | Includes graphite and carbon. Collects configuration, logs, python / pip version and list, packages and service status | -| graylog | Configuration, packages and service status | -| icinga2 | Configuration, packages, service status, logs, Icinga objects, Icinga variables, plugins and icinga-installer | -| icingadb | Includes IcingaDB and IcingaDB redis. Collects configuration, logs, packages and service status | -| icingadirector | Packages or git information, logs, Director health status and service status | -| icingaweb2 | Configuration, logs, packages, modules, PHP, modules and service status | -| influxdb | Configuration, logs, packages and service status | -| keepalived | Configuration, packages and service status | -| mongodb | Configuration, logs, packages and service status | -| mysql | Configuration, logs, packages and service status | -| postgresql | Configuration, logs, packages and service status | -| prometheus | Configuration, packages and service status | -| puppet | Configuration, logs, module list, packages and service status | -| webservers | Includes apache2, httpd and nginx. Collects configuration, logs, packages and service status | -| foreman | Configuration, logs, packages and service status | +| Module name | Description | +|----------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| ansible | Configuration and packages | +| base | Basic information about the system (operating system, kernel, memory, cpu, processes, repositories, firewalls, etc.) | +| corosync | Includes corosync and pacemaker. Collects configuration, logs, packages and service status | +| elastic | Includes elasticsearch, logstash and kibana. Collects configuration, packages and service status | +| grafana | Configuration, logs, plugins, packages and service status | +| graphite | Includes graphite and carbon. Collects configuration, logs, python / pip version and list, packages and service status | +| graylog | Configuration, packages and service status | +| icinga2 | Configuration, packages, service status, logs, Icinga 2 objects, Icinga 2 variables, plugins, icinga-installer and data from API endpoints (if provided) | +| icingadb | Includes IcingaDB and IcingaDB redis. Collects configuration, logs, packages and service status | +| icingadirector | Packages or git information, logs, Director health status and service status | +| icingaweb2 | Configuration, logs, packages, modules, PHP, modules and service status | +| influxdb | Configuration, logs, packages and service status | +| keepalived | Configuration, packages and service status | +| mongodb | Configuration, logs, packages and service status | +| mysql | Configuration, logs, packages and service status | +| postgresql | Configuration, logs, packages and service status | +| prometheus | Configuration, packages and service status | +| puppet | Configuration, logs, module list, packages and service status | +| webservers | Includes apache2, httpd and nginx. Collects configuration, logs, packages and service status | +| foreman | Configuration, logs, packages and service status | ## Supported systems diff --git a/internal/collection/collection.go b/internal/collection/collection.go index 3056879..8e9c62e 100644 --- a/internal/collection/collection.go +++ b/internal/collection/collection.go @@ -3,6 +3,7 @@ package collection import ( "archive/zip" "bytes" + "encoding/json" "fmt" "io" "strings" @@ -140,6 +141,20 @@ func (c *Collection) AddFileYAML(fileName string, data interface{}) { _ = c.AddFileToOutput(file) } +func (c *Collection) AddFileJSON(fileName string, data interface{}) { + var buf bytes.Buffer + + err := json.NewEncoder(&buf).Encode(&data) + if err != nil { + c.Log.Debugf("could not encode JSON data for '%s': %s", fileName, err) + } + + file := NewFile(fileName) + file.Data = buf.Bytes() + + _ = c.AddFileToOutput(file) +} + func (c *Collection) AddFiles(prefix, source string) { c.Log.Debug("Collecting files from ", source) @@ -178,7 +193,7 @@ func (c *Collection) AddFilesIfFound(prefix string, sources ...string) { func (c *Collection) AddCommandOutputWithTimeout(file string, timeout time.Duration, command string, arguments ...string) { - c.Log.Debugf("Collecting command output: %s %s", command, strings.Join(arguments, " ")) + c.Log.Debugf("Collecting command output: '%s %s'", command, strings.Join(arguments, " ")) output, err := LoadCommandOutputWithTimeout(timeout, command, arguments...) if err != nil { diff --git a/main.go b/main.go index db7dca4..44af6c1 100644 --- a/main.go +++ b/main.go @@ -193,17 +193,23 @@ func main() { } func handleArguments() { - flag.StringVarP(&outputFile, "output", "o", buildFileName(), "Output file for the ZIP content") + // arguments for collection handling flag.StringSliceVar(&enabledModules, "enable", moduleOrder, "List of enabled module") flag.StringSliceVar(&disabledModules, "disable", []string{}, "List of disabled module") + flag.StringVarP(&outputFile, "output", "o", buildFileName(), "Output file for the ZIP content") flag.BoolVar(&noDetailedCollection, "nodetails", false, "Disable detailed collection including logs and more") flag.StringArrayVar(&extraObfuscators, "hide", []string{}, "List of additional strings to obfuscate. Can be used multiple times and supports regex.") //nolint:lll flag.DurationVar(&commandTimeout, "command-timeout", commandTimeout, "Timeout for command execution in modules") - flag.BoolVarP(&verbose, "verbose", "v", false, "Enable verbose logging") - flag.BoolVarP(&verbose, "debug", "d", false, "Enable debug logging (use verbose)") + + // api credentials for icinga 2 modules + flag.StringVar(&icinga2.APICred.Username, "icinga2-api-user", "", "Username of global Icinga 2 API user to collect data about Icinga 2 Infrastructure") //nolint:lll + flag.StringVar(&icinga2.APICred.Password, "icinga2-api-pass", "", "Password for global Icinga 2 API user to collect data about Icinga 2 Infrastructure") //nolint:lll + flag.StringSliceVar(&icinga2.APIEndpoints, "icinga2-api-endpoints", []string{}, "List of Icinga 2 API Endpoints (including port) to collect data from. FQDN or IP address must be reachable. (Example: i2-master01.local:5665)") //nolint:lll + + // basic arguments flag.BoolVarP(&printVersion, "version", "V", false, "Print version and exit") + flag.BoolVarP(&verbose, "verbose", "v", false, "Enable verbose logging") - _ = flag.CommandLine.MarkHidden("debug") flag.CommandLine.SortFlags = false // Output a proper help message with details diff --git a/modules/icinga2/api.go b/modules/icinga2/api.go new file mode 100644 index 0000000..c831f98 --- /dev/null +++ b/modules/icinga2/api.go @@ -0,0 +1,128 @@ +package icinga2 + +import ( + "context" + "crypto/tls" + "fmt" + "github.com/NETWAYS/support-collector/internal/collection" + "io" + "net" + "net/http" + "path/filepath" + "strings" + "time" +) + +type UserAuth struct { + Username string + Password string +} + +// APICred saves the user and password. Provided as arguments +var APICred UserAuth + +// APIEndpoints saves the FQDN or ip address for the endpoints, that will be collected. Provided as arguments. +var APIEndpoints []string + +// InitAPICollection starts to collect data from the Icinga 2 API for given endpoints +func InitAPICollection(c *collection.Collection) error { + // return if no endpoints are provided + if len(APIEndpoints) == 0 { + return fmt.Errorf("0 API endpoints provided. No data will be collected from remote targets") + } + + c.Log.Info("Start collection of Icinga 2 API endpoints") + + // return if username or password is not provided + if APICred.Username == "" || APICred.Password == "" { + return fmt.Errorf("API Endpoints provided but username and/or password are missing") + } + + for _, endpoint := range APIEndpoints { + // check if endpoint is reachable + err := endpointIsReachable(endpoint) + if err != nil { + c.Log.Warn(err) + continue + } + + c.Log.Debugf("Endpoint '%s' is reachable", endpoint) + + // collect /v1/status from endpoint + err = collectStatus(endpoint, c) + if err != nil { + c.Log.Warn(err) + } + } + + return nil +} + +// endpointIsReachable checks if the given endpoint is reachable within 5 sec +func endpointIsReachable(endpoint string) error { + timeout := 5 * time.Second + + // try to dial tcp connection within 5 seconds + conn, err := net.DialTimeout("tcp", endpoint, timeout) + if err != nil { + return fmt.Errorf("cant connect to endpoint '%s' within 5 seconds: %w", endpoint, err) + } + defer conn.Close() + + return nil +} + +// collectStatus requests $endpoint$/v1/status with APICred and saves the json result to file +func collectStatus(endpoint string, c *collection.Collection) error { + c.Log.Debugf("request data from endpoint '%s/v1/status'", endpoint) + + // allow insecure connections because of Icinga 2 certificates and add proxy if one is defined in the environments + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + Proxy: http.ProxyFromEnvironment, + } + client := &http.Client{Transport: tr} + + // build context for request + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // build request + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://%s/v1/status", endpoint), nil) + if err != nil { + return fmt.Errorf("cant build new request for '%s': %w", endpoint, err) + } + + // set authentication for request + req.SetBasicAuth(APICred.Username, APICred.Password) + + // make request + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("cant requests status from '%s': %w", endpoint, err) + } + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("cant read from response: %w", err) + } + + // if response code is not '200 OK' throw error and return + if resp.Status != "200 OK" { + return fmt.Errorf("request failed with status code %s: %s", resp.Status, string(body)) + } + + // add body to file + c.AddFileJSON(filepath.Join(ModuleName, fmt.Sprintf("api-v1_status_%s.json", extractHostname(endpoint))), string(body)) + + return nil +} + +// extractsHostname takes the endpoint and extract the hostname of it +func extractHostname(endpoint string) string { + splits := strings.Split(endpoint, ":") + + return splits[0] +} diff --git a/modules/icinga2/collector.go b/modules/icinga2/collector.go index 3d283c1..3449e1e 100644 --- a/modules/icinga2/collector.go +++ b/modules/icinga2/collector.go @@ -143,4 +143,10 @@ func Collect(c *collection.Collection) { c.AddCommandOutput(filepath.Join(ModuleName, name), cmd[0], cmd[1:]...) } } + + // start the collection of remote api endpoints + err = InitAPICollection(c) + if err != nil { + c.Log.Warn(err) + } } diff --git a/modules/icinga2/testdata/api/status-example.txt b/modules/icinga2/testdata/api/status-example.txt new file mode 100644 index 0000000..8080cd3 --- /dev/null +++ b/modules/icinga2/testdata/api/status-example.txt @@ -0,0 +1 @@ +{"results":[{"name":"ApiListener","perfdata":[{"counter":false,"crit":null,"label":"api_num_conn_endpoints","max":null,"min":null,"type":"PerfdataValue","unit":"","value":1,"warn":null},{"counter":false,"crit":null,"label":"api_num_endpoints","max":null,"min":null,"type":"PerfdataValue","unit":"","value":1,"warn":null},{"counter":false,"crit":null,"label":"api_num_http_clients","max":null,"min":null,"type":"PerfdataValue","unit":"","value":1,"warn":null},{"counter":false,"crit":null,"label":"api_num_json_rpc_anonymous_clients","max":null,"min":null,"type":"PerfdataValue","unit":"","value":0,"warn":null},{"counter":false,"crit":null,"label":"api_num_json_rpc_relay_queue_item_rate","max":null,"min":null,"type":"PerfdataValue","unit":"","value":0.3,"warn":null},{"counter":false,"crit":null,"label":"api_num_json_rpc_relay_queue_items","max":null,"min":null,"type":"PerfdataValue","unit":"","value":0,"warn":null},{"counter":false,"crit":null,"label":"api_num_json_rpc_sync_queue_item_rate","max":null,"min":null,"type":"PerfdataValue","unit":"","value":0,"warn":null},{"counter":false,"crit":null,"label":"api_num_json_rpc_sync_queue_items","max":null,"min":null,"type":"PerfdataValue","unit":"","value":0,"warn":null},{"counter":false,"crit":null,"label":"api_num_json_rpc_work_queue_item_rate","max":null,"min":null,"type":"PerfdataValue","unit":"","value":0.08333333333333333,"warn":null},{"counter":false,"crit":null,"label":"api_num_not_conn_endpoints","max":null,"min":null,"type":"PerfdataValue","unit":"","value":0,"warn":null}],"status":{"api":{"conn_endpoints":["tbauriedel-win-icinga"],"http":{"clients":1},"identity":"tbauriedel-icinga","json_rpc":{"anonymous_clients":0,"relay_queue_item_rate":0.3,"relay_queue_items":0,"sync_queue_item_rate":0,"sync_queue_items":0,"work_queue_item_rate":0.08333333333333333},"not_conn_endpoints":[],"num_conn_endpoints":1,"num_endpoints":1,"num_not_conn_endpoints":0,"zones":{"main":{"client_log_lag":0,"connected":true,"endpoints":["tbauriedel-icinga"],"parent_zone":""},"tbauriedel-win-icinga":{"client_log_lag":0,"connected":true,"endpoints":["tbauriedel-win-icinga"],"parent_zone":"main"}}}}},{"name":"CIB","perfdata":[],"status":{"active_host_checks":0.03333333333333333,"active_host_checks_15min":16,"active_host_checks_1min":2,"active_host_checks_5min":6,"active_service_checks":0.03333333333333333,"active_service_checks_15min":37,"active_service_checks_1min":2,"active_service_checks_5min":12,"avg_execution_time":0.8716000080108642,"avg_latency":0,"current_concurrent_checks":0,"current_pending_callbacks":0,"max_execution_time":1.5779998302459717,"max_latency":0,"min_execution_time":0,"min_latency":0,"num_hosts_acknowledged":0,"num_hosts_down":0,"num_hosts_flapping":0,"num_hosts_handled":0,"num_hosts_in_downtime":0,"num_hosts_pending":0,"num_hosts_problem":0,"num_hosts_unreachable":0,"num_hosts_up":2,"num_services_acknowledged":0,"num_services_critical":0,"num_services_flapping":0,"num_services_handled":0,"num_services_in_downtime":0,"num_services_ok":4,"num_services_pending":0,"num_services_problem":1,"num_services_unknown":0,"num_services_unreachable":0,"num_services_warning":1,"passive_host_checks":0,"passive_host_checks_15min":0,"passive_host_checks_1min":0,"passive_host_checks_5min":0,"passive_service_checks":0,"passive_service_checks_15min":0,"passive_service_checks_1min":0,"passive_service_checks_5min":0,"remote_check_queue":0,"uptime":94541.84461402893}},{"name":"CheckerComponent","perfdata":[{"counter":false,"crit":null,"label":"checkercomponent_checker_idle","max":null,"min":null,"type":"PerfdataValue","unit":"","value":7,"warn":null},{"counter":false,"crit":null,"label":"checkercomponent_checker_pending","max":null,"min":null,"type":"PerfdataValue","unit":"","value":0,"warn":null}],"status":{"checkercomponent":{"checker":{"idle":7,"pending":0}}}},{"name":"CompatLogger","perfdata":[],"status":{"compatlogger":{}}},{"name":"ElasticsearchWriter","perfdata":[],"status":{"elasticsearchwriter":{}}},{"name":"ExternalCommandListener","perfdata":[],"status":{"externalcommandlistener":{}}},{"name":"FileLogger","perfdata":[],"status":{"filelogger":{}}},{"name":"GelfWriter","perfdata":[],"status":{"gelfwriter":{}}},{"name":"GraphiteWriter","perfdata":[],"status":{"graphitewriter":{}}},{"name":"IcingaApplication","perfdata":[],"status":{"icingaapplication":{"app":{"enable_event_handlers":true,"enable_flapping":true,"enable_host_checks":true,"enable_notifications":true,"enable_perfdata":true,"enable_service_checks":true,"environment":"","node_name":"tbauriedel-icinga","pid":65872,"program_start":1703059214.645369,"version":"r2.14.0-1"}}}},{"name":"IdoMysqlConnection","perfdata":[],"status":{"idomysqlconnection":{}}},{"name":"IdoPgsqlConnection","perfdata":[],"status":{"idopgsqlconnection":{}}},{"name":"Influxdb2Writer","perfdata":[],"status":{"influxdb2writer":{}}},{"name":"InfluxdbWriter","perfdata":[],"status":{"influxdbwriter":{}}},{"name":"JournaldLogger","perfdata":[],"status":{"journaldlogger":{}}},{"name":"LivestatusListener","perfdata":[],"status":{"livestatuslistener":{}}},{"name":"NotificationComponent","perfdata":[],"status":{"notificationcomponent":{}}},{"name":"OpenTsdbWriter","perfdata":[],"status":{"opentsdbwriter":{}}},{"name":"PerfdataWriter","perfdata":[],"status":{"perfdatawriter":{}}},{"name":"SyslogLogger","perfdata":[],"status":{"sysloglogger":{}}}]} \ No newline at end of file