Skip to content

Commit

Permalink
Refactor configuration of ClusterLogForwarder
Browse files Browse the repository at this point in the history
With this commit configuring the ClusterLogForwarder resource will be done with
a new parameter `openshift4_logging.clusterLogForwarder`, which will follow the
upstream convention / spec.

The old parameter `openshift4_logging.clusterLogForwarding` is deprecated, but
existing legacy config will be respected.
  • Loading branch information
DebakelOrakel committed Aug 10, 2024
1 parent 36a5147 commit 9b545ae
Show file tree
Hide file tree
Showing 24 changed files with 994 additions and 284 deletions.
15 changes: 1 addition & 14 deletions class/defaults.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,20 +71,7 @@ parameters:
memory: 128Mi

clusterLogging: {}

clusterLogForwarding:
enabled: false
forwarders: {}
namespace_groups: {}
application_logs: {}
audit_logs:
enabled: false
infrastructure_logs:
enabled: true
json:
enabled: false
typekey: 'kubernetes.labels.logFormat'
typename: 'nologformat'
clusterLogForwarder: {}

operatorResources:
clusterLogging:
Expand Down
126 changes: 80 additions & 46 deletions component/config_forwarding.libsonnet
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
local com = import 'lib/commodore.libjsonnet';
local kap = import 'lib/kapitan.libjsonnet';
local lib = import 'lib/openshift4-logging.libsonnet';

Expand All @@ -8,70 +9,84 @@ local deployLokistack = params.components.lokistack.enabled;
local deployElasticsearch = params.components.elasticsearch.enabled;
local forwardingOnly = !deployLokistack && !deployElasticsearch;

// -----------------------------------------------------------------------------
// Legacy Rendering
// -----------------------------------------------------------------------------

local legacyConfigParams = std.get(params, 'clusterLogForwarding', {});
local legacyConfig = if std.length(legacyConfigParams) > 0 then std.trace(
'Parameter `clusterLogForwarding` is deprecated. Please update your config to use `clusterLogForwarder`',
legacyConfigParams
) else {};

local pipelineOutputRefs(pipeline) =
local default = if forwardingOnly then [] else [ 'default' ];
std.get(pipeline, 'forwarders', []) + default;

// Apply default config for application logs.
local patchAppLogDefaults = {
local outputRefs = pipelineOutputRefs(params.clusterLogForwarding.application_logs),
local enablePipeline = std.length(outputRefs) > 0,
local patchLegacyAppLogDefaults = {
local pipeline = std.get(legacyConfig, 'application_logs', { enabled: true }),
local pipelineOutputs = pipelineOutputRefs(pipeline),
local pipelineEnabled = std.length(pipelineOutputs) > 0,

pipelines: {
[if enablePipeline then 'application-logs']: {
[if pipelineEnabled then 'application-logs']: {
inputRefs: [ 'application' ],
outputRefs: outputRefs,
outputRefs: pipelineOutputs,
},
},
};

// Apply default config for infra logs.
local patchInfraLogDefaults = {
local outputRefs = pipelineOutputRefs(params.clusterLogForwarding.infrastructure_logs),
local enablePipeline = params.clusterLogForwarding.infrastructure_logs.enabled && std.length(outputRefs) > 0,
local patchLegacyInfraLogDefaults = {
local pipeline = { enabled: true } + std.get(legacyConfig, 'infrastructure_logs', {}),
local pipelineOutputs = pipelineOutputRefs(pipeline),
local pipelineEnabled = pipeline.enabled && std.length(pipelineOutputs) > 0,

pipelines: {
[if enablePipeline then 'infrastructure-logs']: {
[if pipelineEnabled then 'infrastructure-logs']: {
inputRefs: [ 'infrastructure' ],
outputRefs: outputRefs,
outputRefs: pipelineOutputs,
},
},
};

// Apply default config for audit logs.
local patchAuditLogDefaults = {
local outputRefs = pipelineOutputRefs(params.clusterLogForwarding.audit_logs),
local enablePipeline = params.clusterLogForwarding.audit_logs.enabled && std.length(outputRefs) > 0,
local patchLegacyAuditLogDefaults = {
local pipeline = std.get(legacyConfig, 'audit_logs', { enabled: false }),
local pipelineOutputs = pipelineOutputRefs(pipeline),
local pipelineEnabled = pipeline.enabled && std.length(pipelineOutputs) > 0,

pipelines: {
[if enablePipeline then 'audit-logs']: {
[if pipelineEnabled then 'audit-logs']: {
inputRefs: [ 'audit' ],
outputRefs: outputRefs,
outputRefs: pipelineOutputs,
},
},
};

// Enable json parsing for default pipelines if configured.
local patchJsonLogging = {
local enableAppLogs = std.get(params.clusterLogForwarding.application_logs, 'json', false),
local enableInfraLogs = std.get(params.clusterLogForwarding.infrastructure_logs, 'json', false),
local legacyEnableJson = std.get(std.get(legacyConfig, 'json', {}), 'enabled', false);
local patchLegacyJsonLogging = {
local enableAppLogs = std.get(std.get(legacyConfig, 'application_logs', {}), 'json', false),
local enableInfraLogs = std.get(std.get(legacyConfig, 'infrastructure_logs', {}), 'json', false),

pipelines: {
[if enableAppLogs then 'application-logs']: { parse: 'json' },
[if enableInfraLogs then 'infrastructure-logs']: { parse: 'json' },
},
[if deployElasticsearch && params.clusterLogForwarding.json.enabled then 'outputDefaults']: {
[if deployElasticsearch && legacyEnableJson then 'outputDefaults']: {
elasticsearch: {
structuredTypeKey: params.clusterLogForwarding.json.typekey,
structuredTypeName: params.clusterLogForwarding.json.typename,
structuredTypeKey: std.get(legacyConfig.json, 'typekey', 'kubernetes.labels.logFormat'),
structuredTypeName: std.get(legacyConfig.json, 'typename', 'nologformat'),
},
},
};

// Enable detectMultilineErrors for default pipelines if configured.
local patchMultilineErrors = {
local enableAppLogs = std.get(params.clusterLogForwarding.application_logs, 'detectMultilineErrors', false),
local enableInfraLogs = std.get(params.clusterLogForwarding.infrastructure_logs, 'detectMultilineErrors', false),
local patchLegacyMultilineErrors = {
local enableAppLogs = std.get(std.get(legacyConfig, 'application_logs', {}), 'detectMultilineErrors', false),
local enableInfraLogs = std.get(std.get(legacyConfig, 'infrastructure_logs', {}), 'detectMultilineErrors', false),

pipelines: {
[if enableAppLogs then 'application-logs']: { detectMultilineErrors: true },
Expand All @@ -81,19 +96,19 @@ local patchMultilineErrors = {

// --- patch deprecated `clusterLogForwarding.namespace` config
local namespaceGroups = (
if std.objectHas(params.clusterLogForwarding, 'namespaces') then
if std.objectHas(legacyConfig, 'namespaces') then
{
[ns]: {
namespaces: [ ns ],
forwarders: [ params.clusterLogForwarding.namespaces[ns].forwarder ],
forwarders: [ legacyConfig.namespaces[ns].forwarder ],
}
for ns in std.objectFields(params.clusterLogForwarding.namespaces)
for ns in std.objectFields(legacyConfig.namespaces)
} else {}
) + params.clusterLogForwarding.namespace_groups;
) + std.get(legacyConfig, 'namespace_groups', {});
// --- patch end

// Add inputs entry for every namespace_group defined in `clusterLogForwarding.namespace_groups`.
local patchCustomInputs = {
local patchLegacyCustomInputs = {
[if std.length(namespaceGroups) > 0 then 'inputs']: {
[group]: {
application: {
Expand All @@ -105,7 +120,7 @@ local patchCustomInputs = {
};

// Add pipelines entry for every namespace_group defined in `clusterLogForwarding.namespace_groups`.
local patchCustomPipelines = {
local patchLegacyCustomPipelines = {
[if std.length(namespaceGroups) > 0 then 'pipelines']: {
local enableJson = std.get(namespaceGroups[group], 'json', false),
local enableMultilineError = std.get(namespaceGroups[group], 'detectMultilineErrors', false),
Expand All @@ -121,35 +136,52 @@ local patchCustomPipelines = {
};

// Add outputs entry for every forwarder defined in `clusterLogForwarding.forwarders`.
local patchCustomOutputs = {
[if std.length(params.clusterLogForwarding.forwarders) > 0 then 'outputs']: {
[name]: params.clusterLogForwarding.forwarders[name]
for name in std.objectFields(params.clusterLogForwarding.forwarders)
local patchLegacyCustomOutputs = {
[if std.length(std.get(legacyConfig, 'forwarders', {})) > 0 then 'outputs']: {
[name]: legacyConfig.forwarders[name]
for name in std.objectFields(legacyConfig.forwarders)
},
};

// ClusterLogForwarderSpecs:
// -----------------------------------------------------------------------------
// End Legacy Rendering
// -----------------------------------------------------------------------------

// Add defaults to pipelines config
local patchPipelineDefaults = {
local auditPipeline = std.get(std.get(params.clusterLogForwarder, 'pipelines', {}), 'audit-logs', {}),

pipelines: {
[if std.length(auditPipeline) > 0 then 'audit-logs']: {
inputRefs: [ 'audit' ],
[if !forwardingOnly then 'outputRefs']: [ 'default' ],
},
},
};

// clusterLogForwarderSpec:
// Consecutively apply patches to result of previous apply.
local clusterLogForwarderSpec = std.foldl(
// we use std.mergePatch here, because this way we don't need
// to make each patch object mergeable by suffixing all keys with a +.
function(manifest, patch) std.mergePatch(manifest, patch),
[
patchAppLogDefaults,
patchInfraLogDefaults,
patchAuditLogDefaults,
patchJsonLogging,
patchMultilineErrors,
patchCustomInputs,
patchCustomOutputs,
patchCustomPipelines,
patchLegacyAppLogDefaults,
patchLegacyInfraLogDefaults,
patchLegacyAuditLogDefaults,
patchLegacyJsonLogging,
patchLegacyMultilineErrors,
patchLegacyCustomInputs,
patchLegacyCustomOutputs,
patchLegacyCustomPipelines,
patchPipelineDefaults,
],
{
inputs: {},
outputs: {},
pipelines: {},
}
);
},
) + com.makeMergeable(params.clusterLogForwarder);

// ClusterLogForwarder:
// Create definitive ClusterLogForwarder resource from specs.
Expand All @@ -176,8 +208,10 @@ local clusterLogForwarder = lib.ClusterLogForwarder(params.namespace, 'instance'
},
};

local enableLogForwarder = std.length(params.clusterLogForwarder) > 0 || std.get(legacyConfig, 'enabled', false);

// Define outputs below
if params.clusterLogForwarding.enabled then
if enableLogForwarder then
{
'31_cluster_logforwarding': clusterLogForwarder,
}
Expand Down
Loading

0 comments on commit 9b545ae

Please sign in to comment.