Skip to content

Commit

Permalink
Allow customize http vhost config using HttpdCustomization.CustomServ…
Browse files Browse the repository at this point in the history 
…iceConfigSecret

This change allows to customize the httpd vhost config using this parameter
to specify a secret that contains service config data. The content of each
provided snippet gets rendered as a go template and placed into
/etc/httpd/conf/httpd_custom_<endpoint>_<key> .
At the end of the vhost config in the default httpd template these custom
configs get included using `Include conf/httpd_custom_<endpoint>_*`.

For information on how sections in httpd configuration get merged, check section
"How the sections are merged" in https://httpd.apache.org/docs/current/sections.html#merging

All possible parameters which can be use in a template can be looked up
in the <service>-config-data secret of the service like:
$ oc get secret -n openstack keystone-config-data -o json | jq -r .data.templatingParameters | base64 -d

or in the running pod of the service in the file:
$ cat /var/lib/config-data/default/templatingParameters

The content is a versioned dump of the parameters of the service operator, like:

~~~
v1:
    DatabaseConnection: mysql+pymysql://user:[email protected]/keystone?read_default_file=/etc/my.cnf
    KeystoneEndpointInternal: https://keystone-internal.openstack.svc:5000
    KeystoneEndpointPublic: https://keystone-public-openstack.apps-crc.testing
    ProcessNumber: 3
    TransportURL: rabbit://user:[email protected]:5671/?ssl=1
    VHosts:
        internal:
            Override: false
            SSLCertificateFile: /etc/pki/tls/certs/internal.crt
            SSLCertificateKeyFile: /etc/pki/tls/private/internal.key
            ServerName: keystone-internal.openstack.svc
            TLS: true
        public:
            Override: false
            SSLCertificateFile: /etc/pki/tls/certs/public.crt
            SSLCertificateKeyFile: /etc/pki/tls/private/public.key
            ServerName: keystone-public.openstack.svc
            TLS: true
    enableSecureRBAC: true
    fernetMaxActiveKeys: 5
    memcachedServers: memcached-0.memcached.openstack.svc:11212
    memcachedServersWithInet: inet:[memcached-0.memcached.openstack.svc]:11211
    memcachedTLS: true
~~~

Depends-On: openstack-k8s-operators/lib-common#591

Signed-off-by: Martin Schuppert <[email protected]>
  • Loading branch information
@stuggi
stuggi committed Jan 14, 2025
1 parent ece4d21 commit 8220881
Show file tree
Hide file tree
Showing 13 changed files with 199 additions and 35 deletions.
10 changes: 10 additions & 0 deletions api/bases/keystone.openstack.org_keystoneapis.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,16 @@ spec:
httpdCustomization:
description: HttpdCustomization - customize the httpd service
properties:
customServiceConfigSecret:
description: |-
CustomServiceConfigSecret - customize the httpd vhost config using this parameter to specify
a secret that contains service config data. The content of each provided snippet gets
rendered as a go template and placed into /etc/httpd/conf/httpd_custom_<key> .
In the default httpd template at the end of the vhost those custom configs get
included using `Include conf/httpd_custom_<endpoint>_*`.
For information on how sections in httpd configuration get merged, check section
"How the sections are merged" in https://httpd.apache.org/docs/current/sections.html#merging
type: string
processNumber:
default: 3
description: ProcessNumber - Number of processes running in keystone
Expand Down
2 changes: 1 addition & 1 deletion api/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/google/uuid v1.6.0
github.com/gophercloud/gophercloud v1.14.1
github.com/onsi/gomega v1.34.1
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250113155806-c6542cc4eb2b
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e
github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241216113837-d172b3ac0f4e
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
Expand Down
4 changes: 2 additions & 2 deletions api/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 h1:J1wuGhVxpsHykZBa6Beb1gQ96Ptej9AE/BvwCBiRj1E=
github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e h1:hf4kVQBkyG79WcHBxdQ25QrDBbGFdarebS1Tc0Xclq4=
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250113155806-c6542cc4eb2b h1:laxu7pBlpo+lHdRn/XPqxfLGfSW77nnzdOwIBrO4BkI=
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250113155806-c6542cc4eb2b/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e h1:HFo4OqPY0x4ZQeaWI2YGonTXAGTQFt+rOEJlfZVhS7s=
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:IASoGvp5QM/tBJUd/8i8uIjj4DBnI+64Ydh4r7pmnvA=
github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241216113837-d172b3ac0f4e h1:/iWDp3j+ET3gE5IjKHtdZaPd4SQyLHB/4L5jB16cV3I=
Expand Down
10 changes: 10 additions & 0 deletions api/v1beta1/keystoneapi_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,16 @@ type HttpdCustomization struct {
// +kubebuilder:validation:Minimum=1
// ProcessNumber - Number of processes running in keystone API
ProcessNumber *int32 `json:"processNumber"`

// +kubebuilder:validation:Optional
// CustomServiceConfigSecret - customize the httpd vhost config using this parameter to specify
// a secret that contains service config data. The content of each provided snippet gets
// rendered as a go template and placed into /etc/httpd/conf/httpd_custom_<key> .
// In the default httpd template at the end of the vhost those custom configs get
// included using `Include conf/httpd_custom_<endpoint>_*`.
// For information on how sections in httpd configuration get merged, check section
// "How the sections are merged" in https://httpd.apache.org/docs/current/sections.html#merging
CustomServiceConfigSecret *string `json:"customServiceConfigSecret,omitempty"`
}

// KeystoneAPIStatus defines the observed state of KeystoneAPI
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 10 additions & 0 deletions config/crd/bases/keystone.openstack.org_keystoneapis.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,16 @@ spec:
httpdCustomization:
description: HttpdCustomization - customize the httpd service
properties:
customServiceConfigSecret:
description: |-
CustomServiceConfigSecret - customize the httpd vhost config using this parameter to specify
a secret that contains service config data. The content of each provided snippet gets
rendered as a go template and placed into /etc/httpd/conf/httpd_custom_<key> .
In the default httpd template at the end of the vhost those custom configs get
included using `Include conf/httpd_custom_<endpoint>_*`.
For information on how sections in httpd configuration get merged, check section
"How the sections are merged" in https://httpd.apache.org/docs/current/sections.html#merging
type: string
processNumber:
default: 3
description: ProcessNumber - Number of processes running in keystone
Expand Down
77 changes: 62 additions & 15 deletions controllers/keystoneapi_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,17 +233,19 @@ func (r *KeystoneAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request)

// fields to index to reconcile when change
const (
passwordSecretField = ".spec.secret"
caBundleSecretNameField = ".spec.tls.caBundleSecretName"
tlsAPIInternalField = ".spec.tls.api.internal.secretName"
tlsAPIPublicField = ".spec.tls.api.public.secretName"
passwordSecretField = ".spec.secret"
caBundleSecretNameField = ".spec.tls.caBundleSecretName"
tlsAPIInternalField = ".spec.tls.api.internal.secretName"
tlsAPIPublicField = ".spec.tls.api.public.secretName"
httpdCustomServiceConfigSecretField = ".spec.httpdCustomization.customServiceConfigSecret"
)

var allWatchFields = []string{
passwordSecretField,
caBundleSecretNameField,
tlsAPIInternalField,
tlsAPIPublicField,
httpdCustomServiceConfigSecretField,
}

// SetupWithManager -
Expand Down Expand Up @@ -298,6 +300,18 @@ func (r *KeystoneAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl.M
return err
}

// index httpdOverrideSecretField
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &keystonev1.KeystoneAPI{}, httpdCustomServiceConfigSecretField, func(rawObj client.Object) []string {
// Extract the secret name from the spec, if one is provided
cr := rawObj.(*keystonev1.KeystoneAPI)
if cr.Spec.HttpdCustomization.CustomServiceConfigSecret == nil {
return nil
}
return []string{*cr.Spec.HttpdCustomization.CustomServiceConfigSecret}
}); err != nil {
return err
}

memcachedFn := func(ctx context.Context, o client.Object) []reconcile.Request {
result := []reconcile.Request{}

Expand Down Expand Up @@ -1185,7 +1199,7 @@ func (r *KeystoneAPIReconciler) generateServiceConfigMaps(
databaseAccount := db.GetAccount()
dbSecret := db.GetSecret()

templateParameters := map[string]interface{}{
v1 := map[string]interface{}{
"memcachedServers": mc.GetMemcachedServerListString(),
"memcachedServersWithInet": mc.GetMemcachedServerListWithInetString(),
"memcachedTLS": mc.GetMemcachedTLSSupport(),
Expand All @@ -1201,7 +1215,19 @@ func (r *KeystoneAPIReconciler) generateServiceConfigMaps(
"fernetMaxActiveKeys": instance.Spec.FernetMaxActiveKeys,
}

v1["KeystoneEndpointPublic"], _ = instance.GetEndpoint(endpoint.EndpointPublic)
v1["KeystoneEndpointInternal"], _ = instance.GetEndpoint(endpoint.EndpointInternal)

httpdOverrideSecret := &corev1.Secret{}
if instance.Spec.HttpdCustomization.CustomServiceConfigSecret != nil && *instance.Spec.HttpdCustomization.CustomServiceConfigSecret != "" {
httpdOverrideSecret, _, err = oko_secret.GetSecret(ctx, h, *instance.Spec.HttpdCustomization.CustomServiceConfigSecret, instance.Namespace)
if err != nil {
return err
}
}

// create httpd vhost template parameters
customTemplates := map[string]string{}
httpdVhostConfig := map[string]interface{}{}
for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} {
endptConfig := map[string]interface{}{}
Expand All @@ -1212,10 +1238,30 @@ func (r *KeystoneAPIReconciler) generateServiceConfigMaps(
endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String())
endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String())
}

endptConfig["Override"] = false
if len(httpdOverrideSecret.Data) > 0 {
endptConfig["Override"] = true
for key, data := range httpdOverrideSecret.Data {
if len(data) > 0 {
customTemplates["httpd_custom_"+endpt.String()+"_"+key] = string(data)
}
}
}
httpdVhostConfig[endpt.String()] = endptConfig
}
templateParameters["VHosts"] = httpdVhostConfig
templateParameters["TimeOut"] = instance.Spec.APITimeout
v1["VHosts"] = httpdVhostConfig
v1["TimeOut"] = instance.Spec.APITimeout

templateParameters := make(map[string]interface{})
templateParameters["v1"] = v1

// Marshal the templateParameters map to YAML
yamlData, err := yaml.Marshal(templateParameters)
if err != nil {
return fmt.Errorf("Error marshalling to YAML: %w", err)
}
customData["templatingParameters"] = string(yamlData)

tmpl := []util.Template{
// Scripts
Expand All @@ -1228,13 +1274,14 @@ func (r *KeystoneAPIReconciler) generateServiceConfigMaps(
},
// Configs
{
Name: fmt.Sprintf("%s-config-data", instance.Name),
Namespace: instance.Namespace,
Type: util.TemplateTypeConfig,
InstanceType: instance.Kind,
CustomData: customData,
ConfigOptions: templateParameters,
Labels: cmLabels,
Name: fmt.Sprintf("%s-config-data", instance.Name),
Namespace: instance.Namespace,
Type: util.TemplateTypeConfig,
InstanceType: instance.Kind,
StringTemplate: customTemplates,
CustomData: customData,
ConfigOptions: templateParameters,
Labels: cmLabels,
},
}
return oko_secret.EnsureSecrets(ctx, h, instance, tmpl, envVars)
Expand Down Expand Up @@ -1294,7 +1341,7 @@ func (r *KeystoneAPIReconciler) reconcileCloudConfig(
Name: instance.Spec.Secret,
Namespace: instance.Namespace,
},
Type: "Opaque",
Type: corev1.SecretTypeOpaque,
}

err = r.Client.Get(ctx, types.NamespacedName{Name: keystoneSecret.Name, Namespace: instance.Namespace}, keystoneSecret)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ require (
github.com/onsi/gomega v1.34.1
github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20241217075239-1fc4566cc5ab
github.com/openstack-k8s-operators/keystone-operator/api v0.3.1-0.20240213125925-e40975f3db7e
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250113155806-c6542cc4eb2b
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e
github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241216113837-d172b3ac0f4e
github.com/openstack-k8s-operators/mariadb-operator/api v0.5.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 h1:J1wuGhVxpsHykZBa6
github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20241217075239-1fc4566cc5ab h1:Pm9zQyhrs/zGAk9jvyt0hSBP28aHsFdWyI99M/lvFxU=
github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20241217075239-1fc4566cc5ab/go.mod h1:SSYBbFbgQbOwyY2cQNet7fSdQHHPb2rLo6GXE97Awp8=
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e h1:hf4kVQBkyG79WcHBxdQ25QrDBbGFdarebS1Tc0Xclq4=
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250113155806-c6542cc4eb2b h1:laxu7pBlpo+lHdRn/XPqxfLGfSW77nnzdOwIBrO4BkI=
github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20250113155806-c6542cc4eb2b/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e h1:HFo4OqPY0x4ZQeaWI2YGonTXAGTQFt+rOEJlfZVhS7s=
github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:IASoGvp5QM/tBJUd/8i8uIjj4DBnI+64Ydh4r7pmnvA=
github.com/openstack-k8s-operators/lib-common/modules/test v0.5.1-0.20241216113837-d172b3ac0f4e h1:/iWDp3j+ET3gE5IjKHtdZaPd4SQyLHB/4L5jB16cV3I=
Expand Down
9 changes: 7 additions & 2 deletions templates/keystoneapi/config/httpd.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
CustomLog /dev/stdout combined env=!forwarded
CustomLog /dev/stdout proxy env=forwarded

{{ range $endpt, $vhost := .VHosts }}
{{ range $endpt, $vhost := .v1.VHosts }}
# {{ $endpt }} vhost {{ $vhost.ServerName }} configuration
<VirtualHost *:5000>
ServerName {{ $vhost.ServerName }}
Expand Down Expand Up @@ -54,9 +54,14 @@ CustomLog /dev/stdout proxy env=forwarded

## WSGI configuration
WSGIApplicationGroup %{GLOBAL}
WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=keystone processes={{ $.ProcessNumber }} threads=1 user=keystone
WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=keystone processes={{ $.v1.ProcessNumber }} threads=1 user=keystone
WSGIProcessGroup {{ $endpt }}
WSGIScriptAlias / "/usr/bin/keystone-wsgi-public"
WSGIPassAuthorization On

{{- if $vhost.Override }}
Include conf/httpd_override_{{ $endpt }}_*
{{- end }}

</VirtualHost>
{{ end }}
11 changes: 9 additions & 2 deletions templates/keystoneapi/config/keystone-api-config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@
{
"source": "/var/lib/config-data/default/httpd.conf",
"dest": "/etc/httpd/conf/httpd.conf",
"owner": "root",
"owner": "apache",
"perm": "0644"
},
{
"source": "/var/lib/config-data/default/ssl.conf",
"dest": "/etc/httpd/conf.d/ssl.conf",
"owner": "root",
"owner": "apache",
"perm": "0644"
},
{
Expand Down Expand Up @@ -58,6 +58,13 @@
"dest": "/etc/my.cnf",
"owner": "keystone",
"perm": "0644"
},
{
"source": "/var/lib/config-data/default/httpd_override_*",
"dest": "/etc/httpd/conf/",
"owner": "apache",
"perm": "0444",
"optional": true
}
]
}
20 changes: 10 additions & 10 deletions templates/keystoneapi/config/keystone.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,32 @@
use_stderr=true

[cache]
{{if .memcachedTLS}}
{{if .v1.memcachedTLS}}
backend = dogpile.cache.pymemcache
memcache_servers={{ .memcachedServers }}
memcache_servers={{ .v1.memcachedServers }}
{{else}}
backend = dogpile.cache.memcached
memcache_servers={{ .memcachedServersWithInet }}
memcache_servers={{ .v1.memcachedServersWithInet }}
{{end}}
enabled=true
tls_enabled={{ .memcachedTLS }}
tls_enabled={{ .v1.memcachedTLS }}

[database]
max_retries=-1
db_max_retries=-1
connection={{ .DatabaseConnection }}
connection={{ .v1.DatabaseConnection }}

[oslo_policy]
enforce_new_defaults = {{ .enableSecureRBAC }}
enforce_scope = {{ .enableSecureRBAC }}
enforce_new_defaults = {{ .v1.enableSecureRBAC }}
enforce_scope = {{ .v1.enableSecureRBAC }}

[fernet_tokens]
key_repository=/etc/keystone/fernet-keys
max_active_keys={{ .fernetMaxActiveKeys }}
max_active_keys={{ .v1.fernetMaxActiveKeys }}

{{ if (index . "TransportURL") }}
{{ if (index .v1 "TransportURL") }}
[oslo_messaging_notifications]
driver=messagingv2
transport_url={{ .TransportURL }}
transport_url={{ .v1.TransportURL }}
topics = barbican_notifications
{{ end }}
Loading

0 comments on commit 8220881

Please sign in to comment.