Skip to content

Commit

Permalink
[Draft] Split nvidia-mdev arch in two stages
Browse files Browse the repository at this point in the history
  • Loading branch information
sbauza authored and jamepark4 committed Nov 27, 2024
1 parent 2944874 commit 41202f7
Show file tree
Hide file tree
Showing 19 changed files with 347 additions and 54 deletions.
41 changes: 34 additions & 7 deletions automation/vars/nvidia-mdev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
vas:
nvidia-mdev:
stages:
- path: examples/va/nvidia-mdev/nncp
- path: examples/va/nvidia-mdev/control-plane/nncp
wait_conditions:
- >-
oc -n openstack wait nncp
Expand All @@ -14,7 +14,7 @@ vas:
src_file: values.yaml
build_output: nncp.yaml

- path: examples/va/nvidia-mdev
- path: examples/va/nvidia-mdev/control-plane
wait_conditions:
- >-
oc -n openstack wait osctlplane controlplane --for condition=Ready
Expand All @@ -36,20 +36,47 @@ vas:
- name: edpm-nodeset-values
src_file: values.yaml
build_output: nodeset.yaml

- path: examples/va/nvidia-mdev/edpm/deployment
wait_conditions:
- >-
oc -n openstack wait
osdpns openstack-edpm --for condition=Ready
--timeout=60m
values:
- name: edpm-deployment-values
src_file: values.yaml
build_output: deployment.yaml
post_stage_run:
- name: Install nvidia driver
- name: Run phase 1 playbook
type: playbook
# As a reminder, the job needs to set the nvidia driver URL
source: "../../playbooks/nvidia-mdev.yml"
source: "../../playbooks/nvidia-mdev-phase1.yml"
inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml"

- path: examples/va/nvidia-mdev/edpm/deployment
- path: examples/va/nvidia-mdev
wait_conditions:
- >-
oc -n openstack wait
osdpns openstack-edpm --for condition=SetupReady
--timeout=60m
values:
- name: edpm-post-driver-nodeset-values
src_file: values.yaml
build_output: nodeset-post-driver.yaml
post_stage_run:
- name: Run phase 2 playbook
type: playbook
source: "../../playbooks/nvidia-mdev-phase2.yml"
inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml"

- path: examples/va/nvidia-mdev/edpm-post-driver/deployment
wait_conditions:
- >-
oc -n openstack wait
osdpns openstack-edpm --for condition=Ready
--timeout=60m
values:
- name: edpm-deployment-values
- name: edpm-post-driver-deployment-values
src_file: values.yaml
build_output: deployment.yaml
build_output: deployment-post-driver.yaml
98 changes: 98 additions & 0 deletions examples/va/nvidia-mdev/control-plane/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
---
apiVersion: kustomize.config.k8s.io/v1alpha1
kind: Component

transformers:
# Set namespace to OpenStack on all namespaced objects without a namespace
- |-
apiVersion: builtin
kind: NamespaceTransformer
metadata:
name: _ignored_
namespace: openstack
setRoleBindingSubjects: none
unsetOnly: true
fieldSpecs:
- path: metadata/name
kind: Namespace
create: true
components:
- ../../../../lib/networking/metallb
- ../../../../lib/networking/netconfig
- ../../../../lib/networking/nad
- ../../../../lib/control-plane

resources:
- nncp/values.yaml
- service-values.yaml

replacements:
# Control plane customization
- source:
kind: ConfigMap
name: service-values
fieldPath: data.neutron.customServiceConfig
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.neutron.template.customServiceConfig
options:
create: true
# OVN control plane SRIOV customization
- source:
kind: ConfigMap
name: service-values
fieldPath: data.ovn.ovnController.nicMappings
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.ovn.template.ovnController.nicMappings
options:
create: true
- source:
kind: ConfigMap
name: service-values
fieldPath: data.glance.customServiceConfig
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.glance.template.customServiceConfig
options:
create: true
- source:
kind: ConfigMap
name: service-values
fieldPath: data.glance.default.replicas
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.glance.template.glanceAPIs.default.replicas
options:
create: true
- source:
kind: ConfigMap
name: service-values
fieldPath: data.swift.enabled
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.swift.enabled
options:
create: true
- source:
kind: ConfigMap
name: service-values
fieldPath: data.nova.schedulerServiceTemplate.customServiceConfig
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.nova.template.schedulerServiceTemplate.customServiceConfig
options:
create: true
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ transformers:
create: true
components:
- ../../../../lib/nncp
- ../../../../../lib/nncp

resources:
- values.yaml
File renamed without changes.
44 changes: 44 additions & 0 deletions examples/va/nvidia-mdev/control-plane/service-values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: service-values
annotations:
config.kubernetes.io/local-config: "true"
data:
preserveJobs: false
neutron:
customServiceConfig: |
[DEFAULT]
service_plugins=qos,ovn-router,trunk,segments,port_forwarding,log,placement
[ml2]
mechanism_drivers=sriovnicswitch,ovn
extension_drivers=port_security,router,qos,segments,trunk,placement,port_numa_affinity_policy
[ml2_type_vlan]
network_vlan_ranges=datacentre:1:1000,physnet1:2000:2005
ovn:
ovnController:
nicMappings:
datacentre: enp6s0
glance:
customServiceConfig: |
[DEFAULT]
enabled_backends = default_backend:swift
[glance_store]
default_backend = default_backend
[default_backend]
swift_store_create_container_on_put = True
swift_store_auth_version = 3
swift_store_auth_address = {{ .KeystoneInternalURL }}
swift_store_endpoint_type = internalURL
swift_store_user = service:glance
swift_store_key = {{ .ServicePassword }}
default:
replicas: 1
swift:
enabled: true
nova:
schedulerServiceTemplate:
customServiceConfig: |
[filter_scheduler]
enabled_filtes = AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,PciPassthroughFilter,NUMATopologyFilter,AggregateInstanceExtraSpecsFilter
2 changes: 2 additions & 0 deletions examples/va/nvidia-mdev/edpm-post-driver/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
dataplane-deployment.yaml
dataplane-nodeset.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
dataplane-deployment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

components:
- ../../../../../va/nvidia-mdev/edpm-post-driver/deployment
# - https://github.com/openstack-k8s-operators/architecture/va/nvidia-mdev/edpm-post-driver/deployment?ref=main
## It's possible to replace ../../../../../va/nvidia-mdev/edpm-post-driver/deployment/ with a git checkout URL as per:
## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md

resources:
- values.yaml

replacements:
- source:
kind: ConfigMap
name: edpm-nodeset-post-driver-values
fieldPath: data.servicesOverride
targets:
- select:
kind: OpenStackDataPlaneDeployment
fieldPaths:
- spec.servicesOverride
options:
create: true
- source:
kind: ConfigMap
name: edpm-nodeset-post-driver-values
fieldPath: data.deployment.name
targets:
- select:
kind: OpenStackDataPlaneDeployment
fieldPaths:
- metadata.name
options:
create: true
29 changes: 29 additions & 0 deletions examples/va/nvidia-mdev/edpm-post-driver/deployment/values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
---
apiVersion: v1
kind: ConfigMap

metadata:
name: edpm-nodeset-post-driver-values
annotations:
config.kubernetes.io/local-config: "true"

data:
deployment:
name: edpm-deployment-post-driver
servicesOverride:
- bootstrap
- download-cache
- reboot-os
- configure-network
- validate-network
- install-os
- configure-os
- ssh-known-hosts
- run-os
- install-certs
- ovn
- neutron-ovn
- neutron-metadata
- neutron-sriov
- libvirt
- nova-custom-sriov
7 changes: 3 additions & 4 deletions examples/va/nvidia-mdev/edpm/nodeset/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -122,20 +122,19 @@ data:
services:
- bootstrap
- download-cache
- reboot-os
- configure-network
- validate-network
- install-os
- configure-os
- ssh-known-hosts
- run-os
- reboot-os
- install-certs
- libvirt
- ovn
- neutron-ovn
- nova-custom-sriov
- neutron-sriov
- neutron-metadata
- neutron-sriov
- libvirt
nova:
compute:
conf: |
Expand Down
11 changes: 6 additions & 5 deletions examples/va/nvidia-mdev/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

components:
- ../../../va/nvidia-mdev/
# - https://github.com/openstack-k8s-operators/architecture/va/nvidia-mdev?ref=main
## It's possible to replace ../../../va/nvidia-mdev/ with a git checkout URL as per:
- ../../../va/nvidia-mdev/edpm/nodeset
- ../../../va/nvidia-mdev/edpm-post-driver/nodeset
# - https://github.com/openstack-k8s-operators/architecture/va/nvidia-mdev/edpm-post-driver/nodeset?ref=main
## It's possible to replace ../../../../../va/nvidia-mdev/edpm-post-driver/nodeset/ with a git checkout URL as per:
## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md

resources:
- nncp/values.yaml
- service-values.yaml
- values.yaml
- edpm/nodeset/values.yaml
9 changes: 0 additions & 9 deletions examples/va/nvidia-mdev/service-values.yaml

This file was deleted.

29 changes: 29 additions & 0 deletions examples/va/nvidia-mdev/values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
---
apiVersion: v1
kind: ConfigMap

metadata:
name: edpm-nodeset-post-driver-values
annotations:
config.kubernetes.io/local-config: "true"

data:
nodeset:
services:
- bootstrap
- download-cache
- configure-network
- validate-network
- install-os
- ceph-hci-pre
- configure-os
- ssh-known-hosts
- run-os
- reboot-os
- install-certs
- ceph-client
- ovn
- neutron-metadata
- libvirt
- nova-custom-sriov

Check failure on line 29 in examples/va/nvidia-mdev/values.yaml

View workflow job for this annotation

GitHub Actions / yaml-lint

29:1 [empty-lines] too many blank lines (1 > 0)
Loading

0 comments on commit 41202f7

Please sign in to comment.