DOCA workflow support

.github/workflows/package-build-ofed.yml

@@ -1,5 +1,5 @@
 ---
-name: Build OFED packages
+name: Build OFED kernel modules
 on:
   workflow_dispatch:
     inputs:
@@ -23,7 +23,7 @@ env:
   KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 jobs:
   overcloud-ofed-packages:
-    name: Build OFED packages
+    name: Build OFED kernel modules
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: arc-skc-host-image-builder-runner
     permissions: {}
@@ -48,6 +48,11 @@ jobs:
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
           echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
+      - name: Generate OFED tag
+        id: ofed_tag
+        run: |
+          echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+
       - name: Clone StackHPC Kayobe repository
         uses: actions/checkout@v4
         with:
@@ -86,6 +91,7 @@ jobs:
         id: image_tag
         run: |
           echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+        working-directory: ${{ github.workspace }}/src/kayobe-config
 
       # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
       - name: Output image name
@@ -183,23 +189,31 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml \
+          -e seed_bootstrap_user="cloud-user" \
+          -e controller_bootstrap_user="cloud-user" \
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Configure the seed host (Builder VM)
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure --skip-tags network,docker
+          kayobe seed host configure \
+          --skip-tags network,docker,docker-registry \
+          -e seed_bootstrap_user="cloud-user" \
+          -e controller_bootstrap_user="cloud-user" \
+          -e dnf_install_doca=true \
+          -e dnf_enable_doca_modules=false \
+          -e stackhpc_repos_enabled=true
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Run a distro-sync
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host command run --become --command "dnf distro-sync --refresh"
+          kayobe seed host command run --become --command "dnf distro-sync --refresh --assumeyes"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
@@ -240,7 +254,8 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml \
+          -e "ofed_tag=${{ steps.ofed_tag.outputs.ofed_tag }}"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 

etc/kayobe/ansible/build-ofed-rocky.yml

@@ -1,5 +1,5 @@
 ---
-- name: Build OFED packages
+- name: Build OFED kernel modules
   become: true
   hosts: ofed-builder
   gather_facts: false
@@ -22,7 +22,6 @@
           - rpm-build
           - automake
           - patch
-          - kernel
           - kernel-devel
           - autoconf
           - pciutils
@@ -37,37 +36,10 @@
           - cmake-filesystem
           - libnl3-devel
           - python3-devel
+          - doca-extra
         state: latest
         update_cache: true
 
-    - name: Add DOCA host repository package
-      ansible.builtin.dnf:
-        name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_2.8/doca-host-2.8.0-204000_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm
-        disable_gpg_check: true
-
-    - name: Install DOCA extra packages
-      ansible.builtin.dnf:
-        name: doca-extra
-
-    - name: Create build directory
-      ansible.builtin.file:
-        path: /home/cloud-user/ofed
-        state: directory
-        mode: 0777
-
-    - name: Set build directory
-      ansible.builtin.replace:
-        path: /opt/mellanox/doca/tools/doca-kernel-support
-        regexp: 'TMP_DIR=\$1'
-        replace: 'TMP_DIR=/home/cloud-user/ofed'
-
     - name: Build OFED kernel modules
       ansible.builtin.shell:
-        cmd: |
-          /opt/mellanox/doca/tools/doca-kernel-support
-
-    - name: Download OFED userspace packages
-      ansible.builtin.dnf:
-        name: doca-ofed-userspace
-        download_only: true
-        download_dir: /home/cloud-user/ofed
+        cmd: /opt/mellanox/doca/tools/doca-kernel-support

etc/kayobe/ansible/push-ofed.yml

@@ -1,45 +1,80 @@
 ---
 - name: Push OFED packages
   hosts: ofed-builder
+  vars:
+    venv: "/opt/kayobe/venvs/kayobe"
+    doca_extract_path: "/home/stack/doca"
   tasks:
+    - name: Get OFED module repo variables
+      ansible.builtin.set_fact:
+        doca_modules_repo_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='name') | join('') }}"
+        doca_modules_repo_base_path: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='base_path') | join('') }}"
+        doca_modules_repo_distribution_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='distribution_name') | join('') }}"
+
     - name: Install python dependencies
       ansible.builtin.pip:
         name: pulp-cli
 
-    - name: Create Pulp repository for OFED
+    - name: Ensure Pulp configuration directory exists
+      ansible.builtin.file:
+        path: /home/stack/.config/pulp/
+        state: directory
+        recurse: true
+
+    - name: Setup Pulp credentials
+      ansible.builtin.blockinfile:
+        path: /home/stack/.config/pulp/cli.toml
+        create: true
+        block: |
+          [cli]
+          base_url = '{{ stackhpc_release_pulp_url }}'
+          verify_ssl = true
+          format = "json"
+          username = '{{ stackhpc_release_pulp_username }}'
+          password = '{{ stackhpc_release_pulp_password }}'
+      no_log: true
+
+    - name: Find DOCA kernel repo package
+      ansible.builtin.shell:
+        cmd: 'find /tmp/DOCA.* -name doca-kernel-repo-*'
+      register: doca_kernel_repo
+      changed_when: false
+
+    - name: Unpackage kernel repo
+      ansible.builtin.shell:
+        cmd: "rpm2cpio {{ doca_kernel_repo.stdout }} | cpio -idmv -D {{ doca_extract_path }}"
+
+    - name: Find extracted kernel packages
+      ansible.builtin.shell:
+        cmd: 'find {{ doca_extract_path }}/usr/share/doca-host-*/Modules/$(uname -r)/*.rpm'
+      register: doca_kernel_packages
+
+    - name: Create Pulp repository for DOCA kernel modules
       pulp.squeezer.rpm_repository:
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}"
+        name: "{{ doca_modules_repo_name }}"
         state: present
       retries: "{{ pulp_timeout_retries | default(3) }}"
 
-    - name: Lookup Pulp RPMs on builder
-      ansible.builtin.find:
-        paths: "/home/cloud-user/ofed"
-      register: rpm_dir
-
     - name: Upload OFED RPMs to Pulp
       ansible.builtin.shell:
         cmd: |
-          pulp \
-          --base-url '{{ stackhpc_release_pulp_url }}' \
-          --username '{{ stackhpc_release_pulp_username }}' \
-          --password '{{ stackhpc_release_pulp_password }}' \
+          {{ venv }}/bin/pulp \
           rpm content \
           --type package upload \
-          --repository '{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}' \
-          --file {{ item.path }} \
-      with_items: "{{ rpm_dir.files }}"
-      no_log: true
+          --repository '{{ doca_modules_repo_name }}' \
+          --file {{ item }} \
+          --no-publish \
+      with_items: "{{ doca_kernel_packages.stdout_lines }}"
 
     - name: Create Pulp publication for OFED
       pulp.squeezer.rpm_publication:
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        repository: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}"
+        repository: "{{ doca_modules_repo_name }}"
         state: present
       register: publication
 
@@ -48,7 +83,8 @@
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}"
+        name: "{{ doca_modules_repo_distribution_name + ofed_tag }}"
         publication: "{{ publication.publication.pulp_href }}"
-        base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}"
+        content_guard: development
+        base_path: "{{ doca_modules_repo_base_path }}/{{ ofed_tag }}"
         state: present

etc/kayobe/dnf.yml

@@ -47,7 +47,28 @@ dnf_custom_repos: "{{ stackhpc_dnf_repos if stackhpc_repos_enabled | bool else [
 # To use these repos, set stackhpc_repos_enabled to true.
 # This is done by default for hosts in the overcloud group via a group_vars
 # file.
-stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) }}"
+stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) | combine(dnf_custom_repos_doca if dnf_install_doca | bool else {}) }}"
+
+# DOCA repositories
+dnf_custom_repos_doca:
+  doca:
+    baseurl: "{{ stackhpc_repo_rhel9_doca_url }}"
+    description: "DOCA Online Repo {{ stackhpc_pulp_doca_version }} - RHEL $releasever"
+    enabled: "{{ dnf_enable_doca | bool | default(false) }}"
+    priority: -1
+    file: doca
+    gpgcheck: no
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
+  doca-modules:
+    baseurl: "{{ stackhpc_repo_rhel9_doca_modules_url }}"
+    description: "OFED Kernel modules for DOCA {{ stackhpc_pulp_doca_version }} - RHEL $releasever"
+    enabled: "{{ dnf_enable_doca_modules | bool | default(false) }}"
+    priority: -2
+    file: doca
+    gpgcheck: no
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
 
 # Custom repositories shared between all RHEL 9 derivatives.
 dnf_custom_repos_el9:
@@ -132,6 +153,12 @@ dnf_enable_epel: "{{ dnf_install_epel | bool }}"
 # Whether to enable the ELRepo repository. This affects RedHat-based, 9.x release systems only.
 dnf_enable_elrepo_9: "{{ dnf_install_elrepo_9 | bool }}"
 
+# Whether to enable DOCA repositories. This affects RedHat-based systems only.
+dnf_enable_doca: "{{ dnf_install_doca | bool }}"
+
+# Whether to enable the DOCA kernel module repository. This affects RedHat-based systems only.
+dnf_enable_doca_modules: "{{ dnf_install_doca | bool }}"
+
 # URL of EPEL GPG keys.
 dnf_epel_9_gpg_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9"
 
@@ -152,6 +179,9 @@ dnf_enable_docker: true
 #URL of docker repo GPG key
 dnf_docker_gpg_key_url: "https://download.docker.com/linux/centos/gpg"
 
+# Whether to create a repo file for DOCA. This affects RedHat-based
+# systems only. Default value is 'false'.
+#dnf_install_doca:
 
 ###############################################################################
 # DNF Automatic configuration.

etc/kayobe/environments/ci-aio/stackhpc-ci.yml

@@ -50,6 +50,8 @@ stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_vers
 stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}"
 stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}"
 stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
+stackhpc_repo_rhel9_doca_version: "{{ stackhpc_pulp_repo_rhel9_doca_version }}"
+stackhpc_repo_rhel9_doca_modules_version: "{{ stackhpc_pulp_repo_rhel9_doca_modules_version }}"
 
 # Rocky-and-CI-specific Pulp urls
 stackhpc_include_os_minor_version_in_repo_url: true

etc/kayobe/environments/ci-builder/inventory/group_vars/seed/lvm.yml → etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/lvm.yml

@@ -1,4 +1,5 @@
 ---
+
 # List of extra LVs to include in the rootvg VG.
 stackhpc_lvm_group_rootvg_lvs_extra:
   - "{{ stackhpc_lvm_lv_docker }}"
@@ -16,7 +17,7 @@ stackhpc_lvm_lv_root_size: 2g
 stackhpc_lvm_lv_tmp_size: 2g
 
 # StackHPC LVM lv_var LV size.
-stackhpc_lvm_lv_var_size: 2g
+stackhpc_lvm_lv_var_size: 3g
 
 # StackHPC LVM lv_var_tmp LV size.
 stackhpc_lvm_lv_var_tmp_size: 2g

etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/stackhpc-repos

@@ -0,0 +1,3 @@
+---
+
+stackhpc_repos_enabled: true

etc/kayobe/environments/ci-builder/inventory/hosts

@@ -2,6 +2,7 @@
 
 [ofed-builder:children]
 seed
+controllers
 
 # This is neccesary to run `overcloud deployment image build`
 [controllers]

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -75,6 +75,7 @@ stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_vers
 stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}"
 stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}"
 stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
+stackhpc_repo_rhel9_doca_version: "{{ stackhpc_pulp_repo_rhel9_doca_version }}"
 
 # Rocky-and-CI-specific Pulp urls
 stackhpc_include_os_minor_version_in_repo_url: true
@@ -107,3 +108,9 @@ ipa_build_dib_env_extra:
 # builder being a member of the 'overcloud' group for IPA builds.
 stackhpc_repos_enabled: false
 enable_docker_repo: true
+
+# Enable LVM for OFED builds
+seed_lvm_groups:
+  - "{{ stackhpc_lvm_group_rootvg if dnf_install_doca | bool }}"
+controller_lvm_groups:
+  - "{{ stackhpc_lvm_group_rootvg if dnf_install_doca | bool }}"

etc/kayobe/ofed.yml

@@ -1,12 +1,21 @@
 ---
-# DOCA OFED configuration
+# DOCA OFED configuration.
 
-# DOCA OFED version
-stackhpc_pulp_doca_ofed_version: 24.07
+###############################################################################
+# DOCA host version
+stackhpc_pulp_doca_version: 2.9.1
 
-# DOCA OFED repositories
-stackhpc_pulp_repo_doca_ofed_rhel9:
-  name: Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
-  url: "{{ stackhpc_release_pulp_content_url }}/doca_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
-  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}"
-  base_path: "mlnx_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
+###############################################################################
+# Pulp configuration for DOCA OFED
+
+# Whether to sync OFED kernel module packages into the local Pulp service
+stackhpc_pulp_sync_ofed_modules: false
+
+# DOCA Snapshot versions. The defaults use the appropriate version from
+# pulp-repo-versions.yml
+stackhpc_pulp_repo_rhel9_doca_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_doca_version') }}"
+stackhpc_pulp_repo_rhel9_doca_modules_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_doca_modules_version') }}"
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes

etc/kayobe/pulp-repo-versions.yml

@@ -49,3 +49,7 @@ stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20241127T003858
 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20240911T041957
 stackhpc_pulp_repo_ubuntu_jammy_security_version: 20240924T064114
 stackhpc_pulp_repo_ubuntu_jammy_version: 20240924T064114
+stackhpc_pulp_repo_rhel_9_4_doca_version: 20241211T153620
+stackhpc_pulp_repo_rhel_9_4_doca_modules_version: 20241213T112245
+stackhpc_pulp_repo_rhel_9_5_doca_version: 20241211T171301
+stackhpc_pulp_repo_rhel_9_5_doca_modules_version: 20241213T112245