overcloud-host-image-build.yml

---
name: Build overcloud host images
on:
  workflow_dispatch:
    inputs:
      centos:
        description: Build CentOS Stream 8
        type: boolean
        default: true
      rocky8:
        description: Build Rocky Linux 8
        type: boolean
        default: true
      rocky9:
        description: Build Rocky Linux 9
        type: boolean
        default: true
      ubuntu-focal:
        description: Build Ubuntu 20.04 Focal
        type: boolean
        default: true
      ubuntu-jammy:
        description: Build Ubuntu 22.04 Jammy
        type: boolean
        default: true
    secrets:
      KAYOBE_VAULT_PASSWORD:
        required: true
      CLOUDS_YAML:
        required: true
      OS_APPLICATION_CREDENTIAL_ID:
        required: true
      OS_APPLICATION_CREDENTIAL_SECRET:
        required: true

env:
  ANSIBLE_FORCE_COLOR: True
  KAYOBE_ENVIRONMENT: ci-builder
  KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
jobs:
  overcloud-host-image-build:
    name: Build overcloud host images
    if: github.repository == 'stackhpc/stackhpc-kayobe-config'
    runs-on: arc-skc-host-image-builder-runner
    permissions: {}
    steps:
      - name: Install Package
        uses: ConorMacBride/install-package@main
        with:
          apt: git unzip nodejs python3-pip python3-venv ssh

      - uses: actions/checkout@v4
        with:
           submodules: true

      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: src/kayobe-config

      - name: Output image tag of the builder
        id: builder_image_tag
        run: |
          echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT

      - name: Determine OpenStack release
        id: openstack_release
        run: |
          BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
          echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT

      # Generate a tag to apply to all built overcloud host images.
      - name: Generate overcloud host image tag
        id: host_image_tag
        run: |
          echo "host_image_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT

      - name: Display overcloud host image tag
        run: |
          echo "${{ steps.host_image_tag.outputs.host_image_tag }}"

      - name: Clone StackHPC Kayobe repository
        uses: actions/checkout@v4
        with:
          repository: stackhpc/kayobe
          ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
          path: src/kayobe

      - name: Install Kayobe
        run: |
          mkdir -p venvs &&
          pushd venvs &&
          python3 -m venv kayobe &&
          source kayobe/bin/activate &&
          pip install -U pip &&
          pip install ../src/kayobe

      - name: Install terraform
        uses: hashicorp/setup-terraform@v2

      - name: Initialise terraform
        run: terraform init
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder

      - name: Generate SSH keypair
        run: ssh-keygen -f id_rsa -N ''
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder

      - name: Generate clouds.yaml
        run: |
          cat << EOF > clouds.yaml
          ${{ secrets.CLOUDS_YAML }}
          EOF
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder

      - name: Terraform Plan
        run: terraform plan
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder
        env:
          OS_CLOUD: "openstack"
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

      - name: Terraform Apply
        run: |
          for attempt in $(seq 5); do
              if terraform apply -auto-approve; then
                  echo "Created infrastructure on attempt $attempt"
                  exit 0
              fi
              echo "Failed to create infrastructure on attempt $attempt"
              sleep 10
              terraform destroy -auto-approve
              sleep 60
          done
          echo "Failed to create infrastructure after $attempt attempts"
          exit 1
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder
        env:
          OS_CLOUD: "openstack"
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

      - name: Get Terraform outputs
        id: tf_outputs
        run: |
          terraform output -json
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder

      - name: Write Terraform outputs
        run: |
          cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-outputs.yml
          ${{ steps.tf_outputs.outputs.stdout }}
          EOF

      - name: Write Terraform network config
        run: |
          cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml

          admin_oc_net_name: admin
          admin_cidr: "{{ access_cidr.value }}"
          admin_allocation_pool_start: 0.0.0.0
          admin_allocation_pool_end: 0.0.0.0
          admin_gateway: "{{ access_gw.value }}"
          admin_bootproto: dhcp
          admin_ips:
            controller0: "{{ access_ip_v4.value }}"
          EOF

      # Use a heredoc to define a multiline string output
      # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
      - name: Copy SSH key
        run: |
          cat src/kayobe-config/terraform/host-image-builder/id_rsa.pub >> ~/.ssh/authorized_keys 

      - name: Update inventory
        run: |
          cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF
          [seed]
          builder
          EOF

      - name: Debug 3
        run: |
          
          ssh-keyscan -v localhost

      - name: Debug 4
        run: |
          ssh-keyscan -v 127.0.0.1

      - name: Bootstrap the control host
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe control host bootstrap

      - name: Configure the seed host
        run: |
          source venvs/kayobe/bin/activate &&
          source src/kayobe-config/kayobe-env --environment ci-builder &&
          kayobe seed host configure

      # - name: Build a CentOS Stream 8 overcloud host image
      #   id: build_centos_stream_8
      #   continue-on-error: true
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     source src/kayobe-config/kayobe-env --environment ci-builder &&
      #     kayobe overcloud host image build --force-rebuild \
      #     -e os_distribution="centos" \
      #     -e os_release="8-stream" \
      #     -e stackhpc_overcloud_dib_name=overcloud-centos-8-stream
      #   env:
      #     KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      #   if: inputs.centos

      # - name: Upload CentOS Stream 8 overcloud host image to Ark
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     source src/kayobe-config/kayobe-env --environment ci-builder &&
      #     kayobe playbook run \
      #     src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
      #     -e image_path=/opt/kayobe/images/overcloud-centos-8-stream \
      #     -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
      #     -e os_distribution="centos" \
      #     -e os_release="8-stream"
      #   env:
      #     KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      #   if: inputs.centos && steps.build_centos_stream_8.outcome == 'success'

      # - name: Upload CentOS Stream 8 overcloud host image to Cloud
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     openstack image create \
      #     overcloud-centos-8-stream-${{ steps.host_image_tag.outputs.host_image_tag }} \
      #     --container-format bare \
      #     --disk-format qcow2 \
      #     --file /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.qcow2 \
      #     --shared \
      #     --os-cloud openstack \
      #     --progress
      #   env:
      #     OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
      #     OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
      #   if: inputs.centos && steps.build_centos_stream_8.outcome == 'success'

      # - name: Build a Rocky Linux 8 overcloud host image
      #   id: build_rocky_8
      #   continue-on-error: true
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     source src/kayobe-config/kayobe-env --environment ci-builder &&
      #     kayobe overcloud host image build --force-rebuild \
      #     -e os_distribution="rocky" \
      #     -e os_release="8" \
      #     -e stackhpc_overcloud_dib_name=overcloud-rocky-8
      #   env:
      #     KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      #   if: inputs.rocky8

      # - name: Upload Rocky Linux 8 overcloud host image to Ark
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     source src/kayobe-config/kayobe-env --environment ci-builder &&
      #     kayobe playbook run \
      #     src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
      #     -e image_path=/opt/kayobe/images/overcloud-rocky-8 \
      #     -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
      #     -e os_distribution="rocky" \
      #     -e os_release="8"
      #   env:
      #     KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      #   if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success'
        
      # - name: Upload Rocky Linux 8 overcloud host image to Cloud
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     openstack image create \
      #     overcloud-rocky-8-${{ steps.host_image_tag.outputs.host_image_tag }} \
      #     --container-format bare \
      #     --disk-format qcow2 \
      #     --file /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.qcow2 \
      #     --shared \
      #     --os-cloud openstack \
      #     --progress
      #   env:
      #     OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
      #     OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
      #   if: inputs.rocky8 && steps.build_rocky_8.outcome == 'success'

      # - name: Build a Rocky Linux 9 overcloud host image
      #   id: build_rocky_9
      #   continue-on-error: true
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     source src/kayobe-config/kayobe-env --environment ci-builder &&
      #     kayobe overcloud host image build --force-rebuild \
      #     -e os_distribution="rocky" \
      #     -e os_release="9" \
      #     -e stackhpc_overcloud_dib_name=overcloud-rocky-9
      #   env:
      #     KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      #   if: inputs.rocky9

      # - name: Upload Rocky Linux 9 overcloud host image to Ark
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     source src/kayobe-config/kayobe-env --environment ci-builder &&
      #     kayobe playbook run \
      #     src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
      #     -e image_path=/opt/kayobe/images/overcloud-rocky-9 \
      #     -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
      #     -e os_distribution="rocky" \
      #     -e os_release="9"
      #   env:
      #     KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      #   if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'
        
      # - name: Upload Rocky Linux 9 overcloud host image to Cloud
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     openstack image create \
      #     overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} \
      #     --container-format bare \
      #     --disk-format qcow2 \
      #     --file /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2 \
      #     --shared \
      #     --os-cloud openstack \
      #     --progress
      #   env:
      #     OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
      #     OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
      #   if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'

      # - name: Build an Ubuntu Focal 20.04 overcloud host image
      #   id: build_ubuntu_focal
      #   continue-on-error: true
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     source src/kayobe-config/kayobe-env --environment ci-builder &&
      #     kayobe overcloud host image build --force-rebuild \
      #     -e os_distribution="ubuntu" \
      #     -e os_release="focal" \
      #     -e stackhpc_overcloud_dib_name=overcloud-ubuntu-focal
      #   env:
      #     KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      #   if: inputs.ubuntu-focal

      # - name: Upload Ubuntu Focal 20.04 overcloud host image to Ark
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     source src/kayobe-config/kayobe-env --environment ci-builder &&
      #     kayobe playbook run \
      #     src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
      #     -e image_path=/opt/kayobe/images/overcloud-ubuntu-focal \
      #     -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
      #     -e os_distribution="ubuntu" \
      #     -e os_release="focal"
      #   env:
      #     KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      #   if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success'

      # - name: Upload Ubuntu Focal 20.04 overcloud host image to Cloud
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     openstack image create \
      #     overcloud-ubuntu-focal-${{ steps.host_image_tag.outputs.host_image_tag }} \
      #     --container-format bare \
      #     --disk-format qcow2 \
      #     --file /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.qcow2 \
      #     --shared \
      #     --os-cloud openstack \
      #     --progress
      #   env:
      #     OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
      #     OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
      #   if: inputs.ubuntu-focal && steps.build_ubuntu_focal.outcome == 'success'

      # - name: Build an Ubuntu Jammy 22.04 overcloud host image
      #   id: build_ubuntu_jammy
      #   continue-on-error: true
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     source src/kayobe-config/kayobe-env --environment ci-builder &&
      #     kayobe overcloud host image build --force-rebuild \
      #     -e os_distribution="ubuntu" \
      #     -e os_release="jammy" \
      #     -e stackhpc_overcloud_dib_name=overcloud-ubuntu-jammy
      #   env:
      #     KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      #   if: inputs.ubuntu-jammy

      # - name: Upload Ubuntu Jammy 22.04 overcloud host image to Ark
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     source src/kayobe-config/kayobe-env --environment ci-builder &&
      #     kayobe playbook run \
      #     src/kayobe-config/etc/kayobe/ansible/pulp-host-image-upload.yml \
      #     -e image_path=/opt/kayobe/images/overcloud-ubuntu-jammy \
      #     -e host_image_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \
      #     -e os_distribution="ubuntu" \
      #     -e os_release="jammy"
      #   env:
      #     KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      #   if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'

      # - name: Upload Ubuntu Jammy 22.04 overcloud host image to Cloud
      #   run: |
      #     source venvs/kayobe/bin/activate &&
      #     openstack image create \
      #     overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} \
      #     --container-format bare \
      #     --disk-format qcow2 \
      #     --file /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2 \
      #     --shared \
      #     --os-cloud openstack \
      #     --progress
      #   env:
      #     OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
      #     OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
      #   if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'

      # - name: Upload updated images artifact
      #   uses: actions/upload-artifact@v4
      #   with:
      #     name: Updated images list
      #     path: /tmp/updated_images.txt
      #     retention-days: 7
      #   if: steps.build_centos_stream_8.outcome == 'success' ||
      #       steps.build_rocky_8.outcome == 'success' ||
      #       steps.build_rocky_9.outcome == 'success' ||
      #       steps.build_ubuntu_focal.outcome == 'success' ||
      #       steps.build_ubuntu_jammy.outcome == 'success'

      # - name: Upload CentOS build logs if build failed
      #   uses: actions/upload-artifact@v4
      #   with:
      #     name: CentOS build logs
      #     path: |
      #       /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stdout
      #       /opt/kayobe/images/overcloud-centos-8-stream/overcloud-centos-8-stream.stderr
      #     retention-days: 7
      #   if: steps.build_centos_stream_8.outcome == 'failure'

      # - name: Upload Rocky 8 build logs if build failed
      #   uses: actions/upload-artifact@v4
      #   with:
      #     name: Rocky 8 build logs
      #     path: |
      #       /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stdout
      #       /opt/kayobe/images/overcloud-rocky-8/overcloud-rocky-8.stderr
      #     retention-days: 7
      #   if: steps.build_rocky_8.outcome == 'failure'

      # - name: Upload Rocky 9 build logs if build failed
      #   uses: actions/upload-artifact@v4
      #   with:
      #     name: Rocky 9 build logs
      #     path: |
      #       /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout
      #       /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stderr
      #     retention-days: 7
      #   if: steps.build_rocky_9.outcome == 'failure'

      # - name: Upload Ubuntu Focal 20.04 build logs if build failed
      #   uses: actions/upload-artifact@v4
      #   with:
      #     name: Ubuntu Focal 20.04 build logs
      #     path: |
      #       /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stdout
      #       /opt/kayobe/images/overcloud-ubuntu-focal/overcloud-ubuntu-focal.stderr
      #     retention-days: 7
      #   if: steps.build_ubuntu_focal.outcome == 'failure'

      # - name: Upload Ubuntu Jammy 22.04 build logs if build failed
      #   uses: actions/upload-artifact@v4
      #   with:
      #     name: Ubuntu Jammy 22.04 build logs
      #     path: |
      #       /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout
      #       /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stderr
      #     retention-days: 7
      #   if: steps.build_ubuntu_jammy.outcome == 'failure'

      # - name: Fail if any overcloud host image builds failed
      #   run: |
      #     echo "Builds failed. See workflow artifacts for details." &&
      #     exit 1
      #   if: steps.build_centos_stream_8.outcome == 'failure' ||
      #       steps.build_rocky_8.outcome == 'failure' ||
      #       steps.build_rocky_9.outcome == 'failure' ||
      #       steps.build_ubuntu_focal.outcome == 'failure' ||
      #       steps.build_ubuntu_jammy.outcome == 'failure'

      - name: Destroy
        run: terraform destroy -auto-approve
        working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/host-image-builder
        env:
          OS_CLOUD: openstack
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
        if: always()