testo

.github/workflows/overcloud-host-image-build.yml

@@ -39,18 +39,26 @@ jobs:
   overcloud-host-image-build:
     name: Build overcloud host images
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
-    runs-on: arc-skc-host-image-builder-runner
+    runs-on: arc-skc-aio-runner # TODO: switch back to host image builder
     permissions: {}
     steps:
-      - name: Install package dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y debootstrap gdisk zstd build-essential git unzip nodejs python3-wheel python3-pip python3-venv rpm dnf
+      - name: Install Package
+        uses: ConorMacBride/install-package@main
+        with:
+          apt: git unzip nodejs
 
-      - name: Checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
-          path: src/kayobe-config
+           submodules: true
+
+      - name: Make sure dockerd is running and test Docker
+        run: |
+          docker ps
+
+      - name: Output host image builder tag
+        id: builder_image_tag
+        run: |
+          echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
 
       - name: Determine OpenStack release
         id: openstack_release
@@ -68,47 +76,78 @@ jobs:
         run: |
           echo "${{ steps.host_image_tag.outputs.host_image_tag }}"
 
-      - name: Clone StackHPC Kayobe repository
-        uses: actions/checkout@v4
-        with:
-          repository: stackhpc/kayobe
-          ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
-          path: src/kayobe
+      - name: Install terraform
+        uses: hashicorp/setup-terraform@v2
 
-      - name: Install Kayobe
-        run: |
-          mkdir -p venvs &&
-          pushd venvs &&
-          python3 -m venv kayobe &&
-          source kayobe/bin/activate &&
-          pip install -U pip &&
-          pip install ../src/kayobe
-
-      - name: Bootstrap the control host
-        run: |
-          source venvs/kayobe/bin/activate &&
-          source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe control host bootstrap --skip-tags bootstrap
+      - name: Initialise terraform
+        run: terraform init
+        working-directory: ${{ github.workspace }}/terraform/host-image-builder
 
-      - name: Create bifrost_httpboot Docker volume
-        run: |
-          sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data
+      - name: Generate SSH keypair
+        run: ssh-keygen -f id_rsa -N ''
+        working-directory: ${{ github.workspace }}/terraform/host-image-builder
 
       - name: Generate clouds.yaml
         run: |
           cat << EOF > clouds.yaml
           ${{ secrets.CLOUDS_YAML }}
           EOF
+        working-directory: ${{ github.workspace }}/terraform/host-image-builder
 
-      - name: Install OpenStack client
+      - name: Generate terraform.tfvars
         run: |
-          source venvs/kayobe/bin/activate &&
-          pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
+          cat << EOF > terraform.tfvars
+          ssh_public_key = "id_rsa.pub"
+          ssh_username = "${{ env.SSH_USERNAME }}"
+          host_image_builder_vm_interface = "${{ env.VM_INTERFACE }}"
+          host_image_builder_vm_name = "${{ env.VM_NAME }}"
+          host_image_builder_vm_image = "${{ env.VM_IMAGE }}"
+          host_image_builder_vm_flavor = "${{ env.VM_FLAVOR }}"
+          host_image_builder_vm_network = "${{ env.VM_NETWORK }}"
+          host_image_builder_vm_subnet = "${{ env.VM_SUBNET }}"
+          host_image_builder_vm_tags = ${{ env.VM_TAGS }}
+          EOF
+        working-directory: ${{ github.workspace }}/terraform/host-image-builder
+        env:
+          SSH_USERNAME: "cloud-user"
+          VM_NAME: "skc-ci-host-image-builder"
+          VM_IMAGE: "overcloud-rocky-9-yoga-20240124T094316"
+          VM_FLAVOR: "en1.medium"
+          VM_NETWORK: "stackhpc-ci"
+          VM_SUBNET: "stackhpc-ci"
+          VM_INTERFACE: "ens3"
+          VM_TAGS: '["skc-ci-aio", "alex-testo"]'
+
+      - name: Terraform Plan
+        run: terraform plan
+        working-directory: ${{ github.workspace }}/terraform/host-image-builder
+        env:
+          OS_CLOUD: "openstack"
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
 
-      - name: Testo
+      - name: Terraform Apply
         run: |
-          mkdir -p /tmp
-          mkdir -p /dev
+          for attempt in $(seq 5); do
+              if terraform apply -auto-approve; then
+                  echo "Created infrastructure on attempt $attempt"
+                  exit 0
+              fi
+              echo "Failed to create infrastructure on attempt $attempt"
+              sleep 10
+              terraform destroy -auto-approve
+              sleep 60
+          done
+          echo "Failed to create infrastructure after $attempt attempts"
+          exit 1
+        working-directory: ${{ github.workspace }}/terraform/host-image-builder
+        env:
+          OS_CLOUD: "openstack"
+          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
+          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
+
+      - name: fail
+        run: exit 1
 
       - name: Build a CentOS Stream 8 overcloud host image
         id: build_centos_stream_8

terraform/host-image-builder/README.rst

@@ -0,0 +1,191 @@
+==========================
+Terraform All in one (aio)
+==========================
+
+This Terraform configuration deploys a single VM on an OpenStack cloud, to be
+used as an all-in-one Kayobe test environment.
+
+This configuration is used in the GitHub Actions all-in-one.yml workflow for CI
+testing.
+
+Usage
+=====
+
+These instructions show how to use this Terraform configuration manually. They
+assume you are running an Ubuntu host that will be used to run Terraform. The
+machine should have network access to the VM that will be created by this
+configuration.
+
+Install Terraform:
+
+.. code-block:: console
+
+   wget -qO - terraform.gpg https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/terraform-archive-keyring.gpg
+   sudo echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/terraform-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/terraform.list
+   sudo apt update
+   sudo apt install docker.io terraform
+
+Clone and initialise the Kayobe config:
+
+.. code-block:: console
+
+   git clone https://github.com/stackhpc/stackhpc-kayobe-config
+   cd stackhpc-kayobe-config
+   git submodule init
+   git submodule update
+
+Change to the terraform/aio directory:
+
+.. code-block:: console
+
+   cd terraform/aio
+
+Initialise Terraform:
+
+.. code-block:: console
+
+   terraform init
+
+Generate an SSH keypair:
+
+.. code-block:: console
+
+   ssh-keygen -f id_rsa -N ''
+
+Create an OpenStack clouds.yaml file with your credentials to access an
+OpenStack cloud. Alternatively, download one from Horizon.
+
+.. code-block:: console
+
+   cat << EOF > clouds.yaml
+   ---
+   clouds:
+     sms-lab:
+       auth:
+         auth_url: https://api.sms-lab.cloud:5000
+         username: <username>
+         project_name: <project>
+         domain_name: default
+       interface: public
+   EOF
+
+Export environment variables to use the correct cloud and provide a password:
+
+.. code-block:: console
+
+   export OS_CLOUD=sms-lab
+   read -p OS_PASSWORD -s OS_PASSWORD
+   export OS_PASSWORD
+
+Generate Terraform variables:
+
+.. code-block:: console
+
+   cat << EOF > terraform.tfvars
+   ssh_public_key = "id_rsa.pub"
+   aio_vm_name = "kayobe-aio"
+   aio_vm_image = "overcloud-centos-8-stream-yoga-20230525T095243"
+   aio_vm_flavor = "general.v1.medium"
+   aio_vm_network = "stackhpc-ipv4-geneve"
+   aio_vm_subnet = "stackhpc-ipv4-geneve-subnet"
+   EOF
+
+Generate a plan:
+
+.. code-block:: console
+
+   terraform plan
+
+Apply the changes:
+
+.. code-block:: console
+
+   terraform apply -auto-approve
+
+Write Terraform outputs to a Kayobe config file:
+
+.. code-block:: console
+
+   terraform output -json > ../../etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-outputs.yml
+
+Change to the repository root:
+
+.. code-block:: console
+
+   cd ../../
+
+Write Terraform network config:
+
+.. code-block:: console
+
+   cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml
+
+   admin_oc_net_name: admin
+   admin_cidr: "{{ access_cidr.value }}"
+   admin_allocation_pool_start: 0.0.0.0
+   admin_allocation_pool_end: 0.0.0.0
+   admin_gateway: "{{ access_gw.value }}"
+   admin_bootproto: dhcp
+   admin_ips:
+     controller0: "{{ access_ip_v4.value }}"
+   EOF
+
+Write Terraform network interface config:
+
+.. code-block:: console
+
+   cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/controllers/tf-network-interfaces
+   admin_interface: "{{ access_interface.value }}"
+   EOF
+
+Build a Kayobe image:
+
+.. code-block:: console
+
+   sudo DOCKER_BUILDKIT=1 docker build --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest .
+
+Use the ci-aio environment:
+
+.. code-block:: console
+
+   export KAYOBE_ENVIRONMENT=ci-aio
+
+Set the Kayobe Vault password env var:
+
+.. code-block:: console
+
+   read -p KAYOBE_VAULT_PASSWORD -s KAYOBE_VAULT_PASSWORD
+   export KAYOBE_VAULT_PASSWORD
+
+Set the Kayobe SSH private key env var:
+
+.. code-block:: console
+
+   export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY=$(cat terraform/aio/id_rsa)
+
+Host configure:
+
+.. code-block:: console
+
+   sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
+
+Service deploy:
+
+.. code-block:: console
+
+   sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh
+
+Configure aio resources:
+
+.. code-block:: console
+
+   sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml
+
+Run Tempest:
+
+.. code-block:: console
+
+   mkdir -p tempest-artifacts
+   sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack
+
+Tempest results are in tempest-artifacts.

terraform/host-image-builder/outputs.tf

@@ -0,0 +1,15 @@
+output "access_ip_v4" {
+  value = openstack_compute_instance_v2.kayobe-host-image-builder.access_ip_v4
+}
+
+output "access_cidr" {
+  value = data.openstack_networking_subnet_v2.network.cidr
+}
+
+output "access_gw" {
+  value = data.openstack_networking_subnet_v2.network.gateway_ip
+}
+
+output "access_interface" {
+  value = var.host_image_builder_interface
+}

terraform/host-image-builder/provider.tf

@@ -0,0 +1,14 @@
+#provider "openstack" {
+# use environment variables
+#}
+
+terraform {
+  required_version = ">= 0.14"
+  backend "local" {
+  }
+  required_providers {
+    openstack = {
+      source = "terraform-provider-openstack/openstack"
+    }
+  }
+}

terraform/host-image-builder/templates/userdata.cfg.tpl

@@ -0,0 +1,21 @@
+#cloud-config
+# Don't automatically mount ephemeral disk
+mounts:
+  - [/dev/vdb, null]
+# WORKAROUND: internal DNS missing from SMS lab.
+runcmd:
+  - 'echo "10.0.0.34 pelican pelican.service.compute.sms-lab.cloud" >> /etc/hosts'
+  - 'echo "10.205.3.187 pulp-server pulp-server.internal.sms-cloud" >> /etc/hosts'
+# Configure SSH keys here, to avoid creating an ephemeral keypair.
+# This means only the instance needs to be cleaned up if the destroy fails.
+ssh_authorized_keys:
+  - ${ssh_public_key}
+
+write_files:
+  # WORKAROUND: https://bugs.launchpad.net/kolla-ansible/+bug/1995409
+  - content: |
+      #!/bin/bash
+      docker exec openvswitch_vswitchd ovs-vsctl "$@"
+    owner: root:root
+    path: /usr/bin/ovs-vsctl
+    permissions: '0755'