From c071fae8e82547c5cad537611831687def90a819 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Mon, 30 Dec 2024 10:40:38 -0800 Subject: [PATCH] TESTING NEW RUNNER --- .github/workflows/scripts/qemu-2-start.sh | 8 + .github/workflows/scripts/qemu-3-deps-vm.sh | 231 ++++++++++++++++++ .github/workflows/scripts/qemu-3-deps.sh | 240 ++----------------- .github/workflows/scripts/qemu-4-build-vm.sh | 208 ++++++++++++++++ .github/workflows/scripts/qemu-4-build.sh | 156 +----------- .github/workflows/scripts/qemu-6-tests.sh | 4 + .github/workflows/scripts/qemu-7-prepare.sh | 21 +- .github/workflows/zfs-qemu-packages.yml | 93 +++++++ .github/workflows/zfs-qemu.yml | 29 +-- 9 files changed, 590 insertions(+), 400 deletions(-) create mode 100755 .github/workflows/scripts/qemu-3-deps-vm.sh create mode 100755 .github/workflows/scripts/qemu-4-build-vm.sh create mode 100644 .github/workflows/zfs-qemu-packages.yml diff --git a/.github/workflows/scripts/qemu-2-start.sh b/.github/workflows/scripts/qemu-2-start.sh index 0906e438ac0d..23e7c87f8c97 100755 --- a/.github/workflows/scripts/qemu-2-start.sh +++ b/.github/workflows/scripts/qemu-2-start.sh @@ -200,6 +200,14 @@ EOF sudo virsh net-update default add ip-dhcp-host \ "" --live --config +# Make some hostnames for the VMs so we don't have to refer to them with +# hardcoded IP addresses +for i in {0..9} ; do + echo "192.168.122.1$i vm$vm" | sudo tee -a /etc/hosts +done +echo "ETC HOSTS" +sudo cat /etc/hosts + sudo virt-install \ --os-variant $OSv \ --name "openzfs" \ diff --git a/.github/workflows/scripts/qemu-3-deps-vm.sh b/.github/workflows/scripts/qemu-3-deps-vm.sh new file mode 100755 index 000000000000..ad2f876a3a3d --- /dev/null +++ b/.github/workflows/scripts/qemu-3-deps-vm.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash + +###################################################################### +# 3) install dependencies for compiling and loading +# +# $1: OS name (like 'fedora41)' +###################################################################### + +set -eu + +function archlinux() { + echo "##[group]Running pacman -Syu" + sudo btrfs filesystem resize max / + sudo pacman -Syu --noconfirm + echo "##[endgroup]" + + echo "##[group]Install Development Tools" + sudo pacman -Sy --noconfirm base-devel bc cpio cryptsetup dhclient dkms \ + fakeroot fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils \ + parted pax perf python-packaging python-setuptools qemu-guest-agent ksh \ + samba sysstat rng-tools rsync wget xxhash + echo "##[endgroup]" +} + +function debian() { + export DEBIAN_FRONTEND="noninteractive" + + echo "##[group]Running apt-get update+upgrade" + sudo apt-get update -y + sudo apt-get upgrade -y + echo "##[endgroup]" + + echo "##[group]Install Development Tools" + sudo apt-get install -y \ + acl alien attr autoconf bc cpio cryptsetup curl dbench dh-python dkms \ + fakeroot fio gdb gdebi git ksh lcov isc-dhcp-client jq libacl1-dev \ + libaio-dev libattr1-dev libblkid-dev libcurl4-openssl-dev libdevmapper-dev \ + libelf-dev libffi-dev libmount-dev libpam0g-dev libselinux-dev libssl-dev \ + libtool libtool-bin libudev-dev libunwind-dev linux-headers-$(uname -r) \ + lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \ + python3-cffi python3-dev python3-distlib python3-packaging \ + python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \ + rsync samba sysstat uuid-dev watchdog wget xfslibs-dev xxhash zlib1g-dev + echo "##[endgroup]" +} + +function freebsd() { + export ASSUME_ALWAYS_YES="YES" + + echo "##[group]Install Development Tools" + sudo pkg install -y autoconf automake autotools base64 checkbashisms fio \ + gdb gettext gettext-runtime git gmake gsed jq ksh93 lcov libtool lscpu \ + pkgconf python python3 pamtester pamtester qemu-guest-agent rsync xxhash + sudo pkg install -xy \ + '^samba4[[:digit:]]+$' \ + '^py3[[:digit:]]+-cffi$' \ + '^py3[[:digit:]]+-sysctl$' \ + '^py3[[:digit:]]+-packaging$' + echo "##[endgroup]" +} + +# common packages for: almalinux, centos, redhat +function rhel() { + echo "##[group]Running dnf update" + echo "max_parallel_downloads=10" | sudo -E tee -a /etc/dnf/dnf.conf + sudo dnf clean all + sudo dnf update -y --setopt=fastestmirror=1 --refresh + echo "##[endgroup]" + + echo "##[group]Install Development Tools" + + # Alma wants "Development Tools", Fedora 41 wants "development-tools" + if ! sudo dnf group install -y "Development Tools" ; then + echo "Trying 'development-tools' instead of 'Development Tools'" + sudo dnf group install -y development-tools + fi + + sudo dnf install -y \ + acl attr bc bzip2 cryptsetup curl dbench dkms elfutils-libelf-devel fio \ + gdb git jq kernel-rpm-macros ksh libacl-devel libaio-devel \ + libargon2-devel libattr-devel libblkid-devel libcurl-devel libffi-devel \ + ncompress libselinux-devel libtirpc-devel libtool libudev-devel \ + libuuid-devel lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester \ + parted perf python3 python3-cffi python3-devel python3-packaging \ + kernel-devel python3-setuptools qemu-guest-agent rng-tools rpcgen \ + rpm-build rsync samba sysstat systemd watchdog wget xfsprogs-devel xxhash \ + zlib-devel + echo "##[endgroup]" +} + +function tumbleweed() { + echo "##[group]Running zypper is TODO!" + sleep 23456 + echo "##[endgroup]" +} + +# Install dependencies +case "$1" in + almalinux8) + echo "##[group]Enable epel and powertools repositories" + sudo dnf config-manager -y --set-enabled powertools + sudo dnf install -y epel-release + echo "##[endgroup]" + rhel + echo "##[group]Install kernel-abi-whitelists" + sudo dnf install -y kernel-abi-whitelists + echo "##[endgroup]" + ;; + almalinux9|centos-stream9) + echo "##[group]Enable epel and crb repositories" + sudo dnf config-manager -y --set-enabled crb + sudo dnf install -y epel-release + echo "##[endgroup]" + rhel + echo "##[group]Install kernel-abi-stablelists" + sudo dnf install -y kernel-abi-stablelists + echo "##[endgroup]" + ;; + archlinux) + archlinux + ;; + debian*) + echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections + debian + echo "##[group]Install Debian specific" + sudo apt-get install -yq linux-perf dh-sequence-dkms + echo "##[endgroup]" + ;; + fedora*) + rhel + ;; + freebsd*) + freebsd + ;; + tumbleweed) + tumbleweed + ;; + ubuntu*) + debian + echo "##[group]Install Ubuntu specific" + sudo apt-get install -yq linux-tools-common libtirpc-dev \ + linux-modules-extra-$(uname -r) + if [ "$1" != "ubuntu20" ]; then + sudo apt-get install -yq dh-sequence-dkms + fi + echo "##[endgroup]" + echo "##[group]Delete Ubuntu OpenZFS modules" + for i in $(find /lib/modules -name zfs -type d); do sudo rm -rvf $i; done + echo "##[endgroup]" + ;; +esac + +# This script is used for checkstyle + zloop deps also. +# Install only the needed packages and exit - when used this way. +test -z "${ONLY_DEPS:-}" || exit 0 + +# Start services +echo "##[group]Enable services" +case "$1" in + freebsd*) + # add virtio things + echo 'virtio_load="YES"' | sudo -E tee -a /boot/loader.conf + for i in balloon blk console random scsi; do + echo "virtio_${i}_load=\"YES\"" | sudo -E tee -a /boot/loader.conf + done + echo "fdescfs /dev/fd fdescfs rw 0 0" | sudo -E tee -a /etc/fstab + sudo -E mount /dev/fd + sudo -E touch /etc/zfs/exports + sudo -E sysrc mountd_flags="/etc/zfs/exports" + echo '[global]' | sudo -E tee /usr/local/etc/smb4.conf >/dev/null + sudo -E service nfsd enable + sudo -E service qemu-guest-agent enable + sudo -E service samba_server enable + ;; + debian*|ubuntu*) + sudo -E systemctl enable nfs-kernel-server + sudo -E systemctl enable qemu-guest-agent + sudo -E systemctl enable smbd + ;; + *) + # All other linux distros + sudo -E systemctl enable nfs-server + sudo -E systemctl enable qemu-guest-agent + sudo -E systemctl enable smb + ;; +esac +echo "##[endgroup]" + +# Setup Kernel cmdline +CMDLINE="console=tty0 console=ttyS0,115200n8" +CMDLINE="$CMDLINE selinux=0" +CMDLINE="$CMDLINE random.trust_cpu=on" +CMDLINE="$CMDLINE no_timer_check" +case "$1" in + almalinux*|centos*|fedora*) + GRUB_CFG="/boot/grub2/grub.cfg" + GRUB_MKCONFIG="grub2-mkconfig" + CMDLINE="$CMDLINE biosdevname=0 net.ifnames=0" + echo 'GRUB_SERIAL_COMMAND="serial --speed=115200"' \ + | sudo tee -a /etc/default/grub >/dev/null + ;; + ubuntu24) + GRUB_CFG="/boot/grub/grub.cfg" + GRUB_MKCONFIG="grub-mkconfig" + echo 'GRUB_DISABLE_OS_PROBER="false"' \ + | sudo tee -a /etc/default/grub >/dev/null + ;; + *) + GRUB_CFG="/boot/grub/grub.cfg" + GRUB_MKCONFIG="grub-mkconfig" + ;; +esac + +case "$1" in + archlinux|freebsd*) + true + ;; + *) + echo "##[group]Edit kernel cmdline" + sudo sed -i -e '/^GRUB_CMDLINE_LINUX/d' /etc/default/grub || true + echo "GRUB_CMDLINE_LINUX=\"$CMDLINE\"" \ + | sudo tee -a /etc/default/grub >/dev/null + sudo $GRUB_MKCONFIG -o $GRUB_CFG + echo "##[endgroup]" + ;; +esac + +# reset cloud-init configuration and poweroff +sudo cloud-init clean --logs +sleep 2 && sudo poweroff & +exit 0 diff --git a/.github/workflows/scripts/qemu-3-deps.sh b/.github/workflows/scripts/qemu-3-deps.sh index 96979cd02e09..0d6120925122 100755 --- a/.github/workflows/scripts/qemu-3-deps.sh +++ b/.github/workflows/scripts/qemu-3-deps.sh @@ -1,229 +1,17 @@ -#!/usr/bin/env bash - ###################################################################### -# 3) install dependencies for compiling and loading +# 3) Wait for VM to boot from previous step and launch dependencies +# script on it. +# +# $1: OS name (like 'fedora41)' ###################################################################### -set -eu - -function archlinux() { - echo "##[group]Running pacman -Syu" - sudo btrfs filesystem resize max / - sudo pacman -Syu --noconfirm - echo "##[endgroup]" - - echo "##[group]Install Development Tools" - sudo pacman -Sy --noconfirm base-devel bc cpio cryptsetup dhclient dkms \ - fakeroot fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils \ - parted pax perf python-packaging python-setuptools qemu-guest-agent ksh \ - samba sysstat rng-tools rsync wget xxhash - echo "##[endgroup]" -} - -function debian() { - export DEBIAN_FRONTEND="noninteractive" - - echo "##[group]Running apt-get update+upgrade" - sudo apt-get update -y - sudo apt-get upgrade -y - echo "##[endgroup]" - - echo "##[group]Install Development Tools" - sudo apt-get install -y \ - acl alien attr autoconf bc cpio cryptsetup curl dbench dh-python dkms \ - fakeroot fio gdb gdebi git ksh lcov isc-dhcp-client jq libacl1-dev \ - libaio-dev libattr1-dev libblkid-dev libcurl4-openssl-dev libdevmapper-dev \ - libelf-dev libffi-dev libmount-dev libpam0g-dev libselinux-dev libssl-dev \ - libtool libtool-bin libudev-dev libunwind-dev linux-headers-$(uname -r) \ - lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \ - python3-cffi python3-dev python3-distlib python3-packaging \ - python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \ - rsync samba sysstat uuid-dev watchdog wget xfslibs-dev xxhash zlib1g-dev - echo "##[endgroup]" -} - -function freebsd() { - export ASSUME_ALWAYS_YES="YES" - - echo "##[group]Install Development Tools" - sudo pkg install -y autoconf automake autotools base64 checkbashisms fio \ - gdb gettext gettext-runtime git gmake gsed jq ksh93 lcov libtool lscpu \ - pkgconf python python3 pamtester pamtester qemu-guest-agent rsync xxhash - sudo pkg install -xy \ - '^samba4[[:digit:]]+$' \ - '^py3[[:digit:]]+-cffi$' \ - '^py3[[:digit:]]+-sysctl$' \ - '^py3[[:digit:]]+-packaging$' - echo "##[endgroup]" -} - -# common packages for: almalinux, centos, redhat -function rhel() { - echo "##[group]Running dnf update" - echo "max_parallel_downloads=10" | sudo -E tee -a /etc/dnf/dnf.conf - sudo dnf clean all - sudo dnf update -y --setopt=fastestmirror=1 --refresh - echo "##[endgroup]" - - echo "##[group]Install Development Tools" - - # Alma wants "Development Tools", Fedora 41 wants "development-tools" - if ! sudo dnf group install -y "Development Tools" ; then - echo "Trying 'development-tools' instead of 'Development Tools'" - sudo dnf group install -y development-tools - fi - - sudo dnf install -y \ - acl attr bc bzip2 cryptsetup curl dbench dkms elfutils-libelf-devel fio \ - gdb git jq kernel-rpm-macros ksh libacl-devel libaio-devel \ - libargon2-devel libattr-devel libblkid-devel libcurl-devel libffi-devel \ - ncompress libselinux-devel libtirpc-devel libtool libudev-devel \ - libuuid-devel lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester \ - parted perf python3 python3-cffi python3-devel python3-packaging \ - kernel-devel python3-setuptools qemu-guest-agent rng-tools rpcgen \ - rpm-build rsync samba sysstat systemd watchdog wget xfsprogs-devel xxhash \ - zlib-devel - echo "##[endgroup]" -} - -function tumbleweed() { - echo "##[group]Running zypper is TODO!" - sleep 23456 - echo "##[endgroup]" -} - -# Install dependencies -case "$1" in - almalinux8) - echo "##[group]Enable epel and powertools repositories" - sudo dnf config-manager -y --set-enabled powertools - sudo dnf install -y epel-release - echo "##[endgroup]" - rhel - echo "##[group]Install kernel-abi-whitelists" - sudo dnf install -y kernel-abi-whitelists - echo "##[endgroup]" - ;; - almalinux9|centos-stream9) - echo "##[group]Enable epel and crb repositories" - sudo dnf config-manager -y --set-enabled crb - sudo dnf install -y epel-release - echo "##[endgroup]" - rhel - echo "##[group]Install kernel-abi-stablelists" - sudo dnf install -y kernel-abi-stablelists - echo "##[endgroup]" - ;; - archlinux) - archlinux - ;; - debian*) - echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections - debian - echo "##[group]Install Debian specific" - sudo apt-get install -yq linux-perf dh-sequence-dkms - echo "##[endgroup]" - ;; - fedora*) - rhel - ;; - freebsd*) - freebsd - ;; - tumbleweed) - tumbleweed - ;; - ubuntu*) - debian - echo "##[group]Install Ubuntu specific" - sudo apt-get install -yq linux-tools-common libtirpc-dev \ - linux-modules-extra-$(uname -r) - if [ "$1" != "ubuntu20" ]; then - sudo apt-get install -yq dh-sequence-dkms - fi - echo "##[endgroup]" - echo "##[group]Delete Ubuntu OpenZFS modules" - for i in $(find /lib/modules -name zfs -type d); do sudo rm -rvf $i; done - echo "##[endgroup]" - ;; -esac - -# This script is used for checkstyle + zloop deps also. -# Install only the needed packages and exit - when used this way. -test -z "${ONLY_DEPS:-}" || exit 0 - -# Start services -echo "##[group]Enable services" -case "$1" in - freebsd*) - # add virtio things - echo 'virtio_load="YES"' | sudo -E tee -a /boot/loader.conf - for i in balloon blk console random scsi; do - echo "virtio_${i}_load=\"YES\"" | sudo -E tee -a /boot/loader.conf - done - echo "fdescfs /dev/fd fdescfs rw 0 0" | sudo -E tee -a /etc/fstab - sudo -E mount /dev/fd - sudo -E touch /etc/zfs/exports - sudo -E sysrc mountd_flags="/etc/zfs/exports" - echo '[global]' | sudo -E tee /usr/local/etc/smb4.conf >/dev/null - sudo -E service nfsd enable - sudo -E service qemu-guest-agent enable - sudo -E service samba_server enable - ;; - debian*|ubuntu*) - sudo -E systemctl enable nfs-kernel-server - sudo -E systemctl enable qemu-guest-agent - sudo -E systemctl enable smbd - ;; - *) - # All other linux distros - sudo -E systemctl enable nfs-server - sudo -E systemctl enable qemu-guest-agent - sudo -E systemctl enable smb - ;; -esac -echo "##[endgroup]" - -# Setup Kernel cmdline -CMDLINE="console=tty0 console=ttyS0,115200n8" -CMDLINE="$CMDLINE selinux=0" -CMDLINE="$CMDLINE random.trust_cpu=on" -CMDLINE="$CMDLINE no_timer_check" -case "$1" in - almalinux*|centos*|fedora*) - GRUB_CFG="/boot/grub2/grub.cfg" - GRUB_MKCONFIG="grub2-mkconfig" - CMDLINE="$CMDLINE biosdevname=0 net.ifnames=0" - echo 'GRUB_SERIAL_COMMAND="serial --speed=115200"' \ - | sudo tee -a /etc/default/grub >/dev/null - ;; - ubuntu24) - GRUB_CFG="/boot/grub/grub.cfg" - GRUB_MKCONFIG="grub-mkconfig" - echo 'GRUB_DISABLE_OS_PROBER="false"' \ - | sudo tee -a /etc/default/grub >/dev/null - ;; - *) - GRUB_CFG="/boot/grub/grub.cfg" - GRUB_MKCONFIG="grub-mkconfig" - ;; -esac - -case "$1" in - archlinux|freebsd*) - true - ;; - *) - echo "##[group]Edit kernel cmdline" - sudo sed -i -e '/^GRUB_CMDLINE_LINUX/d' /etc/default/grub || true - echo "GRUB_CMDLINE_LINUX=\"$CMDLINE\"" \ - | sudo tee -a /etc/default/grub >/dev/null - sudo $GRUB_MKCONFIG -o $GRUB_CFG - echo "##[endgroup]" - ;; -esac - -# reset cloud-init configuration and poweroff -sudo cloud-init clean --logs -sleep 2 && sudo poweroff & -exit 0 +while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do + ssh 2>/dev/null zfs@vm0 "uname -a" && break +done +scp .github/workflows/scripts/qemu-3-deps-vm.sh zfs@vm0:qemu-3-deps-vm.sh +PID=`pidof /usr/bin/qemu-system-x86_64` +ssh zfs@vm0 '$HOME/qemu-3-deps-vm.sh' $1 +# wait for poweroff to succeed +tail --pid=$PID -f /dev/null +sleep 5 # avoid this: "error: Domain is already active" +rm -f $HOME/.ssh/known_hosts diff --git a/.github/workflows/scripts/qemu-4-build-vm.sh b/.github/workflows/scripts/qemu-4-build-vm.sh new file mode 100755 index 000000000000..6a75fed4482b --- /dev/null +++ b/.github/workflows/scripts/qemu-4-build-vm.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash + +###################################################################### +# 4) configure and build openzfs modules +# +# Usage: +# +# qemu-4-build-vm.sh OS [--dkms] [--save] +# +# OS: OS name like 'fedora41' +# --dkms: Build DKMS RPMs as well (optional) +# --save: Save RPMs to artifacts after they're built +###################################################################### + +DKMS="" +SAVE_RPMS="" +while [[ $# -gt 0 ]]; do + case $1 in + --dkms) + DKMS=1 + shift # past value + ;; + --save) + SAVE_RPMS=1 + shift # past value + ;; + *) + OS=$1 + shift + ;; + esac +done + +set -eu + +function run() { + LOG="/var/tmp/build-stderr.txt" + echo "****************************************************" + echo "$(date) ($*)" + echo "****************************************************" + ($@ || echo $? > /tmp/rv) 3>&1 1>&2 2>&3 | stdbuf -eL -oL tee -a $LOG + if [ -f /tmp/rv ]; then + RV=$(cat /tmp/rv) + echo "****************************************************" + echo "exit with value=$RV ($*)" + echo "****************************************************" + echo 1 > /var/tmp/build-exitcode.txt + exit $RV + fi +} + +function freebsd() { + export MAKE="gmake" + echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure \ + --prefix=/usr/local \ + --with-libintl-prefix=/usr/local \ + --enable-pyzfs \ + --enable-debug \ + --enable-debuginfo + echo "##[endgroup]" + + echo "##[group]Build" + run gmake -j$(sysctl -n hw.ncpu) + echo "##[endgroup]" + + echo "##[group]Install" + run sudo gmake install + echo "##[endgroup]" +} + +function linux() { + echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure \ + --prefix=/usr \ + --enable-pyzfs \ + --enable-debug \ + --enable-debuginfo + echo "##[endgroup]" + + echo "##[group]Build" + run make -j$(nproc) + echo "##[endgroup]" + + echo "##[group]Install" + run sudo make install + echo "##[endgroup]" +} + +function rpm_build_and_install() { + EXTRA_CONFIG="${1:-}" + echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure --enable-debug --enable-debuginfo $EXTRA_CONFIG + echo "##[endgroup]" + + echo "##[group]Build" + run make pkg-kmod pkg-utils + echo "##[endgroup]" + + echo "##[group]Install" + run sudo dnf -y --nobest install $(ls *.rpm | grep -v src.rpm) + echo "##[endgroup]" + +} + +function deb_build_and_install() { +echo "##[group]Autogen.sh" + run ./autogen.sh + echo "##[endgroup]" + + echo "##[group]Configure" + run ./configure \ + --prefix=/usr \ + --enable-pyzfs \ + --enable-debug \ + --enable-debuginfo + echo "##[endgroup]" + + echo "##[group]Build" + run make native-deb-kmod native-deb-utils + echo "##[endgroup]" + + echo "##[group]Install" + # Do kmod install. Note that when you build the native debs, the + # packages themselves are placed in parent directory '../' rather than + # in the source directory like the rpms are. + run sudo apt-get -y install $(find ../ | grep -E '\.deb$' \ + | grep -Ev 'dkms|dracut') + echo "##[endgroup]" +} + +# Debug: show kernel cmdline +if [ -f /proc/cmdline ] ; then + cat /proc/cmdline || true +fi + +# Set our hostname to our OS name and version number. Specifically, we set the +# major and minor number so that when we query the RPMs we build, we can see +# what specific version of RHEL/ALMA we were using to build them. This is +# helpful for matching up KMOD versions. +# +# Examples: +# +# rhel8.10 +# almalinux9.5 +# fedora40 +source /etc/os-release +sudo hostname "$ID$VERSION_ID" + +# save some sysinfo +uname -a > /var/tmp/uname.txt + +cd $HOME/zfs +export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" + +# build +case "$OS" in + freebsd*) + freebsd + ;; + alma*|centos*) + rpm_build_and_install "--with-spec=redhat" + if [ -n "$DKMS" ] ; then + make rpm-dkms + fi + ;; + fedora*) + rpm_build_and_install + if [ -n "$DKMS" ] ; then + make rpm-dkms + fi + ;; + debian*|ubuntu*) + deb_build_and_install + ;; + *) + linux + ;; +esac + +# building the zfs module was ok +echo 0 > /var/tmp/build-exitcode.txt + +if [ -n "$SAVE_RPMS" ] ; then + cp *.rpm /var/tmp +fi + +echo "SETTING VARS, $DKMS, $SAVE_RPMS" +echo "DKMS=$DKMS" >> /var/tmp/env.txt +echo "SAVE_RPMS=$SAVE_RPMS" >> /var/tmp/env.txt +cat /var/tmp/env.txt + +# reset cloud-init configuration and poweroff +sudo cloud-init clean --logs +sync && sleep 2 && sudo poweroff & +exit 0 diff --git a/.github/workflows/scripts/qemu-4-build.sh b/.github/workflows/scripts/qemu-4-build.sh index 955f605f5bce..0ce9b388815d 100755 --- a/.github/workflows/scripts/qemu-4-build.sh +++ b/.github/workflows/scripts/qemu-4-build.sh @@ -3,151 +3,11 @@ ###################################################################### # 4) configure and build openzfs modules ###################################################################### - -set -eu - -function run() { - LOG="/var/tmp/build-stderr.txt" - echo "****************************************************" - echo "$(date) ($*)" - echo "****************************************************" - ($@ || echo $? > /tmp/rv) 3>&1 1>&2 2>&3 | stdbuf -eL -oL tee -a $LOG - if [ -f /tmp/rv ]; then - RV=$(cat /tmp/rv) - echo "****************************************************" - echo "exit with value=$RV ($*)" - echo "****************************************************" - echo 1 > /var/tmp/build-exitcode.txt - exit $RV - fi -} - -function freebsd() { - export MAKE="gmake" - echo "##[group]Autogen.sh" - run ./autogen.sh - echo "##[endgroup]" - - echo "##[group]Configure" - run ./configure \ - --prefix=/usr/local \ - --with-libintl-prefix=/usr/local \ - --enable-pyzfs \ - --enable-debug \ - --enable-debuginfo - echo "##[endgroup]" - - echo "##[group]Build" - run gmake -j$(sysctl -n hw.ncpu) - echo "##[endgroup]" - - echo "##[group]Install" - run sudo gmake install - echo "##[endgroup]" -} - -function linux() { - echo "##[group]Autogen.sh" - run ./autogen.sh - echo "##[endgroup]" - - echo "##[group]Configure" - run ./configure \ - --prefix=/usr \ - --enable-pyzfs \ - --enable-debug \ - --enable-debuginfo - echo "##[endgroup]" - - echo "##[group]Build" - run make -j$(nproc) - echo "##[endgroup]" - - echo "##[group]Install" - run sudo make install - echo "##[endgroup]" -} - -function rpm_build_and_install() { - EXTRA_CONFIG="${1:-}" - echo "##[group]Autogen.sh" - run ./autogen.sh - echo "##[endgroup]" - - echo "##[group]Configure" - run ./configure --enable-debug --enable-debuginfo $EXTRA_CONFIG - echo "##[endgroup]" - - echo "##[group]Build" - run make pkg-kmod pkg-utils - echo "##[endgroup]" - - echo "##[group]Install" - run sudo dnf -y --nobest install $(ls *.rpm | grep -v src.rpm) - echo "##[endgroup]" - -} - -function deb_build_and_install() { -echo "##[group]Autogen.sh" - run ./autogen.sh - echo "##[endgroup]" - - echo "##[group]Configure" - run ./configure \ - --prefix=/usr \ - --enable-pyzfs \ - --enable-debug \ - --enable-debuginfo - echo "##[endgroup]" - - echo "##[group]Build" - run make native-deb-kmod native-deb-utils - echo "##[endgroup]" - - echo "##[group]Install" - # Do kmod install. Note that when you build the native debs, the - # packages themselves are placed in parent directory '../' rather than - # in the source directory like the rpms are. - run sudo apt-get -y install $(find ../ | grep -E '\.deb$' \ - | grep -Ev 'dkms|dracut') - echo "##[endgroup]" -} - -# Debug: show kernel cmdline -if [ -f /proc/cmdline ] ; then - cat /proc/cmdline || true -fi - -# save some sysinfo -uname -a > /var/tmp/uname.txt - -cd $HOME/zfs -export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" - -# build -case "$1" in - freebsd*) - freebsd - ;; - alma*|centos*) - rpm_build_and_install "--with-spec=redhat" - ;; - fedora*) - rpm_build_and_install - ;; - debian*|ubuntu*) - deb_build_and_install - ;; - *) - linux - ;; -esac - -# building the zfs module was ok -echo 0 > /var/tmp/build-exitcode.txt - -# reset cloud-init configuration and poweroff -sudo cloud-init clean --logs -sync && sleep 2 && sudo poweroff & -exit 0 +echo "Build modules in QEMU machine" +sudo virsh start openzfs +while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do + ssh 2>/dev/null zfs@vm0 "uname -a" && break +done +rsync -ar $HOME/work/zfs/zfs zfs@vm0:./ + +ssh zfs@vm0 '$HOME/zfs/.github/workflows/scripts/qemu-4-build-vm.sh' $@ diff --git a/.github/workflows/scripts/qemu-6-tests.sh b/.github/workflows/scripts/qemu-6-tests.sh index 2f023198bbf6..d5fb14bde287 100755 --- a/.github/workflows/scripts/qemu-6-tests.sh +++ b/.github/workflows/scripts/qemu-6-tests.sh @@ -9,6 +9,10 @@ set -eu +cd /var/tmp +echo 0 > tests-exitcode.txt +exit + function prefix() { ID="$1" LINE="$2" diff --git a/.github/workflows/scripts/qemu-7-prepare.sh b/.github/workflows/scripts/qemu-7-prepare.sh index a5fbd7213161..525f4f275188 100755 --- a/.github/workflows/scripts/qemu-7-prepare.sh +++ b/.github/workflows/scripts/qemu-7-prepare.sh @@ -13,6 +13,16 @@ source env.txt mkdir -p $RESPATH +echo "/var/tmp/env.txt" +cat /var/tmp/env.txt + +# Special case, if we're only building RPMs and not testing them, then +# $SAVE_RPMS will be set, but $VMs will not. In that case, just set $VMs to +# the first VM (vm1), since that will be where we built the RPMs. +if [ -z ${VMs:-} ] && [ -z "${SAVE_RPMS:-}" ] ; then + VMs=1 +fi + # check if building the module has failed if [ -z ${VMs:-} ]; then cd $RESPATH @@ -27,12 +37,19 @@ fi BASE="$HOME/work/zfs/zfs" MERGE="$BASE/.github/workflows/scripts/merge_summary.awk" +echo "HOSTS IS" +sudo cat /etc/hosts # catch result files of testings (vm's should be there) for i in $(seq 1 $VMs); do - rsync -arL zfs@192.168.122.1$i:$RESPATH/current $RESPATH/vm$i || true - scp zfs@192.168.122.1$i:"/var/tmp/*.txt" $RESPATH/vm$i || true + rsync -arL zfs@vm$i:$RESPATH/current $RESPATH/vm$i || true + scp zfs@vm$i:"/var/tmp/*.txt" $RESPATH/vm$i || true + scp zfs@vm$i:"/var/tmp/*.rpm" $RESPATH/vm$i || true done cp -f /var/tmp/*.txt $RESPATH || true + +# If a runner wants to save the RPMs to artifacts, they will include them in +# /var/tmp. Copy them to the artifacts if they exist. +cp -f /var/tmp/*.rpm $RESPATH &>/dev/null || true cd $RESPATH # prepare result files for summary diff --git a/.github/workflows/zfs-qemu-packages.yml b/.github/workflows/zfs-qemu-packages.yml new file mode 100644 index 000000000000..d1c8c18bcbee --- /dev/null +++ b/.github/workflows/zfs-qemu-packages.yml @@ -0,0 +1,93 @@ +# This workflow is used to build and test RPM packages. It is intended to be +# used to build RPMs for ZFS releases, and then test install them from a +# repository. +name: zfs-qemu-packages + +on: + workflow_dispatch: + inputs: + dkms: + type: boolean + required: false + default: false + description: 'Build DKMS RPMs in addition to KMOD RPMs' + save_rpms: + type: boolean + required: false + default: false + description: 'Build DKMS & KMOD RPMs and save to artifacts' +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + zfs-qemu-packages-jobs: + outputs: + all_artifacts: ${{ steps.zfs-qemu-packages-jobs.outputs.ALL_ARTIFACTS }} + + name: qemu-VMs + strategy: + fail-fast: false + matrix: + os: ['almalinux9', 'fedora41'] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Setup QEMU + timeout-minutes: 10 + run: .github/workflows/scripts/qemu-1-setup.sh + + - name: Start build machine + timeout-minutes: 10 + run: .github/workflows/scripts/qemu-2-start.sh ${{ matrix.os }} + + - name: Install dependencies + timeout-minutes: 20 + run: | + .github/workflows/scripts/qemu-3-deps.sh ${{ matrix.os }} + + - name: Build modules + timeout-minutes: 30 + run: | + ARGS="" + if [ ${{ github.event.inputs.dkms }} == "true" ] ; then + ARGS+=" --dkms" + fi + if [ ${{ github.event.inputs.save_rpms }} == "true" ] ; then + ARGS+=" --save" + fi + echo "Running with args $ARGS" + .github/workflows/scripts/qemu-4-build.sh $ARGS ${{ matrix.os }} + + - name: Prepare artifacts + if: always() + timeout-minutes: 10 + run: | + mkdir /tmp/rpms + rsync -ar zfs@vm0:zfs/*.rpm /tmp/rpms || true + rsync -ar zfs@192.168.122.10:zfs/*.rpm /tmp/rpms || true + + - uses: actions/upload-artifact@v4 + id: artifact-upload + if: always() + with: + name: rpms-${{ matrix.os }} + path: /tmp/rpms/* + if-no-files-found: ignore + + cleanup: + if: always() + needs: [zfs-qemu-packages-jobs] + name: Cleanup + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + id: artifact-download + if: always() + - name: Test Summary + if: always() + run: | + ls -R diff --git a/.github/workflows/zfs-qemu.yml b/.github/workflows/zfs-qemu.yml index 4748e90db50b..c57fdca0d02e 100644 --- a/.github/workflows/zfs-qemu.yml +++ b/.github/workflows/zfs-qemu.yml @@ -1,8 +1,9 @@ name: zfs-qemu on: - push: - pull_request: +# push: +# pull_request: + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -70,31 +71,11 @@ jobs: - name: Install dependencies timeout-minutes: 20 - run: | - echo "Install dependencies in QEMU machine" - IP=192.168.122.10 - while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do - ssh 2>/dev/null zfs@$IP "uname -a" && break - done - scp .github/workflows/scripts/qemu-3-deps.sh zfs@$IP:qemu-3-deps.sh - PID=`pidof /usr/bin/qemu-system-x86_64` - ssh zfs@$IP '$HOME/qemu-3-deps.sh' ${{ matrix.os }} - # wait for poweroff to succeed - tail --pid=$PID -f /dev/null - sleep 5 # avoid this: "error: Domain is already active" - rm -f $HOME/.ssh/known_hosts + run: .github/workflows/scripts/qemu-3-deps.sh ${{ matrix.os }} - name: Build modules timeout-minutes: 30 - run: | - echo "Build modules in QEMU machine" - sudo virsh start openzfs - IP=192.168.122.10 - while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do - ssh 2>/dev/null zfs@$IP "uname -a" && break - done - rsync -ar $HOME/work/zfs/zfs zfs@$IP:./ - ssh zfs@$IP '$HOME/zfs/.github/workflows/scripts/qemu-4-build.sh' ${{ matrix.os }} + run: .github/workflows/scripts/qemu-4-build.sh ${{ matrix.os }} - name: Setup testing machines timeout-minutes: 5