DLPX-92522 appliance-build changes for LTS upgrade to 24.04

PR URL: https://www.github.com/delphix/appliance-build/pull/769

.ansible-lint-ignore

@@ -0,0 +1,100 @@
+# This file contains ignores rule violations for ansible-lint
+bootstrap/playbook.yml name[play]
+bootstrap/roles/appliance-build.bootstrap role-name
+bootstrap/roles/appliance-build.bootstrap/tasks/main.yml fqcn[action-core]
+bootstrap/roles/appliance-build.bootstrap/tasks/main.yml fqcn[action]
+bootstrap/roles/appliance-build.bootstrap/tasks/main.yml name[missing]
+bootstrap/roles/appliance-build.bootstrap/tasks/main.yml yaml[truthy]
+live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml command-instead-of-module
+live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml fqcn[action]
+live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml no-changed-when
+live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml risky-file-permissions
+live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml yaml[truthy]
+live-build/misc/ansible-roles/appliance-build.dct-common/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.dct-common/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.delphix-autofs/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.delphix-autofs/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.delphix-autofs/tasks/main.yml yaml[octal-values]
+live-build/misc/ansible-roles/appliance-build.delphix-ldap/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.delphix-ldap/tasks/main.yml literal-compare
+live-build/misc/ansible-roles/appliance-build.delphix-ldap/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.delphix-ldap/tasks/main.yml yaml[octal-values]
+live-build/misc/ansible-roles/appliance-build.delphix-ldap/tasks/main.yml yaml[truthy]
+live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.devops-development/tasks/main.yml yaml[truthy]
+live-build/misc/ansible-roles/appliance-build.masking-common/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.masking-common/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml risky-file-permissions
+live-build/misc/ansible-roles/appliance-build.masking-development/tasks/main.yml yaml[truthy]
+live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml command-instead-of-shell
+live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml no-changed-when
+live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml yaml[octal-values]
+live-build/misc/ansible-roles/appliance-build.minimal-common/tasks/main.yml yaml[truthy]
+live-build/misc/ansible-roles/appliance-build.minimal-development/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.minimal-development/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.minimal-development/tasks/main.yml yaml[octal-values]
+live-build/misc/ansible-roles/appliance-build.minimal-internal/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.minimal-internal/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.minimal-internal/tasks/main.yml yaml[octal-values]
+live-build/misc/ansible-roles/appliance-build.qa-internal/handlers/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.qa-internal/handlers/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.qa-internal/handlers/main.yml no-changed-when
+live-build/misc/ansible-roles/appliance-build.qa-internal/handlers/main.yml yaml[new-line-at-end-of-file]
+live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.qa-internal/tasks/main.yml yaml[octal-values]
+live-build/misc/ansible-roles/appliance-build.recovery-environment/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.recovery-environment/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.recovery-environment/tasks/main.yml yaml[empty-lines]
+live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml fqcn[action]
+live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml command-instead-of-module
+live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml no-changed-when
+live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml risky-file-permissions
+live-build/misc/ansible-roles/appliance-build.virtualization-common/tasks/main.yml yaml[truthy]
+live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml fqcn[action]
+live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml key-order[task]
+live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml risky-file-permissions
+live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml yaml[truthy]
+live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml fqcn[action-core]
+live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml name[missing]
+live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml risky-file-permissions
+live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml yaml[truthy]
+live-build/variants/external-dct/ansible/playbook.yml name[play]
+live-build/variants/external-dct/ansible/playbook.yml yaml[truthy]
+live-build/variants/external-standard/ansible/playbook.yml name[play]
+live-build/variants/external-standard/ansible/playbook.yml yaml[truthy]
+live-build/variants/internal-buildserver/ansible/playbook.yml name[play]
+live-build/variants/internal-buildserver/ansible/playbook.yml yaml[truthy]
+live-build/variants/internal-dcenter/ansible/playbook.yml name[play]
+live-build/variants/internal-dcenter/ansible/playbook.yml yaml[truthy]
+live-build/variants/internal-dct/ansible/playbook.yml name[play]
+live-build/variants/internal-dct/ansible/playbook.yml yaml[truthy]
+live-build/variants/internal-dev/ansible/playbook.yml name[play]
+live-build/variants/internal-dev/ansible/playbook.yml yaml[truthy]
+live-build/variants/internal-minimal/ansible/playbook.yml name[play]
+live-build/variants/internal-minimal/ansible/playbook.yml yaml[truthy]
+live-build/variants/internal-package-mirror/ansible/playbook.yml name[play]
+live-build/variants/internal-package-mirror/ansible/playbook.yml yaml[truthy]
+live-build/variants/internal-qa/ansible/playbook.yml name[play]
+live-build/variants/internal-qa/ansible/playbook.yml yaml[truthy]
+live-build/variants/internal-unittest/ansible/playbook.yml name[play]
+live-build/variants/internal-unittest/ansible/playbook.yml yaml[truthy]
+live-build/misc/ansible-roles/appliance-build.dcenter/tasks/main.yml syntax-check[unknown-module]
+live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml syntax-check[unknown-module]
+live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml syntax-check[unknown-module]
+bootstrap/roles/appliance-build.bootstrap/tasks/main.yml syntax-check[unknown-module]

.github/workflows/main.yml

@@ -2,12 +2,16 @@ on: [push, pull_request]
 
 jobs:
   check-ansible:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v1
-      - run: sudo ./.github/scripts/install-gradle.sh
-      - run: sudo -E ./.github/scripts/install-ansible-lint.sh
-      - run: /opt/gradle-5.1/bin/gradle ansibleCheck
+      - uses: actions/checkout@v4
+      - name: Run ansible-lint
+        uses: ansible/ansible-lint@main
+        with:
+          args: ""
+          setup_python: "true"
+          working_directory: ""
+          requirements_file: ""
   check-shellcheck:
     runs-on: ubuntu-20.04
     steps:

bootstrap/roles/appliance-build.bootstrap/tasks/main.yml

@@ -15,17 +15,22 @@
 #
 
 ---
+# The VSDK plugin requires python3.8. The deadsnakes PPA provides python3.8 on 24.04.
+- apt_repository:
+    repo: ppa:deadsnakes/ppa
+    state: present
+
 - apt:
     update_cache: yes
 
 - apt:
     name:
       - ansible
       - aptly
-      - awscli
       - bc
       - coreutils
       - devscripts
+      - docker.io
       - equivs
       - gdisk
       - git
@@ -36,15 +41,38 @@
       - livecd-rootfs
       - make
       - man
-      - openjdk-8-jre-headless
+      - openjdk-8-jdk
       - pigz
-      - qemu
+      # The VSDK plugin requires python3.8
+      - python3.8
+      - qemu-system
       - rename
       - shellcheck
       - vim
       - zfsutils-linux
     state: present
 
+- systemd:
+    name: unattended-upgrades
+    state: stopped
+  register: result_systemd_stop
+  failed_when: "result_systemd_stop is failed and 'Could not find the requested service' not in result_systemd_stop.msg"
+
+- apt:
+    name:
+      - unattended-upgrades
+    state: absent
+    purge: true
+
+- snap:
+    name: aws-cli
+    classic: yes
+
 - modprobe:
     name: zfs
     state: present
+
+- user:
+    name: delphix
+    groups: docker
+    append: true

build.gradle

@@ -105,13 +105,7 @@ task shellCheck(type: Exec) {
     commandLine(["shellcheck", "--exclude=SC1090,SC1091"] + shellScripts.getFiles())
 }
 
-task ansibleCheck(type: Exec) {
-    def ansibleFiles = fileTree("bootstrap").include("**/playbook.yml") +
-                       fileTree("live-build/variants").include("**/playbook.yml")
-    commandLine(["ansible-lint", "--exclude=SC1090,SC1091"] + ansibleFiles.getFiles())
-}
-
-tasks.check.dependsOn shellCheck, shfmtCheck, ansibleCheck
+tasks.check.dependsOn shellCheck, shfmtCheck
 
 task format() {
     dependsOn shfmt

live-build/auto/config

@@ -35,7 +35,7 @@ lb config noauto \
 	--bootstrap-flavour minimal \
 	--chroot-filesystem none \
 	--architectures amd64 \
-	--distribution focal \
+	--distribution noble \
 	--binary-images none \
 	--bootloader none \
 	--system normal \

live-build/config/archives/delphix-secondary-mirror.list.in

@@ -1,5 +1,5 @@
 #
-# Copyright 2019 Delphix
+# Copyright 2019, 2024 Delphix
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,6 @@
 # limitations under the License.
 #
 
-deb @@URL@@ focal main multiverse universe contrib stable
-deb @@URL@@ focal-updates main multiverse universe
-deb @@URL@@ focal-pgdg main
+deb @@URL@@ noble main multiverse universe contrib stable
+deb @@URL@@ noble-pgdg main
+deb @@URL@@ stable main

live-build/config/archives/localhost.list

@@ -22,4 +22,4 @@
 # used to serve the repository.
 #
 
-deb [trusted=yes] http://localhost:8080 focal main
+deb [trusted=yes] http://localhost:8080 noble main

live-build/misc/ansible-roles/appliance-build.buildserver-internal/tasks/main.yml

@@ -17,7 +17,7 @@
 ---
 - apt:
     name:
-      - adoptopenjdk-java8-jdk
+      - openjdk-8-jdk
       - curl
       - docker.io
       - git

live-build/misc/ansible-roles/appliance-build.unittest-internal/tasks/main.yml

@@ -17,7 +17,7 @@
 ---
 - apt:
     name:
-      - adoptopenjdk-java8-jdk
+      - openjdk-8-jdk
       - ant
       - bash
       - build-essential

live-build/variants/internal-dcenter/package-lists/dcenter.list.chroot

@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-adoptopenjdk-java8-jdk
+openjdk-8-jdk
 bind9
 dnsutils
 git

scripts/aptly-repo-from-debs.sh

@@ -79,6 +79,6 @@ rm -rf "$WORK_DIRECTORY"
 #
 # Generate an Aptly/APT repository
 #
-aptly repo create -distribution=focal -component=delphix upgrade-repository
+aptly repo create -distribution=noble -component=delphix upgrade-repository
 aptly repo add upgrade-repository debs
 aptly publish repo -skip-contents -skip-signing upgrade-repository

scripts/aptly-repo-from-image-diff.sh

@@ -93,7 +93,7 @@ popd &>/dev/null || die "'popd' failed"
 # system (e.g. "upgrade-image-from-aptly-repo.sh").
 #
 
-aptly repo create -distribution=focal -component=delphix upgrade-repository ||
+aptly repo create -distribution=noble -component=delphix upgrade-repository ||
 	die "failed to create repository: 'upgrade-repository'"
 aptly repo search image-a | xargs aptly repo copy image-a upgrade-repository ||
 	die "failed to copy packages to repository: 'upgrade-repository'"

scripts/build-ancillary-repository.sh

@@ -47,7 +47,7 @@ function build_ancillary_repository() {
 
 	rm -rf "$HOME/.aptly"
 	aptly repo create \
-		-distribution=focal -component=main ancillary-repository
+		-distribution=noble -component=main ancillary-repository
 	aptly repo add ancillary-repository "$pkg_directory"
 	aptly publish repo -skip-contents -skip-signing ancillary-repository
 

scripts/common.sh

@@ -186,3 +186,15 @@ function get_upstream_or_fail_if_unset() {
 		echo "$UPSTREAM_PRODUCT_BRANCH"
 	fi
 }
+
+function check_env() {
+	#
+	# When the job is ran manually for testing purposes, we do not expect
+	# all environment to be set, so skip the env check.
+	#
+	[[ -n "$JENKINS_URL" ]] || return 0
+
+	local val="${!1}"
+	[[ -n "$val" ]] || die "check_env: $1 must be non-empty"
+	return 0
+}

scripts/create-build-info-package.sh

@@ -68,18 +68,6 @@ mkdir "$target/appliance-build"
 cd "$TOP"
 git rev-parse HEAD >"$target/appliance-build/GIT_HASH"
 
-function check_env() {
-	#
-	# When the job is ran manually for testing purposes, we do not expect
-	# all environment to be set, so skip the env check.
-	#
-	[[ -n "$JENKINS_URL" ]] || return 0
-
-	local val="${!1}"
-	[[ -n "$val" ]] || die "check_env: $1 must be non-empty"
-	return 0
-}
-
 check_env APPLIANCE_BUILD_GIT_URL
 echo "$APPLIANCE_BUILD_GIT_URL" >"$target/appliance-build/GIT_URL"
 check_env APPLIANCE_BUILD_GIT_BRANCH