From f1643696d6c3a653693333169977dbb1ab19e296 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 29 Jun 2021 14:25:38 -0700 Subject: [PATCH 01/12] DLPX-74045 adapt upgrade "execute" script to better handle new hotfix workflow (#520) (#568) Currently when applying a hotfix image generated by the new hotfix workflow being developed in CP-4190, nothing will happen when the upgrade is attempted. This is due to the "delphix-entire" package in the hotfix being installed, matching the version of the package that's already installed. Thus, the package manager chooses not to install the new packagecontained in the upgrade/hotfix image, since it thinks the same package is already installed, due to the two having the same version. We need to adapt the "execute" script that's used during upgrade, such that we always install the new "delphix-entire" package contained in the upgrade/hotfix image, even if the package versions are identical. --- upgrade/upgrade-scripts/execute | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/execute b/upgrade/upgrade-scripts/execute index 2ff40480..16e817d4 100755 --- a/upgrade/upgrade-scripts/execute +++ b/upgrade/upgrade-scripts/execute @@ -303,9 +303,23 @@ dpkg-query -Wf '${Package}\n' "delphix-kernel-*" | xargs apt-mark manual || # # shellcheck disable=SC2153 -apt_get install -y --allow-downgrades "delphix-entire-$platform=$VERSION" || +apt_get install \ + -y --allow-downgrades --reinstall \ + "delphix-entire-$platform=$VERSION" || die "upgrade failed; from '$CURRENT_VERSION' to '$VERSION'" +# +# Since we mark all currently installed packages as "auto" earlier in +# this script, we need to ensure the delphix-entire package that we just +# installed is marked "manual", to avoid it being removed later in this +# script. We must explicitly do this here, as it seems this isn't +# automatially done when installing or upgrading the package; +# particularly in the case of "--reinstall", which replaces the current +# package with a new package of the same version (i.e. for hotfixes). +# +apt-mark manual "delphix-entire-$platform" || + die "failed to mark 'delphix-entire' package as 'manual' installed" + [[ -f "/usr/share/doc/delphix-entire-$platform/packages.list.gz" ]] || die "delphix-entire's packages.list.gz file is missing" From fd57669ffb6f957fc2621d228f1093968c3f7aa9 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 6 Jul 2021 12:59:55 -0700 Subject: [PATCH 02/12] TOOL-11824 Harden against mistakes when running via Jenkins (#569) (#570) This change adds logic to cause a failure when certain environment variables are missing, and the build is being run via Jenkins. When environment variables are missing, we currently will provide default values, such that the build doesn't fail. These default values generally are not sufficient for anything besides allowing the build to complete (e.g. the resultant upgrade images may fail upgrade). Further, the main motivation for generating these default values, is to make it easier to run the build manually, and aid developer iteration. Thus, when it's clear that the build is being run via Jenkins, where these missing environment variables more likely point to an error or broken automation, than an intentional omission of these variables, it's better to report the error than set and use a default value. This way, we can more quickly detect and address the problem, than silently produce (potentially) incorrect or broken artifacts. --- scripts/upgrade-image-from-aptly-repo.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/upgrade-image-from-aptly-repo.sh b/scripts/upgrade-image-from-aptly-repo.sh index 7eb18aa1..54358647 100755 --- a/scripts/upgrade-image-from-aptly-repo.sh +++ b/scripts/upgrade-image-from-aptly-repo.sh @@ -122,6 +122,8 @@ if [[ -n "$DELPHIX_UPGRADE_MINIMUM_VERSION" ]]; then "s/@@MINIMUM_VERSION@@/$DELPHIX_UPGRADE_MINIMUM_VERSION/" \ version.info || die "failed to set MINIMUM_VERSION in version.info" +elif [[ -n "$JENKINS_URL" ]]; then + die "DELPHIX_UPGRADE_MINIMUM_VERSION not specified" else sed -i "s/@@MINIMUM_VERSION@@/0.0.0.0/" version.info || die "failed to set MINIMUM_VERSION in version.info" @@ -132,6 +134,8 @@ if [[ -n "$DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION" ]]; then "s/@@MINIMUM_REBOOT_OPTIONAL_VERSION@@/$DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION/" \ version.info || die "failed to set MINIMUM_REBOOT_OPTIONAL_VERSION in version.info" +elif [[ -n "$JENKINS_URL" ]]; then + die "DELPHIX_UPGRADE_MINIMUM_REBOOT_OPTIONAL_VERSION not specified" else sed -i "s/@@MINIMUM_REBOOT_OPTIONAL_VERSION@@/0.0.0.0/" version.info || die "failed to set MINIMUM_REBOOT_OPTIONAL_VERSION in version.info" @@ -147,6 +151,12 @@ sha256sum payload.tar.gz version.info verification-version.info prepare >SHA256S # DELPHIX_SIGNATURE_TOKEN environment variable contents to stdout. # set +o xtrace +if [[ -n "$JENKINS_URL" ]]; then + [[ -n "$DELPHIX_SIGNATURE_VERSIONS" ]] || die "DELPHIX_SIGNATURE_VERSIONS not specified" + [[ -n "$DELPHIX_SIGNATURE_TOKEN" ]] || die "DELPHIX_SIGNATURE_TOKEN not specified" + [[ -n "$DELPHIX_SIGNATURE_URL" ]] || die "DELPHIX_SIGNATURE_URL not specified" +fi + if [[ -n "${DELPHIX_SIGNATURE_TOKEN:-}" ]] && [[ -n "${DELPHIX_SIGNATURE_URL:-}" ]]; then echo "{\"data\": \"$(base64 -w 0 SHA256SUMS)\"}" >sign-request.payload || die "failed to generate sigh-request.payload file" From b7170df2ef27af205659c2206dbb315f28289b1c Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 9 Jul 2021 10:57:41 -0400 Subject: [PATCH 03/12] [Backport to 6.0] Remove code for building migration images (#571) Co-authored-by: John Gallagher --- README.md | 1 - build.gradle | 3 - live-build/build.gradle | 8 - .../vm-artifacts/84-clear-apt-cache.binary | 4 +- .../90-linux-migration-artifact.binary | 183 ------- live-build/misc/migration-scripts/dx_apply | 362 ------------- live-build/misc/migration-scripts/dx_delete | 122 ----- live-build/misc/migration-scripts/dx_execute | 237 --------- live-build/misc/migration-scripts/dx_prepare | 23 - .../migration-scripts/dx_upg_pause_options | 217 -------- .../migration-scripts/dx_upg_stress_options | 480 ------------------ live-build/misc/migration-scripts/dx_verify | 211 -------- .../migration-scripts/etc_system_whitelist | 10 - scripts/run-live-build.sh | 3 +- 14 files changed, 3 insertions(+), 1861 deletions(-) delete mode 100755 live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary delete mode 100755 live-build/misc/migration-scripts/dx_apply delete mode 100755 live-build/misc/migration-scripts/dx_delete delete mode 100755 live-build/misc/migration-scripts/dx_execute delete mode 100755 live-build/misc/migration-scripts/dx_prepare delete mode 100755 live-build/misc/migration-scripts/dx_upg_pause_options delete mode 100755 live-build/misc/migration-scripts/dx_upg_stress_options delete mode 100755 live-build/misc/migration-scripts/dx_verify delete mode 100644 live-build/misc/migration-scripts/etc_system_whitelist diff --git a/README.md b/README.md index a22bdd99..6875d81a 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,6 @@ the "live-build/build/artifacts/" directory: $ ls -lh live-build/build/artifacts/ total 1.9G -rw-r--r-- 1 root root 275M Jan 11 22:31 internal-minimal-kvm.debs.tar.gz - -rw-r--r-- 1 root root 45 Jan 11 22:31 internal-minimal-kvm.migration.tar.gz -rw-r--r-- 1 root root 636M Jan 11 22:33 internal-minimal-kvm.qcow2 The appliance produced will contain a kernel optimized for the diff --git a/build.gradle b/build.gradle index d7e6f6b2..85e0bcc5 100644 --- a/build.gradle +++ b/build.gradle @@ -82,9 +82,6 @@ def shellScripts = fileTree("scripts") + fileTree("live-build/config/hooks").include({ details -> details.file.canExecute() }) + - fileTree("live-build/misc/migration-scripts", { - exclude "etc_system_whitelist" - }) + fileTree("upgrade", { include "prepare" }) + diff --git a/live-build/build.gradle b/live-build/build.gradle index 89717ece..4443c282 100644 --- a/live-build/build.gradle +++ b/live-build/build.gradle @@ -125,18 +125,10 @@ for (variant in allVariants) { outputs.file "${buildDir}/artifacts/${variant}-${platform}.debs.tar.gz" break case vmArtifactsRunType: - if (variant != "internal-minimal") { - // We don't bother creating a migration image for internal-minimal - outputs.file "${buildDir}/artifacts/${variant}-${platform}.migration.tar.gz" - } outputs.file "${buildDir}/artifacts/${variant}-${platform}.${artifactTypes[platform]}" break case allRunType: outputs.file "${buildDir}/artifacts/${variant}-${platform}.debs.tar.gz" - if (variant != "internal-minimal") { - // We don't bother creating a migration image for internal-minimal - outputs.file "${buildDir}/artifacts/${variant}-${platform}.migration.tar.gz" - } outputs.file "${buildDir}/artifacts/${variant}-${platform}.${artifactTypes[platform]}" break } diff --git a/live-build/config/hooks/vm-artifacts/84-clear-apt-cache.binary b/live-build/config/hooks/vm-artifacts/84-clear-apt-cache.binary index 0ffb3045..2e79f5d2 100755 --- a/live-build/config/hooks/vm-artifacts/84-clear-apt-cache.binary +++ b/live-build/config/hooks/vm-artifacts/84-clear-apt-cache.binary @@ -16,8 +16,8 @@ # # -# To reduce the size of our VM and migration images, we clean the APT -# cache, and remove its "lists" files. +# To reduce the size of our VM images, we clean the APT cache and remove its +# "lists" files. # chroot binary apt-get clean chroot binary rm -rf /var/lib/apt/lists/* diff --git a/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary b/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary deleted file mode 100755 index 269a508f..00000000 --- a/live-build/config/hooks/vm-artifacts/90-linux-migration-artifact.binary +++ /dev/null @@ -1,183 +0,0 @@ -#!/bin/bash -eux -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# This script is intended to be used as part of Delphix's build process. -# Its role is to convert the "binary" directory generated by live-build, -# into a cpio archive that will be wrapper in a tarball. That tarball is -# the migration artifact that illumos-based engines use to migrate to -# Linux. -# -MIGRATION_ARTIFACT="$ARTIFACT_NAME.migration.tar.gz" - -# -# At the time of this writing dx_unpack.sh expects a tarball of the -# following structure: -# -# depot/ the top level container -# depot/dx_prepare top-level script to prepare the archive -# for upgrade -# depot/version.info version and date information about the -# upgrade image -# depot/hashes the sha256sum of any upgrade artifacts -# (see below) -# depot/hashes.sig[.] one cryptographic signature of depot/hashes -# per Delphix release -# depot/ generally cpio archives of OS and appliance -# -# As dx_upack.sh is the only piece of code that runs from the old -# world, we treat it as a contract that lists the bare minimum -# requirements for our artifact. -# -DEPOT_DIRECTORY=depot - -# -# Delete any leftover files from previous runs. -# -rm -rf "$MIGRATION_ARTIFACT" "$DEPOT_DIRECTORY" - -# -# Not all variants include the virtualization application (e.g. -# internal-minimal) so don't bother generating a migration image -# for those. /opt/delphix is always created as part of the -# appliance. Thus, checking if it exists is a quick way for us to -# distinguish between variants that have the appliance from -# the ones that don't. -# -if [[ ! -d binary/opt/delphix ]]; then - echo "This artifact does not contain the appliance." \ - " Skipping migration image..." - exit 0 -fi - -mkdir $DEPOT_DIRECTORY - -# -# Generate archive for os root directory. -# -# Note: We temporarily change directory in the subshell (we move to the -# "binary" directory) because find prints everything as a relative -# path to the current directory and we don't want the generated archive -# to have "binary" as its top-level directory. -# -( - cd binary - find . -print | cpio -oc -) >$DEPOT_DIRECTORY/os-root.cpio - -# -# Copy all migration scripts including dx_prepare which is needed by -# dx_unpack. -# -cp migration-scripts/* $DEPOT_DIRECTORY - -test -n "$DELPHIX_APPLIANCE_VERSION" - -# -# There may be a version.info file in the current directory already. -# That version.info is Linux-specific and does not necessarily have -# all the fields that the illumos upgrade process requires. Thus, we -# create our own special version info directly in the depot directory -# with all the fields expected by dx_unpack. -# -{ - # - # Note that both the following field is bogus and supplied - # solely for making dx_unpack happy. - # - echo "DLPX_OS_VERSION=Linux" - - # - # UpgradeManager expects a valid date for DLPX_DATE. - # - echo "DLPX_DATE=$(date '+%Y.%m.%d.%H.%M.%S')" - - # - # DLPX_MIN_VERSION is specified for ensuring that we are - # migrating from the right version. - # - # Note that this will be checked by the dx_apply script - # of migration, so every time we bump this number we - # need to bump the one from the checks there too. - # - echo "DLPX_MIN_VERSION=5.3.6.0" - - # - # DLPX_VERSION is set explicitly to match the version of the - # virtualization appliance, which is what upgrades in Linux - # do and therefore we try to stay consistent as this field - # really can be anything (e.g. a date, a point-version, a - # nickname, etc.) - # - # shellcheck disable=SC2016 - echo "DLPX_VERSION=$DELPHIX_APPLIANCE_VERSION" -} >$DEPOT_DIRECTORY/version.info - -# -# Generate hashes file. -# -( - cd binary - find . -type f -print0 | xargs -0 sha256sum -) >$DEPOT_DIRECTORY/os-root.hashes -( - cd $DEPOT_DIRECTORY - sha256sum ./* >hashes -) - -# -# DELPHIX_SIGNATURE_TOKEN is used for signing our migration image. -# The signature is later unpacked and verified on the VM -# performing the migration. If DELPHIX_SIGNATURE_TOKEN has not been -# specified by the user when running this hook we skip generating -# the signature for this image. Similarly for DELPHIX_SIGNATURE_URL. -# -if [[ -n "${DELPHIX_SIGNATURE_TOKEN:-}" ]] && [[ -n "${DELPHIX_SIGNATURE_URL:-}" ]]; then - # - # Generate depot/hashes.sig[.] - # - # Assumption: we always migrate from version 5.3 - # - VERSION="5.3" - SIGN_URL="$DELPHIX_SIGNATURE_URL/upgrade/keyVersion/$VERSION/sign" - ( - cd $DEPOT_DIRECTORY - - # Encode payload for signature request - base64 -w 0 hashes >hashes64 - echo "{\"data\": \"$(cat hashes64)\"}" >hashes64.payload - - # Request signature - curl -s -S -f -H "Content-Type: application/json" \ - -u "$DELPHIX_SIGNATURE_TOKEN" "$SIGN_URL" -d @hashes64.payload >hashes.response - - # Decode generated signature - jq -r .signature sig.encoded - base64 -d sig.encoded >hashes.sig.$VERSION - - # Remove intermediate files - rm hashes64 hashes64.payload hashes.response sig.encoded - ) -fi - -# -# Tape-ARchive & Compress Zee File! -# -# We enable verbose output in the hope that all the files that are part of -# the artifact are logged somewhere which could aid debugging in the future. -# -tar -cvf - $DEPOT_DIRECTORY | pigz >"$MIGRATION_ARTIFACT" diff --git a/live-build/misc/migration-scripts/dx_apply b/live-build/misc/migration-scripts/dx_apply deleted file mode 100755 index c02a9df9..00000000 --- a/live-build/misc/migration-scripts/dx_apply +++ /dev/null @@ -1,362 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Creates the dataset layout/hierarchy expected by linux-upgrade for the -# archive from the migration image to be unpacked. It also takes the -# required steps and adds an entry to the FreeBSD bootloader so we can -# optionally boot into Linux. -# - -set -o pipefail - -# -# Reset the umask to the default value. When called from the app-stack the -# umask is set to 0027. Since those scripts can be called manually, we want -# to have a consistent result regardless of the caller. A umask of 0022 -# makes directories created by this script accessible by everyone by default, -# which is important for directories such as /export/home. -# -umask 0022 - -function die() { - echo "$(basename "$0"): $*" >&2 - exit 1 -} - -function usage() { - echo "$(basename "$0"): $*" >&2 - echo "Usage: $(basename "$0") " - exit 2 -} - -function report_progress_inc() { - echo "Progress increment: $(date +%T:%N%z), $1, $2" -} - -[[ "$(uname -s)" == "SunOS" ]] || die "script can only be used in illumos" - -while getopts ':h' opt; do - case $opt in - h) usage ;; - *) usage "Invalid option: -$OPTARG." ;; - esac -done - -ARCHIVE_DIR="$1" -[[ -n $ARCHIVE_DIR ]] || usage -[[ -d $ARCHIVE_DIR ]] || die "$ARCHIVE_DIR is not a directory" -[[ -f $ARCHIVE_DIR/version.info ]] || - die "$ARCHIVE_DIR does not have a version.info file" - -. "$ARCHIVE_DIR/version.info" - -# -# The root filesystem container needs to have the appliance version -# embedded in it as a dataset property, thus if this version was not -# found in the upgrade image, we need to error out (preferably before -# we do anything else, to avoid having to do cleanup later). -# -[[ -n "$DLPX_VERSION" ]] || die "DLPX_VERSION variable is missing" - -MIN_MIGRATION_VERSION="5.3.6.0" -[[ "$DLPX_MIN_VERSION" == "$MIN_MIGRATION_VERSION" ]] || - die "expected DLPX_MIN_VERSION for migration to be" \ - "$MIN_MIGRATION_VERSION" - -# -# Ensure that this VM is version 5.3.5.X or greater (but not 5.4 or 6.X). -# -CURRENT_DDS=$(dirname "$(mount | awk '/^\/opt\/delphix /{ print $3 }')") -CURRENT_VERSION=$(basename "$CURRENT_DDS") -MAJOR_VERSION_0=$(echo "$CURRENT_VERSION" | cut -d. -f1) -MAJOR_VERSION_1=$(echo "$CURRENT_VERSION" | cut -d. -f2) -[[ $MAJOR_VERSION_0 -eq 5 ]] || - die "expected version $MIN_MIGRATION_VERSION or greater but found" \ - "$MAJOR_VERSION_0.$MAJOR_VERSION_1.$MINOR_VERSION" -[[ $MAJOR_VERSION_1 -eq 3 ]] || - die "expected version $MIN_MIGRATION_VERSION or greater but found" \ - "$MAJOR_VERSION_0.$MAJOR_VERSION_1.$MINOR_VERSION" -MINOR_VERSION=$(echo "$CURRENT_VERSION" | cut -d. -f3) -MIGRATION_MINOR_VERSION=$(echo "$MIN_MIGRATION_VERSION" | cut -d. -f3) -[[ $MINOR_VERSION -ge $MIGRATION_MINOR_VERSION ]] || - die "expected version $MIN_MIGRATION_VERSION or greater but found" \ - "$MAJOR_VERSION_0.$MAJOR_VERSION_1.$MINOR_VERSION" - -DX_UPG_STRESS=$ARCHIVE_DIR/dx_upg_stress_options -# shellcheck source=/dev/null -. $DX_UPG_STRESS --source -__trigger_unset_stress_option "STRESS_DX_APPLY_FAIL_AFTER_VERSION_CHECK" - -# -# Get the root dataset and the current ZFS pool that we're currently using. -# -RDS=$(mount | awk '/^\/ /{ print $3 }') -RPOOL=${RDS%%/*} - -# -# Cleanup any previous intermediate state. -# -"${BASH_SOURCE%/*}/dx_delete" || - die "failed to cleanup previous state with dx_delete" - -# -# Save a copy of the boot menu to restore if we aren't upgrading -# -cp "/boot/menu.rc.local" "/boot/menu.rc.local.copy" || - die "failed to save copy of /boot/menu.rc.local" - -report_progress_inc 20 - -# -# Create dataset layout similar to the linux upgrade. -# -TMPDIR=$(mktemp -d -p "/tmp" -t delphix.XXXXXXX) -FSNAME=$(basename "$TMPDIR") -TMP_ROOT="$TMPDIR/root" - -zfs create \ - -o canmount=off \ - -o mountpoint=none \ - "$RPOOL/ROOT" || - die "failed to create linux dataset $RPOOL/ROOT" - -zfs create \ - -o canmount=off \ - -o mountpoint=none \ - -o "com.delphix:initial-version=$DLPX_VERSION" \ - -o "com.delphix:current-version=$DLPX_VERSION" \ - "$RPOOL/ROOT/$FSNAME" || - die "failed to create linux dataset $RPOOL/ROOT/$FSNAME" - -zfs create \ - -o canmount=noauto \ - -o mountpoint="$TMP_ROOT" \ - "$RPOOL/ROOT/$FSNAME/root" || - die "failed to create linux dataset $RPOOL/ROOT/$FSNAME/root" - -zfs mount "$RPOOL/ROOT/$FSNAME/root" || - die "failed to mount $RPOOL/ROOT/$FSNAME/root in temporary dir $TMP_ROOT" - -zfs create \ - -o mountpoint=legacy \ - "$RPOOL/ROOT/$FSNAME/home" || - die "failed to create linux dataset $RPOOL/ROOT/$FSNAME/home" - -zfs create \ - -o mountpoint=legacy \ - "$RPOOL/ROOT/$FSNAME/data" || - die "failed to create linux dataset $RPOOL/ROOT/$FSNAME/data" - -zfs create \ - -o mountpoint=legacy \ - "$RPOOL/ROOT/$FSNAME/log" || - die "failed to create linux dataset $RPOOL/ROOT/$FSNAME/log" - -mkdir -p "$TMP_ROOT/export/home" -mount -F zfs "$RPOOL/ROOT/$FSNAME/home" "$TMP_ROOT/export/home" - -mkdir -p "$TMP_ROOT/var/delphix" -mount -F zfs "$RPOOL/ROOT/$FSNAME/data" "$TMP_ROOT/var/delphix" - -mkdir -p "$TMP_ROOT/var/log" -mount -F zfs "$RPOOL/ROOT/$FSNAME/log" "$TMP_ROOT/var/log" - -( - cd "$TMP_ROOT" - cpio -imu -) <"$ARCHIVE_DIR/os-root.cpio" 2>&1 || - die "failed to unpack os-root.cpio" - -# -# /export/home and /var/delphix datasets are not mounted automatically. -# On Illumos this kind of mounting logic was performed by dxinit and svc-boot. -# On Linux /etc/fstab is used instead. These mounts need to happen before -# the zfs-import service runs. -# -cat <<-EOF >"$TMP_ROOT/etc/fstab" || die "Failed to setup /etc/fstab" - $RPOOL/ROOT/$FSNAME/home /export/home zfs defaults,x-systemd.before=zfs-import-cache.service 0 0 - $RPOOL/ROOT/$FSNAME/data /var/delphix zfs defaults,x-systemd.before=zfs-import-cache.service 0 0 - $RPOOL/ROOT/$FSNAME/log /var/log zfs defaults,x-systemd.before=zfs-import-cache.service 0 0 - $RPOOL/crashdump /var/crash zfs defaults,x-systemd.before=zfs-import-cache.service,x-systemd.before=kdump-tools.service 0 0 -EOF - -# -# On some platforms (currently AWS and new enough versions of ESX), the -# engine's UUID as reported by the os-level tools changes after -# migration. This is because of a confusion over the proper endianness of the -# first three fields of the UUID. To avoid having engine UUIDs change after -# migration, we store the current UUID before we finish the migration, and -# then use that on Linux. -# -/opt/delphix/server/bin/bos_mgmt get_system_uuid appliance >"$TMP_ROOT/etc/engine-uuid" || - die "failed to copy engine uuid" - -# -# Set things up for the FreeBSD bootloader to provide an option for -# booting into the Linux environment This happens in two steps: -# [1] We place vmlinuz and initrd from our migration image into -# the /boot directory of the current system, so the bootloader -# can load them. -# [2] Add a new option in the bootloader's menu for Linux. This is -# done by appending the configuration of the new option in our -# current menu file. -# -# Note: We reverse sort below based on a vmlinuz & initrd's name -# to get the latest version of them. In reality, there is always -# only one version but in the case that there is an issue in -# how we construct the images we always want to get the latest. -# -VMLINUZ=$(find "$TMP_ROOT/boot/vmlinuz-"* | sort -r | head -n 1) -INITRD_IMG=$(find "$TMP_ROOT/boot/initrd.img-"* | sort -r | head -n 1) -cp "$VMLINUZ" /boot || - die "failed to copy $VMLINUZ to /boot" -cp "$INITRD_IMG" /boot || - die "faield to copy $INITRD_IMG to /boot" - -# -# This command loads the Linux Kernel Compressed Executable in memory -# from the OK prompt of the FreeBSD bootloader. Note that we pass it the -# lxcmdline variable, which will be parsed by the bootloader. The reason -# why we create a separate variable for the cmdline arguments instead of -# just adding them in-line is because otherwise the command would be -# greater than 256 characters and fail to be parsed by the bootloader. -# We have to be careful to escape ${lxcmdline} properly so that it is -# not resolved by the current script. -# -LOAD_VMLINUZ_OK_CMD="load /boot/$(basename "$VMLINUZ") \${lxcmdline}" - -# -# Essentially we want the command line to be as close as possible to -# what is set by our grub configuration for fresh Linux installs. Our -# customizations to the grub command line can be found in the -# delphix-platform project at etc/default/grub.d/override.cfg. -# Here is an overview: -# -# - We specify the root filesystem type to be ZFS and its root dataset -# the one that we just created for the migration image. -# -# - We carry-over the following settings from delphix-platform: -# console, ipv6 and crashkernel. -# -# - We specify zfsforce=1 because otherwise we would be failing at -# pool import because of ZFS's hostid check. This argument is added -# specifically for the migration case and doesn't appear on a fresh -# Linux install. -# -# Note that we use an array of strings instead of one whole string -# that we break to multiple lines in order to divide the arguments -# in logical pieces while ensuring that their quotes are escaped -# correctly and in accordance with our linters/checkstyles. This -# should be taken into account for all cases of string arrays for -# the rest of this file that have to do with adding commands in the -# bootloader configuration. -# -LX_CMDLINE=( - "root=ZFS=$RPOOL/ROOT/$FSNAME/root" - 'console=tty0 console=ttyS0,115200n8' - 'ipv6.disable=1' - 'crashkernel=256M,high' - 'crashkernel=256M,low' - 'zfsforce=1' - 'mitigations=off' - 'elevator=noop' - 'init_on_alloc=0' - 'usbcore.nousb=1' -) - -# -# This command loads the initial RAM disk as the initial root filesystem -# which is needed as part of the booting process. -# -LOAD_INITRD_OK_CMD="load -t rootfs /boot/$(basename "$INITRD_IMG")" - -# -# This is the command to be issued whenever our new option in the -# bootloader menu is chosen. It is FICL code (a Forth-inspired -# command language) that is basically read as: -# arg func; arg func; func. -# In our case we: -# [1] Evaluate the command to load vmlinuz -# [2] Evaluate the command to load initrd -# [3] boot (with the above two loaded) -# -MAIN_MENU_FICL=( - "s\\\" $LOAD_VMLINUZ_OK_CMD\\\" evaluate" - "s\\\" $LOAD_INITRD_OK_CMD\\\" evaluate" - boot -) - -# -# Note: We always have 7 options in our bootloader at any given time -# in the latest 5.3 version, thus we use the index 8 for the new -# option below. -# -{ - echo 'set lxcmdline="'"${LX_CMDLINE[*]}"'"' - # - # Add non-ansi and ansi version for the name of the option. - # - echo 'set mainmenu_caption[8]="Boot Delphix on [L]inux"' - echo 'set mainansi_caption[8]="Boot Delphix on ^[1mL^[minux version"' - - # - # 76 is ascii for capital 'L', see caption above - # - echo 'set mainmenu_keycode[8]=76' - - # - # If someone selects our new option, boot into Linux. - # - echo 'set mainmenu_command[8]="'"${MAIN_MENU_FICL[*]}"'"' -} >>/boot/menu.rc.local || - die "failed to update /boot/menu.rc.local" - -# -# Note that the new /var/delphix should not contain any useful data as it will -# be replaced by a clone of the current /var/delphix dataset in dx_execute. -# We do not clone the current /var/delphix right away as the clone will be -# outdated by the time dx_execute is called. Before deleting the dataset we -# first verify that it doesn't contain anything useful. -# -( - cd "$TMP_ROOT/var/delphix" || die "failed to cd into $TMP_ROOT/var/delphix" - # /var/delphix should be empty - [[ $(find . -mindepth 1 | wc -l) -eq 0 ]] || - die "linux dataset for /var/delphix contains unexpected files" -) || die "verification of /var/delphix failed" - -__trigger_unset_stress_option "STRESS_DX_APPLY_FAIL_BEFORE_UNMOUNTING" - -umount "$TMP_ROOT/var/log" || - die "couldn't unmount linux dataset $TMP_ROOT/var/log" -zfs destroy "$RPOOL/ROOT/$FSNAME/data" || - die "failed to destroy linux dataset $RPOOL/ROOT/$FSNAME/data" -umount "$TMP_ROOT/export/home" || - die "couldn't unmount linux dataset $TMP_ROOT/export/home" - -zfs umount "$RPOOL/ROOT/$FSNAME/root" || - die "couldn't unmount linux dataset $RPOOL/ROOT/$FSNAME/root" -zfs set mountpoint=/ "$RPOOL/ROOT/$FSNAME/root" || - die "could not set mountpoint for linux dataset $RPOOL/ROOT/$FSNAME/root" - -rm -rf "$TMPDIR" || - die "failed to destroy temporary directory $TMPDIR" - -report_progress_inc 100 - -exit 0 diff --git a/live-build/misc/migration-scripts/dx_delete b/live-build/misc/migration-scripts/dx_delete deleted file mode 100755 index f90a764e..00000000 --- a/live-build/misc/migration-scripts/dx_delete +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Reset the umask to the default value. When called from the app-stack the -# umask is set to 0027. Since those scripts can be called manually, we want -# to have a consistent result regardless of the caller. A umask of 0022 -# makes directories created by this script accessible by everyone by default, -# which is important for directories such as /export/home. -# -umask 0022 - -set -o pipefail - -function die() { - echo "$(basename "$0"): $*" >&2 - exit 1 -} - -# -# Here, we assume Linux dataset names starts with "rpool/ROOT", and also -# assume that there can be at most one migration dataset applied in current OS -# since migration is only supported from *fixed* 5.3.X version to 6.0.X version. -# Also, it is not possible to get a version from dataset name in Linux since -# names are randomly generated. -# - -LX_DATASET="rpool/ROOT" -if ! zfs list ${LX_DATASET} &>/dev/null; then - echo "Linux dataset '${LX_DATASET}' is not installed." - exit 0 -fi - -LX_RDS_PARENT=$(zfs list -o name -H -d 1 "$LX_DATASET" | tail -n 1) -[[ -n $LX_RDS_PARENT ]] || die "could not find Linux RDS parent dataset" -LX_CONTAINER="${LX_RDS_PARENT##*/}" -LX_RDS_TMP_ROOT_MOUNT="/tmp/$LX_CONTAINER/root" - -MDS_SNAPNAME="MDS-CLONE-upgradeverify" -MDS_CLONE=domain0/$MDS_SNAPNAME -SVC=svc:/system/delphix/postgres:$MDS_SNAPNAME -PG_DATA=/$MDS_CLONE/db - -BOOT_MENU=/boot/menu.rc.local -BOOT_MENU_COPY=$BOOT_MENU.copy - -function dx_apply_cleanup() { - # - # Restore original version of the bootloader - # - if [[ -e $BOOT_MENU_COPY ]]; then - mv $BOOT_MENU_COPY $BOOT_MENU || - die "failed to restore copy of $BOOT_MENU" - fi - - rm -f /boot/vmlinuz-* /boot/initrd.img-* || - die "failed to destroy previously copied Linux kernel data" -} - -function dx_verify_cleanup() { - # - # dx_verify creates a clone of the MDS dataset and the postgres - # service as well as a clone of /var/delphix for masking validation. - # If any are left, clean them up. (This logic is based on dx_manage_pg - # stop and cleanup.) - # - local sta - sta=$(svcs -Ho sta $SVC) - if [[ -n "$sta" ]]; then - /usr/sbin/svcadm disable -s $SVC || - die "unable to disable SMF service: $SVC" - /usr/sbin/svccfg delete -f $SVC || - die "unable to delete SMF service: $SVC" - fi - - # - # Clean up the postmaster.pid leftover from the main MDS in the - # snapshot. This needs to be done after disabling the SMF service in - # case the postmaster.pid was instead from the active clone. - # - if [[ -e $PG_DATA/postmaster.pid ]]; then - echo "renaming $PG_DATA/postmaster.pid to $PG_DATA/postmaster.pid.original" - mv $PG_DATA/postmaster.pid $PG_DATA/postmaster.pid.original || - echo "failed to move postmaster.pid file" - fi - - [[ $(zfs list domain0/mds@$MDS_SNAPNAME) ]] && - zfs destroy -R "domain0/mds@$MDS_SNAPNAME" - [[ $(zfs list domain0/mds@$MDS_SNAPNAME) ]] && - die "unable to cleanup domain0/mds@$MDS_SNAPNAME" - - runningVar=$(mount | awk '/^\/var\/delphix /{ print $3 }') - [[ $(zfs list "$runningVar@$MDS_SNAPNAME") ]] && - zfs destroy -R "$runningVar@$MDS_SNAPNAME" - [[ $(zfs list "$runningVar@$MDS_SNAPNAME") ]] && - die "unable to cleanup $runningVar@$MDS_SNAPNAME" -} - -dx_apply_cleanup -dx_verify_cleanup - -# -# Recursively destroy and force-unmount the Linux Root Dataset. Even though all the -# datasets are cleaned up in a normal execution of Verify, this handles the case when -# it was cancelled early. -# -zfs destroy -rf ${LX_DATASET} || die "Failed to destroy Linux dataset '${LX_DATASET}'" -rm -rf "$LX_RDS_TMP_ROOT_MOUNT" diff --git a/live-build/misc/migration-scripts/dx_execute b/live-build/misc/migration-scripts/dx_execute deleted file mode 100755 index fdef0ca6..00000000 --- a/live-build/misc/migration-scripts/dx_execute +++ /dev/null @@ -1,237 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Finalizes the Linux image in preparation for the reboot now that the app -# stack is quiesced; sets up the FreeBSD bootloader to boot into that Linux -# image next time we reboot; finally reboots the system. -# - -# -# Reset the umask to the default value. When called from the app-stack the -# umask is set to 0027. Since those scripts can be called manually, we want -# to have a consistent result regardless of the caller. A umask of 0022 -# makes directories created by this script accessible by everyone by default, -# which is important for directories such as /export/home. -# -umask 0022 - -set -o pipefail - -DX_UPG_PAUSE="${BASH_SOURCE%/*}/dx_upg_pause_options" -DX_UPG_STRESS="${BASH_SOURCE%/*}/dx_upg_stress_options" -# shellcheck source=/dev/null -. $DX_UPG_STRESS --source - -function die() { - echo "$(basename "$0"): $*" >&2 - exit 1 -} - -function usage() { - [[ -n "$1" ]] && echo "$1" - cat </dev/null; then - zfs destroy -R "$dataset" || - die "Failed to destroy leftover dataset '$dataset'" - fi -} - -opt_s=false -while getopts :hs c; do - case "$c" in - \?) usage "Invalid option: -$OPTARG." ;; - h) usage ;; - s) eval "opt_$c=true" ;; - esac -done - -[[ "$(uname -s)" == "SunOS" ]] || die "script can only be used in illumos" - -# -# Undo any existing default timeout commands in the bootloader's menu. -# -sed -i '/menu_timeout_command/d' /boot/menu.rc.local -[[ "$(grep -cF 'menu_timeout_command' /boot/menu.rc.local)" -eq 0 ]] || - die "failed to cleanup previous default command" - -# -# Get the RDS and the current ZFS pool that we're currently using. -# -RDS=$(mount | awk '/^\/ /{ print $3 }') -RPOOL=${RDS%%/*} - -CURRENT_DDS=$(dirname "$(mount | awk '/^\/opt\/delphix /{ print $3 }')") -CURRENT_VERSION=$(basename "$CURRENT_DDS") - -# -# Ensure that the expected Linux dataset layout exists. -# Note that this excludes the "data" dataset which was destroyed in dx_apply. -# -[[ $(zfs list -o name -Hr "$RPOOL/ROOT" | wc -l) -eq 5 ]] || - die "could not find the expected linux dataset layout" -LX_RDS_PARENT=$(zfs list -o name -H -d 1 "$RPOOL/ROOT" | tail -n 1) -[[ -n $LX_RDS_PARENT ]] || die "could not find Linux RDS parent dataset" -LX_RDS="$LX_RDS_PARENT/root" - -# -# Ensure that the expected bootloader fields are there. -# -[[ "$(grep -cF 'set lxcmdline' /boot/menu.rc.local)" -eq 1 ]] || - die "there is no lxcmdline variable in the bootloader menu" -[[ "$(grep -cF 'mainmenu_caption[8]' /boot/menu.rc.local)" -eq 1 ]] || - die "there is no caption for the Linux option in the bootloader menu" -[[ "$(grep -cF 'mainmenu_caption[8]' /boot/menu.rc.local)" -eq 1 ]] || - die "there is no keycode for the Linux option in the bootloader menu" -[[ "$(grep -cF 'mainmenu_command[8]' /boot/menu.rc.local)" -eq 1 ]] || - die "there is no command for the Linux option in the bootloader menu" -[[ "$(grep -c "$LX_RDS" /boot/menu.rc.local)" -eq 1 ]] || - die "the expected Linux RDS ($LX_RDS) was either not found or has" \ - "been specified more than once in the bootloader's menu file" - -# -# Re-mount the root dataset and run the migration logic. -# -rm -rf /var/delphix/migration -mkdir -p /var/delphix/migration || - die "failed to create /var/delphix/migration" -MIGRATION_SCRIPT="/opt/delphix/migration/migrate_config.py" -LX_CONTAINER="${LX_RDS_PARENT##*/}" -LX_RDS_MOUNT="/tmp/delphix.$LX_CONTAINER/root" -mkdir -p "$LX_RDS_MOUNT" -mount -F zfs -o ignoremountpoint "$LX_RDS" "$LX_RDS_MOUNT" || - die "failed to mount the root Linux dataset" -"${LX_RDS_MOUNT}${MIGRATION_SCRIPT}" pre-upgrade \ - >>/var/delphix/migration/log 2>&1 || die "failed to run migration" - -__trigger_unset_stress_option "STRESS_DX_EXECUTE_FAIL_AFTER_CONFIG_MIGRATION" - -# -# Create a flag file that notifies the delphix-migration service that -# post-reboot migration logic should be run. Note that the /var/delphix -# dataset is carried over into Linux. -# -touch /var/delphix/migration/perform-migration || - die "failed to create delphix-migration flag file" - -# -# Let the app-stack know that an "OS migration" upgrade is being performed. -# -cat <<-EOF >"/var/dlpx-update/upgrade.properties" || - UPGRADE_TYPE=OS_MIGRATION - UPGRADE_BASE_VERSION=$CURRENT_VERSION -EOF - die "failed to create upgrade.properties" - -# -# Create linux /var/delphix dataset from a clone of the current -# /var/delphix dataset. We want to do this last as this will carry-over all -# the logs and the /var/delphix/migration directory into Linux. -# -LX_VAR_DLPX="$LX_RDS_PARENT/data" -CUR_VAR_DLPX=$(zfs list -Ho name /var/delphix) -[[ -n $CUR_VAR_DLPX ]] || die "could not determine current /var/delphix dataset" -cleanup_leftover_dataset "${CUR_VAR_DLPX}@migration" -zfs snapshot "${CUR_VAR_DLPX}@migration" || - die "failed to create snapshot '${CUR_VAR_DLPX}@migration'" -zfs clone \ - -o mountpoint=legacy \ - "${CUR_VAR_DLPX}@migration" \ - "$LX_VAR_DLPX" || - die "failed to clone dataset ${CUR_VAR_DLPX}@migration" - -# -# Create a clone of the current /export/home dataset. It will be kept as -# a backup in case Delphix support or services have left useful files in -# there since all rpool/versions datasets will be cleaned-up on next upgrade. -# -CUR_HOME=$(zfs list -Ho name /export/home) -[[ -n $CUR_HOME ]] || die "could not determine current /export/home dataset" -zfs list "$CUR_HOME" &>/dev/null || - die "'$CUR_HOME' is not a valid zfs dataset" -cleanup_leftover_dataset "${CUR_HOME}@migration" -cleanup_leftover_dataset rpool/illumos-home -zfs snapshot "${CUR_HOME}@migration" || - die "failed to create snapshot '${CUR_HOME}@migration'" -zfs clone \ - -o mountpoint=legacy \ - "${CUR_HOME}@migration" \ - rpool/illumos-home || - die "failed to clone dataset ${CUR_HOME}@migration" - -# -# Create a snapshot of /mds to make it easier to rollback to it if needed. -# -cleanup_leftover_dataset domain0/mds@migration -zfs snapshot domain0/mds@migration || - die "failed to create snapshot domain0/mds@migration" - -# -# Increase the boot delay to 20 seconds to ease bootloader access during -# the migration for debugging purposes. -# -sed -i '/autoboot_delay/c\set autoboot_delay=20' /boot/menu.rc.local || - die "Failed to set boot delay in bootloader" - -# -# Read the command from option 8 which should be the one that boots -# into Linux. Then make it so that the same command runs whenever we -# hit the timer of the FreeBSD bootloader menu. -# -MAIN_MENU_LINUX_CMD=$(grep -F 'mainmenu_command[8]' /boot/menu.rc.local | - cut -d = -f 2-) -echo "set menu_timeout_command=$MAIN_MENU_LINUX_CMD" >>/boot/menu.rc.local - -# -# Notify the UI that the system is rebooting. -# This could hang if the JVM is in a bad state, so we use a timeout. -# -timeout 10 /opt/delphix/server/bin/jmxtool boot upgrade server - -$DX_UPG_PAUSE --pause "PAUSE_IN_DX_EXECUTE_BEFORE_RESTART" || - die "failed to pause fully on stress option" - -__trigger_unset_stress_option "STRESS_DX_EXECUTE_FAIL_BEFORE_REBOOT" - -# Constants used by the uadmin syscall. -A_SHUTDOWN=2 -AD_POWEROFF=6 -AD_BOOT=1 - -# reboot or shutdown -if $opt_s; then - uadmin $A_SHUTDOWN $AD_POWEROFF -else - uadmin $A_SHUTDOWN $AD_BOOT -fi - -exit 0 diff --git a/live-build/misc/migration-scripts/dx_prepare b/live-build/misc/migration-scripts/dx_prepare deleted file mode 100755 index 6430016e..00000000 --- a/live-build/misc/migration-scripts/dx_prepare +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# This script is a no-op and only exist to satisfy dx_unpack.sh -# from illumos. -# - -exit 0 diff --git a/live-build/misc/migration-scripts/dx_upg_pause_options b/live-build/misc/migration-scripts/dx_upg_pause_options deleted file mode 100755 index 6b9e0e79..00000000 --- a/live-build/misc/migration-scripts/dx_upg_pause_options +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2017, 2019 by Delphix. All rights reserved. -# - -# -# Utility for managing upgrade pause options. -# Supported pause options listed below. -# -# Note that those pause options are intended to be run exclusively for -# migration testing, before the upgrade reboot. -# - -PAUSE_OPTIONS_JSON=$( - cat <<-EOF - { - "PAUSE_IN_DX_EXECUTE_BEFORE_RESTART": { - "location": "right before reboot" - }, - "PAUSE_IN_DXSVCINIT_BEFORE_UPGRADE": { - "location": "post-reboot before any app upgrade logic" - }, - "PAUSE_BEFORE_UPGRADE_COMMIT": { - "location": "mgmt service (before point of no return)" - }, - "PAUSE_BEFORE_CHECKPOINT_DISCARD": { - "location": "mgmt service (before point of no return)" - }, - "PAUSE_IN_SVC_ROLLBACK_BEFORE_ROLLBACK": { - "location": "rollback service" - }, - "PAUSE_BEFORE_REWIND_CHECKPOINT": { - "location": "rollback service" - }, - "PAUSE_AFTER_REWIND_CHECKPOINT": { - "location": "rollback service" - }, - "PAUSE_IN_DXSVCINIT_AFTER_ROLLBACK": { - "location": "post-rollback before app stack starts" - }, - "PAUSE_IN_SVC_MGMT_BEFORE_STACK_STARTUP": { - "location": "before app stack starts" - } - } - EOF -) - -PAUSE_DATASET=rpool/pause_options -DLPX_ENTERED=com.delphix:entered - -function die() { - echo "$(basename "$0"): $*" 1>&2 - exit 1 -} - -function usage() { - echo "$(basename "$0"): $*" >&2 - echo "Usage:" - echo " $(basename "$0") --get-options-json" - echo " Prints out a detailed json of all the supported stress options." - echo " $(basename "$0") --set " - echo " Sets the specified pause option. Non-zero exit code on failure." - echo " $(basename "$0") --unset " - echo " Unsets the specified pause option. Non-zero exit code on failure." - echo " $(basename "$0") --get " - echo " Returns exit code 3 if given pause option is set, 0 if unset." - echo " Any other non-zero exit code is an unexpected error." - echo " $(basename "$0") --pause " - echo " Does not terminate until the pause option is unset." - echo " Non-zero exit code if errors hit, 0 if wait successful." - echo " $(basename "$0") --wait-for-pause " - echo " Does not terminate until the pause option is hit." - echo " Non-zero exit code if errors hit, 0 if wait successful." - exit 2 -} - -function check_valid_option() { - local arg="$1" - - local get_names - get_names=$( - cat <<-EOF - import json, sys - obj=json.load(sys.stdin) - print(" ".join(obj.keys())) - EOF - ) - local options - options=$(echo "$PAUSE_OPTIONS_JSON" | python -c "$get_names") - - for option in $options; do - [[ "$option" == "$arg" ]] && return - done - usage "Unsupported pause option '$arg'" -} - -function set_pause_option() { - local option="$1" - local entered="$2" - - if ! zfs list "$PAUSE_DATASET" &>/dev/null; then - zfs create -o mountpoint=legacy "$PAUSE_DATASET" || return 1 - fi - - if ! zfs list "$PAUSE_DATASET/$option" &>/dev/null; then - zfs create -o mountpoint=legacy -o "$DLPX_ENTERED=$entered" \ - "$PAUSE_DATASET/$option" || return 1 - else - zfs set "$DLPX_ENTERED=$entered" "$PAUSE_DATASET/$option" || return 1 - fi - return 0 -} - -# Requires option to be set, otherwise returns non-zero code. -function enter_pause_option() { - local option="$1" - zfs set "$DLPX_ENTERED=$entered" "$PAUSE_DATASET/$option" || return 1 - return 0 -} - -function unset_pause_option() { - local option="$1" - - if zfs list "$PAUSE_DATASET/$option" &>/dev/null; then - zfs destroy "$PAUSE_DATASET/$option" || return 1 - fi - return 0 -} - -function get_pause_option() { - local option="$1" - zfs list "$PAUSE_DATASET/$option" &>/dev/null && return 3 - return 0 -} - -# Verify arguments -[[ $# -ge 1 ]] || usage "missing arguments" - -case "$1" in -"--set") - [[ $# -ge 2 ]] || usage "missing arguments" - check_valid_option "$2" - set_pause_option "$2" "false" - exit $? - ;; -"--unset") - [[ $# -ge 2 ]] || usage "missing arguments" - check_valid_option "$2" - unset_pause_option "$2" - exit $? - ;; -"--get") - [[ $# -ge 2 ]] || usage "missing arguments" - check_valid_option "$2" - get_pause_option "$2" - exit $? - ;; -"--pause") - [[ $# -ge 2 ]] || usage "missing arguments" - check_valid_option "$2" - - # If option not set, nothing to do. - get_pause_option "$2" && exit 0 - - echo -e "Pausing on $2 ... \\c" - - # Mark as entered - enter_pause_option "$2" || die "failed to mark entered" - - get_pause_option "$2" - enabled=$([[ $? -eq 3 ]] && echo true || echo false) - while $enabled; do - sleep 1 - - # - # Enjoy this spinner while you wait... - # (copied this genius piece of code from start_mgmt_server_jvm) - # - case $((SECONDS % 4)) in - 0) echo -e "\\b\\b- \\c" ;; - 1) echo -e "\\b\\b\\ \\c" ;; - 2) echo -e "\\b\\b| \\c" ;; - 3) echo -e "\\b\\b/ \\c" ;; - esac - - get_pause_option "$2" - enabled=$([[ $? -eq 3 ]] && echo true || echo false) - done - echo "\\b\\b\\b done." - ;; -"--wait-for-pause") - [[ $# -ge 2 ]] || usage "missing arguments" - check_valid_option "$2" - - # Check if the option was set - get_pause_option "$2" && die "pause option $2 not set" - - echo -e "Waiting for pause on $2 ... \\c" - entered=false - while ! $entered; do - sleep 1 - entered=$(zfs get -Ho value $DLPX_ENTERED "$PAUSE_DATASET/$2") - # shellcheck disable=SC2181 - [[ $? -eq 0 ]] || die "failed to get property for $2" - done - echo "done." - ;; -"--get-options-json") - echo "$PAUSE_OPTIONS_JSON" - exit $? - ;; -*) - usage "illegal option '$1'" - ;; -esac - -exit 0 diff --git a/live-build/misc/migration-scripts/dx_upg_stress_options b/live-build/misc/migration-scripts/dx_upg_stress_options deleted file mode 100755 index addd500f..00000000 --- a/live-build/misc/migration-scripts/dx_upg_stress_options +++ /dev/null @@ -1,480 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2016, 2019 by Delphix. All rights reserved. -# - -# -# Utility for getting and setting upgrade stress options. -# Supported failure points listed below. -# -# This file may be sourced so the variable/function names are made as unique -# as possible in addition to the double underscores in front. -# -# Note that those stress options are intended to be run exclusively for -# migration testing, before the upgrade reboot. -# - -__STRESS_OPTIONS_JSON=$( - cat <<-EOF - { - "STRESS_DX_APPLY_FAIL_AFTER_VERSION_CHECK": { - "location": "pre-reboot", - "err_msg": "Stress option triggered after version check.", - "auto_unset": true - }, - "STRESS_DX_APPLY_FAIL_BEFORE_UNMOUNTING": { - "location": "pre-reboot", - "err_msg": "Stress option triggered before dx_apply dataset cleanup.", - "auto_unset": true - }, - "STRESS_DX_INSTALL_ARCHIVE_FAIL_AFTER_VERIFY_DX_ARCHIVE": { - "location": "pre-reboot", - "err_msg": "Stress option triggered after verify_dx_archive.", - "auto_unset": true - }, - "STRESS_DX_VERIFY_FAIL_HOTFIX": { - "location": "pre-reboot", - "err_msg": "Stress option triggered to fail hotfix check.", - "auto_unset": true - }, - "STRESS_DX_VERIFY_FAIL_AFTER_TEST_MIGRATION": { - "location": "pre-reboot", - "err_msg": "Stress option triggered after upgrade-verify.jar.", - "auto_unset": true - }, - "STRESS_DX_EXECUTE_FAIL_AFTER_CONFIG_MIGRATION": { - "location": "pre-reboot", - "err_msg": "Stress option triggered after migrating OS config.", - "auto_unset": true - }, - "STRESS_DX_EXECUTE_FAIL_BEFORE_REBOOT": { - "location": "pre-reboot", - "err_msg": "Stress option triggered before reboot.", - "auto_unset": true - }, - "STRESS_APPLY_JOB_CLEANUP_BEFORE_DISABLE_UPGRADE_MODE": { - "location": "pre-reboot", - "err_msg": "Stress option triggered before cleaning up upgrade mode.", - "auto_unset": true - }, - "STRESS_DXSVCINIT_FAIL_AFTER_UPGRADE_MODE": { - "location": "boot service", - "err_msg": "Stress option triggered after upgrade mode.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_MASKING_UPGRADE": { - "location": "mgmt service (before point of no return)", - "err_msg": "Stress option triggered before upgrading masking.", - "auto_unset": true - }, - "STRESS_FAIL_AFTER_MASKING_UPGRADE": { - "location": "mgmt service (before point of no return)", - "err_msg": "Stress option triggered after upgrading masking.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_UPGRADE_COMMIT": { - "location": "mgmt service (before point of no return)", - "err_msg": "Stress option triggered before upgrade commit.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_DOMAIN_UPGRADE": { - "location": "mgmt service (before point of no return)", - "err_msg": "Stress option triggered before domain upgrade.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_CHECKPOINT_DISCARD": { - "location": "mgmt service (before point of no return)", - "err_msg": "Stress option triggered before checkpoint discard.", - "auto_unset": true - }, - "STRESS_ROLLBACK_SVC_FAIL_BEFORE_ROLLBACK": { - "location": "rollback service (all)", - "err_msg": "Stress option triggered before do_rollback.", - "auto_unset": false - }, - "STRESS_ROLLBACK_SVC_FAIL_BEFORE_DX_ROLLBACK": { - "location": "rollback service (all)", - "err_msg": "Stress option triggered before dx_rollback call.", - "auto_unset": false - }, - "STRESS_ROLLBACK_SVC_FAIL_AFTER_DX_ROLLBACK": { - "location": "rollback service (all)", - "err_msg": "Stress option triggered after dx_rollback call.", - "auto_unset": true - }, - "STRESS_ROLLBACK_SVC_FAIL_AFTER_REMOUNT_OPT": { - "location": "rollback service (stack only)", - "err_msg": "Stress option triggered after remounting /opt/delphix.", - "auto_unset": false - }, - "STRESS_ROLLBACK_SVC_FAIL_BEFORE_CLEARING_DEPS": { - "location": "rollback service (stack only)", - "err_msg": "Stress option triggered before clearing dependent services.", - "auto_unset": false - }, - "STRESS_ROLLBACK_SVC_FAIL_BEFORE_REENABLING_BOOT": { - "location": "rollback service (stack only)", - "err_msg": "Stress option triggered before re-enabling boot service.", - "auto_unset": false - }, - "STRESS_CKPT_UTIL_FAIL_AFTER_EXPORT_DOMAIN": { - "location": "rollback service (OS)", - "err_msg": "Stress option triggered after exporting domain0.", - "auto_unset": true - }, - "STRESS_FAIL_AFTER_COMMIT_BEFORE_CHECKPOINT_DISCARD": { - "location": "mgmt service (after point of no return)", - "err_msg": "Stress option triggered between upgrade commit and checkpoint discard", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_BOOTLOADER_REWRITE": { - "location": "mgmt service (after point of no return, migration only)", - "err_msg": "Stress option triggered before rewriting bootloader.", - "auto_unset": true - }, - "STRESS_FAIL_AFTER_UPGRADE_COMMIT": { - "location": "mgmt service (after point of no return)", - "err_msg": "Stress option triggered after upgrade commit.", - "auto_unset": true - }, - "STRESS_UPGRADEMANAGER_FAIL_IN_START": { - "location": "mgmt service (after point of no return)", - "err_msg": "Stress option triggered in upgrade manager.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_POST_MIGRATION_CLEANUP": { - "location": "mgmt service (after point of no return, migation only)", - "err_msg": "Stress option triggered before performing post-migration cleanup.", - "auto_unset": true - }, - "STRESS_FAIL_BEFORE_MIGRATION_MARKED_COMPLETED": { - "location": "mgmt service (after point of no return, migation only)", - "err_msg": "Stress option triggered before marking migration as completed.", - "auto_unset": true - }, - "STRESS_POSTCLEANUP_FAIL_BEFORE_ENABLE_SOURCES": { - "location": "mgmt service (after point of no return)", - "err_msg": "Stress option triggered before enabling sources.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_BEGIN_MIGRATION_SERVICE": { - "location": "migration service", - "err_msg": "Stress option triggered starting migration service.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_BEFORE_DOMAIN0_IMPORT": { - "location": "migration service", - "err_msg": "Stress option triggered before importing domain0.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_AFTER_DOMAIN0_IMPORT": { - "location": "migration service", - "err_msg": "Stress option triggered after importing domain0.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_AFTER_ZFS_MOUNT": { - "location": "migration service", - "err_msg": "Stress option triggered after zfs mount -a.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_AFTER_MOUNT_PERMISSION_FIX": { - "location": "migration service", - "err_msg": "Stress option triggered after fixing mount permissions.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_BEFORE_MIGRATE_CONFIG": { - "location": "migration service", - "err_msg": "Stress option triggered before calling migrate-config.", - "auto_unset": true - }, - "STRESS_MIGRATION_FAIL_AFTER_MIGRATE_CONFIG": { - "location": "migration service", - "err_msg": "Stress option triggered after calling migrate-config.", - "auto_unset": true - } - } - EOF -) - -__STRESS_DATASET=rpool/stress_options -__DLPX_PANIC=com.delphix:panic -__DLPX_SVC_MGMT=svc:/system/delphix/mgmt - -function __dx_upg_stress_options_usage() { - cat <<-EOF >&2 - $(basename "$0"): $* - Usage: - $(basename "$0") --get-options-json - Prints out a detailed json of all the supported stress options. - $(basename "$0") --set - Set the specified stress option to die on trigger. - Non-zero exit status on failure. - $(basename "$0") --set-panic - Set the specified stress option to system panic on trigger. - Non-zero exit status on failure. - $(basename "$0") --unset - Unset the specified stress option. - Non-zero exit status on failure. - $(basename "$0") --get - Returns exit code 3 if given stress option is set to die, - 0 if not. Any other non-zero exit code is an unexpected error. - $(basename "$0") --get-panic - Returns exit code 3 if given stress option is set to panic, - 0 if not. Any other non-zero exit code is an unexpected error. - $(basename "$0") --panic-or-get-unset - If the stress option is set to panic, unset and panic. - Otherwise returns exit code: - 0 - the stress option is unset. - 3 - the stress option is set, and successfully unset it. - 4 - the stress option is set, but unsuccessfully unset it. - Any other non-zero exit code is an unexpected error. - EOF - exit 2 -} - -function __check_valid_option() { - local arg="$1" - - local get_names - get_names=$( - cat <<-EOF - import json - obj=json.loads('''$__STRESS_OPTIONS_JSON''') - print(" ".join(obj.keys())) - EOF - ) - local options - options=$(python -c "$get_names") - - for option in $options; do - [[ "$option" == "$arg" ]] && return - done - __dx_upg_stress_options_usage "Unsupported stress option '$arg'" -} - -function __set_stress_option() { - local option="$1" - local panic="$2" - - if ! zfs list "$__STRESS_DATASET" &>/dev/null; then - zfs create -o mountpoint=legacy "$__STRESS_DATASET" || return 1 - fi - - if ! zfs list "$__STRESS_DATASET/$option" &>/dev/null; then - zfs create -o mountpoint=legacy -o "$__DLPX_PANIC=$panic" \ - "$__STRESS_DATASET/$option" || return 1 - else - zfs set "$__DLPX_PANIC=$panic" "$__STRESS_DATASET/$option" || return 1 - fi - return 0 -} - -function __unset_stress_option() { - local option="$1" - - if zfs list "$__STRESS_DATASET/$option" &>/dev/null; then - zfs destroy "$__STRESS_DATASET/$option" || return 1 - fi - return 0 -} - -function __get_stress_option() { - local option="$1" - local panic="$2" - - zfs list "$__STRESS_DATASET/$option" &>/dev/null && - [[ $(zfs get -Ho value "$__DLPX_PANIC" "$__STRESS_DATASET/$option") == "$panic" ]] && return 3 - return 0 -} - -function __get_unset_stress_option() { - local option="$1" - local panic="$2" - - __get_stress_option "$option" "$panic" - if [[ $? -eq 3 ]]; then - __unset_stress_option "$option" || return 4 - return 3 - fi - return 0 -} - -function __handle_panic_stress_option() { - local option="$1" - local unset="$2" - local get_func="__get_stress_option" - $unset && get_func="__get_unset_stress_option" - - $get_func "$option" "true" - local stress_ret - stress_ret=$? - - if [[ $stress_ret -eq 3 ]]; then - uadmin 2 1 - elif [[ $delphix_debug == "true" ]] && [[ $stress_ret -ne 0 ]]; then - echo "'$get_func' returned $stress_ret" - fi -} - -# -# Fails if the given stress option was set. -# Also fails if we are in debug mode and hit an unexpected return code. -# -# This function should only be used when sourcing this file. The script -# sourcing this file must implement the function "die" or "stress_die" for -# this method to fail correctly. -# -# A caller may define a "stress_die" function if they want extra functionality -# besides the original "die" method. See "dx_execute" for an example of this. -# If no "stress_die" is defined, it will fall back to using the original "die" -# method for failing. -# -function __trigger_stress_option() { - local option=$1 - local delphix_debug - delphix_debug=$(svcprop -p delphix/debug $__DLPX_SVC_MGMT) - - __handle_panic_stress_option "$option" "false" - - local get_err_msg - get_err_msg=$( - cat <<-EOF - import json - obj=json.loads('''$__STRESS_OPTIONS_JSON''') - print(obj["$option"]["err_msg"]) - EOF - ) - local message - message=$(python -c "$get_err_msg") - - __get_stress_option "$option" "false" - local stress_ret=$? - if [[ $stress_ret -eq 3 ]]; then - type stress_die >/dev/null 2>&1 && stress_die "$message" - if ! type die >/dev/null 2>&1; then - echo "ERROR: no die function defined" - exit 1 - fi - die "$message" - elif [[ $delphix_debug == "true" ]] && [[ $stress_ret -ne 0 ]]; then - if ! type die >/dev/null 2>&1; then - echo "ERROR: no die function defined" - exit 1 - fi - die "'__get_stress_option' returned $stress_ret" - fi -} - -# -# Fails if the given stress option was set (unsets it too). -# Also fails if we are in debug mode and hit an unexpected return code. -# -# This function should only be used when sourcing this file. The script -# sourcing this file must implement the function "die" or "stress_die" for -# this method to fail correctly. -# -# A caller may define a "stress_die" function if they want extra functionality -# besides the original "die" method. See "dx_execute" for an example of this. -# If no "stress_die" is defined, it will fall back to using the original "die" -# method for failing. -# -function __trigger_unset_stress_option() { - local option="$1" - local delphix_debug - delphix_debug=$(svcprop -p delphix/debug $__DLPX_SVC_MGMT) - - __handle_panic_stress_option "$option" "true" - - local get_err_msg - get_err_msg=$( - cat <<-EOF - import json - obj=json.loads('''$__STRESS_OPTIONS_JSON''') - print(obj["$option"]["err_msg"]) - EOF - ) - local message - message=$(python -c "$get_err_msg") - - __get_unset_stress_option "$option" "false" - local stress_ret=$? - if [[ $stress_ret -eq 3 ]]; then - type stress_die >/dev/null 2>&1 && stress_die "$message" - if ! type die >/dev/null 2>&1; then - echo "ERROR: no die function defined" - exit 1 - fi - die "$message" - elif [[ $delphix_debug == "true" ]] && [[ $stress_ret -ne 0 ]]; then - if ! type die >/dev/null 2>&1; then - echo "ERROR: no die function defined" - exit 1 - fi - die "'__get_unset_stress_option' returned $stress_ret" - fi -} - -[[ $# -ge 1 ]] || __dx_upg_stress_options_usage "missing arguments" -case "$1" in -"--set") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - __set_stress_option "$2" "false" - exit $? - ;; -"--set-panic") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - __set_stress_option "$2" "true" - exit $? - ;; -"--unset") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - __unset_stress_option "$2" "false" - exit $? - ;; -"--get") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - __get_stress_option "$2" "false" - exit $? - ;; -"--get-panic") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - __get_stress_option "$2" "true" - exit $? - ;; -"--panic-or-get-unset") - [[ $# -ge 2 ]] || __dx_upg_stress_options_usage "missing arguments" - __check_valid_option "$2" - - # Called when the stress option is set and successfully unset, exit 3. - function stress_die() { - echo "$*" - exit 3 - } - - # Called when the stress option is set but failed to unset, exit 4. - function die() { - echo "$*" - exit 4 - } - - __trigger_unset_stress_option "$2" - exit 0 - ;; -"--get-options-json") - echo "$__STRESS_OPTIONS_JSON" - exit $? - ;; -"--source") - # do nothing - ;; -*) - __dx_upg_stress_options_usage "illegal option '$1'" - ;; -esac - -# Do not exit here because sourcing this file will exit the parent. diff --git a/live-build/misc/migration-scripts/dx_verify b/live-build/misc/migration-scripts/dx_verify deleted file mode 100755 index 84b9368b..00000000 --- a/live-build/misc/migration-scripts/dx_verify +++ /dev/null @@ -1,211 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Reset the umask to the default value. When called from the app-stack the -# umask is set to 0027. Since those scripts can be called manually, we want -# to have a consistent result regardless of the caller. A umask of 0022 -# makes directories created by this script accessible by everyone by default. -# -umask 0022 - -export MDS_SNAPNAME="MDS-CLONE-upgradeverify" - -DEBUG=false - -MDS_CLONE=domain0/$MDS_SNAPNAME -UPGRADE_VERIFY_PATH=/opt/delphix/server/lib/exec/upgrade-verify -UPGRADE_VERIFY_JAR=$UPGRADE_VERIFY_PATH/upgrade-verify.jar -LOG_DIR=/var/delphix/server/upgrade-verify -MGMT_FMRI=svc:/system/delphix/mgmt:default -BINDIR=/opt/delphix/server/bin -DX_MANAGE_PG=$BINDIR/dx_manage_pg - -function usage() { - echo "usage: $(basename "$0") -v -o -f -l " - echo " -v : The Delphix version number we're upgrading to" - echo " -o : The output file verification should generate containing an upgrade verification report" - echo " -f : The format of the upgrade verification report to generate" - echo " -l : The locale the upgrade verification report should target" - echo " -d: Disable Consistent MDS ZFS Data Utility feature" - exit 1 -} - -function cleanup() { - if ! $DEBUG; then - if [[ -n "$root" ]]; then - cleanup_postgres - cleanup_datasets - cleanup_masking - fi - fi -} - -function die() { - report "$(basename "$0"): $*" >&2 - - # - # If we have MDS postgres logs, save them before cleaning up so - # they can be analyzed later. - # - if [[ -f "/$MDS_CLONE/db/stdout.log" ]]; then - ( - cd "/$MDS_CLONE/db" || exit 1 - tar -cvf /var/delphix/server/db/pg_log.upgrade.$$.tar \ - stdout.log pg_log - ) - fi - - cleanup - exit 1 -} - -function report() { - echo "$(date +%T:%N:%z): $1" -} - -function report_progress_inc() { - echo "Progress increment: $(date +%T:%N%z), $1, $2" -} - -function mount_datasets() { - local version=$1 - local rds - - rds=$(zfs list -o name -H -d3 rpool/ROOT | grep "delphix.*/root") - [[ -n $rds ]] || die "could not find migration dataset for version $version" - - root=$(mktemp -d) || die "unable to create temporary directory" - chmod 755 "$root" || die "unable to set permissions for $root" - mount -F zfs -o ignoremountpoint "$rds" "$root" || die "unable to mount $rds" - - #TODO dummy file for UpgradeVerify see LX-1808 - touch /var/dlpx-update/"$version"/dx_upg_stress_options - - # - # We need a clone of /var/delphix for masking migration scripts. - # - runningVar=$(mount | awk '/^\/var\/delphix /{ print $3 }') - zfs snapshot "$runningVar@$MDS_SNAPNAME" || - die "unable to create snapshot $runningVar@$MDS_SNAPNAME" - zfs clone "$runningVar@$MDS_SNAPNAME" "$runningVar/$MDS_SNAPNAME" || - die "unable to create $runningVar/$MDS_SNAPNAME" - mount -F zfs "$runningVar/$MDS_SNAPNAME" "$root/var/delphix" || - die "unable to mount $root/var/delphix" -} - -function cleanup_datasets() { - if [[ -n "$runningVar" ]]; then - zfs destroy -R "$runningVar@$MDS_SNAPNAME" - fi - umount -f "$root" - rmdir "$root" -} - -function cleanup_postgres() { - "$root$DX_MANAGE_PG" stop -s $MDS_SNAPNAME || - echo "failed to stop postgres" - "$root$DX_MANAGE_PG" cleanup -s $MDS_SNAPNAME || - echo "failed to clean up postgres" -} - -function cleanup_masking() { - if [[ -n "$masking_temp_dir" ]]; then - rm -r "$masking_temp_dir" - fi -} - -function run_upgrade_verify() { - local output=$1 - local format=$2 - local locale=$3 - local upgrade_verify_opts - $disable_consistent_mds_zfs_data_util && - upgrade_verify_opts="-disableConsistentMdsZfsDataUtil" - local progress_low=$4 - local progress_high=$5 - local debug_mode - debug_mode=$(svcprop -p delphix/debug $MGMT_FMRI) - - local jar=$root$UPGRADE_VERIFY_JAR - $java -Dlog.dir=$LOG_DIR -Dmdsverify=true -Ddelphix.debug="$debug_mode" \ - -DosMigration=true -jar "$jar" -d "$output" -f "$format" \ - -l "$locale" -v "$version" -root "$root" "$upgrade_verify_opts" \ - -droot "$root" -pl "$progress_low" -ph "$progress_high" || - die "upgrade verification failed" -} - -function test_masking() { - masking_temp_dir=$(mktemp -d) || - die "unable to make temporary directory" - - tar xof "$root/opt/delphix/masking/resources/upgrade-verification.tar" \ - -C "$masking_temp_dir" || - die "failed to extract masking upgrade verification tool" - - local verify_tool_dir=$masking_temp_dir/upgrade-verification - local new_conf=$root/opt/delphix/masking/conf/ - local overrides=/var/delphix/dmsuite/overrides/ - - # - # Run the verification tool against the new version of the .properties - # files in conf/ and against override.properties, to ensure that - # override.properties only overrides supported properties. - # - local cp="$verify_tool_dir/*:$new_conf:$overrides" - $java -cp "$cp" com.dmsuite.VerifyProperties || - die "verification of overrides failed" -} - -disable_consistent_mds_zfs_data_util=false -while getopts ':v:o:l:f:d' c; do - case "$c" in - f) format=$OPTARG ;; - l) locale=$OPTARG ;; - o) output=$OPTARG ;; - v) version=$OPTARG ;; - d) disable_consistent_mds_zfs_data_util=true ;; - *) usage ;; - esac -done - -[[ -n $version ]] || usage - -# Where root is or will be mounted. Set in mount_datasets. -root= -# The path of the running dataset mounted at /var/delphix. Set in mount_datasets. -runningVar= -# Temporary diresctory used by the masking checks. Set in test_masking. -masking_temp_dir= -# The installation of java to use when executing our jars. -java=/opt/jdk/bin/java - -report_progress_inc 0 "preparing for verification" -mount_datasets "$version" -report_progress_inc 20 "running upgrade checks" -run_upgrade_verify "$output" "$format" "$locale" 20 95 - -DX_UPG_STRESS="${BASH_SOURCE%/*}/dx_upg_stress_options" -# shellcheck source=/dev/null -. $DX_UPG_STRESS --source -__trigger_unset_stress_option "STRESS_DX_VERIFY_FAIL_AFTER_TEST_MIGRATION" - -test_masking -report_progress_inc 95 "cleaning up post-verification" -cleanup -report_progress_inc 100 "done" -exit 0 diff --git a/live-build/misc/migration-scripts/etc_system_whitelist b/live-build/misc/migration-scripts/etc_system_whitelist deleted file mode 100644 index 5f9b3f77..00000000 --- a/live-build/misc/migration-scripts/etc_system_whitelist +++ /dev/null @@ -1,10 +0,0 @@ -sd:bypass_mode_sense_for_geometry -zfs_recover -zfs_vdev_async_read_max_active -zfs_vdev_async_read_min_active -zfs_vdev_async_write_max_active -zfs_vdev_async_write_min_active -zfs_vdev_sync_read_max_active -zfs_vdev_sync_read_min_active -zfs_vdev_sync_write_max_active -zfs_vdev_sync_write_min_active diff --git a/scripts/run-live-build.sh b/scripts/run-live-build.sh index 04d1ab7c..991d67b3 100755 --- a/scripts/run-live-build.sh +++ b/scripts/run-live-build.sh @@ -94,7 +94,6 @@ if [[ "$RUN_TYPE" == "$ALL_RUN_TYPE" || "$RUN_TYPE" == "$VM_RUN_TYPE" ]]; then fi cp -r "$TOP/live-build/variants/$APPLIANCE_VARIANT/ansible" "$build_dir" -cp -r "$TOP/live-build/misc/migration-scripts" "$build_dir" cd "$build_dir" @@ -256,7 +255,7 @@ esac # user (e.g. other software); this is most useful when multiple variants # are built via a single call to "make" (e.g. using the "all" target). # -for ext in debs.tar.gz migration.tar.gz $vm_artifact_ext; do +for ext in debs.tar.gz $vm_artifact_ext; do if [[ -f "$ARTIFACT_NAME.$ext" ]]; then mv "$ARTIFACT_NAME.$ext" "$TOP/live-build/build/artifacts/" fi From 57ce711229a6c14621132426e351771aa2f34550 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Mon, 12 Jul 2021 13:53:19 -0700 Subject: [PATCH 04/12] DLPX-76433 [Backport of DLPX-76376 to 6.0.10.0] rootfs property "com.delphix:hotfix-version" incorrect after rollback of hotfix (#573) --- upgrade/upgrade-scripts/common.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/common.sh b/upgrade/upgrade-scripts/common.sh index af027af8..e4f8efb2 100644 --- a/upgrade/upgrade-scripts/common.sh +++ b/upgrade/upgrade-scripts/common.sh @@ -223,7 +223,13 @@ function copy_optional_dataset_property() { local DST_DATASET="$3" local PROP_VALUE - PROP_VALUE=$(zfs get -Hpo value "$PROP_NAME" "$SRC_DATASET") + # + # Note, we only want to copy the dataset property when it's a + # local value, rather than a potentially inherited value. Thus, + # we use "-s local" to acheive this; i.e. with that set, if the + # value is not local, no value will be returned. + # + PROP_VALUE=$(zfs get -s local -Hpo value "$PROP_NAME" "$SRC_DATASET") # # Unlike the "copy_required_dataset_property" function, if the From 0194606e800eef04c2a261e32eb74cbe0516054c Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Tue, 13 Jul 2021 11:42:12 -0400 Subject: [PATCH 05/12] DLPX-76478 [Backport of DLPX-71683 to 6.0.10.0] Generate new buildinfo package using metadata from combined packages (#493) (#572) --- .../appliance-build.bootstrap/tasks/main.yml | 1 + build-info/debian/compat | 1 + build-info/debian/control | 25 ++++ build-info/debian/copyright | 18 +++ build-info/debian/install | 1 + build-info/debian/rules | 19 +++ build-info/debian/source/format | 1 + scripts/build-ancillary-repository.sh | 7 ++ scripts/create-build-info-package.sh | 113 ++++++++++++++++++ 9 files changed, 186 insertions(+) create mode 100644 build-info/debian/compat create mode 100644 build-info/debian/control create mode 100644 build-info/debian/copyright create mode 100644 build-info/debian/install create mode 100755 build-info/debian/rules create mode 100644 build-info/debian/source/format create mode 100755 scripts/create-build-info-package.sh diff --git a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml index 9b067a02..a2b2fbb4 100644 --- a/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml +++ b/bootstrap/roles/appliance-build.bootstrap/tasks/main.yml @@ -25,6 +25,7 @@ - awscli - bc - coreutils + - devscripts - equivs - gdisk - git diff --git a/build-info/debian/compat b/build-info/debian/compat new file mode 100644 index 00000000..f599e28b --- /dev/null +++ b/build-info/debian/compat @@ -0,0 +1 @@ +10 diff --git a/build-info/debian/control b/build-info/debian/control new file mode 100644 index 00000000..cdacd76a --- /dev/null +++ b/build-info/debian/control @@ -0,0 +1,25 @@ +# +# Copyright 2020 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +Source: delphix-build-info +Section: misc +Priority: optional +Maintainer: Delphix Engineering +Build-Depends: debhelper (>= 10), devscripts +Standards-Version: 4.1.2 + +Package: delphix-build-info +Architecture: all +Description: Provides build metadata for packages built by Delphix. diff --git a/build-info/debian/copyright b/build-info/debian/copyright new file mode 100644 index 00000000..b4986c55 --- /dev/null +++ b/build-info/debian/copyright @@ -0,0 +1,18 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Files: * +Copyright: 2021 Delphix +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the complete text of the Apache License, Version 2 + can be found in "/usr/share/common-licenses/Apache-2.0". diff --git a/build-info/debian/install b/build-info/debian/install new file mode 100644 index 00000000..338285f1 --- /dev/null +++ b/build-info/debian/install @@ -0,0 +1 @@ +/var/delphix-buildinfo diff --git a/build-info/debian/rules b/build-info/debian/rules new file mode 100755 index 00000000..42f3b90d --- /dev/null +++ b/build-info/debian/rules @@ -0,0 +1,19 @@ +#!/usr/bin/make -f +# +# Copyright 2021 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +%: + dh $@ diff --git a/build-info/debian/source/format b/build-info/debian/source/format new file mode 100644 index 00000000..89ae9db8 --- /dev/null +++ b/build-info/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/scripts/build-ancillary-repository.sh b/scripts/build-ancillary-repository.sh index ae512f26..e216476f 100755 --- a/scripts/build-ancillary-repository.sh +++ b/scripts/build-ancillary-repository.sh @@ -111,6 +111,13 @@ mkdir -p "$WORK_DIRECTORY/artifacts" download_combined_packages_artifacts "$AWS_S3_URI_COMBINED_PACKAGES" \ "$WORK_DIRECTORY/artifacts" +# +# Create a delphix-build-info package from the build metadata of each +# package and of appliance-build itself and store it along with the other +# downloaded artifacts. +# +"$TOP"/scripts/create-build-info-package.sh "$WORK_DIRECTORY/artifacts" + # # Find all debs and put them into a directory that will be fed into Aptly. # diff --git a/scripts/create-build-info-package.sh b/scripts/create-build-info-package.sh new file mode 100755 index 00000000..b25cd70a --- /dev/null +++ b/scripts/create-build-info-package.sh @@ -0,0 +1,113 @@ +#!/bin/bash -ex +# +# Copyright 2021 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This script creates a new package that stores build metadata for packages +# built by Delphix. It takes one argument: the path where the +# combined-packages artifacts have been downloaded. +# + +. "${BASH_SOURCE%/*}/common.sh" + +if [[ -z "$TOP" ]]; then + echo "Must be run inside the git repsitory." + exit 1 +fi + +if [[ $# -ne 1 ]]; then + echo "Must specify the path of the combined-packages artifacts." 1>&2 + exit 1 +fi + +PKG_DIR="$1" +[[ -d "$PKG_DIR/packages" ]] || + die "$PKG_DIR must be a path to combined-packages artifacts." + +# +# Build info files will be installed into /var/delphix-buildinfo on the +# Delphix appliance. +# +target="$TOP/build-info/var/delphix-buildinfo" +mkdir -p "$target" + +# +# Copy build metadata for each package +# +mkdir "$target/packages" +cd "$PKG_DIR/packages" +for pkg in */; do + pushd "$pkg" &>/dev/null + mkdir "$target/packages/$pkg" + for file in GIT_HASH BUILD_INFO PACKAGE_MIRROR_URL_MAIN PACKAGE_MIRROR_URL_SECONDARY metadata.json; do + [[ -f "$file" ]] && cp "$file" "$target/packages/$pkg/" + done + popd &>/dev/null +done + +cp "$PKG_DIR/COMPONENTS" "$target/packages/" +cp "$PKG_DIR/KERNEL_VERSIONS" "$target/packages/" + +# +# Generate build metadata for appliance-build +# +mkdir "$target/appliance-build" +cd "$TOP" +git rev-parse HEAD >"$target/appliance-build/GIT_HASH" + +function check_env() { + # + # When the job is ran manually for testing purposes, we do not expect + # all environment to be set, so skip the env check. + # + [[ -n "$JENKINS_URL" ]] || return 0 + + local val="${!1}" + [[ -n "$val" ]] || die "check_env: $1 must be non-empty" + return 0 +} + +check_env APPLIANCE_BUILD_GIT_URL +echo "$APPLIANCE_BUILD_GIT_URL" >"$target/appliance-build/GIT_URL" +check_env APPLIANCE_BUILD_GIT_BRANCH +echo "$APPLIANCE_BUILD_GIT_BRANCH" >"$target/appliance-build/GIT_BRANCH" +check_env DELPHIX_PACKAGE_MIRROR_MAIN +echo "$DELPHIX_PACKAGE_MIRROR_MAIN" >"$target/appliance-build/DELPHIX_PACKAGE_MIRROR_MAIN" +check_env DELPHIX_PACKAGE_MIRROR_SECONDARY +echo "$DELPHIX_PACKAGE_MIRROR_SECONDARY" >"$target/appliance-build/DELPHIX_PACKAGE_MIRROR_SECONDARY" +check_env AWS_S3_OUTPUT +echo "$AWS_S3_OUTPUT" >"$target/appliance-build/ARTIFACTS_S3_LOCATION" +check_env DELPHIX_APPLIANCE_VERSION +echo "$DELPHIX_APPLIANCE_VERSION" >"$target/appliance-build/DELPHIX_APPLIANCE_VERSION" + +# +# Build the package +# +cd "$TOP/build-info" +# +# We include some random characters in the version string to avoid collisions +# with other build-info packages built for different platforms but included in +# the same upgrade image. The timestamp is insufficient here since the other +# platforms are built in parallel. +# +rnd="$(uuidgen | tr -d '-' | fold -w 8 | head -n 1)" +version="1.0.0-delphix-$(date '+%Y.%m.%d.%H.%M.%S')-$rnd" +dch --create --package delphix-build-info -v "$version" \ + "Automatically generated changelog entry." + +dpkg-buildpackage -uc -us -b + +mv ../delphix-build-info*deb "$PKG_DIR/packages/" From d761972c1b208810290dbf89e8c8a1b962670b40 Mon Sep 17 00:00:00 2001 From: Manoj Joseph Date: Wed, 28 Jul 2021 20:50:21 -0700 Subject: [PATCH 06/12] Add rust to dev environment (#544) (#577) Co-authored-by: Paul Dagnelie --- .../appliance-build.zfsonlinux-development/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml index a24a1a96..e17f4c35 100644 --- a/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.zfsonlinux-development/tasks/main.yml @@ -23,6 +23,7 @@ - alien - autoconf - build-essential + - cargo - cppcheck - curl - emacs-nox @@ -43,7 +44,9 @@ - mandoc - nfs-kernel-server - parted + - pkg-config - python-minimal + - rustc - shellcheck - targetcli-fb - unzip From 3548595e4dc83e925eb99f0325f2247d9bd8d250 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Fri, 30 Jul 2021 09:58:52 -0400 Subject: [PATCH 07/12] Backport a few changes from master to 6.0/stage (#579) * remove crash-python role from appliance-build Now that we can run crash-python outside the git repo from any path and we are halting future development on it, its role on the appliance build is no longer needed for developer VMs. Note that deleting that role uncovered a different bug where the minimal-common role in internal-dev is needed by the ldap role and it would always run before it because it was listed as a dependency under the crash python role. Now that the crash-python role is removed we re-add an old version of that configuration making the minimal-common role run before the ldap role. * Add "shell" subcommand to upgrade-container script (#420) This adds a new "shell" subcommand to the "upgrade-container" script, which is intended to enable users to obtain an interactive shell session within the specified container. Currently, the "run" subcommand can be used, while specifying the shell to run (e.g. /bin/bash), but since that does not specify the "--pty" option with "systemd-run", the terminal may not work correctly (e.g. keyboard input, pager output, etc). Now, when folks want to obtain an interactive shell, they should use this new "shell" subcommand, rather than "run". * Fix "upgrade-container shell" argument handling (#426) Co-authored-by: Serapheim Dimitropoulos Co-authored-by: Prakash Surya --- .../meta/main.yml | 19 ------- .../tasks/main.yml | 51 ------------------- .../internal-dev/ansible/playbook.yml | 13 +++-- upgrade/upgrade-scripts/upgrade-container | 11 ++++ 4 files changed, 21 insertions(+), 73 deletions(-) delete mode 100644 live-build/misc/ansible-roles/appliance-build.crash-python-development/meta/main.yml delete mode 100644 live-build/misc/ansible-roles/appliance-build.crash-python-development/tasks/main.yml diff --git a/live-build/misc/ansible-roles/appliance-build.crash-python-development/meta/main.yml b/live-build/misc/ansible-roles/appliance-build.crash-python-development/meta/main.yml deleted file mode 100644 index 870f3a70..00000000 --- a/live-build/misc/ansible-roles/appliance-build.crash-python-development/meta/main.yml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - ---- -dependencies: - - role: appliance-build.minimal-common diff --git a/live-build/misc/ansible-roles/appliance-build.crash-python-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.crash-python-development/tasks/main.yml deleted file mode 100644 index 4d3526fd..00000000 --- a/live-build/misc/ansible-roles/appliance-build.crash-python-development/tasks/main.yml +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - ---- -- apt: - name: - - git - - python3-pyelftools - - python3-pip - state: present - register: result - until: result is not failed - retries: 3 - delay: 60 - -- pip: - name: mypy - state: present - register: result - until: result is not failed - retries: 3 - delay: 60 - -- git: - repo: "https://gitlab.delphix.com/os-platform/crash-python.git" - dest: "/export/home/delphix/crash-python" - version: "next" - accept_hostkey: yes - update: no - -- file: - path: "/export/home/delphix/{{ item }}" - owner: delphix - group: staff - state: directory - recurse: yes - with_items: - - crash-python diff --git a/live-build/variants/internal-dev/ansible/playbook.yml b/live-build/variants/internal-dev/ansible/playbook.yml index c2ecff0d..7946d852 100644 --- a/live-build/variants/internal-dev/ansible/playbook.yml +++ b/live-build/variants/internal-dev/ansible/playbook.yml @@ -21,7 +21,16 @@ ansible_python_interpreter: /usr/bin/python3 passwordless_ldap_sudo: true roles: - - appliance-build.crash-python-development + # + # In order for the local appliance user (e.g. delphix) to be created + # properly, we need to ensure we attempt to create this user before + # we configure the system to use LDAP. Once we enable LDAP, we'll + # fail to properly create the user. Thus, we need to ensure we apply + # this role (which will create the user) prior to applying the LDAP + # specific configuration/role (which we do below). + # + - appliance-build.minimal-common + - appliance-build.minimal-internal # # In order for LDAP to work properly, we need a specific ordering of the # values contained in the "passwd" line of the "/etc/nsswitch.conf" file. @@ -32,8 +41,6 @@ # roles later. # - appliance-build.delphix-ldap - - appliance-build.minimal-common - - appliance-build.minimal-internal - appliance-build.minimal-development - appliance-build.masking-common - appliance-build.masking-development diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index b7e9610a..1591d5d2 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -508,6 +508,10 @@ function run() { --quiet --wait --pipe -- "$@" } +function shell() { + machinectl shell "$CONTAINER" +} + function convert_to_rootfs() { # # We're relying on the "mountpoint" property for the "data" and @@ -843,6 +847,7 @@ function usage() { echo "$PREFIX_SPACES stop " echo "$PREFIX_SPACES destroy " echo "$PREFIX_SPACES run " + echo "$PREFIX_SPACES shell " echo "$PREFIX_SPACES upgrade " echo "$PREFIX_SPACES convert-to-rootfs " echo "$PREFIX_SPACES get-type " @@ -891,6 +896,12 @@ run) shift 2 run "$@" ;; +shell) + [[ $# -lt 2 ]] && usage "too few arguments specified" + CONTAINER="$2" + shift 2 + shell + ;; upgrade) [[ $# -lt 2 ]] && usage "too few arguments specified" [[ $# -gt 2 ]] && usage "too many arguments specified" From 3ba04d835df60c479813835bf452487af1f502c0 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Tue, 3 Aug 2021 11:15:25 -0700 Subject: [PATCH 08/12] DLPX-76770 [Backport of DLPX-68846] logic for determining device path for grub operations during upgrade is fragile (#581) --- upgrade/upgrade-scripts/rootfs-container | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/upgrade/upgrade-scripts/rootfs-container b/upgrade/upgrade-scripts/rootfs-container index f2af998f..9789531a 100755 --- a/upgrade/upgrade-scripts/rootfs-container +++ b/upgrade/upgrade-scripts/rootfs-container @@ -87,7 +87,17 @@ function get_bootloader_devices() { # zpool list -vH rpool | awk '! /rpool|mirror|replacing|spare/ {print $1}' | - sed 's/p\{0,1\}[0-9]*$//' + while read -r part; do + # + # If the rpool is not installed a parition, we throw + # an error. We expect this to never happen, and the + # calling code is likely untested in that case, so we + # throw an error rather than try to handle it. + # + [[ "$(lsblk --nodeps -no type "/dev/$part")" == "part" ]] || + die "rpool installed on full disk \"$part\"" + lsblk -no pkname "/dev/$part" + done } function set_bootfs_not_mounted_cleanup() { From 63cd72f3d9e042cd0a1f21c83c100d02fb65c243 Mon Sep 17 00:00:00 2001 From: Pavel Zakharov Date: Tue, 3 Aug 2021 14:57:24 -0400 Subject: [PATCH 09/12] DLPX-76769 [Backport of DLPX-76694 to 6.0.10.0] Failure to build DelphixConnector on Delphix Engine, causing gui_sanity testing to fail (#580) --- .../tasks/main.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml index 1add61b9..7a752d03 100644 --- a/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml +++ b/live-build/misc/ansible-roles/appliance-build.virtualization-development/tasks/main.yml @@ -15,6 +15,19 @@ # --- +# +# pkg-config is necessary for building the DelphixConnector, which is built +# when running "ant all". +# +- apt: + name: + - pkg-config + state: present + register: result + until: result is not failed + retries: 3 + delay: 60 + - file: path: "/etc/systemd/system/delphix-mgmt.service.d" owner: root From 4e194afe5f9e2f0debedae525f4654865ee3f386 Mon Sep 17 00:00:00 2001 From: Prakash Surya Date: Tue, 10 Aug 2021 13:31:06 -0700 Subject: [PATCH 10/12] DLPX-76293 Copy Hotfix Metadata Generation to appliance-build (#574) (#590) Co-authored-by: Matt Skinner --- build.gradle | 3 ++- .../configuration/81-upgrade-repository.binary | 14 ++++++++++++++ live-build/config/hooks/configuration/template.ctl | 2 +- scripts/create-build-info-package.sh | 2 ++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 85e0bcc5..178c5bd9 100644 --- a/build.gradle +++ b/build.gradle @@ -78,7 +78,8 @@ for (variant in allVariants) { } } -def shellScripts = fileTree("scripts") + +def shellScripts = fileTree("scripts").include({ details -> + details.file.getName().endsWith(".sh") }) + fileTree("live-build/config/hooks").include({ details -> details.file.canExecute() }) + diff --git a/live-build/config/hooks/configuration/81-upgrade-repository.binary b/live-build/config/hooks/configuration/81-upgrade-repository.binary index a1ce6b40..c871aac8 100755 --- a/live-build/config/hooks/configuration/81-upgrade-repository.binary +++ b/live-build/config/hooks/configuration/81-upgrade-repository.binary @@ -80,6 +80,20 @@ EOF # rename 's/\%3a/:/g' binary/packages/*.deb +# +# Copy over the generated hotfix metadata. +# +if [[ -z "$AWS_S3_HOTFIX_METADATA" ]]; then + touch hotfix_metadata +else + aws s3 cp --only-show-errors "$AWS_S3_HOTFIX_METADATA" hotfix_metadata +fi + +if [[ ! -f "hotfix_metadata" ]]; then + echo "Could not generate hotfix_metadata file." + exit 1 +fi + # # Now we need to create the "delphix-entire" metapackage that will # depend on all of the packages that have been installed in the binary diff --git a/live-build/config/hooks/configuration/template.ctl b/live-build/config/hooks/configuration/template.ctl index 2faa6336..27e913e4 100644 --- a/live-build/config/hooks/configuration/template.ctl +++ b/live-build/config/hooks/configuration/template.ctl @@ -6,7 +6,7 @@ Standards-Version: 3.9.2 Package: delphix-entire-@@PLATFORM@@ Provides: delphix-entire Version: @@VERSION@@ -Extra-Files: packages.list, variant +Extra-Files: packages.list, variant, hotfix_metadata Description: Entirety of Delphix Appliance This package depends on all of the packages that constitute the entirety of the Delphix Appliance. This set of packages provide the necessary tools to run diff --git a/scripts/create-build-info-package.sh b/scripts/create-build-info-package.sh index b25cd70a..eee73912 100755 --- a/scripts/create-build-info-package.sh +++ b/scripts/create-build-info-package.sh @@ -92,6 +92,8 @@ check_env AWS_S3_OUTPUT echo "$AWS_S3_OUTPUT" >"$target/appliance-build/ARTIFACTS_S3_LOCATION" check_env DELPHIX_APPLIANCE_VERSION echo "$DELPHIX_APPLIANCE_VERSION" >"$target/appliance-build/DELPHIX_APPLIANCE_VERSION" +check_env AWS_S3_HOTFIX_METADATA +echo "$AWS_S3_HOTFIX_METADATA" >"$target/appliance-build/HOTFIX_METDATA" # # Build the package From c8ca4c3e26f8f84fa213364f6a87b096fc2de4c1 Mon Sep 17 00:00:00 2001 From: Sanjay Bharadwaj Date: Wed, 11 Aug 2021 12:41:41 -0700 Subject: [PATCH 11/12] install-ansible-lint is failing (#587) (#593) Co-authored-by: Pavel Zakharov --- .github/scripts/install-ansible-lint.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/scripts/install-ansible-lint.sh b/.github/scripts/install-ansible-lint.sh index c36c6568..79d78dcd 100755 --- a/.github/scripts/install-ansible-lint.sh +++ b/.github/scripts/install-ansible-lint.sh @@ -5,7 +5,6 @@ sudo apt-get install ansible git clone https://github.com/willthames/ansible-lint /opt/ansible-lint cd /opt/ansible-lint git checkout v3.4.21 -git branch -D master # # GitHub Actions exposes the GITHUB_ENV file that can be used to From cb6a599e7a7fe87ab7fcf17f665cc32627e89bd6 Mon Sep 17 00:00:00 2001 From: Sanjay Bharadwaj Date: Thu, 12 Aug 2021 00:07:47 -0700 Subject: [PATCH 12/12] DLPX-76893 [Backport of DLPX-76802 to 6.0.10.0] Starting Verification container enables IP forwarding on host (#592) --- upgrade/upgrade-scripts/upgrade-container | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/upgrade/upgrade-scripts/upgrade-container b/upgrade/upgrade-scripts/upgrade-container index 1591d5d2..af6f303d 100755 --- a/upgrade/upgrade-scripts/upgrade-container +++ b/upgrade/upgrade-scripts/upgrade-container @@ -443,6 +443,29 @@ function stop() { sleep 1 done + # + # Starting the upgrade container would have enabled ip forwarding + # on the host (and in turn disabled LRO). Reset these settings + # on a best effor basis. + # + sysctl -w net.ipv4.ip_forward=0 || warn "failed to disable ip port forwarding" + + # + # Find all the interfaces configured on the host and set + # lro on where possible. The sed command retrieves the names of the + # interfaces. The name of the virtual interfaces of containers could + # have an @ symbol followed by device name in the output of the ethtool + # and this needs to be trimmed off. + # + + for i in $(ip -br link | sed 's/^\([^ @]\+\).*/\1/'); do + fixed=$(ethtool -k "$i" | grep large-receive-offload | grep -i fixed) + if [[ -z "$fixed" ]]; then + echo "updating lro setting for nic $i" + ethtool -K "$i" lro on || warn "failed to set lro ON for nic '$i'" + fi + done + machinectl status "$CONTAINER" &>/dev/null && die "timeout waiting for container termination: '$CONTAINER'"