From 65c9ac767d118c93e51a71e90077786244f2e975 Mon Sep 17 00:00:00 2001 From: Marcelo Altmann Date: Thu, 14 Sep 2023 16:36:20 -0300 Subject: [PATCH] Fixed test cases and bootstrap Adjusted keyring to always pass plugin dir. Adjusted mix plugins to ensure we are running on PS Bootstrap to download correct glibc version Bumped default server version Added python3 version of subunit2junitxml (cherry picked from commit d9d47b5ec598c17f8ae98927d934ab8699a55329) Fixed is_xbcloud_credentials_set: On centos7 the expanded command "${XBCLOUD_CREDENTIALS:-unset}" == "unset" return 1 to the caller rather than 0. Fixed by properly adjusting the function to have an if statement. --- storage/innobase/xtrabackup/test/bootstrap.sh | 47 +++++++++++--- .../xtrabackup/test/inc/keyring_common.sh | 8 +-- .../xtrabackup/test/inc/xbcloud_common.sh | 6 +- .../xtrabackup/test/subunit2junitxml_python3 | 64 +++++++++++++++++++ .../keyring/innodb_encryption_mix_plugins.sh | 2 + 5 files changed, 112 insertions(+), 15 deletions(-) create mode 100755 storage/innobase/xtrabackup/test/subunit2junitxml_python3 diff --git a/storage/innobase/xtrabackup/test/bootstrap.sh b/storage/innobase/xtrabackup/test/bootstrap.sh index de29461f1775..6d57684f7949 100755 --- a/storage/innobase/xtrabackup/test/bootstrap.sh +++ b/storage/innobase/xtrabackup/test/bootstrap.sh @@ -22,15 +22,25 @@ function ssl_version() { 100|101) ;; 102) unset sslv; sslv="102.$OS" ;; *) - if ! test -r "${1}"; then - >&2 echo "tarball for your openssl version (${sslv}) is not available" - exit 1 - fi + >&2 echo "tarball for your openssl version (${sslv}) is not available" + exit 1 ;; esac echo ${sslv} } +function glibc_version() { + glibc=$(ldd --version | head -1 | awk '{print $NF}') + case ${glibc} in + 2.12|2.17|2.27|2.28|2.31|2.34|2.35) ;; + *) + >&2 echo "tarball for your glibc version (${glibc}) is not available" + exit 1 + ;; + esac + echo ${glibc} +} + shell_quote_string() { echo "$1" | sed -e 's,\([^a-zA-Z0-9/_.=-]\),\\\1,g' } @@ -64,6 +74,23 @@ parse_arguments() { done } +check_url() { + url=$1 + tarball=$2 + retries=10 + # upstream sometimes reports file does not exists due to transient error + # we should retry a few times before failing back + tries=0 + while [[ ${tries} -lt ${retries} ]]; do + if ! wget --spider "${url}/${tarball}" 2>/dev/null; then + tries=$((tries+1)) + else + return 0; + fi + done + return 1; +} + main () { if [ -f /etc/redhat-release ]; then OS="rpm" @@ -87,23 +114,25 @@ main () { url="https://dev.mysql.com/get/Downloads/MySQL-8.0" fallback_url="https://downloads.mysql.com/archives/get/p/23/file" tarball="mysql-${VERSION}-linux-glibc2.12-${arch}.tar.xz" - if ! wget --spider "${url}/${tarball}" 2>/dev/null; then + if ! check_url "${url}" "${tarball}"; then unset url url=${fallback_url} - fi + fi ;; xtradb80) url="https://www.percona.com/downloads/Percona-Server-8.0/Percona-Server-${VERSION}/binary/tarball" short_version=$(echo ${VERSION} | awk -F "." '{ print $3 }' | cut -d '-' -f1) if [[ ${PXB_TYPE} == "Debug" ]] || [[ ${PXB_TYPE} == "debug" ]]; then SUFFIX="-debug" + else + SUFFIX="-minimal" fi if [[ ${short_version} -lt "20" ]]; then tarball="Percona-Server-${VERSION}-Linux.${arch}.ssl$(ssl_version).tar.gz" elif [[ ${short_version} -ge "20" && ${short_version} -lt "22" ]]; then tarball="Percona-Server-${VERSION}-Linux.${arch}.glibc2.12${SUFFIX}.tar.gz" elif [[ ${short_version} -ge "22" ]]; then - tarball="Percona-Server-${VERSION}-Linux.${arch}.glibc2.17${SUFFIX}.tar.gz" + tarball="Percona-Server-${VERSION}-Linux.${arch}.glibc$(glibc_version)${SUFFIX}.tar.gz" fi ;; *) @@ -115,7 +144,7 @@ main () { esac # Check if tarball exist before any download - if ! wget --spider "${url}/${tarball}" 2>/dev/null; then + if ! check_url "${url}" "${tarball}"; then echo "Version you specified(${VERSION}) does not exist on ${url}/${tarball}" exit 1 else @@ -148,7 +177,7 @@ main () { TYPE="xtradb80" PXB_TYPE="release" -VERSION="8.0.18-9" +VERSION="8.0.34-26" DESTDIR="./server" parse_arguments PICK-ARGS-FROM-ARGV "$@" main diff --git a/storage/innobase/xtrabackup/test/inc/keyring_common.sh b/storage/innobase/xtrabackup/test/inc/keyring_common.sh index 91641a7b7651..126701aaa499 100644 --- a/storage/innobase/xtrabackup/test/inc/keyring_common.sh +++ b/storage/innobase/xtrabackup/test/inc/keyring_common.sh @@ -14,15 +14,15 @@ function test_do() vlog "-- transition_key: ${transition_key} --" vlog "-- keyring_type: ${keyring_type} --" if [[ "$transition_key" = "generate" ]] ; then - backup_options="--generate-transition-key" + backup_options="--generate-transition-key --xtrabackup-plugin-dir=${plugin_dir}" prepare_options="--xtrabackup-plugin-dir=${plugin_dir} ${keyring_args}" copyback_options="--xtrabackup-plugin-dir=${plugin_dir} ${keyring_args}" elif [[ "$transition_key" = "none" ]] ; then - backup_options= + backup_options="--xtrabackup-plugin-dir=${plugin_dir}" prepare_options="--xtrabackup-plugin-dir=${plugin_dir} ${keyring_args}" copyback_options="--xtrabackup-plugin-dir=${plugin_dir} ${keyring_args}" else - backup_options="--transition-key=$transition_key" + backup_options="--transition-key=$transition_key --xtrabackup-plugin-dir=${plugin_dir}" prepare_options="--transition-key=$transition_key" copyback_options="--transition-key=$transition_key --xtrabackup-plugin-dir=${plugin_dir} ${keyring_args}" fi @@ -36,7 +36,7 @@ function test_do() run_cmd $MYSQL $MYSQL_ARGS test -e "SELECT @@server_uuid" # PXB-1540: XB removes and recreate keyring file of 0 size - xtrabackup --backup --target-dir=$topdir/backup0 + xtrabackup --backup --target-dir=$topdir/backup0 $backup_options rm -rf $topdir/backup0 diff --git a/storage/innobase/xtrabackup/test/inc/xbcloud_common.sh b/storage/innobase/xtrabackup/test/inc/xbcloud_common.sh index 5d1037b4c8e4..52dccaa1b88e 100644 --- a/storage/innobase/xtrabackup/test/inc/xbcloud_common.sh +++ b/storage/innobase/xtrabackup/test/inc/xbcloud_common.sh @@ -23,8 +23,10 @@ secure-file-priv=$TEST_VAR_ROOT is_galera && skip_test "skipping" function is_xbcloud_credentials_set() { - [ "${XBCLOUD_CREDENTIALS:-unset}" == "unset" ] && \ - skip_test "Requires XBCLOUD_CREDENTIALS" + if [ -z ${XBCLOUD_CREDENTIALS+x} ]; + then + skip_test "Requires XBCLOUD_CREDENTIALS" + fi } now=$(date +%s) diff --git a/storage/innobase/xtrabackup/test/subunit2junitxml_python3 b/storage/innobase/xtrabackup/test/subunit2junitxml_python3 new file mode 100755 index 000000000000..b898b39220e2 --- /dev/null +++ b/storage/innobase/xtrabackup/test/subunit2junitxml_python3 @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# subunit: extensions to python unittest to get test results from subprocesses. +# Copyright (C) 2009 Robert Collins +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licenses. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + +"""Filter a subunit stream to get aggregate statistics.""" + +from optparse import OptionParser +import sys +import unittest + +from subunit import DiscardStream, ProtocolTestCase +try: + from junitxml import JUnitXmlResult +except ImportError: + sys.stderr.write("python-junitxml (https://launchpad.net/pyjunitxml or " + "http://pypi.python.org/pypi/junitxml) is required for this filter.") + raise + +parser = OptionParser(description=__doc__) +parser.add_option("--no-passthrough", action="store_true", + help="Hide all non subunit input.", default=False, dest="no_passthrough") +parser.add_option("-o", "--output-to", + help="Output the XML to this path rather than stdout.") +parser.add_option("-f", "--forward", action="store_true", default=False, + help="Forward subunit stream on stdout.") +(options, args) = parser.parse_args() +if options.output_to is None: + output_to = sys.stdout +else: + output_to = open(options.output_to, 'wb') +try: + result = JUnitXmlResult(output_to) + if options.no_passthrough: + passthrough_stream = DiscardStream() + else: + passthrough_stream = None + if options.forward: + forward_stream = sys.stdout + else: + forward_stream = None + test = ProtocolTestCase(sys.stdin.buffer, passthrough=passthrough_stream, + forward=forward_stream) + result.startTestRun() + test.run(result) + result.stopTestRun() +finally: + if options.output_to is not None: + output_to.close() +if result.wasSuccessful(): + exit_code = 0 +else: + exit_code = 1 +sys.exit(exit_code) diff --git a/storage/innobase/xtrabackup/test/suites/keyring/innodb_encryption_mix_plugins.sh b/storage/innobase/xtrabackup/test/suites/keyring/innodb_encryption_mix_plugins.sh index 0e73576518bd..3c86b204a423 100644 --- a/storage/innobase/xtrabackup/test/suites/keyring/innodb_encryption_mix_plugins.sh +++ b/storage/innobase/xtrabackup/test/suites/keyring/innodb_encryption_mix_plugins.sh @@ -4,6 +4,8 @@ require_server_version_higher_than 5.7.10 +is_xtradb || skip_test "Keyring vault requires Percona Server" + vlog setup keyring_file . inc/keyring_file.sh