PG-957, PG-958 | Adding a workflow for PPG Server installation & tap …

…test case for testing pg_tde along with install of multiple PPG extensions. Removed reduntant declaration of same variables in tap test cases.

.github/workflows/postgresql-16-ppg-package-pgxs.yml

@@ -0,0 +1,159 @@
+name: postgresql-16-ppg-package-pgxs
+on: 
+  pull_request:
+  workflow_dispatch:
+  push:
+    branches: [main]
+
+jobs:
+  build:
+    name: pg-16-ppg-package-pgxs-build
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Remove old postgres
+        run: |
+          sudo apt-get update
+          sudo apt purge postgresql-client-common postgresql-common \
+            postgresql postgresql*
+          sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
+            /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
+            /etc/postgresql
+          sudo rm -f /usr/bin/pg_config
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get install -y libreadline6-dev systemtap-sdt-dev wget \
+            zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl \
+            libcurl4-openssl-dev  libhttp-server-simple-perl
+          sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
+          sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
+      
+      - name: Install percona-release script
+        run: |
+          sudo apt-get -y update
+          sudo apt-get -y upgrade
+          sudo apt-get install -y wget gnupg2 curl lsb-release
+          sudo wget \
+            https://repo.percona.com/apt/percona-release_latest.generic_all.deb
+          sudo dpkg -i percona-release_latest.generic_all.deb
+
+      - name: Install Percona Distribution Postgresql 16 & Extensions
+        run: |
+          sudo percona-release setup ppg-16
+          sudo apt-get update -y
+          sudo apt-get install -y percona-postgresql-16 \
+            percona-postgresql-contrib percona-postgresql-server-dev-all \
+            percona-pgpool2 libpgpool2 percona-postgresql-16-pgaudit \
+            percona-postgresql-16-pgaudit-dbgsym percona-postgresql-16-repack \
+            percona-postgresql-16-repack-dbgsym percona-pgaudit16-set-user \
+            percona-pgaudit16-set-user-dbgsym percona-postgresql-16-postgis-3 \
+            percona-postgresql-16-postgis-3-scripts \
+            percona-postgresql-postgis-scripts percona-postgresql-postgis \
+            percona-postgis percona-pg-stat-monitor16
+
+      - name: Clone pg_tde repository
+        uses: actions/checkout@master
+        with:
+          path: 'src/pg_tde'
+
+      - name: Change src owner to postgres
+        run: |
+          sudo chmod o+rx ~
+          sudo chown -R postgres:postgres src
+
+      - name: Build pg_tde
+        run: |
+          sudo -u postgres bash -c './configure'
+          sudo -u postgres bash -c 'make USE_PGXS=1'
+          sudo make USE_PGXS=1 install
+        working-directory: src/pg_tde
+
+      - name: Start pg_tde tests
+        run: |
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
+
+          sudo psql -V
+          
+          sudo service postgresql stop
+          echo "shared_preload_libraries = 'pg_tde'" |
+            sudo tee -a /etc/postgresql/16/main/postgresql.conf
+          sudo service postgresql start
+
+          sudo -u postgres bash -c "export ROOT_TOKEN=$ROOT_TOKEN && make installcheck USE_PGXS=1"
+        working-directory: src/pg_tde
+
+      - name: Report on test fail
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: Regressions diff and postgresql log
+          path: |
+            src/pg_tde/regression.diffs
+            src/pg_tde/logfile
+          retention-days: 3
+
+      - name: Report on test fail 2
+        if: ${{ failure() }}
+        run: |
+          cat src/pg_tde/regression.diffs
+
+      - name: Create release directory
+        run: |
+          sudo mkdir pgtde-ppg16
+          sudo mkdir -p pgtde-ppg16/usr/lib/postgresql/16/lib/
+          sudo mkdir -p pgtde-ppg16/share/postgresql/16/extension/
+          sudo cp /usr/share/postgresql/16/extension/pg_tde* pgtde-ppg16/share/postgresql/16/extension/
+          sudo cp /usr/lib/postgresql/16/lib/pg_tde* pgtde-ppg16/usr/lib/postgresql/16/lib/
+
+      - name: Upload tgz
+        uses: actions/upload-artifact@v4
+        with:
+          name: pg_tde_ppg16_binary
+          path: pgtde-ppg16
+
+      - name: Create deb
+        run: |
+          sudo mkdir pgtde-ppg16/DEBIAN
+          sudo sh -c 'echo "Package: pgtde-ppg16" > pgtde-ppg16/DEBIAN/control'
+          sudo sh -c 'echo "Version: 0.1" >> pgtde-ppg16/DEBIAN/control'
+          sudo sh -c 'echo "Architecture: amd64" >> pgtde-ppg16/DEBIAN/control'
+          sudo sh -c 'echo "Maintainer: Percona" >> pgtde-ppg16/DEBIAN/control'
+          sudo sh -c 'echo "Description: Experimental pg_tde extension" >> pgtde-ppg16/DEBIAN/control'
+          sudo dpkg-deb --build --root-owner-group pgtde-ppg16
+
+      - name: Test deb
+        run: |
+          sudo rm -rf /usr/share/postgresql/16/extension/pg_tde*
+          sudo rm -rf /usr/lib/postgresql/16/lib/pg_tde*
+          sudo dpkg -i --debug=7777 pgtde-ppg16.deb
+
+      - name: Upload deb
+        uses: actions/upload-artifact@v4
+        with:
+          name: pg_tde_deb
+          path: pgtde-ppg16.deb
+
+      - name: Create tgz
+        run: |
+          cd pgtde-ppg16 && sudo tar -czvf ../pgtde-ppg16.tar.gz .
+
+      - name: Publish release
+        uses: ncipollo/release-action@v1
+        # Only try and deploy on merged code
+        if: "github.repository == 'Percona-Lab/pg_tde' && github.ref_name == 'main' && (github.event_name == 'push' || github.event_name == 'schedule')"
+        with:
+          artifacts: "pgtde-ppg16.tar.gz,pgtde-ppg16.deb"
+          omitBody: true
+          allowUpdates: true
+          generateReleaseNotes: true
+          makeLatest: true
+          tag: "latest"
+          name: "HEAD"
+          replacesArtifacts: true

t/004_file_config.pl

@@ -37,7 +37,7 @@
 $rt_value = $node->psql('postgres', "SELECT pg_tde_add_key_provider_file('file-provider', json_object( 'type' VALUE 'file', 'path' VALUE '/tmp/datafile-location' ));", extra_params => ['-a']);
 $rt_value = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','file-provider');", extra_params => ['-a']);
 
-my $stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde_basic;', extra_params => ['-a']);
+$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde_basic;', extra_params => ['-a']);
 PGTDE::append_to_file($stdout);
 
 $stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc1 (k) VALUES (5),(6);', extra_params => ['-a']);
@@ -59,7 +59,7 @@
 
 # DROP EXTENSION
 $stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']);
-ok(my $cmdret == 0, "DROP PGTDE EXTENSION");
+ok($cmdret == 0, "DROP PGTDE EXTENSION");
 PGTDE::append_to_file($stdout);
 # Stop the server
 $node->stop();

t/005_multiple_extensions.pl

@@ -0,0 +1,158 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use File::Basename;
+use File::Compare;
+use File::Copy;
+use Test::More;
+use lib 't';
+use pgtde;
+
+# Get file name and CREATE out file name and dirs WHERE requried
+PGTDE::setup_files_dir(basename($0));
+
+my $PG_VERSION_STRING = `pg_config --version`;
+
+if (index(lc($PG_VERSION_STRING), lc("percona")) == -1)
+{
+    plan skip_all => "pg_tde test case only for PPG server package install with extensions.";
+}
+
+# CREATE new PostgreSQL node and do initdb
+my $node = PGTDE->pgtde_init_pg();
+my $pgdata = $node->data_dir;
+
+copy("$pgdata/postgresql.conf", "$pgdata/postgresql.conf.bak");
+
+# UPDATE postgresql.conf to include/load pg_stat_monitor library
+open my $conf, '>>', "$pgdata/postgresql.conf";
+print $conf "shared_preload_libraries = 'pg_tde, pg_stat_monitor, pgaudit, set_user, pg_repack'\n";
+print $conf "pg_stat_monitor.pgsm_bucket_time = 360000\n";
+print $conf "pg_stat_monitor.pgsm_normalized_query = 'yes'\n";
+close $conf;
+
+open my $conf2, '>>', "/tmp/datafile-location";
+print $conf2 "/tmp/keyring_data_file\n";
+close $conf2;
+
+# Start server
+my $rt_value = $node->start;
+ok($rt_value == 1, "Start Server");
+
+# Create PGSM extension
+my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION pg_stat_monitor;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE PGSM EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT pg_stat_monitor_reset();', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']);
+ok($cmdret == 0, "Reset PGSM EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+
+# Create pg_tde extension
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION pg_tde;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE PGTDE EXTENSION");
+PGTDE::append_to_file($stdout);
+
+# Create Other extensions
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pgaudit;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE pgaudit EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS set_user;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE set_user EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_repack;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE pg_repack EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', "SET pgaudit.log = 'none'; CREATE EXTENSION IF NOT EXISTS postgis; SET pgaudit.log = 'all';", extra_params => ['-a']);
+ok($cmdret == 0, "CREATE postgis EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS postgis_raster;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE postgis_raster EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS postgis_sfcgal;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE postgis_sfcgal EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE fuzzystrmatch EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS address_standardizer;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE address_standardizer EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS address_standardizer_data_us;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE address_standardizer_data_us EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE postgis_tiger_geocoder EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+
+$rt_value = $node->psql('postgres', "SELECT pg_tde_add_key_provider_file('file-provider', json_object( 'type' VALUE 'file', 'path' VALUE '/tmp/datafile-location' ));", extra_params => ['-a']);
+$rt_value = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','file-provider');", extra_params => ['-a']);
+
+$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde_basic;', extra_params => ['-a']);
+PGTDE::append_to_file($stdout);
+
+$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc1 (k) VALUES (5),(6);', extra_params => ['-a']);
+PGTDE::append_to_file($stdout);
+
+$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;', extra_params => ['-a']);
+PGTDE::append_to_file($stdout);
+
+# Restart the server
+PGTDE::append_to_file("-- server restart");
+$rt_value = $node->stop();
+$rt_value = $node->start();
+
+$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;', extra_params => ['-a']);
+PGTDE::append_to_file($stdout);
+
+$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc1;', extra_params => ['-a']);
+PGTDE::append_to_file($stdout);
+
+# Print PGSM settings 
+($cmdret, $stdout, $stderr) = $node->psql('postgres', "SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_query_shared_buffer';", extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']);
+ok($cmdret == 0, "Print PGTDE EXTENSION Settings");
+PGTDE::append_to_debug_file($stdout);
+
+# Create example database and run pgbench init
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE database example;', extra_params => ['-a']);
+print "cmdret $cmdret\n";
+ok($cmdret == 0, "CREATE Database example");
+PGTDE::append_to_debug_file($stdout);
+
+my $port = $node->port;
+print "port $port \n";
+
+my $out = system ("pgbench -i -s 20 -p $port example");
+print " out: $out \n";
+ok($cmdret == 0, "Perform pgbench init");
+
+$out = system ("pgbench -c 10 -j 2 -t 5000 -p $port example");
+print " out: $out \n";
+ok($cmdret == 0, "Run pgbench");
+
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT datname, substr(query,0,150) AS query, SUM(calls) AS calls FROM pg_stat_monitor GROUP BY datname, query ORDER BY datname, query, calls;', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']);
+ok($cmdret == 0, "SELECT XXX FROM pg_stat_monitor");
+PGTDE::append_to_debug_file($stdout);
+
+# DROP EXTENSION
+$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']);
+ok($cmdret == 0, "DROP PGTDE EXTENSION");
+PGTDE::append_to_file($stdout);
+
+# DROP EXTENSION
+$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_stat_monitor;', extra_params => ['-a']);
+ok($cmdret == 0, "DROP PGTDE EXTENSION");
+PGTDE::append_to_debug_file($stdout);
+
+# Stop the server
+$node->stop();
+
+# compare the expected and out file
+my $compare = PGTDE->compare_results();
+
+# Test/check if expected and result/out file match. If Yes, test passes.
+is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files.");
+
+# Done testing for this testcase file.
+done_testing();

t/expected/005_multiple_extensions.out

@@ -0,0 +1,12 @@
+CREATE EXTENSION pg_tde;
+CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde_basic;
+INSERT INTO test_enc1 (k) VALUES (5),(6);
+SELECT * FROM test_enc1 ORDER BY id ASC;
+1|5
+2|6
+-- server restart
+SELECT * FROM test_enc1 ORDER BY id ASC;
+1|5
+2|6
+DROP TABLE test_enc1;
+DROP EXTENSION pg_tde;