diff --git a/.github/workflows/postgresql-16-ppg-package-pgxs.yml b/.github/workflows/postgresql-16-ppg-package-pgxs.yml new file mode 100644 index 00000000..2f437ff7 --- /dev/null +++ b/.github/workflows/postgresql-16-ppg-package-pgxs.yml @@ -0,0 +1,159 @@ +name: postgresql-16-ppg-package-pgxs +on: + pull_request: + workflow_dispatch: + push: + branches: [main] + +jobs: + build: + name: pg-16-ppg-package-pgxs-build + runs-on: ubuntu-22.04 + steps: + - name: Remove old postgres + run: | + sudo apt-get update + sudo apt purge postgresql-client-common postgresql-common \ + postgresql postgresql* + sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \ + /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \ + /etc/postgresql + sudo rm -f /usr/bin/pg_config + + - name: Install dependencies + run: | + sudo apt-get install -y libreadline6-dev systemtap-sdt-dev wget \ + zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl \ + libcurl4-openssl-dev libhttp-server-simple-perl + sudo /usr/bin/perl -MCPAN -e 'install IPC::Run' + sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' + wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo apt update && sudo apt install -y vault + + - name: Install percona-release script + run: | + sudo apt-get -y update + sudo apt-get -y upgrade + sudo apt-get install -y wget gnupg2 curl lsb-release + sudo wget \ + https://repo.percona.com/apt/percona-release_latest.generic_all.deb + sudo dpkg -i percona-release_latest.generic_all.deb + + - name: Install Percona Distribution Postgresql 16 & Extensions + run: | + sudo percona-release setup ppg-16 + sudo apt-get update -y + sudo apt-get install -y percona-postgresql-16 \ + percona-postgresql-contrib percona-postgresql-server-dev-all \ + percona-pgpool2 libpgpool2 percona-postgresql-16-pgaudit \ + percona-postgresql-16-pgaudit-dbgsym percona-postgresql-16-repack \ + percona-postgresql-16-repack-dbgsym percona-pgaudit16-set-user \ + percona-pgaudit16-set-user-dbgsym percona-postgresql-16-postgis-3 \ + percona-postgresql-16-postgis-3-scripts \ + percona-postgresql-postgis-scripts percona-postgresql-postgis \ + percona-postgis percona-pg-stat-monitor16 + + - name: Clone pg_tde repository + uses: actions/checkout@master + with: + path: 'src/pg_tde' + + - name: Change src owner to postgres + run: | + sudo chmod o+rx ~ + sudo chown -R postgres:postgres src + + - name: Build pg_tde + run: | + sudo -u postgres bash -c './configure' + sudo -u postgres bash -c 'make USE_PGXS=1' + sudo make USE_PGXS=1 install + working-directory: src/pg_tde + + - name: Start pg_tde tests + run: | + TV=$(mktemp) + { exec >$TV; vault server -dev; } & + sleep 10 + export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n) + echo "Root token: $ROOT_TOKEN" + + sudo psql -V + + sudo service postgresql stop + echo "shared_preload_libraries = 'pg_tde'" | + sudo tee -a /etc/postgresql/16/main/postgresql.conf + sudo service postgresql start + + sudo -u postgres bash -c "export ROOT_TOKEN=$ROOT_TOKEN && make installcheck USE_PGXS=1" + working-directory: src/pg_tde + + - name: Report on test fail + uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: Regressions diff and postgresql log + path: | + src/pg_tde/regression.diffs + src/pg_tde/logfile + retention-days: 3 + + - name: Report on test fail 2 + if: ${{ failure() }} + run: | + cat src/pg_tde/regression.diffs + + - name: Create release directory + run: | + sudo mkdir pgtde-ppg16 + sudo mkdir -p pgtde-ppg16/usr/lib/postgresql/16/lib/ + sudo mkdir -p pgtde-ppg16/share/postgresql/16/extension/ + sudo cp /usr/share/postgresql/16/extension/pg_tde* pgtde-ppg16/share/postgresql/16/extension/ + sudo cp /usr/lib/postgresql/16/lib/pg_tde* pgtde-ppg16/usr/lib/postgresql/16/lib/ + + - name: Upload tgz + uses: actions/upload-artifact@v4 + with: + name: pg_tde_ppg16_binary + path: pgtde-ppg16 + + - name: Create deb + run: | + sudo mkdir pgtde-ppg16/DEBIAN + sudo sh -c 'echo "Package: pgtde-ppg16" > pgtde-ppg16/DEBIAN/control' + sudo sh -c 'echo "Version: 0.1" >> pgtde-ppg16/DEBIAN/control' + sudo sh -c 'echo "Architecture: amd64" >> pgtde-ppg16/DEBIAN/control' + sudo sh -c 'echo "Maintainer: Percona" >> pgtde-ppg16/DEBIAN/control' + sudo sh -c 'echo "Description: Experimental pg_tde extension" >> pgtde-ppg16/DEBIAN/control' + sudo dpkg-deb --build --root-owner-group pgtde-ppg16 + + - name: Test deb + run: | + sudo rm -rf /usr/share/postgresql/16/extension/pg_tde* + sudo rm -rf /usr/lib/postgresql/16/lib/pg_tde* + sudo dpkg -i --debug=7777 pgtde-ppg16.deb + + - name: Upload deb + uses: actions/upload-artifact@v4 + with: + name: pg_tde_deb + path: pgtde-ppg16.deb + + - name: Create tgz + run: | + cd pgtde-ppg16 && sudo tar -czvf ../pgtde-ppg16.tar.gz . + + - name: Publish release + uses: ncipollo/release-action@v1 + # Only try and deploy on merged code + if: "github.repository == 'Percona-Lab/pg_tde' && github.ref_name == 'main' && (github.event_name == 'push' || github.event_name == 'schedule')" + with: + artifacts: "pgtde-ppg16.tar.gz,pgtde-ppg16.deb" + omitBody: true + allowUpdates: true + generateReleaseNotes: true + makeLatest: true + tag: "latest" + name: "HEAD" + replacesArtifacts: true diff --git a/t/004_file_config.pl b/t/004_file_config.pl index 64993622..360c84cb 100644 --- a/t/004_file_config.pl +++ b/t/004_file_config.pl @@ -37,7 +37,7 @@ $rt_value = $node->psql('postgres', "SELECT pg_tde_add_key_provider_file('file-provider', json_object( 'type' VALUE 'file', 'path' VALUE '/tmp/datafile-location' ));", extra_params => ['-a']); $rt_value = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','file-provider');", extra_params => ['-a']); -my $stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde_basic;', extra_params => ['-a']); +$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde_basic;', extra_params => ['-a']); PGTDE::append_to_file($stdout); $stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc1 (k) VALUES (5),(6);', extra_params => ['-a']); @@ -59,7 +59,7 @@ # DROP EXTENSION $stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']); -ok(my $cmdret == 0, "DROP PGTDE EXTENSION"); +ok($cmdret == 0, "DROP PGTDE EXTENSION"); PGTDE::append_to_file($stdout); # Stop the server $node->stop(); diff --git a/t/005_multiple_extensions.pl b/t/005_multiple_extensions.pl new file mode 100644 index 00000000..98088cfb --- /dev/null +++ b/t/005_multiple_extensions.pl @@ -0,0 +1,158 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use File::Basename; +use File::Compare; +use File::Copy; +use Test::More; +use lib 't'; +use pgtde; + +# Get file name and CREATE out file name and dirs WHERE requried +PGTDE::setup_files_dir(basename($0)); + +my $PG_VERSION_STRING = `pg_config --version`; + +if (index(lc($PG_VERSION_STRING), lc("percona")) == -1) +{ + plan skip_all => "pg_tde test case only for PPG server package install with extensions."; +} + +# CREATE new PostgreSQL node and do initdb +my $node = PGTDE->pgtde_init_pg(); +my $pgdata = $node->data_dir; + +copy("$pgdata/postgresql.conf", "$pgdata/postgresql.conf.bak"); + +# UPDATE postgresql.conf to include/load pg_stat_monitor library +open my $conf, '>>', "$pgdata/postgresql.conf"; +print $conf "shared_preload_libraries = 'pg_tde, pg_stat_monitor, pgaudit, set_user, pg_repack'\n"; +print $conf "pg_stat_monitor.pgsm_bucket_time = 360000\n"; +print $conf "pg_stat_monitor.pgsm_normalized_query = 'yes'\n"; +close $conf; + +open my $conf2, '>>', "/tmp/datafile-location"; +print $conf2 "/tmp/keyring_data_file\n"; +close $conf2; + +# Start server +my $rt_value = $node->start; +ok($rt_value == 1, "Start Server"); + +# Create PGSM extension +my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION pg_stat_monitor;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE PGSM EXTENSION"); +PGTDE::append_to_debug_file($stdout); + +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT pg_stat_monitor_reset();', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']); +ok($cmdret == 0, "Reset PGSM EXTENSION"); +PGTDE::append_to_debug_file($stdout); + +# Create pg_tde extension +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION pg_tde;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE PGTDE EXTENSION"); +PGTDE::append_to_file($stdout); + +# Create Other extensions +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pgaudit;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE pgaudit EXTENSION"); +PGTDE::append_to_debug_file($stdout); +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS set_user;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE set_user EXTENSION"); +PGTDE::append_to_debug_file($stdout); +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS pg_repack;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE pg_repack EXTENSION"); +PGTDE::append_to_debug_file($stdout); +($cmdret, $stdout, $stderr) = $node->psql('postgres', "SET pgaudit.log = 'none'; CREATE EXTENSION IF NOT EXISTS postgis; SET pgaudit.log = 'all';", extra_params => ['-a']); +ok($cmdret == 0, "CREATE postgis EXTENSION"); +PGTDE::append_to_debug_file($stdout); +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS postgis_raster;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE postgis_raster EXTENSION"); +PGTDE::append_to_debug_file($stdout); +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS postgis_sfcgal;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE postgis_sfcgal EXTENSION"); +PGTDE::append_to_debug_file($stdout); +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE fuzzystrmatch EXTENSION"); +PGTDE::append_to_debug_file($stdout); +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS address_standardizer;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE address_standardizer EXTENSION"); +PGTDE::append_to_debug_file($stdout); +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS address_standardizer_data_us;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE address_standardizer_data_us EXTENSION"); +PGTDE::append_to_debug_file($stdout); +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;', extra_params => ['-a']); +ok($cmdret == 0, "CREATE postgis_tiger_geocoder EXTENSION"); +PGTDE::append_to_debug_file($stdout); + +$rt_value = $node->psql('postgres', "SELECT pg_tde_add_key_provider_file('file-provider', json_object( 'type' VALUE 'file', 'path' VALUE '/tmp/datafile-location' ));", extra_params => ['-a']); +$rt_value = $node->psql('postgres', "SELECT pg_tde_set_principal_key('test-db-principal-key','file-provider');", extra_params => ['-a']); + +$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde_basic;', extra_params => ['-a']); +PGTDE::append_to_file($stdout); + +$stdout = $node->safe_psql('postgres', 'INSERT INTO test_enc1 (k) VALUES (5),(6);', extra_params => ['-a']); +PGTDE::append_to_file($stdout); + +$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;', extra_params => ['-a']); +PGTDE::append_to_file($stdout); + +# Restart the server +PGTDE::append_to_file("-- server restart"); +$rt_value = $node->stop(); +$rt_value = $node->start(); + +$stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc1 ORDER BY id ASC;', extra_params => ['-a']); +PGTDE::append_to_file($stdout); + +$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc1;', extra_params => ['-a']); +PGTDE::append_to_file($stdout); + +# Print PGSM settings +($cmdret, $stdout, $stderr) = $node->psql('postgres', "SELECT name, setting, unit, context, vartype, source, min_val, max_val, enumvals, boot_val, reset_val, pending_restart FROM pg_settings WHERE name='pg_stat_monitor.pgsm_query_shared_buffer';", extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']); +ok($cmdret == 0, "Print PGTDE EXTENSION Settings"); +PGTDE::append_to_debug_file($stdout); + +# Create example database and run pgbench init +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE database example;', extra_params => ['-a']); +print "cmdret $cmdret\n"; +ok($cmdret == 0, "CREATE Database example"); +PGTDE::append_to_debug_file($stdout); + +my $port = $node->port; +print "port $port \n"; + +my $out = system ("pgbench -i -s 20 -p $port example"); +print " out: $out \n"; +ok($cmdret == 0, "Perform pgbench init"); + +$out = system ("pgbench -c 10 -j 2 -t 5000 -p $port example"); +print " out: $out \n"; +ok($cmdret == 0, "Run pgbench"); + +($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT datname, substr(query,0,150) AS query, SUM(calls) AS calls FROM pg_stat_monitor GROUP BY datname, query ORDER BY datname, query, calls;', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']); +ok($cmdret == 0, "SELECT XXX FROM pg_stat_monitor"); +PGTDE::append_to_debug_file($stdout); + +# DROP EXTENSION +$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']); +ok($cmdret == 0, "DROP PGTDE EXTENSION"); +PGTDE::append_to_file($stdout); + +# DROP EXTENSION +$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_stat_monitor;', extra_params => ['-a']); +ok($cmdret == 0, "DROP PGTDE EXTENSION"); +PGTDE::append_to_debug_file($stdout); + +# Stop the server +$node->stop(); + +# compare the expected and out file +my $compare = PGTDE->compare_results(); + +# Test/check if expected and result/out file match. If Yes, test passes. +is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files."); + +# Done testing for this testcase file. +done_testing(); diff --git a/t/expected/005_multiple_extensions.out b/t/expected/005_multiple_extensions.out new file mode 100644 index 00000000..866de8a7 --- /dev/null +++ b/t/expected/005_multiple_extensions.out @@ -0,0 +1,12 @@ +CREATE EXTENSION pg_tde; +CREATE TABLE test_enc1(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde_basic; +INSERT INTO test_enc1 (k) VALUES (5),(6); +SELECT * FROM test_enc1 ORDER BY id ASC; +1|5 +2|6 +-- server restart +SELECT * FROM test_enc1 ORDER BY id ASC; +1|5 +2|6 +DROP TABLE test_enc1; +DROP EXTENSION pg_tde;