Encrypt fork files with master key and improve tests

Fork file encryption was missing from the previous pull requests,
resulting in the server not initializing the keyring on normal executions,
and because of this, missing if the keyring configuration wasn't specified
at all.

Fixes #46

.github/workflows/postgresql-16-pgdg-package-pgxs.yml

@@ -54,11 +54,14 @@ jobs:
           sudo make USE_PGXS=1 install
         working-directory: src/postgres-tde-ext
 
-      - name: Start pg_stat_monitor_tests
+      - name: Start postgres-tde-ext tests
         run: |
           sudo service postgresql stop
           echo "shared_preload_libraries = 'pg_tde'" |
             sudo tee -a /etc/postgresql/16/main/postgresql.conf
+          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" |
+            sudo tee -a /etc/postgresql/16/main/postgresql.conf
+          cp keyring.json /tmp/keyring.json
           sudo service postgresql start
           sudo psql -V
           sudo -u postgres bash -c 'make installcheck USE_PGXS=1'
@@ -74,6 +77,11 @@ jobs:
             src/postgres-tde-ext/logfile
           retention-days: 3
 
+      - name: Report on test fail 2
+        if: ${{ failure() }}
+        run: |
+          cat src/postgres-tde-ext/regression.diffs
+
       - name: Create release directory
         run: |
           sudo mkdir pgtde-pgdg16

.github/workflows/postgresql-16-src-make.yml

@@ -65,9 +65,12 @@ jobs:
           initdb -D /opt/pgsql/data
           echo "shared_preload_libraries = 'pg_tde'" >> \
             /opt/pgsql/data/postgresql.conf
+          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" >> \
+            /opt/pgsql/data/postgresql.conf
+          cp src/contrib/postgres-tde-ext/keyring.json /tmp/keyring.json
           pg_ctl -D /opt/pgsql/data -l logfile start
 
-      - name: Build postgres-tde-ext
+      - name: Test postgres-tde-ext
         run: |
           make installcheck
         working-directory: src/contrib/postgres-tde-ext
@@ -78,6 +81,11 @@ jobs:
         with:
           name: Regressions diff and postgresql log
           path: |
-            src/postgres-tde-ext/regression.diffs
-            src/postgres-tde-ext/logfile
+            src/contrib/postgres-tde-ext/regression.diffs
+            src/contrib/postgres-tde-ext/logfile
           retention-days: 3
+
+      - name: Report on test fail 2
+        if: ${{ failure() }}
+        run: |
+          cat src/contrib/postgres-tde-ext/regression.diffs

.github/workflows/postgresql-16-src-meson.yml

@@ -48,21 +48,14 @@ jobs:
 
       - name: Build postgres
         run: |
-          meson setup build
-          cd build && ninja && sudo ninja install
+          meson setup build --prefix `pwd`/../inst
+          cd build && ninja && ninja install
         working-directory: src
 
-      - name: Start postgresql cluster with pg_tde
-        run: |
-          export PATH="/usr/local/pgsql/bin:$PATH"
-          sudo cp /usr/local/pgsql/bin/pg_config /usr/bin
-          initdb -D /opt/pgsql/data
-          echo "shared_preload_libraries = 'pg_tde'" >> /opt/pgsql/data/postgresql.conf
-          pg_ctl -D /opt/pgsql/data -l logfile start
-
       - name: Test postgres-tde-ext
         run: |
-          export PATH="/usr/local/pgsql/bin:$PATH"
+          cp ../contrib/postgres-tde-ext/keyring.json /tmp/keyring.json
+          meson test --suite setup -v
           meson test --suite postgres-tde-ext -v
         working-directory: src/build
 

keyring.json

@@ -0,0 +1,4 @@
+{
+        'provider': 'file',
+        'datafile': '/tmp/pgkeyring',
+}

meson.build

@@ -72,4 +72,9 @@ tests += {
     'regress_args': ['--temp-config', files('postgres-tde-ext.conf')],
     'runningcheck': false,
   },
+  'tap': {
+    'tests': [
+      't/001_basic.pl',
+    ],
+  },
 }

postgres-tde-ext.conf

@@ -1 +1,2 @@
 shared_preload_libraries = 'pg_tde'
+pg_tde.keyringConfigFile = '/tmp/keyring.json'

regression/t/001_basic.pl

@@ -0,0 +1,69 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use File::Basename;
+use File::Compare;
+use File::Copy;
+use Test::More;
+use lib 't';
+use pgtde;
+
+# Get file name and CREATE out file name and dirs WHERE requried
+PGTDE::setup_files_dir(basename($0));
+
+# CREATE new PostgreSQL node and do initdb
+my $node = PGTDE->pgtde_init_pg();
+my $pgdata = $node->data_dir;
+
+# UPDATE postgresql.conf to include/load pg_tde library
+open my $conf, '>>', "$pgdata/postgresql.conf";
+print $conf "shared_preload_libraries = 'pg_tde'\n";
+close $conf;
+
+# Start server
+my $rt_value = $node->start;
+ok($rt_value == 1, "Start Server");
+
+# CREATE EXTENSION and change out file permissions
+my ($cmdret, $stdout, $stderr) = $node->psql('postgres', 'CREATE EXTENSION pg_tde;', extra_params => ['-a']);
+ok($cmdret == 0, "CREATE PGTDE EXTENSION");
+PGTDE::append_to_file($stdout);
+
+
+$rt_value = $node->psql('postgres', 'CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde;', extra_params => ['-a']);
+ok($rt_value == 3, "Failing query");
+
+
+# Restart the server
+$node->stop;
+
+# UPDATE postgresql.conf to include/load pg_tde library
+open my $conf, '>>', "$pgdata/postgresql.conf";
+print $conf "pg_tde.keyringConfigFile = '/tmp/keyring.json'\n";
+close $conf;
+
+my $rt_value = $node->start;
+ok($rt_value == 1, "Restart Server");
+
+$stdout = $node->safe_psql('postgres', 'CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde;', extra_params => ['-a']);
+PGTDE::append_to_file($stdout);
+
+$stdout = $node->safe_psql('postgres', 'DROP TABLE test_enc;', extra_params => ['-a']);
+PGTDE::append_to_file($stdout);
+
+# DROP EXTENSION
+$stdout = $node->safe_psql('postgres', 'DROP EXTENSION pg_tde;', extra_params => ['-a']);
+ok($cmdret == 0, "DROP PGTDE EXTENSION");
+PGTDE::append_to_file($stdout);
+# Stop the server
+$node->stop;
+
+# compare the expected and out file
+my $compare = PGTDE->compare_results();
+
+# Test/check if expected and result/out file match. If Yes, test passes.
+is($compare,0,"Compare Files: $PGTDE::expected_filename_with_path and $PGTDE::out_filename_with_path files.");
+
+# Done testing for this testcase file.
+done_testing();

regression/t/expected/001_basic.out

@@ -0,0 +1,4 @@
+CREATE EXTENSION pg_tde;
+CREATE TABLE test_enc(id SERIAL,k INTEGER,PRIMARY KEY (id)) USING pg_tde;
+DROP TABLE test_enc;
+DROP EXTENSION pg_tde;

regression/t/pgtde.pm

@@ -0,0 +1,146 @@
+package PGTDE;
+
+use File::Basename;
+use File::Compare;
+use Test::More;
+
+our @ISA= qw( Exporter );
+
+# These CAN be exported.
+our @EXPORT = qw( pgtde_init_pg pgtde_start_pg pgtde_stop_pg pgtde_psql_cmd pgtde_setup_pg_tde pgtde_create_extension pgtde_drop_extension );
+
+# Instance of pg server that would be spanwed by TAP testing. A new server will be created for each TAP test.
+our $pg_node;
+
+# Expected .out filename of TAP testcase being executed. These are already part of repo under t/expected/*.
+our $expected_filename_with_path;
+
+# Major version of PG Server that we are using.
+our $PG_MAJOR_VERSION;
+
+# Result .out filename of TAP testcase being executed. Where needed, a new *.out will be created for each TAP test.
+our $out_filename_with_path;
+
+# Runtime output file that is used only for debugging purposes for comparison to PGSS, blocks and timings.
+our $debug_out_filename_with_path;
+
+BEGIN {
+    # Get PG Server Major version from pg_config 
+    $PG_MAJOR_VERSION = `pg_config --version | awk {'print \$2'} | cut -f1 -d"." | sed -e 's/[^0-9].*\$//g'`;
+    $PG_MAJOR_VERSION =~ s/^\s+|\s+$//g;
+
+    # Depending upon PG server version load the required module at runtime when pgtde.pm is loaded.
+    my $node_module = $PG_MAJOR_VERSION > 14 ? "PostgreSQL::Test::Cluster" : "PostgresNode";
+    my $node_module_file = $node_module;
+    $node_module_file =~ s[::][/]g;
+    $node_module_file .= '.pm';
+    require $node_module_file;
+    $node_module->import;
+}
+
+sub pgtde_init_pg
+{
+    print "Postgres major version: $PG_MAJOR_VERSION \n";
+
+    # For Server version 15 & above, spawn the server using PostgreSQL::Test::Cluster
+    if ($PG_MAJOR_VERSION > 14) {
+        $pg_node = PostgreSQL::Test::Cluster->new('pgtde_regression');
+    }
+    # For Server version 14 & below, spawn the server using PostgresNode
+    elsif ($PG_MAJOR_VERSION < 15) {
+        $pg_node = PostgresNode->get_new_node('pgtde_regression');
+    }
+
+    $pg_node->dump_info;
+    $pg_node->init;
+    return $pg_node;
+}
+
+sub append_to_file
+{
+    my ($str) = @_;
+
+    # For Server version 15 & above, use PostgreSQL::Test::Utils to write to files
+    if ($PG_MAJOR_VERSION > 14) {
+        PostgreSQL::Test::Utils::append_to_file($out_filename_with_path, $str . "\n");
+    }
+    # For Server version 14 & below, use PostgresNode to write to files
+    elsif ($PG_MAJOR_VERSION < 15) {
+        TestLib::append_to_file($out_filename_with_path, $str . "\n");
+    }
+    chmod(0640 , $out_filename_with_path)
+    or die("unable to set permissions for $out_filename_with_path");
+
+    return;
+}
+
+sub append_to_debug_file
+{
+    my ($str) = @_;
+
+    # For Server version 15 & above, use PostgreSQL::Test::Utils to write to files
+    if ($PG_MAJOR_VERSION > 14) {
+        PostgreSQL::Test::Utils::append_to_file($debug_out_filename_with_path, $str . "\n");
+    }
+    # For Server version 14 & below, use PostgresNode to write to files
+    elsif ($PG_MAJOR_VERSION < 15) {
+        TestLib::append_to_file($debug_out_filename_with_path, $str . "\n");
+    }
+    chmod(0640 , $debug_out_filename_with_path)
+    or die("unable to set permissions for $debug_out_filename_with_path");
+
+    return;
+}
+
+sub setup_files_dir
+{
+    my ($perlfilename) = @_;
+
+    # Expected folder where expected output will be present
+    my $expected_folder = "t/expected";
+
+    # Results/out folder where generated results files will be placed
+    my $results_folder = "t/results";
+
+    # Check if results folder exists or not, create if it doesn't
+    unless (-d $results_folder)
+    {
+        mkdir $results_folder or die "Can't create folder $results_folder: $!\n";
+    }
+
+    # Check if expected folder exists or not, bail out if it doesn't
+    unless (-d $expected_folder)
+    {
+        BAIL_OUT "Expected files folder $expected_folder doesn't exist: \n";
+    }
+
+    #Remove .pl from filename and store in a variable
+    my @split_arr = split /\./, $perlfilename;
+
+    my $filename_without_extension = $split_arr[0];
+
+    # Create expected filename with path
+    my $expected_filename = "${filename_without_extension}.out";
+
+    $expected_filename_with_path = "${expected_folder}/${expected_filename}";
+
+    # Create results filename with path
+    my $out_filename = "${filename_without_extension}.out";
+    $out_filename_with_path = "${results_folder}/${out_filename}";
+
+    # Delete already existing result out file, if it exists.
+    if ( -f $out_filename_with_path)
+    {
+        unlink($out_filename_with_path) or die "Can't delete already existing $out_filename_with_path: $!\n";
+    }
+
+    $debug_out_filename_with_path = "${results_folder}/${out_filename}.debug";
+}
+
+sub compare_results
+{
+    # Compare expected and results files and return the result
+    return compare($expected_filename_with_path, $out_filename_with_path);
+}
+
+1;