Updating CI scripts after recent changes

This commit removes outdated keyring configuration code, and adds vault support to all runners.
percona · Feb 28, 2024 · 60a9b58 · 60a9b58
1 parent 210c95c
commit 60a9b58
Show file tree

Hide file tree

Showing 10 changed files with 48 additions and 53 deletions.
diff --git a/.github/workflows/postgresql-16-pgdg-package-pgxs.yml b/.github/workflows/postgresql-16-pgdg-package-pgxs.yml
@@ -27,6 +27,10 @@ jobs:
             libjson-c-dev libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
+
 
       - name: Install PG Distribution Postgresql 16
         run: |
@@ -57,14 +61,14 @@ jobs:
 
       - name: Start pg_tde tests
         run: |
-          sudo service postgresql stop
-          echo "shared_preload_libraries = 'pg_tde'" |
-            sudo tee -a /etc/postgresql/16/main/postgresql.conf
-          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" |
-            sudo tee -a /etc/postgresql/16/main/postgresql.conf
-          cp keyring.json /tmp/keyring.json
-          sudo service postgresql start
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
+
           sudo psql -V
+          
           sudo -u postgres bash -c 'make installcheck USE_PGXS=1'
         working-directory: src/pg_tde
 

diff --git a/.github/workflows/postgresql-16-src-make-ssl11.yml b/.github/workflows/postgresql-16-src-make-ssl11.yml
@@ -29,6 +29,9 @@ jobs:
             uuid-dev liblz4-dev libjson-c-dev libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
 
       - name: Clone postgres repository
         uses: actions/checkout@v2
@@ -61,14 +64,15 @@ jobs:
 
       - name: Start postgresql cluster with pg_tde
         run: |
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
+
           export PATH="/usr/local/pgsql/bin:$PATH"
           sudo cp /usr/local/pgsql/bin/pg_config /usr/bin
           initdb -D /opt/pgsql/data
-          echo "shared_preload_libraries = 'pg_tde'" >> \
-            /opt/pgsql/data/postgresql.conf
-          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" >> \
-            /opt/pgsql/data/postgresql.conf
-          cp src/contrib/pg_tde/keyring.json /tmp/keyring.json
           pg_ctl -D /opt/pgsql/data -l logfile start
 
       - name: Test pg_tde

diff --git a/.github/workflows/postgresql-16-src-make.yml b/.github/workflows/postgresql-16-src-make.yml
@@ -29,6 +29,9 @@ jobs:
             uuid-dev liblz4-dev libjson-c-dev libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
 
       - name: Clone postgres repository
         uses: actions/checkout@v2
@@ -61,14 +64,15 @@ jobs:
 
       - name: Start postgresql cluster with pg_tde
         run: |
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
+
           export PATH="/usr/local/pgsql/bin:$PATH"
           sudo cp /usr/local/pgsql/bin/pg_config /usr/bin
           initdb -D /opt/pgsql/data
-          echo "shared_preload_libraries = 'pg_tde'" >> \
-            /opt/pgsql/data/postgresql.conf
-          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" >> \
-            /opt/pgsql/data/postgresql.conf
-          cp src/contrib/pg_tde/keyring.json /tmp/keyring.json
           pg_ctl -D /opt/pgsql/data -l logfile start
 
       - name: Test pg_tde

diff --git a/.github/workflows/postgresql-16-src-meson-perf.yml b/.github/workflows/postgresql-16-src-meson-perf.yml
@@ -33,6 +33,10 @@ jobs:
             sysbench libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
+
 
       - name: Clone postgres repository
         uses: actions/checkout@v2
@@ -58,7 +62,12 @@ jobs:
 
       - name: Test pg_tde
         run: |
-          cp ../contrib/pg_tde/keyring.json /tmp/keyring.json
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
+          
           meson test --suite setup -v
           meson test --suite pg_tde -v --num-processes 1
         working-directory: src/build

diff --git a/.github/workflows/postgresql-16-src-meson.yml b/.github/workflows/postgresql-16-src-meson.yml
@@ -56,32 +56,14 @@ jobs:
           cd build && ninja && ninja install
         working-directory: src
 
-      - name: Test pg_tde with keyring_file
-        run: |
-          cp ../contrib/pg_tde/keyring.json /tmp/keyring.json
-          meson test --suite setup -v
-          meson test --suite pg_tde -v --num-processes 1
-        working-directory: src/build
-
-      - name: Report on test fail
-        uses: actions/upload-artifact@v2
-        if: ${{ failure() }}
-        with:
-          name: Regressions diff and postgresql log
-          path: |
-            src/build/testrun/pg_tde/regress/
-          retention-days: 3
-
-      - name: Test pg_tde with keyring_vault
+      - name: Test pg_tde
         run: |
           TV=$(mktemp)
           { exec >$TV; vault server -dev; } &
           sleep 10
-          ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
           echo "Root token: $ROOT_TOKEN"
-          cp ../contrib/pg_tde/keyring-vault.json /tmp/keyring.json
-          sed -i "s/ROOT_TOKEN/$ROOT_TOKEN/g" /tmp/keyring.json
-          cat /tmp/keyring.json
+
           meson test --suite setup -v
           meson test --suite pg_tde -v --num-processes 1
         working-directory: src/build
@@ -93,5 +75,4 @@ jobs:
           name: Regressions diff and postgresql log
           path: |
             src/build/testrun/pg_tde/regress/
-          retention-days: 3
-
+          retention-days: 3
diff --git a/Makefile.in b/Makefile.in
@@ -13,7 +13,8 @@ non_sorted_off_compact \
 update_compare_indexes \
 pgtde_is_encrypted \
 multi_insert \
-trigger_on_view
+trigger_on_view \
+vault_v2_test
 TAP_TESTS = 1
 
 OBJS = src/encryption/enc_tde.o \

diff --git a/keyring-vault.json b/keyring-vault.json
diff --git a/keyring.json b/keyring.json
diff --git a/meson.build b/meson.build
@@ -74,6 +74,7 @@ tests += {
       'pgtde_is_encrypted',
       'multi_insert',
       'trigger_on_view',
+      'vault_v2_test',
     ],
     'regress_args': ['--temp-config', files('pg_tde.conf')],
     'runningcheck': false,

diff --git a/sql/vault_v2_test.sql b/sql/vault_v2_test.sql
@@ -1,6 +1,7 @@
 CREATE EXTENSION pg_tde;
 
-SELECT pg_tde_add_key_provider_vault_v2('vault-v2','ROOT_TOKEN','http://127.0.0.1:8200','secret',NULL);
+\getenv root_token ROOT_TOKEN
+SELECT pg_tde_add_key_provider_vault_v2('vault-v2',:'root_token','http://127.0.0.1:8200','secret',NULL);
 SELECT pg_tde_set_master_key('vault-v2-master-key','vault-v2');
 
 CREATE TABLE test_enc(