Basic KMIP keyring implementation (#325)

Also includes some refactoring because libkmip and postgres headers are not compatible. To avoid compilation errors, keyring_kmip.c does not include postgres headers, and keyring_kmip_ereport.c does not include libkmip headers.
percona · Nov 28, 2024 · 69e82d2 · 69e82d2
1 parent cb3b3c8
commit 69e82d2
Show file tree

Hide file tree

Showing 32 changed files with 812 additions and 244 deletions.
diff --git a/.github/workflows/postgresql-16-ppg-package-pgxs.yml b/.github/workflows/postgresql-16-ppg-package-pgxs.yml
@@ -29,7 +29,7 @@ jobs:
         run: |
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev wget \
             zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl \
-            libcurl4-openssl-dev  libhttp-server-simple-perl
+            libcurl4-openssl-dev  libhttp-server-simple-perl python3-pykmip
           sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
           wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
@@ -64,6 +64,11 @@ jobs:
         with:
           path: 'src/pg_tde'
 
+      - name: Checkout libkmip submodule
+        run: |
+          git submodule update --init --recursive
+        working-directory: src/pg_tde
+
       - name: Change src owner to postgres
         run: |
           sudo chmod o+rx ~
@@ -76,10 +81,18 @@ jobs:
           sudo make USE_PGXS=1 install
         working-directory: src/pg_tde
 
+      - name: Setup kmip 1
+        run: |
+          wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py
+          python3 create_certificates.py
+          cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem
+        working-directory: /tmp
+
       - name: Start pg_tde tests
         run: |
           TV=$(mktemp)
           { exec >$TV; vault server -dev; } &
+          pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log &
           sleep 10
           export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
           echo "Root token: $ROOT_TOKEN"

diff --git a/.github/workflows/postgresql-16-src-make-macos.yml b/.github/workflows/postgresql-16-src-make-macos.yml
diff --git a/.github/workflows/postgresql-16-src-make-ssl11.yml b/.github/workflows/postgresql-16-src-make-ssl11.yml
@@ -29,7 +29,7 @@ jobs:
             libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config \
             libselinux1-dev python3-dev libhttp-server-simple-perl\
-            uuid-dev liblz4-dev libcurl4-openssl-dev
+            uuid-dev liblz4-dev libcurl4-openssl-dev python3-pykmip
           sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
           wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
@@ -47,6 +47,11 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: 'src/contrib/pg_tde'
+
+      - name: Checkout libkmip submodule
+        run: |
+          git submodule update --init --recursive
+        working-directory: src/contrib/pg_tde
 
       - name: Create pgsql dir
         run: mkdir -p /opt/pgsql
@@ -74,10 +79,18 @@ jobs:
             /opt/pgsql/data/postgresql.conf
           pg_ctl -D /opt/pgsql/data -l logfile start
 
+      - name: Setup kmip 1
+        run: |
+          wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py
+          python3 create_certificates.py
+          cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem
+        working-directory: /tmp
+
       - name: Test pg_tde
         run: |
           TV=$(mktemp)
           { exec >$TV; vault server -dev; } &
+          pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log &
           sleep 10
           export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
           echo "Root token: $ROOT_TOKEN"

diff --git a/.github/workflows/postgresql-16-src-make.yml b/.github/workflows/postgresql-16-src-make.yml
@@ -29,7 +29,7 @@ jobs:
             libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
             llvm-11 llvm-11-dev libselinux1-dev python3-dev \
-            uuid-dev liblz4-dev libcurl4-openssl-dev libhttp-server-simple-perl
+            uuid-dev liblz4-dev libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip
           sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
           wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
@@ -47,6 +47,11 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: 'src/contrib/pg_tde'
+
+      - name: Checkout libkmip submodule
+        run: |
+          git submodule update --init --recursive
+        working-directory: src/contrib/pg_tde
 
       - name: Create pgsql dir
         run: mkdir -p /opt/pgsql
@@ -73,11 +78,19 @@ jobs:
           echo "shared_preload_libraries = 'pg_tde'" >> \
             /opt/pgsql/data/postgresql.conf
           pg_ctl -D /opt/pgsql/data -l logfile start
+      
+      - name: Setup kmip 1
+        run: |
+          wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py
+          python3 create_certificates.py
+          cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem
+        working-directory: /tmp
 
       - name: Test pg_tde
         run: |
           TV=$(mktemp)
           { exec >$TV; vault server -dev; } &
+          pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log &
           sleep 10
           export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
           echo "Root token: $ROOT_TOKEN"

diff --git a/.github/workflows/postgresql-16-src-meson-macos.yml b/.github/workflows/postgresql-16-src-meson-macos.yml
diff --git a/.github/workflows/postgresql-16-src-meson.yml b/.github/workflows/postgresql-16-src-meson.yml
@@ -30,7 +30,7 @@ jobs:
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
             llvm-11 llvm-11-dev libselinux1-dev python3-dev \
             uuid-dev liblz4-dev meson ninja-build  \
-            gpg wget libcurl4-openssl-dev libhttp-server-simple-perl
+            gpg wget libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip
           sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
           wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
@@ -48,6 +48,11 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: 'src/contrib/pg_tde'
+
+      - name: Checkout libkmip submodule
+        run: |
+          git submodule update --init --recursive
+        working-directory: src/contrib/pg_tde
 
       - name: Include pg_tde in meson build
         run: |
@@ -58,11 +63,19 @@ jobs:
           meson setup build --prefix `pwd`/../inst --buildtype=debug -Dcassert=true
           cd build && ninja && ninja install
         working-directory: src
+
+      - name: Setup kmip 1
+        run: |
+          wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py
+          python3 create_certificates.py
+          cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem
+        working-directory: /tmp
 
       - name: Test pg_tde
         run: |
           TV=$(mktemp)
           { exec >$TV; vault server -dev; } &
+          pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log &
           sleep 10
           export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
           echo "Root token: $ROOT_TOKEN"

diff --git a/.github/workflows/postgresql-17-src-make.yml b/.github/workflows/postgresql-17-src-make.yml
@@ -29,7 +29,7 @@ jobs:
             libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
             llvm-11 llvm-11-dev libselinux1-dev python3-dev \
-            uuid-dev liblz4-dev libcurl4-openssl-dev libhttp-server-simple-perl
+            uuid-dev liblz4-dev libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip
           sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
           wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
@@ -47,6 +47,11 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: 'src/contrib/pg_tde'
+
+      - name: Checkout libkmip submodule
+        run: |
+          git submodule update --init --recursive
+        working-directory: src/contrib/pg_tde
 
       - name: Create pgsql dir
         run: mkdir -p /opt/pgsql
@@ -74,10 +79,18 @@ jobs:
             /opt/pgsql/data/postgresql.conf
           pg_ctl -D /opt/pgsql/data -l logfile start
 
+      - name: Setup kmip 1
+        run: |
+          wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py
+          python3 create_certificates.py
+          cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem
+        working-directory: /tmp
+
       - name: Test pg_tde
         run: |
           TV=$(mktemp)
           { exec >$TV; vault server -dev; } &
+          pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log &
           sleep 10
           export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
           echo "Root token: $ROOT_TOKEN"

diff --git a/.github/workflows/postgresql-17-src-meson-perf.yml b/.github/workflows/postgresql-17-src-meson-perf.yml
@@ -30,7 +30,7 @@ jobs:
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
             llvm-11 llvm-11-dev libselinux1-dev python3-dev \
             uuid-dev liblz4-dev meson ninja-build \
-            sysbench libcurl4-openssl-dev libhttp-server-simple-perl
+            sysbench libcurl4-openssl-dev libhttp-server-simple-perl python3-pykmip
           sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
           wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
@@ -48,17 +48,30 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: 'src/contrib/pg_tde'
+
+      - name: Checkout libkmip submodule
+        run: |
+          git submodule update --init --recursive
+        working-directory: src/contrib/pg_tde
 
       - name: Build postgres
         run: |
           meson setup build --prefix `pwd`/../inst --buildtype=release
           cd build && ninja && ninja install
         working-directory: src
 
+      - name: Setup kmip 1
+        run: |
+          wget https://raw.githubusercontent.com/OpenKMIP/PyKMIP/refs/heads/master/bin/create_certificates.py
+          python3 create_certificates.py
+          cat client_certificate_jane_doe.pem >> client_key_jane_doe.pem
+        working-directory: /tmp
+
       - name: Test pg_tde
         run: |
           TV=$(mktemp)
           { exec >$TV; vault server -dev; } &
+          pykmip-server -f `pwd`/../contrib/pg_tde/pykmip-server.conf -l /tmp/kmip-server.log &
           sleep 10
           export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
           echo "Root token: $ROOT_TOKEN"