diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index 0e57e170..92ff4733 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -12,11 +12,13 @@ #include "postgres.h" +#include "pg_tde.h" #include "pg_tde_defines.h" #include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xloginsert.h" #include "catalog/pg_tablespace_d.h" +#include "catalog/tde_keyring.h" #include "storage/bufmgr.h" #include "storage/shmem.h" #include "utils/guc.h" @@ -61,12 +63,19 @@ pg_tde_rmgr_redo(XLogReaderState *record) save_principal_key_info(mkey); } - else if (info == XLOG_TDE_CLEAN_PRINCIPAL_KEY) + else if (info == XLOG_TDE_EXTENSION_INSTALL_KEY) { - XLogPrincipalKeyCleanup *xlrec = (XLogPrincipalKeyCleanup *) XLogRecGetData(record); + XLogExtensionInstall *xlrec = (XLogExtensionInstall *)XLogRecGetData(record); - cleanup_principal_key_info(xlrec->databaseId, xlrec->tablespaceId); + extension_install_redo(xlrec); } + + else if (info == XLOG_TDE_ADD_KEY_PROVIDER_KEY) + { + KeyringProviderXLRecord *xlrec = (KeyringProviderXLRecord *)XLogRecGetData(record); + redo_key_provider_info(xlrec); + } + else if (info == XLOG_TDE_ROTATE_KEY) { XLogPrincipalKeyRotate *xlrec = (XLogPrincipalKeyRotate *) XLogRecGetData(record); @@ -96,11 +105,11 @@ pg_tde_rmgr_desc(StringInfo buf, XLogReaderState *record) appendStringInfo(buf, "add tde principal key for db %u/%u", xlrec->databaseId, xlrec->tablespaceId); } - if (info == XLOG_TDE_CLEAN_PRINCIPAL_KEY) + if (info == XLOG_TDE_EXTENSION_INSTALL_KEY) { - XLogPrincipalKeyCleanup *xlrec = (XLogPrincipalKeyCleanup *) XLogRecGetData(record); + XLogExtensionInstall *xlrec = (XLogExtensionInstall *)XLogRecGetData(record); - appendStringInfo(buf, "cleanup tde principal key info for db %u/%u", xlrec->databaseId, xlrec->tablespaceId); + appendStringInfo(buf, "tde extension install for db %u/%u", xlrec->database_id, xlrec->tablespace_id); } if (info == XLOG_TDE_ROTATE_KEY) { @@ -108,6 +117,12 @@ pg_tde_rmgr_desc(StringInfo buf, XLogReaderState *record) appendStringInfo(buf, "rotate principal key for %u", xlrec->databaseId); } + if (info == XLOG_TDE_ADD_KEY_PROVIDER_KEY) + { + KeyringProviderXLRecord *xlrec = (KeyringProviderXLRecord *)XLogRecGetData(record); + + appendStringInfo(buf, "add key provider %s for %u", xlrec->provider.provider_name, xlrec->database_id); + } } const char * @@ -119,8 +134,8 @@ pg_tde_rmgr_identify(uint8 info) if ((info & ~XLR_INFO_MASK) == XLOG_TDE_ADD_PRINCIPAL_KEY) return "XLOG_TDE_ADD_PRINCIPAL_KEY"; - if ((info & ~XLR_INFO_MASK) == XLOG_TDE_CLEAN_PRINCIPAL_KEY) - return "XLOG_TDE_CLEAN_PRINCIPAL_KEY"; + if ((info & ~XLR_INFO_MASK) == XLOG_TDE_EXTENSION_INSTALL_KEY) + return "XLOG_TDE_EXTENSION_INSTALL_KEY"; return NULL; } diff --git a/src/catalog/tde_keyring.c b/src/catalog/tde_keyring.c index 721f9191..6409bad7 100644 --- a/src/catalog/tde_keyring.c +++ b/src/catalog/tde_keyring.c @@ -9,7 +9,10 @@ * *------------------------------------------------------------------------- */ - +#include "postgres.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "access/pg_tde_xlog.h" #include "catalog/tde_keyring.h" #include "catalog/tde_principal_key.h" #include "access/skey.h" @@ -64,8 +67,9 @@ static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, static VaultV2Keyring *load_vaultV2_keyring_provider_options(Datum keyring_options); static void debug_print_kerying(GenericKeyring *keyring); static char *get_keyring_infofile_path(char *resPath, Oid dbOid, Oid spcOid); -static uint32 save_key_provider(KeyringProvideRecord *provider); -static void key_provider_startup_cleanup(int tde_tbl_count, void *arg); +static void key_provider_startup_cleanup(int tde_tbl_count, XLogExtensionInstall *ext_info, bool redo, void *arg); +static uint32 write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, Oid tablespace_id, off_t position, bool redo); +static uint32 save_new_key_provider_info(KeyringProvideRecord *provider); static Size initialize_shared_state(void *start_address); static Size required_shared_mem_size(void); @@ -112,7 +116,7 @@ void InitializeKeyProviderInfo(void) on_ext_install(key_provider_startup_cleanup, NULL); } static void -key_provider_startup_cleanup(int tde_tbl_count, void *arg) +key_provider_startup_cleanup(int tde_tbl_count, XLogExtensionInstall *ext_info, bool redo, void *arg) { if (tde_tbl_count > 0) @@ -121,15 +125,7 @@ key_provider_startup_cleanup(int tde_tbl_count, void *arg) (errmsg("failed to perform initialization. database already has %d TDE tables", tde_tbl_count))); return; } - cleanup_key_provider_info(MyDatabaseId, MyDatabaseTableSpace); - - /* TODO: XLog the key cleanup */ - // XLogPrincipalKeyCleanup xlrec; - // xlrec.databaseId = MyDatabaseId; - // xlrec.tablespaceId = MyDatabaseTableSpace; - // XLogBeginInsert(); - // XLogRegisterData((char *)&xlrec, sizeof(TDEPrincipalKeyInfo)); - // XLogInsert(RM_TDERMGR_ID, XLOG_TDE_CLEAN_PRINCIPAL_KEY); + cleanup_key_provider_info(ext_info->database_id, ext_info->tablespace_id); } ProviderType @@ -314,11 +310,8 @@ fetch_next_key_provider(int fd, off_t* curr_pos, KeyringProvideRecord *provider) return true; } -/* -* Save the key provider info to the file -*/ static uint32 -save_key_provider(KeyringProvideRecord *provider) +write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, Oid tablespace_id, off_t position, bool redo) { off_t bytes_written = 0; off_t curr_pos = 0; @@ -329,7 +322,7 @@ save_key_provider(KeyringProvideRecord *provider) Assert(provider != NULL); - get_keyring_infofile_path(kp_info_path, MyDatabaseId, MyDatabaseTableSpace); + get_keyring_infofile_path(kp_info_path, database_id, tablespace_id); LWLockAcquire(tde_provider_info_lock(), LW_EXCLUSIVE); @@ -341,35 +334,54 @@ save_key_provider(KeyringProvideRecord *provider) (errcode_for_file_access(), errmsg("could not open tde file \"%s\": %m", kp_info_path))); } - - /* we also need to verify the name conflict and generate the next provider ID */ - while (fetch_next_key_provider(fd, &curr_pos, &existing_provider)) + if (!redo) { - if (strcmp(existing_provider.provider_name, provider->provider_name) == 0) + KeyringProviderXLRecord xlrec; + /* we also need to verify the name conflict and generate the next provider ID */ + while (fetch_next_key_provider(fd, &curr_pos, &existing_provider)) { - close(fd); - LWLockRelease(tde_provider_info_lock()); - ereport(ERROR, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("key provider \"%s\" already exists", provider->provider_name))); + if (strcmp(existing_provider.provider_name, provider->provider_name) == 0) + { + close(fd); + LWLockRelease(tde_provider_info_lock()); + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("key provider \"%s\" already exists", provider->provider_name))); + } + if (max_provider_id < existing_provider.provider_id) + max_provider_id = existing_provider.provider_id; } - if (max_provider_id < existing_provider.provider_id) - max_provider_id = existing_provider.provider_id; + provider->provider_id = max_provider_id + 1; + curr_pos = lseek(fd, 0, SEEK_END); + /* emit the xlog here. So that we can handle partial file write errors */ + xlrec.database_id = database_id; + xlrec.tablespace_id = tablespace_id; + xlrec.offset_in_file = curr_pos; + memcpy(&xlrec.provider, provider, sizeof(KeyringProvideRecord)); + + XLogBeginInsert(); + XLogRegisterData((char *)&xlrec, sizeof(KeyringProviderXLRecord)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_KEY_PROVIDER_KEY); + } + else + { + /* we are performing redo, just go to the position received from the + * xlog and write the record there. + * No need to verify the name conflict and generate the provider ID + */ + curr_pos = lseek(fd, position, SEEK_SET); } - provider->provider_id = max_provider_id + 1; /* * All good, Just add a new provider - * Write key to the end of file */ - curr_pos = lseek(fd, 0, SEEK_END); bytes_written = pg_pwrite(fd, provider, sizeof(KeyringProvideRecord), curr_pos); if (bytes_written != sizeof(KeyringProvideRecord)) { close(fd); LWLockRelease(tde_provider_info_lock()); ereport(ERROR, - (errcode_for_file_access(), - errmsg("key provider info file \"%s\" can't be written: %m", + (errcode_for_file_access(), + errmsg("key provider info file \"%s\" can't be written: %m", kp_info_path))); } if (pg_fsync(fd) != 0) @@ -377,8 +389,8 @@ save_key_provider(KeyringProvideRecord *provider) close(fd); LWLockRelease(tde_provider_info_lock()); ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not fsync file \"%s\": %m", + (errcode_for_file_access(), + errmsg("could not fsync file \"%s\": %m", kp_info_path))); } close(fd); @@ -387,10 +399,24 @@ save_key_provider(KeyringProvideRecord *provider) } /* - * Scan the key provider info file and can also apply filter based on scanType + * Save the key provider info to the file */ -static List * -scan_key_provider_file(ProviderScanType scanType, void* scanKey) +static uint32 +save_new_key_provider_info(KeyringProvideRecord* provider) +{ + return write_key_provider_info(provider, MyDatabaseId, MyDatabaseTableSpace, 0, false); +} + +uint32 +redo_key_provider_info(KeyringProviderXLRecord* xlrec) +{ + return write_key_provider_info(&xlrec->provider, xlrec->database_id, xlrec->tablespace_id, xlrec->offset_in_file, true); +} + +/* + * Scan the key provider info file and can also apply filter based on scanType + */ +static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey) { off_t curr_pos = 0; int fd; @@ -483,6 +509,7 @@ pg_tde_add_key_provider_internal(PG_FUNCTION_ARGS) strncpy(provider.options, options, sizeof(provider.options)); strncpy(provider.provider_name, provider_name, sizeof(provider.provider_name)); provider.provider_type = get_keyring_provider_from_typename(provider_type); - save_key_provider(&provider); + save_new_key_provider_info(&provider); + PG_RETURN_INT32(provider.provider_id); } diff --git a/src/catalog/tde_principal_key.c b/src/catalog/tde_principal_key.c index 4bf2129c..0571dc3a 100644 --- a/src/catalog/tde_principal_key.c +++ b/src/catalog/tde_principal_key.c @@ -66,7 +66,7 @@ static void initialize_objects_in_dsa_area(dsa_area *dsa, void *raw_dsa_area); static Size cache_area_size(void); static Size required_shared_mem_size(void); static void shared_memory_shutdown(int code, Datum arg); -static void principal_key_startup_cleanup(int tde_tbl_count, void *arg); +static void principal_key_startup_cleanup(int tde_tbl_count, XLogExtensionInstall *ext_info, bool redo, void *arg); static void clear_principal_key_cache(Oid databaseId) ; static inline dshash_table *get_principal_key_Hash(void); static TDEPrincipalKey *get_principal_key_from_cache(Oid dbOid); @@ -658,10 +658,8 @@ push_principal_key_to_cache(TDEPrincipalKey *principalKey) * but unfortunately we do not have any such mechanism in PG. */ static void -principal_key_startup_cleanup(int tde_tbl_count, void* arg) +principal_key_startup_cleanup(int tde_tbl_count, XLogExtensionInstall *ext_info, bool redo, void *arg) { - XLogPrincipalKeyCleanup xlrec; - if (tde_tbl_count > 0) { ereport(WARNING, @@ -669,14 +667,7 @@ principal_key_startup_cleanup(int tde_tbl_count, void* arg) return; } - cleanup_principal_key_info(MyDatabaseId, MyDatabaseTableSpace); - - /* XLog the key cleanup */ - xlrec.databaseId = MyDatabaseId; - xlrec.tablespaceId = MyDatabaseTableSpace; - XLogBeginInsert(); - XLogRegisterData((char *) &xlrec, sizeof(TDEPrincipalKeyInfo)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_CLEAN_PRINCIPAL_KEY); + cleanup_principal_key_info(ext_info->database_id, ext_info->tablespace_id); } void diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index 1ed2e010..639a70fd 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -17,10 +17,11 @@ #endif /* TDE XLOG resource manager */ -#define XLOG_TDE_ADD_RELATION_KEY 0x00 +#define XLOG_TDE_ADD_RELATION_KEY 0x00 #define XLOG_TDE_ADD_PRINCIPAL_KEY 0x10 -#define XLOG_TDE_CLEAN_PRINCIPAL_KEY 0x20 -#define XLOG_TDE_ROTATE_KEY 0x30 +#define XLOG_TDE_EXTENSION_INSTALL_KEY 0x20 +#define XLOG_TDE_ROTATE_KEY 0x30 +#define XLOG_TDE_ADD_KEY_PROVIDER_KEY 0x40 /* TODO: ID has to be registedred and changed: https://wiki.postgresql.org/wiki/CustomWALResourceManagers */ #define RM_TDERMGR_ID RM_EXPERIMENTAL_ID diff --git a/src/include/catalog/tde_keyring.h b/src/include/catalog/tde_keyring.h index afdf80aa..f35b4b2d 100644 --- a/src/include/catalog/tde_keyring.h +++ b/src/include/catalog/tde_keyring.h @@ -69,6 +69,13 @@ typedef struct KeyringProvideRecord char options[MAX_KEYRING_OPTION_LEN]; ProviderType provider_type; } KeyringProvideRecord; +typedef struct KeyringProviderXLRecord +{ + Oid database_id; + Oid tablespace_id; + off_t offset_in_file; + KeyringProvideRecord provider; +} KeyringProviderXLRecord; extern List *GetAllKeyringProviders(void); extern GenericKeyring *GetKeyProviderByName(const char *provider_name); @@ -76,4 +83,5 @@ extern GenericKeyring *GetKeyProviderByID(int provider_id); extern ProviderType get_keyring_provider_from_typename(char *provider_type); extern void cleanup_key_provider_info(Oid databaseId, Oid tablespaceId); extern void InitializeKeyProviderInfo(void); +extern uint32 redo_key_provider_info(KeyringProviderXLRecord *xlrec); #endif /*TDE_KEYRING_H*/ diff --git a/src/include/catalog/tde_principal_key.h b/src/include/catalog/tde_principal_key.h index ad273000..3cff9c26 100644 --- a/src/include/catalog/tde_principal_key.h +++ b/src/include/catalog/tde_principal_key.h @@ -55,12 +55,6 @@ typedef struct XLogPrincipalKeyRotate #define SizeoOfXLogPrincipalKeyRotate offsetof(XLogPrincipalKeyRotate, buff) -typedef struct XLogPrincipalKeyCleanup -{ - Oid databaseId; - Oid tablespaceId; -} XLogPrincipalKeyCleanup; - extern void InitializePrincipalKeyInfo(void); extern void cleanup_principal_key_info(Oid databaseId, Oid tablespaceId); extern LWLock *tde_lwlock_mk_files(void); diff --git a/src/include/pg_tde.h b/src/include/pg_tde.h index 6625c532..93b29119 100644 --- a/src/include/pg_tde.h +++ b/src/include/pg_tde.h @@ -8,8 +8,15 @@ #ifndef PG_TDE_H #define PG_TDE_H -typedef void (*pg_tde_on_ext_install_callback)(int tde_tbl_count, void* arg); +typedef struct XLogExtensionInstall +{ + Oid database_id; + Oid tablespace_id; +} XLogExtensionInstall; + +typedef void (*pg_tde_on_ext_install_callback)(int tde_tbl_count, XLogExtensionInstall* ext_info, bool redo, void *arg); extern void on_ext_install(pg_tde_on_ext_install_callback function, void* arg); +extern void extension_install_redo(XLogExtensionInstall *xlrec); #endif /*PG_TDE_H*/ \ No newline at end of file diff --git a/src/pg_tde.c b/src/pg_tde.c index ed21b0c3..069758f2 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -22,6 +22,8 @@ #include "access/pg_tde_xlog.h" #include "encryption/enc_aes.h" #include "access/pg_tde_tdemap.h" +#include "access/xlog.h" +#include "access/xloginsert.h" #include "keyring/keyring_config.h" #include "keyring/keyring_api.h" #include "common/pg_tde_shmem.h" @@ -48,7 +50,7 @@ struct OnExtInstall static struct OnExtInstall on_ext_install_list[MAX_ON_INSTALLS]; static int on_ext_install_index = 0; -static void run_extension_install_callbacks(void); +static void run_extension_install_callbacks(XLogExtensionInstall *xlrec, bool redo); void _PG_init(void); Datum pg_tde_extension_initialize(PG_FUNCTION_ARGS); Datum pg_tde_version(PG_FUNCTION_ARGS); @@ -126,9 +128,22 @@ _PG_init(void) Datum pg_tde_extension_initialize(PG_FUNCTION_ARGS) { /* Initialize the TDE map */ - run_extension_install_callbacks(); + XLogExtensionInstall xlrec; + xlrec.database_id = MyDatabaseId; + xlrec.tablespace_id = MyDatabaseTableSpace; + run_extension_install_callbacks(&xlrec, false); + /* Also put this info in xlog, so we can replicate the same on the other side */ + XLogBeginInsert(); + XLogRegisterData((char *)&xlrec, sizeof(XLogExtensionInstall)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_EXTENSION_INSTALL_KEY); + PG_RETURN_NULL(); } +void +extension_install_redo(XLogExtensionInstall *xlrec) +{ + run_extension_install_callbacks(xlrec, true); +} /* ---------------------------------------------------------------- * on_ext_install @@ -137,13 +152,12 @@ Datum pg_tde_extension_initialize(PG_FUNCTION_ARGS) * run at the time of pg_tde extension installs. * ---------------------------------------------------------------- */ -void -on_ext_install(pg_tde_on_ext_install_callback function, void *arg) +void on_ext_install(pg_tde_on_ext_install_callback function, void *arg) { if (on_ext_install_index >= MAX_ON_INSTALLS) ereport(FATAL, - (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg_internal("out of on extension install slots"))); + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg_internal("out of on extension install slots"))); on_ext_install_list[on_ext_install_index].function = function; on_ext_install_list[on_ext_install_index].arg = arg; @@ -156,19 +170,21 @@ on_ext_install(pg_tde_on_ext_install_callback function, void *arg) * ------------------ */ static void -run_extension_install_callbacks(void) +run_extension_install_callbacks(XLogExtensionInstall* xlrec , bool redo) { int i; + int tde_table_count =0; /* * Get the number of tde tables in this database * should always be zero. But still, it prevents * the cleanup if someone explicitly calls this * function. */ - int tde_table_count = get_tde_tables_count(); + if (!redo) + tde_table_count = get_tde_tables_count(); for (i = 0; i < on_ext_install_index; i++) on_ext_install_list[i] - .function(tde_table_count, on_ext_install_list[i].arg); + .function(tde_table_count, xlrec, redo, on_ext_install_list[i].arg); } /* Returns package version */