diff --git a/Makefile.in b/Makefile.in index 33c45b60..c6dca675 100644 --- a/Makefile.in +++ b/Makefile.in @@ -22,7 +22,8 @@ insert_update_delete_basic \ keyprovider_dependency_basic \ vault_v2_test_basic \ alter_index_basic \ -merge_join_basic +merge_join_basic \ +tablespace_basic TAP_TESTS = 1 OBJS = src/encryption/enc_tde.o \ diff --git a/expected/tablespace.out b/expected/tablespace.out new file mode 100644 index 00000000..52448e20 --- /dev/null +++ b/expected/tablespace.out @@ -0,0 +1,41 @@ +\set tde_am tde_heap +\i sql/tablespace.inc +CREATE EXTENSION pg_tde; +SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_key_provider_file +------------------------------ + 1 +(1 row) + +SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); + pg_tde_set_principal_key +-------------------------- + t +(1 row) + +CREATE TABLE test(num1 bigint, num2 double precision, t text) USING :tde_am; +INSERT INTO test(num1, num2, t) + SELECT round(random()*100), random(), 'text' + FROM generate_series(1, 10) s(i); +CREATE INDEX test_idx ON test(num1); +SET allow_in_place_tablespaces = true; +CREATE TABLESPACE test_tblspace LOCATION ''; +ALTER TABLE test SET TABLESPACE test_tblspace; +SELECT count(*) FROM test; + count +------- + 10 +(1 row) + +ALTER TABLE test SET TABLESPACE pg_default; +REINDEX (TABLESPACE test_tblspace, CONCURRENTLY) TABLE test; +INSERT INTO test VALUES (110, 2); +SELECT * FROM test WHERE num1=110; + num1 | num2 | t +------+------+--- + 110 | 2 | +(1 row) + +DROP TABLE test; +DROP TABLESPACE test_tblspace; +DROP EXTENSION pg_tde; diff --git a/expected/tablespace_basic.out b/expected/tablespace_basic.out new file mode 100644 index 00000000..6718dc5a --- /dev/null +++ b/expected/tablespace_basic.out @@ -0,0 +1,41 @@ +\set tde_am tde_heap_basic +\i sql/tablespace.inc +CREATE EXTENSION pg_tde; +SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_key_provider_file +------------------------------ + 1 +(1 row) + +SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); + pg_tde_set_principal_key +-------------------------- + t +(1 row) + +CREATE TABLE test(num1 bigint, num2 double precision, t text) USING :tde_am; +INSERT INTO test(num1, num2, t) + SELECT round(random()*100), random(), 'text' + FROM generate_series(1, 10) s(i); +CREATE INDEX test_idx ON test(num1); +SET allow_in_place_tablespaces = true; +CREATE TABLESPACE test_tblspace LOCATION ''; +ALTER TABLE test SET TABLESPACE test_tblspace; +SELECT count(*) FROM test; + count +------- + 10 +(1 row) + +ALTER TABLE test SET TABLESPACE pg_default; +REINDEX (TABLESPACE test_tblspace, CONCURRENTLY) TABLE test; +INSERT INTO test VALUES (110, 2); +SELECT * FROM test WHERE num1=110; + num1 | num2 | t +------+------+--- + 110 | 2 | +(1 row) + +DROP TABLE test; +DROP TABLESPACE test_tblspace; +DROP EXTENSION pg_tde; diff --git a/meson.build b/meson.build index acead4c4..45688e3c 100644 --- a/meson.build +++ b/meson.build @@ -96,6 +96,7 @@ sql_tests = [ 'trigger_on_view_basic', 'change_access_method_basic', 'insert_update_delete_basic', + 'tablespace_basic', 'vault_v2_test_basic', 'alter_index_basic', 'merge_join_basic', @@ -126,6 +127,7 @@ if get_variable('percona_ext', false) 'trigger_on_view', 'change_access_method', 'insert_update_delete', + 'tablespace', 'vault_v2_test', 'alter_index', 'merge_join', diff --git a/sql/tablespace.inc b/sql/tablespace.inc new file mode 100644 index 00000000..b2f7ebdd --- /dev/null +++ b/sql/tablespace.inc @@ -0,0 +1,26 @@ +CREATE EXTENSION pg_tde; + +SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); +SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault'); + +CREATE TABLE test(num1 bigint, num2 double precision, t text) USING :tde_am; +INSERT INTO test(num1, num2, t) + SELECT round(random()*100), random(), 'text' + FROM generate_series(1, 10) s(i); +CREATE INDEX test_idx ON test(num1); + +SET allow_in_place_tablespaces = true; +CREATE TABLESPACE test_tblspace LOCATION ''; + +ALTER TABLE test SET TABLESPACE test_tblspace; +SELECT count(*) FROM test; +ALTER TABLE test SET TABLESPACE pg_default; + +REINDEX (TABLESPACE test_tblspace, CONCURRENTLY) TABLE test; +INSERT INTO test VALUES (110, 2); + +SELECT * FROM test WHERE num1=110; + +DROP TABLE test; +DROP TABLESPACE test_tblspace; +DROP EXTENSION pg_tde; diff --git a/sql/tablespace.sql b/sql/tablespace.sql new file mode 100644 index 00000000..0d3aa620 --- /dev/null +++ b/sql/tablespace.sql @@ -0,0 +1,2 @@ +\set tde_am tde_heap +\i sql/tablespace.inc \ No newline at end of file diff --git a/sql/tablespace_basic.sql b/sql/tablespace_basic.sql new file mode 100644 index 00000000..0e358723 --- /dev/null +++ b/sql/tablespace_basic.sql @@ -0,0 +1,2 @@ +\set tde_am tde_heap_basic +\i sql/tablespace.inc \ No newline at end of file diff --git a/src/access/pg_tde_slot.c b/src/access/pg_tde_slot.c index 6850d711..1635b2e7 100644 --- a/src/access/pg_tde_slot.c +++ b/src/access/pg_tde_slot.c @@ -531,6 +531,9 @@ PGTdeExecStoreBufferHeapTuple(Relation rel, if (rel->rd_rel->relkind != RELKIND_TOASTVALUE) { RelKeyData *key = get_current_slot_relation_key(bslot, rel); + + Assert(key != NULL); + slot_copytuple(bslot->decrypted_buffer, tuple); PG_TDE_DECRYPT_TUPLE_EX(tuple, (HeapTuple)bslot->decrypted_buffer, key, "ExecStoreBuffer"); tuple->t_data = ((HeapTuple)bslot->decrypted_buffer)->t_data; diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 57e67ff2..699db530 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -87,7 +87,6 @@ typedef struct TDEMapFilePath typedef struct RelKeyCacheRec { RelFileNumber rel_number; - uint32 key_type; RelKeyData key; } RelKeyCacheRec; @@ -176,6 +175,8 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type memset(&int_key, 0, sizeof(InternalKey)); + int_key.rel_type = entry_type; + if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) { LWLockRelease(lock_pk); @@ -188,13 +189,12 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type } /* Encrypt the key */ - rel_key_data = tde_create_rel_key(newrlocator->relNumber, entry_type, &int_key, &principal_key->keyInfo); + rel_key_data = tde_create_rel_key(newrlocator->relNumber, &int_key, &principal_key->keyInfo); enc_rel_key_data = tde_encrypt_rel_key(principal_key, rel_key_data, newrlocator); /* * XLOG internal key */ - xlrec.entry_type = entry_type; xlrec.rlocator = *newrlocator; xlrec.relKey = *enc_rel_key_data; @@ -205,7 +205,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type /* * Add the encrypted key to the key map data file structure. */ - pg_tde_write_key_map_entry(newrlocator, entry_type, enc_rel_key_data, &principal_key->keyInfo); + pg_tde_write_key_map_entry(newrlocator, enc_rel_key_data, &principal_key->keyInfo); LWLockRelease(lock_pk); pfree(enc_rel_key_data); return rel_key_data; @@ -228,7 +228,7 @@ tde_sprint_key(InternalKey *k) * created key. */ RelKeyData * -tde_create_rel_key(RelFileNumber rel_num, uint32 key_type, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info) +tde_create_rel_key(RelFileNumber rel_num, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info) { RelKeyData rel_key_data; memcpy(&rel_key_data.principal_key_id, &principal_key_info->keyId, sizeof(TDEPrincipalKeyId)); @@ -236,7 +236,7 @@ tde_create_rel_key(RelFileNumber rel_num, uint32 key_type, InternalKey *key, TDE rel_key_data.internal_key.ctx = NULL; /* Add to the decrypted key to cache */ - return pg_tde_put_key_into_cache(rel_num, key_type, &rel_key_data); + return pg_tde_put_key_into_cache(rel_num, &rel_key_data); } /* * Encrypts a given key and returns the encrypted one. @@ -480,10 +480,10 @@ pg_tde_write_one_keydata(int fd, int32 key_index, RelKeyData *enc_rel_key_data) Assert(fd != -1); /* Calculate the writing position in the file. */ - curr_pos = (key_index * INTERNAL_KEY_LEN) + TDE_FILE_HEADER_SIZE; + curr_pos = (key_index * INTERNAL_KEY_DAT_LEN) + TDE_FILE_HEADER_SIZE; /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ - if (pg_pwrite(fd, &enc_rel_key_data->internal_key, INTERNAL_KEY_LEN, curr_pos) != INTERNAL_KEY_LEN) + if (pg_pwrite(fd, &enc_rel_key_data->internal_key, INTERNAL_KEY_DAT_LEN, curr_pos) != INTERNAL_KEY_DAT_LEN) { ereport(FATAL, (errcode_for_file_access(), @@ -506,7 +506,7 @@ pg_tde_write_one_keydata(int fd, int32 key_index, RelKeyData *enc_rel_key_data) * The caller must hold an exclusive lock tde_lwlock_enc_keys. */ void -pg_tde_write_key_map_entry(const RelFileLocator *rlocator, uint32 entry_type, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info) +pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info) { int32 key_index = 0; char db_map_path[MAXPGPATH] = {0}; @@ -518,7 +518,7 @@ pg_tde_write_key_map_entry(const RelFileLocator *rlocator, uint32 entry_type, Re pg_tde_set_db_file_paths(rlocator->dbOid, rlocator->spcOid, db_map_path, db_keydata_path); /* Create the map entry and then add the encrypted key to the data file */ - key_index = pg_tde_write_map_entry(rlocator, entry_type, db_map_path, principal_key_info); + key_index = pg_tde_write_map_entry(rlocator, enc_rel_key_data->internal_key.rel_type, db_map_path, principal_key_info); /* Add the encrypted key to the data file. */ pg_tde_write_keydata(db_keydata_path, principal_key_info, key_index, enc_rel_key_data); @@ -572,23 +572,22 @@ void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator, uint32 key_type * * The offset allows us to simply seek to the desired location and mark the entry * as MAP_ENTRY_FREE without needing any further processing. + * + * A caller should hold an EXCLUSIVE tde_lwlock_enc_keys lock. */ void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, uint32 key_type, off_t offset) { int32 key_index = 0; - LWLock *lock_files = tde_lwlock_enc_keys(); char db_map_path[MAXPGPATH] = {0}; - char db_keydata_path[MAXPGPATH] = {0}; + off_t start = 0; Assert(rlocator); /* Get the file paths */ - pg_tde_set_db_file_paths(rlocator->dbOid, rlocator->spcOid, db_map_path, db_keydata_path); + pg_tde_set_db_file_paths(rlocator->dbOid, rlocator->spcOid, db_map_path, NULL); /* Remove the map entry if found */ - LWLockAcquire(lock_files, LW_EXCLUSIVE); key_index = pg_tde_process_map_entry(rlocator, key_type, db_map_path, &offset, true); - LWLockRelease(lock_files); if (key_index == -1) { @@ -598,7 +597,17 @@ void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, uint32 key_type, rlocator->relNumber, db_map_path))); - return; + } + /* + * Remove TDE files it was the last TDE relation in a custom tablespace. + * DROP TABLESPACE needs an empty dir. + */ + if (rlocator->spcOid != GLOBALTABLESPACE_OID && + rlocator->spcOid != DEFAULTTABLESPACE_OID && + pg_tde_process_map_entry(NULL, key_type, db_map_path, &start, false) == -1) + { + pg_tde_delete_tde_files(rlocator->dbOid, rlocator->spcOid); + cleanup_key_provider_info(rlocator->dbOid, rlocator->spcOid); } } @@ -839,6 +848,87 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ return !is_err; } +/* + * Move relation's key to the new physical location and cache it with the new + * relfilenode. It recreates *.map and *.dat files with the old principal key + * and re-encrypted with the new relfilenode internal key. And copies the + * old keyring to the new location. + * Needed by ALTER TABLE SET TABLESPACE for example. + */ +bool +pg_tde_move_rel_key(const RelFileLocator *newrlocator, const RelFileLocator *oldrlocator) +{ + RelKeyData *rel_key; + RelKeyData *enc_key; + TDEPrincipalKey *principal_key; + KeyringProvideRecord provider_rec; + GenericKeyring *keyring; + XLogRelKey xlrec; + char db_map_path[MAXPGPATH] = {0}; + char db_keydata_path[MAXPGPATH] = {0}; + off_t offset = 0; + int32 key_index = 0; + + pg_tde_set_db_file_paths(oldrlocator->dbOid, oldrlocator->spcOid, db_map_path, db_keydata_path); + + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); + + principal_key = GetPrincipalKey(oldrlocator->dbOid, oldrlocator->spcOid, LW_EXCLUSIVE); + Assert(principal_key); + + /* + * Copy kering provider info. + * + * TODO: we can potentially avoid moving keyring and key tde files keeping + * these files always in dbOid+MyDatabaseTableSpace path. But the + * background writer isn't aware of MyDatabaseTableSpace hence it won't + * work with SMGR -> tde_heap. Revisit this after chages in SMGR (mdcreate) + * interface. + */ + keyring = GetKeyProviderByID(principal_key->keyInfo.keyringId, oldrlocator->dbOid, oldrlocator->spcOid); + Assert(keyring); + memcpy(provider_rec.provider_name, keyring->provider_name, sizeof(keyring->provider_name)); + provider_rec.provider_type = keyring->type; + memcpy(provider_rec.options, keyring->options, sizeof(keyring->options)); + copy_key_provider_info(&provider_rec, newrlocator->dbOid, newrlocator->spcOid, true); + + principal_key->keyInfo.keyringId = provider_rec.provider_id; + + key_index = pg_tde_process_map_entry(oldrlocator, MAP_ENTRY_VALID, db_map_path, &offset, false); + Assert(key_index != -1); + /* + * Re-encrypt relation key. We don't use internal_key cache to avoid locking + * complications. + */ + enc_key = pg_tde_read_keydata(db_keydata_path, key_index, principal_key); + rel_key = tde_decrypt_rel_key(principal_key, enc_key, oldrlocator); + enc_key = tde_encrypt_rel_key(principal_key, rel_key, newrlocator); + + xlrec.rlocator = *newrlocator; + xlrec.relKey = *enc_key; + xlrec.pkInfo = principal_key->keyInfo; + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xlrec)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_RELATION_KEY); + + pg_tde_write_key_map_entry(newrlocator, enc_key, &principal_key->keyInfo); + pg_tde_put_key_into_cache(newrlocator->relNumber, rel_key); + + XLogBeginInsert(); + XLogRegisterData((char *) oldrlocator, sizeof(RelFileLocator)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_FREE_MAP_ENTRY); + + /* Clean-up map/dat entries. It will also remove physical files (*.map, + * *.dat and keyring) if it was the last tde_heap_basic relation in the old + * locator AND it was a custom tablespace. + */ + pg_tde_free_key_map_entry(oldrlocator, MAP_ENTRY_VALID, offset); + + LWLockRelease(tde_lwlock_enc_keys()); + + pfree(enc_key); +} + #endif /* !FRONTEND */ /* @@ -846,7 +936,7 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ * reads the key data from the keydata file. */ RelKeyData * -pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type) +pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type, bool no_map_ok) { int32 key_index = 0; TDEPrincipalKey *principal_key; @@ -884,6 +974,11 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type) /* Get the file paths */ pg_tde_set_db_file_paths(rlocator->dbOid, rlocator->spcOid, db_map_path, db_keydata_path); + if (no_map_ok && access(db_map_path, F_OK) == -1) + { + LWLockRelease(lock_pk); + return NULL; + } /* Read the map entry and get the index of the relation key */ key_index = pg_tde_process_map_entry(rlocator, key_type, db_map_path, &offset, false); @@ -1005,7 +1100,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, uint32 key_type, char * /* * Open the file and read the required key data from file and return encrypted key. - * The caller should hold + * The caller should hold a tde_lwlock_enc_keys lock */ static RelKeyData * pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEPrincipalKey *principal_key) @@ -1184,7 +1279,6 @@ pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int fla return found; } - /* * Reads a single keydata from the file. */ @@ -1200,10 +1294,10 @@ pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEPrincipalKey *princi strncpy(enc_rel_key_data->principal_key_id.name, principal_key->keyInfo.keyId.name, PRINCIPAL_KEY_NAME_LEN); /* Calculate the reading position in the file. */ - read_pos += (key_index * INTERNAL_KEY_LEN) + TDE_FILE_HEADER_SIZE; + read_pos += (key_index * INTERNAL_KEY_DAT_LEN) + TDE_FILE_HEADER_SIZE; /* Check if the file has a valid key */ - if ((read_pos + INTERNAL_KEY_LEN) > lseek(keydata_fd, 0, SEEK_END)) + if ((read_pos + INTERNAL_KEY_DAT_LEN) > lseek(keydata_fd, 0, SEEK_END)) { char db_keydata_path[MAXPGPATH] = {0}; pg_tde_set_db_file_paths(principal_key->keyInfo.databaseId, principal_key->keyInfo.tablespaceId, NULL, db_keydata_path); @@ -1216,7 +1310,7 @@ pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEPrincipalKey *princi /* Read the encrypted key */ /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ - if (pg_pread(keydata_fd, &(enc_rel_key_data->internal_key), INTERNAL_KEY_LEN, read_pos) != INTERNAL_KEY_LEN) + if (pg_pread(keydata_fd, &(enc_rel_key_data->internal_key), INTERNAL_KEY_DAT_LEN, read_pos) != INTERNAL_KEY_DAT_LEN) { char db_keydata_path[MAXPGPATH] = {0}; pg_tde_set_db_file_paths(principal_key->keyInfo.databaseId, principal_key->keyInfo.tablespaceId, NULL, db_keydata_path); @@ -1280,7 +1374,7 @@ pg_tde_get_principal_key_info(Oid dbOid, Oid spcOid) * the tde fork file and populates cache. */ RelKeyData * -GetRelationKey(RelFileLocator rel, uint32 key_type) +GetRelationKey(RelFileLocator rel, uint32 key_type, bool no_map_ok) { RelKeyData *key; @@ -1288,11 +1382,11 @@ GetRelationKey(RelFileLocator rel, uint32 key_type) if (key) return key; - key = pg_tde_get_key_from_file(&rel, key_type); + key = pg_tde_get_key_from_file(&rel, key_type, no_map_ok); if (key != NULL) { - RelKeyData* cached_key = pg_tde_put_key_into_cache(rel.relNumber, key_type, key); + RelKeyData* cached_key = pg_tde_put_key_into_cache(rel.relNumber, key); pfree(key); return cached_key; } @@ -1303,19 +1397,19 @@ GetRelationKey(RelFileLocator rel, uint32 key_type) RelKeyData * GetSMGRRelationKey(RelFileLocator rel) { - return GetRelationKey(rel, TDE_KEY_TYPE_SMGR); + return GetRelationKey(rel, TDE_KEY_TYPE_SMGR, true); } RelKeyData * GetHeapBaiscRelationKey(RelFileLocator rel) { - return GetRelationKey(rel, TDE_KEY_TYPE_HEAP_BASIC); + return GetRelationKey(rel, TDE_KEY_TYPE_HEAP_BASIC, false); } RelKeyData * GetTdeGlobaleRelationKey(RelFileLocator rel) { - return GetRelationKey(rel, TDE_KEY_TYPE_GLOBAL); + return GetRelationKey(rel, TDE_KEY_TYPE_GLOBAL, false); } /* @@ -1336,7 +1430,7 @@ pg_tde_get_key_from_cache(RelFileNumber rel_number, uint32 key_type) rec = tde_rel_key_cache->data+i; if (rec != NULL && (rel_number == InvalidOid || (rec->rel_number == rel_number)) && - rec->key_type & key_type) + rec->key.internal_key.rel_type & key_type) { return &rec->key; } @@ -1350,7 +1444,7 @@ pg_tde_get_key_from_cache(RelFileNumber rel_number, uint32 key_type) * TODO: add tests. */ RelKeyData * -pg_tde_put_key_into_cache(RelFileNumber rel_num, uint32 key_type, RelKeyData *key) +pg_tde_put_key_into_cache(RelFileNumber rel_num, RelKeyData *key) { static long pageSize = 0; RelKeyCacheRec *rec; @@ -1420,7 +1514,6 @@ pg_tde_put_key_into_cache(RelFileNumber rel_num, uint32 key_type, RelKeyData *ke rec = tde_rel_key_cache->data + tde_rel_key_cache->len; rec->rel_number = rel_num; - rec->key_type = key_type; memcpy(&rec->key, key, sizeof(RelKeyCacheRec)); tde_rel_key_cache->len++; diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index 9f84b4c8..71854d27 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -37,10 +37,14 @@ tdeheap_rmgr_redo(XLogReaderState *record) if (info == XLOG_TDE_ADD_RELATION_KEY) { + TDEPrincipalKeyInfo *pk = NULL; XLogRelKey *xlrec = (XLogRelKey *) XLogRecGetData(record); + if (xlrec->pkInfo.databaseId != 0) + pk = &xlrec->pkInfo; + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); - pg_tde_write_key_map_entry(&xlrec->rlocator, xlrec->entry_type, &xlrec->relKey, NULL); + pg_tde_write_key_map_entry(&xlrec->rlocator, &xlrec->relKey, pk); LWLockRelease(tde_lwlock_enc_keys()); } else if (info == XLOG_TDE_ADD_PRINCIPAL_KEY) @@ -72,6 +76,16 @@ tdeheap_rmgr_redo(XLogReaderState *record) xl_tde_perform_rotate_key(xlrec); LWLockRelease(tde_lwlock_enc_keys()); } + + else if (info == XLOG_TDE_FREE_MAP_ENTRY) + { + off_t offset = 0; + RelFileLocator *xlrec = (RelFileLocator *) XLogRecGetData(record); + + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); + pg_tde_free_key_map_entry(xlrec, MAP_ENTRY_VALID, offset); + LWLockRelease(tde_lwlock_enc_keys()); + } else { elog(PANIC, "pg_tde_redo: unknown op code %u", info); diff --git a/src/catalog/tde_global_space.c b/src/catalog/tde_global_space.c index 5ceab027..b08aa8b2 100644 --- a/src/catalog/tde_global_space.c +++ b/src/catalog/tde_global_space.c @@ -67,7 +67,7 @@ TDEInitGlobalKeys(const char *dir) if (dir != NULL) pg_tde_set_globalspace_dir(dir); - ikey = pg_tde_get_key_from_file(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), TDE_KEY_TYPE_GLOBAL); + ikey = pg_tde_get_key_from_file(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), TDE_KEY_TYPE_GLOBAL, false); /* * Internal Key should be in the TopMemmoryContext because of SSL @@ -78,7 +78,7 @@ TDEInitGlobalKeys(const char *dir) * backend. (see * https://github.com/percona-Lab/pg_tde/pull/214#discussion_r1648998317) */ - pg_tde_put_key_into_cache(XLOG_TDE_OID, TDE_KEY_TYPE_GLOBAL, ikey); + pg_tde_put_key_into_cache(XLOG_TDE_OID, ikey); } } @@ -113,7 +113,7 @@ init_default_keyring(void) * TODO: should we remove it automaticaly on * pg_tde_rotate_principal_key() ? */ - save_new_key_provider_info(&provider, GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, true); + save_new_key_provider_info(&provider, GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, false); elog(INFO, "default keyring has been created for the global tablespace (WAL)." " Change it with pg_tde_add_key_provider_* and run pg_tde_rotate_principal_key." @@ -146,6 +146,8 @@ init_keys(void) memset(&int_key, 0, sizeof(InternalKey)); + int_key.rel_type = TDE_KEY_TYPE_GLOBAL; + /* Create and store an internal key for XLog */ if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) { @@ -156,9 +158,9 @@ init_keys(void) } rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); - rel_key_data = tde_create_rel_key(rlocator->relNumber, TDE_KEY_TYPE_GLOBAL, &int_key, &mkey->keyInfo); + rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &mkey->keyInfo); enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator); - pg_tde_write_key_map_entry(rlocator, TDE_KEY_TYPE_GLOBAL, enc_rel_key_data, &mkey->keyInfo); + pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo); pfree(enc_rel_key_data); pfree(mkey); } diff --git a/src/catalog/tde_keyring.c b/src/catalog/tde_keyring.c index e1b273c4..f31a878b 100644 --- a/src/catalog/tde_keyring.c +++ b/src/catalog/tde_keyring.c @@ -78,7 +78,8 @@ static void key_provider_startup_cleanup(int tde_tbl_count, XLogExtensionInstall static const char *get_keyring_provider_typename(ProviderType p_type); static uint32 write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, Oid tablespace_id, - off_t position, bool redo, bool recovery); + off_t position, bool error_if_exists, + bool write_xlog); static Size initialize_shared_state(void *start_address); static Size required_shared_mem_size(void); @@ -195,7 +196,7 @@ GetKeyProviderByName(const char *provider_name, Oid dbOid, Oid spcOid) static uint32 write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, - Oid tablespace_id, off_t position, bool redo, bool recovery) + Oid tablespace_id, off_t position, bool error_if_exists, bool write_xlog) { off_t bytes_written = 0; off_t curr_pos = 0; @@ -218,7 +219,7 @@ write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, (errcode_for_file_access(), errmsg("could not open tde file \"%s\": %m", kp_info_path))); } - if (!redo) + if (position == -1) { /* we also need to verify the name conflict and generate the next provider ID */ while (fetch_next_key_provider(fd, &curr_pos, &existing_provider)) @@ -227,9 +228,15 @@ write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, { close(fd); LWLockRelease(tde_provider_info_lock()); - ereport(ERROR, + ereport(error_if_exists ? ERROR : DEBUG1, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("key provider \"%s\" already exists", provider->provider_name))); + + if (!error_if_exists) + { + provider->provider_id = existing_provider.provider_id; + return provider->provider_id; + } } if (max_provider_id < existing_provider.provider_id) max_provider_id = existing_provider.provider_id; @@ -240,7 +247,7 @@ write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, /* emit the xlog here. So that we can handle partial file write errors * but cannot make new WAL entries during recovery. */ - if (!recovery) + if (write_xlog) { KeyringProviderXLRecord xlrec; @@ -293,9 +300,18 @@ write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, * Save the key provider info to the file */ uint32 -save_new_key_provider_info(KeyringProvideRecord* provider, Oid databaseId, Oid tablespaceId, bool recovery) +save_new_key_provider_info(KeyringProvideRecord* provider, Oid databaseId, Oid tablespaceId, bool write_xlog) +{ + return write_key_provider_info(provider, databaseId, tablespaceId, -1, true, write_xlog); +} + +/* + * Save the key provider info to the file but don't fail if it is already exists. + */ +uint32 +copy_key_provider_info(KeyringProvideRecord* provider, Oid newdatabaseId, Oid newtablespaceId, bool write_xlog) { - return write_key_provider_info(provider, databaseId, tablespaceId, 0, false, recovery); + return write_key_provider_info(provider, newdatabaseId, newtablespaceId, -1, false, write_xlog); } uint32 @@ -334,7 +350,7 @@ pg_tde_add_key_provider_internal(PG_FUNCTION_ARGS) strncpy(provider.options, options, sizeof(provider.options)); strncpy(provider.provider_name, provider_name, sizeof(provider.provider_name)); provider.provider_type = get_keyring_provider_from_typename(provider_type); - save_new_key_provider_info(&provider, dbOid, spcOid, false); + save_new_key_provider_info(&provider, dbOid, spcOid, true); PG_RETURN_INT32(provider.provider_id); } diff --git a/src/catalog/tde_principal_key.c b/src/catalog/tde_principal_key.c index cde08ce2..0e1c9383 100644 --- a/src/catalog/tde_principal_key.c +++ b/src/catalog/tde_principal_key.c @@ -788,6 +788,12 @@ get_principal_key_from_keyring(Oid dbOid, Oid spcOid) if (spcOid != GLOBALTABLESPACE_OID) { push_principal_key_to_cache(principalKey); + + /* If we do store key in cache we want to return a cache reference + * rather then a palloc'ed copy. + */ + pfree(principalKey); + principalKey = get_principal_key_from_cache(dbOid); } #endif diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 0db1464a..b6e1666f 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -23,10 +23,18 @@ typedef struct InternalKey { - uint8 key[INTERNAL_KEY_LEN]; - void* ctx; // TODO: shouldn't be here / written to the disk + /* + * DO NOT re-arrange fields! + * Any changes should be aligned with pg_tde_read/write_one_keydata() + */ + uint8 key[INTERNAL_KEY_LEN]; + uint32 rel_type; + + void* ctx; } InternalKey; +#define INTERNAL_KEY_DAT_LEN offsetof(InternalKey, ctx) + typedef struct RelKeyData { TDEPrincipalKeyId principal_key_id; @@ -37,19 +45,19 @@ typedef struct RelKeyData typedef struct XLogRelKey { RelFileLocator rlocator; - uint32 entry_type; RelKeyData relKey; + TDEPrincipalKeyInfo pkInfo; } XLogRelKey; extern RelKeyData *pg_tde_create_smgr_key(const RelFileLocator *newrlocator); extern RelKeyData *pg_tde_create_global_key(const RelFileLocator *newrlocator); extern RelKeyData *pg_tde_create_heap_basic_key(const RelFileLocator *newrlocator); extern RelKeyData *pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type); -extern void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, uint32 entry_type, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info); +extern void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info); extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator, uint32 key_type); extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, uint32 key_type, off_t offset); -extern RelKeyData *GetRelationKey(RelFileLocator rel, uint32 entry_type); +extern RelKeyData *GetRelationKey(RelFileLocator rel, uint32 entry_type, bool no_map_ok); extern RelKeyData *GetSMGRRelationKey(RelFileLocator rel); extern RelKeyData *GetHeapBaiscRelationKey(RelFileLocator rel); extern RelKeyData *GetTdeGlobaleRelationKey(RelFileLocator rel); @@ -60,15 +68,16 @@ extern TDEPrincipalKeyInfo *pg_tde_get_principal_key_info(Oid dbOid, Oid spcOid) extern bool pg_tde_save_principal_key(TDEPrincipalKeyInfo *principal_key_info); extern bool pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_principal_key); extern bool pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_size, char *k_file_data); -extern RelKeyData *tde_create_rel_key(RelFileNumber rel_num, uint32 key_type, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info); +extern RelKeyData *tde_create_rel_key(RelFileNumber rel_num, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info); extern RelKeyData *tde_encrypt_rel_key(TDEPrincipalKey *principal_key, RelKeyData *rel_key_data, const RelFileLocator *rlocator); extern RelKeyData *tde_decrypt_rel_key(TDEPrincipalKey *principal_key, RelKeyData *enc_rel_key_data, const RelFileLocator *rlocator); -extern RelKeyData *pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type); +extern RelKeyData *pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type, bool no_map_ok); +extern bool pg_tde_move_rel_key(const RelFileLocator *newrlocator, const RelFileLocator *oldrlocator); extern void pg_tde_set_db_file_paths(Oid dbOid, Oid spcOid, char *map_path, char *keydata_path); const char * tde_sprint_key(InternalKey *k); -extern RelKeyData *pg_tde_put_key_into_cache(RelFileNumber rel_num, uint32 key_type, RelKeyData *key); +extern RelKeyData *pg_tde_put_key_into_cache(RelFileNumber rel_num, RelKeyData *key); #endif /*PG_TDE_MAP_H*/ diff --git a/src/include/access/pg_tde_xlog.h b/src/include/access/pg_tde_xlog.h index 1ea24cf7..2881f244 100644 --- a/src/include/access/pg_tde_xlog.h +++ b/src/include/access/pg_tde_xlog.h @@ -21,6 +21,7 @@ #define XLOG_TDE_EXTENSION_INSTALL_KEY 0x20 #define XLOG_TDE_ROTATE_KEY 0x30 #define XLOG_TDE_ADD_KEY_PROVIDER_KEY 0x40 +#define XLOG_TDE_FREE_MAP_ENTRY 0x50 /* TODO: ID has to be registedred and changed: https://wiki.postgresql.org/wiki/CustomWALResourceManagers */ #define RM_TDERMGR_ID RM_EXPERIMENTAL_ID diff --git a/src/include/catalog/tde_keyring.h b/src/include/catalog/tde_keyring.h index 2d3a1675..fdfa2585 100644 --- a/src/include/catalog/tde_keyring.h +++ b/src/include/catalog/tde_keyring.h @@ -78,7 +78,12 @@ extern GenericKeyring *GetKeyProviderByID(int provider_id, Oid dbOid, Oid spcOid extern ProviderType get_keyring_provider_from_typename(char *provider_type); extern void cleanup_key_provider_info(Oid databaseId, Oid tablespaceId); extern void InitializeKeyProviderInfo(void); -extern uint32 save_new_key_provider_info(KeyringProvideRecord *provider, Oid databaseId, Oid tablespaceId, bool recovery); +extern uint32 save_new_key_provider_info(KeyringProvideRecord *provider, + Oid databaseId, Oid tablespaceId, + bool write_xlog); +extern uint32 copy_key_provider_info(KeyringProvideRecord* provider, + Oid newdatabaseId, Oid newtablespaceId, + bool write_xlog); extern uint32 redo_key_provider_info(KeyringProviderXLRecord *xlrec); extern bool ParseKeyringJSONOptions(ProviderType provider_type, void *out_opts, diff --git a/src/transam/pg_tde_xact_handler.c b/src/transam/pg_tde_xact_handler.c index 6fb02cd0..2d45aaed 100644 --- a/src/transam/pg_tde_xact_handler.c +++ b/src/transam/pg_tde_xact_handler.c @@ -101,6 +101,8 @@ do_pending_deletes(bool isCommit) PendingMapEntryDelete *prev; PendingMapEntryDelete *next; + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); + prev = NULL; for (pending = pendingDeletes; pending != NULL; pending = next) { @@ -129,6 +131,8 @@ do_pending_deletes(bool isCommit) /* prev does not change */ } + + LWLockRelease(tde_lwlock_enc_keys()); } diff --git a/src16/access/pg_tdeam_handler.c b/src16/access/pg_tdeam_handler.c index 04df262c..a725a5b9 100644 --- a/src16/access/pg_tdeam_handler.c +++ b/src16/access/pg_tdeam_handler.c @@ -707,6 +707,7 @@ pg_tdeam_relation_copy_data(Relation rel, const RelFileLocator *newrlocator) } } + pg_tde_move_rel_key(newrlocator, &rel->rd_locator); /* drop old relation, and close new one */ RelationDropStorage(rel); diff --git a/src17/access/pg_tdeam_handler.c b/src17/access/pg_tdeam_handler.c index dbbe62cf..6a09e325 100644 --- a/src17/access/pg_tdeam_handler.c +++ b/src17/access/pg_tdeam_handler.c @@ -702,6 +702,7 @@ pg_tdeam_relation_copy_data(Relation rel, const RelFileLocator *newrlocator) } } + pg_tde_move_rel_key(newrlocator, &rel->rd_locator); /* drop old relation, and close new one */ RelationDropStorage(rel);