Skip to content

Commit

Permalink
Store rel key type along with internal key
Browse files Browse the repository at this point in the history 
This is needed for ALTER  TABLESPACE as pg_tdeam_relation_copy_data
gets only the old rlocator and needs to know how to properly save the
key in the new rlocator.

Also during ALTER ... TABLESPACE, tde_mdcreate gets called and
tde_smgr_get_key is trying to get the internal key. But there is no key
moved to the new table space since it is called during
RelationCreateStorage() and pg_tde.map file might not exist. And it's ok in
this case but it results in an error. This commit introduces an option not
to have a pg_tde.map (treated as no key).
  • Loading branch information
@dAdAbird
dAdAbird committed Oct 31, 2024
1 parent 05f2e22 commit 5d8436a
Show file tree
Hide file tree
Showing 4 changed files with 53 additions and 40 deletions.
56 changes: 30 additions & 26 deletions src/access/pg_tde_tdemap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ typedef struct TDEMapFilePath
typedef struct RelKeyCacheRec
{
RelFileNumber rel_number;
uint32 key_type;
RelKeyData key;
} RelKeyCacheRec;

Expand Down Expand Up @@ -176,6 +175,8 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type

memset(&int_key, 0, sizeof(InternalKey));

int_key.rel_type = entry_type;

if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN))
{
LWLockRelease(lock_pk);
Expand All @@ -188,13 +189,12 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type
}

/* Encrypt the key */
rel_key_data = tde_create_rel_key(newrlocator->relNumber, entry_type, &int_key, &principal_key->keyInfo);
rel_key_data = tde_create_rel_key(newrlocator->relNumber, &int_key, &principal_key->keyInfo);
enc_rel_key_data = tde_encrypt_rel_key(principal_key, rel_key_data, newrlocator);

/*
* XLOG internal key
*/
xlrec.entry_type = entry_type;
xlrec.rlocator = *newrlocator;
xlrec.relKey = *enc_rel_key_data;

Expand All @@ -205,7 +205,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type
/*
* Add the encrypted key to the key map data file structure.
*/
pg_tde_write_key_map_entry(newrlocator, entry_type, enc_rel_key_data, &principal_key->keyInfo);
pg_tde_write_key_map_entry(newrlocator, enc_rel_key_data, &principal_key->keyInfo);
LWLockRelease(lock_pk);
pfree(enc_rel_key_data);
return rel_key_data;
Expand All @@ -228,15 +228,15 @@ tde_sprint_key(InternalKey *k)
* created key.
*/
RelKeyData *
tde_create_rel_key(RelFileNumber rel_num, uint32 key_type, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info)
tde_create_rel_key(RelFileNumber rel_num, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info)
{
RelKeyData rel_key_data;
memcpy(&rel_key_data.principal_key_id, &principal_key_info->keyId, sizeof(TDEPrincipalKeyId));
memcpy(&rel_key_data.internal_key, key, sizeof(InternalKey));
rel_key_data.internal_key.ctx = NULL;

/* Add to the decrypted key to cache */
return pg_tde_put_key_into_cache(rel_num, key_type, &rel_key_data);
return pg_tde_put_key_into_cache(rel_num, &rel_key_data);
}
/*
* Encrypts a given key and returns the encrypted one.
Expand Down Expand Up @@ -480,10 +480,10 @@ pg_tde_write_one_keydata(int fd, int32 key_index, RelKeyData *enc_rel_key_data)
Assert(fd != -1);

/* Calculate the writing position in the file. */
curr_pos = (key_index * INTERNAL_KEY_LEN) + TDE_FILE_HEADER_SIZE;
curr_pos = (key_index * INTERNAL_KEY_DAT_LEN) + TDE_FILE_HEADER_SIZE;

/* TODO: pgstat_report_wait_start / pgstat_report_wait_end */
if (pg_pwrite(fd, &enc_rel_key_data->internal_key, INTERNAL_KEY_LEN, curr_pos) != INTERNAL_KEY_LEN)
if (pg_pwrite(fd, &enc_rel_key_data->internal_key, INTERNAL_KEY_DAT_LEN, curr_pos) != INTERNAL_KEY_DAT_LEN)
{
ereport(FATAL,
(errcode_for_file_access(),
Expand All @@ -506,7 +506,7 @@ pg_tde_write_one_keydata(int fd, int32 key_index, RelKeyData *enc_rel_key_data)
* The caller must hold an exclusive lock tde_lwlock_enc_keys.
*/
void
pg_tde_write_key_map_entry(const RelFileLocator *rlocator, uint32 entry_type, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info)
pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info)
{
int32 key_index = 0;
char db_map_path[MAXPGPATH] = {0};
Expand All @@ -518,7 +518,7 @@ pg_tde_write_key_map_entry(const RelFileLocator *rlocator, uint32 entry_type, Re
pg_tde_set_db_file_paths(rlocator->dbOid, rlocator->spcOid, db_map_path, db_keydata_path);

/* Create the map entry and then add the encrypted key to the data file */
key_index = pg_tde_write_map_entry(rlocator, entry_type, db_map_path, principal_key_info);
key_index = pg_tde_write_map_entry(rlocator, enc_rel_key_data->internal_key.rel_type, db_map_path, principal_key_info);

/* Add the encrypted key to the data file. */
pg_tde_write_keydata(db_keydata_path, principal_key_info, key_index, enc_rel_key_data);
Expand Down Expand Up @@ -604,7 +604,7 @@ void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, uint32 key_type,
*/
if (rlocator->spcOid != GLOBALTABLESPACE_OID &&
rlocator->spcOid != DEFAULTTABLESPACE_OID &&
pg_tde_process_map_entry(NULL, db_map_path, &start, false) == -1)
pg_tde_process_map_entry(NULL, key_type, db_map_path, &start, false) == -1)
{
pg_tde_delete_tde_files(rlocator->dbOid, rlocator->spcOid);
cleanup_key_provider_info(rlocator->dbOid, rlocator->spcOid);
Expand Down Expand Up @@ -894,7 +894,7 @@ pg_tde_move_rel_key(const RelFileLocator *newrlocator, const RelFileLocator *old

principal_key->keyInfo.keyringId = provider_rec.provider_id;

key_index = pg_tde_process_map_entry(oldrlocator, db_map_path, &offset, false);
key_index = pg_tde_process_map_entry(oldrlocator, MAP_ENTRY_VALID, db_map_path, &offset, false);
Assert(key_index != -1);
/*
* Re-encrypt relation key. We don't use internal_key cache to avoid locking
Expand Down Expand Up @@ -922,7 +922,7 @@ pg_tde_move_rel_key(const RelFileLocator *newrlocator, const RelFileLocator *old
* *.dat and keyring) if it was the last tde_heap_basic relation in the old
* locator AND it was a custom tablespace.
*/
pg_tde_free_key_map_entry(oldrlocator, offset);
pg_tde_free_key_map_entry(oldrlocator, MAP_ENTRY_VALID, offset);

LWLockRelease(tde_lwlock_enc_keys());

Expand All @@ -936,7 +936,7 @@ pg_tde_move_rel_key(const RelFileLocator *newrlocator, const RelFileLocator *old
* reads the key data from the keydata file.
*/
RelKeyData *
pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type)
pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type, bool no_map_ok)
{
int32 key_index = 0;
TDEPrincipalKey *principal_key;
Expand Down Expand Up @@ -974,6 +974,11 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type)
/* Get the file paths */
pg_tde_set_db_file_paths(rlocator->dbOid, rlocator->spcOid, db_map_path, db_keydata_path);

if (no_map_ok && access(db_map_path, F_OK) == -1)
{
LWLockRelease(lock_pk);
return NULL;
}
/* Read the map entry and get the index of the relation key */
key_index = pg_tde_process_map_entry(rlocator, key_type, db_map_path, &offset, false);

Expand Down Expand Up @@ -1289,10 +1294,10 @@ pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEPrincipalKey *princi
strncpy(enc_rel_key_data->principal_key_id.name, principal_key->keyInfo.keyId.name, PRINCIPAL_KEY_NAME_LEN);

/* Calculate the reading position in the file. */
read_pos += (key_index * INTERNAL_KEY_LEN) + TDE_FILE_HEADER_SIZE;
read_pos += (key_index * INTERNAL_KEY_DAT_LEN) + TDE_FILE_HEADER_SIZE;

/* Check if the file has a valid key */
if ((read_pos + INTERNAL_KEY_LEN) > lseek(keydata_fd, 0, SEEK_END))
if ((read_pos + INTERNAL_KEY_DAT_LEN) > lseek(keydata_fd, 0, SEEK_END))
{
char db_keydata_path[MAXPGPATH] = {0};
pg_tde_set_db_file_paths(principal_key->keyInfo.databaseId, principal_key->keyInfo.tablespaceId, NULL, db_keydata_path);
Expand All @@ -1305,7 +1310,7 @@ pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEPrincipalKey *princi

/* Read the encrypted key */
/* TODO: pgstat_report_wait_start / pgstat_report_wait_end */
if (pg_pread(keydata_fd, &(enc_rel_key_data->internal_key), INTERNAL_KEY_LEN, read_pos) != INTERNAL_KEY_LEN)
if (pg_pread(keydata_fd, &(enc_rel_key_data->internal_key), INTERNAL_KEY_DAT_LEN, read_pos) != INTERNAL_KEY_DAT_LEN)
{
char db_keydata_path[MAXPGPATH] = {0};
pg_tde_set_db_file_paths(principal_key->keyInfo.databaseId, principal_key->keyInfo.tablespaceId, NULL, db_keydata_path);
Expand Down Expand Up @@ -1369,19 +1374,19 @@ pg_tde_get_principal_key_info(Oid dbOid, Oid spcOid)
* the tde fork file and populates cache.
*/
RelKeyData *
GetRelationKey(RelFileLocator rel, uint32 key_type)
GetRelationKey(RelFileLocator rel, uint32 key_type, bool no_map_ok)
{
RelKeyData *key;

key = pg_tde_get_key_from_cache(rel.relNumber, key_type);
if (key)
return key;

key = pg_tde_get_key_from_file(&rel, key_type);
key = pg_tde_get_key_from_file(&rel, key_type, no_map_ok);

if (key != NULL)
{
RelKeyData* cached_key = pg_tde_put_key_into_cache(rel.relNumber, key_type, key);
RelKeyData* cached_key = pg_tde_put_key_into_cache(rel.relNumber, key);
pfree(key);
return cached_key;
}
Expand All @@ -1392,19 +1397,19 @@ GetRelationKey(RelFileLocator rel, uint32 key_type)
RelKeyData *
GetSMGRRelationKey(RelFileLocator rel)
{
return GetRelationKey(rel, TDE_KEY_TYPE_SMGR);
return GetRelationKey(rel, TDE_KEY_TYPE_SMGR, true);
}

RelKeyData *
GetHeapBaiscRelationKey(RelFileLocator rel)
{
return GetRelationKey(rel, TDE_KEY_TYPE_HEAP_BASIC);
return GetRelationKey(rel, TDE_KEY_TYPE_HEAP_BASIC, false);
}

RelKeyData *
GetTdeGlobaleRelationKey(RelFileLocator rel)
{
return GetRelationKey(rel, TDE_KEY_TYPE_GLOBAL);
return GetRelationKey(rel, TDE_KEY_TYPE_GLOBAL, false);
}

/*
Expand All @@ -1425,7 +1430,7 @@ pg_tde_get_key_from_cache(RelFileNumber rel_number, uint32 key_type)
rec = tde_rel_key_cache->data+i;
if (rec != NULL &&
(rel_number == InvalidOid || (rec->rel_number == rel_number)) &&
rec->key_type & key_type)
rec->key.internal_key.rel_type & key_type)
{
return &rec->key;
}
Expand All @@ -1439,7 +1444,7 @@ pg_tde_get_key_from_cache(RelFileNumber rel_number, uint32 key_type)
* TODO: add tests.
*/
RelKeyData *
pg_tde_put_key_into_cache(RelFileNumber rel_num, uint32 key_type, RelKeyData *key)
pg_tde_put_key_into_cache(RelFileNumber rel_num, RelKeyData *key)
{
static long pageSize = 0;
RelKeyCacheRec *rec;
Expand Down Expand Up @@ -1509,7 +1514,6 @@ pg_tde_put_key_into_cache(RelFileNumber rel_num, uint32 key_type, RelKeyData *ke
rec = tde_rel_key_cache->data + tde_rel_key_cache->len;

rec->rel_number = rel_num;
rec->key_type = key_type;
memcpy(&rec->key, key, sizeof(RelKeyCacheRec));
tde_rel_key_cache->len++;

Expand Down
4 changes: 2 additions & 2 deletions src/access/pg_tde_xlog.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ tdeheap_rmgr_redo(XLogReaderState *record)
pk = &xlrec->pkInfo;

LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE);
pg_tde_write_key_map_entry(&xlrec->rlocator, xlrec->entry_type, &xlrec->relKey, pk);
pg_tde_write_key_map_entry(&xlrec->rlocator, &xlrec->relKey, pk);
LWLockRelease(tde_lwlock_enc_keys());
}
else if (info == XLOG_TDE_ADD_PRINCIPAL_KEY)
Expand Down Expand Up @@ -83,7 +83,7 @@ tdeheap_rmgr_redo(XLogReaderState *record)
RelFileLocator *xlrec = (RelFileLocator *) XLogRecGetData(record);

LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE);
pg_tde_free_key_map_entry(xlrec, offset);
pg_tde_free_key_map_entry(xlrec, MAP_ENTRY_VALID, offset);
LWLockRelease(tde_lwlock_enc_keys());
}
else
Expand Down
10 changes: 6 additions & 4 deletions src/catalog/tde_global_space.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ TDEInitGlobalKeys(const char *dir)
if (dir != NULL)
pg_tde_set_globalspace_dir(dir);

ikey = pg_tde_get_key_from_file(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), TDE_KEY_TYPE_GLOBAL);
ikey = pg_tde_get_key_from_file(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), TDE_KEY_TYPE_GLOBAL, false);

/*
* Internal Key should be in the TopMemmoryContext because of SSL
Expand All @@ -78,7 +78,7 @@ TDEInitGlobalKeys(const char *dir)
* backend. (see
* https://github.com/percona-Lab/pg_tde/pull/214#discussion_r1648998317)
*/
pg_tde_put_key_into_cache(XLOG_TDE_OID, TDE_KEY_TYPE_GLOBAL, ikey);
pg_tde_put_key_into_cache(XLOG_TDE_OID, ikey);
}
}

Expand Down Expand Up @@ -146,6 +146,8 @@ init_keys(void)

memset(&int_key, 0, sizeof(InternalKey));

int_key.rel_type = TDE_KEY_TYPE_GLOBAL;

/* Create and store an internal key for XLog */
if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN))
{
Expand All @@ -156,9 +158,9 @@ init_keys(void)
}

rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID);
rel_key_data = tde_create_rel_key(rlocator->relNumber, TDE_KEY_TYPE_GLOBAL, &int_key, &mkey->keyInfo);
rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &mkey->keyInfo);
enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator);
pg_tde_write_key_map_entry(rlocator, TDE_KEY_TYPE_GLOBAL, enc_rel_key_data, &mkey->keyInfo);
pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo);
pfree(enc_rel_key_data);
pfree(mkey);
}
Expand Down
23 changes: 15 additions & 8 deletions src/include/access/pg_tde_tdemap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,18 @@

typedef struct InternalKey
{
uint8 key[INTERNAL_KEY_LEN];
void* ctx; // TODO: shouldn't be here / written to the disk
/*
* DO NOT re-arrange fields!
* Any changes should be aligned with pg_tde_read/write_one_keydata()
*/
uint8 key[INTERNAL_KEY_LEN];
uint32 rel_type;

void* ctx;
} InternalKey;

#define INTERNAL_KEY_DAT_LEN offsetof(InternalKey, ctx)

typedef struct RelKeyData
{
TDEPrincipalKeyId principal_key_id;
Expand All @@ -37,7 +45,6 @@ typedef struct RelKeyData
typedef struct XLogRelKey
{
RelFileLocator rlocator;
uint32 entry_type;
RelKeyData relKey;
TDEPrincipalKeyInfo pkInfo;
} XLogRelKey;
Expand All @@ -46,11 +53,11 @@ extern RelKeyData *pg_tde_create_smgr_key(const RelFileLocator *newrlocator);
extern RelKeyData *pg_tde_create_global_key(const RelFileLocator *newrlocator);
extern RelKeyData *pg_tde_create_heap_basic_key(const RelFileLocator *newrlocator);
extern RelKeyData *pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type);
extern void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, uint32 entry_type, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info);
extern void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info);
extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator, uint32 key_type);
extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, uint32 key_type, off_t offset);

extern RelKeyData *GetRelationKey(RelFileLocator rel, uint32 entry_type);
extern RelKeyData *GetRelationKey(RelFileLocator rel, uint32 entry_type, bool no_map_ok);
extern RelKeyData *GetSMGRRelationKey(RelFileLocator rel);
extern RelKeyData *GetHeapBaiscRelationKey(RelFileLocator rel);
extern RelKeyData *GetTdeGlobaleRelationKey(RelFileLocator rel);
Expand All @@ -61,16 +68,16 @@ extern TDEPrincipalKeyInfo *pg_tde_get_principal_key_info(Oid dbOid, Oid spcOid)
extern bool pg_tde_save_principal_key(TDEPrincipalKeyInfo *principal_key_info);
extern bool pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_principal_key);
extern bool pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_size, char *k_file_data);
extern RelKeyData *tde_create_rel_key(RelFileNumber rel_num, uint32 key_type, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info);
extern RelKeyData *tde_create_rel_key(RelFileNumber rel_num, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info);
extern RelKeyData *tde_encrypt_rel_key(TDEPrincipalKey *principal_key, RelKeyData *rel_key_data, const RelFileLocator *rlocator);
extern RelKeyData *tde_decrypt_rel_key(TDEPrincipalKey *principal_key, RelKeyData *enc_rel_key_data, const RelFileLocator *rlocator);
extern RelKeyData *pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type);
extern RelKeyData *pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type, bool no_map_ok);
extern bool pg_tde_move_rel_key(const RelFileLocator *newrlocator, const RelFileLocator *oldrlocator);

extern void pg_tde_set_db_file_paths(Oid dbOid, Oid spcOid, char *map_path, char *keydata_path);

const char * tde_sprint_key(InternalKey *k);

extern RelKeyData *pg_tde_put_key_into_cache(RelFileNumber rel_num, uint32 key_type, RelKeyData *key);
extern RelKeyData *pg_tde_put_key_into_cache(RelFileNumber rel_num, RelKeyData *key);

#endif /*PG_TDE_MAP_H*/

0 comments on commit 5d8436a

Please sign in to comment.