From 8ddbf689d834f3d69c2c1c2f21f8363f6101935e Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Mon, 5 Aug 2024 18:33:16 +0300 Subject: [PATCH] Use common JSON API for keyring This commit replaces dependency of the keyring code on JSON backend functions with common JSON API. Usage of the backend JSON funcs prevents the code to be used by frontend tools like pg_waldump. Besides it requiers extra conversions to Datums and DirectFunctionCall-s. --- Makefile.in | 1 + meson.build | 1 + src/catalog/tde_keyring.c | 81 +++-- src/catalog/tde_keyring_parse_opts.c | 458 +++++++++++++++++++++++++++ src/common/pg_tde_utils.c | 138 -------- src/include/catalog/tde_keyring.h | 3 + src/include/common/pg_tde_utils.h | 4 - src/keyring/keyring_vault.c | 159 ++++++++-- 8 files changed, 640 insertions(+), 205 deletions(-) create mode 100644 src/catalog/tde_keyring_parse_opts.c diff --git a/Makefile.in b/Makefile.in index 5b4d98bf..db1ecfb8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -44,6 +44,7 @@ src/keyring/keyring_vault.o \ src/keyring/keyring_api.o \ src/catalog/tde_global_space.o \ src/catalog/tde_keyring.o \ +src/catalog/tde_keyring_parse_opts.o \ src/catalog/tde_principal_key.o \ src/common/pg_tde_shmem.o \ src/common/pg_tde_utils.o \ diff --git a/meson.build b/meson.build index 6445442d..ec687d58 100644 --- a/meson.build +++ b/meson.build @@ -43,6 +43,7 @@ pg_tde_sources = files( 'src/catalog/tde_global_space.c', 'src/catalog/tde_keyring.c', + 'src/catalog/tde_keyring_parse_opts.c', 'src/catalog/tde_principal_key.c', 'src/common/pg_tde_shmem.c', 'src/common/pg_tde_utils.c', diff --git a/src/catalog/tde_keyring.c b/src/catalog/tde_keyring.c index df3509a5..4b014f3f 100644 --- a/src/catalog/tde_keyring.c +++ b/src/catalog/tde_keyring.c @@ -40,21 +40,6 @@ PG_FUNCTION_INFO_V1(pg_tde_list_all_key_providers); Datum pg_tde_list_all_key_providers(PG_FUNCTION_ARGS); #define PG_TDE_KEYRING_FILENAME "pg_tde_keyrings" -/* - * These token must be exactly same as defined in - * pg_tde_add_key_provider_vault_v2 SQL interface - */ -#define VAULTV2_KEYRING_TOKEN_KEY "token" -#define VAULTV2_KEYRING_URL_KEY "url" -#define VAULTV2_KEYRING_MOUNT_PATH_KEY "mountPath" -#define VAULTV2_KEYRING_CA_PATH_KEY "caPath" - -/* - * These token must be exactly same as defined in - * pg_tde_add_key_provider_file SQL interface - */ -#define FILE_KEYRING_PATH_KEY "path" -#define FILE_KEYRING_TYPE_KEY "type" #define PG_TDE_LIST_PROVIDERS_COLS 4 typedef enum ProviderScanType @@ -67,9 +52,9 @@ typedef enum ProviderScanType static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid, Oid spcOid); -static FileKeyring *load_file_keyring_provider_options(Datum keyring_options); -static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, Datum keyring_options); -static VaultV2Keyring *load_vaultV2_keyring_provider_options(Datum keyring_options); +static FileKeyring *load_file_keyring_provider_options(char *keyring_options); +static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, char *keyring_options); +static VaultV2Keyring *load_vaultV2_keyring_provider_options(char *keyring_options); static void debug_print_kerying(GenericKeyring *keyring); static char *get_keyring_infofile_path(char *resPath, Oid dbOid, Oid spcOid); static void key_provider_startup_cleanup(int tde_tbl_count, XLogExtensionInstall *ext_info, bool redo, void *arg); @@ -165,12 +150,9 @@ get_keyring_provider_typename(ProviderType p_type) static GenericKeyring *load_keyring_provider_from_record(KeyringProvideRecord *provider) { - Datum option_datum; GenericKeyring *keyring = NULL; - option_datum = CStringGetTextDatum(provider->options); - - keyring = load_keyring_provider_options(provider->provider_type, option_datum); + keyring = load_keyring_provider_options(provider->provider_type, provider->options); if (keyring) { keyring->key_id = provider->provider_id; @@ -222,7 +204,7 @@ GetKeyProviderByID(int provider_id, Oid dbOid, Oid spcOid) } static GenericKeyring * -load_keyring_provider_options(ProviderType provider_type, Datum keyring_options) +load_keyring_provider_options(ProviderType provider_type, char *keyring_options) { switch (provider_type) { @@ -239,44 +221,55 @@ load_keyring_provider_options(ProviderType provider_type, Datum keyring_options) } static FileKeyring * -load_file_keyring_provider_options(Datum keyring_options) +load_file_keyring_provider_options(char *keyring_options) { - const char* file_path = extract_json_option_value(keyring_options, FILE_KEYRING_PATH_KEY); - FileKeyring *file_keyring = palloc0(sizeof(FileKeyring)); - - if(file_path == NULL) + FileKeyring *file_keyring = palloc0(sizeof(FileKeyring)); + + file_keyring->keyring.type = FILE_KEY_PROVIDER; + + if (!ParseKeyringJSONOptions(FILE_KEY_PROVIDER, file_keyring, + keyring_options, strlen(keyring_options))) { - ereport(DEBUG2, + return NULL; + } + + if(strlen(file_keyring->file_name) == 0) + { + ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("file path is missing in the keyring options"))); return NULL; } - file_keyring->keyring.type = FILE_KEY_PROVIDER; - strncpy(file_keyring->file_name, file_path, sizeof(file_keyring->file_name)); return file_keyring; } static VaultV2Keyring * -load_vaultV2_keyring_provider_options(Datum keyring_options) +load_vaultV2_keyring_provider_options(char *keyring_options) { - VaultV2Keyring *vaultV2_keyring = palloc0(sizeof(VaultV2Keyring)); - const char* token = extract_json_option_value(keyring_options, VAULTV2_KEYRING_TOKEN_KEY); - const char* url = extract_json_option_value(keyring_options, VAULTV2_KEYRING_URL_KEY); - const char* mount_path = extract_json_option_value(keyring_options, VAULTV2_KEYRING_MOUNT_PATH_KEY); - const char* ca_path = extract_json_option_value(keyring_options, VAULTV2_KEYRING_CA_PATH_KEY); + VaultV2Keyring *vaultV2_keyring = palloc0(sizeof(VaultV2Keyring)); - if(token == NULL || url == NULL || mount_path == NULL) + vaultV2_keyring->keyring.type = VAULT_V2_KEY_PROVIDER; + + if (!ParseKeyringJSONOptions(VAULT_V2_KEY_PROVIDER, vaultV2_keyring, + keyring_options, strlen(keyring_options))) { - /* TODO: report error */ + return NULL; + } + + if(strlen(vaultV2_keyring->vault_token) == 0 || + strlen(vaultV2_keyring->vault_url) == 0 || + strlen(vaultV2_keyring->vault_mount_path) == 0) + { + ereport(WARNING, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("missing in the keyring options:%s%s%s", + !vaultV2_keyring->vault_token ? " token" : "", + !vaultV2_keyring->vault_url ? " url" : "", + !vaultV2_keyring->vault_mount_path ? " mountPath" : ""))); return NULL; } - vaultV2_keyring->keyring.type = VAULT_V2_KEY_PROVIDER; - strncpy(vaultV2_keyring->vault_token, token, sizeof(vaultV2_keyring->vault_token)); - strncpy(vaultV2_keyring->vault_url, url, sizeof(vaultV2_keyring->vault_url)); - strncpy(vaultV2_keyring->vault_mount_path, mount_path, sizeof(vaultV2_keyring->vault_mount_path)); - strncpy(vaultV2_keyring->vault_ca_path, ca_path ? ca_path : "", sizeof(vaultV2_keyring->vault_ca_path)); return vaultV2_keyring; } diff --git a/src/catalog/tde_keyring_parse_opts.c b/src/catalog/tde_keyring_parse_opts.c new file mode 100644 index 00000000..c3e96dca --- /dev/null +++ b/src/catalog/tde_keyring_parse_opts.c @@ -0,0 +1,458 @@ +/*------------------------------------------------------------------------- + * + * tde_keyring_parse_opts.c + * Parser routines for the keyring JSON options + * + * TODO: add unit tests !!! + * + * Each value in the JSON document can be either scalar (string) - a value itself + * or a reference to the external object that contains the value. Though the top + * level field "type" can be only scalar. + * + * Examples: + * {"type" : "file", "path" : "/tmp/keyring_data_file"} + * {"type" : "file", "path" : {"type" : "file", "path" : "/tmp/datafile-location"}} + * in the latter one, /tmp/datafile-location contains not keyring data but the + * location of such. + * + * A field type can be "file", in this case, we expect "path" field. Or "remote", + * when "url" field is expected. + * + * IDENTIFICATION + * contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "common/jsonapi.h" +#include "mb/pg_wchar.h" +#include "storage/fd.h" + +#include "catalog/tde_keyring.h" +#include "keyring/keyring_curl.h" + +#include + +#define MAX_CONFIG_FILE_DATA_LENGTH 1024 + +/* + * JSON parser state + */ + +typedef enum JsonKeringSemState +{ + JK_EXPECT_TOP_FIELD, + JK_EXPECT_EXTERN_VAL, +} JsonKeringSemState; + +#define KEYRING_REMOTE_FIELD_TYPE "remote" +#define KEYRING_FILE_FIELD_TYPE "file" + +typedef enum JsonKeyringField +{ + JK_FIELD_UNKNOWN, + + JK_KRING_TYPE, + + JK_FIELD_TYPE, + JK_REMOTE_URL, + JK_FIELD_PATH, + + JF_FILE_PATH, + + JK_VAULT_TOKEN, + JK_VAULT_URL, + JK_VAULT_MOUNT_PATH, + JK_VAULT_CA_PATH, + + /* must be the last */ + JK_FIELDS_TOTAL +} JsonKeyringField; + +static const char *JK_FIELD_NAMES[JK_FIELDS_TOTAL] = { + [JK_FIELD_UNKNOWN] = "unknownField", + [JK_KRING_TYPE] = "type", + [JK_FIELD_TYPE] = "type", + [JK_REMOTE_URL] = "url", + [JK_FIELD_PATH] = "path", + + /* + * These values should match pg_tde_add_key_provider_vault_v2 and + * pg_tde_add_key_provider_file SQL interfaces + */ + [JF_FILE_PATH] = "path", + [JK_VAULT_TOKEN] = "token", + [JK_VAULT_URL] = "url", + [JK_VAULT_MOUNT_PATH] = "mountPath", + [JK_VAULT_CA_PATH] = "caPath", +}; + +#define MAX_JSON_DEPTH 64 +typedef struct JsonKeyringState +{ + ProviderType provider_type; + + /* + * Caller's options to be set from JSON values. Expected either + * `VaultV2Keyring` or `FileKeyring` + */ + void *provider_opts; + + /* + * A field hierarchy of the current branch, field[level] is the current + * one, field[level-1] is the parent and so on. We need to track parent + * fields because of the external values + */ + JsonKeyringField field[MAX_JSON_DEPTH]; + JsonKeringSemState state; + int level; + + /* + * The rest of the scalar fields might be in the JSON document but has no + * direct value for the caller. Although we need them for the values + * extraction or state tracking. + */ + char *kring_type; + char *field_type; + char *extern_url; + char *extern_path; +} JsonKeyringState; + +static JsonParseErrorType json_kring_scalar(void *state, char *token, JsonTokenType tokentype); +static JsonParseErrorType json_kring_object_field_start(void *state, char *fname, bool isnull); +static JsonParseErrorType json_kring_object_start(void *state); +static JsonParseErrorType json_kring_object_end(void *state); + +static void json_kring_assign_scalar(JsonKeyringState * parse, JsonKeyringField field, char *value); +static char *get_remote_kring_value(const char *url, const char *field_name); +static char *get_file_kring_value(const char *path, const char *field_name); + + +/* + * Parses json input for the given provider type and sets the provided options + * out_opts should be a palloc'd `VaultV2Keyring` or `FileKeyring` struct as the + * respective option values will be mem copied into it. + * Returns `true` if parsing succeded and `false` otherwise. +*/ +bool +ParseKeyringJSONOptions(ProviderType provider_type, void *out_opts, char *in_buf, int buf_len) +{ + JsonLexContext *jlex; + JsonKeyringState parse = {0}; + JsonSemAction sem; + JsonParseErrorType jerr; + + /* Set up parsing context and initial semantic state */ + parse.provider_type = provider_type; + parse.provider_opts = out_opts; + parse.level = -1; + parse.state = JK_EXPECT_TOP_FIELD; + memset(parse.field, 0, MAX_JSON_DEPTH * sizeof(JsonKeyringField)); + +#if PG_VERSION_NUM >= 170000 + jlex = makeJsonLexContextCstringLen(NULL, in_buf, buf_len, PG_UTF8, true); +#else + jlex = makeJsonLexContextCstringLen(in_buf, buf_len, PG_UTF8, true); +#endif + + /* + * Set up semantic actions. The function below will be called when the + * parser reaches the appropriate state. See comments on the functions. + */ + sem.semstate = &parse; + sem.object_start = json_kring_object_start; + sem.object_end = json_kring_object_end; + sem.array_start = NULL; + sem.array_end = NULL; + sem.object_field_start = json_kring_object_field_start; + sem.object_field_end = NULL; + sem.array_element_start = NULL; + sem.array_element_end = NULL; + sem.scalar = json_kring_scalar; + + /* Run the parser */ + jerr = pg_parse_json(jlex, &sem); + if (jerr != JSON_SUCCESS) + { + ereport(WARNING, + (errmsg("parsing of keyring options failed: %s", + json_errdetail(jerr, jlex)))); + + } +#if PG_VERSION_NUM >= 170000 + freeJsonLexContext(jlex); +#endif + + return jerr == JSON_SUCCESS; +} + +/* + * JSON parser semantic actions +*/ + +/* + * Invoked at the start of each object in the JSON document. + * + * Every new object increases the level of nesting as the whole document is the + * object itself (level 0) and every next one means going deeper into nesting. + * + * On the top level, we expect either scalar (string) values or objects referencing + * the external value of the field. Hence, if we are on level 1, we expect an + * "external field object" e.g. ({"type" : "remote", "url" : "http://localhost:8888/hello"}) + */ +static JsonParseErrorType +json_kring_object_start(void *state) +{ + JsonKeyringState *parse = state; + + if (MAX_JSON_DEPTH == ++parse->level) + { + elog(WARNING, "reached max depth of JSON nesting"); + return JSON_SEM_ACTION_FAILED; + } + + switch (parse->level) + { + case 0: + parse->state = JK_EXPECT_TOP_FIELD; + break; + case 1: + parse->state = JK_EXPECT_EXTERN_VAL; + break; + } + + return JSON_SUCCESS; +} + +/* + * Invoked at the start of each object in the JSON document. + * + * First, it means we are going back to the higher level. Plus, if it was the + * level 1, we expect only external objects there, which means we have all + * the necessary info to extract the value and assign the result to the + * appropriate (parent) field. + */ +static JsonParseErrorType +json_kring_object_end(void *state) +{ + JsonKeyringState *parse = state; + + /* + * we're done with the nested object and if it's an external field, the + * value should be extracted and assigned to the parent "field". for + * example if : "field" : {"type" : "remote", "url" : + * "http://localhost:8888/hello"} or "field" : {"type" : "file", "path" : + * "/tmp/datafile-location"} the "field"'s value should be the content of + * "path" or "url" respectively + */ + if (parse->level == 1) + { + if (parse->state == JK_EXPECT_EXTERN_VAL) + { + JsonKeyringField parent_field = parse->field[0]; + + char *value = NULL; + + if (strcmp(parse->field_type, KEYRING_REMOTE_FIELD_TYPE) == 0) + value = get_remote_kring_value(parse->extern_url, JK_FIELD_NAMES[parent_field]); + if (strcmp(parse->field_type, KEYRING_FILE_FIELD_TYPE) == 0) + value = get_file_kring_value(parse->extern_path, JK_FIELD_NAMES[parent_field]); + + json_kring_assign_scalar(parse, parent_field, value); + } + + parse->state = JK_EXPECT_TOP_FIELD; + } + + parse->level--; + + return JSON_SUCCESS; +} + +/* + * Invoked at the start of each object field in the JSON document. + * + * Based on the given field name and the semantic state (we expect a top-level + * field or an external object) we set the state so that when we get the value, + * we know what is it and where to assign it. + */ +static JsonParseErrorType +json_kring_object_field_start(void *state, char *fname, bool isnull) +{ + JsonKeyringState *parse = state; + JsonKeyringField *field; + + Assert(parse->level >= 0); + + field = &parse->field[parse->level]; + + switch (parse->state) + { + case JK_EXPECT_TOP_FIELD: + + /* + * On the top level, "type" stores a keyring type and this field + * is common for all keyrings. The rest of the fields depend on + * the keyring type. + */ + if (strcmp(fname, JK_FIELD_NAMES[JK_KRING_TYPE]) == 0) + { + *field = JK_KRING_TYPE; + break; + } + switch (parse->provider_type) + { + case FILE_KEY_PROVIDER: + if (strcmp(fname, JK_FIELD_NAMES[JF_FILE_PATH]) == 0) + *field = JF_FILE_PATH; + else + { + *field = JK_FIELD_UNKNOWN; + elog(DEBUG1, "parse file keyring config: unexpected field %s", fname); + } + break; + + case VAULT_V2_KEY_PROVIDER: + if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_TOKEN]) == 0) + *field = JK_VAULT_TOKEN; + else if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_URL]) == 0) + *field = JK_VAULT_URL; + else if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_MOUNT_PATH]) == 0) + *field = JK_VAULT_MOUNT_PATH; + else if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_CA_PATH]) == 0) + *field = JK_VAULT_CA_PATH; + else + { + *field = JK_FIELD_UNKNOWN; + elog(DEBUG1, "parse json keyring config: unexpected field %s", fname); + } + break; + } + break; + + case JK_EXPECT_EXTERN_VAL: + if (strcmp(fname, JK_FIELD_NAMES[JK_FIELD_TYPE]) == 0) + *field = JK_FIELD_TYPE; + else if (strcmp(fname, JK_FIELD_NAMES[JK_REMOTE_URL]) == 0) + *field = JK_REMOTE_URL; + else if (strcmp(fname, JK_FIELD_NAMES[JK_FIELD_PATH]) == 0) + *field = JK_FIELD_PATH; + break; + } + + return JSON_SUCCESS; +} + +/* + * Invoked at the start of each scalar in the JSON document. + * + * We have only the string value of the field. And rely on the state set by + * `json_kring_object_field_start` for defining what the field is. + */ +static JsonParseErrorType +json_kring_scalar(void *state, char *token, JsonTokenType tokentype) +{ + JsonKeyringState *parse = state; + + json_kring_assign_scalar(parse, parse->field[parse->level], token); + + return JSON_SUCCESS; +} + +static void +json_kring_assign_scalar(JsonKeyringState * parse, JsonKeyringField field, char *value) +{ + VaultV2Keyring *vault = parse->provider_opts; + FileKeyring *file = parse->provider_opts; + + switch (field) + { + case JK_KRING_TYPE: + parse->kring_type = value; + break; + + case JK_FIELD_TYPE: + parse->field_type = value; + break; + case JK_REMOTE_URL: + parse->extern_url = value; + break; + case JK_FIELD_PATH: + parse->extern_path = value; + break; + + case JF_FILE_PATH: + strncpy(file->file_name, value, sizeof(file->file_name)); + break; + + case JK_VAULT_TOKEN: + strncpy(vault->vault_token, value, sizeof(vault->vault_token)); + break; + case JK_VAULT_URL: + strncpy(vault->vault_url, value, sizeof(vault->vault_url)); + break; + case JK_VAULT_MOUNT_PATH: + strncpy(vault->vault_mount_path, value, sizeof(vault->vault_mount_path)); + break; + case JK_VAULT_CA_PATH: + strncpy(vault->vault_ca_path, value, sizeof(vault->vault_ca_path)); + break; + + default: + elog(DEBUG1, "json keyring: unexpected scalar field %d", field); + Assert(0); + break; + } +} + +static char * +get_remote_kring_value(const char *url, const char *field_name) +{ + long httpCode; + CurlString outStr; + + /* TODO: we never pfree it */ + outStr.ptr = palloc0(1); + outStr.len = 0; + + if (!curlSetupSession(url, NULL, &outStr)) + { + elog(WARNING, "CURL error for remote object %s", field_name); + return NULL; + } + if (curl_easy_perform(keyringCurl) != CURLE_OK) + { + elog(WARNING, "HTTP request error for remote object %s", field_name); + return NULL; + } + if (curl_easy_getinfo(keyringCurl, CURLINFO_RESPONSE_CODE, &httpCode) != CURLE_OK) + { + elog(WARNING, "HTTP error for remote object %s, HTTP code %li", field_name, httpCode); + return NULL; + } + + return outStr.ptr; +} + +static char * +get_file_kring_value(const char *path, const char *field_name) +{ + int fd = -1; + char *val; + + fd = BasicOpenFile(path, O_RDONLY); + if (fd < 0) + { + elog(WARNING, "filed to open file %s for %s", path, field_name); + return NULL; + } + + /* TODO: we never pfree it */ + val = palloc0(MAX_CONFIG_FILE_DATA_LENGTH); + pg_pread(fd, val, MAX_CONFIG_FILE_DATA_LENGTH, 0); + val[strcspn(val, "\r\n")] = 0; + + close(fd); + return val; +} diff --git a/src/common/pg_tde_utils.c b/src/common/pg_tde_utils.c index 7bd4030a..b3dfd370 100644 --- a/src/common/pg_tde_utils.c +++ b/src/common/pg_tde_utils.c @@ -9,24 +9,13 @@ *------------------------------------------------------------------------- */ -#define MAX_CONFIG_FILE_DATA_LENGTH 1024 - #include "postgres.h" #include "access/genam.h" #include "access/heapam.h" -#include "access/htup_details.h" -#include "catalog/pg_class.h" -#include "catalog/pg_am.h" #include "catalog/pg_tablespace_d.h" -#include "utils/fmgroids.h" -#include "utils/rel.h" #include "utils/snapmgr.h" #include "commands/defrem.h" #include "common/pg_tde_utils.h" -#include "fmgr.h" -#include "keyring/keyring_curl.h" -#include "utils/builtins.h" -#include "unistd.h" Oid get_tde_basic_table_am_oid(void) @@ -90,133 +79,6 @@ get_tde_tables_count(void) return count; } - -const char * -extract_json_cstr(Datum json, const char* field_name) -{ - Datum field = DirectFunctionCall2(json_object_field_text, json, CStringGetTextDatum(field_name)); - const char* cstr = TextDatumGetCString(field); - - return cstr; -} - -const char * -extract_json_option_value(Datum top_json, const char* field_name) -{ - Datum field; - Datum field_type; - const char* field_type_cstr; - - field = DirectFunctionCall2(json_object_field, top_json, CStringGetTextDatum(field_name)); - - field_type = DirectFunctionCall1(json_typeof, field); - field_type_cstr = TextDatumGetCString(field_type); - - if(field_type_cstr == NULL) - { - return NULL; - } - - if(strcmp(field_type_cstr, "string") == 0) - { - return extract_json_cstr(top_json, field_name); - } - - if(strcmp(field_type_cstr, "object") != 0) - { - elog(ERROR, "Unsupported type for object %s: %s", field_name, field_type_cstr); - return NULL; - } - - /* Now it is definitely an object */ - - { - const char* type_cstr = extract_json_cstr(field, "type"); - - if(type_cstr == NULL) - { - elog(ERROR, "Missing type property for remote object %s", field_name); - return NULL; - } - - if(strncmp("remote", type_cstr, 7) == 0) - { - const char* url_cstr = extract_json_cstr(field, "url"); - - long httpCode; - CurlString outStr; - - if(url_cstr == NULL) - { - elog(ERROR, "Missing url property for remote object %s", field_name); - return NULL; - } - - /* Http request should return the value of the option */ - outStr.ptr = palloc0(1); - outStr.len = 0; - if(!curlSetupSession(url_cstr, NULL, &outStr)) - { - elog(ERROR, "CURL error for remote object %s", field_name); - return NULL; - } - if(curl_easy_perform(keyringCurl) != CURLE_OK) - { - elog(ERROR, "HTTP request error for remote object %s", field_name); - return NULL; - } - if(curl_easy_getinfo(keyringCurl, CURLINFO_RESPONSE_CODE, &httpCode) != CURLE_OK) - { - elog(ERROR, "HTTP error for remote object %s, HTTP code %li", field_name, httpCode); - return NULL; - } -#if KEYRING_DEBUG - elog(DEBUG2, "HTTP response for config [%s] '%s'", field_name, outStr->ptr != NULL ? outStr->ptr : ""); -#endif - return outStr.ptr; - } - - if(strncmp("file", type_cstr, 5) == 0) - { - const char* path_cstr = extract_json_cstr(field, "path"); - FILE* f; - char* out; - - if(path_cstr == NULL) - { - elog(ERROR, "Missing path property for file object %s", field_name); - return NULL; - } - - if(access(path_cstr, R_OK) != 0) - { - elog(ERROR, "The file referenced by %s doesn't exists, or is not readable to postgres: %s", field_name, path_cstr); - return NULL; - } - - f = fopen(path_cstr, "r"); - - if(!f) - { - elog(ERROR, "The file referenced by %s doesn't exists, or is not readable to postgres: %s", field_name, path_cstr); - return NULL; - } - - /* File should contain the value for the option */ - out = palloc(MAX_CONFIG_FILE_DATA_LENGTH); - fgets(out, MAX_CONFIG_FILE_DATA_LENGTH, f); - out[strcspn(out, "\r\n")] = 0; - - fclose(f); - - return out; - } - - elog(ERROR, "Unknown type for object %s: %s", field_name, type_cstr); - return NULL; - } -} - /* returns the palloc'd string */ char * pg_tde_get_tde_file_dir(Oid dbOid, Oid spcOid) diff --git a/src/include/catalog/tde_keyring.h b/src/include/catalog/tde_keyring.h index 2256b495..2d3a1675 100644 --- a/src/include/catalog/tde_keyring.h +++ b/src/include/catalog/tde_keyring.h @@ -80,4 +80,7 @@ extern void cleanup_key_provider_info(Oid databaseId, Oid tablespaceId); extern void InitializeKeyProviderInfo(void); extern uint32 save_new_key_provider_info(KeyringProvideRecord *provider, Oid databaseId, Oid tablespaceId, bool recovery); extern uint32 redo_key_provider_info(KeyringProviderXLRecord *xlrec); + +extern bool ParseKeyringJSONOptions(ProviderType provider_type, void *out_opts, + char *in_buf, int buf_len); #endif /*TDE_KEYRING_H*/ diff --git a/src/include/common/pg_tde_utils.h b/src/include/common/pg_tde_utils.h index 82a9c28b..4f80e5b2 100644 --- a/src/include/common/pg_tde_utils.h +++ b/src/include/common/pg_tde_utils.h @@ -10,15 +10,11 @@ #include "postgres.h" #include "nodes/pg_list.h" -#include "common/relpath.h" extern Oid get_tde_basic_table_am_oid(void); extern Oid get_tde_table_am_oid(void); extern List *get_all_tde_tables(void); extern int get_tde_tables_count(void); -extern const char *extract_json_cstr(Datum json, const char* field_name); -const char *extract_json_option_value(Datum top_json, const char* field_name); - extern char *pg_tde_get_tde_file_dir(Oid dbOid, Oid spcOid); #endif /*PG_TDE_UTILS_H*/ diff --git a/src/keyring/keyring_vault.c b/src/keyring/keyring_vault.c index ef38fa70..648b7f81 100644 --- a/src/keyring/keyring_vault.c +++ b/src/keyring/keyring_vault.c @@ -16,8 +16,8 @@ #include "keyring/keyring_curl.h" #include "keyring/keyring_api.h" #include "pg_tde_defines.h" -#include "fmgr.h" -#include "utils/fmgrprotos.h" +#include "common/jsonapi.h" +#include "mb/pg_wchar.h" #include "utils/builtins.h" #include @@ -26,6 +26,39 @@ #include "common/base64.h" +/* + * JSON parser state +*/ + +typedef enum +{ + JRESP_EXPECT_TOP_DATA, + JRESP_EXPECT_DATA, + JRESP_EXPECT_KEY +} JsonVaultRespSemState; + +typedef enum +{ + JRESP_F_UNUSED, + + JRESP_F_KEY +} JsonVaultRespField; + +typedef struct JsonVaultRespState +{ + JsonVaultRespSemState state; + JsonVaultRespField field; + int level; + + char *key; +} JsonVaultRespState; + +static JsonParseErrorType json_resp_object_start(void *state); +static JsonParseErrorType json_resp_object_end(void *state); +static JsonParseErrorType json_resp_scalar(void *state, char *token, JsonTokenType tokentype); +static JsonParseErrorType json_resp_object_field_start(void *state, char *fname, bool isnull); +static JsonParseErrorType parse_json_response(JsonVaultRespState *parse, JsonLexContext *lex); + struct curl_slist *curlList = NULL; static bool curl_setup_token(VaultV2Keyring *keyring); @@ -178,10 +211,9 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error, char url[VAULT_URL_MAX_LEN]; CurlString str; long httpCode = 0; - - Datum dataJson; - Datum data2Json; - Datum keyJson; + JsonParseErrorType json_error; + JsonLexContext *jlex = NULL; + JsonVaultRespState parse; const char* responseKey; @@ -213,25 +245,26 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error, goto cleanup; } - PG_TRY(); - { - dataJson = DirectFunctionCall2(json_object_field_text, CStringGetTextDatum(str.ptr), CStringGetTextDatum("data")); - data2Json = DirectFunctionCall2(json_object_field_text, dataJson, CStringGetTextDatum("data")); - keyJson = DirectFunctionCall2(json_object_field_text, data2Json, CStringGetTextDatum("key")); - responseKey = TextDatumGetCString(keyJson); - } - PG_CATCH(); +#if PG_VERSION_NUM < 170000 + jlex = makeJsonLexContextCstringLen(str.ptr, str.len, PG_UTF8, true); +#else + jlex = makeJsonLexContextCstringLen(NULL, str.ptr, str.len, PG_UTF8, true); +#endif + json_error = parse_json_response(&parse, jlex); + + if (json_error != JSON_SUCCESS) { *return_code = KEYRING_CODE_INVALID_RESPONSE; ereport(throw_error ? ERROR : WARNING, - (errmsg("HTTP(S) request to keyring provider \"%s\" returned incorrect JSON response", - vault_keyring->keyring.provider_name))); + (errmsg("HTTP(S) request to keyring provider \"%s\" returned incorrect JSON: %s", + vault_keyring->keyring.provider_name, json_errdetail(json_error, jlex)))); goto cleanup; } - PG_END_TRY(); + + responseKey = parse.key; #if KEYRING_DEBUG - elog(DEBUG1, "Retrieved base64 key: %s", response_key); + elog(DEBUG1, "Retrieved base64 key: %s", responseKey); #endif key = palloc(sizeof(keyInfo)); @@ -249,6 +282,94 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error, } cleanup: - if(str.ptr != NULL) pfree(str.ptr); + if(str.ptr != NULL) + pfree(str.ptr); +#if PG_VERSION_NUM >= 170000 + if (jlex != NULL) + freeJsonLexContext(jlex); +#endif return key; } + +/* + * JSON parser routines +*/ + +static JsonParseErrorType +parse_json_response(JsonVaultRespState *parse, JsonLexContext *lex) +{ + JsonSemAction sem; + + parse->state = JRESP_EXPECT_TOP_DATA; + parse->level = -1; + parse->field = JRESP_F_UNUSED; + parse->key = NULL; + + sem.semstate = parse; + sem.object_start = json_resp_object_start; + sem.object_end = json_resp_object_end; + sem.array_start = NULL; + sem.array_end = NULL; + sem.object_field_start = json_resp_object_field_start; + sem.object_field_end = NULL; + sem.array_element_start = NULL; + sem.array_element_end = NULL; + sem.scalar = json_resp_scalar; + + return pg_parse_json(lex, &sem); +} + +static JsonParseErrorType +json_resp_object_start(void *state) +{ + ((JsonVaultRespState *) state)->level++; + + return JSON_SUCCESS; +} + +static JsonParseErrorType +json_resp_object_end(void *state) +{ + ((JsonVaultRespState *) state)->level--; + + return JSON_SUCCESS; +} + +static JsonParseErrorType +json_resp_scalar(void *state, char *token, JsonTokenType tokentype) +{ + JsonVaultRespState *parse = state; + + switch (parse->field) + { + case JRESP_F_KEY: + parse->key = token; + parse->field = JRESP_F_UNUSED; + break; + } + return JSON_SUCCESS; +} + +static JsonParseErrorType +json_resp_object_field_start(void *state, char *fname, bool isnull) +{ + JsonVaultRespState *parse = state; + + switch (parse->state) + { + case JRESP_EXPECT_TOP_DATA: + if (strcmp(fname, "data") == 0 && parse->level == 0) + parse->state = JRESP_EXPECT_DATA; + break; + case JRESP_EXPECT_DATA: + if (strcmp(fname, "data") == 0 && parse->level == 1) + parse->state = JRESP_EXPECT_KEY; + break; + case JRESP_EXPECT_KEY: + if (strcmp(fname, "key") == 0 && parse->level == 2) + parse->field = JRESP_F_KEY; + break; + } + + return JSON_SUCCESS; +}