diff --git a/Makefile.in b/Makefile.in index 5b4d98bf..db1ecfb8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -44,6 +44,7 @@ src/keyring/keyring_vault.o \ src/keyring/keyring_api.o \ src/catalog/tde_global_space.o \ src/catalog/tde_keyring.o \ +src/catalog/tde_keyring_parse_opts.o \ src/catalog/tde_principal_key.o \ src/common/pg_tde_shmem.o \ src/common/pg_tde_utils.o \ diff --git a/meson.build b/meson.build index 6445442d..ec687d58 100644 --- a/meson.build +++ b/meson.build @@ -43,6 +43,7 @@ pg_tde_sources = files( 'src/catalog/tde_global_space.c', 'src/catalog/tde_keyring.c', + 'src/catalog/tde_keyring_parse_opts.c', 'src/catalog/tde_principal_key.c', 'src/common/pg_tde_shmem.c', 'src/common/pg_tde_utils.c', diff --git a/src/catalog/tde_keyring.c b/src/catalog/tde_keyring.c index df3509a5..4b014f3f 100644 --- a/src/catalog/tde_keyring.c +++ b/src/catalog/tde_keyring.c @@ -40,21 +40,6 @@ PG_FUNCTION_INFO_V1(pg_tde_list_all_key_providers); Datum pg_tde_list_all_key_providers(PG_FUNCTION_ARGS); #define PG_TDE_KEYRING_FILENAME "pg_tde_keyrings" -/* - * These token must be exactly same as defined in - * pg_tde_add_key_provider_vault_v2 SQL interface - */ -#define VAULTV2_KEYRING_TOKEN_KEY "token" -#define VAULTV2_KEYRING_URL_KEY "url" -#define VAULTV2_KEYRING_MOUNT_PATH_KEY "mountPath" -#define VAULTV2_KEYRING_CA_PATH_KEY "caPath" - -/* - * These token must be exactly same as defined in - * pg_tde_add_key_provider_file SQL interface - */ -#define FILE_KEYRING_PATH_KEY "path" -#define FILE_KEYRING_TYPE_KEY "type" #define PG_TDE_LIST_PROVIDERS_COLS 4 typedef enum ProviderScanType @@ -67,9 +52,9 @@ typedef enum ProviderScanType static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid, Oid spcOid); -static FileKeyring *load_file_keyring_provider_options(Datum keyring_options); -static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, Datum keyring_options); -static VaultV2Keyring *load_vaultV2_keyring_provider_options(Datum keyring_options); +static FileKeyring *load_file_keyring_provider_options(char *keyring_options); +static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, char *keyring_options); +static VaultV2Keyring *load_vaultV2_keyring_provider_options(char *keyring_options); static void debug_print_kerying(GenericKeyring *keyring); static char *get_keyring_infofile_path(char *resPath, Oid dbOid, Oid spcOid); static void key_provider_startup_cleanup(int tde_tbl_count, XLogExtensionInstall *ext_info, bool redo, void *arg); @@ -165,12 +150,9 @@ get_keyring_provider_typename(ProviderType p_type) static GenericKeyring *load_keyring_provider_from_record(KeyringProvideRecord *provider) { - Datum option_datum; GenericKeyring *keyring = NULL; - option_datum = CStringGetTextDatum(provider->options); - - keyring = load_keyring_provider_options(provider->provider_type, option_datum); + keyring = load_keyring_provider_options(provider->provider_type, provider->options); if (keyring) { keyring->key_id = provider->provider_id; @@ -222,7 +204,7 @@ GetKeyProviderByID(int provider_id, Oid dbOid, Oid spcOid) } static GenericKeyring * -load_keyring_provider_options(ProviderType provider_type, Datum keyring_options) +load_keyring_provider_options(ProviderType provider_type, char *keyring_options) { switch (provider_type) { @@ -239,44 +221,55 @@ load_keyring_provider_options(ProviderType provider_type, Datum keyring_options) } static FileKeyring * -load_file_keyring_provider_options(Datum keyring_options) +load_file_keyring_provider_options(char *keyring_options) { - const char* file_path = extract_json_option_value(keyring_options, FILE_KEYRING_PATH_KEY); - FileKeyring *file_keyring = palloc0(sizeof(FileKeyring)); - - if(file_path == NULL) + FileKeyring *file_keyring = palloc0(sizeof(FileKeyring)); + + file_keyring->keyring.type = FILE_KEY_PROVIDER; + + if (!ParseKeyringJSONOptions(FILE_KEY_PROVIDER, file_keyring, + keyring_options, strlen(keyring_options))) { - ereport(DEBUG2, + return NULL; + } + + if(strlen(file_keyring->file_name) == 0) + { + ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("file path is missing in the keyring options"))); return NULL; } - file_keyring->keyring.type = FILE_KEY_PROVIDER; - strncpy(file_keyring->file_name, file_path, sizeof(file_keyring->file_name)); return file_keyring; } static VaultV2Keyring * -load_vaultV2_keyring_provider_options(Datum keyring_options) +load_vaultV2_keyring_provider_options(char *keyring_options) { - VaultV2Keyring *vaultV2_keyring = palloc0(sizeof(VaultV2Keyring)); - const char* token = extract_json_option_value(keyring_options, VAULTV2_KEYRING_TOKEN_KEY); - const char* url = extract_json_option_value(keyring_options, VAULTV2_KEYRING_URL_KEY); - const char* mount_path = extract_json_option_value(keyring_options, VAULTV2_KEYRING_MOUNT_PATH_KEY); - const char* ca_path = extract_json_option_value(keyring_options, VAULTV2_KEYRING_CA_PATH_KEY); + VaultV2Keyring *vaultV2_keyring = palloc0(sizeof(VaultV2Keyring)); - if(token == NULL || url == NULL || mount_path == NULL) + vaultV2_keyring->keyring.type = VAULT_V2_KEY_PROVIDER; + + if (!ParseKeyringJSONOptions(VAULT_V2_KEY_PROVIDER, vaultV2_keyring, + keyring_options, strlen(keyring_options))) { - /* TODO: report error */ + return NULL; + } + + if(strlen(vaultV2_keyring->vault_token) == 0 || + strlen(vaultV2_keyring->vault_url) == 0 || + strlen(vaultV2_keyring->vault_mount_path) == 0) + { + ereport(WARNING, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("missing in the keyring options:%s%s%s", + !vaultV2_keyring->vault_token ? " token" : "", + !vaultV2_keyring->vault_url ? " url" : "", + !vaultV2_keyring->vault_mount_path ? " mountPath" : ""))); return NULL; } - vaultV2_keyring->keyring.type = VAULT_V2_KEY_PROVIDER; - strncpy(vaultV2_keyring->vault_token, token, sizeof(vaultV2_keyring->vault_token)); - strncpy(vaultV2_keyring->vault_url, url, sizeof(vaultV2_keyring->vault_url)); - strncpy(vaultV2_keyring->vault_mount_path, mount_path, sizeof(vaultV2_keyring->vault_mount_path)); - strncpy(vaultV2_keyring->vault_ca_path, ca_path ? ca_path : "", sizeof(vaultV2_keyring->vault_ca_path)); return vaultV2_keyring; } diff --git a/src/catalog/tde_keyring_parse_opts.c b/src/catalog/tde_keyring_parse_opts.c new file mode 100644 index 00000000..c3e96dca --- /dev/null +++ b/src/catalog/tde_keyring_parse_opts.c @@ -0,0 +1,458 @@ +/*------------------------------------------------------------------------- + * + * tde_keyring_parse_opts.c + * Parser routines for the keyring JSON options + * + * TODO: add unit tests !!! + * + * Each value in the JSON document can be either scalar (string) - a value itself + * or a reference to the external object that contains the value. Though the top + * level field "type" can be only scalar. + * + * Examples: + * {"type" : "file", "path" : "/tmp/keyring_data_file"} + * {"type" : "file", "path" : {"type" : "file", "path" : "/tmp/datafile-location"}} + * in the latter one, /tmp/datafile-location contains not keyring data but the + * location of such. + * + * A field type can be "file", in this case, we expect "path" field. Or "remote", + * when "url" field is expected. + * + * IDENTIFICATION + * contrib/pg_tde/src/catalog/tde_keyring_parse_opts.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "common/jsonapi.h" +#include "mb/pg_wchar.h" +#include "storage/fd.h" + +#include "catalog/tde_keyring.h" +#include "keyring/keyring_curl.h" + +#include + +#define MAX_CONFIG_FILE_DATA_LENGTH 1024 + +/* + * JSON parser state + */ + +typedef enum JsonKeringSemState +{ + JK_EXPECT_TOP_FIELD, + JK_EXPECT_EXTERN_VAL, +} JsonKeringSemState; + +#define KEYRING_REMOTE_FIELD_TYPE "remote" +#define KEYRING_FILE_FIELD_TYPE "file" + +typedef enum JsonKeyringField +{ + JK_FIELD_UNKNOWN, + + JK_KRING_TYPE, + + JK_FIELD_TYPE, + JK_REMOTE_URL, + JK_FIELD_PATH, + + JF_FILE_PATH, + + JK_VAULT_TOKEN, + JK_VAULT_URL, + JK_VAULT_MOUNT_PATH, + JK_VAULT_CA_PATH, + + /* must be the last */ + JK_FIELDS_TOTAL +} JsonKeyringField; + +static const char *JK_FIELD_NAMES[JK_FIELDS_TOTAL] = { + [JK_FIELD_UNKNOWN] = "unknownField", + [JK_KRING_TYPE] = "type", + [JK_FIELD_TYPE] = "type", + [JK_REMOTE_URL] = "url", + [JK_FIELD_PATH] = "path", + + /* + * These values should match pg_tde_add_key_provider_vault_v2 and + * pg_tde_add_key_provider_file SQL interfaces + */ + [JF_FILE_PATH] = "path", + [JK_VAULT_TOKEN] = "token", + [JK_VAULT_URL] = "url", + [JK_VAULT_MOUNT_PATH] = "mountPath", + [JK_VAULT_CA_PATH] = "caPath", +}; + +#define MAX_JSON_DEPTH 64 +typedef struct JsonKeyringState +{ + ProviderType provider_type; + + /* + * Caller's options to be set from JSON values. Expected either + * `VaultV2Keyring` or `FileKeyring` + */ + void *provider_opts; + + /* + * A field hierarchy of the current branch, field[level] is the current + * one, field[level-1] is the parent and so on. We need to track parent + * fields because of the external values + */ + JsonKeyringField field[MAX_JSON_DEPTH]; + JsonKeringSemState state; + int level; + + /* + * The rest of the scalar fields might be in the JSON document but has no + * direct value for the caller. Although we need them for the values + * extraction or state tracking. + */ + char *kring_type; + char *field_type; + char *extern_url; + char *extern_path; +} JsonKeyringState; + +static JsonParseErrorType json_kring_scalar(void *state, char *token, JsonTokenType tokentype); +static JsonParseErrorType json_kring_object_field_start(void *state, char *fname, bool isnull); +static JsonParseErrorType json_kring_object_start(void *state); +static JsonParseErrorType json_kring_object_end(void *state); + +static void json_kring_assign_scalar(JsonKeyringState * parse, JsonKeyringField field, char *value); +static char *get_remote_kring_value(const char *url, const char *field_name); +static char *get_file_kring_value(const char *path, const char *field_name); + + +/* + * Parses json input for the given provider type and sets the provided options + * out_opts should be a palloc'd `VaultV2Keyring` or `FileKeyring` struct as the + * respective option values will be mem copied into it. + * Returns `true` if parsing succeded and `false` otherwise. +*/ +bool +ParseKeyringJSONOptions(ProviderType provider_type, void *out_opts, char *in_buf, int buf_len) +{ + JsonLexContext *jlex; + JsonKeyringState parse = {0}; + JsonSemAction sem; + JsonParseErrorType jerr; + + /* Set up parsing context and initial semantic state */ + parse.provider_type = provider_type; + parse.provider_opts = out_opts; + parse.level = -1; + parse.state = JK_EXPECT_TOP_FIELD; + memset(parse.field, 0, MAX_JSON_DEPTH * sizeof(JsonKeyringField)); + +#if PG_VERSION_NUM >= 170000 + jlex = makeJsonLexContextCstringLen(NULL, in_buf, buf_len, PG_UTF8, true); +#else + jlex = makeJsonLexContextCstringLen(in_buf, buf_len, PG_UTF8, true); +#endif + + /* + * Set up semantic actions. The function below will be called when the + * parser reaches the appropriate state. See comments on the functions. + */ + sem.semstate = &parse; + sem.object_start = json_kring_object_start; + sem.object_end = json_kring_object_end; + sem.array_start = NULL; + sem.array_end = NULL; + sem.object_field_start = json_kring_object_field_start; + sem.object_field_end = NULL; + sem.array_element_start = NULL; + sem.array_element_end = NULL; + sem.scalar = json_kring_scalar; + + /* Run the parser */ + jerr = pg_parse_json(jlex, &sem); + if (jerr != JSON_SUCCESS) + { + ereport(WARNING, + (errmsg("parsing of keyring options failed: %s", + json_errdetail(jerr, jlex)))); + + } +#if PG_VERSION_NUM >= 170000 + freeJsonLexContext(jlex); +#endif + + return jerr == JSON_SUCCESS; +} + +/* + * JSON parser semantic actions +*/ + +/* + * Invoked at the start of each object in the JSON document. + * + * Every new object increases the level of nesting as the whole document is the + * object itself (level 0) and every next one means going deeper into nesting. + * + * On the top level, we expect either scalar (string) values or objects referencing + * the external value of the field. Hence, if we are on level 1, we expect an + * "external field object" e.g. ({"type" : "remote", "url" : "http://localhost:8888/hello"}) + */ +static JsonParseErrorType +json_kring_object_start(void *state) +{ + JsonKeyringState *parse = state; + + if (MAX_JSON_DEPTH == ++parse->level) + { + elog(WARNING, "reached max depth of JSON nesting"); + return JSON_SEM_ACTION_FAILED; + } + + switch (parse->level) + { + case 0: + parse->state = JK_EXPECT_TOP_FIELD; + break; + case 1: + parse->state = JK_EXPECT_EXTERN_VAL; + break; + } + + return JSON_SUCCESS; +} + +/* + * Invoked at the start of each object in the JSON document. + * + * First, it means we are going back to the higher level. Plus, if it was the + * level 1, we expect only external objects there, which means we have all + * the necessary info to extract the value and assign the result to the + * appropriate (parent) field. + */ +static JsonParseErrorType +json_kring_object_end(void *state) +{ + JsonKeyringState *parse = state; + + /* + * we're done with the nested object and if it's an external field, the + * value should be extracted and assigned to the parent "field". for + * example if : "field" : {"type" : "remote", "url" : + * "http://localhost:8888/hello"} or "field" : {"type" : "file", "path" : + * "/tmp/datafile-location"} the "field"'s value should be the content of + * "path" or "url" respectively + */ + if (parse->level == 1) + { + if (parse->state == JK_EXPECT_EXTERN_VAL) + { + JsonKeyringField parent_field = parse->field[0]; + + char *value = NULL; + + if (strcmp(parse->field_type, KEYRING_REMOTE_FIELD_TYPE) == 0) + value = get_remote_kring_value(parse->extern_url, JK_FIELD_NAMES[parent_field]); + if (strcmp(parse->field_type, KEYRING_FILE_FIELD_TYPE) == 0) + value = get_file_kring_value(parse->extern_path, JK_FIELD_NAMES[parent_field]); + + json_kring_assign_scalar(parse, parent_field, value); + } + + parse->state = JK_EXPECT_TOP_FIELD; + } + + parse->level--; + + return JSON_SUCCESS; +} + +/* + * Invoked at the start of each object field in the JSON document. + * + * Based on the given field name and the semantic state (we expect a top-level + * field or an external object) we set the state so that when we get the value, + * we know what is it and where to assign it. + */ +static JsonParseErrorType +json_kring_object_field_start(void *state, char *fname, bool isnull) +{ + JsonKeyringState *parse = state; + JsonKeyringField *field; + + Assert(parse->level >= 0); + + field = &parse->field[parse->level]; + + switch (parse->state) + { + case JK_EXPECT_TOP_FIELD: + + /* + * On the top level, "type" stores a keyring type and this field + * is common for all keyrings. The rest of the fields depend on + * the keyring type. + */ + if (strcmp(fname, JK_FIELD_NAMES[JK_KRING_TYPE]) == 0) + { + *field = JK_KRING_TYPE; + break; + } + switch (parse->provider_type) + { + case FILE_KEY_PROVIDER: + if (strcmp(fname, JK_FIELD_NAMES[JF_FILE_PATH]) == 0) + *field = JF_FILE_PATH; + else + { + *field = JK_FIELD_UNKNOWN; + elog(DEBUG1, "parse file keyring config: unexpected field %s", fname); + } + break; + + case VAULT_V2_KEY_PROVIDER: + if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_TOKEN]) == 0) + *field = JK_VAULT_TOKEN; + else if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_URL]) == 0) + *field = JK_VAULT_URL; + else if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_MOUNT_PATH]) == 0) + *field = JK_VAULT_MOUNT_PATH; + else if (strcmp(fname, JK_FIELD_NAMES[JK_VAULT_CA_PATH]) == 0) + *field = JK_VAULT_CA_PATH; + else + { + *field = JK_FIELD_UNKNOWN; + elog(DEBUG1, "parse json keyring config: unexpected field %s", fname); + } + break; + } + break; + + case JK_EXPECT_EXTERN_VAL: + if (strcmp(fname, JK_FIELD_NAMES[JK_FIELD_TYPE]) == 0) + *field = JK_FIELD_TYPE; + else if (strcmp(fname, JK_FIELD_NAMES[JK_REMOTE_URL]) == 0) + *field = JK_REMOTE_URL; + else if (strcmp(fname, JK_FIELD_NAMES[JK_FIELD_PATH]) == 0) + *field = JK_FIELD_PATH; + break; + } + + return JSON_SUCCESS; +} + +/* + * Invoked at the start of each scalar in the JSON document. + * + * We have only the string value of the field. And rely on the state set by + * `json_kring_object_field_start` for defining what the field is. + */ +static JsonParseErrorType +json_kring_scalar(void *state, char *token, JsonTokenType tokentype) +{ + JsonKeyringState *parse = state; + + json_kring_assign_scalar(parse, parse->field[parse->level], token); + + return JSON_SUCCESS; +} + +static void +json_kring_assign_scalar(JsonKeyringState * parse, JsonKeyringField field, char *value) +{ + VaultV2Keyring *vault = parse->provider_opts; + FileKeyring *file = parse->provider_opts; + + switch (field) + { + case JK_KRING_TYPE: + parse->kring_type = value; + break; + + case JK_FIELD_TYPE: + parse->field_type = value; + break; + case JK_REMOTE_URL: + parse->extern_url = value; + break; + case JK_FIELD_PATH: + parse->extern_path = value; + break; + + case JF_FILE_PATH: + strncpy(file->file_name, value, sizeof(file->file_name)); + break; + + case JK_VAULT_TOKEN: + strncpy(vault->vault_token, value, sizeof(vault->vault_token)); + break; + case JK_VAULT_URL: + strncpy(vault->vault_url, value, sizeof(vault->vault_url)); + break; + case JK_VAULT_MOUNT_PATH: + strncpy(vault->vault_mount_path, value, sizeof(vault->vault_mount_path)); + break; + case JK_VAULT_CA_PATH: + strncpy(vault->vault_ca_path, value, sizeof(vault->vault_ca_path)); + break; + + default: + elog(DEBUG1, "json keyring: unexpected scalar field %d", field); + Assert(0); + break; + } +} + +static char * +get_remote_kring_value(const char *url, const char *field_name) +{ + long httpCode; + CurlString outStr; + + /* TODO: we never pfree it */ + outStr.ptr = palloc0(1); + outStr.len = 0; + + if (!curlSetupSession(url, NULL, &outStr)) + { + elog(WARNING, "CURL error for remote object %s", field_name); + return NULL; + } + if (curl_easy_perform(keyringCurl) != CURLE_OK) + { + elog(WARNING, "HTTP request error for remote object %s", field_name); + return NULL; + } + if (curl_easy_getinfo(keyringCurl, CURLINFO_RESPONSE_CODE, &httpCode) != CURLE_OK) + { + elog(WARNING, "HTTP error for remote object %s, HTTP code %li", field_name, httpCode); + return NULL; + } + + return outStr.ptr; +} + +static char * +get_file_kring_value(const char *path, const char *field_name) +{ + int fd = -1; + char *val; + + fd = BasicOpenFile(path, O_RDONLY); + if (fd < 0) + { + elog(WARNING, "filed to open file %s for %s", path, field_name); + return NULL; + } + + /* TODO: we never pfree it */ + val = palloc0(MAX_CONFIG_FILE_DATA_LENGTH); + pg_pread(fd, val, MAX_CONFIG_FILE_DATA_LENGTH, 0); + val[strcspn(val, "\r\n")] = 0; + + close(fd); + return val; +} diff --git a/src/common/pg_tde_utils.c b/src/common/pg_tde_utils.c index 7bd4030a..b3dfd370 100644 --- a/src/common/pg_tde_utils.c +++ b/src/common/pg_tde_utils.c @@ -9,24 +9,13 @@ *------------------------------------------------------------------------- */ -#define MAX_CONFIG_FILE_DATA_LENGTH 1024 - #include "postgres.h" #include "access/genam.h" #include "access/heapam.h" -#include "access/htup_details.h" -#include "catalog/pg_class.h" -#include "catalog/pg_am.h" #include "catalog/pg_tablespace_d.h" -#include "utils/fmgroids.h" -#include "utils/rel.h" #include "utils/snapmgr.h" #include "commands/defrem.h" #include "common/pg_tde_utils.h" -#include "fmgr.h" -#include "keyring/keyring_curl.h" -#include "utils/builtins.h" -#include "unistd.h" Oid get_tde_basic_table_am_oid(void) @@ -90,133 +79,6 @@ get_tde_tables_count(void) return count; } - -const char * -extract_json_cstr(Datum json, const char* field_name) -{ - Datum field = DirectFunctionCall2(json_object_field_text, json, CStringGetTextDatum(field_name)); - const char* cstr = TextDatumGetCString(field); - - return cstr; -} - -const char * -extract_json_option_value(Datum top_json, const char* field_name) -{ - Datum field; - Datum field_type; - const char* field_type_cstr; - - field = DirectFunctionCall2(json_object_field, top_json, CStringGetTextDatum(field_name)); - - field_type = DirectFunctionCall1(json_typeof, field); - field_type_cstr = TextDatumGetCString(field_type); - - if(field_type_cstr == NULL) - { - return NULL; - } - - if(strcmp(field_type_cstr, "string") == 0) - { - return extract_json_cstr(top_json, field_name); - } - - if(strcmp(field_type_cstr, "object") != 0) - { - elog(ERROR, "Unsupported type for object %s: %s", field_name, field_type_cstr); - return NULL; - } - - /* Now it is definitely an object */ - - { - const char* type_cstr = extract_json_cstr(field, "type"); - - if(type_cstr == NULL) - { - elog(ERROR, "Missing type property for remote object %s", field_name); - return NULL; - } - - if(strncmp("remote", type_cstr, 7) == 0) - { - const char* url_cstr = extract_json_cstr(field, "url"); - - long httpCode; - CurlString outStr; - - if(url_cstr == NULL) - { - elog(ERROR, "Missing url property for remote object %s", field_name); - return NULL; - } - - /* Http request should return the value of the option */ - outStr.ptr = palloc0(1); - outStr.len = 0; - if(!curlSetupSession(url_cstr, NULL, &outStr)) - { - elog(ERROR, "CURL error for remote object %s", field_name); - return NULL; - } - if(curl_easy_perform(keyringCurl) != CURLE_OK) - { - elog(ERROR, "HTTP request error for remote object %s", field_name); - return NULL; - } - if(curl_easy_getinfo(keyringCurl, CURLINFO_RESPONSE_CODE, &httpCode) != CURLE_OK) - { - elog(ERROR, "HTTP error for remote object %s, HTTP code %li", field_name, httpCode); - return NULL; - } -#if KEYRING_DEBUG - elog(DEBUG2, "HTTP response for config [%s] '%s'", field_name, outStr->ptr != NULL ? outStr->ptr : ""); -#endif - return outStr.ptr; - } - - if(strncmp("file", type_cstr, 5) == 0) - { - const char* path_cstr = extract_json_cstr(field, "path"); - FILE* f; - char* out; - - if(path_cstr == NULL) - { - elog(ERROR, "Missing path property for file object %s", field_name); - return NULL; - } - - if(access(path_cstr, R_OK) != 0) - { - elog(ERROR, "The file referenced by %s doesn't exists, or is not readable to postgres: %s", field_name, path_cstr); - return NULL; - } - - f = fopen(path_cstr, "r"); - - if(!f) - { - elog(ERROR, "The file referenced by %s doesn't exists, or is not readable to postgres: %s", field_name, path_cstr); - return NULL; - } - - /* File should contain the value for the option */ - out = palloc(MAX_CONFIG_FILE_DATA_LENGTH); - fgets(out, MAX_CONFIG_FILE_DATA_LENGTH, f); - out[strcspn(out, "\r\n")] = 0; - - fclose(f); - - return out; - } - - elog(ERROR, "Unknown type for object %s: %s", field_name, type_cstr); - return NULL; - } -} - /* returns the palloc'd string */ char * pg_tde_get_tde_file_dir(Oid dbOid, Oid spcOid) diff --git a/src/include/catalog/tde_keyring.h b/src/include/catalog/tde_keyring.h index 2256b495..2d3a1675 100644 --- a/src/include/catalog/tde_keyring.h +++ b/src/include/catalog/tde_keyring.h @@ -80,4 +80,7 @@ extern void cleanup_key_provider_info(Oid databaseId, Oid tablespaceId); extern void InitializeKeyProviderInfo(void); extern uint32 save_new_key_provider_info(KeyringProvideRecord *provider, Oid databaseId, Oid tablespaceId, bool recovery); extern uint32 redo_key_provider_info(KeyringProviderXLRecord *xlrec); + +extern bool ParseKeyringJSONOptions(ProviderType provider_type, void *out_opts, + char *in_buf, int buf_len); #endif /*TDE_KEYRING_H*/ diff --git a/src/include/common/pg_tde_utils.h b/src/include/common/pg_tde_utils.h index 82a9c28b..4f80e5b2 100644 --- a/src/include/common/pg_tde_utils.h +++ b/src/include/common/pg_tde_utils.h @@ -10,15 +10,11 @@ #include "postgres.h" #include "nodes/pg_list.h" -#include "common/relpath.h" extern Oid get_tde_basic_table_am_oid(void); extern Oid get_tde_table_am_oid(void); extern List *get_all_tde_tables(void); extern int get_tde_tables_count(void); -extern const char *extract_json_cstr(Datum json, const char* field_name); -const char *extract_json_option_value(Datum top_json, const char* field_name); - extern char *pg_tde_get_tde_file_dir(Oid dbOid, Oid spcOid); #endif /*PG_TDE_UTILS_H*/ diff --git a/src/keyring/keyring_vault.c b/src/keyring/keyring_vault.c index ef38fa70..648b7f81 100644 --- a/src/keyring/keyring_vault.c +++ b/src/keyring/keyring_vault.c @@ -16,8 +16,8 @@ #include "keyring/keyring_curl.h" #include "keyring/keyring_api.h" #include "pg_tde_defines.h" -#include "fmgr.h" -#include "utils/fmgrprotos.h" +#include "common/jsonapi.h" +#include "mb/pg_wchar.h" #include "utils/builtins.h" #include @@ -26,6 +26,39 @@ #include "common/base64.h" +/* + * JSON parser state +*/ + +typedef enum +{ + JRESP_EXPECT_TOP_DATA, + JRESP_EXPECT_DATA, + JRESP_EXPECT_KEY +} JsonVaultRespSemState; + +typedef enum +{ + JRESP_F_UNUSED, + + JRESP_F_KEY +} JsonVaultRespField; + +typedef struct JsonVaultRespState +{ + JsonVaultRespSemState state; + JsonVaultRespField field; + int level; + + char *key; +} JsonVaultRespState; + +static JsonParseErrorType json_resp_object_start(void *state); +static JsonParseErrorType json_resp_object_end(void *state); +static JsonParseErrorType json_resp_scalar(void *state, char *token, JsonTokenType tokentype); +static JsonParseErrorType json_resp_object_field_start(void *state, char *fname, bool isnull); +static JsonParseErrorType parse_json_response(JsonVaultRespState *parse, JsonLexContext *lex); + struct curl_slist *curlList = NULL; static bool curl_setup_token(VaultV2Keyring *keyring); @@ -178,10 +211,9 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error, char url[VAULT_URL_MAX_LEN]; CurlString str; long httpCode = 0; - - Datum dataJson; - Datum data2Json; - Datum keyJson; + JsonParseErrorType json_error; + JsonLexContext *jlex = NULL; + JsonVaultRespState parse; const char* responseKey; @@ -213,25 +245,26 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error, goto cleanup; } - PG_TRY(); - { - dataJson = DirectFunctionCall2(json_object_field_text, CStringGetTextDatum(str.ptr), CStringGetTextDatum("data")); - data2Json = DirectFunctionCall2(json_object_field_text, dataJson, CStringGetTextDatum("data")); - keyJson = DirectFunctionCall2(json_object_field_text, data2Json, CStringGetTextDatum("key")); - responseKey = TextDatumGetCString(keyJson); - } - PG_CATCH(); +#if PG_VERSION_NUM < 170000 + jlex = makeJsonLexContextCstringLen(str.ptr, str.len, PG_UTF8, true); +#else + jlex = makeJsonLexContextCstringLen(NULL, str.ptr, str.len, PG_UTF8, true); +#endif + json_error = parse_json_response(&parse, jlex); + + if (json_error != JSON_SUCCESS) { *return_code = KEYRING_CODE_INVALID_RESPONSE; ereport(throw_error ? ERROR : WARNING, - (errmsg("HTTP(S) request to keyring provider \"%s\" returned incorrect JSON response", - vault_keyring->keyring.provider_name))); + (errmsg("HTTP(S) request to keyring provider \"%s\" returned incorrect JSON: %s", + vault_keyring->keyring.provider_name, json_errdetail(json_error, jlex)))); goto cleanup; } - PG_END_TRY(); + + responseKey = parse.key; #if KEYRING_DEBUG - elog(DEBUG1, "Retrieved base64 key: %s", response_key); + elog(DEBUG1, "Retrieved base64 key: %s", responseKey); #endif key = palloc(sizeof(keyInfo)); @@ -249,6 +282,94 @@ get_key_by_name(GenericKeyring *keyring, const char *key_name, bool throw_error, } cleanup: - if(str.ptr != NULL) pfree(str.ptr); + if(str.ptr != NULL) + pfree(str.ptr); +#if PG_VERSION_NUM >= 170000 + if (jlex != NULL) + freeJsonLexContext(jlex); +#endif return key; } + +/* + * JSON parser routines +*/ + +static JsonParseErrorType +parse_json_response(JsonVaultRespState *parse, JsonLexContext *lex) +{ + JsonSemAction sem; + + parse->state = JRESP_EXPECT_TOP_DATA; + parse->level = -1; + parse->field = JRESP_F_UNUSED; + parse->key = NULL; + + sem.semstate = parse; + sem.object_start = json_resp_object_start; + sem.object_end = json_resp_object_end; + sem.array_start = NULL; + sem.array_end = NULL; + sem.object_field_start = json_resp_object_field_start; + sem.object_field_end = NULL; + sem.array_element_start = NULL; + sem.array_element_end = NULL; + sem.scalar = json_resp_scalar; + + return pg_parse_json(lex, &sem); +} + +static JsonParseErrorType +json_resp_object_start(void *state) +{ + ((JsonVaultRespState *) state)->level++; + + return JSON_SUCCESS; +} + +static JsonParseErrorType +json_resp_object_end(void *state) +{ + ((JsonVaultRespState *) state)->level--; + + return JSON_SUCCESS; +} + +static JsonParseErrorType +json_resp_scalar(void *state, char *token, JsonTokenType tokentype) +{ + JsonVaultRespState *parse = state; + + switch (parse->field) + { + case JRESP_F_KEY: + parse->key = token; + parse->field = JRESP_F_UNUSED; + break; + } + return JSON_SUCCESS; +} + +static JsonParseErrorType +json_resp_object_field_start(void *state, char *fname, bool isnull) +{ + JsonVaultRespState *parse = state; + + switch (parse->state) + { + case JRESP_EXPECT_TOP_DATA: + if (strcmp(fname, "data") == 0 && parse->level == 0) + parse->state = JRESP_EXPECT_DATA; + break; + case JRESP_EXPECT_DATA: + if (strcmp(fname, "data") == 0 && parse->level == 1) + parse->state = JRESP_EXPECT_KEY; + break; + case JRESP_EXPECT_KEY: + if (strcmp(fname, "key") == 0 && parse->level == 2) + parse->field = JRESP_F_KEY; + break; + } + + return JSON_SUCCESS; +}