From a0457b68cd5261c568dadfa609ba08728ea2a1fe Mon Sep 17 00:00:00 2001 From: Everistus Oputa Date: Thu, 12 Oct 2023 12:08:12 +0100 Subject: [PATCH 1/2] stp_ccg_snapshots added the snapshots --- .../config_ccg_code_mapping.json | 66 ++++++++++++++++++ .../config_stp_code_mapping.json | 69 +++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 config/pipelines/reference_tables/config_ccg_code_mapping.json create mode 100644 config/pipelines/reference_tables/config_stp_code_mapping.json diff --git a/config/pipelines/reference_tables/config_ccg_code_mapping.json b/config/pipelines/reference_tables/config_ccg_code_mapping.json new file mode 100644 index 0000000..bfbd98c --- /dev/null +++ b/config/pipelines/reference_tables/config_ccg_code_mapping.json @@ -0,0 +1,66 @@ +{ + "metadata": { + "products": [ + { + "product_name": "Reference Data", + "topic": "ccg code mapping", + "metrics": [ + { + "id": "", + "full_name": "Reference table with ccg code mapping", + "short_name":"ccg_code_mapping" + } + ] + } + ] + }, + "pipeline": { + "name": "trigger_ccg_code_mapping", + "folder": "reference_tables/stp_code_mapping/", + "ingestion": { + }, + "raw": { + "source_dbo": "dbo", + "source_table": "ccg_code_mapping_snapshot", + "source_query": "SELECT * FROM ccg_code_mapping_snapshot;", + "sink_path": "raw/ncdr_etp/adf_v2/table/ccg_code_mapping/snapshot/", + "sink_file": "table_ccg_code_mapping_snapshot.parquet" + }, + "proc": { + "source_path":"raw/ncdr_etp/adf_v2/table/ccg_code_mapping/snapshot/", + "source_file": "table_ccg_code_mapping_snapshot.parquet", + "source_schema": [ + {"name": "[ONS CCG code]", "type": "String"}, + {"name": "[CCG name]", "type": "String"}, + {"name": "[ODS CCG code]", "type": "String"} + + ], + "sink_path": "proc/sources/ncdr_etp/adf_v2/table/ccg_code_mapping/snapshot/", + "sink_file": "table_ccg_code_mapping_snapshot.parquet" + }, + "project": { + "source_path": "proc/sources/ncdr_etp/adf_v2/table/ccg_code_mapping/snapshot/", + "source_file": "table_ccg_code_mapping_snapshot.parquet", + "sink_path": "proc/projects/reference_tables/ods_codes/ccg_code_mapping/", + "sink_file": "table_ccg_code_mapping_snapshot.parquet" + }, + "staging":[ + { + "metric": "ccg_code_mapping_snapshot", + "stored_procedure": "[dbo].[stored_proc_ccg_code_mapping_snapshot]", + "source_path": "proc/projects/reference_tables/ods_codes/ccg_code_mapping/", + "source_file": "table_ccg_code_mapping_snapshot.parquet", + "sink_table": "ccg_code_mapping_snapshot" + } + ] + + + + } +} + + + + + + diff --git a/config/pipelines/reference_tables/config_stp_code_mapping.json b/config/pipelines/reference_tables/config_stp_code_mapping.json new file mode 100644 index 0000000..e041ea0 --- /dev/null +++ b/config/pipelines/reference_tables/config_stp_code_mapping.json @@ -0,0 +1,69 @@ +{ + "metadata": { + "products": [ + { + "product_name": "Reference Data", + "topic": "stp code mapping", + "metrics": [ + { + "id": "", + "full_name": "Reference table with stp code mapping", + "short_name":"stp_code_mapping" + } + ] + } + ] + }, + "pipeline": { + "name": "trigger_stp_code_mapping", + "folder": "reference_tables/stp_code_mapping/", + "ingestion": { + }, + "raw": { + "source_dbo": "dbo", + "source_table": "stp_code_mapping_snapshot", + "source_query": "SELECT * FROM stp_code_mapping_snapshot;", + "sink_path": "raw/ncdr_etp/adf_v2/table/stp_code_mapping/snapshot/", + "sink_file": "table_stp_code_mapping_snapshot.parquet" + }, + "proc": { + "source_path":"raw/ncdr_etp/adf_v2/table/stp_code_mapping/snapshot/", + "source_file": "table_stp_code_mapping_snapshot.parquet", + "source_schema": [ + {"name": "[ONS STP code]", "type": "String"}, + {"name": "[STP name]", "type": "String"}, + {"name": "[ODS STP code]", "type": "String"} + + ], + "sink_path": "proc/sources/ncdr_etp/adf_v2/table/stp_code_mapping/snapshot/", + "sink_file": "table_stp_code_mapping_snapshot.parquet" + }, + "project": { + "source_path": "proc/sources/ncdr_etp/adf_v2/table/stp_code_mapping/snapshot/", + "source_file": "table_stp_code_mapping_snapshot.parquet", + "sink_path": "proc/projects/reference_tables/ods_codes/stp_code_mapping/", + "sink_file": "table_stp_code_mapping_snapshot.parquet" + }, + "staging":[ + { + "metric": "stp_code_mapping_snapshot", + "stored_procedure": "[dbo].[stored_proc_stp_code_mapping_snapshot]", + "source_path": "proc/projects/reference_tables/ods_codes/stp_code_mapping/", + "source_file": "table_stp_code_mapping_snapshot.parquet", + "sink_table": "stp_code_mapping_snapshot" + } + ] + + + + } +} + + + + + + + + + \ No newline at end of file From 0dd416e6fac8468f47d6133072b3941390160031 Mon Sep 17 00:00:00 2001 From: Everistus Oputa Date: Thu, 12 Oct 2023 12:12:08 +0100 Subject: [PATCH 2/2] Stp_ccg_snapshots2 added 2 snapshots --- config/pipelines/reference_tables/config_ccg_code_mapping.json | 2 +- config/pipelines/reference_tables/config_stp_code_mapping.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/pipelines/reference_tables/config_ccg_code_mapping.json b/config/pipelines/reference_tables/config_ccg_code_mapping.json index bfbd98c..a7385ca 100644 --- a/config/pipelines/reference_tables/config_ccg_code_mapping.json +++ b/config/pipelines/reference_tables/config_ccg_code_mapping.json @@ -3,7 +3,7 @@ "products": [ { "product_name": "Reference Data", - "topic": "ccg code mapping", + "topic": "ccg_code_mapping", "metrics": [ { "id": "", diff --git a/config/pipelines/reference_tables/config_stp_code_mapping.json b/config/pipelines/reference_tables/config_stp_code_mapping.json index e041ea0..29effd3 100644 --- a/config/pipelines/reference_tables/config_stp_code_mapping.json +++ b/config/pipelines/reference_tables/config_stp_code_mapping.json @@ -3,7 +3,7 @@ "products": [ { "product_name": "Reference Data", - "topic": "stp code mapping", + "topic": "stp_code_mapping", "metrics": [ { "id": "",