An 5218/overflow traces/an 4956 macro traces (#205)

* stash

* fr

* source

* traces2 workflow

* fr flag

.github/workflows/dbt_run_overflowed_traces2.yml

@@ -0,0 +1,46 @@
+name: dbt_run_overflowed_traces2
+run-name: dbt_run_overflowed_traces2
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m "base_models,tag:overflowed_traces2" --vars '{"OVERFLOWED_TRACES":True}'

.github/workflows/dbt_run_temp_traces2.yml

@@ -0,0 +1,50 @@
+name: dbt_run_temp_traces2
+run-name: dbt_run_temp_traces2
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs “At minute 12 past every hour.” (see https://crontab.guru)
+    - cron: '12 * * * *'
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod_2xl
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m "base_models,tag:traces_reload"
+  
+
+

dbt_project.yml

@@ -63,6 +63,7 @@ vars:
   UPDATE_UDFS_AND_SPS: False
   UPDATE_SNOWFLAKE_TAGS: True
   OBSERV_FULL_TEST: False
+  OVERFLOWED_TRACES: False
   WAIT: 0
   HEAL_MODEL: False
   HEAL_MODELS: []

macros/streamline/models.sql

@@ -58,7 +58,7 @@
             )
 {% endmacro %}
 
-{% macro streamline_external_table_FR_query(
+{% macro streamline_external_table_fr_query(
         model,
         partition_function,
         partition_name,

models/bronze/overflow/bronze__overflowed_traces2.sql

@@ -0,0 +1,80 @@
+{{ config (
+    materialized = "view",
+    tags = ['overflowed_traces2']
+) }}
+
+{% for item in range(
+        1,
+        11
+    ) %}
+
+    SELECT
+        o.file_name,
+        f.block_number,
+        f.index_vals,
+        f.path,
+        f.key,
+        f.value_
+    FROM
+        (
+            SELECT
+                file_name,
+                file_url,
+                index_cols,
+                [overflowed_block, overflowed_tx] AS index_vals
+            FROM
+                (
+                    SELECT
+                        block_number,
+                        POSITION,
+                        file_name,
+                        file_url,
+                        index_cols,
+                        VALUE [0] AS overflowed_block,
+                        VALUE [1] AS overflowed_tx,
+                        block_number = overflowed_block
+                        AND POSITION = overflowed_tx AS missing
+                    FROM
+                        (
+                            SELECT
+                                block_number,
+                                POSITION,
+                                file_name,
+                                file_url,
+                                index_cols,
+                                utils.udf_detect_overflowed_responses(
+                                    file_url,
+                                    index_cols
+                                ) AS index_vals
+                            FROM
+                                {{ ref("bronze__potential_overflowed_traces2") }}
+                            WHERE
+                                row_no = {{ item }}
+                        ),
+                        LATERAL FLATTEN (
+                            input => index_vals
+                        )
+                )
+            WHERE
+                missing = TRUE
+        ) o,
+        TABLE(
+            utils.udtf_flatten_overflowed_responses(
+                o.file_url,
+                o.index_cols,
+                [o.index_vals]
+            )
+        ) f
+    WHERE
+        NOT IS_OBJECT(
+            f.value_
+        )
+        AND NOT IS_ARRAY(
+            f.value_
+        )
+        AND NOT IS_NULL_VALUE(
+            f.value_
+        ) {% if not loop.last %}
+        UNION ALL
+        {% endif %}
+    {% endfor %}

models/bronze/overflow/bronze__potential_overflowed_traces2.sql

@@ -0,0 +1,77 @@
+{{ config (
+    materialized = "view",
+    tags = ['overflowed_traces2']
+) }}
+
+WITH impacted_blocks AS (
+
+    SELECT
+        blocks_impacted_array
+    FROM
+        {{ ref("silver_observability__traces_completeness") }}
+    ORDER BY
+        test_timestamp DESC
+    LIMIT
+        1
+), all_missing AS (
+    SELECT
+        DISTINCT VALUE :: INT AS block_number
+    FROM
+        impacted_blocks,
+        LATERAL FLATTEN (
+            input => blocks_impacted_array
+        )
+),
+all_txs AS (
+    SELECT
+        block_number,
+        POSITION AS tx_position,
+        tx_hash
+    FROM
+        {{ ref("silver__transactions") }}
+        JOIN all_missing USING (block_number)
+),
+missing_txs AS (
+    SELECT
+        DISTINCT txs.block_number,
+        txs.tx_position,
+        file_name
+    FROM
+        all_txs txs
+        LEFT JOIN {{ ref("silver__traces2") }}
+        tr2 USING (
+            block_number,
+            tx_position
+        )
+        JOIN {{ ref("streamline__complete_debug_traceBlockByNumber") }} USING (block_number)
+        LEFT JOIN {{ source(
+            'base_silver',
+            'overflowed_traces2'
+        ) }}
+        ot USING (
+            block_number,
+            tx_position
+        )
+    WHERE
+        tr2.block_number IS NULL
+        AND ot.block_number IS NULL
+)
+SELECT
+    block_number,
+    tx_position AS POSITION,
+    file_name,
+    build_scoped_file_url(
+        @streamline.bronze.external_tables,
+        file_name
+    ) AS file_url,
+    ['block_number', 'array_index'] AS index_cols,
+    ROW_NUMBER() over (
+        ORDER BY
+            block_number ASC,
+            POSITION ASC
+    ) AS row_no
+FROM
+    missing_txs
+ORDER BY
+    block_number ASC,
+    POSITION ASC

models/silver/core/silver__blocks.sql

@@ -70,7 +70,7 @@ WHERE
             {{ this }}
     )
 {% else %}
-    {{ ref('bronze__streamline_FR_blocks') }}
+    {{ ref('bronze__streamline_fr_blocks') }}
 {% endif %}
 
 qualify(ROW_NUMBER() over (PARTITION BY block_number

models/silver/core/silver__confirmed_blocks.sql

@@ -32,7 +32,7 @@ WHERE
             {{ this }}
     )
 {% else %}
-    {{ ref('bronze__streamline_FR_confirm_blocks') }}
+    {{ ref('bronze__streamline_fr_confirm_blocks') }}
 {% endif %}
 
 qualify(ROW_NUMBER() over (PARTITION BY block_number

models/silver/core/silver__fact_traces2.sql

@@ -0,0 +1,15 @@
+{{ config (
+    materialized = "incremental",
+    incremental_strategy = 'delete+insert',
+    unique_key = ['block_number'],
+    cluster_by = "block_timestamp::date",
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
+    tags = ['traces_reload'],
+    full_refresh = false
+) }}
+{{ fsc_evm.gold_traces_v1(
+    full_reload_start_block = 3000000,
+    full_reload_blocks = 1000000,
+    full_reload_mode = true,
+    uses_overflow_steps = true
+) }}

models/silver/core/silver__receipts.sql

@@ -28,7 +28,7 @@ WHERE
     )
     AND IS_OBJECT(DATA)
 {% else %}
-    {{ ref('bronze__streamline_FR_receipts') }}
+    {{ ref('bronze__streamline_fr_receipts') }}
 WHERE
     IS_OBJECT(DATA)
 {% endif %}

models/silver/core/silver__traces.sql

@@ -29,7 +29,7 @@ WHERE
     )
     AND DATA :result IS NOT NULL
 {% else %}
-    {{ ref('bronze__streamline_FR_traces') }}
+    {{ ref('bronze__streamline_fr_traces') }}
 WHERE
     _partition_by_block_id <= 2300000
     AND DATA :result IS NOT NULL