Skip to content

Commit

Permalink
Merge pull request #60 from FlipsideCrypto/upd_schedule_core_tags
Browse files Browse the repository at this point in the history
upd tags core vs non core and workflow schedule csv
  • Loading branch information
eric-laurello authored Nov 10, 2023
2 parents dc63f75 + 43062ec commit be1302a
Show file tree
Hide file tree
Showing 19 changed files with 56 additions and 21 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/dbt_run_scheduled.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
with:
dbt_command: >
dbt run -s "bitcoin_models,tag:load" "bitcoin_models,tag:core"
dbt run -s "bitcoin_models,tag:scheduled_core"
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
34 changes: 34 additions & 0 deletions .github/workflows/dbt_run_scheduled_non_core.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
name: dbt_run_scheduled
run-name: dbt_run_scheduled

on:
workflow_dispatch:
# schedule:
# # Runs 8 minutes after each dbt job kicks off (which is set to 15 min interval) (see https://crontab.guru)
# - cron: '8,23,38,53 * * * *'

env:
USE_VARS: "${{ vars.USE_VARS }}"
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
DBT_VERSION: "${{ vars.DBT_VERSION }}"
ACCOUNT: "${{ vars.ACCOUNT }}"
ROLE: "${{ vars.ROLE }}"
USER: "${{ vars.USER }}"
PASSWORD: "${{ secrets.PASSWORD }}"
REGION: "${{ vars.REGION }}"
DATABASE: "${{ vars.DATABASE }}"
WAREHOUSE: "${{ vars.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"

concurrency:
group: ${{ github.workflow }}

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
with:
dbt_command: >
dbt run -s "bitcoin_models,tag:scheduled_non_core"
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
9 changes: 5 additions & 4 deletions data/github_actions__workflows.csv
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
workflow_name,workflow_schedule
dbt_run_scheduled,"8,23,38,53 * * * *"
dbt_run_streamline_blocks_hash_realtime,"*/10 * * * *"
dbt_run_streamline_realtime,"3,13,23,33,43,53 * * * *"
dbt_test_tasks,"0,30 * * * *"
dbt_run_scheduled,"*/30 * * * *"
dbt_run_scheduled_non_core,"7 * * * *"
dbt_run_streamline_blocks_hash_realtime,"0,15,30,45 * * * *"
dbt_run_streamline_realtime,"5,20,35,50 * * * *"
dbt_test_tasks,"0,30 * * * *"
2 changes: 1 addition & 1 deletion models/bronze/core/bronze__blocks.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
materialized = 'incremental',
cluster_by = ["_inserted_timestamp::DATE"],
unique_key = 'block_number',
tags = ["load"],
tags = ["load", "scheduled_core"],
incremental_strategy = 'delete+insert'
) }}
-- depends on {{ref('bronze__streamline_blocks')}}
Expand Down
2 changes: 1 addition & 1 deletion models/bronze/core/bronze__transactions.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
materialized = 'incremental',
cluster_by = ["_inserted_timestamp::DATE"],
unique_key = "tx_id",
tags = ["load"],
tags = ["load", "scheduled_core"],
incremental_strategy = 'delete+insert',
incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
) }}
Expand Down
2 changes: 1 addition & 1 deletion models/bronze/labels/bronze__labels.sql
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{{ config(
materialized = 'view',
tags = ['core']
tags = ['core', 'scheduled_non_core']

) }}

Expand Down
2 changes: 1 addition & 1 deletion models/silver/core/silver__blocks.sql
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
incremental_strategy = 'delete+insert',
unique_key = 'block_number',
cluster_by = ["_inserted_timestamp::DATE", "block_number"],
tags = ["core"]
tags = ["core", "scheduled_core"]
) }}

WITH bronze_blocks AS (
Expand Down
2 changes: 1 addition & 1 deletion models/silver/core/silver__inputs.sql
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
incremental_strategy = 'delete+insert',
incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
unique_key = 'input_id',
tags = ["core"],
tags = ["core", "scheduled_core"],
cluster_by = ["_inserted_timestamp"],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
) }}
Expand Down
2 changes: 1 addition & 1 deletion models/silver/core/silver__inputs_final.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
unique_key = 'input_id',
cluster_by = ["block_number", "tx_id"],
tags = ["core"],
tags = ["core", "scheduled_core"],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
) }}

Expand Down
2 changes: 1 addition & 1 deletion models/silver/core/silver__outputs.sql
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
incremental_strategy = 'delete+insert',
incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
unique_key = 'output_id',
tags = ["core"],
tags = ["core", "scheduled_core"],
cluster_by = ["_partition_by_block_id", "tx_id"],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
) }}
Expand Down
2 changes: 1 addition & 1 deletion models/silver/core/silver__transaction_index.sql
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
unique_key = 'tx_id',
incremental_strategy = 'delete+insert',
cluster_by = ["_partition_by_block_id", "tx_id"],
tags = ["core"]
tags = ["core", "scheduled_core"]
) }}

WITH blocks AS (
Expand Down
2 changes: 1 addition & 1 deletion models/silver/core/silver__transactions.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
unique_key = 'tx_id',
cluster_by = ["_inserted_timestamp::DATE", "block_number"],
tags = ["core"]
tags = ["core", "scheduled_core"]
) }}
-- depends_on: {{ ref('silver__blocks') }}
WITH bronze_transactions AS (
Expand Down
2 changes: 1 addition & 1 deletion models/silver/core/silver__transactions_final.sql
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
unique_key = 'tx_id',
cluster_by = ["block_number", "tx_id"],
tags = ["core"],
tags = ["core", "scheduled_core"],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
) }}

Expand Down
2 changes: 1 addition & 1 deletion models/silver/ez/silver__block_miner_rewards.sql
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
incremental_strategy = 'delete+insert',
unique_key = 'block_number',
cluster_by = ["block_number", "block_timestamp::DATE"],
tags = ["core", "ez"],
tags = ["core", "ez", "scheduled_non_core" ],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
) }}

Expand Down
2 changes: 1 addition & 1 deletion models/silver/labels/silver__labels.sql
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{{ config(
materialized = 'incremental',
unique_key = 'address',
tags = ['core']
tags = ["core", "scheduled_non_core"]
) }}

SELECT
Expand Down
2 changes: 1 addition & 1 deletion models/silver/price/silver__price_all_providers_hourly.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
materialized = 'incremental',
unique_key = 'id',
cluster_by = ['hour::DATE'],
tags = ['prices', 'core']
tags = ["prices", "core", "scheduled_non_core"]
) }}

WITH coinmarketcap AS (
Expand Down
2 changes: 1 addition & 1 deletion models/silver/price/silver__price_coingecko_hourly.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
materialized = 'incremental',
unique_key = 'recorded_hour',
cluster_by = ['recorded_hour ::DATE'],
tags = ['prices', 'core']
tags = ["prices", "core", "scheduled_non_core"]
) }}

WITH prices AS (
Expand Down
2 changes: 1 addition & 1 deletion models/silver/price/silver__price_coinmarketcap_hourly.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
materialized = 'incremental',
unique_key = 'recorded_hour',
cluster_by = ['recorded_hour ::DATE'],
tags = ['prices', 'core']
tags = ["prices", "core", "scheduled_non_core"]
) }}

WITH prices AS (
Expand Down
2 changes: 1 addition & 1 deletion models/silver/price/silver__price_coinpaprika_hourly.sql
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
unique_key = 'recorded_hour',
incremental_strategy='delete+insert',
cluster_by = ['recorded_hour ::DATE'],
tags = ['prices', 'core']
tags = ["prices", "core", "scheduled_non_core"]
) }}

WITH prices AS (
Expand Down

0 comments on commit be1302a

Please sign in to comment.