Merge pull request #98 from fivetran/bugfix/updated-line-item-enhanced

line item enhanced updates

CHANGELOG.md

@@ -1,3 +1,16 @@
+# dbt_stripe v0.15.1
+
+## Bug Fixes
+- Updated the logic in `stripe__line_item_enhanced` to properly bring in refund data by adjusting the joins on balance transactions, refunds and charges. 
+  - Since charges and refunds are both types of balance transactions, included an additional join between refunds and balance transactions to bring in refunds at the same level as charges. 
+    - Updated balance transactions join on `connected_account_id` and `source_relation` to look at both charge and refund balance transactions.
+  - Fixed `fee_amount` logic to sum together charge and refund amounts.
+    - Coalesced `fee_amount` with zero for invoice-only (non-header) rows and updated downstream summing logic accordingly. 
+  - Updated `transaction_type` logic to not only bring in `charge`, but also return `charge + refund` if the balance transaction has a charge and a refund associated with it, or `payment intent + refund` if the refund balance transaction is not yet tied to a charge. 
+
+## Under the Hood
+- Modified the consistency tests to better compare differences between production and development rows.
+
 # dbt_stripe v0.15.0
 
 ## Breaking Changes

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 name: 'stripe'
 
-version: '0.15.0'
+version: '0.15.1'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   stripe:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: stripe_integrations_tests_14
+      schema: stripe_integrations_tests_15
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: stripe_integrations_tests_14
+      schema: stripe_integrations_tests_15
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: stripe_integrations_tests_14
+      schema: stripe_integrations_tests_15
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: stripe_integrations_tests_14
+      schema: stripe_integrations_tests_15
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: stripe_integrations_tests_14
+      schema: stripe_integrations_tests_15
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,16 +1,16 @@
 config-version: 2
 
 name: 'stripe_integration_tests'
-version: '0.15.0'
+version: '0.15.1'
 
 profile: 'integration_tests'
 
 # For use with validations
 models:
   +schema: "stripe_{{ var('directed_schema','dev') }}"
 
-vars:
-  stripe_schema: stripe_integrations_tests_14
+vars:  
+  stripe_schema: stripe_integrations_tests_15
   stripe__standardized_billing_model_enabled: true
   stripe__using_credit_notes: true
   stripe_source:
@@ -39,7 +39,7 @@ vars:
     stripe_credit_note_identifier:            "credit_note_data"
     stripe_credit_note_line_item_identifier:  "credit_note_line_item_data"
 
-    stripe__subscription_history: false
+  stripe__subscription_history: false
 
 seeds:
   stripe_integration_tests:

integration_tests/tests/consistency/consistency_activity_itemized_2.sql

@@ -4,18 +4,42 @@
 ) }}
 
 with prod as (
-    select balance_transaction_id, dispute_id as dispute_ids -- we don't have multi-dispute records in our data
+    select balance_transaction_id, dispute_ids
     from {{ target.schema }}_stripe_prod.stripe__activity_itemized_2
 ),
 
 dev as (
     select balance_transaction_id, dispute_ids
     from {{ target.schema }}_stripe_dev.stripe__activity_itemized_2
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
 )
 
--- test will return values and fail if the values are different (which they shouldn't be in our test data)
 select *
-from prod
-join dev
-    on prod.balance_transaction_id = dev.balance_transaction_id
-where prod.dispute_ids != dev.dispute_ids
+from final

integration_tests/tests/consistency/consistency_balance_change_from_activity_itemized_3.sql

@@ -4,18 +4,42 @@
 ) }}
 
 with prod as (
-    select balance_transaction_id, dispute_reason as dispute_reasons -- we don't have multi-dispute records in our data
+    select balance_transaction_id, dispute_reasons
     from {{ target.schema }}_stripe_prod.stripe__balance_change_from_activity_itemized_3
 ),
 
 dev as (
     select balance_transaction_id, dispute_reasons
     from {{ target.schema }}_stripe_dev.stripe__balance_change_from_activity_itemized_3
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
 )
 
--- test will return values and fail if the values are different (which they shouldn't be in our test data)
 select *
-from prod
-join dev
-    on prod.balance_transaction_id = dev.balance_transaction_id
-where prod.dispute_reasons != dev.dispute_reasons
+from final

integration_tests/tests/consistency/consistency_daily_overview.sql

@@ -19,21 +19,35 @@ with
         from {{ target.schema }}_stripe_{{ prod_or_dev }}.stripe__daily_overview
         group by 1,2 -- need to group to remove randomization stemming from rolling totals
     ),
-{% endfor %}
+{% endfor %} 
 
-final as (
-    -- test will fail if any rows from prod are not found in dev
-    (select * from prod
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
     except distinct
-    select * from dev)
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
 
     union all -- union since we only care if rows are produced
 
-    -- test will fail if any rows from dev are not found in prod
-    (select * from dev
-    except distinct
-    select * from prod)
-    )
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
 
 select *
 from final

integration_tests/tests/consistency/consistency_ending_balance_reconciliation_itemized_4_count.sql

@@ -4,18 +4,42 @@
 ) }}
 
 with prod as (
-    select balance_transaction_id, dispute_reason as dispute_reasons -- we don't have multi-dispute records in our data
+    select balance_transaction_id, dispute_reasons
     from {{ target.schema }}_stripe_prod.stripe__ending_balance_reconciliation_itemized_4
 ),
 
 dev as (
     select balance_transaction_id, dispute_reasons
     from {{ target.schema }}_stripe_dev.stripe__ending_balance_reconciliation_itemized_4
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
 )
 
--- test will return values and fail if the values are different (which they shouldn't be in our test data)
 select *
-from prod
-join dev
-    on prod.balance_transaction_id = dev.balance_transaction_id
-where prod.dispute_reasons != dev.dispute_reasons
+from final

integration_tests/tests/consistency/consistency_line_item_enhanced.sql

@@ -5,27 +5,43 @@
 
 with prod as (
     select *
+    except(fee_amount, refund_amount, transaction_type) --remove before merge
     from {{ target.schema }}_stripe_prod.stripe__line_item_enhanced
 ),
 
 dev as (
     select *
+    except(fee_amount, refund_amount, transaction_type) --remove before merge
     from {{ target.schema }}_stripe_dev.stripe__line_item_enhanced
-), 
+),
 
-final as (
-    -- test will fail if any rows from prod are not found in dev
-    (select * from prod
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
     except distinct
-    select * from dev)
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
 
     union all -- union since we only care if rows are produced
 
-    -- test will fail if any rows from dev are not found in prod
-    (select * from dev
-    except distinct
-    select * from prod)
-    )
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
 
 select *
 from final