feat: restructure integration test

zanebclark · Nov 15, 2024 · fb2f2a4 · fb2f2a4
1 parent f35f1a6
commit fb2f2a4
Show file tree

Hide file tree

Showing 35 changed files with 50 additions and 99 deletions.
diff --git a/.github/workflows/master-pytest.yml b/.github/workflows/master-pytest.yml
@@ -58,6 +58,19 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Create and populate connections.toml
+        run: |
+          touch connections.toml
+          echo [default] >> connections.toml
+          echo account = "$SNOWFLAKE_ACCOUNT" >> connections.toml
+          echo user = "$SNOWFLAKE_USER" >> connections.toml
+          echo role = "$SNOWFLAKE_ROLE" >> connections.toml
+          echo warehouse = "$SNOWFLAKE_WAREHOUSE" >> connections.toml
+          echo database = "$SNOWFLAKE_DATABASE" >> connections.toml
+          echo schema = "$MY_TARGET_SCHEMA" >> connections.toml
+          echo password = "$SCHEMACHANGE_SNOWFLAKE_PASSWORD" >> connections.toml
+          echo "cat connections.toml"
+          cat connections.toml
       - name: Test with pytest
         id: pytest
         run: |
@@ -66,16 +79,39 @@ jobs:
       - name: Test Schemachange on ${{ matrix.os }} targeting ${{ env.SNOWFLAKE_DATABASE }}.${{ env.MY_TARGET_SCHEMA }} schema
         run: |
           echo "::group::Setting up ${MY_TARGET_SCHEMA}"
-          schemachange deploy --config-folder ./demo/setup/${SCENARIO_NAME}
-          echo "::endgroup::"
+          schemachange deploy \
+            --config-folder ./demo \
+            --config-file-name schemachange-config-setup.yml \
+            --root-folder ./demo/${SCENARIO_NAME}/1_setup \
+            --connection-name default \
+            --connections-file-path connections.toml
+          echo "::endgroup::"'
+
           echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}"
-          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/A__render.sql
-          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/R__render.sql
-          schemachange render --config-folder ./demo/${SCENARIO_NAME} ./demo/${SCENARIO_NAME}/V1.0.0__render.sql
+          schemachange render \
+            --config-folder ./demo/${SCENARIO_NAME} \
+            --connection-name default \
+            --connections-file-path connections.toml \
+            ./demo/${SCENARIO_NAME}/2_test/A__render.sql
+          schemachange render \
+            --config-folder ./demo/${SCENARIO_NAME} \
+            --connection-name default \
+            --connections-file-path connections.toml \
+            ./demo/${SCENARIO_NAME}/2_test/R__render.sql
+          schemachange render \
+            --config-folder ./demo/${SCENARIO_NAME} \
+            --connection-name default \
+            --connections-file-path connections.toml 
+            ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql
           echo "::endgroup::"
+
           echo "::group::Testing Deployment using ${MY_TARGET_SCHEMA}"
           set +e
-          schemachange deploy --config-folder ./demo/${SCENARIO_NAME}
+          schemachange deploy \
+            --config-folder ./demo/${SCENARIO_NAME} \
+            --connection-name default \
+            --connections-file-path connections.toml \
+            --root-folder ./demo/${SCENARIO_NAME}/2_test
           RESULT=$?
           if [ $RESULT -eq 0 ]; then
             echo "Deployment Completed!"
@@ -84,9 +120,16 @@ jobs:
           fi
           echo "::endgroup::"
           set -e
+
           echo "::group::Tearing down up ${MY_TARGET_SCHEMA}"
-          schemachange deploy --config-folder ./demo/teardown/${SCENARIO_NAME}
+          schemachange deploy \
+            --config-folder ./demo \
+            --config-file-name schemachange-config-teardown.yml \
+            --connection-name default \
+            --connections-file-path connections.toml \
+            --root-folder ./demo/${SCENARIO_NAME}/3_teardown \
           echo "::endgroup::"
+
           if [ $RESULT -ne 0 ]; then
             exit 1
           fi
diff --git a/...etup/basics_demo/A__setup_basics_demo.sql → demo/basics_demo/1_setup/A__setup.sql b/...etup/basics_demo/A__setup_basics_demo.sql → demo/basics_demo/1_setup/A__setup.sql
diff --git a/demo/basics_demo/A__basic001.sql → demo/basics_demo/2_test/A__basic001.sql b/demo/basics_demo/A__basic001.sql → demo/basics_demo/2_test/A__basic001.sql
diff --git a/demo/basics_demo/A__render.sql → demo/basics_demo/2_test/A__render.sql b/demo/basics_demo/A__render.sql → demo/basics_demo/2_test/A__render.sql
diff --git a/demo/basics_demo/R__basic001.sql → demo/basics_demo/2_test/R__basic001.sql b/demo/basics_demo/R__basic001.sql → demo/basics_demo/2_test/R__basic001.sql
diff --git a/demo/basics_demo/R__render.sql → demo/basics_demo/2_test/R__render.sql b/demo/basics_demo/R__render.sql → demo/basics_demo/2_test/R__render.sql
diff --git a/demo/basics_demo/V1.0.0__render.sql → demo/basics_demo/2_test/V1.0.0__render.sql b/demo/basics_demo/V1.0.0__render.sql → demo/basics_demo/2_test/V1.0.0__render.sql
diff --git a/demo/basics_demo/V1.0.1__EOF_FIle.sql → demo/basics_demo/2_test/V1.0.1__EOF_FIle.sql b/demo/basics_demo/V1.0.1__EOF_FIle.sql → demo/basics_demo/2_test/V1.0.1__EOF_FIle.sql
diff --git a/demo/basics_demo/V1.0.2__StoredProc.sql → ...basics_demo/2_test/V1.0.2__StoredProc.sql b/demo/basics_demo/V1.0.2__StoredProc.sql → ...basics_demo/2_test/V1.0.2__StoredProc.sql
diff --git a/...n/basics_demo/A__teardown_basics_demo.sql → demo/basics_demo/3_teardown/A__teardown.sql b/...n/basics_demo/A__teardown_basics_demo.sql → demo/basics_demo/3_teardown/A__teardown.sql
diff --git a/demo/basics_demo/schemachange-config.yml b/demo/basics_demo/schemachange-config.yml
@@ -2,12 +2,6 @@ config-version: 1
 
 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}"
 
-snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}"
-snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}"
-snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}"
-snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}"
-snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}"
-snowflake-schema: "{{ env_var('MY_TARGET_SCHEMA')}}"
 change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY"
 create-change-history-table: true
 

diff --git a/.../citibike_demo/A__setup_citibike_demo.sql → demo/citibike_demo/1_setup/A__setup.sql b/.../citibike_demo/A__setup_citibike_demo.sql → demo/citibike_demo/1_setup/A__setup.sql
diff --git a/demo/citibike_demo/A__checks.sql → demo/citibike_demo/2_test/A__checks.sql b/demo/citibike_demo/A__checks.sql → demo/citibike_demo/2_test/A__checks.sql
diff --git a/demo/citibike_demo/A__render.sql → demo/citibike_demo/2_test/A__render.sql b/demo/citibike_demo/A__render.sql → demo/citibike_demo/2_test/A__render.sql
diff --git a/demo/citibike_demo/R__checks.sql → demo/citibike_demo/2_test/R__checks.sql b/demo/citibike_demo/R__checks.sql → demo/citibike_demo/2_test/R__checks.sql
diff --git a/demo/citibike_demo/R__render.sql → demo/citibike_demo/2_test/R__render.sql b/demo/citibike_demo/R__render.sql → demo/citibike_demo/2_test/R__render.sql
diff --git a/demo/citibike_demo/V1.0.0__render.sql → demo/citibike_demo/2_test/V1.0.0__render.sql b/demo/citibike_demo/V1.0.0__render.sql → demo/citibike_demo/2_test/V1.0.0__render.sql
diff --git a/...demo/V1.1.0__initial_database_objects.sql → ...test/V1.1.0__initial_database_objects.sql b/...demo/V1.1.0__initial_database_objects.sql → ...test/V1.1.0__initial_database_objects.sql
diff --git a/...bike_demo/V1.2.0__load_tables_from_s3.sql → ...mo/2_test/V1.2.0__load_tables_from_s3.sql b/...bike_demo/V1.2.0__load_tables_from_s3.sql → ...mo/2_test/V1.2.0__load_tables_from_s3.sql
diff --git a/...tibike_demo/A__teardown_citibike_demo.sql → .../citibike_demo/3_teardown/A__teardown.sql b/...tibike_demo/A__teardown_citibike_demo.sql → .../citibike_demo/3_teardown/A__teardown.sql
diff --git a/demo/citibike_demo/schemachange-config.yml b/demo/citibike_demo/schemachange-config.yml
@@ -2,12 +2,6 @@ config-version: 1
 
 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}"
 
-snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}"
-snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}"
-snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}"
-snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}"
-snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}"
-snowflake-schema: "{{ env_var('MY_TARGET_SCHEMA')}}"
 change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY"
 create-change-history-table: true
 

diff --git a/...mo_jinja/A__setup_citibike_demo_jinja.sql → .../citibike_demo_jinja/1_setup/A__setup.sql b/...mo_jinja/A__setup_citibike_demo_jinja.sql → .../citibike_demo_jinja/1_setup/A__setup.sql
diff --git a/demo/citibike_demo_jinja/A__render.sql → .../citibike_demo_jinja/2_test/A__render.sql b/demo/citibike_demo_jinja/A__render.sql → .../citibike_demo_jinja/2_test/A__render.sql
diff --git a/demo/citibike_demo_jinja/R__render.sql → .../citibike_demo_jinja/2_test/R__render.sql b/demo/citibike_demo_jinja/R__render.sql → .../citibike_demo_jinja/2_test/R__render.sql
diff --git a/demo/citibike_demo_jinja/V1.0.0__render.sql → ...bike_demo_jinja/2_test/V1.0.0__render.sql b/demo/citibike_demo_jinja/V1.0.0__render.sql → ...bike_demo_jinja/2_test/V1.0.0__render.sql
diff --git a/...inja/V1.1.0__initial_database_objects.sql → ...test/V1.1.0__initial_database_objects.sql b/...inja/V1.1.0__initial_database_objects.sql → ...test/V1.1.0__initial_database_objects.sql
diff --git a/...emo_jinja/V1.2.0__load_tables_from_s3.sql → ...ja/2_test/V1.2.0__load_tables_from_s3.sql b/...emo_jinja/V1.2.0__load_tables_from_s3.sql → ...ja/2_test/V1.2.0__load_tables_from_s3.sql
diff --git a/...jinja/A__teardown_citibike_demo_jinja.sql → ...ike_demo_jinja/3_teardown/A__teardown.sql b/...jinja/A__teardown_citibike_demo_jinja.sql → ...ike_demo_jinja/3_teardown/A__teardown.sql
diff --git a/demo/citibike_demo_jinja/schemachange-config.yml b/demo/citibike_demo_jinja/schemachange-config.yml
@@ -3,12 +3,6 @@ config-version: 1
 root-folder: "./demo/{{ env_var('SCENARIO_NAME')}}"
 modules-folder: "./demo/{{ env_var('SCENARIO_NAME')}}/modules"
 
-snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}"
-snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}"
-snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}"
-snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}"
-snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}"
-snowflake-schema: "{{ env_var('MY_TARGET_SCHEMA')}}"
 change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.{{ env_var('MY_TARGET_SCHEMA')}}.CHANGE_HISTORY"
 create-change-history-table: true
 

diff --git a/...setup/basics_demo/schemachange-config.yml → demo/schemachange-config-setup.yml b/...setup/basics_demo/schemachange-config.yml → demo/schemachange-config-setup.yml
@@ -2,11 +2,6 @@ config-version: 1
 
 root-folder: "./demo/setup/{{ env_var('SCENARIO_NAME')}}"
 
-snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}"
-snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}"
-snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}"
-snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}"
-snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}"
 # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment.
 change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY"
 create-change-history-table: true

diff --git a/...own/citibike_demo/schemachange-config.yml → demo/schemachange-config-teardown.yml b/...own/citibike_demo/schemachange-config.yml → demo/schemachange-config-teardown.yml
@@ -2,11 +2,6 @@ config-version: 1
 
 root-folder: "./demo/teardown/{{ env_var('SCENARIO_NAME')}}"
 
-snowflake-user: "{{ env_var('SNOWFLAKE_USER')}}"
-snowflake-account: "{{ env_var('SNOWFLAKE_ACCOUNT')}}"
-snowflake-role: "{{ env_var('SNOWFLAKE_ROLE')}}"
-snowflake-warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE')}}"
-snowflake-database: "{{ env_var('SNOWFLAKE_DATABASE')}}"
 # tracking the setup step in a different change history table to use schemachange setup and teardown separate from deployment.
 change-history-table: "{{ env_var('SNOWFLAKE_DATABASE')}}.SCHEMACHANGE.{{ env_var('SCENARIO_NAME')}}_CHANGE_HISTORY"
 create-change-history-table: true

diff --git a/demo/setup/citibike_demo/schemachange-config.yml b/demo/setup/citibike_demo/schemachange-config.yml
diff --git a/demo/setup/citibike_demo_jinja/schemachange-config.yml b/demo/setup/citibike_demo_jinja/schemachange-config.yml
diff --git a/demo/teardown/basics_demo/schemachange-config.yml b/demo/teardown/basics_demo/schemachange-config.yml
diff --git a/demo/teardown/citibike_demo_jinja/schemachange-config.yml b/demo/teardown/citibike_demo_jinja/schemachange-config.yml