From c07f97cfd3ade48ec155f94ba81d2e4df3c833c4 Mon Sep 17 00:00:00 2001 From: Zane Clark Date: Thu, 14 Nov 2024 19:10:33 -0800 Subject: [PATCH] fix: fix integration tests --- .github/workflows/master-pytest.yml | 47 +++++++++++------------- schemachange/session/SnowflakeSession.py | 14 ++++--- tests/session/test_SnowflakeSession.py | 25 +++++++------ tests/test_main.py | 10 ++++- 4 files changed, 54 insertions(+), 42 deletions(-) diff --git a/.github/workflows/master-pytest.yml b/.github/workflows/master-pytest.yml index 1cbe21b..385403e 100644 --- a/.github/workflows/master-pytest.yml +++ b/.github/workflows/master-pytest.yml @@ -29,13 +29,13 @@ jobs: runs-on: ${{ matrix.os }} if: ${{ github.event.label.name == 'ci-run-tests' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} env: - SNOWFLAKE_PASSWORD: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_PASSWORD }} - SNOWFLAKE_USER: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_USER }} SNOWFLAKE_ACCOUNT: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_ACCOUNT }} - SNOWFLAKE_DATABASE: SCHEMACHANGE_DEMO - SNOWFLAKE_WAREHOUSE: SCHEMACHANGE_DEMO_WH + SNOWFLAKE_USER: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_USER }} SNOWFLAKE_ROLE: SCHEMACHANGE_DEMO-DEPLOY + SNOWFLAKE_WAREHOUSE: SCHEMACHANGE_DEMO_WH + SNOWFLAKE_DATABASE: SCHEMACHANGE_DEMO MY_TARGET_SCHEMA: ${{ matrix.scenario-name }}_${{ github.run_number }}_${{ strategy.job-index }} + SNOWFLAKE_PASSWORD: ${{ secrets.SCHEMACHANGE_SNOWFLAKE_PASSWORD }} SCENARIO_NAME: ${{ matrix.scenario-name }} steps: - uses: actions/checkout@v4 @@ -60,17 +60,16 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Create and populate connections.toml run: | - touch connections.toml - echo [default] >> connections.toml - echo account = "$SNOWFLAKE_ACCOUNT" >> connections.toml - echo user = "$SNOWFLAKE_USER" >> connections.toml - echo role = "$SNOWFLAKE_ROLE" >> connections.toml - echo warehouse = "$SNOWFLAKE_WAREHOUSE" >> connections.toml - echo database = "$SNOWFLAKE_DATABASE" >> connections.toml - echo schema = "$MY_TARGET_SCHEMA" >> connections.toml - echo password = "$SCHEMACHANGE_SNOWFLAKE_PASSWORD" >> connections.toml + touch ./connections.toml + echo [default] >> ./connections.toml + echo account = \"${SNOWFLAKE_ACCOUNT}\" >> ./connections.toml + echo user = \"${SNOWFLAKE_USER}\" >> ./connections.toml + echo role = \"${SNOWFLAKE_ROLE}\" >> ./connections.toml + echo warehouse = \"${SNOWFLAKE_WAREHOUSE}\" >> ./connections.toml + echo database = \"${SNOWFLAKE_DATABASE}\" >> ./connections.toml + echo password = \"${SNOWFLAKE_PASSWORD}\" >> ./connections.toml echo "cat connections.toml" - cat connections.toml + cat ./connections.toml - name: Test with pytest id: pytest run: | @@ -84,24 +83,20 @@ jobs: --config-file-name schemachange-config-setup.yml \ --root-folder ./demo/${SCENARIO_NAME}/1_setup \ --connection-name default \ - --connections-file-path connections.toml - echo "::endgroup::"' + --connections-file-path ./connections.toml \ + --verbose + echo "::endgroup::" echo "::group::Testing Rendering to ${MY_TARGET_SCHEMA}" + schemachange render \ --config-folder ./demo/${SCENARIO_NAME} \ - --connection-name default \ - --connections-file-path connections.toml \ ./demo/${SCENARIO_NAME}/2_test/A__render.sql schemachange render \ --config-folder ./demo/${SCENARIO_NAME} \ - --connection-name default \ - --connections-file-path connections.toml \ ./demo/${SCENARIO_NAME}/2_test/R__render.sql schemachange render \ --config-folder ./demo/${SCENARIO_NAME} \ - --connection-name default \ - --connections-file-path connections.toml ./demo/${SCENARIO_NAME}/2_test/V1.0.0__render.sql echo "::endgroup::" @@ -110,8 +105,9 @@ jobs: schemachange deploy \ --config-folder ./demo/${SCENARIO_NAME} \ --connection-name default \ - --connections-file-path connections.toml \ - --root-folder ./demo/${SCENARIO_NAME}/2_test + --connections-file-path ./connections.toml \ + --root-folder ./demo/${SCENARIO_NAME}/2_test \ + --verbose RESULT=$? if [ $RESULT -eq 0 ]; then echo "Deployment Completed!" @@ -126,8 +122,9 @@ jobs: --config-folder ./demo \ --config-file-name schemachange-config-teardown.yml \ --connection-name default \ - --connections-file-path connections.toml \ + --connections-file-path ./connections.toml \ --root-folder ./demo/${SCENARIO_NAME}/3_teardown \ + --verbose echo "::endgroup::" if [ $RESULT -ne 0 ]; then diff --git a/schemachange/session/SnowflakeSession.py b/schemachange/session/SnowflakeSession.py index 4531db1..ba987f4 100644 --- a/schemachange/session/SnowflakeSession.py +++ b/schemachange/session/SnowflakeSession.py @@ -9,6 +9,7 @@ import structlog from schemachange.config.ChangeHistoryTable import ChangeHistoryTable +from schemachange.config.utils import get_snowflake_identifier_string from schemachange.session.Script import VersionedScript, RepeatableScript, AlwaysScript @@ -79,15 +80,18 @@ def __init__( "application": application, "session_parameters": self.session_parameters, } + connect_kwargs = {k: v for k, v in connect_kwargs.items() if v is not None} self.logger.debug("snowflake.connector.connect kwargs", **connect_kwargs) self.con = snowflake.connector.connect(**connect_kwargs) print(f"Current session ID: {self.con.session_id}") self.account = self.con.account - self.user = self.con.user - self.role = self.con.role - self.warehouse = self.con.warehouse - self.database = self.con.database - self.schema = self.con.schema + self.user = get_snowflake_identifier_string(self.con.user, "user") + self.role = get_snowflake_identifier_string(self.con.role, "role") + self.warehouse = get_snowflake_identifier_string( + self.con.warehouse, "warehouse" + ) + self.database = get_snowflake_identifier_string(self.con.database, "database") + self.schema = get_snowflake_identifier_string(self.con.schema, "schema") if not self.autocommit: self.con.autocommit(False) diff --git a/tests/session/test_SnowflakeSession.py b/tests/session/test_SnowflakeSession.py index 32b9d4b..647852d 100644 --- a/tests/session/test_SnowflakeSession.py +++ b/tests/session/test_SnowflakeSession.py @@ -15,17 +15,20 @@ def session() -> SnowflakeSession: logger = structlog.testing.CapturingLogger() with mock.patch("snowflake.connector.connect"): - # noinspection PyTypeChecker - return SnowflakeSession( - user="user", - account="account", - role="role", - warehouse="warehouse", - schemachange_version="3.6.1.dev", - application="schemachange", - change_history_table=change_history_table, - logger=logger, - ) + with mock.patch( + "schemachange.session.SnowflakeSession.get_snowflake_identifier_string" + ): + # noinspection PyTypeChecker + return SnowflakeSession( + user="user", + account="account", + role="role", + warehouse="warehouse", + schemachange_version="3.6.1.dev", + application="schemachange", + change_history_table=change_history_table, + logger=logger, + ) class TestSnowflakeSession: diff --git a/tests/test_main.py b/tests/test_main.py index 17d96e1..cdbc661 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -66,7 +66,9 @@ "snowflake_warehouse": "warehouse", "snowflake_role": "role", } -script_path = Path(__file__).parent.parent / "demo" / "basics_demo" / "A__basic001.sql" +script_path = ( + Path(__file__).parent.parent / "demo" / "basics_demo" / "2_test" / "A__basic001.sql" +) no_command = pytest.param( "schemachange.cli.deploy", @@ -418,9 +420,11 @@ ) @mock.patch("pathlib.Path.is_file", return_value=True) @mock.patch("schemachange.session.SnowflakeSession.snowflake.connector.connect") +@mock.patch("schemachange.session.SnowflakeSession.get_snowflake_identifier_string") def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( _, __, + ___, to_mock: str, cli_args: list[str], expected_config: dict, @@ -476,8 +480,10 @@ def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( ], ) @mock.patch("schemachange.session.SnowflakeSession.snowflake.connector.connect") +@mock.patch("schemachange.session.SnowflakeSession.get_snowflake_identifier_string") def test_main_deploy_config_folder( _, + __, to_mock: str, args: list[str], expected_config: dict, @@ -541,8 +547,10 @@ def test_main_deploy_config_folder( ], ) @mock.patch("schemachange.session.SnowflakeSession.snowflake.connector.connect") +@mock.patch("schemachange.session.SnowflakeSession.get_snowflake_identifier_string") def test_main_deploy_modules_folder( _, + __, to_mock: str, args: list[str], expected_config: dict,