Merge branch 'master' into feat/integrate-connections-toml

# Conflicts:
#	schemachange/session/Credential.py
#	schemachange/session/SnowflakeSession.py

.github/CONTRIBUTING.md

@@ -7,43 +7,44 @@ operating environment, schemachange version and python version. Whenever possibl
 also include a brief, self-contained code example that demonstrates the problem.
 
 We have
-included [issue templates for reporting bugs, requesting features and seeking clarifications.](https://github.com/Snowflake-Labs/schemachange/issues/new/choose)
-Choose the appropriate issue template to contribute to the repository.
+included [issue templates](https://github.com/Snowflake-Labs/schemachange/issues/new/choose) for reporting bugs, requesting features and seeking clarifications. Choose the appropriate issue template to contribute to the repository.
 
 ## Contributing code
 
-Thanks for your interest in contributing code to schemachange!
+Thank you for your interest in contributing code to schemachange!
 
-+ If this is your first time contributing to a project on GitHub, please read through
-  our [guide to contributing to schemachange](guide-to-contributing-to-schemachange).
++ If this is your first time contributing to a project on GitHub, please continue reading through
+  [our guide to contributing to schemachange](#guide-to-contributing-to-schemachange).
 + There are many online tutorials to help you [learn git](https://try.github.io/). For discussions of specific git
   workflows, see these discussions
   on [linux git workflow](https://www.mail-archive.com/[email protected]/msg39091.html),
   and [ipython git workflow](https://mail.python.org/pipermail/ipython-dev/2010-October/005632.html).
 
 ### Guide to contributing to schemachange
 
+> **IMPORTANT** : You will need to follow the [provisioning and schemachange setup instructions](../demo/README.MD) to ensure you can run GitHub actions against your Snowflake account before placing a PR with main schemachange repository so that your PR can be merged into schemachange master branch.
+
 1. If you are a first-time contributor
     + Go to [Snowflake-Labs/Schemachange](https://github.com/Snowflake-Labs/schemachange) and click the "fork" button to
       create your own copy of the project.
     + [Clone](https://github.com/git-guides/git-clone) the project to your local computer
 
     ```shell
-    git clone https://github.com/your-username/schemachange.git
-
+    # Replace <you-github-username> with your Github User Name otherwise
+    # you will not be able to clone from the fork you created earlier.
+    git clone https://github.com/<your-github-username>/schemachange.git
     ```
 
     + Change the directory
 
     ```shell
     cd schemachange
-
     ```
+
     + Add upstream repository:
 
     ```shell
     git remote add upstream https://github.com/Snowflake-Labs/schemachange
-
     ```
 
     + Now, `git remote -v` will show two [remote](https://github.com/git-guides/git-remote) repositories named:
@@ -54,7 +55,6 @@ Thanks for your interest in contributing code to schemachange!
     ```shell
     git checkout main
     git pull upstream main --tags
-
     ```
 
 2. Create and Activate a Virtual Environment
@@ -68,20 +68,22 @@ Thanks for your interest in contributing code to schemachange!
        of [this](https://docs.python.org/3/library/venv.html#how-venvs-work) table:
 
        | Platform | Shell      | Command                               |
-                                   |----------|------------|---------------------------------------|
+       |----------|------------|---------------------------------------|
        | POSIX    | bash/zsh   | `$ source <venv>/bin/activate`        |
        | POSIX    | fish       | `$ source <venv>/bin/activate.fish`   |
        | POSIX    | csh/tcsh   | `$ source <venv>/bin/activate.csh`    |
        | POSIX    | PowerShell | `$ <venv>/bin/Activate.ps1`           |
        | Windows  | cmd.exe    | `C:\> <venv>\Scripts\activate.bat`    |
        | Windows  | PowerShell | `PS C:\> <venv>\Scripts\Activate.ps1` |
 
-    3. With your virtual environment activated, upgrade pip
+    3. With your virtual environment activated, upgrade `pip`
+
        ```bash
        python -m pip install --upgrade pip
        ```
 
     4. Install the repo as an "editable" package with development dependencies
+
        ```bash
        pip install -e .[dev]
        ```
@@ -90,27 +92,30 @@ Thanks for your interest in contributing code to schemachange!
     + Create a branch for the features you want to work on. Since the branch name will appear in the merge message, use
       a sensible name such as 'update-build-library-dependencies':
 
-    ```shell
-    git checkout -b update-build-library-dependencies
-    ```
+      ```shell
+      git checkout -b update-build-library-dependencies
+      ```
 
     + Commit locally as you progress ( [git add](https://github.com/git-guides/git-add)
       and [git commit](https://github.com/git-guides/git-commit) ). Use a properly formatted commit message. Be sure to
-      document any changed behavior.
+      document any changed behavior in the [CHANGELOG.md](../CHANGELOG.md) file to help us collate the changes for a specific release.
+
+4. Test your contribution locally
 
-4. Test your contribution
    ```bash
    python -m pytest
    ```
+   PS: Please add test cases to the features you are developing so that over time, we can capture any lapse in functionality changes.
+
+5. Push your contribution to GitHub
 
-5. To submit your contribution
-    + [Push](https://github.com/git-guides/git-push) your changes back to your fork on GitHub
+    [Push](https://github.com/git-guides/git-push) your changes back to your fork on GitHub
 
     ```shell
     git push origin update-build-library-dependencies
-
     ```
 
+6. Raise a Pull Request to merge your contribution into the a Schemachange Release
     + Go to GitHub. The new branch will show up with a
       green [Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#initiating-the-pull-request)
       button. Make sure the title and message are clear, concise and self-explanatory. Then click the button to submit

.github/workflows/dev-pytest.yml

@@ -20,7 +20,7 @@ jobs:
       matrix:
         scenario-name: [ 'basics_demo', 'citibike_demo', 'citibike_demo_jinja']
         os: ["ubuntu-latest"]
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     runs-on: ${{ matrix.os }}
     if: ${{ github.event.label.name == 'ci-run-tests' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
 

.github/workflows/master-pytest.yml

@@ -25,7 +25,7 @@ jobs:
       matrix:
         scenario-name: [ 'basics_demo', 'citibike_demo', 'citibike_demo_jinja']
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     runs-on: ${{ matrix.os }}
     if: ${{ github.event.label.name == 'ci-run-tests' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
     env:

CHANGELOG.md

@@ -3,6 +3,17 @@ All notable changes to this project will be documented in this file.
 
 *The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).*
 
+## [4.0.0] - TBD
+### Added
+- Use of `structlog~=24.1.0` for standard log outputs
+- Verified Schemachange against Python 3.12
+
+### Changed
+- Refactored the main cli.py into multiple modules - config, session.
+- Updated contributing guidelines and demo readme content to help contributors setup local snowflake account to run the github actions in their fork before pushing the PR to upstream repository.
+- Removed tests against Python 3.8 [End of Life on 2024-10-07](https://devguide.python.org/versions/#supported-versions)
+
+
 ## [3.7.0] - 2024-07-22
 ### Added
 - Improved unit test coverage

README.md

@@ -23,6 +23,8 @@ blog post.
 
 For the complete list of changes made to schemachange check out the [CHANGELOG](CHANGELOG.md).
 
+To learn more about making a contribution to schemachange, please see our [Contributing guide](.github/CONTRIBUTING.md).
+
 **Please note** that schemachange is a community-developed tool, not an official Snowflake offering. It comes with no
 support or warranty.
 
@@ -490,7 +492,9 @@ compatibility with versions prior to 3.2.
 
 This is the main command that runs the deployment process.
 
-`usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG]`
+```bash
+usage: schemachange deploy [-h] [--config-folder CONFIG_FOLDER] [-f ROOT_FOLDER] [-m MODULES_FOLDER] [-a SNOWFLAKE_ACCOUNT] [-u SNOWFLAKE_USER] [-r SNOWFLAKE_ROLE] [-w SNOWFLAKE_WAREHOUSE] [-d SNOWFLAKE_DATABASE] [-s SNOWFLAKE_SCHEMA] [-c CHANGE_HISTORY_TABLE] [--vars VARS] [--create-change-history-table] [-ac] [-v] [--dry-run] [--query-tag QUERY_TAG]
+```
 
 | Parameter                                                            | Description                                                                                                                                                             |
 |----------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------|

demo/README.MD

@@ -13,30 +13,24 @@ not break any existing functionality.
     - [setup_schemachange_schema.sql](provision/setup_schemachange_schema.sql): Contains the SQL variables to track the
       individual demo scenarios in its own change history table.
 
-### Contributors
-
-As a contributor, you will have to set up schemachange demo database and schemachange schema (See Initialize and Setup
-scripts below). Along with that you will also set up the following Secrets in your forked repository so that the GitHub
-actions can set up, test and teardown the temporary schema it creates to test the changes to your code in the master and
-dev branches respectively.
-
-- SCHEMACHANGE_SNOWFLAKE_PASSWORD
-- SCHEMACHANGE_SNOWFLAKE_USER
-- SCHEMACHANGE_SNOWFLAKE_ACCOUNT
-
 ### Consumers
 
 - If you are consumer who is installing schemachange and wants to test-run the demo, then you will have to set the
   following environment variables.
-    - SNOWFLAKE_ACCOUNT: This will be the account identifier for your snowflake account.
-    - SNOWFLAKE_USER: This will be the user that will connect to you snowflake account.
-    - SNOWFLAKE_PASSWORD: This is the password for the user (SNOWFLAKE_USER) that will connect to the snowflake account.
+    - SNOWFLAKE_ACCOUNT: This will be the account identifier for your Snowflake account.
+    - SNOWFLAKE_USER: This will be the user that will connect to your Snowflake account (`SNOWFLAKE_ACCOUNT`).
+    - SNOWFLAKE_PASSWORD: This is the password for the user (`SNOWFLAKE_USER`) that will connect to your Snowflake
+      account (`SNOWFLAKE_ACCOUNT`).
     - SCENARIO_NAME: This will be demo folder you intend to experiment with. For
       starters, `basics_demo`, `citibike_demo` or `citibike_demo_jinja` are included with the repo that will set the
       root folder value in the respective schemachange-config.yml file.
-    - SNOWFLAKE_WAREHOUSE: This will be the warehouse you set up for the demo. Default setup is SCHEMACHANGE_DEMO_WH
-    - SNOWFLAKE_DATABASE Keyed to SCHEMACHANGE_DEMO
-    - SNOWFLAKE_ROLE Keyed to SCHEMACHANGE_DEMO-DEPLOY
+    - SNOWFLAKE_WAREHOUSE: This will be the warehouse you set up for the demo in your Snowflake
+      account (`SNOWFLAKE_ACCOUNT`). Default setup is SCHEMACHANGE_DEMO_WH.
+    - SNOWFLAKE_DATABASE Keyed to SCHEMACHANGE_DEMO. You can update this to the database in your snowflake account
+      (`SNOWFLAKE_ACCOUNT`). You will also need to update the provision scripts accordingly.
+    - SNOWFLAKE_ROLE Keyed to SCHEMACHANGE_DEMO-DEPLOY. This particular role name was chosen to test hyphenated roles
+      in schemachange. You can update this value to match the role you have setup in your snowflake
+      account (`SNOWFLAKE_ACCOUNT`).
 
 The scripts in the `provision` folder can be used to set up up your demo database along with a schema in that database
 that will house the change tracking tables needed to set up and teardown the schemas used to test a working version of
@@ -47,6 +41,17 @@ the demo DDL scripts.
 - The [setup](provision/setup_schemachange_schema.sql) script creates the `SCHEMACHANGE` schema in the database that you
   created in the initialize step.
 
+### Contributors
+
+As a contributor, you will have to set up schemachange demo database and schemachange schema (See Initialize and Setup
+scripts below). Along with that you will also set up the following Secrets in your forked repository so that the GitHub
+actions can set up, test and teardown the temporary schema it creates to test the changes to your code in the master and
+dev branches respectively.
+
+- SCHEMACHANGE_SNOWFLAKE_PASSWORD
+- SCHEMACHANGE_SNOWFLAKE_USER
+- SCHEMACHANGE_SNOWFLAKE_ACCOUNT
+
 # Setup
 
 The setup scripts are included to build the schema needed by the GitHub Actions Workflow to avoid conflict across jobs

demo/basics_demo/V1.0.2__StoredProc.sql

@@ -1,3 +1,5 @@
+use database {{ database_name }};
+use schema {{ schema_name }};
 -- This block of code executes in Visual Studio Code but fails in Schemachange.
 -- Use the $$ ... $$ to mark the block and execute the code block successfully.
 -- The comment from a community user help find the root cause.

demo/provision/setup_schemachange_schema.sql

@@ -20,6 +20,8 @@ CREATE DATABASE ROLE IF NOT EXISTS DB_R;
 CREATE DATABASE ROLE IF NOT EXISTS DB_W;
 CREATE DATABASE ROLE IF NOT EXISTS DB_C;
 
+GRANT CREATE SCHEMA ON DATABASE IDENTIFIER($TARGET_DB_NAME) TO DATABASE ROLE DB_C;
+
 GRANT DATABASE ROLE DB_C TO ROLE IDENTIFIER($DEPLOY_ROLE);
 
 CREATE DATABASE ROLE IF NOT EXISTS IDENTIFIER($SC_M);
@@ -38,6 +40,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C);
 CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS;
 -- USE SCHEMA INFORMATION_SCHEMA;
 -- DROP SCHEMA IF EXISTS PUBLIC;
+GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE);
 
 USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE);
 -- SCHEMA

demo/setup/basics_demo/A__setup_basics_demo.sql

@@ -31,6 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C);
 CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS;
 -- USE SCHEMA INFORMATION_SCHEMA;
 -- DROP SCHEMA IF EXISTS PUBLIC;
+GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE);
 
 USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE);
 -- SCHEMA

demo/setup/citibike_demo/A__setup_citibike_demo.sql

@@ -31,6 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C);
 CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS;
 -- USE SCHEMA INFORMATION_SCHEMA;
 -- DROP SCHEMA IF EXISTS PUBLIC;
+GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE);
 
 USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE);
 -- SCHEMA

demo/setup/citibike_demo_jinja/A__setup_citibike_demo_jinja.sql

@@ -31,6 +31,7 @@ GRANT DATABASE ROLE IDENTIFIER($SC_W) TO DATABASE ROLE IDENTIFIER($SC_C);
 CREATE SCHEMA IF NOT EXISTS IDENTIFIER($TARGET_SCHEMA_NAME) WITH MANAGED ACCESS;
 -- USE SCHEMA INFORMATION_SCHEMA;
 -- DROP SCHEMA IF EXISTS PUBLIC;
+GRANT OWNERSHIP ON SCHEMA IDENTIFIER($TARGET_SCHEMA_NAME) TO ROLE IDENTIFIER($DEPLOY_ROLE);
 
 USE SCHEMA IDENTIFIER($SCHEMACHANGE_NAMESPACE);
 -- SCHEMA

schemachange/cli.py

@@ -13,7 +13,7 @@
 
 # region Global Variables
 # metadata
-SCHEMACHANGE_VERSION = "3.7.0"
+SCHEMACHANGE_VERSION = "4.0.0"
 SNOWFLAKE_APPLICATION_NAME = "schemachange"
 module_logger = structlog.getLogger(__name__)
 

schemachange/config/BaseConfig.py

@@ -18,7 +18,7 @@
 T = TypeVar("T", bound="BaseConfig")
 
 
-@dataclasses.dataclass(frozen=True, kw_only=True)
+@dataclasses.dataclass(frozen=True)
 class BaseConfig(ABC):
     default_config_file_name: ClassVar[str] = "schemachange-config.yml"
 

schemachange/config/DeployConfig.py

@@ -13,7 +13,7 @@
 )
 
 
-@dataclasses.dataclass(frozen=True, kw_only=True)
+@dataclasses.dataclass(frozen=True)
 class DeployConfig(BaseConfig):
     subcommand: Literal["deploy"] = "deploy"
     snowflake_account: str | None = None

schemachange/config/RenderConfig.py

@@ -8,10 +8,10 @@
 from schemachange.config.utils import validate_file_path
 
 
-@dataclasses.dataclass(frozen=True, kw_only=True)
+@dataclasses.dataclass(frozen=True)
 class RenderConfig(BaseConfig):
+    script_path: Path | None = None
     subcommand: Literal["render"] = "render"
-    script_path: Path
 
     @classmethod
     def factory(
@@ -31,3 +31,9 @@ def factory(
             script_path=validate_file_path(file_path=script_path),
             **kwargs,
         )
+
+    def __post_init__(self):
+        if self.script_path is None:
+            raise TypeError(
+                "RenderConfig is missing 1 required argument: 'script_path'"
+            )

schemachange/deploy.py

@@ -23,7 +23,9 @@ def alphanum_convert(text: str):
 # Each number is converted to and integer and string parts are left as strings
 # This will enable correct sorting in python when the lists are compared
 # e.g. get_alphanum_key('1.2.2') results in ['', 1, '.', 2, '.', 2, '']
-def get_alphanum_key(key):
+def get_alphanum_key(key: str | int | None) -> list:
+    if key == "" or key is None:
+        return []
     alphanum_key = [alphanum_convert(c) for c in re.split("([0-9]+)", key)]
     return alphanum_key
 
@@ -100,7 +102,7 @@ def deploy(config: DeployConfig, session: SnowflakeSession):
             script_metadata = versioned_scripts.get(script.name)
 
             if (
-                max_published_version != ""
+                max_published_version is not None
                 and get_alphanum_key(script.version) <= max_published_version
             ):
                 if script_metadata is None:
@@ -113,7 +115,7 @@ def deploy(config: DeployConfig, session: SnowflakeSession):
                 else:
                     script_log.debug(
                         "Script has already been applied",
-                        max_published_version=str(max_published_version),
+                        max_published_version=max_published_version,
                     )
                     if script_metadata["checksum"] != checksum_current:
                         script_log.info("Script checksum has drifted since application")