From a022dae699d48bb1b3b6a907306d86fe1733362a Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Thu, 24 Oct 2024 08:59:30 -0700 Subject: [PATCH 01/12] Server-side Snowpark: Update package dependencies --- setup.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 8539d125d4a..2d512b01579 100644 --- a/setup.py +++ b/setup.py @@ -13,13 +13,16 @@ MODIN_DEPENDENCY_VERSION = ( "==0.28.1" # Snowpark pandas requires modin 0.28.1, which depends on pandas 2.2.1 ) -CONNECTOR_DEPENDENCY_VERSION = ">=3.10.0, <4.0.0" +CONNECTOR_DEPENDENCY_VERSION = ">=3.12.0, <4.0.0" +CONNECTOR_DEPENDENCY = f"snowflake-connector-python{CONNECTOR_DEPENDENCY_VERSION}" INSTALL_REQ_LIST = [ "setuptools>=40.6.0", "wheel", - f"snowflake-connector-python{CONNECTOR_DEPENDENCY_VERSION}", + CONNECTOR_DEPENDENCY, # snowpark directly depends on typing-extension, so we should not remove it even if connector also depends on it. "typing-extensions>=4.1.0, <5.0.0", + "protobuf>=5.28", + "tzlocal", "pyyaml", "cloudpickle>=1.6.0,<=2.2.1,!=2.1.0,!=2.2.0;python_version<'3.11'", "cloudpickle==2.2.1;python_version~='3.11'", # backend only supports cloudpickle 2.2.1 + python 3.11 at the moment @@ -39,6 +42,7 @@ DEVELOPMENT_REQUIREMENTS = [ "pytest<8.0.0", # check SNOW-1022240 for more details on the pin here "pytest-cov", + "wrapt", "coverage", "sphinx==5.0.2", "cachetools", # used in UDF doctest @@ -50,6 +54,15 @@ "graphviz", # used in plot tests "pytest-assume", # sql counter check "decorator", # sql counter check + "protoc-wheel-0", + "aiohttp", # vcrpy requirements. + "boto", # vcrpy requirements. + "httplib2", # vcrpy requirements. + "httpx", # vcrpy requirements. + "tornado", # vcrpy requirements. + "graphviz", # used in plot tests + "pytest-assume", # sql counter check + "decorator", # sql counter check ] # read the version From 13cbebf9d10a4668289521547ca96058bead51a7 Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Thu, 24 Oct 2024 08:41:47 -0700 Subject: [PATCH 02/12] Update CHANGELOG.md with details of the new dependencies --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ce7fee72f4..4c3ae600a62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,11 @@ - Fixed a bug where the automatic cleanup of temporary tables could interfere with the results of async query execution. - Fixed a bug in `DataFrame.analytics.time_series_agg` function to handle multiple data points in same sliding interval. +#### Dependency Updates + +- Added a dependency on `protobuf>=5.28` and `tzlocal` at runtime. +- Added a dependency on `protoc-wheel-0` for the development profile. + ### Snowpark pandas API Updates #### New Features From 0aab748a4c3e4d0b024511da8f33c2de1ce0bd6a Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Thu, 24 Oct 2024 10:22:14 -0700 Subject: [PATCH 03/12] Server-side Snowpark: Drop unnecessary dependency --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 2d512b01579..25222c43411 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,6 @@ DEVELOPMENT_REQUIREMENTS = [ "pytest<8.0.0", # check SNOW-1022240 for more details on the pin here "pytest-cov", - "wrapt", "coverage", "sphinx==5.0.2", "cachetools", # used in UDF doctest From afe1ca74ec481058a8fb602ffe3498c5d7a5330b Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Thu, 24 Oct 2024 10:22:46 -0700 Subject: [PATCH 04/12] Server-side Snowpark: Drop unused vcrpy dependencies and remove duplicate entries --- setup.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/setup.py b/setup.py index 25222c43411..854701b1aeb 100644 --- a/setup.py +++ b/setup.py @@ -54,14 +54,6 @@ "pytest-assume", # sql counter check "decorator", # sql counter check "protoc-wheel-0", - "aiohttp", # vcrpy requirements. - "boto", # vcrpy requirements. - "httplib2", # vcrpy requirements. - "httpx", # vcrpy requirements. - "tornado", # vcrpy requirements. - "graphviz", # used in plot tests - "pytest-assume", # sql counter check - "decorator", # sql counter check ] # read the version From 77161dbd9c87f770f72621f0cfeae86c472e9d5b Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Thu, 24 Oct 2024 10:25:23 -0700 Subject: [PATCH 05/12] Server-side Snowpark: Call out the change in connector requirements --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c3ae600a62..d71d6acf2d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - Added a dependency on `protobuf>=5.28` and `tzlocal` at runtime. - Added a dependency on `protoc-wheel-0` for the development profile. +- Require `snowflake-connector-python>=3.12.0, <4.0.0` (was `>=3.10.0`). ### Snowpark pandas API Updates From 116a17aba36bd699332fadbd7a475e77cf39aaff Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Thu, 24 Oct 2024 13:36:08 -0700 Subject: [PATCH 06/12] Server-side Snowpark: Run Snyk unconditionally --- .github/workflows/snyk-pr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml index 36c8294ae27..4384f40317b 100644 --- a/.github/workflows/snyk-pr.yml +++ b/.github/workflows/snyk-pr.yml @@ -12,7 +12,6 @@ permissions: jobs: snyk: runs-on: ubuntu-latest - if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }} steps: - name: Checkout uses: actions/checkout@v4 From e218134b02b8bc3e107e05b987714a2a2dfd75ee Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Thu, 24 Oct 2024 13:49:17 -0700 Subject: [PATCH 07/12] Server-side Snowpark: Make pr_title uppercase for Snyk --- .github/workflows/snyk-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml index 4384f40317b..848fcb0a4b4 100644 --- a/.github/workflows/snyk-pr.yml +++ b/.github/workflows/snyk-pr.yml @@ -29,7 +29,7 @@ jobs: - name: Snyk Pull Request Scan Check uses: ./whitesource-actions/snyk-pr env: - pr_title: ${{ github.event.pull_request.title }} + PR_TITLE: ${{ github.event.pull_request.title }} with: jira_token: ${{ secrets.jira_token_public_repo }} gh_token: ${{ secrets.github_token }} From c502394c7328523c829036cc33fb15e7673354cc Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Fri, 25 Oct 2024 09:49:00 -0700 Subject: [PATCH 08/12] Revert "Server-side Snowpark: Make pr_title uppercase for Snyk" This reverts commit e218134b02b8bc3e107e05b987714a2a2dfd75ee. --- .github/workflows/snyk-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml index 848fcb0a4b4..4384f40317b 100644 --- a/.github/workflows/snyk-pr.yml +++ b/.github/workflows/snyk-pr.yml @@ -29,7 +29,7 @@ jobs: - name: Snyk Pull Request Scan Check uses: ./whitesource-actions/snyk-pr env: - PR_TITLE: ${{ github.event.pull_request.title }} + pr_title: ${{ github.event.pull_request.title }} with: jira_token: ${{ secrets.jira_token_public_repo }} gh_token: ${{ secrets.github_token }} From 02f95fc57245216eafb545692f04f6f4523c0515 Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Fri, 25 Oct 2024 09:49:14 -0700 Subject: [PATCH 09/12] Revert "Server-side Snowpark: Run Snyk unconditionally" This reverts commit 116a17aba36bd699332fadbd7a475e77cf39aaff. --- .github/workflows/snyk-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml index 4384f40317b..36c8294ae27 100644 --- a/.github/workflows/snyk-pr.yml +++ b/.github/workflows/snyk-pr.yml @@ -12,6 +12,7 @@ permissions: jobs: snyk: runs-on: ubuntu-latest + if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }} steps: - name: Checkout uses: actions/checkout@v4 From 29cf1420a519a8a7193dc57880726b1bfe381f9a Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Fri, 25 Oct 2024 09:57:29 -0700 Subject: [PATCH 10/12] Server-side Snowpark: Fix incorrect merge conflict resolution --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a6b31c5ac5..141f92cc620 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,7 +47,8 @@ - Added support for `on` parameter with `Resampler`. - Added support for timedelta inputs in `value_counts()`. - Added support for applying Snowpark Python function `snowflake_cortex_summarize`. -- Added support for `DataFrame`/`Series.attrs` +- Added support for `DataFrame.attrs` and `Series.attrs`. +- Added support for `DataFrame.style`. #### Improvements From bb74ce4e406764a64cd9ecb2247c33446eaf2ce4 Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Fri, 25 Oct 2024 11:30:25 -0700 Subject: [PATCH 11/12] Server-side Snowpark: Explain rationale for dependencies --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 854701b1aeb..ebbcd107764 100644 --- a/setup.py +++ b/setup.py @@ -21,11 +21,11 @@ CONNECTOR_DEPENDENCY, # snowpark directly depends on typing-extension, so we should not remove it even if connector also depends on it. "typing-extensions>=4.1.0, <5.0.0", - "protobuf>=5.28", - "tzlocal", "pyyaml", "cloudpickle>=1.6.0,<=2.2.1,!=2.1.0,!=2.2.0;python_version<'3.11'", "cloudpickle==2.2.1;python_version~='3.11'", # backend only supports cloudpickle 2.2.1 + python 3.11 at the moment + "protobuf>=5.28", # Server-side Snowpark + "tzlocal", # Server-side Snowpark ] REQUIRED_PYTHON_VERSION = ">=3.8, <3.12" @@ -53,7 +53,7 @@ "graphviz", # used in plot tests "pytest-assume", # sql counter check "decorator", # sql counter check - "protoc-wheel-0", + "protoc-wheel-0", # Protocol buffer compiler, for server-side Snowpark ] # read the version From 5dcddd17108c52c1121fa1bbe1268d86661c842c Mon Sep 17 00:00:00 2001 From: Ovidiu Platon Date: Fri, 25 Oct 2024 15:36:40 -0700 Subject: [PATCH 12/12] Server-side Snowpark: Refer to the project as 'Snowpark IR' in setup.py --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index ebbcd107764..fc50573905f 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,8 @@ "pyyaml", "cloudpickle>=1.6.0,<=2.2.1,!=2.1.0,!=2.2.0;python_version<'3.11'", "cloudpickle==2.2.1;python_version~='3.11'", # backend only supports cloudpickle 2.2.1 + python 3.11 at the moment - "protobuf>=5.28", # Server-side Snowpark - "tzlocal", # Server-side Snowpark + "protobuf>=5.28", # Snowpark IR + "tzlocal", # Snowpark IR ] REQUIRED_PYTHON_VERSION = ">=3.8, <3.12" @@ -53,7 +53,7 @@ "graphviz", # used in plot tests "pytest-assume", # sql counter check "decorator", # sql counter check - "protoc-wheel-0", # Protocol buffer compiler, for server-side Snowpark + "protoc-wheel-0", # Protocol buffer compiler, for Snowpark IR ] # read the version