Consolidate all requirements (#2597)

* Empty

Signed-off-by: Juan Luis Cano Rodríguez <[email protected]>

* Avoid adding problematic lines to requirements.txt

kedro is preinstalled in the behave environment,
so this should not be needed.
In exchange, requirements.txt files can always be read
by setuptools automatic parsing.

See McK-Private/private-kedro#352 (comment)
for motivation of the original logic.

Signed-off-by: Juan Luis Cano Rodríguez <[email protected]>

* Consolidate all requirements

Fix gh-2588.

Signed-off-by: Juan Luis Cano Rodríguez <[email protected]>

* Fix dependencies

Signed-off-by: Juan Luis Cano Rodríguez <[email protected]>

* Ignore import order in pylint in favour of isort

Signed-off-by: Juan Luis Cano Rodríguez <[email protected]>

* Do not replace kedro plugins in e2e tests

Signed-off-by: Juan Luis Cano Rodríguez <[email protected]>

* Fix dependabot config

Signed-off-by: Juan Luis Cano Rodríguez <[email protected]>

* Add release notes

Signed-off-by: Juan Luis Cano Rodríguez <[email protected]>

---------

Signed-off-by: Juan Luis Cano Rodríguez <[email protected]>
Signed-off-by: Nok <[email protected]>

.circleci/continue_config.yml

@@ -61,7 +61,7 @@ commands:
           command: conda install -c conda-forge pytables -y
       - run:
           name: Install requirements and test requirements
-          command: pip install --upgrade -r test_requirements.txt
+          command: pip install --upgrade .[test]
       - run:
           # this is needed to fix java cacerts so
           # spark can automatically download packages from mvn
@@ -146,7 +146,7 @@ commands:
     steps:
       - restore_cache:
           name: Restore package cache
-          key: kedro-deps-v1-win-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+          key: kedro-deps-v1-win-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
         # We don't restore the conda environment cache for python 3.10 as it conflicts with the
         # 'Install GDAL, Fiona and pytables' step breaking the conda environment (missing zlib.dll).
       - unless:
@@ -155,7 +155,7 @@ commands:
           steps:
             - restore_cache:
                 name: Restore conda environment cache
-                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
         # pytables and Fiona have a series of binary dependencies under Windows that
         # are best handled by conda-installing instead of pip-installing them.
         # Dependency resolution works best when installing these altogether in one
@@ -168,7 +168,7 @@ commands:
           command: conda activate kedro_builder; pip debug --verbose
       - run:
           name: Install all requirements
-          command: conda activate kedro_builder; pip install -v -r test_requirements.txt -U
+          command: conda activate kedro_builder; pip install -v -U .[test]
       - run:
           name: Print Python environment
           command: conda activate kedro_builder; make print-python-env
@@ -337,7 +337,7 @@ jobs:
           steps:
           - save_cache:
               name: Save Python package cache
-              key: kedro-deps-v1-win-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+              key: kedro-deps-v1-win-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
               paths:
                 # Cache pip cache and conda packages directories
                 - c:\tools\miniconda3\pkgs
@@ -350,7 +350,7 @@ jobs:
           steps:
             - save_cache:
                 name: Save conda environment cache
-                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "dependency/requirements.txt" }}-{{ checksum "test_requirements.txt" }}
+                key: kedro-deps-v1-win-<<parameters.python_version>>-{{ checksum "pyproject.toml" }}-{{ checksum "setup.py" }}
                 paths:
                   - c:\tools\miniconda3\envs\kedro_builder
       - run:

.github/dependabot.yml

@@ -6,7 +6,7 @@
 version: 2
 updates:
   - package-ecosystem: "pip" # See documentation for possible values
-    directory: "/dependency" # Location of package manifests
+    directory: "/" # Location of package manifests
     schedule:
       interval: "weekly"
     labels:

.gitpod.yml

@@ -5,10 +5,9 @@ tasks:
 
     init: |
       make sign-off
-      pip install -e /workspace/kedro
+      pip install -e /workspace/kedro[test]
       cd /workspace
       yes project | kedro new -s pandas-iris --checkout main
-      pip install -r /workspace/kedro/test_requirements.txt
       cd /workspace/kedro
       pre-commit install --install-hooks
 

.readthedocs.yml

@@ -36,4 +36,4 @@ python:
       path: .
       extra_requirements:
         - docs
-    - requirements: test_requirements.txt
+        - test

MANIFEST.in

@@ -1,7 +1,5 @@
 include README.md
 include LICENSE.md
-include dependency/requirements.txt
-include test_requirements.txt
 include kedro/framework/project/default_logging.yml
 include kedro/ipython/*.png
 include kedro/ipython/*.svg

Makefile

@@ -48,7 +48,7 @@ package: clean install
 	python -m pip install build && python -m build
 
 install-test-requirements:
-	pip install -r test_requirements.txt
+	pip install .[test]
 
 install-pre-commit: install-test-requirements
 	pre-commit install --install-hooks

RELEASE.md

@@ -15,6 +15,7 @@
 * Activated all built-in resolvers by default for `OmegaConfigLoader` except for `oc.env`.
 
 ## Bug fixes and other changes
+* Consolidated dependencies and optional dependencies in `pyproject.toml`.
 
 ## Documentation changes
 

features/environment.py

@@ -56,7 +56,6 @@ def _setup_context_with_venv(context, venv_dir):
     context.pip = str(bin_dir / "pip")
     context.python = str(bin_dir / "python")
     context.kedro = str(bin_dir / "kedro")
-    context.requirements_path = Path("dependency/requirements.txt").resolve()
 
     # clone the environment, remove any condas and venvs and insert our venv
     context.env = os.environ.copy()

features/steps/cli_steps.py

@@ -11,6 +11,7 @@
 import toml
 import yaml
 from behave import given, then, when
+from packaging.requirements import Requirement
 
 import kedro
 from features.steps import util
@@ -407,18 +408,16 @@ def update_pyproject_toml(context: behave.runner.Context, new_source_dir):
 
 @given("I have updated kedro requirements")
 def update_kedro_req(context: behave.runner.Context):
-    """Replace kedro as a standalone requirement with a line
-    that includes all of kedro's dependencies (-r kedro/requirements.txt)
-    """
+    """Remove kedro as a standalone requirement."""
     reqs_path = context.root_project_dir / "src" / "requirements.txt"
-    kedro_reqs = f"-r {context.requirements_path.as_posix()}"
 
     if reqs_path.is_file():
         old_reqs = reqs_path.read_text().splitlines()
         new_reqs = []
         for req in old_reqs:
-            if req.startswith("kedro"):
-                new_reqs.append(kedro_reqs)
+            if req.startswith("kedro") and Requirement(req).name.lower() == "kedro":
+                # Do not include kedro as it's preinstalled in the environment
+                pass
             else:
                 new_reqs.append(req)
         new_reqs = "\n".join(new_reqs)

features/windows_reqs.txt

@@ -1,4 +1,4 @@
-# same versions as `test_requirements`
+# same versions as [test] optional requirements
 # e2e tests on Windows are slow but we don't need to install
 # everything, so just this subset will be enough for CI
 behave==1.2.6
@@ -7,3 +7,4 @@ psutil~=5.8
 requests~=2.20
 toml~=0.10.1
 PyYAML>=4.2, <7.0
+packaging>=20.0

pyproject.toml

@@ -11,6 +11,32 @@ authors = [
 ]
 description = "Kedro helps you build production-ready data and analytics pipelines"
 requires-python = ">=3.7"
+dependencies = [
+    "anyconfig~=0.10.0",
+    "attrs>=21.3",
+    "build",
+    "cachetools~=5.3",
+    "click<9.0",
+    "cookiecutter>=2.1.1, <3.0",
+    "dynaconf>=3.1.2, <4.0",
+    "fsspec>=2021.4, <2024.1",  # Upper bound set arbitrarily, to be reassessed in early 2024
+    "gitpython~=3.0",
+    "importlib-metadata>=3.6; python_version >= '3.8'",
+    "importlib_metadata>=3.6, <5.0; python_version < '3.8'",  # The "selectable" entry points were introduced in `importlib_metadata` 3.6 and Python 3.10. Bandit on Python 3.7 relies on a library with `importlib_metadata` < 5.0
+    "importlib_resources>=1.3",  # The `files()` API was introduced in `importlib_resources` 1.3 and Python 3.9.
+    "jmespath>=0.9.5, <1.0",
+    "more_itertools~=9.0",
+    "omegaconf~=2.3",
+    "parse~=1.19.0",
+    "pip-tools~=6.5",
+    "pluggy~=1.0",
+    "PyYAML>=4.2, <7.0",
+    "rich>=12.0, <14.0",
+    "rope>=0.21, <2.0",  # subject to LGPLv3 license
+    "setuptools>=65.5.1",
+    "toml~=0.10",
+    "toposort~=1.5",  # Needs to be at least 1.5 to be able to raise CircularDependencyError
+]
 keywords = [
     "pipelines",
     "machine learning",
@@ -26,7 +52,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
 ]
-dynamic = ["readme", "version", "dependencies", "optional-dependencies"]
+dynamic = ["readme", "version", "optional-dependencies"]
 
 [project.urls]
 Homepage = "https://kedro.org"
@@ -46,7 +72,6 @@ include = ["kedro*"]
 [tool.setuptools.dynamic]
 readme = {file = "README.md", content-type = "text/markdown"}
 version = {attr = "kedro.__version__"}
-dependencies = {file = "dependency/requirements.txt"}
 
 [tool.black]
 exclude = "/templates/|^features/steps/test_starter"
@@ -67,7 +92,8 @@ unsafe-load-any-extension = false
 [tool.pylint.messages_control]
 disable = [
     "ungrouped-imports",
-    "duplicate-code"
+    "duplicate-code",
+    "wrong-import-order",  # taken care of by isort
 ]
 enable = ["useless-suppression"]
 [tool.pylint.refactoring]

setup.py

@@ -1,23 +1,14 @@
-from codecs import open
 from glob import glob
 from itertools import chain
-from os import path
 
 from setuptools import setup
 
-name = "kedro"
-here = path.abspath(path.dirname(__file__))
-
 # at least 1.3 to be able to use XMLDataSet and pandas integration with fsspec
 PANDAS = "pandas~=1.3"
 SPARK = "pyspark>=2.2, <4.0"
 HDFS = "hdfs>=2.5.8, <3.0"
 S3FS = "s3fs>=0.3.0, <0.5"
 
-# get the dependencies and installs
-with open("dependency/requirements.txt", encoding="utf-8") as f:
-    requires = [x.strip() for x in f if x.strip()]
-
 template_files = []
 for pattern in ["**/*", "**/.*", "**/.*/**", "**/.*/.**"]:
     template_files.extend(
@@ -80,7 +71,9 @@ def _collect_requirements(requires):
     "tensorflow.TensorflowModelDataset": [
         # currently only TensorFlow V2 supported for saving and loading.
         # V1 requires HDF5 and serialises differently
-        "tensorflow~=2.0"
+        "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'",
+        # https://developer.apple.com/metal/tensorflow-plugin/
+        "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'",
     ]
 }
 yaml_require = {"yaml.YAMLDataSet": [PANDAS, "PyYAML>=4.2, <7.0"]}
@@ -139,10 +132,77 @@ def _collect_requirements(requires):
 }
 
 extras_require["all"] = _collect_requirements(extras_require)
+extras_require["test"] = [
+    "adlfs>=2021.7.1, <=2022.2; python_version == '3.7'",
+    "adlfs~=2023.1; python_version >= '3.8'",
+    "bandit>=1.6.2, <2.0",
+    "behave==1.2.6",
+    "biopython~=1.73",
+    "blacken-docs==1.9.2",
+    "black~=22.0",
+    "compress-pickle[lz4]~=2.1.0",
+    "coverage[toml]",
+    "dask[complete]~=2021.10",  # pinned by Snyk to avoid a vulnerability
+    "delta-spark~=1.2.1",  # 1.2.0 has a bug that breaks some of our tests: https://github.com/delta-io/delta/issues/1070
+    "dill~=0.3.1",
+    "filelock>=3.4.0, <4.0",
+    "gcsfs>=2021.4, <=2023.1; python_version == '3.7'",
+    "gcsfs>=2023.1, <2023.3; python_version >= '3.8'",
+    "geopandas>=0.6.0, <1.0",
+    "hdfs>=2.5.8, <3.0",
+    "holoviews~=1.13.0",
+    "import-linter[toml]==1.8.0",
+    "ipython>=7.31.1, <8.0; python_version < '3.8'",
+    "ipython~=8.10; python_version >= '3.8'",
+    "isort~=5.0",
+    "Jinja2<3.1.0",
+    "joblib>=0.14",
+    "jupyterlab_server>=2.11.1, <2.16.0",  # 2.16.0 requires importlib_metedata >= 4.8.3 which conflicts with flake8 requirement
+    "jupyterlab~=3.0, <3.6.0",  # 3.6.0 requires jupyterlab_server~=2.19
+    "jupyter~=1.0",
+    "lxml~=4.6",
+    "matplotlib>=3.0.3, <3.4; python_version < '3.10'",  # 3.4.0 breaks holoviews
+    "matplotlib>=3.5, <3.6; python_version == '3.10'",
+    "memory_profiler>=0.50.0, <1.0",
+    "moto==1.3.7; python_version < '3.10'",
+    "moto==3.0.4; python_version == '3.10'",
+    "networkx~=2.4",
+    "opencv-python~=4.5.5.64",
+    "openpyxl>=3.0.3, <4.0",
+    "pandas-gbq>=0.12.0, <0.18.0",
+    "pandas~=1.3  # 1.3 for read_xml/to_xml",
+    "Pillow~=9.0",
+    "plotly>=4.8.0, <6.0",
+    "pre-commit>=2.9.2, <3.0",  # The hook `mypy` requires pre-commit version 2.9.2.
+    "psutil~=5.8",
+    "pyarrow>=6.0",
+    "pylint>=2.17.0, <3.0",
+    "pyproj~=3.0",
+    "pyspark>=2.2, <4.0",
+    "pytest-cov~=3.0",
+    "pytest-mock>=1.7.1, <2.0",
+    "pytest-xdist[psutil]~=2.2.1",
+    "pytest~=7.2",
+    "redis~=4.1",
+    "requests-mock~=1.6",
+    "requests~=2.20",
+    "s3fs>=0.3.0, <0.5",  # Needs to be at least 0.3.0 to make use of `cachable` attribute on S3FileSystem.
+    "scikit-learn~=1.0.2",
+    "scipy~=1.7.3",
+    "SQLAlchemy~=1.2",
+    "tables~=3.6.0; platform_system == 'Windows' and python_version<'3.9'",
+    "tables~=3.6; platform_system != 'Windows'",
+    "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'",
+    # https://developer.apple.com/metal/tensorflow-plugin/
+    "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'",
+    "triad>=0.6.7, <1.0",
+    "trufflehog~=2.1",
+    "xlsxwriter~=1.0",
+]
 
 setup(
     package_data={
-        name: ["py.typed", "test_requirements.txt"] + template_files
+        "kedro": ["py.typed"] + template_files
     },
     extras_require=extras_require,
 )