From 27af3982ac517d1928df985a5e7a80c9b086b0fb Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Mon, 14 Oct 2024 16:29:17 -0700 Subject: [PATCH 01/17] move to pyproject.toml --- .gitignore | 3 + .pre-commit-config.yaml | 70 +- MANIFEST.in | 3 - dev-requirements.txt | 5 - docs-requirements.txt | 4 - lume_model/__init__.py | 8 +- lume_model/_version.py | 556 ------------ pyproject.toml | 70 ++ setup.cfg | 6 - setup.py | 39 - versioneer.py | 1885 --------------------------------------- 11 files changed, 121 insertions(+), 2528 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 dev-requirements.txt delete mode 100644 docs-requirements.txt delete mode 100644 lume_model/_version.py create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 versioneer.py diff --git a/.gitignore b/.gitignore index 25a1906..23d8836 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Auto-generated version file +lume_model/_version.py + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f94e2e8..556b4b4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,28 +1,46 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.1.0 + - repo: https://github.com/pre-commit/pre-commit-hooks.git + rev: v4.6.0 hooks: - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace -- repo: local - hooks: - - id: black - name: black - entry: black - language: python - language_version: python3 - types: [python] - args: # arguments to configure black - - --line-length=88 - - id: flake8 - name: flake8 - entry: flake8 - language: python - language_version: python3 - types: [python] - # only in lume_services - files: ^lume_services - args: # arguments to configure flake8 - # making isort line length compatible with black - - "--max-line-length=88" + - id: no-commit-to-branch + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-ast + - id: check-case-conflict + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-xml + - id: check-yaml + exclude: "^(python/lume_model/_version.py)$" + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.0 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format + +#- repo: local +# hooks: +# - id: black +# name: black +# entry: black +# language: python +# language_version: python3 +# types: [python] +# args: # arguments to configure black +# - --line-length=88 +# - id: flake8 +# name: flake8 +# entry: flake8 +# language: python +# language_version: python3 +# types: [python] +# # only in lume_services +# files: ^lume_services +# args: # arguments to configure flake8 +# # making isort line length compatible with black +# - "--max-line-length=88" diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 7b6e3eb..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include versioneer.py -include lume_model/_version.py -include tests/test_files/*/* diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 1fc4470..0000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -black -pre-commit -pytest -tensorflow -botorch diff --git a/docs-requirements.txt b/docs-requirements.txt deleted file mode 100644 index f53ecff..0000000 --- a/docs-requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -mkdocs -mkdocstrings -mkdocstrings-python -mkdocs-material diff --git a/lume_model/__init__.py b/lume_model/__init__.py index 80edaf0..654639f 100644 --- a/lume_model/__init__.py +++ b/lume_model/__init__.py @@ -1,4 +1,4 @@ -from ._version import get_versions - -__version__ = get_versions()["version"] -del get_versions +try: + from ._version import __version__ +except ImportError: + __version__ = "0.0.0" diff --git a/lume_model/_version.py b/lume_model/_version.py deleted file mode 100644 index f6c15b8..0000000 --- a/lume_model/_version.py +++ /dev/null @@ -1,556 +0,0 @@ -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "v" - cfg.parentdir_prefix = "None" - cfg.versionfile_source = "lume_model/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..180b8c1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,70 @@ +[build-system] +requires = ["setuptools>=64.0", "setuptools_scm[toml]"] +build-backend = "setuptools.build_meta" + +[project] +name = "lume-model" +description = "Data structures used in the LUME modeling toolset." +readme = "README.md" +authors = [ + { name = "SLAC National Accelerator Laboratory"}, +] +keywords = ["machine learning", "accelerator physics"] +classifiers = [ + "Natural Language :: English", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering", +] +requires-python = ">=3.9" +dependencies = [ + "pydantic", + "numpy", + "pyyaml" +] +dynamic = ["version"] +[tool.setuptools_scm] +version_file = "lume_model/_version.py" + +[project.optional-dependencies] +dev = [ + "tensorflow<2.16.1", + "keras<3", + "botorch<0.11", + "pre-commit", + "ruff", + "pytest" +] +docs = [ + "mkdocs", + "mkdocstrings", + "mkdocstrings-python", + "mkdocs-material" +] + +[project.urls] +Homepage = "https://github.com/slaclab/lume-model" +Documentation = "https://slaclab.github.io/lume-model/" +"Bug Tracker" = "https://github.com/slaclab/lume-model/issues" + +[project.license] +file = "LICENSE" + +[options] +zip_safe = false +include_package_data = true + +[tool.setuptools.packages.find] +include = [ "lume_model", ] +namespaces = false + +[tool.ruff] + +[tool.pytest.ini_options] +log_cli_level = "info" +log_level = "debug" +testpaths = ["tests"] \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 2fc218f..0000000 --- a/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[versioneer] -VCS = git -style = pep440 -versionfile_source = lume_model/_version.py -versionfile_build = lume_model/_version.py -tag_prefix = v diff --git a/setup.py b/setup.py deleted file mode 100644 index 268d831..0000000 --- a/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -from setuptools import setup, find_packages -from os import path, environ -import versioneer - -cur_dir = path.abspath(path.dirname(__file__)) - -# parse requirements -with open(path.join(cur_dir, "requirements.txt"), "r") as f: - requirements = f.read().split() - -# set up additional dev requirements -dev_requirements = [] -with open(path.join(cur_dir, "dev-requirements.txt"), "r") as f: - dev_requirements = f.read().split() - -docs_requirements = [] -with open(path.join(cur_dir, "docs-requirements.txt"), "r") as f: - docs_requirements = f.read().split() - - -setup( - name="lume-model", - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - author="SLAC National Accelerator Laboratory", - license="SLAC Open", - packages=find_packages(), - install_requires=requirements, - # set up development requirements - extras_require={ - "dev": dev_requirements, - "docs": docs_requirements, - "keras": ["tensorflow"], - "torch": ["botorch"], - }, - url="https://github.com/slaclab/lume-model", - include_package_data=True, - python_requires=">=3.9", -) diff --git a/versioneer.py b/versioneer.py deleted file mode 100644 index 2b54540..0000000 --- a/versioneer.py +++ /dev/null @@ -1,1885 +0,0 @@ -# Version: 0.18 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - -from __future__ import print_function - -try: - import configparser -except ImportError: - import ConfigParser as configparser -import errno -import json -import os -import re -import subprocess -import sys - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -LONG_VERSION_PY[ - "git" -] = ''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S - ) - if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S - ) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Eexceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if "py2exe" in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except ( - EnvironmentError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", file=sys.stderr) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print( - " appending versionfile_source ('%s') to MANIFEST.in" - % cfg.versionfile_source - ) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) From b083db20e830d5b79941db33eab6e84bea43e9e0 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Mon, 14 Oct 2024 16:33:09 -0700 Subject: [PATCH 02/17] adjust test env in workflow --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 62723d0..d62d25e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: name: Python ${{ matrix.python-version }}, ${{ matrix.os }} build steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: conda-incubator/setup-miniconda@v2 with: python-version: ${{ matrix.python-version }} @@ -27,7 +27,7 @@ jobs: - name: Install lume-model shell: bash -l {0} run: | - pip install -e . + pip install -e ".[dev]" - name: Run Tests shell: bash -l {0} From 823e432119b7d644723b4aacb4007db3c93f0fb5 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Mon, 14 Oct 2024 16:36:32 -0700 Subject: [PATCH 03/17] adjust test env in workflow --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d62d25e..6c39ad3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ["3.9"] + python-version: ["3.9", "3.11", "3.12"] name: Python ${{ matrix.python-version }}, ${{ matrix.os }} build steps: From bc40d3c3a6c58b1d0bdb91b84e93d1da55a8f349 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Mon, 14 Oct 2024 16:54:58 -0700 Subject: [PATCH 04/17] adjust test env in workflow --- .github/workflows/build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6c39ad3..95a77b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,13 +16,12 @@ jobs: name: Python ${{ matrix.python-version }}, ${{ matrix.os }} build steps: - uses: actions/checkout@v4 - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} miniforge-variant: Mambaforge - channels: conda-forge activate-environment: lume-model-dev - environment-file: dev-environment.yml + #environment-file: dev-environment.yml - name: Install lume-model shell: bash -l {0} From 6c4db00c5376bca9b4b0627948f5e0afb1a8e359 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Mon, 14 Oct 2024 17:07:58 -0700 Subject: [PATCH 05/17] adjust deps --- dev-environment.yml | 3 +-- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dev-environment.yml b/dev-environment.yml index b256a6b..ab19eef 100644 --- a/dev-environment.yml +++ b/dev-environment.yml @@ -14,8 +14,7 @@ dependencies: # dev requirements - pytest - pre-commit - - black - - boa + - ruff # docs requirements - mkdocs diff --git a/pyproject.toml b/pyproject.toml index 180b8c1..9dedea5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,8 @@ version_file = "lume_model/_version.py" [project.optional-dependencies] dev = [ - "tensorflow<2.16.1", - "keras<3", + "tensorflow", # has to be >=2.16 to be compatible with python 3.12 + "keras", "botorch<0.11", "pre-commit", "ruff", From 823550e72212d24f0ff8df3c3febaf6e0cca9080 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Mon, 14 Oct 2024 17:12:25 -0700 Subject: [PATCH 06/17] adjust conda dev in test workflow --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95a77b4..d28fbe0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,8 @@ jobs: - uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge + miniforge-version: latest + use-mamba: true activate-environment: lume-model-dev #environment-file: dev-environment.yml From 9143615261449ddd099453764abec05cf874b335 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Tue, 29 Oct 2024 15:52:42 -0700 Subject: [PATCH 07/17] minor adjustments --- pyproject.toml | 2 +- tests/models/test_keras_model.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9dedea5..9fa5e09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ version_file = "lume_model/_version.py" [project.optional-dependencies] dev = [ "tensorflow", # has to be >=2.16 to be compatible with python 3.12 - "keras", + "keras<3.0", "botorch<0.11", "pre-commit", "ruff", diff --git a/tests/models/test_keras_model.py b/tests/models/test_keras_model.py index 45c3783..9e0c1e3 100644 --- a/tests/models/test_keras_model.py +++ b/tests/models/test_keras_model.py @@ -9,7 +9,7 @@ try: from lume_model.models import KerasModel from lume_model.variables import InputVariable, OutputVariable, ScalarOutputVariable -except ImportError: +except ImportError: # TODO: add info when this happens pass From 8cc723177973f602c9ed9d81026ce2c894020877 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Tue, 29 Oct 2024 15:53:51 -0700 Subject: [PATCH 08/17] remove py3.12 from matrix for now --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d28fbe0..2cd51f9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ["3.9", "3.11", "3.12"] + python-version: ["3.9", "3.11"] #, "3.12"] # Until Keras 3.0 compatibility is resolved name: Python ${{ matrix.python-version }}, ${{ matrix.os }} build steps: From 223b8d357ea6b9e3bec274e20617165d88ca8be5 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Wed, 30 Oct 2024 12:36:17 -0700 Subject: [PATCH 09/17] remove keras from reqs for now --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9fa5e09..3d334a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ version_file = "lume_model/_version.py" [project.optional-dependencies] dev = [ "tensorflow", # has to be >=2.16 to be compatible with python 3.12 - "keras<3.0", "botorch<0.11", "pre-commit", "ruff", From 0661a31f1a82fd1ac795eeda64d64f2963dd0d4d Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Wed, 30 Oct 2024 12:49:36 -0700 Subject: [PATCH 10/17] Update build.yml to add channel --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f38dcf3..a5aa03d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,7 @@ jobs: python-version: ${{ matrix.python-version }} miniforge-version: latest use-mamba: true + channels: conda-forge activate-environment: lume-model-dev #environment-file: dev-environment.yml From 52de7307a5e62adb6d2d5b9946244f2f7298260b Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Wed, 30 Oct 2024 12:58:51 -0700 Subject: [PATCH 11/17] fix conda build and add docs build test --- .github/workflows/build.yml | 4 +--- .github/workflows/build_docs.yml | 7 +++---- .github/workflows/test_build_docs.yml | 27 +++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/test_build_docs.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f38dcf3..5ce01b5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,6 @@ jobs: os: [ubuntu-latest] python-version: ["3.9", "3.11", "3.12"] - name: Python ${{ matrix.python-version }}, ${{ matrix.os }} build steps: - uses: actions/checkout@v4 @@ -23,7 +22,6 @@ jobs: miniforge-version: latest use-mamba: true activate-environment: lume-model-dev - #environment-file: dev-environment.yml - name: Install lume-model shell: bash -l {0} @@ -39,4 +37,4 @@ jobs: shell: bash -l {0} run: | mamba install conda-build - conda mambabuild -q conda-recipe --python=${{ matrix.python-version }} --output-folder bld-dir + conda build -q conda-recipe --python=${{ matrix.python-version }} --output-folder bld-dir diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index 1999027..f7db596 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -11,18 +11,17 @@ jobs: runs-on: ubuntu-latest name: Build docs steps: - - uses: actions/checkout@v3 - - uses: conda-incubator/setup-miniconda@v2 + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 with: python-version: 3.9 mamba-version: "*" channels: conda-forge activate-environment: lume-model-dev - environment-file: dev-environment.yml - name: Build package shell: bash -l {0} run: | - pip install -e . + pip install -e ".[docs]" - name: Build docs shell: bash -l {0} run: | diff --git a/.github/workflows/test_build_docs.yml b/.github/workflows/test_build_docs.yml new file mode 100644 index 0000000..8b236df --- /dev/null +++ b/.github/workflows/test_build_docs.yml @@ -0,0 +1,27 @@ +name: Build docs + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + build: + if: ${{ github.repository == 'slaclab/lume-model' }} + runs-on: ubuntu-latest + name: Build docs + steps: + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: 3.9 + mamba-version: "*" + channels: conda-forge + activate-environment: lume-model-dev + - name: Build package + shell: bash -l {0} + run: | + pip install -e ".[docs]" + - name: Build docs + shell: bash -l {0} + run: | + mkdocs build From daf1b45514ffb4e834b13781467c32a3d5fdf4a9 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Wed, 30 Oct 2024 13:07:51 -0700 Subject: [PATCH 12/17] remove conda build test and recipe since it's on conda-forge --- .github/workflows/build.yml | 6 ------ conda-recipe/build.sh | 1 - conda-recipe/meta.yaml | 41 ------------------------------------- 3 files changed, 48 deletions(-) delete mode 100644 conda-recipe/build.sh delete mode 100644 conda-recipe/meta.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4cc096e..bc620c6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,9 +33,3 @@ jobs: shell: bash -l {0} run: | pytest -ra --pyargs tests -k "not keras" - - - name: Build package - shell: bash -l {0} - run: | - mamba install conda-build - conda build -q conda-recipe --python=${{ matrix.python-version }} --output-folder bld-dir diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh deleted file mode 100644 index a660906..0000000 --- a/conda-recipe/build.sh +++ /dev/null @@ -1 +0,0 @@ -$PYTHON setup.py install --single-version-externally-managed --record=record.txt diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml deleted file mode 100644 index 7a23356..0000000 --- a/conda-recipe/meta.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{% set data = load_setup_py_data(setup_file='../setup.py', from_recipe_dir=True) %} - -package: - name: "lume-model" - version: {{ data.get('version') }} - -build: - noarch: python - number: 2 - -source: - path: .. - -requirements: - host: - - python - - setuptools - - pip - - pydantic>2.3 - run: - - python - - pydantic>2.3 - - numpy - - pyyaml - -test: - imports: - - lume_model - requires: - - pytest - commands: - - pytest -ra --pyargs tests - -about: - home: https://github.com/slaclab/lume-model - license: SLAC Open - license_family: OTHER - license_file: LICENSE - summary: Lume-model hosts the variable classes required for running LUME models. - doc_url: https://slaclab.github.io/lume-model/ - dev_url: https://github.com/slaclab/lume-model From 3fecb6925a88f13304239b0d89761c5ad5349290 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Fri, 1 Nov 2024 13:50:04 -0700 Subject: [PATCH 13/17] drop support for keras/tensorflow --- .github/workflows/build.yml | 2 +- README.md | 49 ---- dev-environment.yml | 1 - docs/models.md | 5 - examples/keras_model.ipynb | 220 -------------- github_deploy_key_slaclab_lume_model.enc | 1 - lume_model/base.py | 33 --- lume_model/models/__init__.py | 7 - lume_model/models/keras_layers.py | 178 ------------ lume_model/models/keras_model.py | 268 ------------------ pyproject.toml | 1 - requirements.txt | 2 +- tests/conftest.py | 28 -- tests/models/test_keras_layers.py | 38 --- tests/models/test_keras_model.py | 176 ------------ tests/models/test_models.py | 3 +- .../iris_classification/keras_model.yml | 27 -- .../iris_classification/model.keras | Bin 29368 -> 0 bytes .../iris_classification/test_input_array.npy | Bin 224 -> 0 bytes .../iris_classification/variables.yml | 23 -- 20 files changed, 3 insertions(+), 1059 deletions(-) delete mode 100644 examples/keras_model.ipynb delete mode 100644 github_deploy_key_slaclab_lume_model.enc delete mode 100644 lume_model/models/keras_layers.py delete mode 100644 lume_model/models/keras_model.py delete mode 100644 tests/models/test_keras_layers.py delete mode 100644 tests/models/test_keras_model.py delete mode 100644 tests/test_files/iris_classification/keras_model.yml delete mode 100644 tests/test_files/iris_classification/model.keras delete mode 100644 tests/test_files/iris_classification/test_input_array.npy delete mode 100644 tests/test_files/iris_classification/variables.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bc620c6..775cd0f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,4 +32,4 @@ jobs: - name: Run Tests shell: bash -l {0} run: | - pytest -ra --pyargs tests -k "not keras" + pytest -ra --pyargs tests diff --git a/README.md b/README.md index edcbee7..1e81cf5 100644 --- a/README.md +++ b/README.md @@ -164,55 +164,6 @@ input_variables: ``` -## Keras/tensorflow toolkit - -At present, only the tensorflow v2 backend is supported for this toolkit. - -The `KerasModel` packaged in the toolkit will be compatible with models saved using the `keras.save_model()` method. - -### Development requirements - -* The model must be trained using the custom scaling layers provided in `lume_model.keras.layers` OR using preprocessing layers packaged with Keras OR the custom layers must be defined during build and made accessible during loading by the user. Custom layers are not supported out-of-the box by this toolkit. - -* The keras model must use named input layers such that the model will accept a dictionary input OR the `KerasModel` must be subclassed and the `format_input` and `format_output` member functions must be overwritten with proper formatting of model input from a dictionary mapping input variable names to values and proper output parsing into a dictionary, respectively. This will require use of the Keras functional API for model construction. - -An example of a model built using the functional API is given below: - -```python -from tensorflow import keras -import tensorflow as tf - -sepal_length_input = keras.Input(shape=(1,), name="SepalLength") -sepal_width_input = keras.Input(shape=(1,), name="SepalWidth") -petal_length_input = keras.Input(shape=(1,), name="PetalLength") -petal_width_input = keras.Input(shape=(1,), name="PetalWidth") -inputs = [sepal_length_input, sepal_width_input, petal_length_input, petal_width_input] -merged = keras.layers.concatenate(inputs) -dense1 = Dense(8, activation='relu')(merged) -output = Dense(3, activation='softmax', name="Species")(dense1) - -# Compile model -model = keras.Model(inputs=inputs, outputs=[output]) -optimizer = tf.keras.optimizers.Adam() -model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) - -``` - -Models built in this way will accept inputs in dictionary form mapping variable name to a numpy array of values. - -### Configuration file - -The KerasModel can be instantiated using the utility function `lume_model.utils.model_from_yaml` method. - -KerasModel can be specified in the `model_class` of the model configuration. - -```yaml -model: - model_class: lume_model.keras.KerasModel -``` - -Custom parsing will require a custom model class. - ## PyTorch Toolkit In the same way as the KerasModel, a PyTorchModel can also be loaded using the `lume_model.utils.model_from_yaml` method, specifying `PyTorchModel` in the `model_class` of the configuration file. diff --git a/dev-environment.yml b/dev-environment.yml index 3283d61..a3e14bb 100644 --- a/dev-environment.yml +++ b/dev-environment.yml @@ -7,7 +7,6 @@ dependencies: - pydantic>2.3 - numpy - pyyaml - - tensorflow - botorch<0.11 # dev requirements diff --git a/docs/models.md b/docs/models.md index c66eb0a..d382201 100644 --- a/docs/models.md +++ b/docs/models.md @@ -14,8 +14,3 @@ options: members: - TorchModule - -::: lume_model.models.keras_model - options: - members: - - KerasModel diff --git a/examples/keras_model.ipynb b/examples/keras_model.ipynb deleted file mode 100644 index c9afcf7..0000000 --- a/examples/keras_model.ipynb +++ /dev/null @@ -1,220 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "235c92cd-cc05-42b8-a516-1185eeac5f0c", - "metadata": {}, - "source": [ - "# Creating a KerasModel\n", - "Base models built in Keras are already supported by LUME-model. We demonstrate how to create and execute a `KerasModel` below." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "56725817-2b21-4bea-98b0-151dea959f77", - "metadata": {}, - "outputs": [], - "source": [ - "import keras\n", - "import numpy as np\n", - "\n", - "from lume_model.models import KerasModel\n", - "from lume_model.variables import ScalarInputVariable, ScalarOutputVariable" - ] - }, - { - "cell_type": "markdown", - "id": "79c62b18-7dc1-44ca-b578-4dea5cc4a4b4", - "metadata": {}, - "source": [ - "## Building a Model from Scratch\n", - "Instantiation of a `KerasModel` requires specification of the base model (`keras.Model` with named inputs) and in-/output variables." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "f96d9863-269c-49d8-9671-cc73a783bcbc", - "metadata": {}, - "outputs": [], - "source": [ - "# exemplary model definition\n", - "inputs = [keras.Input(name=\"input1\", shape=(1,)), keras.Input(name=\"input2\", shape=(1,))]\n", - "outputs = keras.layers.Dense(1, activation=keras.activations.relu)(keras.layers.concatenate(inputs))\n", - "base_model = keras.Model(inputs=inputs, outputs=outputs)\n", - "\n", - "# variable specification\n", - "input_variables = [\n", - " ScalarInputVariable(name=inputs[0].name, default=0.1, value_range=[0.0, 1.0]),\n", - " ScalarInputVariable(name=inputs[1].name, default=0.2, value_range=[0.0, 1.0]),\n", - "]\n", - "output_variables = [\n", - " ScalarOutputVariable(name=\"output\"),\n", - "]\n", - "\n", - "# creation of KerasModel\n", - "example_model = KerasModel(\n", - " model=base_model,\n", - " input_variables=input_variables,\n", - " output_variables=output_variables,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "d22e1cdd-0ea7-4a75-a2ed-47e6a77dac85", - "metadata": {}, - "source": [ - "## Loading a Model from File\n", - "An already created model can be saved to a YAML file by calling the `dump` method. The model can then be loaded by simply passing the file to the constructor." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "b32234ad-adcb-4431-940b-e5377cfa4e4a", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "model_class: KerasModel\n", - "input_variables:\n", - " SepalLength:\n", - " variable_type: scalar\n", - " default: 4.3\n", - " is_constant: false\n", - " value_range: [4.3, 7.9]\n", - " SepalWidth:\n", - " variable_type: scalar\n", - " default: 2.0\n", - " is_constant: false\n", - " value_range: [2.0, 6.9]\n", - " PetalLength:\n", - " variable_type: scalar\n", - " default: 1.0\n", - " is_constant: false\n", - " value_range: [1.0, 6.9]\n", - " PetalWidth:\n", - " variable_type: scalar\n", - " default: 0.1\n", - " is_constant: false\n", - " value_range: [0.1, 2.5]\n", - "output_variables:\n", - " Species: {variable_type: scalar}\n", - "model: model.keras\n", - "output_format: array\n", - "output_transforms: [softmax]\n", - "\n" - ] - } - ], - "source": [ - "keras_model = KerasModel(\"../tests/test_files/iris_classification/keras_model.yml\")\n", - "print(keras_model.yaml())" - ] - }, - { - "cell_type": "markdown", - "id": "868fff4d-1f46-48e2-8bd0-c9d831df79e6", - "metadata": {}, - "source": [ - "## Model Execution\n", - "Calling the `evaluate` method allows for model execution on dictionary input." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "97946e64-062d-47d4-8d0c-d7e02a335a56", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'SepalLength': array([7.40696632]),\n", - " 'SepalWidth': array([6.5843979]),\n", - " 'PetalLength': array([1.06113014]),\n", - " 'PetalWidth': array([1.31041352])}" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# generate exemplary input\n", - "input_dict = keras_model.random_input(n_samples=1)\n", - "input_dict" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "50aae4be-0d6e-456f-83e8-3a84d6d78f84", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1/1 [==============================] - 0s 45ms/step\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2023-11-09 11:53:54.522723: W tensorflow/tsl/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n" - ] - }, - { - "data": { - "text/plain": [ - "{'Species': array(0)}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# execute KerasModel\n", - "keras_model.evaluate(input_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "55d12bdc-ed38-401d-8bf8-bea92f4456bc", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python [conda env:lume-model-dev]", - "language": "python", - "name": "conda-env-lume-model-dev-py" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/github_deploy_key_slaclab_lume_model.enc b/github_deploy_key_slaclab_lume_model.enc deleted file mode 100644 index 43b4b7b..0000000 --- a/github_deploy_key_slaclab_lume_model.enc +++ /dev/null @@ -1 +0,0 @@ -gAAAAABfNvgSoQV2o_Qh9uJNhuJR22cSzxQgkvki_9AXXHFGtgESuQPLAYbSfhhZrs7kYwujJ_REBryXIoH5DCoB3g9wWxVFgpgW3gnjQWVujrCA4rl4el2hB2klW1F1q3yW7e5r3pOmFQ2CpKZdbzji7bvpKlLSwtpB-DZQePRqLofDWz1PcpWrRi29tAcJn5k4p9awhBuWpCCrbzRXsk9cmugR-q3j3tGMNsEhVFmsymP_x2o92aZCJB5Q7FwoKbkatT8Mg2h0aF8FHKJGizRk0Utvoc0UYjMcfd9eTxsBVs3ftvViruyApMZ8gaVEh_cDEKc6xxgFj7vdU6jU4jw9CBan_9jYQ5717ExFSvHKpMJeE4-REP4Apjpx9UhBKJR_D4Vg9HJ9K9qMf0lFY5AQFL1PL3euunWGbOxlOOFfG7Mi3fdpN02fmjqw2AJvnNiYsO7cly_rg_TdNUg92J81CeTC9YN2NNvzQQ9ZXQES6LHGTM30DkOq5viM1TTsMY-aqzqCArUFaIJOoF14DdKLoq2k0ZM5TPtw15wu4ammSZDRujAQVKkqVp4qHFtGgU_lDNNqsDrHoiqtOSmuJlWtdYZp78R4u2XxG1pPY7eiqhSrgg5ZxHeOkoLojvCLxa1d5xrN1BwUTpRA3jrjUT52hrWoep_QPy_pWK0k8LPU5bjg8WFLXzTKA3YK7sxPDHCPucA-43BSuCz3UBnyKMzGX4R3MTVokq-gyxNXEdeW1yq8Ld5BGq0zmvczQ1relPVzoObbMQcFB7GRmlxs0qnSpHI8W5dlCmb4yI6T-kSmGCwCgcLlmhrHRzdyUcT38soZ8VICwty1zAEvlA-aoY7TNGiroHaBdX3F7LPaIOQUwjIRYRBx29GDeRm-kL1PFRi6vfFfiMXxlaYHdcykObcZcnktD7Wpl9a1XP2Ss1xWZTw3NfybG_WpzpfFpWEo-ynctkQznfXeLlq4WJy603L9I0wHk4Jsr1l5vKgWBol2b-MAcXz9aEZ9SWr8d-AyHCu9ZevmFYibI8ZUOTSz-16tD1O9q9Yx_Ft1f5GJXJnpNAuilwOBRGYeG53ZYdxOy7aq9tLOPeekJxyG5b9H1J7w4AP-twbuKqBJ_Yyx7NyhakU2zg2WLeReH0relvmHZxiwpNsNiIMeRdIJ6S1scPnZbNcuCih10jMJlbcM-dKyp_ITD2ZV8H1S37Voku--V2nDdM52hRLtpIEc4ys0KUiVkEbfDrelyuRT_zkebmBm7cFggf3XY3ouKo2f6JkfpVfphU92s0VM0jV04hx0xNfi0MdrTgHOB6DDYpgkyJasQNJVr2BffmtTkPLGVIMNvtp8Xgk6E4dXhiHojZ-aW9r3T0aY1Rb2yvdvpmGxMRckWSJ26uHQBL1hD0oUajh18mblj0FA0ZwoZONPK5ut-U4jKUrgwLHAbtiETT_GmMnFdnZcUZ8RNnB8FrN_iUibCwNOn2cngdu3ukURUvP0d5qh0NKdsC9KjJpRYB-LbaPiBZ_JlZ3FVXqOw2bryIxw3FbPO_enjL731pBm9a0tNaKgF7XxQcFuNeqUw-hEhCFYV4PEBYW10g3bLj8Z_NLZoU9muMa4TAaC5kRpBhh2rvVaV0s291giaVxMkKS8D4Ui_Gy3IaUXKLt5QBedEAmD49fRu7Z8I-XDu0ZdGwll76TTcsEka1Neyo-xiikE5-NNbwJ7hqhs7Nkqgp402-jU-U8IaG8Cpr51wxj75tcij3-8y3Eog2R65oOl6ZVUw2Jf-2qzDhjhOVDHtyHWyBii_Vs-rObvYWabvS2UyMLH4YfitoqcZfBRP1nwNAgkcBJCQRDHh4BFXhws1cj8uad0UoFjXXAxpBIIvIkuoz7-U-BxShTv9HMB0drNWpsMedhjeTknwESNhLjrwUSddlNB67WscDnKVwlem69rSZMGfBTwTimGHZktqJ981OGhIGJaHJN_6Wl1z2zMasL0VEzrU6weh1ViXvPtQL7Kr4Z7GdzN2wclYRdN32-qlzMrC63KjRFszdYJlUragAV2P07cNZfy_-OwaX0WsZ0kOkxT08uN5U619Hn1mWIEplgPeQd5T3CWlcCeVch_gzvRNGOsqXjX6fZdJwXFT9tDx5QwQRbupC32_gBa6nnsAyHUSKLoQrymGxUY3HjznHF2c5N-vLKZpg5p7neXt7qTvCXijiTOYLZ6YnlQdwVzZA2zGSd0jCg4rJjYMQ09TYRny5seFflrOUKONjb9GfTRi4TjuiIuwLD-404wQDlmrIRRxumDU7DLgKoTMKz6Wghl2C5ag2zjeSMB-Pg6lMjcvaxAJt5zDUHKqzS-EWZJLVb4MhDC1DJCAWDOnYKm6aNm7JZdMQtRUaI2QZUZxz3uAQ47l39a5pxrQP3fPcTIXMxycjYTJD2k-FJRIRrqD8RE7ya3-Ap9ZI0Qxq5aAvafarXlbbEFHvZLD5uN9LHfbGNXVlog1sZPcRoNe_hOflO_zx4F4DHvXfQkCQFKTiC6oIFBljec3J0C-W9Wzx0ml0I9o3bkZzhnITH-xbhevBmr2lwY0H7Kb5FYS1BhpGVt6tDU4CsjPu-0S2BcQ7wCBZ-lsnFuUdL02KbLFgF8UTydjIhG6Xk_AaRy_yuFxKDFPrRzkkajdznfjgMyezayHAnI9Vg8kOz2FRT8X8B5K04jp2eTerQjD7nPjQH-0QC9L5E5RxgPeuabrVH3n_lXvjEaPsMquBY_sGYFsgxQYN4j3K7MvO2jB-uLwTt5D7jpuG_evcwRCXmvESKmsPHVSnQlPRpr8LS3hBS1EnNSrwy03OEzRCFgiHyNpfLpk1KxdQg9H9Kbil1UlsfTCb64vMZIaA1AWWC3W_W-JKBTuQ6ktNsj_FmOUu8QiMNoxRmSYB1SPfJ0XchQ16Q01shGsWNP22kKw-_w2oETRr-xaG2b-bOLbk_r8TJMd0Sf69Ut8vrFKLVjnzPQiiS-aaqc-iOaLS_qfLuEiH-ffjSwD2S_hQaMupND0Tbe3T2IZUhAnUzb-W7wmlZxsdnyk3kJ7si-78tWnc2DtN1xlC8HXuj2tj3LkkXe3gwdHM5Bgh27nfl4A_VgQsx34KhUqBfD8jUkMJC36D-Cbofz0J91LLWXtC6RUwhRoIF-Ad80pH8LkoHKlKRry0SKrenNrqPgYMl_mWLAELATcrk32DJ25xfXwKvQj_qScY6CpYO4oz5z-fJ8g41A2Piav3pLmt3uMFONwqXVcV7pHey6KaJrntUs3lrJKjl-XbRQhZ6oQq7qnixqF6wpej4cqgEAFBtMlvInyfwlvQlQ57q8Hv7It2x9b0NdTZz3PT_2UKbti3BjFRC9F1d40xNG2W6zZcjzvkZXO15zGb6mNXrScWWkYbL-qTgDEz7c-fPOae8rL6UPt99prtB8coiVBiq6EzUQPThrFykwMpg-BvXWX5iB1cHh7e8zYn8kpFipBznvGxPZBp3DvhzedGXe3Cdbmtp3cabNlVL9SjfMs7KggRe23mcOL-zZntGj9mNi2VZIlsWTv8G38Q4vPyPopZVFpX7QpBGgjFy-YZDAboQMozXlyeiogSyVOF1hG5YonDlPzNNLgdf4KltaRB752miHcMWqKWFfWWu7_HDau6heoF8Os2I5kgA4NBxcZ9mMDsO-L1vCWe_Z7EXlkkdApz6z4zC-mKY-CN92mSRY_XcLgISoGBqey9pwS7vk4SVt9NqfpztulvElqbkK4ynNiu5r31G5CXhfNkXz6wCtdEqowW7NmOi8Sc-UQ9jspJX4Y20zujGfWk7DESOk-ML3FYVWQwOOp5IwMiVUMCd_QUe60KuE3i_GkG9n2XbZuPkYr6n-Bga1_m4XCZs9mwJCTFKtRuzjNr4r1rjfNU7UYM9g90m8kyMtVsIqEZ1iLNPsUyZ_oWZbizHQCkaAUXRYNEVjM5TUWeIQnjzSk07qlrTueLpw1_23uSEubViy7QCHJjNtK-TpfqhcyLFcN_ceAmJUInttj4pVD2uC8qs0Ehta83w3xkipLX4EbgixLZQ_Ec-4VZB-kepjnGSMZzRTfBuuWFSdLr0cIGoYkL5p4cQG9pgK5nEzo4JLW_dIhI3B_RPi9i2Ve3MAYux3M0BjXGb22h_iVAaROOa8nPIrIlUIcMjEGGV5BbEgwYotHIhRaX9h8-pxKMjvVpw92TlELuAncOHTrl-C20yhakmUTg8G8RZ62idauxDab6aN6Ekodnh9bkyyCjoEcs0P_pz41UuBKpNkPiYdRGkD39W3IMzEZa2LD2j1Goy1Z7LFRBhrBww2mYxNJib9ZuvBxzgtzPVIMwsHM4eHYHxSoUZ-en1aO7ANrfYQcZob0wqXlHHv2hq9E75A2IeIXOUyC_DhvgXviieLg8mqx_mvlvHq8uLh_Sn3mhLobilIfRcO7NINmT44EkefTXmRJyAIX2Vq9Gjjvw== diff --git a/lume_model/base.py b/lume_model/base.py index 3f3338a..513bebc 100644 --- a/lume_model/base.py +++ b/lume_model/base.py @@ -67,36 +67,6 @@ def process_torch_module( torch.save(module, filepath) return filename - -def process_keras_model( - model, - base_key: str = "", - key: str = "", - file_prefix: Union[str, os.PathLike] = "", - save_models: bool = True, -): - """Optionally saves the given keras model to file and returns the filename. - - Args: - base_key: Base key at this stage of serialization. - key: Key corresponding to the torch module. - model: The keras model to process. - file_prefix: Prefix for generated filenames. - save_models: Determines whether keras models are saved to file. - - Returns: - Filename under which the keras model is (or would be) saved. - """ - prefixes = [ele for ele in [file_prefix, base_key] if not ele == ""] - if not prefixes: - model_name = "{}.keras".format(key) - else: - model_name = "{}.keras".format("_".join((*prefixes, key))) - if save_models: - model.save(model_name) - return model_name - - def recursive_serialize( v: dict[str, Any], base_key: str = "", @@ -116,7 +86,6 @@ def recursive_serialize( """ # try to import modules for LUMEBaseModel child classes torch = try_import_module("torch") - keras = try_import_module("keras") # serialize v = serialize_variables(v) for key, value in v.items(): @@ -132,8 +101,6 @@ def recursive_serialize( save_models) for i in range(len(value)) ] - elif keras is not None and isinstance(value, keras.Model): - v[key] = process_keras_model(value, base_key, key, file_prefix, save_models) else: for _type, func in JSON_ENCODERS.items(): if isinstance(value, _type): diff --git a/lume_model/models/__init__.py b/lume_model/models/__init__.py index 4e2664e..f73950e 100644 --- a/lume_model/models/__init__.py +++ b/lume_model/models/__init__.py @@ -12,13 +12,6 @@ except ModuleNotFoundError: pass -# models requiring keras -try: - from lume_model.models.keras_model import KerasModel - registered_models += [KerasModel] -except ModuleNotFoundError: - pass - def get_model(name: str): """Returns the LUME model class for the given name. diff --git a/lume_model/models/keras_layers.py b/lume_model/models/keras_layers.py deleted file mode 100644 index d92dd93..0000000 --- a/lume_model/models/keras_layers.py +++ /dev/null @@ -1,178 +0,0 @@ -""" -This module contains layers for use in building toolkit compatible models. -""" - -import numpy as np -from tensorflow import keras -import logging - -logger = logging.getLogger(__name__) - - -class LumeModelLayer(keras.layers.Layer): - def _validate_args(self): - - for attr in ("_offset", "_scale", "_lower", "_upper"): - try: - val = getattr(self, attr) - except: - val = None - - if val: - if not isinstance(val, (float, int,)): - raise ValueError(f"{attr} must be a float or int.") - - -class ScaleLayer(LumeModelLayer): - """Layer for scaling float values. - - Attributes: - _offset (float): Data offset - _scale (float): Scale multiplier - _lower (float): Lower range - _upper (float): Upper range - - """ - - trainable = False - - def __init__( - self, offset: float, scale: float, lower: float, upper: float, **kwargs - ) -> None: - """Sets up scaling. - - Args: - offset (float): Data offset - scale (float): Scale multiplier - lower (float): Lower range - upper (float): Upper range - - """ - super(ScaleLayer, self).__init__(**kwargs) - self._scale = scale - self._offset = offset - self._lower = lower - self._upper = upper - self._validate_args() - - def call(self, inputs: np.ndarray) -> np.ndarray: - """Execute scaling on an array. - - Args: - inputs (np.ndarray) - - Returns: - np.ndarray - """ - return self._lower + ( - (inputs - self._offset) * (self._upper - self._lower) / self._scale - ) - - def get_config(self) -> dict: - """Get layer config. - - Returns: - dict - - """ - return { - "scale": self._scale, - "offset": self._offset, - "lower": self._lower, - "upper": self._upper, - } - - -class UnscaleLayer(LumeModelLayer): - """Layer used for unscaling float values. - - Attributes: - _offset (float): Data offset - _scale (float): Scale multiplier - _lower (float): Lower range - _upper (float): Upper range - - """ - - trainable = False - - def __init__( - self, offset: float, scale: float, lower: float, upper: float, **kwargs - ): - """Sets up scaling. - - Args: - offset (float): Data offset - scale (float): Scale multiplier - lower (float): Lower range - upper (float): Upper range - """ - super(UnscaleLayer, self).__init__(**kwargs) - self._scale = scale - self._offset = offset - self._lower = lower - self._upper = upper - self._validate_args() - - def call(self, inputs: np.ndarray) -> np.ndarray: - """Unscale an array - - Args: - inputs (np.ndarray) - - """ - return ( - ((inputs - self._lower) * self._scale) / (self._upper - self._lower) - ) + self._offset - - def get_config(self) -> dict: - """Get layer config. - - Returns: - dict - - """ - return { - "scale": self.scale, - "offset": self.offset, - "lower": self.lower, - "upper": self.upper, - } - - -class UnscaleImgLayer(LumeModelLayer): - """Layer used to unscale images. - - - """ - - trainable = False - - def __init__(self, offset: float, scale: float, **kwargs): - """ - Args: - offset (float): Data offset - scale (float): Scale multiplier - """ - super(UnscaleImgLayer, self).__init__(**kwargs) - self._scale = scale - self._offset = offset - self._validate_args() - - def call(self, inputs: np.ndarray) -> np.ndarray: - """Unscale an image. - - Returns: - np.ndarray - - """ - return (inputs + self._offset) * self._scale - - def get_config(self) -> dict: - """Get layer config. - - Returns: - dict - - """ - return {"img_scale": self.img_scale, "img_offset": self.img_offset} diff --git a/lume_model/models/keras_model.py b/lume_model/models/keras_model.py deleted file mode 100644 index f46f375..0000000 --- a/lume_model/models/keras_model.py +++ /dev/null @@ -1,268 +0,0 @@ -import os -import logging -from typing import Union - -import keras -import numpy as np -from pydantic import field_validator - -from lume_model.base import LUMEBaseModel -from lume_model.variables import ( - InputVariable, - OutputVariable, - ScalarInputVariable, - # ScalarOutputVariable, - # ImageOutputVariable, -) - -logger = logging.getLogger(__name__) - - -class KerasModel(LUMEBaseModel): - """LUME-model class for keras models. - - Attributes: - model: The keras base model. - output_format: Determines format of outputs: "array", "variable" or "raw". - output_transforms: List of strings defining additional transformations applied to the outputs. For now, - only "softmax" is supported. - """ - model: keras.Model - output_format: str = "array" - output_transforms: list[str] = [] - - def __init__( - self, - *args, - **kwargs, - ): - """Initializes KerasModel. - - Args: - *args: Accepts a single argument which is the model configuration as dictionary, YAML or JSON - formatted string or file path. - **kwargs: See class attributes. - """ - super().__init__(*args, **kwargs) - - @field_validator("model", mode="before") - def validate_keras_model(cls, v): - if isinstance(v, (str, os.PathLike)): - if os.path.exists(v): - v = keras.models.load_model(v) - else: - raise OSError(f"File {v} is not found.") - return v - - @field_validator("output_format") - def validate_output_format(cls, v): - supported_formats = ["array", "variable", "raw"] - if v not in supported_formats: - raise ValueError(f"Unknown output format {v}, expected one of {supported_formats}.") - return v - - @property - def dtype(self): - return np.double - - def evaluate( - self, - input_dict: dict[str, Union[InputVariable, float, np.ndarray]], - ) -> dict[str, Union[OutputVariable, float, np.ndarray]]: - """Evaluates model on the given input dictionary. - - Args: - input_dict: Input dictionary on which to evaluate the model. - - Returns: - Dictionary of output variable names to values. - """ - formatted_inputs = self._format_inputs(input_dict) - complete_input_dict = self._complete_inputs(formatted_inputs) - output_array = self.model.predict(complete_input_dict).astype(self.dtype) - output_array = self._output_transform(output_array) - parsed_outputs = self._parse_outputs(output_array) - output_dict = self._prepare_outputs(parsed_outputs) - return output_dict - - def random_input(self, n_samples: int = 1) -> dict[str, np.ndarray]: - """Generates random input(s) for the model. - - Args: - n_samples: Number of random samples to generate. - - Returns: - Dictionary of input variable names to arrays. - """ - input_dict = {} - for var in self.input_variables: - if isinstance(var, ScalarInputVariable): - input_dict[var.name] = np.random.uniform(*var.value_range, size=n_samples) - else: - default_array = np.array(var.default, dtype=self.dtype) - input_dict[var.name] = np.repeat(default_array.reshape((1, *default_array.shape)), - n_samples, axis=0) - return input_dict - - def random_evaluate(self, n_samples: int = 1) -> dict[str, Union[OutputVariable, float, np.ndarray]]: - """Returns random evaluation(s) of the model. - - Args: - n_samples: Number of random samples to evaluate. - - Returns: - Dictionary of variable names to outputs. - """ - random_input = self.random_input(n_samples) - return self.evaluate(random_input) - - def _format_inputs( - self, - input_dict: dict[str, Union[InputVariable, float, np.ndarray]], - ) -> dict[str, np.ndarray]: - """Formats values of the input dictionary as arrays. - - Args: - input_dict: Dictionary of input variable names to values. - - Returns: - Dictionary of input variable names to arrays. - """ - # NOTE: The input variable is only updated if a singular value is given (ambiguous otherwise) - formatted_inputs = {} - for var_name, var in input_dict.items(): - if isinstance(var, InputVariable): - formatted_inputs[var_name] = np.array(var.value, dtype=self.dtype) - # self.input_variables[self.input_names.index(var_name)].value = var.value - elif isinstance(var, float): - formatted_inputs[var_name] = np.array(var, dtype=self.dtype) - # self.input_variables[self.input_names.index(var_name)].value = var - elif isinstance(var, np.ndarray): - var = var.astype(self.dtype).squeeze() - formatted_inputs[var_name] = var - # if var.ndim == 0: - # self.input_variables[self.input_names.index(var_name)].value = var.item() - else: - TypeError( - f"Unknown type {type(var)} passed to evaluate." - f"Should be one of InputVariable, float or np.ndarray." - ) - return formatted_inputs - - def _complete_inputs(self, formatted_inputs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: - """Completes input dictionary by filling in default values. - - Args: - formatted_inputs: Dictionary of input variable names to arrays. - - Returns: - Completed input dictionary to be passed to the model. - """ - # determine input shape - input_shapes = [formatted_inputs[k].shape for k in formatted_inputs.keys()] - if not all(ele == input_shapes[0] for ele in input_shapes): - raise ValueError("Inputs have inconsistent shapes.") - - for i, key in enumerate(self.input_names): - if key not in formatted_inputs.keys(): - default_array = np.array(self.input_variables[i].default, dtype=self.dtype) - formatted_inputs[key] = np.tile(default_array, reps=input_shapes[0]) - - if not input_shapes[0]: - for key in self.input_names: - formatted_inputs[key] = formatted_inputs[key].reshape((1, *formatted_inputs[key].shape)) - return formatted_inputs - - def _output_transform(self, output_array: np.ndarray) -> np.ndarray: - """Applies additional transformations to the model output array. - - Args: - output_array: Output array from the model. - - Returns: - Transformed output array. - """ - if "softmax" in self.output_transforms: - output_array = np.argmax(output_array, axis=-1) - return output_array - - def _parse_outputs(self, output_array: np.ndarray) -> dict[str, np.ndarray]: - """Constructs dictionary from model output array. - - Args: - output_array: Transformed output array from the model. - - Returns: - Dictionary of output variable names to transformed arrays. - """ - parsed_outputs = {} - if output_array.ndim in [0, 1]: - output_array = output_array.reshape((1, *output_array.shape)) - if len(self.output_names) == 1: - parsed_outputs[self.output_names[0]] = output_array.squeeze() - else: - for idx, output_name in enumerate(self.output_names): - parsed_outputs[output_name] = output_array[..., idx].squeeze() - return parsed_outputs - - def _prepare_outputs( - self, - parsed_outputs: dict[str, np.ndarray], - ) -> dict[str, Union[OutputVariable, np.ndarray]]: - """Updates and returns outputs according to output_format. - - Updates the output variables within the model to reflect the new values. - - Args: - parsed_outputs: Dictionary of output variable names to transformed arrays. - - Returns: - Dictionary of output variable names to values depending on output_format. - """ - # for var in self.output_variables: - # if parsed_outputs[var.name].ndim == 0: - # idx = self.output_names.index(var.name) - # if isinstance(var, ScalarOutputVariable): - # self.output_variables[idx].value = parsed_outputs[var.name].item() - # elif isinstance(var, ImageOutputVariable): - # # OutputVariables should be arrays - # self.output_variables[idx].value = (parsed_outputs[var.name].reshape(var.shape).numpy()) - # self._update_image_limits(var, parsed_outputs) - - if self.output_format == "array": - return parsed_outputs - elif self.output_format == "variable": - output_dict = {var.name: var for var in self.output_variables} - for var in output_dict.values(): - var.value = parsed_outputs[var.name].item() - return output_dict - # return {var.name: var for var in self.output_variables} - else: - return {key: value.item() if value.squeeze().ndim == 0 else value - for key, value in parsed_outputs.items()} - # return {var.name: var.value for var in self.output_variables} - - def _update_image_limits( - self, - variable: OutputVariable, predicted_output: dict[str, np.ndarray], - ): - output_idx = self.output_names.index(variable.name) - if self.output_variables[output_idx].x_min_variable: - self.output_variables[output_idx].x_min = predicted_output[ - self.output_variables[output_idx].x_min_variable - ].item() - - if self.output_variables[output_idx].x_max_variable: - self.output_variables[output_idx].x_max = predicted_output[ - self.output_variables[output_idx].x_max_variable - ].item() - - if self.output_variables[output_idx].y_min_variable: - self.output_variables[output_idx].y_min = predicted_output[ - self.output_variables[output_idx].y_min_variable - ].item() - - if self.output_variables[output_idx].y_max_variable: - self.output_variables[output_idx].y_max = predicted_output[ - self.output_variables[output_idx].y_max_variable - ].item() diff --git a/pyproject.toml b/pyproject.toml index 3d334a4..aeee8c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ version_file = "lume_model/_version.py" [project.optional-dependencies] dev = [ - "tensorflow", # has to be >=2.16 to be compatible with python 3.12 "botorch<0.11", "pre-commit", "ruff", diff --git a/requirements.txt b/requirements.txt index 58df03c..d7fdd15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -pydantic>2.3 +pydantic numpy pyyaml diff --git a/tests/conftest.py b/tests/conftest.py index 399609d..a0ff552 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,12 +15,6 @@ except ModuleNotFoundError: pass -try: - import keras - from lume_model.models import KerasModel -except ModuleNotFoundError: - pass - @pytest.fixture(scope="session") def rootdir() -> str: @@ -147,21 +141,6 @@ def iris_variables(rootdir) -> tuple[list[InputVariable], list[OutputVariable]]: pytest.skip(str(e)) -@pytest.fixture(scope="module") -def iris_model_kwargs(rootdir, iris_variables) -> dict[str, Any]: - keras = pytest.importorskip("keras") - - input_variables, output_variables = iris_variables - model_kwargs = { - "model": keras.models.load_model(f"{rootdir}/test_files/iris_classification/model.keras"), - "input_variables": input_variables, - "output_variables": output_variables, - "output_format": "array", - "output_transforms": ["softmax"], - } - return model_kwargs - - @pytest.fixture(scope="module") def iris_test_input_array(rootdir: str): try: @@ -178,10 +157,3 @@ def iris_test_input_dict(iris_test_input_array, iris_variables) -> dict: var.name: iris_test_input_array[0, idx] for idx, var in enumerate(input_variables) } return test_input_dict - - -@pytest.fixture(scope="module") -def iris_model(iris_model_kwargs): - keras = pytest.importorskip("keras") - - return KerasModel(**iris_model_kwargs) diff --git a/tests/models/test_keras_layers.py b/tests/models/test_keras_layers.py deleted file mode 100644 index 6e77f38..0000000 --- a/tests/models/test_keras_layers.py +++ /dev/null @@ -1,38 +0,0 @@ -import pytest - - -# test value and failed initialization with characters -@pytest.mark.parametrize( - "offset,scale,lower,upper", - [ - (1, 2, 0, 1), - (5, 4, -1, 1), - pytest.param("t", "e", "s", "t", marks=pytest.mark.xfail), - ], -) -def test_scale_layer(offset, scale, lower, upper): - layers = pytest.importorskip("lume_model.models.keras_layers") - scale_layer = layers.ScaleLayer(offset, scale, lower, upper) - - -# test value and failed initialization with characters -@pytest.mark.parametrize( - "offset,scale,lower,upper", - [ - (1, 2, 0, 1), - (5, 4, -1, 1), - pytest.param("t", "e", "s", "t", marks=pytest.mark.xfail), - ], -) -def test_unscale_layer(offset, scale, lower, upper): - layers = pytest.importorskip("lume_model.models.keras_layers") - unscale_layer = layers.UnscaleLayer(offset, scale, lower, upper) - - -# test value and failed initialization with characters -@pytest.mark.parametrize( - "offset,scale", [(1, 2), (5, 4), pytest.param("t", "e", marks=pytest.mark.xfail),], -) -def test_unscale_image_layer(offset, scale): - layers = pytest.importorskip("lume_model.models.keras_layers") - unscale_layer = layers.UnscaleImgLayer(offset, scale) diff --git a/tests/models/test_keras_model.py b/tests/models/test_keras_model.py deleted file mode 100644 index 9e0c1e3..0000000 --- a/tests/models/test_keras_model.py +++ /dev/null @@ -1,176 +0,0 @@ -import os -import random -from typing import Union -from copy import deepcopy - -import pytest -import numpy as np - -try: - from lume_model.models import KerasModel - from lume_model.variables import InputVariable, OutputVariable, ScalarOutputVariable -except ImportError: # TODO: add info when this happens - pass - - -# def assert_variables_updated( -# input_value: float, -# output_value: float, -# model, -# input_name: str, -# output_name: str, -# ): -# """helper function to verify that model input_variables and output_variables -# have been updated correctly with float values (NOT arrays)""" -# assert isinstance(model.input_variables[model.input_names.index(input_name)].value, float) -# assert model.input_variables[model.input_names.index(input_name)].value == pytest.approx(input_value) -# assert isinstance(model.output_variables[model.output_names.index(output_name)].value, int) -# assert model.output_variables[model.output_names.index(output_name)].value == pytest.approx(output_value) - - -# def assert_iris_model_result(iris_test_input_dict: dict, model): -# assert_variables_updated( -# input_value=iris_test_input_dict["SepalLength"].item(), -# output_value=2, -# model=model, -# input_name="SepalLength", -# output_name="Species", -# ) - - -def assert_model_equality(m1: KerasModel, m2: KerasModel): - assert m1.input_variables == m2.input_variables - assert m1.output_variables == m2.output_variables - for l1, l2 in zip(m1.model.layers, m2.model.layers): - assert l1.get_config() == l2.get_config() - for l1, l2 in zip(m1.model.layers, m2.model.layers): - w1, w2 = l1.get_weights(), l2.get_weights() - assert len(w1) == len(w2) - for i, w in enumerate(w1): - assert np.array_equal(w, w2[i]) - assert m1.output_format == m2.output_format - assert m1.output_transforms == m2.output_transforms - - -class TestKerasModel: - def test_model_from_objects( - self, - iris_model_kwargs: dict[str, Union[list, dict, str]], - iris_variables: tuple[list[InputVariable], list[OutputVariable]], - iris_model, - ): - input_variables, output_variables = iris_variables - - assert isinstance(iris_model, KerasModel) - assert iris_model.input_variables == input_variables - assert iris_model.output_variables == output_variables - - def test_model_from_yaml(self, rootdir: str, iris_model): - file = f"{rootdir}/test_files/iris_classification/keras_model.yml" - yaml_model = KerasModel(file) - assert_model_equality(yaml_model, iris_model) - - def test_model_as_yaml(self, rootdir: str, iris_model): - filename = "test_keras_model" - file = f"{filename}.yml" - iris_model.dump(file) - yaml_model = KerasModel(file) - assert_model_equality(yaml_model, iris_model) - os.remove(file) - os.remove(f"{filename}_model.keras") - - def test_model_evaluate_variable( - self, - iris_test_input_dict: dict, - iris_model_kwargs: dict[str, Union[list, dict, str]], - ): - kwargs = {k: v if not k == "output_format" else "variable" for k, v in iris_model_kwargs.items()} - iris_model = KerasModel(**kwargs) - input_variables = deepcopy(iris_model.input_variables) - for var in input_variables: - var.value = iris_test_input_dict[var.name].item() - results = iris_model.evaluate({var.name: var for var in input_variables}) - - assert isinstance(results["Species"], ScalarOutputVariable) - assert results["Species"].value == 2 - # assert_iris_model_result(iris_test_input_dict, iris_model) - - def test_model_evaluate_single_sample(self, iris_test_input_dict: dict, iris_model): - results = iris_model.evaluate(iris_test_input_dict) - - assert isinstance(results["Species"], np.ndarray) - assert results["Species"] == 2 - # assert_iris_model_result(iris_test_input_dict, iris_model) - - def test_model_evaluate_n_samples(self, iris_test_input_array, iris_model): - test_dict = { - key: iris_test_input_array[:, idx] for idx, key in enumerate(iris_model.input_names) - } - results = iris_model.evaluate(test_dict) - # in this case we don't expect the input/output variables to be updated, because we don't know which value - # to update them with so we only check for the resulting values - target_array = np.array([2, 0, 1], dtype=results["Species"].dtype) - - assert all(np.isclose(results["Species"], target_array)) - - # def test_model_evaluate_batch_n_samples( - # self, - # iris_test_input_array, - # iris_model, - # ): - # # model should be able to handle input of shape [batch_size, n_samples, n_dim] - # input_dict = { - # key: iris_test_input_array.reshape((1, *iris_test_input_array.shape)).repeat(2, axis=0) - # for idx, key in enumerate(iris_model.input_names) - # } - # results = iris_model.evaluate(input_dict) - # - # # output shape should be [batch_size, n_samples] - # assert tuple(results["Species"].shape) == (2, 3) - - def test_model_evaluate_raw( - self, - iris_test_input_dict: dict, - iris_model_kwargs: dict[str, Union[list, dict, str]], - ): - kwargs = {k: v if not k == "output_format" else "raw" for k, v in iris_model_kwargs.items()} - iris_model = KerasModel(**kwargs) - float_dict = {key: value.item() for key, value in iris_test_input_dict.items()} - results = iris_model.evaluate(float_dict) - - assert isinstance(results["Species"], int) - assert results["Species"] == 2 - # assert_iris_model_result(iris_test_input_dict, iris_model) - - def test_model_evaluate_shuffled_input(self, iris_test_input_dict: dict, iris_model): - shuffled_input = deepcopy(iris_test_input_dict) - item_list = list(shuffled_input.items()) - random.shuffle(item_list) - shuffled_input = dict(item_list) - results = iris_model.evaluate(shuffled_input) - - assert isinstance(results["Species"], np.ndarray) - assert results["Species"] == 2 - # assert_iris_model_result(iris_test_input_dict, iris_model) - - @pytest.mark.parametrize("test_idx,expected", [(0, 2), (1, 0), (2, 1)]) - def test_model_evaluate_different_values( - self, - test_idx: int, - expected: float, - iris_test_input_array, - iris_model, - ): - input_dict = { - key: iris_test_input_array[test_idx, idx] for idx, key in enumerate(iris_model.input_names) - } - results = iris_model.evaluate(input_dict) - - assert results["Species"].item() == expected - # assert_variables_updated( - # input_value=input_dict["SepalWidth"].item(), - # output_value=expected, - # model=iris_model, - # input_name="SepalWidth", - # output_name="Species", - # ) diff --git a/tests/models/test_models.py b/tests/models/test_models.py index e5e1fce..43a4452 100644 --- a/tests/models/test_models.py +++ b/tests/models/test_models.py @@ -1,7 +1,7 @@ import pytest try: - from lume_model.models import TorchModel, TorchModule, KerasModel + from lume_model.models import TorchModel, TorchModule from lume_model.models import model_from_yaml except ImportError: pass @@ -10,7 +10,6 @@ @pytest.mark.parametrize("filename,expected", [ ("test_files/california_regression/torch_model.yml", TorchModel), ("test_files/california_regression/torch_module.yml", TorchModule), - ("test_files/iris_classification/keras_model.yml", KerasModel), ]) def test_model_from_yaml(rootdir, filename, expected): model = model_from_yaml(f"{rootdir}/{filename}") diff --git a/tests/test_files/iris_classification/keras_model.yml b/tests/test_files/iris_classification/keras_model.yml deleted file mode 100644 index 27c556b..0000000 --- a/tests/test_files/iris_classification/keras_model.yml +++ /dev/null @@ -1,27 +0,0 @@ -model_class: KerasModel -input_variables: - SepalLength: - variable_type: scalar - value_range: [4.3, 7.9] - default: 4.3 - is_constant: false - SepalWidth: - variable_type: scalar - value_range: [2.0, 6.9] - default: 2.0 - is_constant: false - PetalLength: - variable_type: scalar - value_range: [1.0, 6.9] - default: 1.0 - is_constant: false - PetalWidth: - variable_type: scalar - value_range: [0.1, 2.5] - default: 0.1 - is_constant: false -output_variables: - Species: {variable_type: scalar} -model: model.keras -output_format: array -output_transforms: [softmax] diff --git a/tests/test_files/iris_classification/model.keras b/tests/test_files/iris_classification/model.keras deleted file mode 100644 index 2eccce5dac30ac2295c584abc1c26bcd4621c957..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29368 zcmeGkZERcB^*JfbEddf%Xj)3^W)rj8w6W8)N%LVF^(#Orp)I2$>YNwnB{p$vV?VcP zt85(_oj0KsX{f6JX+Y_S5R@WyB7?-P#(->Er}D8W8b3M>X=6x0g%Ar98oM9o*zaAx zXOobo0l%l#=iYnHJ@?*o&pG$r^X|(}+c#~wa>0@X5`FA;X`WQclKAsEJR@;lf_cPu zLBa+w1#q~GCv4K?62(!^H&mb!eM0=F5f1eWS3#tiH7>%i+fzSaZt|_CmsIN|}XacK1Uq!vHklI-d zn{<`5P_j$?N?cC(_bTy3C>kM{#WowONv(;uWe0i&h*gk>7cqQ=1L zEXHqy4j%Z;BT1Wd85=L!G5Uk{#RN&77)(qte=xBW=sB={OH2aj4^F|jQvi>x)yDBb4zj;C$(Y`;Oiz#fLw^M(APs&a%ZepH-K7qF z@w9WM{ae9VisRY3@%C-357Z%$2m8bP6Uo=Fal6;JBnjOt-zZ3uszegecyBnmk5wa5 z;Scj4BzdqV5S9}Oe?%ToYMLE2TauB0N?fBHuBmm@aOXkF57r<-ZM;8G%SYu^ z+GAu@n}|~~6Gacj`Rf~|AcSNjq*5s~5h2OMd+n8}2O|YFaV4CjjFN<9cSxp0AWX6k zjF1f|*(0eTIZQX61SeYm)^Ic)Rqu?1dZY0HVd@eJb3W8-4N>7!_C?s;t;C}VJsVaP z`Vv?Al3_Uxec|qqvo*x9k%rX$enTwqB&v{Ls-%+PX<6nWvELT9@s8tQ2KqVZ5b z4*LVdAQMVNjYnhqsoD%EDnU@P&b-~sZHgSZjp6T#%duGArUrev90(-ia)9wOa@oDi zGiiH~tuZvj#>r2`N{>I+9WY%w20S3=3_5pqCWcAQj|3xnvYUEj(Sc!q5fxVCI6E=M z={nWyaIJB<+%En!G&XE#Y*^=}HrJyBxf|-;h^=Q> zf}_0L^-YZp>s@OpCnc5$g_)yvE2~|Nq(mM_^u^`urfz6x=rT5o5;?u|NAG$hem*BC z+_$5u;`SXOXn2iwU`IUed z_w_h_gfN~uy6iNQegD-^4~_>C;`utz?80)SLpdRXDyWYzi$?~4zXsqp0Sa_=dueCL zgZMmtw8L2{iNq@SgM9V<&JkVL#T4+P)?W~A8RPfMuIR% zcAQx{F0VSITmtRke1=pdj(wph7arJv&H|PMj3lsg`|X=-n5A<{mWJ$==DQ4|Jo~F&?(uL#5+IH#Ml#3&q#mfux zW9kR}qH7tC4@>6niwKSNl&%E0U$bZZnhOjF{nZF8pu3?Q{n|N!E(LMjVF#C`6GWO? z!4t^u6y(o+i}1MQ zSW*?t?BoKhODK$sWhde~I9qlSgZ0z06P#fS7LtIOom7J!Vcm;mC*nSEw(O*R6(3m& z2FTYF^Lux}HJs3n0VP!9Qb*sNYi1zP9LKBGlMDg5KAq80tn`C@WGQB?s=Ohf{B4qLhW27;6 zZ9Ipji`nxT7asty;2G<;WF#Qkvjaw634R_0A-tZE@O=>41I9Oyz8K{37<#*vKNAp7 zMtQ!BvHP2A5YAelUNO`^lTn zWmfIF+`Dz%En~Mfypf*&%Z1**&lE_#0~K5*C*J+a!G`o%MClf(Pc$3I$@9_atM z=bay|&HV4R+RTa5wVA&?f1l@ux;wq|>fTOAD>CU5fx67Cq3gZf+mCw&YO692uQ`+c zqwiU7XVcj(Yr%vFv5fJi8xVb8kH#?D^%wD+pdUKqnU zA7L+1Fvp?2y{+Y==N}pEpIC@Xv5w&M3rAXf_rLtob0@Dc#D6%lb#&=R^{uZTy1@|d zTkaS=a`i~-r%JUU{(*T-Ee|iaadgQ&#`Sfb(PTy^&$n#3`gbj5^X?d3_F8iEz~MVa z|NQHBTGyXkTIOk7UUtvF7u}~J*CikN{e|BiUtarCbJHylW#;I&gU`0QcAagzdFQyU z94jAv;+dAV`6pZ7+wzp5HGcfpU9FAoAGiJC-UkfppYMOS`TVE$*6`!k8rFYh`Ss1G zpV`oI{{0&^V4GhArk{JL`IhR#t!G`2w5B)LxA@$hEvt^7ZTWENlZ{7S{jBL&be}1| zCx_;3xaaI|nw_7F820Ilv~ST^60js-NnkEYAb(%UcU1Vk4)>EeINg0@5cZLHV-2N| z!o-|_)_ayYF%5IJ6XH1<&lS7jJv??C&$l!EK2m(|KU?wR2_GLBT5GX*lVcO7NBN*! zELa*65bXrd>4$v!@r&_Tv>Sm;U#UP2ovU|v&+%(PZK|`L#YR$ewP2I#h zKfIB{Vs_t#EFfN@(M zi_Lv+BIc(bw^d=keMsLSX;8?|ZdQ!jp4nQsee`=l@>gQ4Cnte?z0#lCP;LmPuSXw) z9yxN2vs6mRU)rqMJ-P?sK5G!VXPxgZ?L5u(*2VP}?WYO!2>Xxg;7p(I#CzDYb-o+f z&PO&011z>LJp%(r`Jh}ZSQ-)#?ZgSg?gBkRDd6{s;yDlR=b#9?cw42XI9ji|OhrPbD_IBk9U z4Xd_3{Z=)9z2}Xg{K`oUnyw_p9OEOoLhtn#u4M$+NuD9Uk7ZMHIl=dl<&pzB>FeSN zlt&Qqscv4*y$@N!(A#zDW4PSkhoZ>hli{gIOm{o=>j zmcl;jkRa+oAwRoW)GO4NbGr+-kNP(lDNI*tShMv-w98`a%jEaD`Uiy5j}t!%e$atK0hkiT>T#l%Jr7hLc8@-ud9EP|JN@TXqjm`jXu^sfXnEqA=IX(Ib?5_n&Ljq>{ z8wCA9Da`b`j_VleT~~;=gy+E& z{r%D??3cO(QO~V&qrFe)o{&ZTI~EfVC7w0gABlEZY<+PJa6zJ+OI)laH+-}8WgPrE z3S_3&mwdmdKR2R3%)#k?7qYa0k6~_q7lP|8`hQ0lC2tJTUWe~B#+PbozFz6qTk-y* zn4kW7>jXVJB#8dfuDAK}(Ayd68^Y<2e;CH^5XPSp@|QN7FAu$b19?EzWEIG#P zz+_w?^_5Y?VV28`d3+U84%Lq9zwBN3+1_!}cZX`{H}>AiXHTt55OKKrvGE^Mgs6kV Ue5cPx-ibK4TSmJtUii}<0QT@l#sB~S diff --git a/tests/test_files/iris_classification/variables.yml b/tests/test_files/iris_classification/variables.yml deleted file mode 100644 index cfbc9d6..0000000 --- a/tests/test_files/iris_classification/variables.yml +++ /dev/null @@ -1,23 +0,0 @@ -input_variables: - SepalLength: - variable_type: scalar - value_range: [4.3, 7.9] - default: 4.3 - is_constant: false - SepalWidth: - variable_type: scalar - value_range: [2.0, 6.9] - default: 2.0 - is_constant: false - PetalLength: - variable_type: scalar - value_range: [1.0, 6.9] - default: 1.0 - is_constant: false - PetalWidth: - variable_type: scalar - value_range: [0.1, 2.5] - default: 0.1 - is_constant: false -output_variables: - Species: {variable_type: scalar} From 34d6408e0d0110bc6aa8347ebc09ec116beac461 Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Tue, 5 Nov 2024 12:03:39 -0800 Subject: [PATCH 14/17] remove keras examples and adjust README --- README.md | 155 +++++++++++++++++++++--------------------------------- 1 file changed, 59 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index 1e81cf5..dfacc68 100644 --- a/README.md +++ b/README.md @@ -24,87 +24,64 @@ conda env create -f dev-environment.yml ## Variables -The LUME-model variables are intended to enforce requirements for input and output variables by variable type. Current variable implementations are scalar (float) or image (numpy array) type. +The lume-model variables are intended to enforce requirements for input and output variables by variable type. For now, only scalar variables (floats) are supported. -Example of minimal implementation of scalar input and output variables: +Minimal example of scalar input and output variables: ```python from lume_model.variables import ScalarInputVariable, ScalarOutputVariable -input_variable = ScalarInputVariable(name="test_input", default=0.1, value_range=[1, 2]) -output_variable = ScalarOutputVariable(name="test_output") -``` - -Example of minimal implementation of image input and output variables: - -```python -from lume_model.variables import ImageInputVariable, ImageOutputVariable -import numpy as np - -input_variable = ImageInputVariable( - name="test_input", - default= np.array([[1, 2,], [3, 4]]), - value_range=[1, 10], - axis_labels=["count_1", "count_2"], - x_min=0, - y_min=0, - x_max=5, - y_max=5, -) - -output_variable = ImageOutputVariable( - name="test_output", - axis_labels=["count_1", "count_2"], +input_variable = ScalarInputVariable( + name="example_input", + default=0.1, + value_range=[0.0, 1.0], ) +output_variable = ScalarOutputVariable(name="example_output") ``` All input variables may be made into constants by passing the `is_constant=True` keyword argument. Value assingments on these constant variables will raise an error message. ## Models -LUME-model model classes are intended to guide user development while allowing for flexibility and customizability. The base class `lume_model.models.BaseModel` is used to enforce LUME tool compatable classes for the execution of trained models. For this case, model loading and execution should be organized into class methods. +The lume-model base class `lume_model.base.LUMEBaseModel` is intended to guide user development while allowing for flexibility and customizability. It is used to enforce LUME tool compatible classes for the execution of trained models. -Model Requirements: +Requirements for model classes: -* input_variables, output_variables: LUME-model input and output variables are required for use with lume-epics tools. The user can optionally define these as class attributes or design the subclass so that these are passed during initialization . Names of all variables must be unique in order to be served using the EPICS tools. A utility function for saving these variables, which also enforces the uniqueness constraint, is provided (lume_model.utils.save_variables). -* evaluate: The evaluate method is called by the serving model. Subclasses must implement the method, accepting a list of input variables and returning a list of the model's output variables with value attributes updated based on model execution. +* input_variables: A list defining the input variables for the model. Variable names must be unique. Required for use with lume-epics tools. +* output_variables: A list defining the output variables for the model. Variable names must be unique. Required for use with lume-epics tools. +* evaluate: The evaluate method is called by the serving model. Subclasses must implement this method, accepting and returning a dictionary. -Example model implementation: +Example model implementation and instantiation: ```python -from lume_model.models import BaseModel - -class ExampleModel(BaseModel): - input_variables = { - "input1": ScalarInputVariable(name="input1", default=1, range=[0.0, 5.0]), - "input2": ScalarInputVariable(name="input2", default=2, range=[0.0, 5.0]), - } - - output_variables = { - "output1": ScalarOutputVariable(name="output1"), - "output2": ScalarOutputVariable(name="output2"), - } +from lume_model.base import LUMEBaseModel +from lume_model.variables import ScalarInputVariable, ScalarOutputVariable - def evaluate(self, input_variables): - self.input_variables = { - variable.name: variable for variable in input_variables +class ExampleModel(LUMEBaseModel): + def evaluate(self, input_dict): + output_dict = { + "output1": input_dict[self.input_variables[0].name] ** 2, + "output2": input_dict[self.input_variables[1].name] ** 2, } + return output_dict + - self.output_variables["output1"].value = ( - self.input_variables["input1"].value * 2 - ) - self.output_variables["output2"].value = ( - self.input_variables["input2"].value * 2 - ) +input_variables = [ + ScalarInputVariable(name="input1", default=0.1, value_range=[0.0, 1.0]), + ScalarInputVariable(name="input2", default=0.2, value_range=[0.0, 1.0]), +] +output_variables = [ + ScalarOutputVariable(name="output1"), + ScalarOutputVariable(name="output2"), +] - # return inputs * 2 - return list(self.output_variables.values()) +m = ExampleModel(input_variables=input_variables, output_variables=output_variables) ``` ## Configuration files -Models and variables may be constructed using a yaml configuration file. The configuration file consists of three sections: +Models and variables may be constructed using a YAML configuration file. The configuration file consists of three sections: * model (optional, can alternatively pass a custom model class into the `model_from_yaml` method) * input_variables @@ -112,56 +89,42 @@ Models and variables may be constructed using a yaml configuration file. The con The model section is used for the initialization of model classes. The `model_class` entry is used to specify the model class to initialize. The `model_from_yaml` method will attempt to import the specified class. Additional model-specific requirements may be provided. These requirements will be checked before model construction. Model keyword arguments may be passed via the config file or with the function kwarg `model_kwargs`. All models are assumed to accept `input_variables` and `output_variables` as keyword arguments. -The below example outlines the specification for a model compatible with the `lume-model` keras/tensorflow toolkit. +For example, `m.dump("example_model.yml")` writes the following to file ```yaml -model: - model_class: lume_model.keras.KerasModel - requirements: - tensorflow: 2.3.1 - args: - model_file: examples/files/iris_model.h5 - output_format: - type: softmax - +model_class: ExampleModel +input_variables: + input1: + variable_type: scalar + default: 0.1 + is_constant: false + value_range: [0.0, 1.0] + input2: + variable_type: scalar + default: 0.2 + is_constant: false + value_range: [0.0, 1.0] +output_variables: + output1: {variable_type: scalar} + output2: {variable_type: scalar} ``` -Variables are constructed the minimal data requirements for inputs/outputs. - -An example ScalarInputVariable: - -```yaml -input_variables: - SepalLength: - name: SepalLength - type: scalar - default: 4.3 - lower: 4.3 - upper: 7.9 +and can be loaded by simply passing the file to the model constructor: -``` +```python +from lume_model.base import LUMEBaseModel -For image variables, default values must point to files associated with a default numpy array representation. The file import will be relative to PYTHONPATH. -An example ImageInputVariable: +class ExampleModel(LUMEBaseModel): + def evaluate(self, input_dict): + output_dict = { + "output1": input_dict[self.input_variables[0].name] ** 2, + "output2": input_dict[self.input_variables[1].name] ** 2, + } + return output_dict -```yaml -input_variables: - InputImage: - name: test - type: image - default: examples/files/example_input_image.npy - range: [0, 100] - x_min: 0 - x_max: 10 - y_min: 0 - y_max: 10 - axis_labels: ["x", "y"] - x_min_variable: xmin_pv - y_min_variable: ymin_pv - x_max_variable: xmax_pv - y_max_variable: ymax_pv +m = ExampleModel("example_model.yml") ``` ## PyTorch Toolkit From fa538ccdb5ab49d4012bdde78ea8585a2fb8548a Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Tue, 5 Nov 2024 16:03:02 -0800 Subject: [PATCH 15/17] add pypi release workflow --- .github/workflows/publish.yml | 31 +++++++++++++++++++ .../workflows/{build.yml => test_build.yml} | 10 +++++- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/publish.yml rename .github/workflows/{build.yml => test_build.yml} (80%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..567a8f7 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,31 @@ +name: Publish Python distributions to PyPI + +on: + release: + types: [published] + +jobs: + build-n-publish: + if: ${{ github.repository == 'slaclab/lume-model' }} + name: Build and publish Python distributions to PyPI + runs-on: ubuntu-latest + environment: deployment + permissions: + id-token: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install build + run: | + python -m pip install build + - name: Build a source tarball + run: | + sudo python3 -m build + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/build.yml b/.github/workflows/test_build.yml similarity index 80% rename from .github/workflows/build.yml rename to .github/workflows/test_build.yml index 775cd0f..39b4ee2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/test_build.yml @@ -1,4 +1,4 @@ -name: Build lume-model +name: Test build lume-model on: push: @@ -33,3 +33,11 @@ jobs: shell: bash -l {0} run: | pytest -ra --pyargs tests + + - name: Install build + run: | + python -m pip install build + + - name: Build a source tarball + run: | + sudo python -m build From 4efe8c93496a436c9e3621f0806668eed072c9de Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Tue, 5 Nov 2024 16:11:34 -0800 Subject: [PATCH 16/17] fix build install --- .github/workflows/test_build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 39b4ee2..cd7f166 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -35,9 +35,11 @@ jobs: pytest -ra --pyargs tests - name: Install build + shell: bash -l {0} run: | python -m pip install build - name: Build a source tarball + shell: bash -l {0} run: | sudo python -m build From 921d35c3762f78dc225dbe4412551db0d8dd3f5c Mon Sep 17 00:00:00 2001 From: "Sara A. Miskovich" Date: Tue, 5 Nov 2024 16:16:30 -0800 Subject: [PATCH 17/17] fix build install --- .github/workflows/test_build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index cd7f166..dff9787 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -37,9 +37,9 @@ jobs: - name: Install build shell: bash -l {0} run: | - python -m pip install build + python -m pip install build --user - name: Build a source tarball shell: bash -l {0} run: | - sudo python -m build + python -m build