diff --git a/doc/sphinx/source/changelog.rst b/doc/sphinx/source/changelog.rst index c2051e7e7b..2d67cc43fe 100644 --- a/doc/sphinx/source/changelog.rst +++ b/doc/sphinx/source/changelog.rst @@ -1,6 +1,128 @@ Changelog ========= +.. _changelog-v2-2-0: + +v2.2.0 +------ + +Highlights +~~~~~~~~~~ + +ESMValTool is now using the recently released `Iris 3 `__. +We acknowledge that this change may impact your work, as Iris 3 introduces +several changes that are not backward-compatible, but we think that moving forward is the best +decision for the tool in the long term. + + +This release includes + +Bug fixes +~~~~~~~~~ + +- Bugfix: time weights in time_operations (`#1956 `__) `Axel Lauer `__ +- Fix issues with bibtex references (`#1955 `__) `Stef Smeets `__ +- Fix ImportError for `configure_logging` (`#1976 `__) `Stef Smeets `__ +- Add required functional parameters for extract time in recipe_er5.yml (`#1978 `__) `Valeriu Predoi `__ +- Revert "Fix ImportError for `configure_logging`" (`#1992 `__) `Bouwe Andela `__ +- Fix import of esmvalcore _logging module in cmorize_obs.py (`#2020 `__) `Valeriu Predoi `__ +- Fix logging import in cmorize_obs again since last merge was nulled by pre-commit hooks (`#2022 `__) `Valeriu Predoi `__ +- Refactor the functions in derive_evspsblpot due to new iris (`#2023 `__) `SarahAlidoost `__ +- Avoid importing private ESMValCore functions in CMORizer (`#2027 `__) `Bouwe Andela `__ +- Fix extract_seasons in validation recipe (`#2054 `__) `Javier Vegas-Regidor `__ + +Deprecations +~~~~~~~~~~~~ + +- Deprecate classes Variable, Variables and Datasets (`#1944 `__) `Manuel Schlund `__ +- Python 3.9: remove pynio as dependency and replace with rasterio and pin Matplotlib>3.3.1 and pin cartopy>=0.18 (`#1997 `__) `Valeriu Predoi `__ +- Removed write_plots and write_netcdf in some python diagnostics (`#2036 `__) `Manuel Schlund `__ + +Documentation +~~~~~~~~~~~~~ + +- Update instructions on making a release (`#1867 `__) `Bouwe Andela `__ +- Update review.rst (`#1917 `__) `Axel Lauer `__ +- Add guidance on how to review a pull request (`#1872 `__) `Bouwe Andela `__ +- Adding tutorial links to documentation (`#1927 `__) `Birgit Hassler `__ +- Added bibtex file for schlund20jgr (`#1928 `__) `Manuel Schlund `__ +- Documentation contact added the actual email for the mailing list (`#1938 `__) `Valeriu Predoi `__ +- Make CircleCI badge specific to master branch (`#1831 `__) `Bouwe Andela `__ +- Documentation on how to move code from a private repository to a public repository (`#1920 `__) `Birgit Hassler `__ +- Refine pull request review guidelines (`#1924 `__) `Stef Smeets `__ +- Update release schedule (`#1948 `__) `Klaus Zimmermann `__ +- Improve contact info and move to more prominent location (`#1950 `__) `Bouwe Andela `__ +- Add some maintainers to some recipes that are missing them (`#1970 `__) `Valeriu Predoi `__ +- Update core team info (`#1973 `__) `Axel Lauer `__ +- Combine installation from source instructions and add common issues (`#1971 `__) `Bouwe Andela `__ +- Update iris documentation URL for sphinx (`#2003 `__) `Bouwe Andela `__ +- Fix iris documentation link(s) with new iris3 location on readthedocs (`#2012 `__) `Valeriu Predoi `__ +- Document how to run tests for installation verification (`#1847 `__) `Valeriu Predoi `__ +- List Remi Kazeroni as a code owner and sole merger of CMORizers (`#2017 `__) `Bouwe Andela `__ +- Install documentation: mention that we build conda package with python>=3.7 (`#2030 `__) `Valeriu Predoi `__ +- Recipe and documentation update for ERA5-Land. (`#1906 `__) `katjaweigel `__ + +Diagnostics +~~~~~~~~~~~ + +- Add mapplot diagnostic to ClimWIP (`#1864 `__) `Lukas Brunner `__ +- Add the option to weight variable groups in ClimWIP (`#1856 `__) `Lukas Brunner `__ +- Implementation of ensemble member recognition to the ClimWIP diagnostic (`#1852 `__) `Lukas Brunner `__ +- Restructure ClimWIP (`#1919 `__) `Lukas Brunner `__ +- Diagnostic for recipe_eyring13jgr.yml Fig. 12 (`#1922 `__) `Lisa Bock `__ +- Added changes in shared functions necessary for schlund20esd (`#1967 `__) `Manuel Schlund `__ +- Adding recipe and diagnostics for Gier et al 2020 (`#1914 `__) `Bettina Gier `__ +- Added recipe, diagnostics and documentation for Schlund et al., ESD (2020) (`#2015 `__) `Manuel Schlund `__ +- Add PRIMAVERA Eady Growth Rate diagnostic (`#1285 `__) `sloosvel `__ +- Implement shape parameter calibration for ClimWIP (`#1905 `__) `Lukas Brunner `__ + +Observational and re-analysis dataset support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Extended ESRL cmorizer (`#1937 `__) `Bettina Gier `__ +- Cmorizer for GRACE data (`#1694 `__) `bascrezee `__ +- Cmorizer for latest ESACCI-SST data (`#1895 `__) `Valeriu Predoi `__ +- Fix longitude in ESRL cmorizer (`#1988 `__) `Bettina Gier `__ +- Selectively turn off fixing bounds for coordinates during cmorization with utilities.py (`#2014 `__) `Valeriu Predoi `__ +- Cmorize hadcrut5 (`#1977 `__) `mwjury `__ +- Cmorize gpcc masking (`#1995 `__) `mwjury `__ +- Cmorize_utils_save_1mon_Amon (`#1990 `__) `mwjury `__ +- Cmorize gpcc fix (`#1982 `__) `mwjury `__ +- Fix flake8 raised by develop test in cmorize_obs_gpcc.py (`#2038 `__) `Valeriu Predoi `__ + +Automatic testing +~~~~~~~~~~~~~~~~~ + +- Switched miniconda conda setup hooks for Github Actions workflows (`#1913 `__) `Valeriu Predoi `__ +- Fix style issue (`#1929 `__) `Bouwe Andela `__ +- Fix mlr test with solution that works for CentOS too (`#1936 `__) `Valeriu Predoi `__ +- Temporary deactivation Github Actions on OSX (`#1939 `__) `Valeriu Predoi `__ +- Fix conda installation test on CircleCI (`#1952 `__) `Bouwe Andela `__ +- Github Actions: change time for cron job that installs from conda (`#1969 `__) `Valeriu Predoi `__ +- CI upload relevant artifacts for test job (`#1999 `__) `Valeriu Predoi `__ +- Github Actions test that runs with the latest ESMValCore master (`#1989 `__) `Valeriu Predoi `__ +- Introduce python 39 in Github Actions tests (`#2029 `__) `Valeriu Predoi `__ +- Remove test for conda package installation on Python 3.6 (`#2033 `__) `Valeriu Predoi `__ +- Update codacy coverage reporter to fix coverage (`#2039 `__) `Bouwe Andela `__ + +Installation +~~~~~~~~~~~~ + +- Simplify installation of R development dependencies (`#1930 `__) `Bouwe Andela `__ +- Fix docker build (`#1934 `__) `Bouwe Andela `__ +- Use new conda environment for installing ESMValTool in Docker containers (`#1993 `__) `Bouwe Andela `__ +- Fix conda build (`#2026 `__) `Bouwe Andela `__ + +Improvements +~~~~~~~~~~~~ + +- Allow multiple references for a cmorizer script (`#1953 `__) `SarahAlidoost `__ +- Add GRACE to the recipe check_obs (`#1963 `__) `Rémi Kazeroni `__ +- Align ESMValTool to ESMValCore=2.2.0 (adopt iris3, fix environment for new Core release) (`#1874 `__) `Stef Smeets `__ +- Make it possible to use write_plots and write_netcdf from recipe instead of config-user.yml (`#2018 `__) `Bouwe Andela `__ +- Revise lisflood and hype recipes (`#2035 `__) `SarahAlidoost `__ +- Set version to 2.2.0 (`#2042 `__) `Javier Vegas-Regidor `__ + .. _changelog-v2-1-1: v2.1.1 @@ -52,7 +174,6 @@ Documentation - Add more info on documenting a recipe (`#1795 `__) `Bouwe Andela `__ - Improve the Python example diagnostic and documentation (`#1827 `__) `Bouwe Andela `__ - Improve description of how to use draft_release_notes.py (`#1848 `__) `Bouwe Andela `__ -- Increase version to 2.1.0 and add release notes (`#1868 `__) `Valeriu Predoi `__ - Update changelog for release 2.1 (`#1886 `__) `Valeriu Predoi `__ Improvements diff --git a/doc/sphinx/source/utils.rst b/doc/sphinx/source/utils.rst index fcc76f6e32..9e08fdca39 100644 --- a/doc/sphinx/source/utils.rst +++ b/doc/sphinx/source/utils.rst @@ -24,14 +24,15 @@ To use the tool, install the package pygithub_: Create a `GitHub access token`_ (leave all boxes for additional permissions unchecked) and store it in the file ``~/.github_api_key``. -Edit the script and update the date and time of the previous release. If needed, -change it so it uses the correct repository. - -Run the script: +Edit the script and update the date and time of the previous release and run +the script: .. code-block:: bash - python esmvaltool/utils/draft_release_notes.py + python esmvaltool/utils/draft_release_notes.py ${REPOSITORY} + +``REPOSITORY`` can be either ``esmvalcore`` or ``esmvaltool`` depending on the +release notes you want to create. Review the resulting output (in ``.rst`` format) and if anything needs changing, change it on GitHub and re-run the script until the changelog looks acceptable. diff --git a/esmvaltool/utils/draft_release_notes.py b/esmvaltool/utils/draft_release_notes.py index 2e5c851e93..1ec9690e39 100644 --- a/esmvaltool/utils/draft_release_notes.py +++ b/esmvaltool/utils/draft_release_notes.py @@ -5,10 +5,18 @@ 2) Create an access token and store it in the file ~/.github_api_key, see: https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line 3) set PREVIOUS_RELEASE to the date/time of the previous release in code below - +4) Call the script passing the project to create release notes on: esmvalcore +or esmvaltool """ import datetime from pathlib import Path +from typing import DefaultDict + +import dateutil +import esmvalcore +import fire + +import esmvaltool try: from github import Github @@ -23,93 +31,158 @@ "authenticating-to-github/creating-a-personal-access-token-" "for-the-command-line") -from esmvaltool import __version__ +VERSION = { + 'esmvalcore': f"v{esmvalcore.__version__}", + 'esmvaltool': f"v{esmvaltool.__version__}" +} +GITHUB_REPO = { + 'esmvalcore': "ESMValGroup/ESMValCore", + 'esmvaltool': "ESMValGroup/ESMValTool", +} -VERSION = f"v{__version__}" -GITHUB_REPO = "ESMValGroup/ESMValTool" +PREVIOUS_RELEASE = { + 'esmvalcore': datetime.datetime(2020, 10, 13, 00), + 'esmvaltool': datetime.datetime(2020, 10, 26, 00), +} +LABELS = { + 'esmvalcore': ( + 'bug', + 'deprecated feature', + 'documentation', + 'fix for dataset', + 'cmor', + 'preprocessor', + 'api', + 'testing', + 'installation', + 'enhancement', + ), + 'esmvaltool': ( + 'bug', + 'deprecated feature', + 'documentation', + 'diagnostic', + 'preprocessor', + 'observations', + 'testing', + 'installation', + 'enhancement', + ) +} TITLES = { 'bug': 'Bug fixes', + 'deprecated feature': 'Deprecations', + 'cmor': 'CMOR standard', 'diagnostic': 'Diagnostics', 'fix for dataset': 'Fixes for datasets', 'observations': 'Observational and re-analysis dataset support', + 'testing': 'Automatic testing', + 'api': 'Notebook API (experimental)', 'enhancement': 'Improvements', } -def draft_notes_since(previous_release_date, labels): +def draft_notes_since(project, previous_release_date=None, labels=None): """Draft release notes containing the merged pull requests. Arguments --------- + project: str + Project to draft release notes from. Valid options are esmvaltool and + esmvalcore previous_release_date: datetime.datetime date of the previous release labels: list list of GitHub labels that deserve separate sections """ - session = Github(GITHUB_API_KEY) - repo = session.get_repo(GITHUB_REPO) - pulls = repo.get_pulls( - state='closed', - sort='updated', - direction='desc', - ) - - lines = {} + project = project.lower() + if previous_release_date is None: + previous_release_date = PREVIOUS_RELEASE[project] + else: + previous_release_date = dateutil.parse(previous_release_date) + if labels is None: + labels = LABELS[project] + + pulls = _get_pull_requests(project) + + lines = DefaultDict(list) + labelless_pulls = [] for pull in pulls: print(pull.updated_at, pull.merged_at, pull.number, pull.title) if pull.updated_at < previous_release_date: break - if pull.merged: - if pull.merged_at < previous_release_date: - continue - pr_labels = {label.name for label in pull.labels} - for label in labels: - if label in pr_labels: - break - else: - label = 'enhancement' - - user = pull.user - username = user.login if user.name is None else user.name - title = pull.title - title = title[0].upper() + title[1:] - line = ( - f"- {title} (`#{pull.number} " - f"`__) " - f"`{username} `__") - if label not in lines: - lines[label] = [] - lines[label].append((pull.closed_at, line)) + if not pull.merged or pull.merged_at < previous_release_date: + continue + pr_labels = {label.name for label in pull.labels} + for label in labels: + if label in pr_labels: + break + else: + labelless_pulls.append(pull) + label = 'enhancement' + lines[label].append((pull.closed_at, _compose_note(pull))) + + # Warn about label-less PR: + _list_labelless_pulls(labelless_pulls) # Create sections sections = [ - VERSION, - '-' * len(VERSION), + VERSION[project], + '-' * len(VERSION[project]), '', "This release includes", ] - for label in sorted(lines): - entries = sorted(lines[label]) # sort by merge time + for label in labels: + try: + entries = sorted(lines[label]) # sort by merge time + except KeyError: + continue title = TITLES.get(label, label.title()) sections.append('\n'.join(['', title, '~' * len(title), ''])) sections.append('\n'.join(entry for _, entry in entries)) notes = '\n'.join(sections) - print(notes) -if __name__ == '__main__': - - PREVIOUS_RELEASE = datetime.datetime(2020, 8, 3, 18) - LABELS = ( - 'bug', - 'documentation', - 'diagnostic', - 'fix for dataset', - 'preprocessor', - 'observations', - 'enhancement', +def _get_pull_requests(project): + session = Github(GITHUB_API_KEY) + repo = session.get_repo(GITHUB_REPO[project]) + pulls = repo.get_pulls( + state='closed', + sort='updated', + direction='desc', ) + return pulls + + +def _list_labelless_pulls(labelless_pulls): + if labelless_pulls: + print('\nPlease add labels to the following PR:') + for pull in labelless_pulls: + print(pull.html_url) + print('\n') + - draft_notes_since(PREVIOUS_RELEASE, LABELS) +def _compose_note(pull): + user = pull.user + username = user.login if user.name is None else user.name + title = pull.title + title = title[0].upper() + title[1:] + return (f"- {title} (`#{pull.number} " + f"<{pull.html_url}>`__) " + f"`{username} `__") + + +def main(): + """Entry point for the scrip.""" + def display(lines, out): + text = "\n".join(lines) + "\n" + out.write(text) + + fire.core.Display = display + fire.Fire(draft_notes_since) + + +if __name__ == '__main__': + main() diff --git a/tests/unit/test_docs.py b/tests/unit/test_docs.py new file mode 100644 index 0000000000..73a7701c7f --- /dev/null +++ b/tests/unit/test_docs.py @@ -0,0 +1,21 @@ +"""Tests for the documentation.""" +import collections +import os +import re + + +def test_duplications_in_changelog(): + changelog_path = os.path.join(os.path.dirname(__file__), '../..', + 'doc/sphinx/source/changelog.rst') + with open(changelog_path) as changelog: + changelog = changelog.read() + + pr_links = re.compile( + "") + links = pr_links.findall(changelog) + if len(links) != len(set(links)): + print('The following PR are duplicated in the changelog:') + print('\n'.join((link + for link, count in collections.Counter(links).items() + if count > 1))) + assert False