From 721b3309c2d91ab980fdabfbc0be90ada5629452 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 11 Feb 2019 13:36:06 +0100 Subject: [PATCH 001/225] Fixed #42. Added support for file open `**kwargs` --- portalocker/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 371e0fc..585f0f2 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -68,7 +68,7 @@ class Lock(object): def __init__( self, filename, mode='a', timeout=DEFAULT_TIMEOUT, check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, - flags=LOCK_METHOD): + flags=LOCK_METHOD, **file_open_kwargs): '''Lock manager with build-in timeout filename -- filename @@ -79,6 +79,7 @@ def __init__( check_interval -- check interval while waiting fail_when_locked -- after the initial lock failed, return an error or lock the file + **file_open_kwargs -- The kwargs for the `open(...)` call fail_when_locked is useful when multiple threads/processes can race when creating a file. If set to true than the system will wait till @@ -102,6 +103,7 @@ def __init__( self.check_interval = check_interval self.fail_when_locked = fail_when_locked self.flags = flags + self.file_open_kwargs = file_open_kwargs def acquire( self, timeout=None, check_interval=None, fail_when_locked=None): @@ -168,7 +170,7 @@ def release(self): def _get_fh(self): '''Get a new filehandle''' - return open(self.filename, self.mode) + return open(self.filename, self.mode, **self.file_open_kwargs) def _get_lock(self, fh): ''' From 6bf1d3de19b632568f01a84b05a10937d2fa4335 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 11 Feb 2019 13:42:28 +0100 Subject: [PATCH 002/225] Incrementing version to v1.4.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 7bb8266..973ab75 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '1.3.0' +__version__ = '1.4.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 5c47782..2fb2988 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -11,7 +11,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '1.3.0' +__version__ = '1.4.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 88ee9ca800beab52fadeb575ee325d4bd92be6bf Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 12 Jul 2019 15:32:01 +0200 Subject: [PATCH 003/225] renamed tests directory to fix #44 --- .travis.yml | 2 +- MANIFEST.in | 2 +- {tests => portalocker_tests}/__init__.py | 0 {tests => portalocker_tests}/conftest.py | 0 {tests => portalocker_tests}/requirements.txt | 0 {tests => portalocker_tests}/temporary_file_lock.py | 0 {tests => portalocker_tests}/test_combined.py | 0 {tests => portalocker_tests}/tests.py | 0 pytest.ini | 2 +- tox.ini | 4 ++-- 10 files changed, 5 insertions(+), 5 deletions(-) rename {tests => portalocker_tests}/__init__.py (100%) rename {tests => portalocker_tests}/conftest.py (100%) rename {tests => portalocker_tests}/requirements.txt (100%) rename {tests => portalocker_tests}/temporary_file_lock.py (100%) rename {tests => portalocker_tests}/test_combined.py (100%) rename {tests => portalocker_tests}/tests.py (100%) diff --git a/.travis.yml b/.travis.yml index fb62491..5373e28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ before_install: - wheel version install: - pip install -U setuptools wheel pip -- pip install -r tests/requirements.txt +- pip install -r portalocker_tests/requirements.txt - pip install -e . - pip install coveralls script: diff --git a/MANIFEST.in b/MANIFEST.in index 076f508..10cc305 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include CHANGELOG include README.rst include LICENSE -recursive-include tests *.py +recursive-include portalocker_tests *.py diff --git a/tests/__init__.py b/portalocker_tests/__init__.py similarity index 100% rename from tests/__init__.py rename to portalocker_tests/__init__.py diff --git a/tests/conftest.py b/portalocker_tests/conftest.py similarity index 100% rename from tests/conftest.py rename to portalocker_tests/conftest.py diff --git a/tests/requirements.txt b/portalocker_tests/requirements.txt similarity index 100% rename from tests/requirements.txt rename to portalocker_tests/requirements.txt diff --git a/tests/temporary_file_lock.py b/portalocker_tests/temporary_file_lock.py similarity index 100% rename from tests/temporary_file_lock.py rename to portalocker_tests/temporary_file_lock.py diff --git a/tests/test_combined.py b/portalocker_tests/test_combined.py similarity index 100% rename from tests/test_combined.py rename to portalocker_tests/test_combined.py diff --git a/tests/tests.py b/portalocker_tests/tests.py similarity index 100% rename from tests/tests.py rename to portalocker_tests/tests.py diff --git a/pytest.ini b/pytest.ini index 401cca6..b6ec2ab 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] python_files = - tests/*.py + portalocker_tests/*.py addopts = --ignore setup.py diff --git a/tox.ini b/tox.ini index d3b9be2..cdfe768 100644 --- a/tox.ini +++ b/tox.ini @@ -10,13 +10,13 @@ basepython = py36: python3.6 pypy: pypy -deps = -r{toxinidir}/tests/requirements.txt +deps = -r{toxinidir}/portalocker_tests/requirements.txt commands = python -m pytest {posargs} [testenv:flake8] basepython = python2.7 deps = flake8 -commands = flake8 --ignore=W391 {toxinidir}/portalocker {toxinidir}/tests +commands = flake8 --ignore=W391 {toxinidir}/portalocker {toxinidir}/portalocker_tests [testenv:docs] basepython = python2.7 From 25c1d45d8c7d8a4d45e626fdc2b674a858c76962 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 12 Jul 2019 15:38:34 +0200 Subject: [PATCH 004/225] attempting to fix appveyor --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index cdfe768..802c2c6 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,10 @@ basepython = py36: python3.6 pypy: pypy -deps = -r{toxinidir}/portalocker_tests/requirements.txt +deps = + setuptools >= 41.0.1 + pip >= 19.1.1 + -r{toxinidir}/portalocker_tests/requirements.txt commands = python -m pytest {posargs} [testenv:flake8] From c5d7e4c2829bafca8521934f3423e9f01fc1a72d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 12 Jul 2019 15:43:57 +0200 Subject: [PATCH 005/225] attempting to fix appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 2d67dec..7c5851c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,7 +24,7 @@ install: build: false # Not a C# project, build stuff at the test step instead. test_script: - - "%PYTHON%/Scripts/tox -e %TOX_ENV%" + - "%PYTHON%/Scripts/tox -r -e %TOX_ENV%" after_test: - "%PYTHON%/python setup.py bdist_wheel" From 8c869403ae89f0e9d730af1be7022803abad2b4f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 12 Jul 2019 15:50:02 +0200 Subject: [PATCH 006/225] attempting to fix appveyor --- appveyor.yml | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 7c5851c..9c64e2b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,6 +19,7 @@ init: install: - "%PYTHON%/Scripts/easy_install -U pip" + - "%PYTHON%/python -m pip install --upgrade pip" - "%PYTHON%/Scripts/pip install -U setuptools tox wheel" build: false # Not a C# project, build stuff at the test step instead. diff --git a/tox.ini b/tox.ini index 802c2c6..12ab8d4 100644 --- a/tox.ini +++ b/tox.ini @@ -11,8 +11,8 @@ basepython = pypy: pypy deps = - setuptools >= 41.0.1 pip >= 19.1.1 + setuptools >= 41.0.1 -r{toxinidir}/portalocker_tests/requirements.txt commands = python -m pytest {posargs} From ebb21c3573d3d4c9c1d2c58c6c5ba29e8d61f088 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 12 Jul 2019 15:57:51 +0200 Subject: [PATCH 007/225] attempting to fix appveyor --- tox.ini | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 12ab8d4..762086c 100644 --- a/tox.ini +++ b/tox.ini @@ -10,10 +10,7 @@ basepython = py36: python3.6 pypy: pypy -deps = - pip >= 19.1.1 - setuptools >= 41.0.1 - -r{toxinidir}/portalocker_tests/requirements.txt +deps = -e{toxinidir}[tests] commands = python -m pytest {posargs} [testenv:flake8] From 35c0369dd9d8dd98f3bb9ba3d069302842f32580 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 12 Jul 2019 16:10:53 +0200 Subject: [PATCH 008/225] attempting to fix appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 9c64e2b..22d8430 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -25,7 +25,7 @@ install: build: false # Not a C# project, build stuff at the test step instead. test_script: - - "%PYTHON%/Scripts/tox -r -e %TOX_ENV%" + - "%PYTHON%/Scripts/tox --sitepackages -r -e %TOX_ENV%" after_test: - "%PYTHON%/python setup.py bdist_wheel" From 747419f98a504469a755a81622a303d6237eaf0c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 12 Jul 2019 16:17:32 +0200 Subject: [PATCH 009/225] attempting to fix appveyor --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 762086c..4d8a8c7 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ basepython = pypy: pypy deps = -e{toxinidir}[tests] +commands_pre = python -m pip install --upgrade pip commands = python -m pytest {posargs} [testenv:flake8] From 6c247031575b6033358f40adc67bb1be2f83d491 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 12 Jul 2019 16:17:41 +0200 Subject: [PATCH 010/225] attempting to fix appveyor --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4d8a8c7..c3a9eaf 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ basepython = pypy: pypy deps = -e{toxinidir}[tests] -commands_pre = python -m pip install --upgrade pip +commands_pre = python -m pip install --upgrade pip setuptools wheel commands = python -m pytest {posargs} [testenv:flake8] From 86211e8bfe8f5e59a651176c0c97686753efd512 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 12 Jul 2019 16:22:26 +0200 Subject: [PATCH 011/225] attempting to fix appveyor --- appveyor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 22d8430..35889c5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,13 +19,12 @@ init: install: - "%PYTHON%/Scripts/easy_install -U pip" - - "%PYTHON%/python -m pip install --upgrade pip" - "%PYTHON%/Scripts/pip install -U setuptools tox wheel" build: false # Not a C# project, build stuff at the test step instead. test_script: - - "%PYTHON%/Scripts/tox --sitepackages -r -e %TOX_ENV%" + - "%PYTHON%/Scripts/tox --force-dep=pip>10.0.0 -r -e %TOX_ENV%" after_test: - "%PYTHON%/python setup.py bdist_wheel" From 3f2138b896d8b6a55261efec09f13d154e598dfa Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 12 Jul 2019 16:22:30 +0200 Subject: [PATCH 012/225] attempting to fix appveyor --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index c3a9eaf..762086c 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,6 @@ basepython = pypy: pypy deps = -e{toxinidir}[tests] -commands_pre = python -m pip install --upgrade pip setuptools wheel commands = python -m pytest {posargs} [testenv:flake8] From ad234fd1dce7395c24a3fd838956b86a94aff71d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 13 Jul 2019 22:26:23 +0200 Subject: [PATCH 013/225] attempting to fix appveyor --- appveyor.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 35889c5..fedcff2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,12 +19,13 @@ init: install: - "%PYTHON%/Scripts/easy_install -U pip" - - "%PYTHON%/Scripts/pip install -U setuptools tox wheel" + - "%PYTHON%/Scripts/pip install -U setuptools wheel" + - "%PYTHON%/Scripts/pip install -e .[tests]" build: false # Not a C# project, build stuff at the test step instead. test_script: - - "%PYTHON%/Scripts/tox --force-dep=pip>10.0.0 -r -e %TOX_ENV%" + - "%PYTHON%/python -m pytest" after_test: - "%PYTHON%/python setup.py bdist_wheel" From cddcd37d8220fdce730606dcf4a050ed20e4e9d5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 13 Jul 2019 22:34:04 +0200 Subject: [PATCH 014/225] Incrementing version to v1.5.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 973ab75..b40c63f 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '1.4.0' +__version__ = '1.5.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 2fb2988..34b1d84 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -11,7 +11,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '1.4.0' +__version__ = '1.5.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 6708552f4465b7142446fde19d67ce15da14c4ff Mon Sep 17 00:00:00 2001 From: Igor Gnatenko Date: Sun, 4 Aug 2019 14:44:54 +0200 Subject: [PATCH 015/225] Include proper CHANGELOG file --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 10cc305..cbbc4f8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include CHANGELOG +include CHANGELOG.rst include README.rst include LICENSE recursive-include portalocker_tests *.py From 1e33ac812d5cff732cc75d241f3c3e2bea86383f Mon Sep 17 00:00:00 2001 From: Igor Gnatenko Date: Sun, 4 Aug 2019 14:46:15 +0200 Subject: [PATCH 016/225] Exclude portalocker_tests from installed packages --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 79df3fc..3919ba1 100644 --- a/setup.py +++ b/setup.py @@ -127,7 +127,7 @@ def run(self): author_email=about['__email__'], url=about['__url__'], license='PSF', - packages=setuptools.find_packages(exclude=['ez_setup', 'examples']), + packages=setuptools.find_packages(exclude=['ez_setup', 'examples', 'portalocker_tests']), # zip_safe=False, platforms=['any'], cmdclass={ From 696a964c922012506be391a9fa6218a84411250c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 5 Aug 2019 03:58:26 +0200 Subject: [PATCH 017/225] updated changelog, for more details look at git commit log: https://github.com/WoLpH/portalocker/commits/develop --- CHANGELOG.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 36811a9..203a2d2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,26 @@ +1.5: + + * Moved tests to prevent collisions with other packages + +1.4: + + * Added optional file open parameters + +1.3: + + * Improved documentation + * Added file handle to locking exceptions + +1.2: + + * Added signed releases and tags to PyPI and Git + + +1.1: + + * Added support for Python 3.6+ + * Using real time to calculate timeout + 1.0: * Complete code refactor. From 673f25581129c7de092daa8f533d82ac4e091ac6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 5 Aug 2019 03:58:31 +0200 Subject: [PATCH 018/225] Incrementing version to v1.5.1 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index b40c63f..3517380 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '1.5.0' +__version__ = '1.5.1' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 34b1d84..a8ad34f 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -11,7 +11,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '1.5.0' +__version__ = '1.5.1' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From dccaa68199aa8f5ce13bf017e8a106dead69bc3b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 5 Aug 2019 04:20:58 +0200 Subject: [PATCH 019/225] Incrementing version to v1.5.1 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index b40c63f..3517380 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '1.5.0' +__version__ = '1.5.1' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 34b1d84..a8ad34f 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -11,7 +11,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '1.5.0' +__version__ = '1.5.1' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 7741925738c7e66ae9c4a0944a04b6a3088037d5 Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Mon, 28 Oct 2019 19:21:21 -0700 Subject: [PATCH 020/225] Allow for development setuptools --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 3919ba1..8a8444f 100644 --- a/setup.py +++ b/setup.py @@ -4,11 +4,11 @@ import sys import setuptools from setuptools.command.test import test as TestCommand -from distutils.version import StrictVersion +from distutils.version import LooseVersion from setuptools import __version__ as setuptools_version -if StrictVersion(setuptools_version) < StrictVersion('38.3.0'): +if LooseVersion(setuptools_version) < LooseVersion('38.3.0'): raise SystemExit( 'Your `setuptools` version is old. ' 'Please upgrade setuptools by running `pip install -U setuptools` ' @@ -47,7 +47,7 @@ def run_tests(self): import pytest errno = pytest.main(shlex.split(self.pytest_args)) sys.exit(errno) - + class Combine(setuptools.Command): description = 'Build single combined portalocker file' From dcda0ecdd7d0ff9da5e2f09bdcde7f9586b9cd9c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 12 Nov 2019 00:08:05 +0100 Subject: [PATCH 021/225] Blacklisted pywin32 version 226 to fix #48 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8a8444f..7351616 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ def run(self): 'test': PyTest, }, install_requires=[ - 'pypiwin32; platform_system == "Windows"', + 'pywin32!=226; platform_system == "Windows"', ], tests_require=tests_require, extras_require=dict( From 929fbf6221fb8ce6db5f0389ca148f8248fd61ed Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 12 Nov 2019 01:23:23 +0100 Subject: [PATCH 022/225] Incrementing version to v1.5.2 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 3517380..aba35ad 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '1.5.1' +__version__ = '1.5.2' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index a8ad34f..7a46cb0 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -11,7 +11,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '1.5.1' +__version__ = '1.5.2' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 782a1ca5b043be1c2e6de0222d4e2b749ba50d4a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Mar 2020 02:37:45 +0100 Subject: [PATCH 023/225] updated docs to new test location --- docs/tests.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tests.rst b/docs/tests.rst index 7fd5c81..12bbb50 100644 --- a/docs/tests.rst +++ b/docs/tests.rst @@ -4,7 +4,7 @@ tests package Module contents --------------- -.. automodule:: tests.tests +.. automodule:: portalocker_tests.tests :members: :private-members: :special-members: @@ -12,12 +12,12 @@ Module contents :undoc-members: :show-inheritance: -.. automodule:: tests.test_combined +.. automodule:: portalocker_tests.test_combined :members: :undoc-members: :show-inheritance: -.. automodule:: tests.temporary_file_lock +.. automodule:: portalocker_tests.temporary_file_lock :members: :private-members: :special-members: From dd5392c0df7ccc7c7814e1d9ebc6b9f331307164 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Mar 2020 02:40:05 +0100 Subject: [PATCH 024/225] modernized tests --- .travis.yml | 4 ++++ tox.ini | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5373e28..bfad044 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,14 @@ sudo: false +dist: xenial language: python python: - '2.7' - '3.4' - '3.5' - '3.6' +- '3.7' +- '3.8' +- '3.9' - pypy before_install: - wheel version diff --git a/tox.ini b/tox.ini index 762086c..2c622a8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py33, py34, py35, py36, pypy, flake8, docs +envlist = py27, py33, py34, py35, py36, py37, py38, py39, pypy, flake8, docs skip_missing_interpreters = True [testenv] @@ -8,6 +8,9 @@ basepython = py34: python3.4 py35: python3.5 py36: python3.6 + py37: python3.7 + py38: python3.8 + py39: python3.9 pypy: pypy deps = -e{toxinidir}[tests] From 921cc92583e8ab238b510c18e58b65331b358fd2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Mar 2020 02:41:39 +0100 Subject: [PATCH 025/225] Fixed unlocking bug on Windows. Fixes #49 --- README.rst | 11 +++++++---- portalocker/utils.py | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 2cd4b49..c013490 100644 --- a/README.rst +++ b/README.rst @@ -35,7 +35,9 @@ requests can be submitted there. Patches are also very welcome. Tips ---- -On some networked filesystems it might be needed to force a `os.fsync()` before closing the file so it's actually written before another client reads the file. Effectively this comes down to: +On some networked filesystems it might be needed to force a `os.fsync()` before +closing the file so it's actually written before another client reads the file. +Effectively this comes down to: :: @@ -79,9 +81,10 @@ To customize the opening and locking a manual approach is also possible: >>> file.write('foo') >>> file.close() -There is no explicit need to unlock the file as it is automatically unlocked -after `file.close()`. If you still feel the need to manually unlock a file -than you can do it like this: +Explicitly unlocking might not be needed in all cases: +https://github.com/AzureAD/microsoft-authentication-extensions-for-python/issues/42#issuecomment-601108266 + +But can be done through: >>> portalocker.unlock(file) diff --git a/portalocker/utils.py b/portalocker/utils.py index 585f0f2..c1faa26 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -165,6 +165,7 @@ def acquire( def release(self): '''Releases the currently locked file handle''' if self.fh: + portalocker.unlock(self.fh) self.fh.close() self.fh = None From cb9e7b024212c65ec84f96e6931295c4f557ff02 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Mar 2020 02:44:17 +0100 Subject: [PATCH 026/225] fixed tests --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bfad044..7e0ae05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,8 @@ python: - '3.6' - '3.7' - '3.8' -- '3.9' +# TODO: Enable when available +# - '3.9' - pypy before_install: - wheel version From d9147bf3cdb999b75888e223935475cc2f63bac3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Mar 2020 02:45:37 +0100 Subject: [PATCH 027/225] Incrementing version to v1.6.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index aba35ad..6ac8c0a 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '1.5.2' +__version__ = '1.6.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 7a46cb0..8d81d5f 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -11,7 +11,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '1.5.2' +__version__ = '1.6.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From ff2d0420d7a20b883eb80959a8e8505e7aaae6ff Mon Sep 17 00:00:00 2001 From: ahauan4 <41740080+ahauan4@users.noreply.github.com> Date: Sun, 12 Apr 2020 17:26:01 +0200 Subject: [PATCH 028/225] Fixes ResourceWarning 'unclosed file' if LockException is raised ResourceWarning: unclosed file Object allocated at (most recent call last): File "portalocker\utils.py", lineno 174 return open(self.filename, self.mode, **self.file_open_kwargs) --- portalocker/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/portalocker/utils.py b/portalocker/utils.py index c1faa26..1e7e7a0 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -153,6 +153,7 @@ def acquire( pass else: + fh.close() # We got a timeout... reraising raise exceptions.LockException(exception) From de9a7c468ca5be601cce4d5f271861bb3c2c1959 Mon Sep 17 00:00:00 2001 From: ahauan4 <41740080+ahauan4@users.noreply.github.com> Date: Sun, 12 Apr 2020 17:42:08 +0200 Subject: [PATCH 029/225] fixed tox error W291 trailing whitespace --- portalocker/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 1e7e7a0..8baebc2 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -153,7 +153,7 @@ def acquire( pass else: - fh.close() + fh.close() # We got a timeout... reraising raise exceptions.LockException(exception) From 98f6bfc1a80bc93a0169beb5cbd14287a81b8d4b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 12 Apr 2020 17:56:54 +0200 Subject: [PATCH 030/225] Incrementing version to v1.7.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 6ac8c0a..4b765d8 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '1.6.0' +__version__ = '1.7.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 8d81d5f..044ca50 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -11,7 +11,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '1.6.0' +__version__ = '1.7.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From ac0cea5c124d62b0a878d478307dac858cd2aba9 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 25 Apr 2020 23:56:09 +0200 Subject: [PATCH 031/225] Create FUNDING.yml --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..5ca3704 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: wolph From 42e4c0a16bbc987c7e33b5cbc7676a63a164ceb5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 6 May 2020 10:10:23 +0200 Subject: [PATCH 032/225] switched pytest to flake8 from unmaintained packages --- pytest.ini | 11 ++--------- setup.py | 14 ++++++-------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/pytest.ini b/pytest.ini index b6ec2ab..9542cd2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,16 +10,9 @@ addopts = --cov-report term-missing --cov-report html --no-cov-on-fail - --pep8 - --flakes + --flake8 -pep8ignore = +flake8-ignore = *.py W391 - runtests.py ALL - docs/*.py ALL - -flakes-ignore = - *.py W391 - runtests.py ALL docs/*.py ALL diff --git a/setup.py b/setup.py index 7351616..7679cf7 100644 --- a/setup.py +++ b/setup.py @@ -24,13 +24,10 @@ tests_require = [ - 'flake8>=3.5.0', - 'pytest>=3.4.0', - 'pytest-cache>=1.0', - 'pytest-cov>=2.5.1', - 'pytest-flakes>=2.0.0', - 'pytest-pep8>=1.0.6', - 'sphinx>=1.7.1', + 'pytest>=4.6.9', + 'pytest-cov>=2.8.1', + 'sphinx>=1.8.5', + 'pytest-flake8>=1.0.5', ] @@ -127,7 +124,8 @@ def run(self): author_email=about['__email__'], url=about['__url__'], license='PSF', - packages=setuptools.find_packages(exclude=['ez_setup', 'examples', 'portalocker_tests']), + packages=setuptools.find_packages(exclude=[ + 'examples', 'portalocker_tests']), # zip_safe=False, platforms=['any'], cmdclass={ From 23bc2fc4e8387c4a3d2e1c212c8ad2831f177456 Mon Sep 17 00:00:00 2001 From: Callan Bryant Date: Wed, 20 May 2020 21:45:55 +0100 Subject: [PATCH 033/225] Remove potentially old code for LockFileEx LockFileEx is no longer used. --- portalocker/portalocker.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 460cf06..6c8eb67 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -48,10 +48,6 @@ def lock(file_, flags): # here? raise else: - mode = win32con.LOCKFILE_EXCLUSIVE_LOCK - if flags & constants.LOCK_NB: - mode |= win32con.LOCKFILE_FAIL_IMMEDIATELY - if flags & constants.LOCK_NB: mode = msvcrt.LK_NBLCK else: From 242e166d2bb677a247ea0a545b727d56766d90bb Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 16 Jul 2020 22:58:17 +0200 Subject: [PATCH 034/225] Incrementing version to v1.7.1 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 4b765d8..f16fe0c 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '1.7.0' +__version__ = '1.7.1' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 044ca50..9bf27fe 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -11,7 +11,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '1.7.0' +__version__ = '1.7.1' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 65c318a536efffdccc0a1aa7f2a68d7e33dbd452 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 2 Aug 2020 02:06:09 +0200 Subject: [PATCH 035/225] added boudned semaphore to fix #57 --- README.rst | 27 ++++++++- portalocker/__init__.py | 3 + portalocker/utils.py | 90 +++++++++++++++++++++++++++++ portalocker_tests/test_semaphore.py | 22 +++++++ portalocker_tests/tests.py | 1 + 5 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 portalocker_tests/test_semaphore.py diff --git a/README.rst b/README.rst index c013490..9c91b36 100644 --- a/README.rst +++ b/README.rst @@ -81,16 +81,39 @@ To customize the opening and locking a manual approach is also possible: >>> file.write('foo') >>> file.close() -Explicitly unlocking might not be needed in all cases: +Explicitly unlocking is not needed in most cases but omitting it has been known +to cause issues: https://github.com/AzureAD/microsoft-authentication-extensions-for-python/issues/42#issuecomment-601108266 -But can be done through: +If needed, it can be done through: >>> portalocker.unlock(file) Do note that your data might still be in a buffer so it is possible that your data is not available until you `flush()` or `close()`. +To create a cross platform bounded semaphore across multiple processes you can +use the `BoundedSemaphore` class which functions somewhat similar to +`threading.BoundedSemaphore`: + +>>> import portalocker +>>> n = 2 +>>> timeout = 0.1 + +>>> semaphore_a = portalocker.BoundedSemaphore(n, timeout=timeout) +>>> semaphore_b = portalocker.BoundedSemaphore(n, timeout=timeout) +>>> semaphore_c = portalocker.BoundedSemaphore(n, timeout=timeout) + +>>> semaphore_a.acquire() + +>>> semaphore_b.acquire() + +>>> semaphore_c.acquire() +Traceback (most recent call last): + ... +portalocker.exceptions.AlreadyLocked + + More examples can be found in the `tests `_. diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 9bf27fe..f425ea2 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -49,6 +49,7 @@ #: context wrappers Lock = utils.Lock RLock = utils.RLock +BoundedSemaphore = utils.BoundedSemaphore TemporaryFileLock = utils.TemporaryFileLock open_atomic = utils.open_atomic @@ -61,7 +62,9 @@ 'LOCK_UN', 'LockException', 'Lock', + 'RLock', 'AlreadyLocked', + 'BoundedSemaphore', 'open_atomic', ] diff --git a/portalocker/utils.py b/portalocker/utils.py index 8baebc2..d5f2315 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -1,6 +1,9 @@ import os +import abc import time import atexit +import random +import pathlib import tempfile import contextlib from . import exceptions @@ -254,3 +257,90 @@ def release(self): Lock.release(self) if os.path.isfile(self.filename): # pragma: no branch os.unlink(self.filename) + + +class BoundedSemaphore(LockBase): + ''' + Bounded semaphore to prevent too many parallel processes from running + + It's also possible to specify a timeout when acquiring the lock to wait + for a resource to become available. This is very similar to + threading.BoundedSemaphore but works across multiple processes and across + multiple operating systems. + + >>> semaphore = BoundedSemaphore(2, directory='') + >>> semaphore.get_filenames()[0] + PosixPath('bounded_semaphore.00.lock') + >>> sorted(semaphore.get_random_filenames())[1] + PosixPath('bounded_semaphore.01.lock') + ''' + + def __init__(self, maximum: int, name: str = 'bounded_semaphore', + filename_pattern: str = '{name}.{number:02d}.lock', directory: + str = tempfile.gettempdir(), timeout=DEFAULT_TIMEOUT, + check_interval=DEFAULT_CHECK_INTERVAL): + self.maximum = maximum + self.name = name + self.filename_pattern = filename_pattern + self.directory = directory + self.lock: Lock = None + self.timeout = timeout + self.check_interval = check_interval + + def get_filenames(self): + return [self.get_filename(n) for n in range(self.maximum)] + + def get_random_filenames(self): + filenames = self.get_filenames() + random.shuffle(filenames) + return filenames + + def get_filename(self, number): + return pathlib.Path(self.directory) / self.filename_pattern.format( + name=self.name, + number=number, + ) + + def acquire( + self, timeout=None, check_interval=None): + assert not self.lock, 'Already locked' + + if timeout is None: + timeout = self.timeout + if timeout is None: + timeout = 0 + + assert timeout < 0.2, f'timeout: {timeout} :: {self.timeout}' + if check_interval is None: + check_interval = self.check_interval + + filenames = self.get_filenames() + + if self.try_lock(filenames): + return self.lock + + if not timeout: + raise exceptions.AlreadyLocked() + + timeout_end = current_time() + timeout + while timeout_end > current_time(): # pragma: no branch + if self.try_lock(filenames): # pragma: no branch + return self.lock # pragma: no cover + + time.sleep(check_interval) + + raise exceptions.AlreadyLocked() + + def try_lock(self, filenames): + for filename in filenames: + self.lock = Lock(filename, fail_when_locked=True) + try: + self.lock.acquire() + return True + except exceptions.AlreadyLocked: + pass + + def release(self): # pragma: no cover + self.lock.release() + self.lock = None + diff --git a/portalocker_tests/test_semaphore.py b/portalocker_tests/test_semaphore.py new file mode 100644 index 0000000..0512460 --- /dev/null +++ b/portalocker_tests/test_semaphore.py @@ -0,0 +1,22 @@ +import random +import pytest +import portalocker +from portalocker import utils + + +@pytest.mark.parametrize('timeout', [None, 0, 0.001]) +@pytest.mark.parametrize('check_interval', [None, 0, 0.0005]) +def test_bounded_semaphore(timeout, check_interval, monkeypatch): + n = 2 + name = random.random() + print('args', timeout, check_interval) + monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) + monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) + semaphore_a = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) + semaphore_b = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) + semaphore_c = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) + + semaphore_a.acquire(timeout=timeout) + semaphore_b.acquire() + with pytest.raises(portalocker.AlreadyLocked): + semaphore_c.acquire(check_interval=check_interval, timeout=timeout) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index a656742..6f5fe3d 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -3,6 +3,7 @@ import pytest import portalocker +from portalocker import utils def test_exceptions(tmpfile): From 596da9f1e5d69cae17fe547a38788d06f48936c9 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 2 Aug 2020 02:06:41 +0200 Subject: [PATCH 036/225] cleanup --- README.rst | 10 +++++----- portalocker/utils.py | 17 ++++------------- portalocker_tests/tests.py | 5 +++++ 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index 9c91b36..8b4ad39 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ portalocker - Cross-platform locking library .. image:: https://travis-ci.org/WoLpH/portalocker.svg?branch=master :alt: Linux Test Status :target: https://travis-ci.org/WoLpH/portalocker - + .. image:: https://ci.appveyor.com/api/projects/status/mgqry98hgpy4prhh?svg=true :alt: Windows Tests Status :target: https://ci.appveyor.com/project/WoLpH/portalocker @@ -40,11 +40,11 @@ closing the file so it's actually written before another client reads the file. Effectively this comes down to: :: - + with portalocker.Lock('some_file', 'rb+', timeout=60) as fh: # do what you need to do ... - + # flush and sync to filesystem fh.flush() os.fsync(fh.fileno()) @@ -56,7 +56,7 @@ Links - http://portalocker.readthedocs.org/en/latest/ * Source - https://github.com/WoLpH/portalocker -* Bug reports +* Bug reports - https://github.com/WoLpH/portalocker/issues * Package homepage - https://pypi.python.org/pypi/portalocker @@ -70,7 +70,7 @@ To make sure your cache generation scripts don't race, use the `Lock` class: >>> import portalocker >>> with portalocker.Lock('somefile', timeout=1) as fh: - print >>fh, 'writing some stuff to my cache...' +... print >>fh, 'writing some stuff to my cache...' To customize the opening and locking a manual approach is also possible: diff --git a/portalocker/utils.py b/portalocker/utils.py index d5f2315..7fe6b61 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -134,8 +134,8 @@ def acquire( fh = self._get_lock(fh) except exceptions.LockException as exception: # Try till the timeout has passed - timeoutend = current_time() + timeout - while timeoutend > current_time(): + timeout_end = current_time() + timeout + while timeout_end > current_time(): # Wait a bit time.sleep(check_interval) @@ -198,22 +198,13 @@ def _prepare_fh(self, fh): return fh - def __enter__(self): - return self.acquire() - - def __exit__(self, type_, value, tb): - self.release() - - def __delete__(self, instance): # pragma: no cover - instance.release() - class RLock(Lock): - """ + ''' A reentrant lock, functions in a similar way to threading.RLock in that it can be acquired multiple times. When the corresponding number of release() calls are made the lock will finally release the underlying file lock. - """ + ''' def __init__( self, filename, mode='a', timeout=DEFAULT_TIMEOUT, check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 6f5fe3d..3cc9758 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -26,6 +26,11 @@ def test_exceptions(tmpfile): b.close() +def test_utils_base(): + class Test(utils.LockBase): + pass + + def test_with_timeout(tmpfile): # Open the file 2 times with pytest.raises(portalocker.AlreadyLocked): From c8f749d4aedf7c68e288b50a8dff0ee935ef91ab Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 2 Aug 2020 02:06:51 +0200 Subject: [PATCH 037/225] added base class for lock --- portalocker/utils.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 7fe6b61..d33b761 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -66,7 +66,28 @@ def open_atomic(filename, binary=True): pass -class Lock(object): +class LockBase(abc.ABC): # pragma: no cover + + @abc.abstractmethod + def acquire( + self, timeout=None, check_interval=None, fail_when_locked=None): + return NotImplemented + + @abc.abstractmethod + def release(self): + return NotImplemented + + def __enter__(self): + return self.acquire() + + def __exit__(self, type_, value, tb): + self.release() + + def __delete__(self, instance): + instance.release() + + +class Lock(LockBase): def __init__( self, filename, mode='a', timeout=DEFAULT_TIMEOUT, From 159d232de2006052a3a59d04e8bc12d40daf779b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 2 Aug 2020 02:14:08 +0200 Subject: [PATCH 038/225] removed old python versions --- .travis.yml | 2 -- tox.ini | 10 ++++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7e0ae05..d3acfc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,6 @@ sudo: false dist: xenial language: python python: -- '2.7' -- '3.4' - '3.5' - '3.6' - '3.7' diff --git a/tox.ini b/tox.ini index 2c622a8..33de79b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,28 +1,26 @@ [tox] -envlist = py27, py33, py34, py35, py36, py37, py38, py39, pypy, flake8, docs +envlist = py35, py36, py37, py38, py39, pypy3, flake8, docs skip_missing_interpreters = True [testenv] basepython = - py27: python2.7 - py34: python3.4 py35: python3.5 py36: python3.6 py37: python3.7 py38: python3.8 py39: python3.9 - pypy: pypy + pypy3: pypy3 deps = -e{toxinidir}[tests] commands = python -m pytest {posargs} [testenv:flake8] -basepython = python2.7 +basepython = python3 deps = flake8 commands = flake8 --ignore=W391 {toxinidir}/portalocker {toxinidir}/portalocker_tests [testenv:docs] -basepython = python2.7 +basepython = python3 deps = -r{toxinidir}/docs/requirements.txt whitelist_externals = rm From be67c963edaf8236feea0c0b138507265b3ed344 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 2 Aug 2020 02:19:08 +0200 Subject: [PATCH 039/225] removed type hint to support python 3.5 --- portalocker/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index d33b761..3f0830d 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -295,7 +295,7 @@ def __init__(self, maximum: int, name: str = 'bounded_semaphore', self.name = name self.filename_pattern = filename_pattern self.directory = directory - self.lock: Lock = None + self.lock = None self.timeout = timeout self.check_interval = check_interval @@ -322,7 +322,6 @@ def acquire( if timeout is None: timeout = 0 - assert timeout < 0.2, f'timeout: {timeout} :: {self.timeout}' if check_interval is None: check_interval = self.check_interval From 588b013d2fe1924edb2d571413eed6415b8d9153 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 2 Aug 2020 02:19:53 +0200 Subject: [PATCH 040/225] replaced pypy with pypy3 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d3acfc1..759677b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ python: - '3.8' # TODO: Enable when available # - '3.9' -- pypy +- pypy3 before_install: - wheel version install: From fff2a4cb71fb94aae9fcb2ae40ea5705866d0232 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 2 Aug 2020 02:24:29 +0200 Subject: [PATCH 041/225] force converting pathlib to filename --- portalocker/utils.py | 2 +- portalocker_tests/test_semaphore.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 3f0830d..d7b7b41 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -120,7 +120,7 @@ def __init__( truncate = False self.fh = None - self.filename = filename + self.filename = str(filename) self.mode = mode self.truncate = truncate self.timeout = timeout diff --git a/portalocker_tests/test_semaphore.py b/portalocker_tests/test_semaphore.py index 0512460..9dfc9dc 100644 --- a/portalocker_tests/test_semaphore.py +++ b/portalocker_tests/test_semaphore.py @@ -9,7 +9,6 @@ def test_bounded_semaphore(timeout, check_interval, monkeypatch): n = 2 name = random.random() - print('args', timeout, check_interval) monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) semaphore_a = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) From 6d681848c39acee660e631c7b8d2069535e2c4a7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 2 Aug 2020 02:32:12 +0200 Subject: [PATCH 042/225] Incrementing version to v2.0.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index f16fe0c..93cc02a 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '1.7.1' +__version__ = '2.0.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index f425ea2..835dc95 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -11,7 +11,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '1.7.1' +__version__ = '2.0.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 68347a44a2a7edc198dd6f3b704098d2c5f4d680 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 15 Dec 2020 13:46:20 +0100 Subject: [PATCH 043/225] Testing fix for `ResourceWarning` reported in #59 --- portalocker/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index d7b7b41..583691b 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -166,7 +166,8 @@ def acquire( # We already tried to the get the lock # If fail_when_locked is true, then stop trying if fail_when_locked: - raise exceptions.AlreadyLocked(exception) + fh.close() + raise exceptions.BaseLockException(exception) else: # pragma: no cover # We've got the lock From addca52b22345372aee3629e264b5aece1e805a9 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 21 Jan 2021 04:39:53 +0100 Subject: [PATCH 044/225] fixed AlreadyLocked issue thanks to @BoniLindsley --- portalocker/utils.py | 56 ++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index d7b7b41..61644e2 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -150,36 +150,40 @@ def acquire( # Get a new filehandler fh = self._get_fh() - try: - # Try to lock - fh = self._get_lock(fh) - except exceptions.LockException as exception: - # Try till the timeout has passed - timeout_end = current_time() + timeout - while timeout_end > current_time(): - # Wait a bit - time.sleep(check_interval) - - # Try again - try: - # We already tried to the get the lock - # If fail_when_locked is true, then stop trying - if fail_when_locked: - raise exceptions.AlreadyLocked(exception) + def try_close(): # pragma: no cover + # Silently try to close the handle if possible, ignore all issues + try: + fh.close() + except Exception: + pass - else: # pragma: no cover - # We've got the lock - fh = self._get_lock(fh) - break + # Try till the timeout has passed + timeout_end = current_time() + timeout + exception = None + while timeout_end > current_time(): + try: + # Try to lock + fh = self._get_lock(fh) + break + except exceptions.LockException as exc: + # Python will automatically remove the variable from memory + # unless you save it in a different location + exception = exc + + # We already tried to the get the lock + # If fail_when_locked is True, stop trying + if fail_when_locked: + try_close() + raise exceptions.AlreadyLocked(exception) - except exceptions.LockException: - pass + # Wait a bit + time.sleep(check_interval) - else: - fh.close() - # We got a timeout... reraising - raise exceptions.LockException(exception) + else: + try_close() + # We got a timeout... reraising + raise exceptions.LockException(exception) # Prepare the filehandle (truncate if needed) fh = self._prepare_fh(fh) From d3872fc3ada6d9a2bc510ac24ea510b63c8a7d2e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 21 Jan 2021 15:12:41 +0100 Subject: [PATCH 045/225] fixed appveyor tests --- portalocker/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 61644e2..6868403 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -285,10 +285,10 @@ class BoundedSemaphore(LockBase): multiple operating systems. >>> semaphore = BoundedSemaphore(2, directory='') - >>> semaphore.get_filenames()[0] - PosixPath('bounded_semaphore.00.lock') - >>> sorted(semaphore.get_random_filenames())[1] - PosixPath('bounded_semaphore.01.lock') + >>> str(semaphore.get_filenames()[0]) + 'bounded_semaphore.00.lock' + >>> str(sorted(semaphore.get_random_filenames())[1]) + 'bounded_semaphore.01.lock' ''' def __init__(self, maximum: int, name: str = 'bounded_semaphore', From 216c6133cca5c7e2d8db59f59162c2a116dff0ea Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 21 Jan 2021 15:16:21 +0100 Subject: [PATCH 046/225] attempting appveyor config update --- appveyor.yml | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index fedcff2..baa982e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,32 +3,24 @@ environment: matrix: - - PYTHON: "C:\\Python27" - TOX_ENV: "py27" - - - PYTHON: "C:\\Python35" - TOX_ENV: "py35" - - - PYTHON: "C:\\Python36" - TOX_ENV: "py36" - - -init: - - "%PYTHON%/python -V" - - "%PYTHON%/python -c \"import struct;print( 8 * struct.calcsize(\'P\'))\"" + - TOXENV: py27 + - TOXENV: py35 + - TOXENV: py36 + - TOXENV: py37 + - TOXENV: py38 + - TOXENV: py39 install: - - "%PYTHON%/Scripts/easy_install -U pip" - - "%PYTHON%/Scripts/pip install -U setuptools wheel" - - "%PYTHON%/Scripts/pip install -e .[tests]" + - pip install -U tox setuptools wheel + - pip install -Ue .[tests] -build: false # Not a C# project, build stuff at the test step instead. +build: off # Not a C# project, build stuff at the test step instead. test_script: - - "%PYTHON%/python -m pytest" + - tox after_test: - - "%PYTHON%/python setup.py bdist_wheel" + - python setup.py sdist bdist_wheel - ps: "ls dist" artifacts: From cdbefd4a5f272b8ad8c4d0468931e97d01854e62 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 21 Jan 2021 15:20:01 +0100 Subject: [PATCH 047/225] removed python 2.7 from appveyor as well --- appveyor.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index baa982e..cbd6cc8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,7 +3,6 @@ environment: matrix: - - TOXENV: py27 - TOXENV: py35 - TOXENV: py36 - TOXENV: py37 From 41de2a09b6a2f52f2afe0a9dd0cc3a2f381046c0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 22 Jan 2021 02:47:17 +0100 Subject: [PATCH 048/225] added full type hinting --- CHANGELOG.rst | 6 ++ README.rst | 50 +++++++++++++++- docs/changelog.rst | 7 --- docs/conf.py | 4 +- docs/index.rst | 1 - mypy.ini | 6 ++ portalocker/__init__.py | 12 ++-- portalocker/constants.py | 15 +++-- portalocker/exceptions.py | 4 +- portalocker/portalocker.py | 45 +++++++------- portalocker/utils.py | 116 +++++++++++++++++++++++-------------- pytest.ini | 1 + setup.py | 8 ++- 13 files changed, 188 insertions(+), 87 deletions(-) delete mode 100644 docs/changelog.rst create mode 100644 mypy.ini diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 203a2d2..d2a7ca5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,9 @@ +For newer changes please look at the comments for the Git tags: +https://github.com/WoLpH/portalocker/tags + +For more details the commit log for the master branch could be useful: +https://github.com/WoLpH/portalocker/commits/master + 1.5: * Moved tests to prevent collisions with other packages diff --git a/README.rst b/README.rst index 8b4ad39..4bcb119 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,18 @@ The module is currently maintained by Rick van Hattem . The project resides at https://github.com/WoLpH/portalocker . Bugs and feature requests can be submitted there. Patches are also very welcome. +Python 2 +-------- + +Python 2 was supported in versions before Portalocker 2.0. If you are still +using +Python 2, +you can run this to install: + +:: + + pip install "portalocker<2" + Tips ---- @@ -74,6 +86,38 @@ To make sure your cache generation scripts don't race, use the `Lock` class: To customize the opening and locking a manual approach is also possible: +>>> import portalocker +>>> file = open('somefile', 'r+') +>>> portalocker.lock(file, portalocker.EXCLUSIVE) +>>> file.seek(12) +>>> file.write('foo') +>>> file.close() + +Explicitly unlocking is not needed in most cases but omitting it has been known +to cause issues: + +>>> import portalocker +>>> with portalocker.Lock('somefile', timeout=1) as fh: +... print >>fh, 'writing some stuff to my cache...' + +To customize the opening and locking a manual approach is also possible: + +>>> import portalocker +>>> file = open('somefile', 'r+') +>>> portalocker.lock(file, portalocker.EXCLUSIVE) +>>> file.seek(12) +>>> file.write('foo') +>>> file.close() + +Explicitly unlocking is not needed in most cases but omitting it has been known +to cause issues: + +>>> import portalocker +>>> with portalocker.Lock('somefile', timeout=1) as fh: +... print >>fh, 'writing some stuff to my cache...' + +To customize the opening and locking a manual approach is also possible: + >>> import portalocker >>> file = open('somefile', 'r+') >>> portalocker.lock(file, portalocker.LOCK_EX) @@ -120,7 +164,11 @@ More examples can be found in the Changelog --------- -See the `changelog `_ page. +Every realease has a ``git tag`` with a commit message for the tag +explaining what was added +and/or changed. The list of tags including the commit messages can be found +here: https://github.com/WoLpH/portalocker/tags + License ------- diff --git a/docs/changelog.rst b/docs/changelog.rst deleted file mode 100644 index 21263ed..0000000 --- a/docs/changelog.rst +++ /dev/null @@ -1,7 +0,0 @@ -Changelog -========= - -For a more detailed overview of the changelog please view the Git history - -.. include :: ../CHANGELOG.rst - diff --git a/docs/conf.py b/docs/conf.py index eaccf38..ce2e742 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -196,7 +196,7 @@ # -- Options for LaTeX output --------------------------------------------- -latex_elements = { +# latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', @@ -205,7 +205,7 @@ # Additional stuff for the LaTeX preamble. #'preamble': '', -} +# } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). diff --git a/docs/index.rst b/docs/index.rst index f41abbd..48d2768 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,7 +10,6 @@ Contents: portalocker tests - changelog license Indices and tables diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..30861cf --- /dev/null +++ b/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +warn_return_any = True +warn_unused_configs = True +files = portalocker + +ignore_missing_imports = True diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 835dc95..9b0cbd1 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -32,18 +32,21 @@ #: Place an exclusive lock. #: Only one process may hold an exclusive lock for a given file at a given #: time. -LOCK_EX = constants.LOCK_EX +LOCK_EX = constants.LockFlags.EXCLUSIVE #: Place a shared lock. #: More than one process may hold a shared lock for a given file at a given #: time. -LOCK_SH = constants.LOCK_SH +LOCK_SH = constants.LockFlags.SHARED #: Acquire the lock in a non-blocking fashion. -LOCK_NB = constants.LOCK_NB +LOCK_NB = constants.LockFlags.NON_BLOCKING #: Remove an existing lock held by this process. -LOCK_UN = constants.LOCK_UN +LOCK_UN = constants.LockFlags.UNBLOCK + +#: Locking flags enum +LockFlags = constants.LockFlags #: Locking utility class to automatically handle opening with timeouts and #: context wrappers @@ -60,6 +63,7 @@ 'LOCK_SH', 'LOCK_NB', 'LOCK_UN', + 'LockFlags', 'LockException', 'Lock', 'RLock', diff --git a/portalocker/constants.py b/portalocker/constants.py index fb0927e..2c13ece 100644 --- a/portalocker/constants.py +++ b/portalocker/constants.py @@ -3,17 +3,18 @@ Lock types: -- `LOCK_EX` exclusive lock -- `LOCK_SH` shared lock +- `EXCLUSIVE` exclusive lock +- `SHARED` shared lock Lock flags: -- `LOCK_NB` non-blocking +- `NON_BLOCKING` non-blocking Manually unlock, only needed internally -- `LOCK_UN` unlock +- `UNBLOCK` unlock ''' +import enum import os # The actual tests will execute the code anyhow so the following code can @@ -37,3 +38,9 @@ else: # pragma: no cover raise RuntimeError('PortaLocker only defined for nt and posix platforms') + +class LockFlags(enum.IntFlag): + EXCLUSIVE = LOCK_EX #: exclusive lock + SHARED = LOCK_SH #: shared lock + NON_BLOCKING = LOCK_NB #: non-blocking + UNBLOCK = LOCK_UN #: unlock diff --git a/portalocker/exceptions.py b/portalocker/exceptions.py index bb2b35e..0a815b9 100644 --- a/portalocker/exceptions.py +++ b/portalocker/exceptions.py @@ -2,8 +2,8 @@ class BaseLockException(Exception): # Error codes: LOCK_FAILED = 1 - def __init__(self, *args, **kwargs): - self.fh = kwargs.pop('fh', None) + def __init__(self, *args, fh=None, **kwargs): + self.fh = fh Exception.__init__(self, *args, **kwargs) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 6c8eb67..6bc7447 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -1,8 +1,10 @@ import os import sys -from . import exceptions -from . import constants +import typing + +from . import constants +from . import exceptions if os.name == 'nt': # pragma: no cover import win32con @@ -17,16 +19,16 @@ else: lock_length = int(2**31 - 1) - def lock(file_, flags): - if flags & constants.LOCK_SH: + def lock(file_: typing.IO, flags: constants.LockFlags): + if flags & constants.LockFlags.SHARED: if sys.version_info.major == 2: - if flags & constants.LOCK_NB: + if flags & constants.LockFlags.NON_BLOCKING: mode = win32con.LOCKFILE_FAIL_IMMEDIATELY else: mode = 0 else: - if flags & constants.LOCK_NB: + if flags & constants.LockFlags.NON_BLOCKING: mode = msvcrt.LK_NBRLCK else: mode = msvcrt.LK_RLCK @@ -48,7 +50,7 @@ def lock(file_, flags): # here? raise else: - if flags & constants.LOCK_NB: + if flags & constants.LockFlags.NON_BLOCKING: mode = msvcrt.LK_NBLCK else: mode = msvcrt.LK_LOCK @@ -81,22 +83,25 @@ def lock(file_, flags): exceptions.LockException.LOCK_FAILED, exc_value.strerror, fh=file_) - def unlock(file_): + def unlock(file_: typing.IO): try: savepos = file_.tell() if savepos: file_.seek(0) try: - msvcrt.locking(file_.fileno(), constants.LOCK_UN, lock_length) - except IOError as exc_value: - if exc_value.strerror == 'Permission denied': + msvcrt.locking(file_.fileno(), constants.LockFlags.UNBLOCK, + lock_length) + except IOError as exc: + exception = exc + if exc.strerror == 'Permission denied': hfile = win32file._get_osfhandle(file_.fileno()) try: win32file.UnlockFileEx( hfile, 0, -0x10000, __overlapped) - except pywintypes.error as exc_value: - if exc_value.winerror == winerror.ERROR_NOT_LOCKED: + except pywintypes.error as exc: + exception = exc + if exc.winerror == winerror.ERROR_NOT_LOCKED: # error: (158, 'UnlockFileEx', # 'The segment is already unlocked.') # To match the 'posix' implementation, silently @@ -109,23 +114,23 @@ def unlock(file_): else: raise exceptions.LockException( exceptions.LockException.LOCK_FAILED, - exc_value.strerror, + exception.strerror, fh=file_) finally: if savepos: file_.seek(savepos) - except IOError as exc_value: + except IOError as exc: raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, exc_value.strerror, + exceptions.LockException.LOCK_FAILED, exc.strerror, fh=file_) elif os.name == 'posix': # pragma: no cover import fcntl - def lock(file_, flags): + def lock(file_: typing.IO, flags: constants.LockFlags): locking_exceptions = IOError, try: # pragma: no cover - locking_exceptions += BlockingIOError, + locking_exceptions += BlockingIOError, # type: ignore except NameError: # pragma: no cover pass @@ -136,8 +141,8 @@ def lock(file_, flags): # every IO error raise exceptions.LockException(exc_value, fh=file_) - def unlock(file_): - fcntl.flock(file_.fileno(), constants.LOCK_UN) + def unlock(file_: typing.IO, ): + fcntl.flock(file_.fileno(), constants.LockFlags.UNBLOCK) else: # pragma: no cover raise RuntimeError('PortaLocker only defined for nt and posix platforms') diff --git a/portalocker/utils.py b/portalocker/utils.py index 6868403..e9e32d5 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -1,29 +1,33 @@ -import os import abc -import time import atexit -import random +import contextlib +import os import pathlib +import random import tempfile -import contextlib -from . import exceptions +import time +import typing + from . import constants +from . import exceptions from . import portalocker current_time = getattr(time, "monotonic", time.time) DEFAULT_TIMEOUT = 5 DEFAULT_CHECK_INTERVAL = 0.25 -LOCK_METHOD = constants.LOCK_EX | constants.LOCK_NB +LOCK_METHOD = constants.LockFlags.EXCLUSIVE | constants.LockFlags.NON_BLOCKING __all__ = [ 'Lock', 'open_atomic', ] +Filename = typing.Union[str, pathlib.Path] + @contextlib.contextmanager -def open_atomic(filename, binary=True): +def open_atomic(filename: Filename, binary: bool = True): '''Open a file for atomic writing. Instead of locking this method allows you to write the entire file and move it to the actual location. Note that this makes the assumption that a rename is atomic on your platform which @@ -40,17 +44,25 @@ def open_atomic(filename, binary=True): >>> assert os.path.exists(filename) >>> os.remove(filename) + >>> import pathlib + >>> path_filename = pathlib.Path('test_file.txt') + + >>> with open_atomic(path_filename) as fh: + ... written = fh.write(b'test') + >>> assert path_filename.exists() + >>> path_filename.unlink() ''' - assert not os.path.exists(filename), '%r exists' % filename - path, name = os.path.split(filename) + # `pathlib.Path` cast in case `path` is a `str` + path: pathlib.Path = pathlib.Path(filename) + + assert not path.exists(), '%r exists' % path # Create the parent directory if it doesn't exist - if path and not os.path.isdir(path): # pragma: no cover - os.makedirs(path) + path.parent.mkdir(parents=True, exist_ok=True) temp_fh = tempfile.NamedTemporaryFile( mode=binary and 'wb' or 'w', - dir=path, + dir=str(path.parent), delete=False, ) yield temp_fh @@ -58,7 +70,7 @@ def open_atomic(filename, binary=True): os.fsync(temp_fh.fileno()) temp_fh.close() try: - os.rename(temp_fh.name, filename) + os.rename(temp_fh.name, path) finally: try: os.remove(temp_fh.name) @@ -70,7 +82,8 @@ class LockBase(abc.ABC): # pragma: no cover @abc.abstractmethod def acquire( - self, timeout=None, check_interval=None, fail_when_locked=None): + self, timeout: float = None, check_interval: float = None, + fail_when_locked: bool = None): return NotImplemented @abc.abstractmethod @@ -90,9 +103,13 @@ def __delete__(self, instance): class Lock(LockBase): def __init__( - self, filename, mode='a', timeout=DEFAULT_TIMEOUT, - check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, - flags=LOCK_METHOD, **file_open_kwargs): + self, + filename: Filename, + mode: str = 'a', + timeout: float = DEFAULT_TIMEOUT, + check_interval: float = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool = False, + flags: constants.LockFlags = LOCK_METHOD, **file_open_kwargs): '''Lock manager with build-in timeout filename -- filename @@ -119,18 +136,19 @@ def __init__( else: truncate = False - self.fh = None - self.filename = str(filename) - self.mode = mode - self.truncate = truncate - self.timeout = timeout - self.check_interval = check_interval - self.fail_when_locked = fail_when_locked - self.flags = flags + self.fh: typing.Optional[typing.IO] = None + self.filename: str = str(filename) + self.mode: str = mode + self.truncate: bool = truncate + self.timeout: float = timeout + self.check_interval: float = check_interval + self.fail_when_locked: bool = fail_when_locked + self.flags: constants.LockFlags = flags self.file_open_kwargs = file_open_kwargs def acquire( - self, timeout=None, check_interval=None, fail_when_locked=None): + self, timeout: float = None, check_interval: float = None, + fail_when_locked: bool = None) -> typing.IO: '''Acquire the locked filehandle''' if timeout is None: timeout = self.timeout @@ -198,11 +216,11 @@ def release(self): self.fh.close() self.fh = None - def _get_fh(self): + def _get_fh(self) -> typing.IO: '''Get a new filehandle''' return open(self.filename, self.mode, **self.file_open_kwargs) - def _get_lock(self, fh): + def _get_lock(self, fh: typing.IO) -> typing.IO: ''' Try to lock the given filehandle @@ -210,7 +228,7 @@ def _get_lock(self, fh): portalocker.lock(fh, self.flags) return fh - def _prepare_fh(self, fh): + def _prepare_fh(self, fh: typing.IO) -> typing.IO: ''' Prepare the filehandle for usage @@ -230,6 +248,7 @@ class RLock(Lock): can be acquired multiple times. When the corresponding number of release() calls are made the lock will finally release the underlying file lock. ''' + def __init__( self, filename, mode='a', timeout=DEFAULT_TIMEOUT, check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, @@ -239,13 +258,15 @@ def __init__( self._acquire_count = 0 def acquire( - self, timeout=None, check_interval=None, fail_when_locked=None): + self, timeout: float = None, check_interval: float = None, + fail_when_locked: bool = None) -> typing.IO: if self._acquire_count >= 1: fh = self.fh else: fh = super(RLock, self).acquire(timeout, check_interval, fail_when_locked) self._acquire_count += 1 + assert fh return fh def release(self): @@ -263,7 +284,6 @@ class TemporaryFileLock(Lock): def __init__(self, filename='.lock', timeout=DEFAULT_TIMEOUT, check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=True, flags=LOCK_METHOD): - Lock.__init__(self, filename=filename, mode='w', timeout=timeout, check_interval=check_interval, fail_when_locked=fail_when_locked, flags=flags) @@ -290,35 +310,43 @@ class BoundedSemaphore(LockBase): >>> str(sorted(semaphore.get_random_filenames())[1]) 'bounded_semaphore.01.lock' ''' + lock: typing.Optional[Lock] - def __init__(self, maximum: int, name: str = 'bounded_semaphore', - filename_pattern: str = '{name}.{number:02d}.lock', directory: - str = tempfile.gettempdir(), timeout=DEFAULT_TIMEOUT, - check_interval=DEFAULT_CHECK_INTERVAL): + def __init__( + self, + maximum: int, + name: str = 'bounded_semaphore', + filename_pattern: str = '{name}.{number:02d}.lock', + directory: str = tempfile.gettempdir(), + timeout=DEFAULT_TIMEOUT, + check_interval=DEFAULT_CHECK_INTERVAL): self.maximum = maximum self.name = name self.filename_pattern = filename_pattern self.directory = directory - self.lock = None + self.lock: typing.Optional[Lock] = None self.timeout = timeout self.check_interval = check_interval - def get_filenames(self): + def get_filenames(self) -> typing.Sequence[pathlib.Path]: return [self.get_filename(n) for n in range(self.maximum)] - def get_random_filenames(self): - filenames = self.get_filenames() + def get_random_filenames(self) -> typing.Sequence[pathlib.Path]: + filenames = list(self.get_filenames()) random.shuffle(filenames) return filenames - def get_filename(self, number): + def get_filename(self, number) -> pathlib.Path: return pathlib.Path(self.directory) / self.filename_pattern.format( name=self.name, number=number, ) def acquire( - self, timeout=None, check_interval=None): + self, + timeout: float = None, + check_interval: float = None, + fail_when_locked: bool = None) -> typing.Optional[Lock]: assert not self.lock, 'Already locked' if timeout is None: @@ -346,7 +374,8 @@ def acquire( raise exceptions.AlreadyLocked() - def try_lock(self, filenames): + def try_lock(self, filenames: typing.Sequence[Filename]) -> bool: + filename: Filename for filename in filenames: self.lock = Lock(filename, fail_when_locked=True) try: @@ -355,7 +384,8 @@ def try_lock(self, filenames): except exceptions.AlreadyLocked: pass + return False + def release(self): # pragma: no cover self.lock.release() self.lock = None - diff --git a/pytest.ini b/pytest.ini index 9542cd2..a02e3ec 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,6 +11,7 @@ addopts = --cov-report html --no-cov-on-fail --flake8 + --mypy flake8-ignore = *.py W391 diff --git a/setup.py b/setup.py index 7679cf7..3263046 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ import os import sys import setuptools +import typing from setuptools.command.test import test as TestCommand from distutils.version import LooseVersion from setuptools import __version__ as setuptools_version @@ -18,16 +19,17 @@ # To prevent importing about and thereby breaking the coverage info we use this # exec hack -about = {} +about: typing.Dict[str, str] = {} with open('portalocker/__about__.py') as fp: exec(fp.read(), about) tests_require = [ - 'pytest>=4.6.9', + 'pytest>=5.4.1', 'pytest-cov>=2.8.1', - 'sphinx>=1.8.5', + 'sphinx>=3.0.3', 'pytest-flake8>=1.0.5', + 'pytest-mypy>=0.8.0', ] From 50151b89053d5f2233d01c160e9b6aabbfb38be1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 03:49:41 +0100 Subject: [PATCH 049/225] added redis locking --- .travis.yml | 2 + portalocker/redis.py | 68 +++++++++++++++++++++++++++++++++ portalocker_tests/test_redis.py | 61 +++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 portalocker/redis.py create mode 100644 portalocker_tests/test_redis.py diff --git a/.travis.yml b/.travis.yml index 759677b..9c58543 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ sudo: false dist: xenial language: python +services: +- redis python: - '3.5' - '3.6' diff --git a/portalocker/redis.py b/portalocker/redis.py new file mode 100644 index 0000000..50dc35e --- /dev/null +++ b/portalocker/redis.py @@ -0,0 +1,68 @@ +import typing + +from redis import client + +from . import exceptions +from . import utils + + +class RedisLock(utils.LockBase): + channel: str + timeout: float + connection: typing.Optional[client.Redis] + pubsub: typing.Optional[client.PubSub] = None + close_connection: bool + + def __init__( + self, + channel: str, + connection: typing.Optional[client.Redis] = None, + timeout: typing.Optional[float] = None, + check_interval: typing.Optional[float] = None, + fail_when_locked: typing.Optional[bool] = False, + ): + # We don't want to close connections given as an argument + self.close_connection = not connection + + self.channel = channel + self.connection = connection + + super(RedisLock, self).__init__(timeout=timeout, + check_interval=check_interval, + fail_when_locked=fail_when_locked) + + def get_connection(self) -> client.Redis: + if not self.connection: + self.connection = client.Redis() + + return self.connection + + def acquire( + self, timeout: float = None, check_interval: float = None, + fail_when_locked: typing.Optional[bool] = True): + + assert not self.pubsub, 'This lock is already active' + connection = self.get_connection() + + timeout_generator = self._timeout_generator(timeout, check_interval) + for _ in timeout_generator: # pragma: no branch + subscribers = connection.pubsub_numsub(self.channel)[0][1] + + if not subscribers: + self.pubsub = connection.pubsub() + self.pubsub.subscribe(self.channel) + + subscribers = connection.pubsub_numsub(self.channel)[0][1] + if subscribers == 1: # pragma: no branch + return self + else: # pragma: no cover + # Race condition, let's try again + self.release() + + if fail_when_locked: # pragma: no branch + raise exceptions.AlreadyLocked(exceptions) + + def release(self): + if self.pubsub: + self.pubsub.unsubscribe(self.channel) + self.pubsub = None diff --git a/portalocker_tests/test_redis.py b/portalocker_tests/test_redis.py new file mode 100644 index 0000000..210a431 --- /dev/null +++ b/portalocker_tests/test_redis.py @@ -0,0 +1,61 @@ +import random + +import pytest +from redis import client +from redis import exceptions + +import portalocker +from portalocker import redis + + +@pytest.mark.xfail(raises=exceptions.ConnectionError) +def test_redis_lock(): + channel = str(random.random()) + + lock_a = redis.RedisLock(channel) + lock_a.acquire(fail_when_locked=True) + + lock_b = redis.RedisLock(channel) + with pytest.raises(portalocker.AlreadyLocked): + lock_b.acquire(fail_when_locked=True) + + +@pytest.mark.parametrize('timeout', [None, 0, 0.001]) +@pytest.mark.parametrize('check_interval', [None, 0, 0.0005]) +def test_redis_lock_timeout(timeout, check_interval): + connection = client.Redis() + channel = str(random.random()) + lock_a = redis.RedisLock(channel) + lock_a.acquire(timeout=timeout, check_interval=check_interval) + + lock_b = redis.RedisLock(channel, connection=connection) + with pytest.raises(portalocker.AlreadyLocked): + lock_b.acquire(timeout=timeout, check_interval=check_interval) + + +@pytest.mark.xfail(raises=exceptions.ConnectionError) +def test_redis_lock_context(): + channel = str(random.random()) + + lock_a = redis.RedisLock(channel, fail_when_locked=True) + with lock_a: + lock_b = redis.RedisLock(channel, fail_when_locked=True) + with pytest.raises(portalocker.AlreadyLocked): + with lock_b: + pass + + +@pytest.mark.xfail(raises=exceptions.ConnectionError) +def test_redis_relock(): + channel = str(random.random()) + + lock_a = redis.RedisLock(channel, fail_when_locked=True) + with lock_a: + with pytest.raises(AssertionError): + lock_a.acquire() + + lock_a.release() + + +if __name__ == '__main__': + test_redis_lock() From 364f580fa63dbcba09703b259dc66e4869b3d0ab Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 03:52:34 +0100 Subject: [PATCH 050/225] refactored and modernized code --- mypy.ini | 1 + portalocker/__init__.py | 15 ++++-- portalocker/utils.py | 78 ++++++++++++++++------------- portalocker_tests/test_semaphore.py | 1 + setup.py | 15 +++--- 5 files changed, 65 insertions(+), 45 deletions(-) diff --git a/mypy.ini b/mypy.ini index 30861cf..b66f60c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,3 +4,4 @@ warn_unused_configs = True files = portalocker ignore_missing_imports = True + diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 9b0cbd1..dc4e361 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -4,6 +4,12 @@ from . import portalocker from . import utils +try: + from .redis import RedisLock +except ImportError: + RedisLock = None + + #: The package name on Pypi __package_name__ = __about__.__package_name__ #: Current author and maintainer, view the git history for the previous ones @@ -32,18 +38,18 @@ #: Place an exclusive lock. #: Only one process may hold an exclusive lock for a given file at a given #: time. -LOCK_EX = constants.LockFlags.EXCLUSIVE +LOCK_EX: constants.LockFlags = constants.LockFlags.EXCLUSIVE #: Place a shared lock. #: More than one process may hold a shared lock for a given file at a given #: time. -LOCK_SH = constants.LockFlags.SHARED +LOCK_SH: constants.LockFlags = constants.LockFlags.SHARED #: Acquire the lock in a non-blocking fashion. -LOCK_NB = constants.LockFlags.NON_BLOCKING +LOCK_NB: constants.LockFlags = constants.LockFlags.NON_BLOCKING #: Remove an existing lock held by this process. -LOCK_UN = constants.LockFlags.UNBLOCK +LOCK_UN: constants.LockFlags = constants.LockFlags.UNBLOCK #: Locking flags enum LockFlags = constants.LockFlags @@ -70,5 +76,6 @@ 'AlreadyLocked', 'BoundedSemaphore', 'open_atomic', + 'RedisLock', ] diff --git a/portalocker/utils.py b/portalocker/utils.py index e9e32d5..53d613d 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -12,10 +12,9 @@ from . import exceptions from . import portalocker -current_time = getattr(time, "monotonic", time.time) - DEFAULT_TIMEOUT = 5 DEFAULT_CHECK_INTERVAL = 0.25 +DEFAULT_FAIL_WHEN_LOCKED = False LOCK_METHOD = constants.LockFlags.EXCLUSIVE | constants.LockFlags.NON_BLOCKING __all__ = [ @@ -79,6 +78,20 @@ def open_atomic(filename: Filename, binary: bool = True): class LockBase(abc.ABC): # pragma: no cover + #: timeout when trying to acquire a lock + timeout: typing.Optional[float] = DEFAULT_TIMEOUT + #: check interval while waiting for `timeout` + check_interval: typing.Optional[float] = DEFAULT_CHECK_INTERVAL + #: skip the timeout and immediately fail if the initial lock fails + fail_when_locked: typing.Optional[bool] = DEFAULT_FAIL_WHEN_LOCKED + + def __init__(self, timeout: typing.Optional[float] = None, + check_interval: typing.Optional[float] = None, + fail_when_locked: typing.Optional[bool] = None): + if timeout is not None: + self.timeout = timeout + if check_interval is not None: + self.check_interval: float = check_interval @abc.abstractmethod def acquire( @@ -86,6 +99,23 @@ def acquire( fail_when_locked: bool = None): return NotImplemented + def _timeout_generator(self, timeout, check_interval): + if timeout is None: + timeout = self.timeout + + if timeout is None: + timeout = 0 + + if check_interval is None: + check_interval = self.check_interval + + yield + + timeout_end = time.perf_counter() + timeout + while timeout_end > time.perf_counter(): + yield + time.sleep(check_interval) + @abc.abstractmethod def release(self): return NotImplemented @@ -108,7 +138,7 @@ def __init__( mode: str = 'a', timeout: float = DEFAULT_TIMEOUT, check_interval: float = DEFAULT_CHECK_INTERVAL, - fail_when_locked: bool = False, + fail_when_locked: bool = DEFAULT_FAIL_WHEN_LOCKED, flags: constants.LockFlags = LOCK_METHOD, **file_open_kwargs): '''Lock manager with build-in timeout @@ -119,7 +149,7 @@ def __init__( timeout -- timeout when trying to acquire a lock check_interval -- check interval while waiting fail_when_locked -- after the initial lock failed, return an error - or lock the file + or lock the file. This does not wait for the timeout. **file_open_kwargs -- The kwargs for the `open(...)` call fail_when_locked is useful when multiple threads/processes can race @@ -150,13 +180,6 @@ def acquire( self, timeout: float = None, check_interval: float = None, fail_when_locked: bool = None) -> typing.IO: '''Acquire the locked filehandle''' - if timeout is None: - timeout = self.timeout - if timeout is None: - timeout = 0 - - if check_interval is None: - check_interval = self.check_interval if fail_when_locked is None: fail_when_locked = self.fail_when_locked @@ -176,10 +199,10 @@ def try_close(): # pragma: no cover except Exception: pass - # Try till the timeout has passed - timeout_end = current_time() + timeout exception = None - while timeout_end > current_time(): + # Try till the timeout has passed + for _ in self._timeout_generator(timeout, check_interval): + exception = None try: # Try to lock fh = self._get_lock(fh) @@ -196,9 +219,8 @@ def try_close(): # pragma: no cover raise exceptions.AlreadyLocked(exception) # Wait a bit - time.sleep(check_interval) - else: + if exception: try_close() # We got a timeout... reraising raise exceptions.LockException(exception) @@ -349,37 +371,25 @@ def acquire( fail_when_locked: bool = None) -> typing.Optional[Lock]: assert not self.lock, 'Already locked' - if timeout is None: - timeout = self.timeout - if timeout is None: - timeout = 0 - - if check_interval is None: - check_interval = self.check_interval - filenames = self.get_filenames() + print('filenames', filenames) - if self.try_lock(filenames): - return self.lock - - if not timeout: - raise exceptions.AlreadyLocked() - - timeout_end = current_time() + timeout - while timeout_end > current_time(): # pragma: no branch + for _ in self._timeout_generator(timeout, check_interval): # pragma: + print('trying lock', filenames) + # no branch if self.try_lock(filenames): # pragma: no branch return self.lock # pragma: no cover - time.sleep(check_interval) - raise exceptions.AlreadyLocked() def try_lock(self, filenames: typing.Sequence[Filename]) -> bool: filename: Filename for filename in filenames: + print('trying lock for', filename) self.lock = Lock(filename, fail_when_locked=True) try: self.lock.acquire() + print('locked', filename) return True except exceptions.AlreadyLocked: pass diff --git a/portalocker_tests/test_semaphore.py b/portalocker_tests/test_semaphore.py index 9dfc9dc..b0c57aa 100644 --- a/portalocker_tests/test_semaphore.py +++ b/portalocker_tests/test_semaphore.py @@ -11,6 +11,7 @@ def test_bounded_semaphore(timeout, check_interval, monkeypatch): name = random.random() monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) + semaphore_a = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) semaphore_b = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) semaphore_c = portalocker.BoundedSemaphore(n, name=name, timeout=timeout) diff --git a/setup.py b/setup.py index 3263046..28b0be2 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,14 @@ from __future__ import print_function -import re + import os +import re import sys -import setuptools import typing -from setuptools.command.test import test as TestCommand from distutils.version import LooseVersion -from setuptools import __version__ as setuptools_version +import setuptools +from setuptools import __version__ as setuptools_version +from setuptools.command.test import test as TestCommand if LooseVersion(setuptools_version) < LooseVersion('38.3.0'): raise SystemExit( @@ -16,14 +17,12 @@ 'and try again.' ) - # To prevent importing about and thereby breaking the coverage info we use this # exec hack about: typing.Dict[str, str] = {} with open('portalocker/__about__.py') as fp: exec(fp.read(), about) - tests_require = [ 'pytest>=5.4.1', 'pytest-cov>=2.8.1', @@ -143,6 +142,8 @@ def run(self): 'sphinx>=1.7.1', ], tests=tests_require, + redis=[ + 'redis', + ] ), ) - From 125b760cc8963de5d076aee8467e56ad7d916bb4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 04:18:09 +0100 Subject: [PATCH 051/225] appveyor fix? --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index cbd6cc8..908c201 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,8 +10,8 @@ environment: - TOXENV: py39 install: - - pip install -U tox setuptools wheel - - pip install -Ue .[tests] + - pip3 install -U tox setuptools wheel + - pip3 install -Ue .[tests] build: off # Not a C# project, build stuff at the test step instead. @@ -19,7 +19,7 @@ test_script: - tox after_test: - - python setup.py sdist bdist_wheel + - python3 setup.py sdist bdist_wheel - ps: "ls dist" artifacts: From 76aff98d134bf70261f639105ef07ba43b1860af Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 04:18:25 +0100 Subject: [PATCH 052/225] enabled redis tests --- .travis.yml | 2 +- docs/portalocker.redis.rst | 7 +++++++ docs/portalocker.rst | 1 + portalocker/__init__.py | 6 +++--- portalocker_tests/mypy.ini | 5 +++++ pytest.ini | 1 - setup.py | 1 + tox.ini | 2 +- 8 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 docs/portalocker.redis.rst create mode 100644 portalocker_tests/mypy.ini diff --git a/.travis.yml b/.travis.yml index 9c58543..93d346c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ before_install: install: - pip install -U setuptools wheel pip - pip install -r portalocker_tests/requirements.txt -- pip install -e . +- pip install -e '.[redis]' - pip install coveralls script: - python setup.py test diff --git a/docs/portalocker.redis.rst b/docs/portalocker.redis.rst new file mode 100644 index 0000000..4406655 --- /dev/null +++ b/docs/portalocker.redis.rst @@ -0,0 +1,7 @@ +portalocker.redis module +======================== + +.. automodule:: portalocker.redis + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/portalocker.rst b/docs/portalocker.rst index ce88674..9050d7a 100644 --- a/docs/portalocker.rst +++ b/docs/portalocker.rst @@ -6,6 +6,7 @@ Submodules .. toctree:: + portalocker.redis portalocker.constants portalocker.exceptions portalocker.portalocker diff --git a/portalocker/__init__.py b/portalocker/__init__.py index dc4e361..0ec0292 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -4,10 +4,10 @@ from . import portalocker from . import utils -try: +try: # pragma: no cover from .redis import RedisLock -except ImportError: - RedisLock = None +except ImportError: # pragma: no cover + RedisLock = None # type: ignore #: The package name on Pypi diff --git a/portalocker_tests/mypy.ini b/portalocker_tests/mypy.ini new file mode 100644 index 0000000..2f91b47 --- /dev/null +++ b/portalocker_tests/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +warn_return_any = True +warn_unused_configs = True + +ignore_missing_imports = True diff --git a/pytest.ini b/pytest.ini index a02e3ec..9542cd2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,7 +11,6 @@ addopts = --cov-report html --no-cov-on-fail --flake8 - --mypy flake8-ignore = *.py W391 diff --git a/setup.py b/setup.py index 28b0be2..4365703 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ 'sphinx>=3.0.3', 'pytest-flake8>=1.0.5', 'pytest-mypy>=0.8.0', + 'redis', ] diff --git a/tox.ini b/tox.ini index 33de79b..e92ee70 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ basepython = py39: python3.9 pypy3: pypy3 -deps = -e{toxinidir}[tests] +deps = -e{toxinidir}[tests,redis] commands = python -m pytest {posargs} [testenv:flake8] From a7d12c5ceb95352e93c6bdc9269d1abbb0fc42fb Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 04:31:47 +0100 Subject: [PATCH 053/225] added redis docs --- README.rst | 29 +++++++++++++++++++++++++++++ docs/conf.py | 1 + portalocker/redis.py | 20 ++++++++++++++++++++ portalocker/utils.py | 40 ++++++++++++++++++++-------------------- 4 files changed, 70 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index 4bcb119..aab394b 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,35 @@ The module is currently maintained by Rick van Hattem . The project resides at https://github.com/WoLpH/portalocker . Bugs and feature requests can be submitted there. Patches are also very welcome. +Redis Locks +----------- + +This library now features a lock based on Redis which allows for locks across +multiple threads, processes and even distributed locks across multiple +computers. + +It is an extremely reliable Redis lock that is based on pubsub. + +As opposed to most Redis locking systems based on key/value pairs, +this locking method is based on the pubsub system. The big advantage is +that if the connection gets killed due to network issues, crashing +processes or otherwise, it will still immediately unlock instead of +waiting for a lock timeout. + +Usage is really easy: + +:: + + import portalocker + + lock = portalocker.RedisLock('some_lock_channel_name') + + with lock: + print('do something here') + +The API is essentially identical to the other ``Lock`` classes so in addition +to the ``with`` statement you can also use ``lock.acquire(...)``. + Python 2 -------- diff --git a/docs/conf.py b/docs/conf.py index ce2e742..5680b57 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,6 +40,7 @@ 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon', ] # Add any paths that contain templates here, relative to this directory. diff --git a/portalocker/redis.py b/portalocker/redis.py index 50dc35e..71ab09e 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -7,6 +7,26 @@ class RedisLock(utils.LockBase): + ''' + An extremely reliable Redis lock based on pubsub + + As opposed to most Redis locking systems based on key/value pairs, + this locking method is based on the pubsub system. The big advantage is + that if the connection gets killed due to network issues, crashing + processes or otherwise, it will still immediately unlock instead of + waiting for a lock timeout. + + Args: + channel: the redis channel to use as locking key. + connection: an optional redis connection if you already have one + or if you need to specify the redis connection + timeout: timeout when trying to acquire a lock + check_interval: check interval while waiting + fail_when_locked: after the initial lock failed, return an error + or lock the file. This does not wait for the timeout. + + ''' + channel: str timeout: float connection: typing.Optional[client.Redis] diff --git a/portalocker/utils.py b/portalocker/utils.py index 53d613d..9f253b5 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -131,6 +131,26 @@ def __delete__(self, instance): class Lock(LockBase): + '''Lock manager with build-in timeout + + Args: + filename: filename + mode: the open mode, 'a' or 'ab' should be used for writing + truncate: use truncate to emulate 'w' mode, None is disabled, 0 is + truncate to 0 bytes + timeout: timeout when trying to acquire a lock + check_interval: check interval while waiting + fail_when_locked: after the initial lock failed, return an error + or lock the file. This does not wait for the timeout. + **file_open_kwargs: The kwargs for the `open(...)` call + + fail_when_locked is useful when multiple threads/processes can race + when creating a file. If set to true than the system will wait till + the lock was acquired and then return an AlreadyLocked exception. + + Note that the file is opened first and locked later. So using 'w' as + mode will result in truncate _BEFORE_ the lock is checked. + ''' def __init__( self, @@ -140,26 +160,6 @@ def __init__( check_interval: float = DEFAULT_CHECK_INTERVAL, fail_when_locked: bool = DEFAULT_FAIL_WHEN_LOCKED, flags: constants.LockFlags = LOCK_METHOD, **file_open_kwargs): - '''Lock manager with build-in timeout - - filename -- filename - mode -- the open mode, 'a' or 'ab' should be used for writing - truncate -- use truncate to emulate 'w' mode, None is disabled, 0 is - truncate to 0 bytes - timeout -- timeout when trying to acquire a lock - check_interval -- check interval while waiting - fail_when_locked -- after the initial lock failed, return an error - or lock the file. This does not wait for the timeout. - **file_open_kwargs -- The kwargs for the `open(...)` call - - fail_when_locked is useful when multiple threads/processes can race - when creating a file. If set to true than the system will wait till - the lock was acquired and then return an AlreadyLocked exception. - - Note that the file is opened first and locked later. So using 'w' as - mode will result in truncate _BEFORE_ the lock is checked. - ''' - if 'w' in mode: truncate = True mode = mode.replace('w', 'a') From 10d5df0618865f905066e9d5a9bb7f22bd227b04 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 14:53:34 +0100 Subject: [PATCH 054/225] updated python versions --- .travis.yml | 4 +--- appveyor.yml | 5 ++--- tox.ini | 4 ++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 93d346c..a2eabc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,10 @@ language: python services: - redis python: -- '3.5' - '3.6' - '3.7' - '3.8' -# TODO: Enable when available -# - '3.9' +- '3.9' - pypy3 before_install: - wheel version diff --git a/appveyor.yml b/appveyor.yml index 908c201..6a06cfa 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,15 +3,14 @@ environment: matrix: - - TOXENV: py35 - TOXENV: py36 - TOXENV: py37 - TOXENV: py38 - TOXENV: py39 install: - - pip3 install -U tox setuptools wheel - - pip3 install -Ue .[tests] + - python3 -m pip install -U tox setuptools wheel + - python3 -m pip install -Ue .[tests] build: off # Not a C# project, build stuff at the test step instead. diff --git a/tox.ini b/tox.ini index e92ee70..0274dfb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,14 @@ [tox] -envlist = py35, py36, py37, py38, py39, pypy3, flake8, docs +envlist = py36, py37, py38, py39, py310, pypy3, flake8, docs skip_missing_interpreters = True [testenv] basepython = - py35: python3.5 py36: python3.6 py37: python3.7 py38: python3.8 py39: python3.9 + py310: python3.10 pypy3: pypy3 deps = -e{toxinidir}[tests,redis] From 051190b857dd3a70d3bd31dbae4e52201289f949 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 15:09:25 +0100 Subject: [PATCH 055/225] attempt to fix appveyor --- appveyor.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 6a06cfa..dc41c2f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,8 @@ # http://www.appveyor.com/docs/installed-software#python environment: + global: + PYTHON: "C:\\Python39-x64" matrix: - TOXENV: py36 - TOXENV: py37 @@ -9,8 +11,8 @@ environment: - TOXENV: py39 install: - - python3 -m pip install -U tox setuptools wheel - - python3 -m pip install -Ue .[tests] + - "%PYTHON%\\python -m pip install -U tox setuptools wheel" + - "%PYTHON%\\python -m pip install -Ue .[tests]" build: off # Not a C# project, build stuff at the test step instead. @@ -18,7 +20,7 @@ test_script: - tox after_test: - - python3 setup.py sdist bdist_wheel + - "%PYTHON%\\python setup.py sdist bdist_wheel" - ps: "ls dist" artifacts: From ee3d354dcaa50c0f9e6e1438bd3c05e542780734 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 15:12:19 +0100 Subject: [PATCH 056/225] attempting travis fix --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index a2eabc0..698e466 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,14 @@ sudo: false dist: xenial language: python + services: - redis + +jobs: + allow_failures: + - python: pypy3 + python: - '3.6' - '3.7' From a18add3d25712bdca32f610711aa5aca5cb38c7f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 15:13:45 +0100 Subject: [PATCH 057/225] attempt to fix appveyor --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index dc41c2f..28b69c5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,8 +11,8 @@ environment: - TOXENV: py39 install: - - "%PYTHON%\\python -m pip install -U tox setuptools wheel" - - "%PYTHON%\\python -m pip install -Ue .[tests]" + - "%PYTHON%\\python.exe -m pip install -U tox setuptools wheel" + - "%PYTHON%\\python.exe -m pip install -Ue .[tests]" build: off # Not a C# project, build stuff at the test step instead. @@ -20,7 +20,7 @@ test_script: - tox after_test: - - "%PYTHON%\\python setup.py sdist bdist_wheel" + - "%PYTHON%\\python.exe setup.py sdist bdist_wheel" - ps: "ls dist" artifacts: From d0eda361e3b026b1998593d41302aecd54fa7b3e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 15:17:19 +0100 Subject: [PATCH 058/225] attempt to fix appveyor --- appveyor.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 28b69c5..579769f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,8 +2,6 @@ # http://www.appveyor.com/docs/installed-software#python environment: - global: - PYTHON: "C:\\Python39-x64" matrix: - TOXENV: py36 - TOXENV: py37 @@ -12,7 +10,7 @@ environment: install: - "%PYTHON%\\python.exe -m pip install -U tox setuptools wheel" - - "%PYTHON%\\python.exe -m pip install -Ue .[tests]" + - "C:\\Python39-x64\\python.exe -m pip install -Ue .[tests]" build: off # Not a C# project, build stuff at the test step instead. @@ -20,7 +18,7 @@ test_script: - tox after_test: - - "%PYTHON%\\python.exe setup.py sdist bdist_wheel" + - "C:\\Python39-x64\\python.exe setup.py sdist bdist_wheel" - ps: "ls dist" artifacts: From f9dc18f6f65e5c394d87ae66b9179dfe05820965 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 15:18:02 +0100 Subject: [PATCH 059/225] attempt to fix appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 579769f..a35bced 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,7 +9,7 @@ environment: - TOXENV: py39 install: - - "%PYTHON%\\python.exe -m pip install -U tox setuptools wheel" + - "python -m pip install -U tox setuptools wheel" - "C:\\Python39-x64\\python.exe -m pip install -Ue .[tests]" build: off # Not a C# project, build stuff at the test step instead. From 38e59d707ca14251575875fac4e57e525d42c03f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 15:20:37 +0100 Subject: [PATCH 060/225] attempt to fix appveyor --- appveyor.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index a35bced..cf6e81e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,8 @@ # http://www.appveyor.com/docs/installed-software#python environment: + global: + PYTHON: "C:\\Python38-x64" matrix: - TOXENV: py36 - TOXENV: py37 @@ -9,8 +11,8 @@ environment: - TOXENV: py39 install: - - "python -m pip install -U tox setuptools wheel" - - "C:\\Python39-x64\\python.exe -m pip install -Ue .[tests]" + - "%PYTHON%\\python.exe -m pip install -U tox setuptools wheel" + - "%PYTHON%\\python.exe -m pip install -Ue .[tests]" build: off # Not a C# project, build stuff at the test step instead. @@ -18,7 +20,7 @@ test_script: - tox after_test: - - "C:\\Python39-x64\\python.exe setup.py sdist bdist_wheel" + - "%PYTHON%\\python.exe setup.py sdist bdist_wheel" - ps: "ls dist" artifacts: From 0d876ddf15c691dc07812a2cf0f3371ebf8a392d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 15:23:06 +0100 Subject: [PATCH 061/225] attempt to fix appveyor --- appveyor.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index cf6e81e..efa8003 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,7 +3,7 @@ environment: global: - PYTHON: "C:\\Python38-x64" + PYTHON: "C:\\Python39-x64\\python.exe" matrix: - TOXENV: py36 - TOXENV: py37 @@ -11,16 +11,16 @@ environment: - TOXENV: py39 install: - - "%PYTHON%\\python.exe -m pip install -U tox setuptools wheel" - - "%PYTHON%\\python.exe -m pip install -Ue .[tests]" + - "%PYTHON% -m pip install -U tox setuptools wheel" + - "%PYTHON% -m pip install -Ue .[tests]" build: off # Not a C# project, build stuff at the test step instead. test_script: - - tox + - "%PYTHON% -m pip tox" after_test: - - "%PYTHON%\\python.exe setup.py sdist bdist_wheel" + - "%PYTHON% setup.py sdist bdist_wheel" - ps: "ls dist" artifacts: From 79f184884e19a00669db9c1f0ca222f262e8d513 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 15:24:19 +0100 Subject: [PATCH 062/225] attempt to fix appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index efa8003..cbfc19b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,7 +3,7 @@ environment: global: - PYTHON: "C:\\Python39-x64\\python.exe" + PYTHON: "C:\\Python38-x64\\python.exe" matrix: - TOXENV: py36 - TOXENV: py37 From df37e4a0508d0380c23d9d685252f83d8dffcd43 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 15:25:21 +0100 Subject: [PATCH 063/225] attempt to fix appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index cbfc19b..32dc6e6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,7 +17,7 @@ install: build: off # Not a C# project, build stuff at the test step instead. test_script: - - "%PYTHON% -m pip tox" + - "%PYTHON% -m tox" after_test: - "%PYTHON% setup.py sdist bdist_wheel" From afca0302898b3aed389892356d747f07db97baa1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 15:40:09 +0100 Subject: [PATCH 064/225] redis doesnt use redis exceptions on windows apparently --- portalocker_tests/test_redis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/portalocker_tests/test_redis.py b/portalocker_tests/test_redis.py index 210a431..8f98bd2 100644 --- a/portalocker_tests/test_redis.py +++ b/portalocker_tests/test_redis.py @@ -34,6 +34,7 @@ def test_redis_lock_timeout(timeout, check_interval): @pytest.mark.xfail(raises=exceptions.ConnectionError) +@pytest.mark.xfail(raises=ConnectionRefusedError) def test_redis_lock_context(): channel = str(random.random()) From c1682e882353ccde3e8d02d43274414d9d0f6909 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 16:09:58 +0100 Subject: [PATCH 065/225] ignoring redis from coverage --- .coveragerc | 3 ++- portalocker_tests/test_redis.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.coveragerc b/.coveragerc index 8bad7fd..9ba46c2 100644 --- a/.coveragerc +++ b/.coveragerc @@ -13,4 +13,5 @@ exclude_lines = [run] source = src branch = True - +omit = + portalocker/redis.py diff --git a/portalocker_tests/test_redis.py b/portalocker_tests/test_redis.py index 8f98bd2..93c49e1 100644 --- a/portalocker_tests/test_redis.py +++ b/portalocker_tests/test_redis.py @@ -8,7 +8,14 @@ from portalocker import redis -@pytest.mark.xfail(raises=exceptions.ConnectionError) +def xfail(function): + # Apply both xfail decorators + function = pytest.mark.xfail(raises=exceptions.ConnectionError)(function) + function = pytest.mark.xfail(raises=ConnectionRefusedError)(function) + return function + + +@xfail def test_redis_lock(): channel = str(random.random()) @@ -22,6 +29,7 @@ def test_redis_lock(): @pytest.mark.parametrize('timeout', [None, 0, 0.001]) @pytest.mark.parametrize('check_interval', [None, 0, 0.0005]) +@xfail def test_redis_lock_timeout(timeout, check_interval): connection = client.Redis() channel = str(random.random()) @@ -33,8 +41,7 @@ def test_redis_lock_timeout(timeout, check_interval): lock_b.acquire(timeout=timeout, check_interval=check_interval) -@pytest.mark.xfail(raises=exceptions.ConnectionError) -@pytest.mark.xfail(raises=ConnectionRefusedError) +@xfail def test_redis_lock_context(): channel = str(random.random()) @@ -46,7 +53,7 @@ def test_redis_lock_context(): pass -@pytest.mark.xfail(raises=exceptions.ConnectionError) +@xfail def test_redis_relock(): channel = str(random.random()) From 0330730d5875611deac75118d9c3a0a756923f15 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 17:33:02 +0100 Subject: [PATCH 066/225] Incrementing version to v2.1.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 93cc02a..731b59a 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.0.0' +__version__ = '2.1.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 0ec0292..7fa213f 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.0.0' +__version__ = '2.1.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From be0518f3a1068d7a2aa8e0864ddeada4c2b21522 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 23 Jan 2021 21:46:33 +0100 Subject: [PATCH 067/225] Update README.rst --- README.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index aab394b..a76e3ed 100644 --- a/README.rst +++ b/README.rst @@ -193,11 +193,10 @@ More examples can be found in the Changelog --------- -Every realease has a ``git tag`` with a commit message for the tag -explaining what was added -and/or changed. The list of tags including the commit messages can be found -here: https://github.com/WoLpH/portalocker/tags - +Every release has a ``git tag`` with a commit message for the tag +explaining what was added and/or changed. The list of tags/releases +including the commit messages can be found here: +https://github.com/WoLpH/portalocker/releases License ------- From ee98f12ddc40ff2713531bebab7dfa0cab318360 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 30 Jan 2021 02:11:21 +0100 Subject: [PATCH 068/225] made redis locks survive network drops as well --- .coveragerc | 7 +- portalocker/redis.py | 145 ++++++++++++++++++++++++++++++-- portalocker/utils.py | 67 ++++++++++----- portalocker_tests/conftest.py | 5 +- portalocker_tests/test_redis.py | 45 +++++++--- pytest.ini | 1 - 6 files changed, 226 insertions(+), 44 deletions(-) diff --git a/.coveragerc b/.coveragerc index 9ba46c2..2611ef0 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,5 @@ [report] +ignore_errors = True fail_under = 100 exclude_lines = pragma: no cover @@ -10,8 +11,10 @@ exclude_lines = if 0: if __name__ == .__main__.: +omit = + portalocker/redis.py + [run] source = src branch = True -omit = - portalocker/redis.py + diff --git a/portalocker/redis.py b/portalocker/redis.py index 71ab09e..1a0eb3c 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -1,14 +1,36 @@ +import _thread +import json +import logging +import random +import time import typing +from typing import Any +from typing import Dict from redis import client from . import exceptions from . import utils +logger = logging.getLogger(__name__) + +DEFAULT_UNAVAILABLE_TIMEOUT = 1 +DEFAULT_THREAD_SLEEP_TIME = 0.1 + + +class PubSubWorkerThread(client.PubSubWorkerThread): + + def run(self): + try: + super().run() + except Exception: # pragma: no cover + _thread.interrupt_main() + raise + class RedisLock(utils.LockBase): ''' - An extremely reliable Redis lock based on pubsub + An extremely reliable Redis lock based on pubsub with a keep-alive thread As opposed to most Redis locking systems based on key/value pairs, this locking method is based on the pubsub system. The big advantage is @@ -16,6 +38,10 @@ class RedisLock(utils.LockBase): processes or otherwise, it will still immediately unlock instead of waiting for a lock timeout. + To make sure both sides of the lock know about the connection state it is + recommended to set the `health_check_interval` when creating the redis + connection.. + Args: channel: the redis channel to use as locking key. connection: an optional redis connection if you already have one @@ -24,15 +50,32 @@ class RedisLock(utils.LockBase): check_interval: check interval while waiting fail_when_locked: after the initial lock failed, return an error or lock the file. This does not wait for the timeout. + thread_sleep_time: sleep time between fetching messages from redis to + prevent a busy/wait loop. In the case of lock conflicts this + increases the time it takes to resolve the conflict. This should + be smaller than the `check_interval` to be useful. + unavailable_timeout: If the conflicting lock is properly connected + this should never exceed twice your redis latency. Note that this + will increase the wait time possibly beyond your `timeout` and is + always executed if a conflict arises. + redis_kwargs: The redis connection arguments if no connection is + given. The `DEFAULT_REDIS_KWARGS` are used as default, if you want + to override these you need to explicitly specify a value (e.g. + `health_check_interval=0`) ''' - + redis_kwargs: Dict[str, Any] + thread: typing.Optional[PubSubWorkerThread] channel: str timeout: float connection: typing.Optional[client.Redis] pubsub: typing.Optional[client.PubSub] = None close_connection: bool + DEFAULT_REDIS_KWARGS = dict( + health_check_interval=10, + ) + def __init__( self, channel: str, @@ -40,12 +83,22 @@ def __init__( timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = False, + thread_sleep_time: float = DEFAULT_THREAD_SLEEP_TIME, + unavailable_timeout: float = DEFAULT_UNAVAILABLE_TIMEOUT, + redis_kwargs: typing.Optional[typing.Dict] = None, ): # We don't want to close connections given as an argument self.close_connection = not connection + self.thread = None self.channel = channel self.connection = connection + self.thread_sleep_time = thread_sleep_time + self.unavailable_timeout = unavailable_timeout + self.redis_kwargs = redis_kwargs or dict() + + for key, value in self.DEFAULT_REDIS_KWARGS.items(): + self.redis_kwargs.setdefault(key, value) super(RedisLock, self).__init__(timeout=timeout, check_interval=check_interval, @@ -53,13 +106,35 @@ def __init__( def get_connection(self) -> client.Redis: if not self.connection: - self.connection = client.Redis() + self.connection = client.Redis(**self.redis_kwargs) return self.connection + def channel_handler(self, message): + if message.get('type') != 'message': # pragma: no cover + return + + try: + data = json.loads(message.get('data')) + except TypeError: # pragma: no cover + logger.debug('TypeError while parsing: %r', message) + return + + self.connection.publish(data['response_channel'], str(time.time())) + + @property + def client_name(self): + return self.channel + '-lock' + def acquire( self, timeout: float = None, check_interval: float = None, - fail_when_locked: typing.Optional[bool] = True): + fail_when_locked: typing.Optional[bool] = None): + + timeout = utils.coalesce(timeout, self.timeout, 0.0) + check_interval = utils.coalesce(check_interval, self.check_interval, + 0.0) + fail_when_locked = utils.coalesce(fail_when_locked, + self.fail_when_locked) assert not self.pubsub, 'This lock is already active' connection = self.get_connection() @@ -68,9 +143,26 @@ def acquire( for _ in timeout_generator: # pragma: no branch subscribers = connection.pubsub_numsub(self.channel)[0][1] + if subscribers: + logger.debug('Found %d lock subscribers for %s', + subscribers, self.channel) + + if self.check_or_kill_lock( + connection, + self.unavailable_timeout): # pragma: no branch + continue + else: # pragma: no cover + subscribers = None + + # Note: this should not be changed to an elif because the if + # above can still end up here if not subscribers: + connection.client_setname(self.client_name) self.pubsub = connection.pubsub() - self.pubsub.subscribe(self.channel) + self.pubsub.subscribe(**{self.channel: self.channel_handler}) + self.thread = PubSubWorkerThread( + self.pubsub, sleep_time=self.thread_sleep_time) + self.thread.start() subscribers = connection.pubsub_numsub(self.channel)[0][1] if subscribers == 1: # pragma: no branch @@ -79,10 +171,49 @@ def acquire( # Race condition, let's try again self.release() - if fail_when_locked: # pragma: no branch + if fail_when_locked: # pragma: no cover raise exceptions.AlreadyLocked(exceptions) + raise exceptions.AlreadyLocked(exceptions) + + def check_or_kill_lock(self, connection, timeout): + # Random channel name to get messages back from the lock + response_channel = f'{self.channel}-{random.random()}' + + pubsub = connection.pubsub() + pubsub.subscribe(response_channel) + connection.publish(self.channel, json.dumps(dict( + response_channel=response_channel, + message='ping', + ))) + + check_interval = min(self.thread_sleep_time, timeout / 10) + for _ in self._timeout_generator( + timeout, check_interval): # pragma: no branch + message = pubsub.get_message(timeout=check_interval) + if message: # pragma: no branch + pubsub.close() + return True + + for client_ in connection.client_list('pubsub'): # pragma: no cover + if client_.get('name') == self.client_name: + logger.warning( + 'Killing unavailable redis client: %r', client_) + connection.client_kill_filter(client_.get('id')) + def release(self): - if self.pubsub: + logger.error('releasing: %r', self.thread) + if self.thread: # pragma: no branch + self.thread.stop() + self.thread.join() + self.thread = None + time.sleep(0.01) + + if self.pubsub: # pragma: no branch self.pubsub.unsubscribe(self.channel) + self.pubsub.close() self.pubsub = None + + def __del__(self): + self.release() + diff --git a/portalocker/utils.py b/portalocker/utils.py index 9f253b5..4226e9e 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -25,6 +25,33 @@ Filename = typing.Union[str, pathlib.Path] +def coalesce(*args, test_value=None): + '''Simple coalescing function that returns the first value that is not + equal to the `test_value`. Or `None` if no value is valid. Usually this + means that the last given value is the default value. + + Note that the `test_value` is compared using an identity check + (i.e. `value is not test_value`) so changing the `test_value` won't work + for all values. + + >>> coalesce(None, 1) + 1 + >>> coalesce() + + >>> coalesce(0, False, True) + 0 + >>> coalesce(0, False, True, test_value=0) + False + + # This won't work because of the `is not test_value` type testing: + >>> coalesce([], dict(spam='eggs'), test_value=[]) + [] + ''' + for arg in args: + if arg is not test_value: + return arg + + @contextlib.contextmanager def open_atomic(filename: Filename, binary: bool = True): '''Open a file for atomic writing. Instead of locking this method allows @@ -79,19 +106,19 @@ def open_atomic(filename: Filename, binary: bool = True): class LockBase(abc.ABC): # pragma: no cover #: timeout when trying to acquire a lock - timeout: typing.Optional[float] = DEFAULT_TIMEOUT + timeout: float #: check interval while waiting for `timeout` - check_interval: typing.Optional[float] = DEFAULT_CHECK_INTERVAL + check_interval: float #: skip the timeout and immediately fail if the initial lock fails - fail_when_locked: typing.Optional[bool] = DEFAULT_FAIL_WHEN_LOCKED + fail_when_locked: bool def __init__(self, timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = None): - if timeout is not None: - self.timeout = timeout - if check_interval is not None: - self.check_interval: float = check_interval + self.timeout = coalesce(timeout, DEFAULT_TIMEOUT) + self.check_interval = coalesce(check_interval, DEFAULT_CHECK_INTERVAL) + self.fail_when_locked = coalesce(fail_when_locked, + DEFAULT_FAIL_WHEN_LOCKED) @abc.abstractmethod def acquire( @@ -100,21 +127,20 @@ def acquire( return NotImplemented def _timeout_generator(self, timeout, check_interval): - if timeout is None: - timeout = self.timeout - - if timeout is None: - timeout = 0 + timeout = coalesce(timeout, self.timeout, 0.0) + check_interval = coalesce(check_interval, self.check_interval, 0.0) - if check_interval is None: - check_interval = self.check_interval + yield 0 + i = 0 - yield + start_time = time.perf_counter() + while start_time + timeout > time.perf_counter(): + i += 1 + yield i - timeout_end = time.perf_counter() + timeout - while timeout_end > time.perf_counter(): - yield - time.sleep(check_interval) + # Take low lock checks into account to stay within the interval + since_start_time = time.perf_counter() - start_time + time.sleep(max(0.001, (i * check_interval) - since_start_time)) @abc.abstractmethod def release(self): @@ -181,8 +207,7 @@ def acquire( fail_when_locked: bool = None) -> typing.IO: '''Acquire the locked filehandle''' - if fail_when_locked is None: - fail_when_locked = self.fail_when_locked + fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) # If we already have a filehandle, return it fh = self.fh diff --git a/portalocker_tests/conftest.py b/portalocker_tests/conftest.py index a92117e..f751d38 100644 --- a/portalocker_tests/conftest.py +++ b/portalocker_tests/conftest.py @@ -1,7 +1,11 @@ import py +import logging import pytest +logger = logging.getLogger(__name__) + + @pytest.fixture def tmpfile(tmpdir_factory): tmpdir = tmpdir_factory.mktemp('temp') @@ -11,4 +15,3 @@ def tmpfile(tmpdir_factory): filename.remove(ignore_errors=True) except (py.error.EBUSY, py.error.ENOENT): pass - diff --git a/portalocker_tests/test_redis.py b/portalocker_tests/test_redis.py index 93c49e1..694c9af 100644 --- a/portalocker_tests/test_redis.py +++ b/portalocker_tests/test_redis.py @@ -1,4 +1,7 @@ +import _thread +import logging import random +import time import pytest from redis import client @@ -6,30 +9,43 @@ import portalocker from portalocker import redis +from portalocker import utils +logger = logging.getLogger(__name__) -def xfail(function): - # Apply both xfail decorators - function = pytest.mark.xfail(raises=exceptions.ConnectionError)(function) - function = pytest.mark.xfail(raises=ConnectionRefusedError)(function) - return function +try: + client.Redis().ping() +except (exceptions.ConnectionError, ConnectionRefusedError): + pytest.skip('Unable to connect to redis', allow_module_level=True) + + +@pytest.fixture(autouse=True) +def set_redis_timeouts(monkeypatch): + monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) + monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) + monkeypatch.setattr(redis, 'DEFAULT_UNAVAILABLE_TIMEOUT', 0.01) + monkeypatch.setattr(redis, 'DEFAULT_THREAD_SLEEP_TIME', 0.001) + monkeypatch.setattr(_thread, 'interrupt_main', lambda: None) -@xfail def test_redis_lock(): channel = str(random.random()) lock_a = redis.RedisLock(channel) lock_a.acquire(fail_when_locked=True) + time.sleep(0.01) lock_b = redis.RedisLock(channel) - with pytest.raises(portalocker.AlreadyLocked): - lock_b.acquire(fail_when_locked=True) + try: + with pytest.raises(portalocker.AlreadyLocked): + lock_b.acquire(fail_when_locked=True) + finally: + lock_a.release() + lock_a.connection.close() @pytest.mark.parametrize('timeout', [None, 0, 0.001]) @pytest.mark.parametrize('check_interval', [None, 0, 0.0005]) -@xfail def test_redis_lock_timeout(timeout, check_interval): connection = client.Redis() channel = str(random.random()) @@ -38,29 +54,34 @@ def test_redis_lock_timeout(timeout, check_interval): lock_b = redis.RedisLock(channel, connection=connection) with pytest.raises(portalocker.AlreadyLocked): - lock_b.acquire(timeout=timeout, check_interval=check_interval) + try: + lock_b.acquire(timeout=timeout, check_interval=check_interval) + finally: + lock_a.release() + lock_a.connection.close() -@xfail def test_redis_lock_context(): channel = str(random.random()) lock_a = redis.RedisLock(channel, fail_when_locked=True) with lock_a: + time.sleep(0.01) lock_b = redis.RedisLock(channel, fail_when_locked=True) with pytest.raises(portalocker.AlreadyLocked): with lock_b: pass -@xfail def test_redis_relock(): channel = str(random.random()) lock_a = redis.RedisLock(channel, fail_when_locked=True) with lock_a: + time.sleep(0.01) with pytest.raises(AssertionError): lock_a.acquire() + time.sleep(0.01) lock_a.release() diff --git a/pytest.ini b/pytest.ini index 9542cd2..f739bce 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,7 +9,6 @@ addopts = --cov portalocker --cov-report term-missing --cov-report html - --no-cov-on-fail --flake8 flake8-ignore = From 34a8c1b619f05243694cc920928dcae32a0ddfb7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 30 Jan 2021 02:11:33 +0100 Subject: [PATCH 069/225] Incrementing version to v2.2.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 731b59a..61c0ad6 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.1.0' +__version__ = '2.2.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 7fa213f..d45b0c1 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.1.0' +__version__ = '2.2.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 3c5ea798a5bb8163a5deabc866934ff3d08b3191 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 30 Jan 2021 02:27:50 +0100 Subject: [PATCH 070/225] updated travis location --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index a76e3ed..52a43e0 100644 --- a/README.rst +++ b/README.rst @@ -2,9 +2,9 @@ portalocker - Cross-platform locking library ############################################ -.. image:: https://travis-ci.org/WoLpH/portalocker.svg?branch=master +.. image:: https://travis-ci.com/WoLpH/portalocker.svg?branch=master :alt: Linux Test Status - :target: https://travis-ci.org/WoLpH/portalocker + :target: https://travis-ci.com/WoLpH/portalocker .. image:: https://ci.appveyor.com/api/projects/status/mgqry98hgpy4prhh?svg=true :alt: Windows Tests Status From 8e52808224e2aad89f46142c8c3489d582738704 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 3 Feb 2021 02:10:41 +0100 Subject: [PATCH 071/225] removed debug statement --- appveyor.yml | 2 +- portalocker/redis.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 32dc6e6..f677fa6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,7 +14,7 @@ install: - "%PYTHON% -m pip install -U tox setuptools wheel" - "%PYTHON% -m pip install -Ue .[tests]" -build: off # Not a C# project, build stuff at the test step instead. +build: false # Not a C# project, build stuff at the test step instead. test_script: - "%PYTHON% -m tox" diff --git a/portalocker/redis.py b/portalocker/redis.py index 1a0eb3c..b2468b9 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -202,7 +202,6 @@ def check_or_kill_lock(self, connection, timeout): connection.client_kill_filter(client_.get('id')) def release(self): - logger.error('releasing: %r', self.thread) if self.thread: # pragma: no branch self.thread.stop() self.thread.join() From ad33bc19419c7308af0139cffd64e955b912bcbc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 3 Feb 2021 02:10:45 +0100 Subject: [PATCH 072/225] Incrementing version to v2.2.1 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 61c0ad6..3bed460 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.2.0' +__version__ = '2.2.1' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index d45b0c1..b20b3d7 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.2.0' +__version__ = '2.2.1' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 0e5da0568bdeb9efeffed1c66d167d83aa06735e Mon Sep 17 00:00:00 2001 From: Laszlo Kindrat Date: Fri, 26 Mar 2021 09:45:47 -0400 Subject: [PATCH 073/225] replaced prints with debug logs --- portalocker/utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 4226e9e..aefe299 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -7,11 +7,14 @@ import tempfile import time import typing +import logging from . import constants from . import exceptions from . import portalocker +logger = logging.getLogger(__name__) + DEFAULT_TIMEOUT = 5 DEFAULT_CHECK_INTERVAL = 0.25 DEFAULT_FAIL_WHEN_LOCKED = False @@ -397,10 +400,9 @@ def acquire( assert not self.lock, 'Already locked' filenames = self.get_filenames() - print('filenames', filenames) - for _ in self._timeout_generator(timeout, check_interval): # pragma: - print('trying lock', filenames) + for n in self._timeout_generator(timeout, check_interval): # pragma: + logger.debug('trying lock (attempt %d) %r', n, filenames) # no branch if self.try_lock(filenames): # pragma: no branch return self.lock # pragma: no cover @@ -410,11 +412,11 @@ def acquire( def try_lock(self, filenames: typing.Sequence[Filename]) -> bool: filename: Filename for filename in filenames: - print('trying lock for', filename) + logger.debug('trying lock for %r', filename) self.lock = Lock(filename, fail_when_locked=True) try: self.lock.acquire() - print('locked', filename) + logger.debug('locked %r', filename) return True except exceptions.AlreadyLocked: pass From 4ac4ade602466b2741b35bb2d6d37ab91326617c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 27 Mar 2021 00:48:27 +0100 Subject: [PATCH 074/225] added PEP 561 compliance thanks to @BoniLindsley --- portalocker/py.typed | 0 portalocker/redis.py | 2 +- setup.py | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 portalocker/py.typed diff --git a/portalocker/py.typed b/portalocker/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/portalocker/redis.py b/portalocker/redis.py index b2468b9..9d34b24 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -152,7 +152,7 @@ def acquire( self.unavailable_timeout): # pragma: no branch continue else: # pragma: no cover - subscribers = None + subscribers = 0 # Note: this should not be changed to an elif because the if # above can still end up here diff --git a/setup.py b/setup.py index 4365703..2e5c552 100644 --- a/setup.py +++ b/setup.py @@ -126,6 +126,7 @@ def run(self): author_email=about['__email__'], url=about['__url__'], license='PSF', + package_data=dict(portalocker=['py.typed']), packages=setuptools.find_packages(exclude=[ 'examples', 'portalocker_tests']), # zip_safe=False, From e99c2d684eef8e6baaf7130c7eaa01be25c7d3a2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 28 Mar 2021 04:18:27 +0200 Subject: [PATCH 075/225] Incrementing version to v2.3.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 3bed460..9fe427c 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.2.1' +__version__ = '2.3.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index b20b3d7..64b7c80 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.2.1' +__version__ = '2.3.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From dde0ae65614fc26c040c81f0f2db734c327e4dad Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 25 May 2021 21:57:10 +0200 Subject: [PATCH 076/225] renamed redis module to fix #65 --- docs/portalocker.redis.rst | 7 ------- docs/portalocker.redis_lock.rst | 7 +++++++ docs/portalocker.rst | 2 +- portalocker/__init__.py | 2 +- portalocker/{redis.py => redis_lock.py} | 0 portalocker_tests/test_redis.py | 20 ++++++++++---------- 6 files changed, 19 insertions(+), 19 deletions(-) delete mode 100644 docs/portalocker.redis.rst create mode 100644 docs/portalocker.redis_lock.rst rename portalocker/{redis.py => redis_lock.py} (100%) diff --git a/docs/portalocker.redis.rst b/docs/portalocker.redis.rst deleted file mode 100644 index 4406655..0000000 --- a/docs/portalocker.redis.rst +++ /dev/null @@ -1,7 +0,0 @@ -portalocker.redis module -======================== - -.. automodule:: portalocker.redis - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/portalocker.redis_lock.rst b/docs/portalocker.redis_lock.rst new file mode 100644 index 0000000..2b88151 --- /dev/null +++ b/docs/portalocker.redis_lock.rst @@ -0,0 +1,7 @@ +portalocker.redis\_lock module +============================== + +.. automodule:: portalocker.redis_lock + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/portalocker.rst b/docs/portalocker.rst index 9050d7a..486241b 100644 --- a/docs/portalocker.rst +++ b/docs/portalocker.rst @@ -6,7 +6,7 @@ Submodules .. toctree:: - portalocker.redis + portalocker.redis_lock portalocker.constants portalocker.exceptions portalocker.portalocker diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 64b7c80..cd68c62 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -5,7 +5,7 @@ from . import utils try: # pragma: no cover - from .redis import RedisLock + from .redis_lock import RedisLock except ImportError: # pragma: no cover RedisLock = None # type: ignore diff --git a/portalocker/redis.py b/portalocker/redis_lock.py similarity index 100% rename from portalocker/redis.py rename to portalocker/redis_lock.py diff --git a/portalocker_tests/test_redis.py b/portalocker_tests/test_redis.py index 694c9af..e622c45 100644 --- a/portalocker_tests/test_redis.py +++ b/portalocker_tests/test_redis.py @@ -8,7 +8,7 @@ from redis import exceptions import portalocker -from portalocker import redis +from portalocker import redis_lock from portalocker import utils logger = logging.getLogger(__name__) @@ -23,19 +23,19 @@ def set_redis_timeouts(monkeypatch): monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) - monkeypatch.setattr(redis, 'DEFAULT_UNAVAILABLE_TIMEOUT', 0.01) - monkeypatch.setattr(redis, 'DEFAULT_THREAD_SLEEP_TIME', 0.001) + monkeypatch.setattr(redis_lock, 'DEFAULT_UNAVAILABLE_TIMEOUT', 0.01) + monkeypatch.setattr(redis_lock, 'DEFAULT_THREAD_SLEEP_TIME', 0.001) monkeypatch.setattr(_thread, 'interrupt_main', lambda: None) def test_redis_lock(): channel = str(random.random()) - lock_a = redis.RedisLock(channel) + lock_a = redis_lock.RedisLock(channel) lock_a.acquire(fail_when_locked=True) time.sleep(0.01) - lock_b = redis.RedisLock(channel) + lock_b = redis_lock.RedisLock(channel) try: with pytest.raises(portalocker.AlreadyLocked): lock_b.acquire(fail_when_locked=True) @@ -49,10 +49,10 @@ def test_redis_lock(): def test_redis_lock_timeout(timeout, check_interval): connection = client.Redis() channel = str(random.random()) - lock_a = redis.RedisLock(channel) + lock_a = redis_lock.RedisLock(channel) lock_a.acquire(timeout=timeout, check_interval=check_interval) - lock_b = redis.RedisLock(channel, connection=connection) + lock_b = redis_lock.RedisLock(channel, connection=connection) with pytest.raises(portalocker.AlreadyLocked): try: lock_b.acquire(timeout=timeout, check_interval=check_interval) @@ -64,10 +64,10 @@ def test_redis_lock_timeout(timeout, check_interval): def test_redis_lock_context(): channel = str(random.random()) - lock_a = redis.RedisLock(channel, fail_when_locked=True) + lock_a = redis_lock.RedisLock(channel, fail_when_locked=True) with lock_a: time.sleep(0.01) - lock_b = redis.RedisLock(channel, fail_when_locked=True) + lock_b = redis_lock.RedisLock(channel, fail_when_locked=True) with pytest.raises(portalocker.AlreadyLocked): with lock_b: pass @@ -76,7 +76,7 @@ def test_redis_lock_context(): def test_redis_relock(): channel = str(random.random()) - lock_a = redis.RedisLock(channel, fail_when_locked=True) + lock_a = redis_lock.RedisLock(channel, fail_when_locked=True) with lock_a: time.sleep(0.01) with pytest.raises(AssertionError): From 489e88ae5405e8e18b4ecdda20264b61091f469e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 26 May 2021 15:13:24 +0200 Subject: [PATCH 077/225] Revert "renamed redis module to fix #65". Turned out not to be a bug :) This reverts commit dde0ae65614fc26c040c81f0f2db734c327e4dad. --- docs/portalocker.redis.rst | 7 +++++++ docs/portalocker.redis_lock.rst | 7 ------- docs/portalocker.rst | 2 +- portalocker/__init__.py | 2 +- portalocker/{redis_lock.py => redis.py} | 0 portalocker_tests/test_redis.py | 20 ++++++++++---------- 6 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 docs/portalocker.redis.rst delete mode 100644 docs/portalocker.redis_lock.rst rename portalocker/{redis_lock.py => redis.py} (100%) diff --git a/docs/portalocker.redis.rst b/docs/portalocker.redis.rst new file mode 100644 index 0000000..4406655 --- /dev/null +++ b/docs/portalocker.redis.rst @@ -0,0 +1,7 @@ +portalocker.redis module +======================== + +.. automodule:: portalocker.redis + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/portalocker.redis_lock.rst b/docs/portalocker.redis_lock.rst deleted file mode 100644 index 2b88151..0000000 --- a/docs/portalocker.redis_lock.rst +++ /dev/null @@ -1,7 +0,0 @@ -portalocker.redis\_lock module -============================== - -.. automodule:: portalocker.redis_lock - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/portalocker.rst b/docs/portalocker.rst index 486241b..9050d7a 100644 --- a/docs/portalocker.rst +++ b/docs/portalocker.rst @@ -6,7 +6,7 @@ Submodules .. toctree:: - portalocker.redis_lock + portalocker.redis portalocker.constants portalocker.exceptions portalocker.portalocker diff --git a/portalocker/__init__.py b/portalocker/__init__.py index cd68c62..64b7c80 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -5,7 +5,7 @@ from . import utils try: # pragma: no cover - from .redis_lock import RedisLock + from .redis import RedisLock except ImportError: # pragma: no cover RedisLock = None # type: ignore diff --git a/portalocker/redis_lock.py b/portalocker/redis.py similarity index 100% rename from portalocker/redis_lock.py rename to portalocker/redis.py diff --git a/portalocker_tests/test_redis.py b/portalocker_tests/test_redis.py index e622c45..694c9af 100644 --- a/portalocker_tests/test_redis.py +++ b/portalocker_tests/test_redis.py @@ -8,7 +8,7 @@ from redis import exceptions import portalocker -from portalocker import redis_lock +from portalocker import redis from portalocker import utils logger = logging.getLogger(__name__) @@ -23,19 +23,19 @@ def set_redis_timeouts(monkeypatch): monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) - monkeypatch.setattr(redis_lock, 'DEFAULT_UNAVAILABLE_TIMEOUT', 0.01) - monkeypatch.setattr(redis_lock, 'DEFAULT_THREAD_SLEEP_TIME', 0.001) + monkeypatch.setattr(redis, 'DEFAULT_UNAVAILABLE_TIMEOUT', 0.01) + monkeypatch.setattr(redis, 'DEFAULT_THREAD_SLEEP_TIME', 0.001) monkeypatch.setattr(_thread, 'interrupt_main', lambda: None) def test_redis_lock(): channel = str(random.random()) - lock_a = redis_lock.RedisLock(channel) + lock_a = redis.RedisLock(channel) lock_a.acquire(fail_when_locked=True) time.sleep(0.01) - lock_b = redis_lock.RedisLock(channel) + lock_b = redis.RedisLock(channel) try: with pytest.raises(portalocker.AlreadyLocked): lock_b.acquire(fail_when_locked=True) @@ -49,10 +49,10 @@ def test_redis_lock(): def test_redis_lock_timeout(timeout, check_interval): connection = client.Redis() channel = str(random.random()) - lock_a = redis_lock.RedisLock(channel) + lock_a = redis.RedisLock(channel) lock_a.acquire(timeout=timeout, check_interval=check_interval) - lock_b = redis_lock.RedisLock(channel, connection=connection) + lock_b = redis.RedisLock(channel, connection=connection) with pytest.raises(portalocker.AlreadyLocked): try: lock_b.acquire(timeout=timeout, check_interval=check_interval) @@ -64,10 +64,10 @@ def test_redis_lock_timeout(timeout, check_interval): def test_redis_lock_context(): channel = str(random.random()) - lock_a = redis_lock.RedisLock(channel, fail_when_locked=True) + lock_a = redis.RedisLock(channel, fail_when_locked=True) with lock_a: time.sleep(0.01) - lock_b = redis_lock.RedisLock(channel, fail_when_locked=True) + lock_b = redis.RedisLock(channel, fail_when_locked=True) with pytest.raises(portalocker.AlreadyLocked): with lock_b: pass @@ -76,7 +76,7 @@ def test_redis_lock_context(): def test_redis_relock(): channel = str(random.random()) - lock_a = redis_lock.RedisLock(channel, fail_when_locked=True) + lock_a = redis.RedisLock(channel, fail_when_locked=True) with lock_a: time.sleep(0.01) with pytest.raises(AssertionError): From 0f420c8e0b533471754c40b561a350df13bef50f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 26 May 2021 15:14:27 +0200 Subject: [PATCH 078/225] alternative fix for #65 --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 52a43e0..a83a80c 100644 --- a/README.rst +++ b/README.rst @@ -47,6 +47,12 @@ that if the connection gets killed due to network issues, crashing processes or otherwise, it will still immediately unlock instead of waiting for a lock timeout. +First make sure you have everything installed correctly: + +:: + + pip install "portalocker[redis]" + Usage is really easy: :: From 5a23e40e5735054f832b6ae555839a766795985e Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 20 Aug 2021 08:13:48 -0700 Subject: [PATCH 079/225] Remove Python 2 classifier --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 2e5c552..7a1915f 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,6 @@ def run(self): classifiers=[ 'Intended Audience :: Developers', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', From a89f21a17c93bfdb0b28b57c868bddfaf3b3f64a Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 20 Aug 2021 08:37:43 -0700 Subject: [PATCH 080/225] Explicitly mentions Semantic Versioning --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index a83a80c..3bab409 100644 --- a/README.rst +++ b/README.rst @@ -196,6 +196,13 @@ portalocker.exceptions.AlreadyLocked More examples can be found in the `tests `_. + +Versioning +---------- + +This library follows `Semantic Versioning `_. + + Changelog --------- From 90d267a19a61002732d85478a09a53755c179a7f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 21 Aug 2021 11:22:08 +0200 Subject: [PATCH 081/225] Incrementing version to v2.3.1 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 9fe427c..03d461c 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.3.0' +__version__ = '2.3.1' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 64b7c80..746292f 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.3.0' +__version__ = '2.3.1' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From b28e6ed3077974f53b1df32d41edb31710714c2f Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Sat, 21 Aug 2021 10:02:06 -0700 Subject: [PATCH 082/225] Actually regulate the Python version requirement --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 7a1915f..65c77e9 100644 --- a/setup.py +++ b/setup.py @@ -120,6 +120,7 @@ def run(self): 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], + python_requires=">=3.3", keywords='locking, locks, with statement, windows, linux, unix', author=about['__author__'], author_email=about['__email__'], From d480d4ecca9130f9a9f48b8bef759451f2238c47 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 27 Aug 2021 14:41:23 +0200 Subject: [PATCH 083/225] updated pywin32 due to CVE-2021-32559 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a1915f..a465378 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,8 @@ def run(self): 'test': PyTest, }, install_requires=[ - 'pywin32!=226; platform_system == "Windows"', + # Due to CVE-2021-32559 updating the pywin32 requirement + 'pywin32>=226; platform_system == "Windows"', ], tests_require=tests_require, extras_require=dict( From 6f89fcdb1179201cd4a82e1da150ecea1679076d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 27 Aug 2021 14:42:56 +0200 Subject: [PATCH 084/225] requiring python 3.5 or above due to type hinting --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2239ad5..fed530f 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ class PyTest(TestCommand): - user_options = [('pytest-args=', 'a', "Arguments to pass to pytest")] + user_options = [('pytest-args=', 'a', 'Arguments to pass to pytest')] def initialize_options(self): TestCommand.initialize_options(self) @@ -120,7 +120,7 @@ def run(self): 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], - python_requires=">=3.3", + python_requires='>=3.5', keywords='locking, locks, with statement, windows, linux, unix', author=about['__author__'], author_email=about['__email__'], From ee0f8c099a5245a72078593ac60385eaee99e020 Mon Sep 17 00:00:00 2001 From: Ray Luo Date: Fri, 20 Aug 2021 08:37:43 -0700 Subject: [PATCH 085/225] Explicitly mentions Semantic Versioning --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index a83a80c..3bab409 100644 --- a/README.rst +++ b/README.rst @@ -196,6 +196,13 @@ portalocker.exceptions.AlreadyLocked More examples can be found in the `tests `_. + +Versioning +---------- + +This library follows `Semantic Versioning `_. + + Changelog --------- From 4257cedb01705c2ea60b1669245f5b20dbf23ecc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 27 Aug 2021 15:43:36 +0200 Subject: [PATCH 086/225] Incrementing version to v2.3.2 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 03d461c..69b961e 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.3.1' +__version__ = '2.3.2' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 746292f..89e1d51 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.3.1' +__version__ = '2.3.2' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 871814f18376163da1641072b40ec0abbc9e9632 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 28 Aug 2021 03:31:36 +0200 Subject: [PATCH 087/225] updated python version classifiers --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index fed530f..055a458 100644 --- a/setup.py +++ b/setup.py @@ -113,10 +113,12 @@ def run(self): classifiers=[ 'Intended Audience :: Developers', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], From 3025e66895ce819d806ac7201041af9d0b4f67bb Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 14 Sep 2021 03:58:18 +0200 Subject: [PATCH 088/225] added more type hinting to fix #60 --- msvcrt.pyi | 7 +++++++ portalocker/exceptions.py | 12 ++++++++++-- portalocker/redis.py | 2 +- portalocker/utils.py | 24 ++++++++++++++++-------- setup.py | 2 +- tox.ini | 5 +++++ 6 files changed, 40 insertions(+), 12 deletions(-) create mode 100644 msvcrt.pyi diff --git a/msvcrt.pyi b/msvcrt.pyi new file mode 100644 index 0000000..73c309a --- /dev/null +++ b/msvcrt.pyi @@ -0,0 +1,7 @@ +LK_LOCK: int +LK_NBLCK: int +LK_NBRLCK: int +LK_RLCK: int +LK_UNLCK: int + +def locking(file: int, mode: int, lock_length: int) -> int: ... diff --git a/portalocker/exceptions.py b/portalocker/exceptions.py index 0a815b9..0a8594d 100644 --- a/portalocker/exceptions.py +++ b/portalocker/exceptions.py @@ -1,10 +1,18 @@ +import typing + + class BaseLockException(Exception): # Error codes: LOCK_FAILED = 1 - def __init__(self, *args, fh=None, **kwargs): + def __init__( + self, + *args: typing.Any, + fh: typing.Optional[typing.IO] = None, + **kwargs: typing.Any, + ) -> None: self.fh = fh - Exception.__init__(self, *args, **kwargs) + Exception.__init__(self, *args) class LockException(BaseLockException): diff --git a/portalocker/redis.py b/portalocker/redis.py index 9d34b24..08dbd4a 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -18,7 +18,7 @@ DEFAULT_THREAD_SLEEP_TIME = 0.1 -class PubSubWorkerThread(client.PubSubWorkerThread): +class PubSubWorkerThread(client.PubSubWorkerThread): # type: ignore def run(self): try: diff --git a/portalocker/utils.py b/portalocker/utils.py index aefe299..b50de3e 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -28,7 +28,7 @@ Filename = typing.Union[str, pathlib.Path] -def coalesce(*args, test_value=None): +def coalesce(*args: typing.Any, test_value: typing.Any = None) -> typing.Any: '''Simple coalescing function that returns the first value that is not equal to the `test_value`. Or `None` if no value is valid. Usually this means that the last given value is the default value. @@ -56,7 +56,8 @@ def coalesce(*args, test_value=None): @contextlib.contextmanager -def open_atomic(filename: Filename, binary: bool = True): +def open_atomic(filename: Filename, binary: bool = True) \ + -> typing.Iterator[typing.IO]: '''Open a file for atomic writing. Instead of locking this method allows you to write the entire file and move it to the actual location. Note that this makes the assumption that a rename is atomic on your platform which @@ -129,21 +130,23 @@ def acquire( fail_when_locked: bool = None): return NotImplemented - def _timeout_generator(self, timeout, check_interval): - timeout = coalesce(timeout, self.timeout, 0.0) - check_interval = coalesce(check_interval, self.check_interval, 0.0) + def _timeout_generator(self, timeout: typing.Optional[float], + check_interval: typing.Optional[float]) \ + -> typing.Iterator[int]: + f_timeout = coalesce(timeout, self.timeout, 0.0) + f_check_interval = coalesce(check_interval, self.check_interval, 0.0) yield 0 i = 0 start_time = time.perf_counter() - while start_time + timeout > time.perf_counter(): + while start_time + f_timeout > time.perf_counter(): i += 1 yield i # Take low lock checks into account to stay within the interval since_start_time = time.perf_counter() - start_time - time.sleep(max(0.001, (i * check_interval) - since_start_time)) + time.sleep(max(0.001, (i * f_check_interval) - since_start_time)) @abc.abstractmethod def release(self): @@ -152,8 +155,13 @@ def release(self): def __enter__(self): return self.acquire() - def __exit__(self, type_, value, tb): + def __exit__(self, + exc_type: typing.Optional[typing.Type[BaseException]], + exc_value: typing.Optional[BaseException], + traceback: typing.Any, # Should be typing.TracebackType + ) -> typing.Optional[bool]: self.release() + return None def __delete__(self, instance): instance.release() diff --git a/setup.py b/setup.py index 055a458..8b532ac 100644 --- a/setup.py +++ b/setup.py @@ -128,7 +128,7 @@ def run(self): author_email=about['__email__'], url=about['__url__'], license='PSF', - package_data=dict(portalocker=['py.typed']), + package_data=dict(portalocker=['py.typed', 'msvcrt.pyi']), packages=setuptools.find_packages(exclude=[ 'examples', 'portalocker_tests']), # zip_safe=False, diff --git a/tox.ini b/tox.ini index 0274dfb..84454fa 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,11 @@ basepython = deps = -e{toxinidir}[tests,redis] commands = python -m pytest {posargs} +[testenv:mypy] +basepython = python3 +deps = mypy +commands = mypy {toxinidir}/portalocker + [testenv:flake8] basepython = python3 deps = flake8 From 6989f2a4d887a1d954de2f742de6748ebd7909e4 Mon Sep 17 00:00:00 2001 From: "Kian-Meng, Ang" Date: Mon, 15 Nov 2021 11:17:47 +0800 Subject: [PATCH 089/225] Fix typo --- portalocker/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index b50de3e..69c302a 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -168,7 +168,7 @@ def __delete__(self, instance): class Lock(LockBase): - '''Lock manager with build-in timeout + '''Lock manager with built-in timeout Args: filename: filename From f5823f8a9423fce7fe8d43286c6ca649f73cb4bf Mon Sep 17 00:00:00 2001 From: Joshua Newton Date: Wed, 16 Feb 2022 13:44:39 -0500 Subject: [PATCH 090/225] `README.rst`: Trim outdated `.EXCLUSIVE` snippets It looks like these sections got duplicated somehow. I've kept only the portalocker.LOCK_EX snippet, since it seems to be the only valid flag presently. --- README.rst | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/README.rst b/README.rst index 3bab409..362beea 100644 --- a/README.rst +++ b/README.rst @@ -121,38 +121,6 @@ To make sure your cache generation scripts don't race, use the `Lock` class: To customize the opening and locking a manual approach is also possible: ->>> import portalocker ->>> file = open('somefile', 'r+') ->>> portalocker.lock(file, portalocker.EXCLUSIVE) ->>> file.seek(12) ->>> file.write('foo') ->>> file.close() - -Explicitly unlocking is not needed in most cases but omitting it has been known -to cause issues: - ->>> import portalocker ->>> with portalocker.Lock('somefile', timeout=1) as fh: -... print >>fh, 'writing some stuff to my cache...' - -To customize the opening and locking a manual approach is also possible: - ->>> import portalocker ->>> file = open('somefile', 'r+') ->>> portalocker.lock(file, portalocker.EXCLUSIVE) ->>> file.seek(12) ->>> file.write('foo') ->>> file.close() - -Explicitly unlocking is not needed in most cases but omitting it has been known -to cause issues: - ->>> import portalocker ->>> with portalocker.Lock('somefile', timeout=1) as fh: -... print >>fh, 'writing some stuff to my cache...' - -To customize the opening and locking a manual approach is also possible: - >>> import portalocker >>> file = open('somefile', 'r+') >>> portalocker.lock(file, portalocker.LOCK_EX) From 7750eb7a9d750501a6f3def87aa182d92f3e49e9 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 17 Feb 2022 03:14:46 +0100 Subject: [PATCH 091/225] Added github actions tests --- .github/workflows/python-package.yml | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..b25ea52 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,37 @@ +name: pytest + +on: + push: + branches: [ develop, master ] + pull_request: + branches: [ develop ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest From 94c935183b5d5b0e4b9c9a59883de9c0d3cc7466 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 17 Feb 2022 03:09:40 +0100 Subject: [PATCH 092/225] cleaned up old distutils and sphinx directives --- setup.cfg | 10 +--------- setup.py | 8 -------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/setup.cfg b/setup.cfg index 3232b33..68d4f40 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,13 +1,5 @@ [metadata] -description-file = README.rst - -[build_sphinx] -source-dir = docs/ -build-dir = docs/_build -all_files = 1 - -[upload_sphinx] -upload-dir = docs/_build/html +description_file = README.rst [bdist_wheel] universal = 1 diff --git a/setup.py b/setup.py index 8b532ac..b568327 100644 --- a/setup.py +++ b/setup.py @@ -4,19 +4,11 @@ import re import sys import typing -from distutils.version import LooseVersion import setuptools from setuptools import __version__ as setuptools_version from setuptools.command.test import test as TestCommand -if LooseVersion(setuptools_version) < LooseVersion('38.3.0'): - raise SystemExit( - 'Your `setuptools` version is old. ' - 'Please upgrade setuptools by running `pip install -U setuptools` ' - 'and try again.' - ) - # To prevent importing about and thereby breaking the coverage info we use this # exec hack about: typing.Dict[str, str] = {} From 6dac3604e256a4e6c6d708c8c06fba445c961dad Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 17 Feb 2022 03:12:53 +0100 Subject: [PATCH 093/225] fixing appveyor tests? --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f677fa6..8ad9373 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ after_test: - "%PYTHON% setup.py sdist bdist_wheel" - ps: "ls dist" -artifacts: - - path: dist\* +# artifacts: +# - path: dist\* #on_success: # - TODO: upload the content of dist/*.whl to a public wheelhouse From 449b1ef084a988a5e416735a4cb71f317d3df2f9 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 17 Feb 2022 04:27:15 +0100 Subject: [PATCH 094/225] github actions using tox --- .github/workflows/python-package.yml | 17 +++++------------ README.rst | 2 +- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b25ea52..81f2919 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,4 +1,4 @@ -name: pytest +name: test on: push: @@ -23,15 +23,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 + python -m pip install --upgrade pip setuptools wheel + python -m pip install tox + - name: Test with tox run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest + tox diff --git a/README.rst b/README.rst index 362beea..221f824 100644 --- a/README.rst +++ b/README.rst @@ -123,7 +123,7 @@ To customize the opening and locking a manual approach is also possible: >>> import portalocker >>> file = open('somefile', 'r+') ->>> portalocker.lock(file, portalocker.LOCK_EX) +>>> portalocker.lock(file, portalocker.LockFlags.EXCLUSIVE) >>> file.seek(12) >>> file.write('foo') >>> file.close() From ee8305c6b273449c72912b86924cb320060f1a8b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 17 Feb 2022 04:29:04 +0100 Subject: [PATCH 095/225] replacing travis --- .travis.yml | 42 ------------------------------------------ README.rst | 4 ++-- 2 files changed, 2 insertions(+), 44 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 698e466..0000000 --- a/.travis.yml +++ /dev/null @@ -1,42 +0,0 @@ -sudo: false -dist: xenial -language: python - -services: -- redis - -jobs: - allow_failures: - - python: pypy3 - -python: -- '3.6' -- '3.7' -- '3.8' -- '3.9' -- pypy3 -before_install: -- wheel version -install: -- pip install -U setuptools wheel pip -- pip install -r portalocker_tests/requirements.txt -- pip install -e '.[redis]' -- pip install coveralls -script: -- python setup.py test -after_success: -- coveralls -- python setup.py bdist_wheel -- ls -la dist/ -- pip install codecov -- codecov -before_deploy: -- python setup.py combine -o dist/portalocker.py -deploy: - provider: releases - api_key: - secure: rqSN1zaF8/9mZYpiU1Em1txC6ZSBG2VjqmeUH7Brp9z0GcIfr8JKoPkIB5W6NyrOPXmBa+eA1nsUsrcDkyMr/q1T5pvJwZl2QtOWayAb0jjvBFv/hDK9VO2eqSADU81YzTWA+U8lRSDSTqmdgWa76LG8Hxc1m76Ns5t2KGxOO5k= - file: dist/portalocker.py - skip_cleanup: true - on: - repo: WoLpH/portalocker diff --git a/README.rst b/README.rst index 221f824..b07d74d 100644 --- a/README.rst +++ b/README.rst @@ -2,9 +2,9 @@ portalocker - Cross-platform locking library ############################################ -.. image:: https://travis-ci.com/WoLpH/portalocker.svg?branch=master +.. image:: https://github.com/WoLpH/portalocker/actions/workflows/python-package.yml/badge.svg?branch=master :alt: Linux Test Status - :target: https://travis-ci.com/WoLpH/portalocker + :target: https://github.com/WoLpH/portalocker/actions/ .. image:: https://ci.appveyor.com/api/projects/status/mgqry98hgpy4prhh?svg=true :alt: Windows Tests Status From cc989fe2ea0c51273390f2a65de2499ac0aa6988 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 17 Feb 2022 14:33:46 +0100 Subject: [PATCH 096/225] added warning when combining timeout with blocking mode to fix #74 --- portalocker/utils.py | 14 ++++++++++++-- portalocker_tests/tests.py | 11 +++++++++++ pytest.ini | 1 - setup.py | 1 - 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 69c302a..0bbc67e 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -1,13 +1,14 @@ import abc import atexit import contextlib +import logging import os import pathlib import random import tempfile import time import typing -import logging +import warnings from . import constants from . import exceptions @@ -193,7 +194,7 @@ def __init__( self, filename: Filename, mode: str = 'a', - timeout: float = DEFAULT_TIMEOUT, + timeout: float = None, check_interval: float = DEFAULT_CHECK_INTERVAL, fail_when_locked: bool = DEFAULT_FAIL_WHEN_LOCKED, flags: constants.LockFlags = LOCK_METHOD, **file_open_kwargs): @@ -203,6 +204,12 @@ def __init__( else: truncate = False + print('flags:', bin(flags), bin(constants.LockFlags.NON_BLOCKING), timeout) + if timeout is None: + timeout = DEFAULT_TIMEOUT + elif not (flags & constants.LockFlags.NON_BLOCKING): + warnings.warn('timeout has no effect in blocking mode') + self.fh: typing.Optional[typing.IO] = None self.filename: str = str(filename) self.mode: str = mode @@ -220,6 +227,9 @@ def acquire( fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) + if not (self.flags & constants.LockFlags.NON_BLOCKING) and timeout is not None: + warnings.warn('timeout has no effect in blocking mode') + # If we already have a filehandle, return it fh = self.fh if fh: diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 3cc9758..514b028 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -206,3 +206,14 @@ def test_shared(tmpfile): portalocker.unlock(f) f.close() + +def test_blocking_timeout(tmpfile): + flags = portalocker.LockFlags.SHARED + + with pytest.warns(UserWarning): + with portalocker.Lock(tmpfile, timeout=5, flags=flags): + pass + + lock = portalocker.Lock(tmpfile, flags=flags) + with pytest.warns(UserWarning): + lock.acquire(timeout=5) diff --git a/pytest.ini b/pytest.ini index f739bce..1ef7c1b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,7 +9,6 @@ addopts = --cov portalocker --cov-report term-missing --cov-report html - --flake8 flake8-ignore = *.py W391 diff --git a/setup.py b/setup.py index b568327..974e0aa 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,6 @@ 'pytest>=5.4.1', 'pytest-cov>=2.8.1', 'sphinx>=3.0.3', - 'pytest-flake8>=1.0.5', 'pytest-mypy>=0.8.0', 'redis', ] From 277c2e4d810314429a466a10c2a826663f752553 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 17 Feb 2022 14:37:30 +0100 Subject: [PATCH 097/225] fixed flake8 issues --- portalocker/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 0bbc67e..d7c94ca 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -204,7 +204,6 @@ def __init__( else: truncate = False - print('flags:', bin(flags), bin(constants.LockFlags.NON_BLOCKING), timeout) if timeout is None: timeout = DEFAULT_TIMEOUT elif not (flags & constants.LockFlags.NON_BLOCKING): @@ -227,7 +226,8 @@ def acquire( fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) - if not (self.flags & constants.LockFlags.NON_BLOCKING) and timeout is not None: + if not (self.flags & constants.LockFlags.NON_BLOCKING) \ + and timeout is not None: warnings.warn('timeout has no effect in blocking mode') # If we already have a filehandle, return it From 970b98169788d338ce932f123e6b537740891a44 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 17 Feb 2022 16:04:24 +0100 Subject: [PATCH 098/225] Incrementing version to v2.4.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 69b961e..ad60e5f 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.3.2' +__version__ = '2.4.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 89e1d51..13699df 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.3.2' +__version__ = '2.4.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From c69616efba1a3492416018535797389ac840deef Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 2 Jul 2022 15:01:46 +0200 Subject: [PATCH 099/225] Run tests on Linux, OS X and Windows --- .github/workflows/python-package.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 81f2919..9f4f1c3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -9,11 +9,12 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ['3.8', '3.9', '3.10', '3.11'] + os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] steps: - uses: actions/checkout@v2 @@ -21,6 +22,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Python version + run: python --version - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From 309fabc709b0b06ad8e68d4c33077b1a0599e86c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 2 Jul 2022 15:02:54 +0200 Subject: [PATCH 100/225] added python 3.11 to builds --- tox.ini | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 84454fa..3d2d080 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,13 @@ [tox] -envlist = py36, py37, py38, py39, py310, pypy3, flake8, docs +envlist = py38, py39, py310, py311, pypy3, flake8, docs skip_missing_interpreters = True [testenv] basepython = - py36: python3.6 - py37: python3.7 py38: python3.8 py39: python3.9 py310: python3.10 + py311: python3.11 pypy3: pypy3 deps = -e{toxinidir}[tests,redis] From ca9ea75bdc6dd427a47f4facfc6c15e20eab641c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 2 Jul 2022 15:04:42 +0200 Subject: [PATCH 101/225] disabling 3.11 github builds for now as they're not supported --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9f4f1c3..09571f8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10'] os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] steps: From 7acfa6bba387de150e3f2db1e658181f16aa8967 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 2 Jul 2022 15:06:27 +0200 Subject: [PATCH 102/225] run tox in parallel --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 09571f8..fa5c18a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,4 +30,4 @@ jobs: python -m pip install tox - name: Test with tox run: | - tox + tox --parallel=all From 5297d6732ed6fb18d76e143165a9df30128a89ad Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 2 Jul 2022 15:19:53 +0200 Subject: [PATCH 103/225] added pip caching to build --- .github/workflows/python-package.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fa5c18a..2484e7e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,11 +17,12 @@ jobs: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Python version run: python --version - name: Install dependencies From bc61e11e1ac07ec62617cdb2c708fb161571c7ff Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 2 Jul 2022 16:29:33 +0200 Subject: [PATCH 104/225] More efficient testing --- .github/workflows/python-package.yml | 38 +++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 2484e7e..c5ea426 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -7,17 +7,43 @@ on: branches: [ develop ] jobs: - build: - + # Run os specific tests on the slower OS X/Windows machines + windows_osx: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: python-version: ['3.8', '3.9', '3.10'] - os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] + os: ['macos-latest', 'windows-latest'] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Python version + run: python --version + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install --develop ".[tests]" + - name: Test with pytest + run: python -m pytest + + # Run all tests including Redis on Linux + linux: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v3 + - name: Start Redis + uses: supercharge/redis-github-action@1.4.0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -27,8 +53,6 @@ jobs: run: python --version - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools wheel python -m pip install tox - - name: Test with tox - run: | - tox --parallel=all + - name: Test with pytest + run: tox -p all From a80fbeb50b23e03b1a7807120ac4c895d9f0b430 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 2 Jul 2022 16:30:35 +0200 Subject: [PATCH 105/225] More efficient testing --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c5ea426..7cb780d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,7 +28,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install --develop ".[tests]" + python -m pip install -e ".[tests]" - name: Test with pytest run: python -m pytest From 136eba994b5f2475c28d81ed88d5e4f9518aeb75 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 2 Jul 2022 17:17:26 +0200 Subject: [PATCH 106/225] Added extra shared locks tests --- portalocker_tests/tests.py | 46 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 514b028..9674246 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -1,6 +1,9 @@ from __future__ import print_function from __future__ import with_statement +import multiprocessing +import time + import pytest import portalocker from portalocker import utils @@ -36,8 +39,10 @@ def test_with_timeout(tmpfile): with pytest.raises(portalocker.AlreadyLocked): with portalocker.Lock(tmpfile, timeout=0.1) as fh: print('writing some stuff to my cache...', file=fh) - with portalocker.Lock(tmpfile, timeout=0.1, mode='wb', - fail_when_locked=True): + with portalocker.Lock( + tmpfile, timeout=0.1, mode='wb', + fail_when_locked=True + ): pass print('writing more stuff to my cache...', file=fh) @@ -217,3 +222,40 @@ def test_blocking_timeout(tmpfile): lock = portalocker.Lock(tmpfile, flags=flags) with pytest.warns(UserWarning): lock.acquire(timeout=5) + + +def shared_lock(filename, **kwargs): + with portalocker.Lock( + filename, + timeout=0.1, + fail_when_locked=False, + flags=portalocker.LockFlags.SHARED | portalocker.LockFlags.NON_BLOCKING, + ): + time.sleep(0.2) + return True + + +def shared_lock_fail(filename, **kwargs): + with portalocker.Lock( + filename, + timeout=0.1, + fail_when_locked=True, + flags=portalocker.LockFlags.SHARED | portalocker.LockFlags.NON_BLOCKING, + ): + time.sleep(0.2) + return True + + +def test_shared_processes(tmpfile): + # Force spawning the process so we don't accidently inherit the lock + # I'm not a 100% certain this will work correctly unfortunately... there + # is some potential for breaking other tests + multiprocessing.set_start_method('spawn') + + with multiprocessing.Pool(processes=2) as pool: + for result in pool.imap_unordered(shared_lock, 3 * [tmpfile]): + assert result is True + + with multiprocessing.Pool(processes=2) as pool: + for result in pool.imap_unordered(shared_lock_fail, 3 * [tmpfile]): + assert result is True From 8521cc5883f8b68b15a8fced6f7a3547019e8a63 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 5 Jul 2022 01:55:07 +0200 Subject: [PATCH 107/225] Added multiprocessing test cases for more test scenarios --- portalocker/exceptions.py | 4 +- portalocker_tests/conftest.py | 20 ++++++--- portalocker_tests/tests.py | 78 +++++++++++++++++++++++++++++++---- 3 files changed, 85 insertions(+), 17 deletions(-) diff --git a/portalocker/exceptions.py b/portalocker/exceptions.py index 0a8594d..b360c77 100644 --- a/portalocker/exceptions.py +++ b/portalocker/exceptions.py @@ -19,9 +19,9 @@ class LockException(BaseLockException): pass -class AlreadyLocked(BaseLockException): +class AlreadyLocked(LockException): pass -class FileToLarge(BaseLockException): +class FileToLarge(LockException): pass diff --git a/portalocker_tests/conftest.py b/portalocker_tests/conftest.py index f751d38..52d31dd 100644 --- a/portalocker_tests/conftest.py +++ b/portalocker_tests/conftest.py @@ -1,17 +1,25 @@ +import multiprocessing + import py import logging import pytest - +import random logger = logging.getLogger(__name__) @pytest.fixture -def tmpfile(tmpdir_factory): - tmpdir = tmpdir_factory.mktemp('temp') - filename = tmpdir.join('tmpfile') +def tmpfile(tmp_path): + filename = tmp_path / str(random.random()) yield str(filename) try: - filename.remove(ignore_errors=True) - except (py.error.EBUSY, py.error.ENOENT): + filename.unlink(missing_ok=True) + except PermissionError: pass + + +def pytest_sessionstart(session): + # Force spawning the process so we don't accidently inherit locks. + # I'm not a 100% certain this will work correctly unfortunately... there + # is some potential for breaking tests + multiprocessing.set_start_method('spawn') diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 9674246..1c3fcfd 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -1,8 +1,10 @@ from __future__ import print_function from __future__ import with_statement +import dataclasses import multiprocessing import time +import typing import pytest import portalocker @@ -246,16 +248,74 @@ def shared_lock_fail(filename, **kwargs): return True -def test_shared_processes(tmpfile): - # Force spawning the process so we don't accidently inherit the lock - # I'm not a 100% certain this will work correctly unfortunately... there - # is some potential for breaking other tests - multiprocessing.set_start_method('spawn') +def exclusive_lock(filename, **kwargs): + with portalocker.Lock( + filename, + timeout=0.1, + fail_when_locked=False, + flags=portalocker.LockFlags.EXCLUSIVE | + portalocker.LockFlags.NON_BLOCKING, + ): + time.sleep(0.2) + return True + + +@dataclasses.dataclass(order=True) +class LockResult: + exception_class: typing.Union[typing.Type[Exception], None] = None + exception_message: typing.Union[str, None] = None + exception_repr: typing.Union[str, None] = None + + +def lock( + filename: str, + fail_when_locked: bool, + flags: portalocker.LockFlags +) -> LockResult: + # Returns a case of True, False or FileNotFound + # https://thedailywtf.com/articles/what_is_truth_0x3f_ + # But seriously, the exception properties cannot be safely pickled so we + # only return string representations of the exception properties + try: + with portalocker.Lock( + filename, + timeout=0.1, + fail_when_locked=fail_when_locked, + flags=flags, + ): + time.sleep(0.2) + return LockResult() + + except Exception as exception: + # The exceptions cannot be pickled so we cannot return them through + # multiprocessing + return LockResult( + type(exception), + str(exception), + repr(exception), + ) + + +@pytest.mark.parametrize('fail_when_locked', [True, False]) +def test_shared_processes(tmpfile, fail_when_locked): + flags = portalocker.LockFlags.SHARED | portalocker.LockFlags.NON_BLOCKING with multiprocessing.Pool(processes=2) as pool: - for result in pool.imap_unordered(shared_lock, 3 * [tmpfile]): - assert result is True + args = tmpfile, fail_when_locked, flags + results = pool.starmap_async(lock, 3 * [args]) + + for result in results.get(timeout=1): + assert result == LockResult() + + +@pytest.mark.parametrize('fail_when_locked', [True, False]) +def test_exclusive_processes(tmpfile, fail_when_locked): + flags = portalocker.LockFlags.EXCLUSIVE | portalocker.LockFlags.NON_BLOCKING with multiprocessing.Pool(processes=2) as pool: - for result in pool.imap_unordered(shared_lock_fail, 3 * [tmpfile]): - assert result is True + # filename, fail_when_locked, flags + args = tmpfile, fail_when_locked, flags + a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=1) + + assert a.exception_class is None + assert issubclass(b.exception_class, portalocker.LockException) From 89e2e32a15ff37f42c447cbbedbd8f46a195b1da Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 5 Jul 2022 01:56:02 +0200 Subject: [PATCH 108/225] Rewrote locking systems for Windows in an attempt to fix #64 --- msvcrt.pyi | 7 -- portalocker/portalocker.py | 136 +++++++++++++------------------------ 2 files changed, 47 insertions(+), 96 deletions(-) delete mode 100644 msvcrt.pyi diff --git a/msvcrt.pyi b/msvcrt.pyi deleted file mode 100644 index 73c309a..0000000 --- a/msvcrt.pyi +++ /dev/null @@ -1,7 +0,0 @@ -LK_LOCK: int -LK_NBLCK: int -LK_NBRLCK: int -LK_RLCK: int -LK_UNLCK: int - -def locking(file: int, mode: int, lock_length: int) -> int: ... diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 6bc7447..bf9b3c5 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -3,85 +3,54 @@ import typing +import win32con +import win32file + from . import constants from . import exceptions if os.name == 'nt': # pragma: no cover - import win32con - import win32file import pywintypes import winerror import msvcrt __overlapped = pywintypes.OVERLAPPED() - if sys.version_info.major == 2: - lock_length = -1 - else: - lock_length = int(2**31 - 1) - def lock(file_: typing.IO, flags: constants.LockFlags): - if flags & constants.LockFlags.SHARED: - if sys.version_info.major == 2: - if flags & constants.LockFlags.NON_BLOCKING: - mode = win32con.LOCKFILE_FAIL_IMMEDIATELY - else: - mode = 0 + mode = 0 + if flags & constants.LockFlags.NON_BLOCKING: + mode |= win32con.LOCKFILE_FAIL_IMMEDIATELY - else: - if flags & constants.LockFlags.NON_BLOCKING: - mode = msvcrt.LK_NBRLCK - else: - mode = msvcrt.LK_RLCK + if flags & constants.LockFlags.EXCLUSIVE: + mode |= win32con.LOCKFILE_EXCLUSIVE_LOCK - # is there any reason not to reuse the following structure? - hfile = win32file._get_osfhandle(file_.fileno()) - try: - win32file.LockFileEx(hfile, mode, 0, -0x10000, __overlapped) - except pywintypes.error as exc_value: - # error: (33, 'LockFileEx', 'The process cannot access the file - # because another process has locked a portion of the file.') - if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION: - raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, - exc_value.strerror, - fh=file_) - else: - # Q: Are there exceptions/codes we should be dealing with - # here? - raise - else: - if flags & constants.LockFlags.NON_BLOCKING: - mode = msvcrt.LK_NBLCK - else: - mode = msvcrt.LK_LOCK + # Save the old position so we can go back to that position but + # still lock from the beginning of the file + savepos = file_.tell() + if savepos: + file_.seek(0) - # windows locks byte ranges, so make sure to lock from file start - try: - savepos = file_.tell() - if savepos: - # [ ] test exclusive lock fails on seek here - # [ ] test if shared lock passes this point - file_.seek(0) - # [x] check if 0 param locks entire file (not documented in - # Python) - # [x] fails with "IOError: [Errno 13] Permission denied", - # but -1 seems to do the trick - - try: - msvcrt.locking(file_.fileno(), mode, lock_length) - except IOError as exc_value: - # [ ] be more specific here - raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, - exc_value.strerror, - fh=file_) - finally: - if savepos: - file_.seek(savepos) - except IOError as exc_value: - raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, exc_value.strerror, + os_fh = msvcrt.get_osfhandle(file_.fileno()) + try: + win32file.LockFileEx(os_fh, mode, 0, -0x10000, __overlapped) + except pywintypes.error as exc_value: + # import winerror + # errors = {k for k, v in winerror.__dict__.items() if v == exc_value.winerror} + # print(errors) + + # error: (33, 'LockFileEx', 'The process cannot access the file + # because another process has locked a portion of the file.') + if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION: + raise exceptions.AlreadyLocked( + exceptions.LockException.LOCK_FAILED, + exc_value.strerror, fh=file_) + else: + # Q: Are there exceptions/codes we should be dealing with + # here? + raise + finally: + if savepos: + file_.seek(savepos) def unlock(file_: typing.IO): try: @@ -89,33 +58,22 @@ def unlock(file_: typing.IO): if savepos: file_.seek(0) + os_fh = msvcrt.get_osfhandle(file_.fileno()) try: - msvcrt.locking(file_.fileno(), constants.LockFlags.UNBLOCK, - lock_length) - except IOError as exc: + win32file.UnlockFileEx( + os_fh, 0, -0x10000, __overlapped) + except pywintypes.error as exc: exception = exc - if exc.strerror == 'Permission denied': - hfile = win32file._get_osfhandle(file_.fileno()) - try: - win32file.UnlockFileEx( - hfile, 0, -0x10000, __overlapped) - except pywintypes.error as exc: - exception = exc - if exc.winerror == winerror.ERROR_NOT_LOCKED: - # error: (158, 'UnlockFileEx', - # 'The segment is already unlocked.') - # To match the 'posix' implementation, silently - # ignore this error - pass - else: - # Q: Are there exceptions/codes we should be - # dealing with here? - raise + if exc.winerror == winerror.ERROR_NOT_LOCKED: + # error: (158, 'UnlockFileEx', + # 'The segment is already unlocked.') + # To match the 'posix' implementation, silently + # ignore this error + pass else: - raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, - exception.strerror, - fh=file_) + # Q: Are there exceptions/codes we should be + # dealing with here? + raise finally: if savepos: file_.seek(savepos) From 75cdf6ef7bf3b0bd8166db1cd3861587642aff5f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 5 Jul 2022 02:14:06 +0200 Subject: [PATCH 109/225] PyCharm can be stupid at times --- portalocker/portalocker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index bf9b3c5..621704c 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -1,18 +1,18 @@ import os -import sys import typing -import win32con -import win32file from . import constants from . import exceptions + if os.name == 'nt': # pragma: no cover + import msvcrt import pywintypes + import win32con + import win32file import winerror - import msvcrt __overlapped = pywintypes.OVERLAPPED() def lock(file_: typing.IO, flags: constants.LockFlags): From 1e9cb83150cb1dec44a345302859dfcd7197df30 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 5 Jul 2022 02:18:01 +0200 Subject: [PATCH 110/225] Removing legacy testing integration into setup.py and added pytest-timeout so tests can't run too long --- portalocker_tests/tests.py | 7 +++++-- pytest.ini | 1 + setup.py | 20 +------------------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 1c3fcfd..dfd919b 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -317,5 +317,8 @@ def test_exclusive_processes(tmpfile, fail_when_locked): args = tmpfile, fail_when_locked, flags a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=1) - assert a.exception_class is None - assert issubclass(b.exception_class, portalocker.LockException) + assert not a.exception_class or not b.exception_class + assert issubclass( + a.exception_class or b.exception_class, + portalocker.LockException + ) diff --git a/pytest.ini b/pytest.ini index 1ef7c1b..a9aa19e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -14,3 +14,4 @@ flake8-ignore = *.py W391 docs/*.py ALL +timeout = 10 diff --git a/setup.py b/setup.py index 974e0aa..bd15ad6 100644 --- a/setup.py +++ b/setup.py @@ -2,12 +2,9 @@ import os import re -import sys import typing import setuptools -from setuptools import __version__ as setuptools_version -from setuptools.command.test import test as TestCommand # To prevent importing about and thereby breaking the coverage info we use this # exec hack @@ -18,27 +15,13 @@ tests_require = [ 'pytest>=5.4.1', 'pytest-cov>=2.8.1', + 'pytest-timeout>=2.1.0', 'sphinx>=3.0.3', 'pytest-mypy>=0.8.0', 'redis', ] -class PyTest(TestCommand): - user_options = [('pytest-args=', 'a', 'Arguments to pass to pytest')] - - def initialize_options(self): - TestCommand.initialize_options(self) - self.pytest_args = '' - - def run_tests(self): - import shlex - # import here, cause outside the eggs aren't loaded - import pytest - errno = pytest.main(shlex.split(self.pytest_args)) - sys.exit(errno) - - class Combine(setuptools.Command): description = 'Build single combined portalocker file' relative_import_re = re.compile(r'^from \. import (?P.+)$', @@ -126,7 +109,6 @@ def run(self): platforms=['any'], cmdclass={ 'combine': Combine, - 'test': PyTest, }, install_requires=[ # Due to CVE-2021-32559 updating the pywin32 requirement From 213b31d12f30dd5ade9673815d6a975653fe5604 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 5 Jul 2022 02:27:38 +0200 Subject: [PATCH 111/225] Code and debug statement cleanup --- portalocker/portalocker.py | 22 +++++++++++----------- portalocker_tests/conftest.py | 4 +--- portalocker_tests/tests.py | 18 +++++++++--------- pytest.ini | 4 ---- setup.cfg | 5 +++++ 5 files changed, 26 insertions(+), 27 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 621704c..3be4d66 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -2,19 +2,19 @@ import typing - from . import constants from . import exceptions - if os.name == 'nt': # pragma: no cover import msvcrt import pywintypes import win32con import win32file import winerror + __overlapped = pywintypes.OVERLAPPED() + def lock(file_: typing.IO, flags: constants.LockFlags): mode = 0 if flags & constants.LockFlags.NON_BLOCKING: @@ -33,17 +33,14 @@ def lock(file_: typing.IO, flags: constants.LockFlags): try: win32file.LockFileEx(os_fh, mode, 0, -0x10000, __overlapped) except pywintypes.error as exc_value: - # import winerror - # errors = {k for k, v in winerror.__dict__.items() if v == exc_value.winerror} - # print(errors) - # error: (33, 'LockFileEx', 'The process cannot access the file # because another process has locked a portion of the file.') if exc_value.winerror == winerror.ERROR_LOCK_VIOLATION: raise exceptions.AlreadyLocked( exceptions.LockException.LOCK_FAILED, exc_value.strerror, - fh=file_) + fh=file_ + ) else: # Q: Are there exceptions/codes we should be dealing with # here? @@ -52,6 +49,7 @@ def lock(file_: typing.IO, flags: constants.LockFlags): if savepos: file_.seek(savepos) + def unlock(file_: typing.IO): try: savepos = file_.tell() @@ -61,9 +59,9 @@ def unlock(file_: typing.IO): os_fh = msvcrt.get_osfhandle(file_.fileno()) try: win32file.UnlockFileEx( - os_fh, 0, -0x10000, __overlapped) + os_fh, 0, -0x10000, __overlapped + ) except pywintypes.error as exc: - exception = exc if exc.winerror == winerror.ERROR_NOT_LOCKED: # error: (158, 'UnlockFileEx', # 'The segment is already unlocked.') @@ -80,11 +78,13 @@ def unlock(file_: typing.IO): except IOError as exc: raise exceptions.LockException( exceptions.LockException.LOCK_FAILED, exc.strerror, - fh=file_) + fh=file_ + ) elif os.name == 'posix': # pragma: no cover import fcntl + def lock(file_: typing.IO, flags: constants.LockFlags): locking_exceptions = IOError, try: # pragma: no cover @@ -99,9 +99,9 @@ def lock(file_: typing.IO, flags: constants.LockFlags): # every IO error raise exceptions.LockException(exc_value, fh=file_) + def unlock(file_: typing.IO, ): fcntl.flock(file_.fileno(), constants.LockFlags.UNBLOCK) else: # pragma: no cover raise RuntimeError('PortaLocker only defined for nt and posix platforms') - diff --git a/portalocker_tests/conftest.py b/portalocker_tests/conftest.py index 52d31dd..cf59e2b 100644 --- a/portalocker_tests/conftest.py +++ b/portalocker_tests/conftest.py @@ -1,9 +1,7 @@ -import multiprocessing - -import py import logging import pytest import random +import multiprocessing logger = logging.getLogger(__name__) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index dfd919b..f01a6bd 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -9,6 +9,7 @@ import pytest import portalocker from portalocker import utils +from portalocker import LockFlags def test_exceptions(tmpfile): @@ -215,7 +216,7 @@ def test_shared(tmpfile): def test_blocking_timeout(tmpfile): - flags = portalocker.LockFlags.SHARED + flags = LockFlags.SHARED with pytest.warns(UserWarning): with portalocker.Lock(tmpfile, timeout=5, flags=flags): @@ -231,7 +232,7 @@ def shared_lock(filename, **kwargs): filename, timeout=0.1, fail_when_locked=False, - flags=portalocker.LockFlags.SHARED | portalocker.LockFlags.NON_BLOCKING, + flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, ): time.sleep(0.2) return True @@ -242,7 +243,7 @@ def shared_lock_fail(filename, **kwargs): filename, timeout=0.1, fail_when_locked=True, - flags=portalocker.LockFlags.SHARED | portalocker.LockFlags.NON_BLOCKING, + flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, ): time.sleep(0.2) return True @@ -253,8 +254,7 @@ def exclusive_lock(filename, **kwargs): filename, timeout=0.1, fail_when_locked=False, - flags=portalocker.LockFlags.EXCLUSIVE | - portalocker.LockFlags.NON_BLOCKING, + flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, ): time.sleep(0.2) return True @@ -270,7 +270,7 @@ class LockResult: def lock( filename: str, fail_when_locked: bool, - flags: portalocker.LockFlags + flags: LockFlags ) -> LockResult: # Returns a case of True, False or FileNotFound # https://thedailywtf.com/articles/what_is_truth_0x3f_ @@ -298,7 +298,7 @@ def lock( @pytest.mark.parametrize('fail_when_locked', [True, False]) def test_shared_processes(tmpfile, fail_when_locked): - flags = portalocker.LockFlags.SHARED | portalocker.LockFlags.NON_BLOCKING + flags = LockFlags.SHARED | LockFlags.NON_BLOCKING with multiprocessing.Pool(processes=2) as pool: args = tmpfile, fail_when_locked, flags @@ -310,14 +310,14 @@ def test_shared_processes(tmpfile, fail_when_locked): @pytest.mark.parametrize('fail_when_locked', [True, False]) def test_exclusive_processes(tmpfile, fail_when_locked): - flags = portalocker.LockFlags.EXCLUSIVE | portalocker.LockFlags.NON_BLOCKING + flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING with multiprocessing.Pool(processes=2) as pool: # filename, fail_when_locked, flags args = tmpfile, fail_when_locked, flags a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=1) - assert not a.exception_class or not b.exception_class + assert not a.exception_class or not b.exception_class assert issubclass( a.exception_class or b.exception_class, portalocker.LockException diff --git a/pytest.ini b/pytest.ini index a9aa19e..f87a508 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,8 +10,4 @@ addopts = --cov-report term-missing --cov-report html -flake8-ignore = - *.py W391 - docs/*.py ALL - timeout = 10 diff --git a/setup.cfg b/setup.cfg index 68d4f40..70e8910 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,8 @@ description_file = README.rst [bdist_wheel] universal = 1 + +[flake8] +ignore = + *.py W391,E303 + docs/*.py ALL From 7951ed5c60af46fb0ec0b8dd6c7cef6f5d993f89 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 5 Jul 2022 02:35:20 +0200 Subject: [PATCH 112/225] Github actions is slow... increase the timeouts --- portalocker_tests/tests.py | 4 ++-- pytest.ini | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index f01a6bd..ff9e8b9 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -304,7 +304,7 @@ def test_shared_processes(tmpfile, fail_when_locked): args = tmpfile, fail_when_locked, flags results = pool.starmap_async(lock, 3 * [args]) - for result in results.get(timeout=1): + for result in results.get(timeout=2): assert result == LockResult() @@ -315,7 +315,7 @@ def test_exclusive_processes(tmpfile, fail_when_locked): with multiprocessing.Pool(processes=2) as pool: # filename, fail_when_locked, flags args = tmpfile, fail_when_locked, flags - a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=1) + a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=2) assert not a.exception_class or not b.exception_class assert issubclass( diff --git a/pytest.ini b/pytest.ini index f87a508..64f7177 100644 --- a/pytest.ini +++ b/pytest.ini @@ -10,4 +10,4 @@ addopts = --cov-report term-missing --cov-report html -timeout = 10 +timeout = 20 From d80b55b4efe90b938c74e9b3201e124acbed9f6f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 5 Jul 2022 02:41:04 +0200 Subject: [PATCH 113/225] Made the tox flake8 task use the config instead of overriding it... sometimes I just make my life hard --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 3d2d080..9f9b3f5 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ commands = mypy {toxinidir}/portalocker [testenv:flake8] basepython = python3 deps = flake8 -commands = flake8 --ignore=W391 {toxinidir}/portalocker {toxinidir}/portalocker_tests +commands = flake8 {toxinidir}/portalocker {toxinidir}/portalocker_tests [testenv:docs] basepython = python3 @@ -36,4 +36,3 @@ commands = sphinx-apidoc -e -o docs/ portalocker rm -f docs/modules.rst sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} - From 45aad9843584706366d525a04f0f2a807504dbdb Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 5 Jul 2022 02:48:24 +0200 Subject: [PATCH 114/225] windows tests on github actions can be _really_ slow --- portalocker_tests/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index ff9e8b9..bd3ba51 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -302,9 +302,9 @@ def test_shared_processes(tmpfile, fail_when_locked): with multiprocessing.Pool(processes=2) as pool: args = tmpfile, fail_when_locked, flags - results = pool.starmap_async(lock, 3 * [args]) + results = pool.starmap_async(lock, 2 * [args]) - for result in results.get(timeout=2): + for result in results.get(timeout=3): assert result == LockResult() @@ -315,7 +315,7 @@ def test_exclusive_processes(tmpfile, fail_when_locked): with multiprocessing.Pool(processes=2) as pool: # filename, fail_when_locked, flags args = tmpfile, fail_when_locked, flags - a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=2) + a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=3) assert not a.exception_class or not b.exception_class assert issubclass( From ec5a037ea332f4b191cbcf0d0e83775e51990eef Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 9 Jul 2022 12:29:24 +0200 Subject: [PATCH 115/225] Incrementing version to v2.5.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index ad60e5f..0762d9d 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.4.0' +__version__ = '2.5.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 13699df..03eaee9 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.4.0' +__version__ = '2.5.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 0bfa4b171cad3f565cab1922f933f17b7e86a8d5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 9 Jul 2022 16:04:18 +0200 Subject: [PATCH 116/225] Added clear warning when locking in non-blocking mode without specifying either EXCLUSIVE or SHARED mode. Fixes #77 --- portalocker/portalocker.py | 23 ++++++++++++++++++----- portalocker_tests/tests.py | 6 ++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 3be4d66..91a6666 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -5,6 +5,12 @@ from . import constants from . import exceptions + +# Alias for readability. Due to import recursion issues we cannot do: +# from .constants import LockFlags +LockFlags = constants.LockFlags + + if os.name == 'nt': # pragma: no cover import msvcrt import pywintypes @@ -15,12 +21,12 @@ __overlapped = pywintypes.OVERLAPPED() - def lock(file_: typing.IO, flags: constants.LockFlags): + def lock(file_: typing.IO, flags: LockFlags): mode = 0 - if flags & constants.LockFlags.NON_BLOCKING: + if flags & LockFlags.NON_BLOCKING: mode |= win32con.LOCKFILE_FAIL_IMMEDIATELY - if flags & constants.LockFlags.EXCLUSIVE: + if flags & LockFlags.EXCLUSIVE: mode |= win32con.LOCKFILE_EXCLUSIVE_LOCK # Save the old position so we can go back to that position but @@ -85,13 +91,20 @@ def unlock(file_: typing.IO): import fcntl - def lock(file_: typing.IO, flags: constants.LockFlags): + def lock(file_: typing.IO, flags: LockFlags): locking_exceptions = IOError, try: # pragma: no cover locking_exceptions += BlockingIOError, # type: ignore except NameError: # pragma: no cover pass + # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled results + # in an error + if ((flags & LockFlags.NON_BLOCKING) and \ + not flags & (LockFlags.SHARED | LockFlags.EXCLUSIVE)): + raise RuntimeError('When locking in non-blocking mode the SHARED ' + 'or EXCLUSIVE flag must be specified as well') + try: fcntl.flock(file_.fileno(), flags) except locking_exceptions as exc_value: @@ -101,7 +114,7 @@ def lock(file_: typing.IO, flags: constants.LockFlags): def unlock(file_: typing.IO, ): - fcntl.flock(file_.fileno(), constants.LockFlags.UNBLOCK) + fcntl.flock(file_.fileno(), LockFlags.UNBLOCK) else: # pragma: no cover raise RuntimeError('PortaLocker only defined for nt and posix platforms') diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index bd3ba51..9c141ff 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -227,6 +227,12 @@ def test_blocking_timeout(tmpfile): lock.acquire(timeout=5) +def test_nonblocking(tmpfile): + with open(tmpfile, 'w') as fh: + with pytest.raises(RuntimeError): + portalocker.lock(fh, LockFlags.NON_BLOCKING) + + def shared_lock(filename, **kwargs): with portalocker.Lock( filename, From f8adb504d91ba36cda185b29bbc5b6a4df533981 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 9 Jul 2022 16:07:32 +0200 Subject: [PATCH 117/225] Updated Python 2 example to Python 3 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b07d74d..9b363c2 100644 --- a/README.rst +++ b/README.rst @@ -117,7 +117,7 @@ To make sure your cache generation scripts don't race, use the `Lock` class: >>> import portalocker >>> with portalocker.Lock('somefile', timeout=1) as fh: -... print >>fh, 'writing some stuff to my cache...' +... print('writing some stuff to my cache...', file=fh) To customize the opening and locking a manual approach is also possible: From 9b7bfab394a6ab8b3ad787b1d19eaad27a17851e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 9 Jul 2022 17:00:21 +0200 Subject: [PATCH 118/225] skip linux/unix/mac tests on windows --- portalocker_tests/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 9c141ff..f79678f 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -1,6 +1,7 @@ from __future__ import print_function from __future__ import with_statement +import os import dataclasses import multiprocessing import time @@ -227,6 +228,7 @@ def test_blocking_timeout(tmpfile): lock.acquire(timeout=5) +pytest.mark.skipif(os.name == 'nt') def test_nonblocking(tmpfile): with open(tmpfile, 'w') as fh: with pytest.raises(RuntimeError): From 41e059c396b6444bcf2bb5927cbe42c97ec23489 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 9 Jul 2022 17:05:08 +0200 Subject: [PATCH 119/225] flake8 compliance --- portalocker/portalocker.py | 4 ++-- portalocker_tests/tests.py | 2 +- setup.cfg | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 91a6666..4bae5e3 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -100,8 +100,8 @@ def lock(file_: typing.IO, flags: LockFlags): # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled results # in an error - if ((flags & LockFlags.NON_BLOCKING) and \ - not flags & (LockFlags.SHARED | LockFlags.EXCLUSIVE)): + if ((flags & LockFlags.NON_BLOCKING) + and not flags & (LockFlags.SHARED | LockFlags.EXCLUSIVE)): raise RuntimeError('When locking in non-blocking mode the SHARED ' 'or EXCLUSIVE flag must be specified as well') diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index f79678f..fe92d93 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -228,7 +228,7 @@ def test_blocking_timeout(tmpfile): lock.acquire(timeout=5) -pytest.mark.skipif(os.name == 'nt') +@pytest.mark.skipif(os.name == 'nt') def test_nonblocking(tmpfile): with open(tmpfile, 'w') as fh: with pytest.raises(RuntimeError): diff --git a/setup.cfg b/setup.cfg index 70e8910..9f4fba1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,5 +6,5 @@ universal = 1 [flake8] ignore = - *.py W391,E303 + *.py W391,E303,W503 docs/*.py ALL From fccafdd66eda3d7752067785bb28f9387fe74d48 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 9 Jul 2022 17:08:29 +0200 Subject: [PATCH 120/225] skip linux/unix/mac tests on windows --- portalocker_tests/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index fe92d93..e41cb03 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -228,7 +228,8 @@ def test_blocking_timeout(tmpfile): lock.acquire(timeout=5) -@pytest.mark.skipif(os.name == 'nt') +@pytest.mark.skipif(os.name == 'nt', + reason='Windows uses an entirely different lockmechanism') def test_nonblocking(tmpfile): with open(tmpfile, 'w') as fh: with pytest.raises(RuntimeError): From 22d7639458093cecb858a6303a1bacd6aff31565 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 9 Jul 2022 17:17:48 +0200 Subject: [PATCH 121/225] Incrementing version to v2.5.1 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 0762d9d..17bf0ed 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.5.0' +__version__ = '2.5.1' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 03eaee9..c0edcc1 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.5.0' +__version__ = '2.5.1' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 35b500bb6fecdffd372b0022e417baa210a88462 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 2 Aug 2022 00:30:37 +0200 Subject: [PATCH 122/225] Updated license to 3-clause BSD License. Fixes #79 The 3-clause BSD license is effectively the same as the old license but the old license had invalid references to the Python project instead of this project. --- LICENSE | 49 ++++++------------------------------------------- 1 file changed, 6 insertions(+), 43 deletions(-) diff --git a/LICENSE b/LICENSE index adb8038..b638bda 100644 --- a/LICENSE +++ b/LICENSE @@ -1,48 +1,11 @@ -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- +Copyright 2022 Rick van Hattem -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 -Python Software Foundation; All Rights Reserved" are retained in Python alone or -in any derivative version prepared by Licensee. +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 26a1522520f0d8b656db210117fcdcbf6c526ede Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 18 Oct 2022 17:02:30 +0200 Subject: [PATCH 123/225] testing fix for #80 --- portalocker/portalocker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 4bae5e3..f25269f 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -35,7 +35,7 @@ def lock(file_: typing.IO, flags: LockFlags): if savepos: file_.seek(0) - os_fh = msvcrt.get_osfhandle(file_.fileno()) + os_fh = msvcrt.get_osfhandle(file_) try: win32file.LockFileEx(os_fh, mode, 0, -0x10000, __overlapped) except pywintypes.error as exc_value: @@ -62,7 +62,7 @@ def unlock(file_: typing.IO): if savepos: file_.seek(0) - os_fh = msvcrt.get_osfhandle(file_.fileno()) + os_fh = msvcrt.get_osfhandle(file_) try: win32file.UnlockFileEx( os_fh, 0, -0x10000, __overlapped @@ -106,7 +106,7 @@ def lock(file_: typing.IO, flags: LockFlags): 'or EXCLUSIVE flag must be specified as well') try: - fcntl.flock(file_.fileno(), flags) + fcntl.flock(file_, flags) except locking_exceptions as exc_value: # The exception code varies on different systems so we'll catch # every IO error From 568d213bae9fa2ea8861117f3a83507865fff1df Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 18 Oct 2022 17:16:12 +0200 Subject: [PATCH 124/225] Added fileno() support. Fixes #80 on posix systems --- portalocker/portalocker.py | 4 ++-- portalocker_tests/tests.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index f25269f..72260a7 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -35,7 +35,7 @@ def lock(file_: typing.IO, flags: LockFlags): if savepos: file_.seek(0) - os_fh = msvcrt.get_osfhandle(file_) + os_fh = msvcrt.get_osfhandle(file_.fileno()) try: win32file.LockFileEx(os_fh, mode, 0, -0x10000, __overlapped) except pywintypes.error as exc_value: @@ -62,7 +62,7 @@ def unlock(file_: typing.IO): if savepos: file_.seek(0) - os_fh = msvcrt.get_osfhandle(file_) + os_fh = msvcrt.get_osfhandle(file_.fileno()) try: win32file.UnlockFileEx( os_fh, 0, -0x10000, __overlapped diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index e41cb03..7a00405 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -331,3 +331,27 @@ def test_exclusive_processes(tmpfile, fail_when_locked): a.exception_class or b.exception_class, portalocker.LockException ) + + +@pytest.mark.skipif( + os.name == 'nt', + reason='Locking on Windows requires a file object', +) +def test_lock_fileno(tmpfile): + # Open the file 2 times + a = open(tmpfile, 'a') + b = open(tmpfile, 'a') + + # Lock exclusive non-blocking + flags = LockFlags.SHARED | LockFlags.NON_BLOCKING + + # First lock file a + portalocker.lock(a, flags) + + # Now see if we can lock using fileno() + portalocker.lock(b.fileno(), flags) + + # Cleanup + a.close() + b.close() + From faf78f15528edf05efadb885bb81ca32372a6496 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 18 Oct 2022 17:19:07 +0200 Subject: [PATCH 125/225] Incrementing version to v2.6.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 17bf0ed..2264351 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.5.1' +__version__ = '2.6.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index c0edcc1..65ddb4a 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.5.1' +__version__ = '2.6.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From b5c5db11f2dda13e83ffdd03ad60b95abc6d29f2 Mon Sep 17 00:00:00 2001 From: Alexander Shadchin Date: Wed, 2 Nov 2022 11:16:17 +0300 Subject: [PATCH 126/225] Fix license Missed change after #79 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bd15ad6..6225b75 100644 --- a/setup.py +++ b/setup.py @@ -101,7 +101,7 @@ def run(self): author=about['__author__'], author_email=about['__email__'], url=about['__url__'], - license='PSF', + license='BSD-3-Clause', package_data=dict(portalocker=['py.typed', 'msvcrt.pyi']), packages=setuptools.find_packages(exclude=[ 'examples', 'portalocker_tests']), From e8939e80788d2743e5b02eca77ef9e4cbbb53ee2 Mon Sep 17 00:00:00 2001 From: Flavien Solt Date: Thu, 12 Jan 2023 11:15:14 +0100 Subject: [PATCH 127/225] BoundedSemaphore: do not fail if fail_when_locked is False Hi there! It is misleading that a BoundedSemaphore fails regardless of `fail_when_locked`. This PR proposes to fix this. Thanks! Flavien --- portalocker/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index d7c94ca..493d8b5 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -425,7 +425,9 @@ def acquire( if self.try_lock(filenames): # pragma: no branch return self.lock # pragma: no cover - raise exceptions.AlreadyLocked() + if fail_when_locked: + raise exceptions.AlreadyLocked() + return None def try_lock(self, filenames: typing.Sequence[Filename]) -> bool: filename: Filename From a3b717f47c8cc4e15b5ab9da12945c85a17679db Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 18 Jan 2023 21:31:25 +0100 Subject: [PATCH 128/225] Added `fail_when_locked=False` support to `BoundedSemaphore` thanks to @flaviens --- portalocker/utils.py | 17 ++++++++++++----- portalocker_tests/test_semaphore.py | 6 ++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 493d8b5..c4fc11a 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -386,15 +386,20 @@ def __init__( name: str = 'bounded_semaphore', filename_pattern: str = '{name}.{number:02d}.lock', directory: str = tempfile.gettempdir(), - timeout=DEFAULT_TIMEOUT, - check_interval=DEFAULT_CHECK_INTERVAL): + timeout: typing.Optional[float] = DEFAULT_TIMEOUT, + check_interval: typing.Optional[float] = DEFAULT_CHECK_INTERVAL, + fail_when_locked: typing.Optional[bool] = True, + ): self.maximum = maximum self.name = name self.filename_pattern = filename_pattern self.directory = directory self.lock: typing.Optional[Lock] = None - self.timeout = timeout - self.check_interval = check_interval + super().__init__( + timeout=timeout, + check_interval=check_interval, + fail_when_locked=fail_when_locked, + ) def get_filenames(self) -> typing.Sequence[pathlib.Path]: return [self.get_filename(n) for n in range(self.maximum)] @@ -425,8 +430,10 @@ def acquire( if self.try_lock(filenames): # pragma: no branch return self.lock # pragma: no cover + fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) if fail_when_locked: raise exceptions.AlreadyLocked() + return None def try_lock(self, filenames: typing.Sequence[Filename]) -> bool: @@ -439,7 +446,7 @@ def try_lock(self, filenames: typing.Sequence[Filename]) -> bool: logger.debug('locked %r', filename) return True except exceptions.AlreadyLocked: - pass + self.lock = None return False diff --git a/portalocker_tests/test_semaphore.py b/portalocker_tests/test_semaphore.py index b0c57aa..c6847ab 100644 --- a/portalocker_tests/test_semaphore.py +++ b/portalocker_tests/test_semaphore.py @@ -20,3 +20,9 @@ def test_bounded_semaphore(timeout, check_interval, monkeypatch): semaphore_b.acquire() with pytest.raises(portalocker.AlreadyLocked): semaphore_c.acquire(check_interval=check_interval, timeout=timeout) + + semaphore_c.acquire( + check_interval=check_interval, + timeout=timeout, + fail_when_locked=False, + ) From 789f69d8ab77066f04fa35885948811ac148fe0e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 18 Jan 2023 21:39:07 +0100 Subject: [PATCH 129/225] fixed issue with new flake8 release --- setup.cfg | 5 +++-- tox.ini | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9f4fba1..9960540 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,5 +6,6 @@ universal = 1 [flake8] ignore = - *.py W391,E303,W503 - docs/*.py ALL + W391,E303,W503 +exclude = + docs/*.py diff --git a/tox.ini b/tox.ini index 9f9b3f5..6437a37 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ commands = mypy {toxinidir}/portalocker [testenv:flake8] basepython = python3 -deps = flake8 +deps = flake8>=6.0.0 commands = flake8 {toxinidir}/portalocker {toxinidir}/portalocker_tests [testenv:docs] From 19c2dac6743a6472a22f37337af98ad5347df76a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 19 Jan 2023 00:25:47 +0100 Subject: [PATCH 130/225] Disabling warning as error for sphinx in tox. Somehow the docs build fails in tox but not outside of it --- docs/conf.py | 4 ++-- portalocker/constants.py | 36 +++++++++++++++++++++++------------- setup.py | 2 +- tox.ini | 5 ++++- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 5680b57..07e4caf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -87,7 +87,7 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +default_role = 'py:obj' # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True @@ -351,4 +351,4 @@ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {'http://docs.python.org/3/': None} diff --git a/portalocker/constants.py b/portalocker/constants.py index 2c13ece..f683a45 100644 --- a/portalocker/constants.py +++ b/portalocker/constants.py @@ -13,7 +13,7 @@ Manually unlock, only needed internally - `UNBLOCK` unlock -''' +'''.strip() import enum import os @@ -22,25 +22,35 @@ if os.name == 'nt': # pragma: no cover import msvcrt - LOCK_EX = 0x1 #: exclusive lock - LOCK_SH = 0x2 #: shared lock - LOCK_NB = 0x4 #: non-blocking + #: exclusive lock + LOCK_EX = 0x1 + #: shared lock + LOCK_SH = 0x2 + #: non-blocking + LOCK_NB = 0x4 LOCK_UN = msvcrt.LK_UNLCK #: unlock elif os.name == 'posix': # pragma: no cover import fcntl - - LOCK_EX = fcntl.LOCK_EX #: exclusive lock - LOCK_SH = fcntl.LOCK_SH #: shared lock - LOCK_NB = fcntl.LOCK_NB #: non-blocking - LOCK_UN = fcntl.LOCK_UN #: unlock + #: exclusive lock + LOCK_EX = fcntl.LOCK_EX + #: shared lock + LOCK_SH = fcntl.LOCK_SH + #: non-blocking + LOCK_NB = fcntl.LOCK_NB + #: unlock + LOCK_UN = fcntl.LOCK_UN else: # pragma: no cover raise RuntimeError('PortaLocker only defined for nt and posix platforms') class LockFlags(enum.IntFlag): - EXCLUSIVE = LOCK_EX #: exclusive lock - SHARED = LOCK_SH #: shared lock - NON_BLOCKING = LOCK_NB #: non-blocking - UNBLOCK = LOCK_UN #: unlock + #: exclusive lock + EXCLUSIVE = LOCK_EX + #: shared lock + SHARED = LOCK_SH + #: non-blocking + NON_BLOCKING = LOCK_NB + #: unlock + UNBLOCK = LOCK_UN diff --git a/setup.py b/setup.py index 6225b75..5c7bfd6 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ 'pytest>=5.4.1', 'pytest-cov>=2.8.1', 'pytest-timeout>=2.1.0', - 'sphinx>=3.0.3', + 'sphinx>=6.0.0', 'pytest-mypy>=0.8.0', 'redis', ] diff --git a/tox.ini b/tox.ini index 6437a37..b9960b0 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,9 @@ commands = flake8 {toxinidir}/portalocker {toxinidir}/portalocker_tests [testenv:docs] basepython = python3 deps = -r{toxinidir}/docs/requirements.txt +allowlist_externals = + rm + mkdir whitelist_externals = rm cd @@ -35,4 +38,4 @@ commands = mkdir -p docs/_static sphinx-apidoc -e -o docs/ portalocker rm -f docs/modules.rst - sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} + sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html {posargs} From 2b24ca24c97d3394c4ff8ededd9a1f1ac5dd9d44 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 19 Jan 2023 00:32:34 +0100 Subject: [PATCH 131/225] silly editor inserting tabs... --- portalocker/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portalocker/constants.py b/portalocker/constants.py index f683a45..dce44f6 100644 --- a/portalocker/constants.py +++ b/portalocker/constants.py @@ -22,7 +22,7 @@ if os.name == 'nt': # pragma: no cover import msvcrt - #: exclusive lock + #: exclusive lock LOCK_EX = 0x1 #: shared lock LOCK_SH = 0x2 From 821ec365f05abf7c2afbff439348042a617422bf Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 19 Jan 2023 00:35:49 +0100 Subject: [PATCH 132/225] Incrementing version to v2.7.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 2264351..5fde983 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,7 +1,7 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.6.0' +__version__ = '2.7.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 65ddb4a..d71a7b9 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -17,7 +17,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.6.0' +__version__ = '2.7.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 1935ea69d874ca355d0d123b53c99841b39cb25a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 22 Jan 2023 00:33:09 +0100 Subject: [PATCH 133/225] improved docs to fix #82 --- portalocker/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index c4fc11a..b02773d 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -173,9 +173,8 @@ class Lock(LockBase): Args: filename: filename - mode: the open mode, 'a' or 'ab' should be used for writing - truncate: use truncate to emulate 'w' mode, None is disabled, 0 is - truncate to 0 bytes + mode: the open mode, 'a' or 'ab' should be used for writing. When mode + contains `w` the file will be truncated to 0 bytes. timeout: timeout when trying to acquire a lock check_interval: check interval while waiting fail_when_locked: after the initial lock failed, return an error From 6ffc39ee398213f8894bbc5019d004920540964d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 13 Mar 2023 22:14:57 +0100 Subject: [PATCH 134/225] added security contact information --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 9b363c2..c5ef42f 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,13 @@ The module is currently maintained by Rick van Hattem . The project resides at https://github.com/WoLpH/portalocker . Bugs and feature requests can be submitted there. Patches are also very welcome. +Security contact information +------------------------------------------------------------------------------ + +To report a security vulnerability, please use the +`Tidelift security contact `_. +Tidelift will coordinate the fix and disclosure. + Redis Locks ----------- From 393c5958757936821f10de2e5f8300328e0d0ce1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 21 Aug 2023 02:36:49 +0200 Subject: [PATCH 135/225] Added stale action --- .github/workflows/stale.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..3169ca3 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,20 @@ +name: Close stale issues and pull requests + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' # Run every day at midnight + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + days-before-stale: 30 + exempt-issue-labels: | + in-progress + help-wanted + pinned + security + enhancement \ No newline at end of file From 0f8d86990ea5f35e2f8d4bf7565129a9c4356814 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 29 Aug 2023 03:09:17 +0200 Subject: [PATCH 136/225] updated stale file --- .github/workflows/stale.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 3169ca3..0740d1a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -12,9 +12,4 @@ jobs: - uses: actions/stale@v8 with: days-before-stale: 30 - exempt-issue-labels: | - in-progress - help-wanted - pinned - security - enhancement \ No newline at end of file + exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement From 98d143e4db289307d0cfea861844615f395c18a4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 29 Aug 2023 23:50:01 +0200 Subject: [PATCH 137/225] updated stale file --- .github/workflows/stale.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0740d1a..7101b3f 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,3 +13,5 @@ jobs: with: days-before-stale: 30 exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement + exempt-all-pr-assignees: true + From deb1474d1eae210bae910e2ea0efa7de255a5c69 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:35:37 +0200 Subject: [PATCH 138/225] Add support for Python 3.11 --- .github/workflows/python-package.yml | 4 ++-- appveyor.yml | 2 ++ setup.cfg | 3 --- setup.py | 1 + 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7cb780d..ec23157 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10', '3.11'] os: ['macos-latest', 'windows-latest'] steps: @@ -38,7 +38,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v3 diff --git a/appveyor.yml b/appveyor.yml index 8ad9373..23cbf4e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,6 +9,8 @@ environment: - TOXENV: py37 - TOXENV: py38 - TOXENV: py39 + - TOXENV: py310 + - TOXENV: py311 install: - "%PYTHON% -m pip install -U tox setuptools wheel" diff --git a/setup.cfg b/setup.cfg index 9960540..c9bc65e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,6 @@ [metadata] description_file = README.rst -[bdist_wheel] -universal = 1 - [flake8] ignore = W391,E303,W503 diff --git a/setup.py b/setup.py index 5c7bfd6..b13224f 100644 --- a/setup.py +++ b/setup.py @@ -93,6 +93,7 @@ def run(self): 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], From e5906d3216e8b68d86fd869d9b75f4f7df0b5e4f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:36:41 +0200 Subject: [PATCH 139/225] Drop support for EOL Python 3.5 and 3.6 --- appveyor.yml | 1 - setup.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 23cbf4e..d3a9b65 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,7 +5,6 @@ environment: global: PYTHON: "C:\\Python38-x64\\python.exe" matrix: - - TOXENV: py36 - TOXENV: py37 - TOXENV: py38 - TOXENV: py39 diff --git a/setup.py b/setup.py index b13224f..99dbe24 100644 --- a/setup.py +++ b/setup.py @@ -87,8 +87,6 @@ def run(self): classifiers=[ 'Intended Audience :: Developers', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', @@ -97,7 +95,7 @@ def run(self): 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], - python_requires='>=3.5', + python_requires='>=3.7', keywords='locking, locks, with statement, windows, linux, unix', author=about['__author__'], author_email=about['__email__'], From 7c146b6126e5a18df84563b8fc9aae2810e10cc0 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 16 Nov 2022 18:37:08 +0200 Subject: [PATCH 140/225] Upgrade Python syntax with pyupgrade --py37-plus --- docs/conf.py | 9 ++++----- portalocker/portalocker.py | 2 +- portalocker/redis.py | 6 +++--- portalocker/utils.py | 10 +++++----- portalocker_tests/tests.py | 11 ++++------- setup.py | 2 -- 6 files changed, 17 insertions(+), 23 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 07e4caf..761d7f3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Documentation build configuration file, created by # sphinx-quickstart on Thu Feb 27 20:00:23 2014. @@ -57,7 +56,7 @@ # General information about the project. project = metadata.__package_name__.replace('-', ' ').capitalize() -copyright = u'%s, %s' % ( +copyright = '{}, {}'.format( datetime.date.today().year, metadata.__author__, ) @@ -213,7 +212,7 @@ latex_documents = [( 'index', '%s.tex' % metadata.__package_name__, - u'%s Documentation' % metadata.__package_name__.replace('-', ' ').capitalize(), + '%s Documentation' % metadata.__package_name__.replace('-', ' ').capitalize(), metadata.__author__, 'manual', )] @@ -246,7 +245,7 @@ man_pages = [( 'index', metadata.__package_name__, - u'%s Documentation' % metadata.__package_name__.replace('-', ' ').capitalize(), + '%s Documentation' % metadata.__package_name__.replace('-', ' ').capitalize(), [metadata.__author__], 1, )] @@ -263,7 +262,7 @@ texinfo_documents = [( 'index', metadata.__package_name__, - u'%s Documentation' % metadata.__package_name__.replace('-', ' ').capitalize(), + '%s Documentation' % metadata.__package_name__.replace('-', ' ').capitalize(), metadata.__author__, metadata.__package_name__, metadata.__description__, diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 72260a7..b1e0964 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -81,7 +81,7 @@ def unlock(file_: typing.IO): finally: if savepos: file_.seek(savepos) - except IOError as exc: + except OSError as exc: raise exceptions.LockException( exceptions.LockException.LOCK_FAILED, exc.strerror, fh=file_ diff --git a/portalocker/redis.py b/portalocker/redis.py index 08dbd4a..26c337d 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -100,9 +100,9 @@ def __init__( for key, value in self.DEFAULT_REDIS_KWARGS.items(): self.redis_kwargs.setdefault(key, value) - super(RedisLock, self).__init__(timeout=timeout, - check_interval=check_interval, - fail_when_locked=fail_when_locked) + super().__init__(timeout=timeout, + check_interval=check_interval, + fail_when_locked=fail_when_locked) def get_connection(self) -> client.Redis: if not self.connection: diff --git a/portalocker/utils.py b/portalocker/utils.py index b02773d..5bf4cab 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -320,8 +320,8 @@ def __init__( self, filename, mode='a', timeout=DEFAULT_TIMEOUT, check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, flags=LOCK_METHOD): - super(RLock, self).__init__(filename, mode, timeout, check_interval, - fail_when_locked, flags) + super().__init__(filename, mode, timeout, check_interval, + fail_when_locked, flags) self._acquire_count = 0 def acquire( @@ -330,8 +330,8 @@ def acquire( if self._acquire_count >= 1: fh = self.fh else: - fh = super(RLock, self).acquire(timeout, check_interval, - fail_when_locked) + fh = super().acquire(timeout, check_interval, + fail_when_locked) self._acquire_count += 1 assert fh return fh @@ -342,7 +342,7 @@ def release(self): "Cannot release more times than acquired") if self._acquire_count == 1: - super(RLock, self).release() + super().release() self._acquire_count -= 1 diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 7a00405..7c75239 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -1,6 +1,3 @@ -from __future__ import print_function -from __future__ import with_statement - import os import dataclasses import multiprocessing @@ -173,12 +170,12 @@ def test_exlusive(tmpfile): with open(tmpfile, 'w') as fh: fh.write('spam and eggs') - fh = open(tmpfile, 'r') + fh = open(tmpfile) portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB) # Make sure we can't read the locked file with pytest.raises(portalocker.LockException): - with open(tmpfile, 'r') as fh2: + with open(tmpfile) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.read() @@ -197,11 +194,11 @@ def test_shared(tmpfile): with open(tmpfile, 'w') as fh: fh.write('spam and eggs') - f = open(tmpfile, 'r') + f = open(tmpfile) portalocker.lock(f, portalocker.LOCK_SH | portalocker.LOCK_NB) # Make sure we can read the locked file - with open(tmpfile, 'r') as fh2: + with open(tmpfile) as fh2: portalocker.lock(fh2, portalocker.LOCK_SH | portalocker.LOCK_NB) assert fh2.read() == 'spam and eggs' diff --git a/setup.py b/setup.py index 99dbe24..dd62dbf 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os import re import typing From c5e6ee22945ce4fade73b768b436b8fda4dd58b5 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 11 Sep 2023 21:49:56 +0300 Subject: [PATCH 141/225] Bump GitHub Actions --- .github/workflows/python-package.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ec23157..ac5b462 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,7 +17,7 @@ jobs: os: ['macos-latest', 'windows-latest'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -41,9 +41,9 @@ jobs: python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Start Redis - uses: supercharge/redis-github-action@1.4.0 + uses: supercharge/redis-github-action@1.7.0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 9d31f8ed5d166675fb012e779f31509938986349 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 11 Sep 2023 21:55:03 +0300 Subject: [PATCH 142/225] Drop support for EOL Python 3.7 --- appveyor.yml | 1 - setup.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d3a9b65..0574689 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,7 +5,6 @@ environment: global: PYTHON: "C:\\Python38-x64\\python.exe" matrix: - - TOXENV: py37 - TOXENV: py38 - TOXENV: py39 - TOXENV: py310 diff --git a/setup.py b/setup.py index dd62dbf..796936e 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,6 @@ def run(self): classifiers=[ 'Intended Audience :: Developers', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', @@ -93,7 +92,7 @@ def run(self): 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], - python_requires='>=3.7', + python_requires='>=3.8', keywords='locking, locks, with statement, windows, linux, unix', author=about['__author__'], author_email=about['__email__'], From aa0a674121839d0fc577121a56d3b20e713ec32f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 11 Sep 2023 22:00:57 +0300 Subject: [PATCH 143/225] Add colour to CI logs for readability --- .github/workflows/python-package.yml | 3 +++ tox.ini | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ac5b462..71209cc 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [ develop ] +env: + FORCE_COLOR: 1 + jobs: # Run os specific tests on the slower OS X/Windows machines windows_osx: diff --git a/tox.ini b/tox.ini index b9960b0..6c1dfa6 100644 --- a/tox.ini +++ b/tox.ini @@ -3,6 +3,8 @@ envlist = py38, py39, py310, py311, pypy3, flake8, docs skip_missing_interpreters = True [testenv] +pass_env = + FORCE_COLOR basepython = py38: python3.8 py39: python3.9 From c9a4d2a228686bebfb87cf2ff629ea004349dc85 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 13 Sep 2023 03:18:30 +0200 Subject: [PATCH 144/225] Attempting appveyor build fix --- appveyor.yml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0574689..0b57035 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,9 +1,10 @@ # What Python version is installed where: # http://www.appveyor.com/docs/installed-software#python +image: + - Visual Studio 2022 + environment: - global: - PYTHON: "C:\\Python38-x64\\python.exe" matrix: - TOXENV: py38 - TOXENV: py39 @@ -11,20 +12,10 @@ environment: - TOXENV: py311 install: - - "%PYTHON% -m pip install -U tox setuptools wheel" - - "%PYTHON% -m pip install -Ue .[tests]" + - py -m pip install -U tox setuptools wheel + - py -m pip install -Ue ".[tests]" build: false # Not a C# project, build stuff at the test step instead. test_script: - - "%PYTHON% -m tox" - -after_test: - - "%PYTHON% setup.py sdist bdist_wheel" - - ps: "ls dist" - -# artifacts: -# - path: dist\* - -#on_success: -# - TODO: upload the content of dist/*.whl to a public wheelhouse + - py -m tox" From d84fca1c09e50f47a01ceb3ae33df4c824422617 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 01:08:12 +0200 Subject: [PATCH 145/225] Made fully ruff, pyright and mypy compliant with automatic testing through tox and github actions --- .github/workflows/lint.yml | 63 +++++++ docs/conf.py | 2 +- portalocker/__about__.py | 1 - portalocker/__init__.py | 9 +- portalocker/__main__.py | 98 +++++++++++ portalocker/constants.py | 6 +- portalocker/exceptions.py | 4 +- portalocker/portalocker.py | 69 ++++---- portalocker/redis.py | 102 ++++++----- portalocker/utils.py | 215 ++++++++++++++--------- portalocker_tests/conftest.py | 12 +- portalocker_tests/temporary_file_lock.py | 2 +- portalocker_tests/test_combined.py | 14 +- portalocker_tests/test_redis.py | 19 +- portalocker_tests/test_semaphore.py | 4 +- portalocker_tests/tests.py | 142 +++++++-------- pyproject.toml | 100 +++++++++++ ruff.toml | 78 ++++++++ setup.cfg | 8 - setup.py | 123 ------------- sourcery.yaml | 2 + tox.ini | 39 +++- 22 files changed, 703 insertions(+), 409 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 portalocker/__main__.py create mode 100644 pyproject.toml create mode 100644 ruff.toml delete mode 100644 setup.cfg delete mode 100644 setup.py create mode 100644 sourcery.yaml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..12157b3 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,63 @@ +name: lint + +on: + push: + branches: [ develop, master ] + pull_request: + branches: [ develop ] + +env: + FORCE_COLOR: 1 + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Python version + run: python --version + - name: Install dependencies + run: | + python -m pip install tox + - name: Test with pytest + run: tox -p all + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel mypy + python -m pip install -e '.[tests]' + + - name: Linting with pyright + uses: jakebailey/pyright-action@v1 + with: + extra-args: portalocker portalocker_tests + + - name: Linting with ruff + uses: jpetrucciani/ruff-check@main + with: + extra-args: portalocker portalocker_tests + + - name: Linting with mypy + run: | + python -m mypy portalocker portalocker_tests + diff --git a/docs/conf.py b/docs/conf.py index 761d7f3..10570a6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -320,7 +320,7 @@ # The format is a list of tuples containing the path and title. #epub_pre_files = [] -# HTML files shat should be inserted after the pages created by sphinx. +# HTML files that should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. #epub_post_files = [] diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 5fde983..51415ff 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -4,4 +4,3 @@ __version__ = '2.7.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' - diff --git a/portalocker/__init__.py b/portalocker/__init__.py index d71a7b9..f52941c 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -1,8 +1,4 @@ -from . import __about__ -from . import constants -from . import exceptions -from . import portalocker -from . import utils +from . import __about__, constants, exceptions, portalocker, utils try: # pragma: no cover from .redis import RedisLock @@ -17,7 +13,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.7.0' +__version__ = __about__.__version__ #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage @@ -78,4 +74,3 @@ 'open_atomic', 'RedisLock', ] - diff --git a/portalocker/__main__.py b/portalocker/__main__.py new file mode 100644 index 0000000..658a3ec --- /dev/null +++ b/portalocker/__main__.py @@ -0,0 +1,98 @@ +import argparse +import logging +import os +import pathlib +import re + +base_path = pathlib.Path(__file__).parent.parent +src_path = base_path / 'portalocker' +dist_path = base_path / 'dist' +_default_output_path = base_path / 'dist' / 'portalocker.py' + +_RELATIVE_IMPORT_RE = re.compile(r'^from \. import (?P.+)$') +_USELESS_ASSIGNMENT_RE = re.compile(r'^(?P\w+) = \1\n$') + +_TEXT_TEMPLATE = """''' +{} +''' + +""" + +logger = logging.getLogger(__name__) + + +def main(argv=None): + parser = argparse.ArgumentParser() + + subparsers = parser.add_subparsers(required=True) + combine_parser = subparsers.add_parser( + 'combine', + help='Combine all Python files into a single unified `portalocker.py` ' + 'file for easy distribution', + ) + combine_parser.add_argument( + '--output-file', + '-o', + type=argparse.FileType('w'), + default=str(_default_output_path), + ) + + combine_parser.set_defaults(func=combine) + args = parser.parse_args(argv) + args.func(args) + + +def _read_file(path, seen_files): + if path in seen_files: + return + + names = set() + seen_files.add(path) + for line in path.open(): + if match := _RELATIVE_IMPORT_RE.match(line): + for name in match.group('names').split(','): + name = name.strip() + names.add(name) + yield from _read_file(src_path / f'{name}.py', seen_files) + else: + yield _clean_line(line, names) + + +def _clean_line(line, names): + # Replace `some_import.spam` with `spam` + if names: + joined_names = '|'.join(names) + line = re.sub(fr'\b({joined_names})\.', '', line) + + # Replace useless assignments (e.g. `spam = spam`) + return _USELESS_ASSIGNMENT_RE.sub('', line) + + +def combine(args): + output_file = args.output_file + pathlib.Path(output_file.name).parent.mkdir(parents=True, exist_ok=True) + + output_file.write( + _TEXT_TEMPLATE.format((base_path / 'README.rst').read_text()), + ) + output_file.write( + _TEXT_TEMPLATE.format((base_path / 'LICENSE').read_text()), + ) + + seen_files = set() + for line in _read_file(src_path / '__init__.py', seen_files): + output_file.write(line) + + output_file.flush() + output_file.close() + + logger.info(f'Wrote combined file to {output_file.name}') + # Run black and ruff if available. If not then just run the file. + os.system(f'black {output_file.name}') + os.system(f'ruff --fix {output_file.name}') + os.system(f'python3 {output_file.name}') + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() diff --git a/portalocker/constants.py b/portalocker/constants.py index dce44f6..72733c8 100644 --- a/portalocker/constants.py +++ b/portalocker/constants.py @@ -13,7 +13,7 @@ Manually unlock, only needed internally - `UNBLOCK` unlock -'''.strip() +''' import enum import os @@ -28,10 +28,12 @@ LOCK_SH = 0x2 #: non-blocking LOCK_NB = 0x4 - LOCK_UN = msvcrt.LK_UNLCK #: unlock + #: unlock + LOCK_UN = msvcrt.LK_UNLCK # type: ignore elif os.name == 'posix': # pragma: no cover import fcntl + #: exclusive lock LOCK_EX = fcntl.LOCK_EX #: shared lock diff --git a/portalocker/exceptions.py b/portalocker/exceptions.py index b360c77..e871d13 100644 --- a/portalocker/exceptions.py +++ b/portalocker/exceptions.py @@ -1,14 +1,14 @@ import typing -class BaseLockException(Exception): +class BaseLockException(Exception): # noqa: N818 # Error codes: LOCK_FAILED = 1 def __init__( self, *args: typing.Any, - fh: typing.Optional[typing.IO] = None, + fh: typing.Union[typing.IO, None, int] = None, **kwargs: typing.Any, ) -> None: self.fh = fh diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index b1e0964..fea66d2 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -1,10 +1,8 @@ +import contextlib import os - import typing -from . import constants -from . import exceptions - +from . import constants, exceptions # Alias for readability. Due to import recursion issues we cannot do: # from .constants import LockFlags @@ -13,6 +11,7 @@ if os.name == 'nt': # pragma: no cover import msvcrt + import pywintypes import win32con import win32file @@ -20,8 +19,7 @@ __overlapped = pywintypes.OVERLAPPED() - - def lock(file_: typing.IO, flags: LockFlags): + def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): mode = 0 if flags & LockFlags.NON_BLOCKING: mode |= win32con.LOCKFILE_FAIL_IMMEDIATELY @@ -29,13 +27,15 @@ def lock(file_: typing.IO, flags: LockFlags): if flags & LockFlags.EXCLUSIVE: mode |= win32con.LOCKFILE_EXCLUSIVE_LOCK + # Windows locking does not support locking through `fh.fileno()` + assert isinstance(file_, typing.IO) # Save the old position so we can go back to that position but # still lock from the beginning of the file savepos = file_.tell() if savepos: file_.seek(0) - os_fh = msvcrt.get_osfhandle(file_.fileno()) + os_fh = msvcrt.get_osfhandle(file_.fileno()) # type: ignore try: win32file.LockFileEx(os_fh, mode, 0, -0x10000, __overlapped) except pywintypes.error as exc_value: @@ -45,8 +45,8 @@ def lock(file_: typing.IO, flags: LockFlags): raise exceptions.AlreadyLocked( exceptions.LockException.LOCK_FAILED, exc_value.strerror, - fh=file_ - ) + fh=file_, + ) from exc_value else: # Q: Are there exceptions/codes we should be dealing with # here? @@ -55,26 +55,22 @@ def lock(file_: typing.IO, flags: LockFlags): if savepos: file_.seek(savepos) - def unlock(file_: typing.IO): try: savepos = file_.tell() if savepos: file_.seek(0) - os_fh = msvcrt.get_osfhandle(file_.fileno()) + os_fh = msvcrt.get_osfhandle(file_.fileno()) # type: ignore try: win32file.UnlockFileEx( - os_fh, 0, -0x10000, __overlapped + os_fh, + 0, + -0x10000, + __overlapped, ) except pywintypes.error as exc: - if exc.winerror == winerror.ERROR_NOT_LOCKED: - # error: (158, 'UnlockFileEx', - # 'The segment is already unlocked.') - # To match the 'posix' implementation, silently - # ignore this error - pass - else: + if exc.winerror != winerror.ERROR_NOT_LOCKED: # Q: Are there exceptions/codes we should be # dealing with here? raise @@ -83,37 +79,36 @@ def unlock(file_: typing.IO): file_.seek(savepos) except OSError as exc: raise exceptions.LockException( - exceptions.LockException.LOCK_FAILED, exc.strerror, - fh=file_ - ) + exceptions.LockException.LOCK_FAILED, + exc.strerror, + fh=file_, + ) from exc elif os.name == 'posix': # pragma: no cover import fcntl - - def lock(file_: typing.IO, flags: LockFlags): - locking_exceptions = IOError, - try: # pragma: no cover - locking_exceptions += BlockingIOError, # type: ignore - except NameError: # pragma: no cover - pass - + def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): + locking_exceptions = (IOError,) + with contextlib.suppress(NameError): + locking_exceptions += (BlockingIOError,) # type: ignore # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled results # in an error - if ((flags & LockFlags.NON_BLOCKING) - and not flags & (LockFlags.SHARED | LockFlags.EXCLUSIVE)): - raise RuntimeError('When locking in non-blocking mode the SHARED ' - 'or EXCLUSIVE flag must be specified as well') + if (flags & LockFlags.NON_BLOCKING) and not flags & ( + LockFlags.SHARED | LockFlags.EXCLUSIVE + ): + raise RuntimeError( + 'When locking in non-blocking mode the SHARED ' + 'or EXCLUSIVE flag must be specified as well', + ) try: fcntl.flock(file_, flags) except locking_exceptions as exc_value: # The exception code varies on different systems so we'll catch # every IO error - raise exceptions.LockException(exc_value, fh=file_) - + raise exceptions.LockException(exc_value, fh=file_) from exc_value - def unlock(file_: typing.IO, ): + def unlock(file_: typing.IO): fcntl.flock(file_.fileno(), LockFlags.UNBLOCK) else: # pragma: no cover diff --git a/portalocker/redis.py b/portalocker/redis.py index 26c337d..59ee5ff 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -4,13 +4,10 @@ import random import time import typing -from typing import Any -from typing import Dict from redis import client -from . import exceptions -from . import utils +from . import exceptions, utils logger = logging.getLogger(__name__) @@ -19,7 +16,6 @@ class PubSubWorkerThread(client.PubSubWorkerThread): # type: ignore - def run(self): try: super().run() @@ -64,7 +60,8 @@ class RedisLock(utils.LockBase): `health_check_interval=0`) ''' - redis_kwargs: Dict[str, Any] + + redis_kwargs: typing.Dict[str, typing.Any] thread: typing.Optional[PubSubWorkerThread] channel: str timeout: float @@ -72,20 +69,20 @@ class RedisLock(utils.LockBase): pubsub: typing.Optional[client.PubSub] = None close_connection: bool - DEFAULT_REDIS_KWARGS = dict( + DEFAULT_REDIS_KWARGS: typing.ClassVar[typing.Dict[str, typing.Any]] = dict( health_check_interval=10, ) def __init__( - self, - channel: str, - connection: typing.Optional[client.Redis] = None, - timeout: typing.Optional[float] = None, - check_interval: typing.Optional[float] = None, - fail_when_locked: typing.Optional[bool] = False, - thread_sleep_time: float = DEFAULT_THREAD_SLEEP_TIME, - unavailable_timeout: float = DEFAULT_UNAVAILABLE_TIMEOUT, - redis_kwargs: typing.Optional[typing.Dict] = None, + self, + channel: str, + connection: typing.Optional[client.Redis] = None, + timeout: typing.Optional[float] = None, + check_interval: typing.Optional[float] = None, + fail_when_locked: typing.Optional[bool] = False, + thread_sleep_time: float = DEFAULT_THREAD_SLEEP_TIME, + unavailable_timeout: float = DEFAULT_UNAVAILABLE_TIMEOUT, + redis_kwargs: typing.Optional[typing.Dict] = None, ): # We don't want to close connections given as an argument self.close_connection = not connection @@ -100,9 +97,11 @@ def __init__( for key, value in self.DEFAULT_REDIS_KWARGS.items(): self.redis_kwargs.setdefault(key, value) - super().__init__(timeout=timeout, - check_interval=check_interval, - fail_when_locked=fail_when_locked) + super().__init__( + timeout=timeout, + check_interval=check_interval, + fail_when_locked=fail_when_locked, + ) def get_connection(self) -> client.Redis: if not self.connection: @@ -120,21 +119,29 @@ def channel_handler(self, message): logger.debug('TypeError while parsing: %r', message) return + assert self.connection is not None self.connection.publish(data['response_channel'], str(time.time())) @property def client_name(self): - return self.channel + '-lock' + return f'{self.channel}-lock' def acquire( - self, timeout: float = None, check_interval: float = None, - fail_when_locked: typing.Optional[bool] = None): - + self, + timeout: typing.Optional[float] = None, + check_interval: typing.Optional[float] = None, + fail_when_locked: typing.Optional[bool] = None, + ): timeout = utils.coalesce(timeout, self.timeout, 0.0) - check_interval = utils.coalesce(check_interval, self.check_interval, - 0.0) - fail_when_locked = utils.coalesce(fail_when_locked, - self.fail_when_locked) + check_interval = utils.coalesce( + check_interval, + self.check_interval, + 0.0, + ) + fail_when_locked = utils.coalesce( + fail_when_locked, + self.fail_when_locked, + ) assert not self.pubsub, 'This lock is already active' connection = self.get_connection() @@ -144,12 +151,16 @@ def acquire( subscribers = connection.pubsub_numsub(self.channel)[0][1] if subscribers: - logger.debug('Found %d lock subscribers for %s', - subscribers, self.channel) + logger.debug( + 'Found %d lock subscribers for %s', + subscribers, + self.channel, + ) if self.check_or_kill_lock( - connection, - self.unavailable_timeout): # pragma: no branch + connection, + self.unavailable_timeout, + ): # pragma: no branch continue else: # pragma: no cover subscribers = 0 @@ -161,7 +172,9 @@ def acquire( self.pubsub = connection.pubsub() self.pubsub.subscribe(**{self.channel: self.channel_handler}) self.thread = PubSubWorkerThread( - self.pubsub, sleep_time=self.thread_sleep_time) + self.pubsub, + sleep_time=self.thread_sleep_time, + ) self.thread.start() subscribers = connection.pubsub_numsub(self.channel)[0][1] @@ -182,24 +195,30 @@ def check_or_kill_lock(self, connection, timeout): pubsub = connection.pubsub() pubsub.subscribe(response_channel) - connection.publish(self.channel, json.dumps(dict( - response_channel=response_channel, - message='ping', - ))) + connection.publish( + self.channel, + json.dumps( + dict( + response_channel=response_channel, + message='ping', + ), + ), + ) check_interval = min(self.thread_sleep_time, timeout / 10) for _ in self._timeout_generator( - timeout, check_interval): # pragma: no branch - message = pubsub.get_message(timeout=check_interval) - if message: # pragma: no branch + timeout, + check_interval, + ): # pragma: no branch + if pubsub.get_message(timeout=check_interval): pubsub.close() return True for client_ in connection.client_list('pubsub'): # pragma: no cover if client_.get('name') == self.client_name: - logger.warning( - 'Killing unavailable redis client: %r', client_) + logger.warning('Killing unavailable redis client: %r', client_) connection.client_kill_filter(client_.get('id')) + return None def release(self): if self.thread: # pragma: no branch @@ -215,4 +234,3 @@ def release(self): def __del__(self): self.release() - diff --git a/portalocker/utils.py b/portalocker/utils.py index 5bf4cab..fe2fa60 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -10,9 +10,7 @@ import typing import warnings -from . import constants -from . import exceptions -from . import portalocker +from . import constants, exceptions, portalocker logger = logging.getLogger(__name__) @@ -51,14 +49,14 @@ def coalesce(*args: typing.Any, test_value: typing.Any = None) -> typing.Any: >>> coalesce([], dict(spam='eggs'), test_value=[]) [] ''' - for arg in args: - if arg is not test_value: - return arg + return next((arg for arg in args if arg is not test_value), None) @contextlib.contextmanager -def open_atomic(filename: Filename, binary: bool = True) \ - -> typing.Iterator[typing.IO]: +def open_atomic( + filename: Filename, + binary: bool = True, +) -> typing.Iterator[typing.IO]: '''Open a file for atomic writing. Instead of locking this method allows you to write the entire file and move it to the actual location. Note that this makes the assumption that a rename is atomic on your platform which @@ -103,10 +101,8 @@ def open_atomic(filename: Filename, binary: bool = True) \ try: os.rename(temp_fh.name, path) finally: - try: + with contextlib.suppress(Exception): os.remove(temp_fh.name) - except Exception: - pass class LockBase(abc.ABC): # pragma: no cover @@ -117,23 +113,33 @@ class LockBase(abc.ABC): # pragma: no cover #: skip the timeout and immediately fail if the initial lock fails fail_when_locked: bool - def __init__(self, timeout: typing.Optional[float] = None, - check_interval: typing.Optional[float] = None, - fail_when_locked: typing.Optional[bool] = None): + def __init__( + self, + timeout: typing.Optional[float] = None, + check_interval: typing.Optional[float] = None, + fail_when_locked: typing.Optional[bool] = None, + ): self.timeout = coalesce(timeout, DEFAULT_TIMEOUT) self.check_interval = coalesce(check_interval, DEFAULT_CHECK_INTERVAL) - self.fail_when_locked = coalesce(fail_when_locked, - DEFAULT_FAIL_WHEN_LOCKED) + self.fail_when_locked = coalesce( + fail_when_locked, + DEFAULT_FAIL_WHEN_LOCKED, + ) @abc.abstractmethod def acquire( - self, timeout: float = None, check_interval: float = None, - fail_when_locked: bool = None): + self, + timeout: typing.Optional[float] = None, + check_interval: typing.Optional[float] = None, + fail_when_locked: typing.Optional[bool] = None, + ): return NotImplemented - def _timeout_generator(self, timeout: typing.Optional[float], - check_interval: typing.Optional[float]) \ - -> typing.Iterator[int]: + def _timeout_generator( + self, + timeout: typing.Optional[float], + check_interval: typing.Optional[float], + ) -> typing.Iterator[int]: f_timeout = coalesce(timeout, self.timeout, 0.0) f_check_interval = coalesce(check_interval, self.check_interval, 0.0) @@ -156,11 +162,12 @@ def release(self): def __enter__(self): return self.acquire() - def __exit__(self, - exc_type: typing.Optional[typing.Type[BaseException]], - exc_value: typing.Optional[BaseException], - traceback: typing.Any, # Should be typing.TracebackType - ) -> typing.Optional[bool]: + def __exit__( + self, + exc_type: typing.Optional[typing.Type[BaseException]], + exc_value: typing.Optional[BaseException], + traceback: typing.Any, # Should be typing.TracebackType + ) -> typing.Optional[bool]: self.release() return None @@ -190,13 +197,15 @@ class Lock(LockBase): ''' def __init__( - self, - filename: Filename, - mode: str = 'a', - timeout: float = None, - check_interval: float = DEFAULT_CHECK_INTERVAL, - fail_when_locked: bool = DEFAULT_FAIL_WHEN_LOCKED, - flags: constants.LockFlags = LOCK_METHOD, **file_open_kwargs): + self, + filename: Filename, + mode: str = 'a', + timeout: typing.Optional[float] = None, + check_interval: float = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool = DEFAULT_FAIL_WHEN_LOCKED, + flags: constants.LockFlags = LOCK_METHOD, + **file_open_kwargs, + ): if 'w' in mode: truncate = True mode = mode.replace('w', 'a') @@ -206,7 +215,10 @@ def __init__( if timeout is None: timeout = DEFAULT_TIMEOUT elif not (flags & constants.LockFlags.NON_BLOCKING): - warnings.warn('timeout has no effect in blocking mode') + warnings.warn( + 'timeout has no effect in blocking mode', + stacklevel=1, + ) self.fh: typing.Optional[typing.IO] = None self.filename: str = str(filename) @@ -219,18 +231,26 @@ def __init__( self.file_open_kwargs = file_open_kwargs def acquire( - self, timeout: float = None, check_interval: float = None, - fail_when_locked: bool = None) -> typing.IO: + self, + timeout: typing.Optional[float] = None, + check_interval: typing.Optional[float] = None, + fail_when_locked: typing.Optional[bool] = None, + ) -> typing.IO: '''Acquire the locked filehandle''' fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) - if not (self.flags & constants.LockFlags.NON_BLOCKING) \ - and timeout is not None: - warnings.warn('timeout has no effect in blocking mode') + if ( + not (self.flags & constants.LockFlags.NON_BLOCKING) + and timeout is not None + ): + warnings.warn( + 'timeout has no effect in blocking mode', + stacklevel=1, + ) # If we already have a filehandle, return it - fh = self.fh + fh: typing.Optional[typing.IO] = self.fh if fh: return fh @@ -239,10 +259,9 @@ def acquire( def try_close(): # pragma: no cover # Silently try to close the handle if possible, ignore all issues - try: - fh.close() - except Exception: - pass + if fh is not None: + with contextlib.suppress(Exception): + fh.close() exception = None # Try till the timeout has passed @@ -261,7 +280,7 @@ def try_close(): # pragma: no cover # If fail_when_locked is True, stop trying if fail_when_locked: try_close() - raise exceptions.AlreadyLocked(exception) + raise exceptions.AlreadyLocked(exception) from exc # Wait a bit @@ -285,7 +304,11 @@ def release(self): def _get_fh(self) -> typing.IO: '''Get a new filehandle''' - return open(self.filename, self.mode, **self.file_open_kwargs) + return open( # noqa: SIM115 + self.filename, + self.mode, + **self.file_open_kwargs, + ) def _get_lock(self, fh: typing.IO) -> typing.IO: ''' @@ -317,21 +340,34 @@ class RLock(Lock): ''' def __init__( - self, filename, mode='a', timeout=DEFAULT_TIMEOUT, - check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False, - flags=LOCK_METHOD): - super().__init__(filename, mode, timeout, check_interval, - fail_when_locked, flags) + self, + filename, + mode='a', + timeout=DEFAULT_TIMEOUT, + check_interval=DEFAULT_CHECK_INTERVAL, + fail_when_locked=False, + flags=LOCK_METHOD, + ): + super().__init__( + filename, + mode, + timeout, + check_interval, + fail_when_locked, + flags, + ) self._acquire_count = 0 def acquire( - self, timeout: float = None, check_interval: float = None, - fail_when_locked: bool = None) -> typing.IO: + self, + timeout: typing.Optional[float] = None, + check_interval: typing.Optional[float] = None, + fail_when_locked: typing.Optional[bool] = None, + ) -> typing.IO: if self._acquire_count >= 1: fh = self.fh else: - fh = super().acquire(timeout, check_interval, - fail_when_locked) + fh = super().acquire(timeout, check_interval, fail_when_locked) self._acquire_count += 1 assert fh return fh @@ -339,7 +375,8 @@ def acquire( def release(self): if self._acquire_count == 0: raise exceptions.LockException( - "Cannot release more times than acquired") + 'Cannot release more times than acquired', + ) if self._acquire_count == 1: super().release() @@ -347,13 +384,23 @@ def release(self): class TemporaryFileLock(Lock): - - def __init__(self, filename='.lock', timeout=DEFAULT_TIMEOUT, - check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=True, - flags=LOCK_METHOD): - Lock.__init__(self, filename=filename, mode='w', timeout=timeout, - check_interval=check_interval, - fail_when_locked=fail_when_locked, flags=flags) + def __init__( + self, + filename='.lock', + timeout=DEFAULT_TIMEOUT, + check_interval=DEFAULT_CHECK_INTERVAL, + fail_when_locked=True, + flags=LOCK_METHOD, + ): + Lock.__init__( + self, + filename=filename, + mode='w', + timeout=timeout, + check_interval=check_interval, + fail_when_locked=fail_when_locked, + flags=flags, + ) atexit.register(self.release) def release(self): @@ -377,17 +424,18 @@ class BoundedSemaphore(LockBase): >>> str(sorted(semaphore.get_random_filenames())[1]) 'bounded_semaphore.01.lock' ''' + lock: typing.Optional[Lock] def __init__( - self, - maximum: int, - name: str = 'bounded_semaphore', - filename_pattern: str = '{name}.{number:02d}.lock', - directory: str = tempfile.gettempdir(), - timeout: typing.Optional[float] = DEFAULT_TIMEOUT, - check_interval: typing.Optional[float] = DEFAULT_CHECK_INTERVAL, - fail_when_locked: typing.Optional[bool] = True, + self, + maximum: int, + name: str = 'bounded_semaphore', + filename_pattern: str = '{name}.{number:02d}.lock', + directory: str = tempfile.gettempdir(), + timeout: typing.Optional[float] = DEFAULT_TIMEOUT, + check_interval: typing.Optional[float] = DEFAULT_CHECK_INTERVAL, + fail_when_locked: typing.Optional[bool] = True, ): self.maximum = maximum self.name = name @@ -415,10 +463,11 @@ def get_filename(self, number) -> pathlib.Path: ) def acquire( - self, - timeout: float = None, - check_interval: float = None, - fail_when_locked: bool = None) -> typing.Optional[Lock]: + self, + timeout: typing.Optional[float] = None, + check_interval: typing.Optional[float] = None, + fail_when_locked: typing.Optional[bool] = None, + ) -> typing.Optional[Lock]: assert not self.lock, 'Already locked' filenames = self.get_filenames() @@ -429,8 +478,10 @@ def acquire( if self.try_lock(filenames): # pragma: no branch return self.lock # pragma: no cover - fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) - if fail_when_locked: + if fail_when_locked := coalesce( + fail_when_locked, + self.fail_when_locked, + ): raise exceptions.AlreadyLocked() return None @@ -442,13 +493,15 @@ def try_lock(self, filenames: typing.Sequence[Filename]) -> bool: self.lock = Lock(filename, fail_when_locked=True) try: self.lock.acquire() - logger.debug('locked %r', filename) - return True except exceptions.AlreadyLocked: self.lock = None + else: + logger.debug('locked %r', filename) + return True return False def release(self): # pragma: no cover - self.lock.release() - self.lock = None + if self.lock is not None: + self.lock.release() + self.lock = None diff --git a/portalocker_tests/conftest.py b/portalocker_tests/conftest.py index cf59e2b..6c56a6a 100644 --- a/portalocker_tests/conftest.py +++ b/portalocker_tests/conftest.py @@ -1,7 +1,9 @@ +import contextlib import logging -import pytest -import random import multiprocessing +import random + +import pytest logger = logging.getLogger(__name__) @@ -10,14 +12,12 @@ def tmpfile(tmp_path): filename = tmp_path / str(random.random()) yield str(filename) - try: + with contextlib.suppress(PermissionError): filename.unlink(missing_ok=True) - except PermissionError: - pass def pytest_sessionstart(session): - # Force spawning the process so we don't accidently inherit locks. + # Force spawning the process so we don't accidentally inherit locks. # I'm not a 100% certain this will work correctly unfortunately... there # is some potential for breaking tests multiprocessing.set_start_method('spawn') diff --git a/portalocker_tests/temporary_file_lock.py b/portalocker_tests/temporary_file_lock.py index b250bad..ad35373 100644 --- a/portalocker_tests/temporary_file_lock.py +++ b/portalocker_tests/temporary_file_lock.py @@ -1,4 +1,5 @@ import os + import portalocker @@ -11,4 +12,3 @@ def test_temporary_file_lock(tmpfile): lock = portalocker.TemporaryFileLock(tmpfile) lock.acquire() del lock - diff --git a/portalocker_tests/test_combined.py b/portalocker_tests/test_combined.py index 594de74..bbd9eb2 100644 --- a/portalocker_tests/test_combined.py +++ b/portalocker_tests/test_combined.py @@ -1,15 +1,13 @@ import sys +from portalocker import __main__ -def test_combined(tmpdir): - from distutils import dist - import setup +def test_combined(tmpdir): output_file = tmpdir.join('combined.py') - combine = setup.Combine(dist.Distribution()) - combine.output_file = str(output_file) - combine.run() + __main__.main(['combine', '--output-file', output_file.strpath]) sys.path.append(output_file.dirname) - import combined - assert combined + # Combined is being generated above but linters won't understand that + import combined # type: ignore + assert combined diff --git a/portalocker_tests/test_redis.py b/portalocker_tests/test_redis.py index 694c9af..e9bec02 100644 --- a/portalocker_tests/test_redis.py +++ b/portalocker_tests/test_redis.py @@ -4,12 +4,10 @@ import time import pytest -from redis import client -from redis import exceptions import portalocker -from portalocker import redis -from portalocker import utils +from portalocker import redis, utils +from redis import client, exceptions logger = logging.getLogger(__name__) @@ -31,7 +29,7 @@ def set_redis_timeouts(monkeypatch): def test_redis_lock(): channel = str(random.random()) - lock_a = redis.RedisLock(channel) + lock_a: redis.RedisLock = redis.RedisLock(channel) lock_a.acquire(fail_when_locked=True) time.sleep(0.01) @@ -41,7 +39,8 @@ def test_redis_lock(): lock_b.acquire(fail_when_locked=True) finally: lock_a.release() - lock_a.connection.close() + if lock_a.connection is not None: + lock_a.connection.close() @pytest.mark.parametrize('timeout', [None, 0, 0.001]) @@ -58,7 +57,8 @@ def test_redis_lock_timeout(timeout, check_interval): lock_b.acquire(timeout=timeout, check_interval=check_interval) finally: lock_a.release() - lock_a.connection.close() + if lock_a.connection is not None: + lock_a.connection.close() def test_redis_lock_context(): @@ -68,9 +68,8 @@ def test_redis_lock_context(): with lock_a: time.sleep(0.01) lock_b = redis.RedisLock(channel, fail_when_locked=True) - with pytest.raises(portalocker.AlreadyLocked): - with lock_b: - pass + with pytest.raises(portalocker.AlreadyLocked), lock_b: + pass def test_redis_relock(): diff --git a/portalocker_tests/test_semaphore.py b/portalocker_tests/test_semaphore.py index c6847ab..b6d4594 100644 --- a/portalocker_tests/test_semaphore.py +++ b/portalocker_tests/test_semaphore.py @@ -1,5 +1,7 @@ import random + import pytest + import portalocker from portalocker import utils @@ -8,7 +10,7 @@ @pytest.mark.parametrize('check_interval', [None, 0, 0.0005]) def test_bounded_semaphore(timeout, check_interval, monkeypatch): n = 2 - name = random.random() + name: str = str(random.random()) monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.0001) monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.0005) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 7c75239..b475396 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -1,33 +1,26 @@ -import os import dataclasses import multiprocessing +import os import time import typing import pytest + import portalocker -from portalocker import utils -from portalocker import LockFlags +from portalocker import LockFlags, utils def test_exceptions(tmpfile): - # Open the file 2 times - a = open(tmpfile, 'a') - b = open(tmpfile, 'a') + with open(tmpfile, 'a') as a, open(tmpfile, 'a') as b: + # Lock exclusive non-blocking + lock_flags = portalocker.LOCK_EX | portalocker.LOCK_NB - # Lock exclusive non-blocking - lock_flags = portalocker.LOCK_EX | portalocker.LOCK_NB - - # First lock file a - portalocker.lock(a, lock_flags) - - # Now see if we can lock file b - with pytest.raises(portalocker.LockException): - portalocker.lock(b, lock_flags) + # First lock file a + portalocker.lock(a, lock_flags) - # Cleanup - a.close() - b.close() + # Now see if we can lock file b + with pytest.raises(portalocker.LockException): + portalocker.lock(b, lock_flags) def test_utils_base(): @@ -41,8 +34,10 @@ def test_with_timeout(tmpfile): with portalocker.Lock(tmpfile, timeout=0.1) as fh: print('writing some stuff to my cache...', file=fh) with portalocker.Lock( - tmpfile, timeout=0.1, mode='wb', - fail_when_locked=True + tmpfile, + timeout=0.1, + mode='wb', + fail_when_locked=True, ): pass print('writing more stuff to my cache...', file=fh) @@ -71,18 +66,17 @@ def test_simple(tmpfile): with open(tmpfile, 'w') as fh: fh.write('spam and eggs') - fh = open(tmpfile, 'r+') - portalocker.lock(fh, portalocker.LOCK_EX) + with open(tmpfile, 'r+') as fh: + portalocker.lock(fh, portalocker.LOCK_EX) - fh.seek(13) - fh.write('foo') + fh.seek(13) + fh.write('foo') - # Make sure we didn't overwrite the original text - fh.seek(0) - assert fh.read(13) == 'spam and eggs' + # Make sure we didn't overwrite the original text + fh.seek(0) + assert fh.read(13) == 'spam and eggs' - portalocker.unlock(fh) - fh.close() + portalocker.unlock(fh) def test_truncate(tmpfile): @@ -106,9 +100,8 @@ def test_class(tmpfile): with lock: lock.acquire() - with pytest.raises(portalocker.LockException): - with lock2: - pass + with pytest.raises(portalocker.LockException), lock2: + pass with lock2: pass @@ -170,47 +163,46 @@ def test_exlusive(tmpfile): with open(tmpfile, 'w') as fh: fh.write('spam and eggs') - fh = open(tmpfile) - portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB) + with open(tmpfile) as fh: + portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB) - # Make sure we can't read the locked file - with pytest.raises(portalocker.LockException): - with open(tmpfile) as fh2: + # Make sure we can't read the locked file + with pytest.raises(portalocker.LockException), open(tmpfile) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.read() - # Make sure we can't write the locked file - with pytest.raises(portalocker.LockException): - with open(tmpfile, 'w+') as fh2: + # Make sure we can't write the locked file + with pytest.raises(portalocker.LockException), open( + tmpfile, 'w+', + ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.write('surprise and fear') - # Make sure we can explicitly unlock the file - portalocker.unlock(fh) - fh.close() + # Make sure we can explicitly unlock the file + portalocker.unlock(fh) def test_shared(tmpfile): with open(tmpfile, 'w') as fh: fh.write('spam and eggs') - f = open(tmpfile) - portalocker.lock(f, portalocker.LOCK_SH | portalocker.LOCK_NB) + with open(tmpfile) as f: + portalocker.lock(f, portalocker.LOCK_SH | portalocker.LOCK_NB) - # Make sure we can read the locked file - with open(tmpfile) as fh2: - portalocker.lock(fh2, portalocker.LOCK_SH | portalocker.LOCK_NB) - assert fh2.read() == 'spam and eggs' + # Make sure we can read the locked file + with open(tmpfile) as fh2: + portalocker.lock(fh2, portalocker.LOCK_SH | portalocker.LOCK_NB) + assert fh2.read() == 'spam and eggs' - # Make sure we can't write the locked file - with pytest.raises(portalocker.LockException): - with open(tmpfile, 'w+') as fh2: + # Make sure we can't write the locked file + with pytest.raises(portalocker.LockException), open( + tmpfile, 'w+', + ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.write('surprise and fear') - # Make sure we can explicitly unlock the file - portalocker.unlock(f) - f.close() + # Make sure we can explicitly unlock the file + portalocker.unlock(f) def test_blocking_timeout(tmpfile): @@ -225,12 +217,13 @@ def test_blocking_timeout(tmpfile): lock.acquire(timeout=5) -@pytest.mark.skipif(os.name == 'nt', - reason='Windows uses an entirely different lockmechanism') +@pytest.mark.skipif( + os.name == 'nt', + reason='Windows uses an entirely different lockmechanism', +) def test_nonblocking(tmpfile): - with open(tmpfile, 'w') as fh: - with pytest.raises(RuntimeError): - portalocker.lock(fh, LockFlags.NON_BLOCKING) + with open(tmpfile, 'w') as fh, pytest.raises(RuntimeError): + portalocker.lock(fh, LockFlags.NON_BLOCKING) def shared_lock(filename, **kwargs): @@ -276,7 +269,7 @@ class LockResult: def lock( filename: str, fail_when_locked: bool, - flags: LockFlags + flags: LockFlags, ) -> LockResult: # Returns a case of True, False or FileNotFound # https://thedailywtf.com/articles/what_is_truth_0x3f_ @@ -325,8 +318,8 @@ def test_exclusive_processes(tmpfile, fail_when_locked): assert not a.exception_class or not b.exception_class assert issubclass( - a.exception_class or b.exception_class, - portalocker.LockException + a.exception_class or b.exception_class, # type: ignore + portalocker.LockException, ) @@ -335,20 +328,13 @@ def test_exclusive_processes(tmpfile, fail_when_locked): reason='Locking on Windows requires a file object', ) def test_lock_fileno(tmpfile): - # Open the file 2 times - a = open(tmpfile, 'a') - b = open(tmpfile, 'a') - - # Lock exclusive non-blocking - flags = LockFlags.SHARED | LockFlags.NON_BLOCKING - - # First lock file a - portalocker.lock(a, flags) - - # Now see if we can lock using fileno() - portalocker.lock(b.fileno(), flags) + with open(tmpfile, 'a') as a: + with open(tmpfile, 'a') as b: + # Lock exclusive non-blocking + flags = LockFlags.SHARED | LockFlags.NON_BLOCKING - # Cleanup - a.close() - b.close() + # First lock file a + portalocker.lock(a, flags) + # Now see if we can lock using fileno() + portalocker.lock(b.fileno(), flags) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3a76033 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,100 @@ +[build-system] +build-backend = 'setuptools.build_meta' +requires = ['setuptools', 'setuptools-scm', 'wheel'] + +[project] +name = 'portalocker' +dynamic = ['version'] +authors = [{name = 'Rick van Hattem', email = 'wolph@wol.ph'}] +license = {text = 'BSD-3-Clause'} +description = 'Wraps the portalocker recipe for easy usage' +keywords = [ + 'locking', + 'locks', + 'with', + 'statement', + 'windows', + 'linux', + 'unix', +] +readme = 'README.rst' +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Development Status :: 6 - Mature', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Natural Language :: English', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: MacOS', + 'Operating System :: Microsoft :: MS-DOS', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: Microsoft', + 'Operating System :: POSIX :: BSD :: FreeBSD', + 'Operating System :: POSIX :: BSD', + 'Operating System :: POSIX :: Linux', + 'Operating System :: POSIX :: SunOS/Solaris', + 'Operating System :: POSIX', + 'Operating System :: Unix', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: IronPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Programming Language :: Python :: Implementation', + 'Programming Language :: Python', + 'Topic :: Education :: Testing', + 'Topic :: Office/Business', + 'Topic :: Other/Nonlisted Topic', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Software Development :: Libraries', + 'Topic :: System :: Monitoring', +] +requires-python = '>=3.8' +dependencies = ['pywin32>=226; platform_system == "Windows"'] + +[project.urls] +bugs = 'https://github.com/wolph/portalocker/issues' +documentation = 'https://portalocker.readthedocs.io/en/latest/' +repository = 'https://github.com/wolph/portalocker/' + +[project.optional-dependencies] +docs = ['sphinx>=1.7.1'] +tests = [ + 'pytest>=5.4.1', + 'pytest-cov>=2.8.1', + 'pytest-timeout>=2.1.0', + 'sphinx>=6.0.0', + 'pytest-mypy>=0.8.0', + 'types-redis', + 'redis', +] +redis = ['redis'] + +[tool.setuptools] +platforms = ['any'] +include-package-data = false + +[tool.setuptools.dynamic] +version = { attr = 'portalocker.__about__.__version__' } + +[tool.setuptools.packages.find] +exclude = ['examples','portalocker_tests'] + +[tool.setuptools.package-data] +portalocker = ['py.typed', 'msvcrt.pyi'] + +[tool.black] +line-length = 79 +skip-string-normalization = true + +[tool.codespell] +skip = '*/htmlcov,./docs/_build,*.asc' + +[tool.pyright] +include = ['portalocker', 'portalocker_tests'] +exclude = ['dist/*'] diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..e484f82 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,78 @@ +# We keep the ruff configuration separate so it can easily be shared across +# all projects + +target-version = 'py38' + +src = ['portalocker'] +exclude = ['docs'] + +format = 'grouped' +ignore = [ + 'A001', # Variable {name} is shadowing a Python builtin + 'A002', # Argument {name} is shadowing a Python builtin + 'A003', # Class attribute {name} is shadowing a Python builtin + 'B023', # function-uses-loop-variable + 'B024', # `FormatWidgetMixin` is an abstract base class, but it has no abstract methods + 'D205', # blank-line-after-summary + 'D212', # multi-line-summary-first-line + 'RET505', # Unnecessary `else` after `return` statement + 'TRY003', # Avoid specifying long messages outside the exception class + 'RET507', # Unnecessary `elif` after `continue` statement + 'C405', # Unnecessary {obj_type} literal (rewrite as a set literal) + 'C406', # Unnecessary {obj_type} literal (rewrite as a dict literal) + 'C408', # Unnecessary {obj_type} call (rewrite as a literal) + 'SIM114', # Combine `if` branches using logical `or` operator + 'RET506', # Unnecessary `else` after `raise` statement + 'FA100', # Missing `from __future__ import annotations`, but uses `typing.Optional` +] + +line-length = 80 +select = [ + 'A', # flake8-builtins + 'ASYNC', # flake8 async checker + 'B', # flake8-bugbear + 'C4', # flake8-comprehensions + 'C90', # mccabe + 'COM', # flake8-commas + + ## Require docstrings for all public methods, would be good to enable at some point + # 'D', # pydocstyle + + 'E', # pycodestyle error ('W' for warning) + 'F', # pyflakes + 'FA', # flake8-future-annotations + 'I', # isort + 'ICN', # flake8-import-conventions + 'INP', # flake8-no-pep420 + 'ISC', # flake8-implicit-str-concat + 'N', # pep8-naming + 'NPY', # NumPy-specific rules + 'PERF', # perflint, + 'PIE', # flake8-pie + 'Q', # flake8-quotes + + 'RET', # flake8-return + 'RUF', # Ruff-specific rules + 'SIM', # flake8-simplify + 'T20', # flake8-print + 'TD', # flake8-todos + 'TRY', # tryceratops + 'UP', # pyupgrade +] + +[per-file-ignores] +'portalocker_tests/tests.py' = ['SIM115', 'SIM117'] + +[pydocstyle] +convention = 'google' +ignore-decorators = ['typing.overload'] + +[isort] +case-sensitive = true +combine-as-imports = true +force-wrap-aliases = true + +[flake8-quotes] +docstring-quotes = 'single' +inline-quotes = 'single' +multiline-quotes = 'single' diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c9bc65e..0000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[metadata] -description_file = README.rst - -[flake8] -ignore = - W391,E303,W503 -exclude = - docs/*.py diff --git a/setup.py b/setup.py deleted file mode 100644 index 796936e..0000000 --- a/setup.py +++ /dev/null @@ -1,123 +0,0 @@ -import os -import re -import typing - -import setuptools - -# To prevent importing about and thereby breaking the coverage info we use this -# exec hack -about: typing.Dict[str, str] = {} -with open('portalocker/__about__.py') as fp: - exec(fp.read(), about) - -tests_require = [ - 'pytest>=5.4.1', - 'pytest-cov>=2.8.1', - 'pytest-timeout>=2.1.0', - 'sphinx>=6.0.0', - 'pytest-mypy>=0.8.0', - 'redis', -] - - -class Combine(setuptools.Command): - description = 'Build single combined portalocker file' - relative_import_re = re.compile(r'^from \. import (?P.+)$', - re.MULTILINE) - user_options = [ - ('output-file=', 'o', 'Path to the combined output file'), - ] - - def initialize_options(self): - self.output_file = os.path.join( - 'dist', '%(package_name)s_%(version)s.py' % dict( - package_name=about['__package_name__'], - version=about['__version__'].replace('.', '-'), - )) - - def finalize_options(self): - pass - - def run(self): - dirname = os.path.dirname(self.output_file) - if dirname and not os.path.isdir(dirname): - os.makedirs(dirname) - - output = open(self.output_file, 'w') - print("'''", file=output) - with open('README.rst') as fh: - output.write(fh.read().rstrip()) - print('', file=output) - print('', file=output) - - with open('LICENSE') as fh: - output.write(fh.read().rstrip()) - - print('', file=output) - print("'''", file=output) - - names = set() - lines = [] - for line in open('portalocker/__init__.py'): - match = self.relative_import_re.match(line) - if match: - names.add(match.group('name')) - with open('portalocker/%(name)s.py' % match.groupdict()) as fh: - line = fh.read() - line = self.relative_import_re.sub('', line) - - lines.append(line) - - import_attributes = re.compile(r'\b(%s)\.' % '|'.join(names)) - for line in lines[:]: - line = import_attributes.sub('', line) - output.write(line) - - print('Wrote combined file to %r' % self.output_file) - - -if __name__ == '__main__': - setuptools.setup( - name=about['__package_name__'], - version=about['__version__'], - description=about['__description__'], - long_description=open('README.rst').read(), - classifiers=[ - 'Intended Audience :: Developers', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - ], - python_requires='>=3.8', - keywords='locking, locks, with statement, windows, linux, unix', - author=about['__author__'], - author_email=about['__email__'], - url=about['__url__'], - license='BSD-3-Clause', - package_data=dict(portalocker=['py.typed', 'msvcrt.pyi']), - packages=setuptools.find_packages(exclude=[ - 'examples', 'portalocker_tests']), - # zip_safe=False, - platforms=['any'], - cmdclass={ - 'combine': Combine, - }, - install_requires=[ - # Due to CVE-2021-32559 updating the pywin32 requirement - 'pywin32>=226; platform_system == "Windows"', - ], - tests_require=tests_require, - extras_require=dict( - docs=[ - 'sphinx>=1.7.1', - ], - tests=tests_require, - redis=[ - 'redis', - ] - ), - ) diff --git a/sourcery.yaml b/sourcery.yaml new file mode 100644 index 0000000..311a5b5 --- /dev/null +++ b/sourcery.yaml @@ -0,0 +1,2 @@ +ignore: + - portalocker_tests \ No newline at end of file diff --git a/tox.ini b/tox.ini index 6c1dfa6..f90bdc1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,19 @@ [tox] -envlist = py38, py39, py310, py311, pypy3, flake8, docs +envlist = + py38 + py39 + py310 + py311 + py312 + pypy3 + flake8 + docs + mypy + pyright + ruff + codespell + black + skip_missing_interpreters = True [testenv] @@ -10,6 +24,7 @@ basepython = py39: python3.9 py310: python3.10 py311: python3.11 + py312: python3.12 pypy3: pypy3 deps = -e{toxinidir}[tests,redis] @@ -20,11 +35,22 @@ basepython = python3 deps = mypy commands = mypy {toxinidir}/portalocker +[testenv:pyright] +changedir = +basepython = python3 +deps = pyright +commands = pyright {toxinidir}/portalocker + [testenv:flake8] basepython = python3 deps = flake8>=6.0.0 commands = flake8 {toxinidir}/portalocker {toxinidir}/portalocker_tests +[testenv:black] +basepython = python3 +deps = black +commands = black {toxinidir}/portalocker {toxinidir}/portalocker_tests + [testenv:docs] basepython = python3 deps = -r{toxinidir}/docs/requirements.txt @@ -41,3 +67,14 @@ commands = sphinx-apidoc -e -o docs/ portalocker rm -f docs/modules.rst sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html {posargs} + +[testenv:ruff] +commands = ruff check . +deps = ruff +skip_install = true + +[testenv:codespell] +commands = codespell . +deps = codespell +skip_install = true +command = codespell From ea5e1809e28d9197aaea22c4bbefe1dae2bcaef4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 02:01:35 +0200 Subject: [PATCH 146/225] fixed several linting/testing issues --- .github/workflows/lint.yml | 21 ++------------------- portalocker/portalocker.py | 6 ++++-- portalocker_tests/tests.py | 6 ++++-- tox.ini | 12 ++++++++---- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 12157b3..5a7a57c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,20 +2,18 @@ name: lint on: push: - branches: [ develop, master ] pull_request: - branches: [ develop ] env: FORCE_COLOR: 1 jobs: - lint: + tox: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.8', '3.10', '3.11'] steps: - uses: actions/checkout@v4 @@ -26,21 +24,6 @@ jobs: cache: 'pip' - name: Python version run: python --version - - name: Install dependencies - run: | - python -m pip install tox - - name: Test with pytest - run: tox -p all - - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - cache: 'pip' - name: Install dependencies run: | diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index fea66d2..90307b7 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -20,6 +20,10 @@ __overlapped = pywintypes.OVERLAPPED() def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): + # Windows locking does not support locking through `fh.fileno()` so + # we cast it to make mypy and pyright happy + file_ = typing.cast(typing.IO, file_) + mode = 0 if flags & LockFlags.NON_BLOCKING: mode |= win32con.LOCKFILE_FAIL_IMMEDIATELY @@ -27,8 +31,6 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): if flags & LockFlags.EXCLUSIVE: mode |= win32con.LOCKFILE_EXCLUSIVE_LOCK - # Windows locking does not support locking through `fh.fileno()` - assert isinstance(file_, typing.IO) # Save the old position so we can go back to that position but # still lock from the beginning of the file savepos = file_.tell() diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index b475396..a6fe310 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -173,7 +173,8 @@ def test_exlusive(tmpfile): # Make sure we can't write the locked file with pytest.raises(portalocker.LockException), open( - tmpfile, 'w+', + tmpfile, + 'w+', ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.write('surprise and fear') @@ -196,7 +197,8 @@ def test_shared(tmpfile): # Make sure we can't write the locked file with pytest.raises(portalocker.LockException), open( - tmpfile, 'w+', + tmpfile, + 'w+', ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.write('surprise and fear') diff --git a/tox.ini b/tox.ini index f90bdc1..29bf765 100644 --- a/tox.ini +++ b/tox.ini @@ -33,13 +33,17 @@ commands = python -m pytest {posargs} [testenv:mypy] basepython = python3 deps = mypy -commands = mypy {toxinidir}/portalocker +commands = + mypy --install-types --non-interactive + mypy [testenv:pyright] changedir = basepython = python3 -deps = pyright -commands = pyright {toxinidir}/portalocker +deps = + pyright + -e{toxinidir}[tests,redis] +commands = pyright {toxinidir}/portalocker {toxinidir}/portalocker_tests [testenv:flake8] basepython = python3 @@ -69,7 +73,7 @@ commands = sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html {posargs} [testenv:ruff] -commands = ruff check . +commands = ruff check {toxinidir}/portalocker {toxinidir}/portalocker_tests deps = ruff skip_install = true From 1021d1f5c5a09260ce254bd3ba3c9329eb2e7bb4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 04:02:15 +0200 Subject: [PATCH 147/225] Added NamedBoundedSemaphore to fix #87 --- portalocker/utils.py | 63 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index fe2fa60..b74c355 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -413,10 +413,10 @@ class BoundedSemaphore(LockBase): ''' Bounded semaphore to prevent too many parallel processes from running - It's also possible to specify a timeout when acquiring the lock to wait - for a resource to become available. This is very similar to - threading.BoundedSemaphore but works across multiple processes and across - multiple operating systems. + This method is deprecated because multiple processes that are completely + unrelated could end up using the same semaphore. To prevent this, + use `NamedBoundedSemaphore` instead. The + `NamedBoundedSemaphore` is a drop-in replacement for this class. >>> semaphore = BoundedSemaphore(2, directory='') >>> str(semaphore.get_filenames()[0]) @@ -448,6 +448,13 @@ def __init__( fail_when_locked=fail_when_locked, ) + if not name or name == 'bounded_semaphore': + warnings.warn( + '`BoundedSemaphore` without an explicit `name` ' + 'argument is deprecated, use NamedBoundedSemaphore', + DeprecationWarning, + ) + def get_filenames(self) -> typing.Sequence[pathlib.Path]: return [self.get_filename(n) for n in range(self.maximum)] @@ -505,3 +512,51 @@ def release(self): # pragma: no cover if self.lock is not None: self.lock.release() self.lock = None + + +class NamedBoundedSemaphore(BoundedSemaphore): + ''' + Bounded semaphore to prevent too many parallel processes from running + + It's also possible to specify a timeout when acquiring the lock to wait + for a resource to become available. This is very similar to + `threading.BoundedSemaphore` but works across multiple processes and across + multiple operating systems. + + Because this works across multiple processes it's important to give the + semaphore a name. This name is used to create the lock files. If you + don't specify a name, a random name will be generated. This means that + you can't use the same semaphore in multiple processes unless you pass the + semaphore object to the other processes. + + >>> semaphore = NamedBoundedSemaphore(2, name='test') + >>> str(semaphore.get_filenames()[0]) + '...test.00.lock' + + >>> semaphore = NamedBoundedSemaphore(2) + >>> 'bounded_semaphore' in str(semaphore.get_filenames()[0]) + True + + ''' + + def __init__( + self, + maximum: int, + name: typing.Optional[str] = None, + filename_pattern: str = '{name}.{number:02d}.lock', + directory: str = tempfile.gettempdir(), + timeout: typing.Optional[float] = DEFAULT_TIMEOUT, + check_interval: typing.Optional[float] = DEFAULT_CHECK_INTERVAL, + fail_when_locked: typing.Optional[bool] = True, + ): + if name is None: + name = 'bounded_semaphore.%d' % random.randint(0, 1000000) + super().__init__( + maximum, + name, + filename_pattern, + directory, + timeout, + check_interval, + fail_when_locked, + ) From 0b3940c425daf19633514d8764467ee90ed31a5b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 11:56:27 +0200 Subject: [PATCH 148/225] added explicit stacklevel to make ruff happy --- portalocker/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/portalocker/utils.py b/portalocker/utils.py index b74c355..3b5682e 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -453,6 +453,7 @@ def __init__( '`BoundedSemaphore` without an explicit `name` ' 'argument is deprecated, use NamedBoundedSemaphore', DeprecationWarning, + stacklevel=1, ) def get_filenames(self) -> typing.Sequence[pathlib.Path]: From ff7d2c24e415a6dbc39f10df4488456fcf987c99 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 12:40:11 +0200 Subject: [PATCH 149/225] testing with python 3.12 --- .github/workflows/lint.yml | 6 +++--- .github/workflows/python-package.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5a7a57c..3c5dd50 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,12 +8,12 @@ env: FORCE_COLOR: 1 jobs: - tox: + lint: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ['3.8', '3.8', '3.10', '3.11'] + python-version: ['3.8', '3.8', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 @@ -33,7 +33,7 @@ jobs: - name: Linting with pyright uses: jakebailey/pyright-action@v1 with: - extra-args: portalocker portalocker_tests + path: portalocker portalocker_tests - name: Linting with ruff uses: jpetrucciani/ruff-check@main diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 71209cc..e23feaf 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] os: ['macos-latest', 'windows-latest'] steps: @@ -41,7 +41,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 From 58d2552b36a9cc3829d0a03a146843fa064f5829 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 12:41:35 +0200 Subject: [PATCH 150/225] disabled python 3.12 testing until the test environments supports it --- .github/workflows/lint.yml | 2 +- .github/workflows/python-package.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3c5dd50..4c92829 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.8', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.8', '3.10', '3.11'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e23feaf..71209cc 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11'] os: ['macos-latest', 'windows-latest'] steps: @@ -41,7 +41,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v4 From 1bf6d4cd38b0a3898325c70aa5bef1906668b36f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 12:55:31 +0200 Subject: [PATCH 151/225] Incrementing version to v2.8.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 51415ff..b6e8cfe 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,6 +1,6 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.7.0' +__version__ = '2.8.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index f52941c..d996231 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -13,7 +13,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = __about__.__version__ +__version__ = '2.8.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 17b9be50c2ec00af2bb5ce36d890e48928f0f5b5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 16:20:41 +0200 Subject: [PATCH 152/225] Removed docs from build to fix #88 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3a76033..fd7551d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ include-package-data = false version = { attr = 'portalocker.__about__.__version__' } [tool.setuptools.packages.find] -exclude = ['examples','portalocker_tests'] +exclude = ['docs','portalocker_tests'] [tool.setuptools.package-data] portalocker = ['py.typed', 'msvcrt.pyi'] From 9b8ee01699f577be66372f064c9c86ba2542d76e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 16:20:59 +0200 Subject: [PATCH 153/225] Incrementing version to v2.8.1 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index b6e8cfe..baac4fe 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,6 +1,6 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.8.0' +__version__ = '2.8.1' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index d996231..2dc4b7e 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -13,7 +13,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.8.0' +__version__ = '2.8.1' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 3b43b84b23a3fcb3c99ffe09456eec66fbfa397b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 16:57:46 +0200 Subject: [PATCH 154/225] attempt #2 to fix #88 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fd7551d..e4312c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ include-package-data = false version = { attr = 'portalocker.__about__.__version__' } [tool.setuptools.packages.find] -exclude = ['docs','portalocker_tests'] +include = ['portalocker'] [tool.setuptools.package-data] portalocker = ['py.typed', 'msvcrt.pyi'] From 365c5f645c7d0af9c2d66c5a6520d1ee056f0a7b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 16 Sep 2023 16:57:52 +0200 Subject: [PATCH 155/225] Incrementing version to v2.8.2 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index baac4fe..e45c443 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,6 +1,6 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.8.1' +__version__ = '2.8.2' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 2dc4b7e..9170e33 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -13,7 +13,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.8.1' +__version__ = '2.8.2' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 37dd63cdab5ce4d33d5c5f7a0342acfad002d418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Sun, 17 Sep 2023 06:46:24 +0200 Subject: [PATCH 156/225] Remove redundant wheel dep from pyproject.toml Remove the redundant `wheel` dependency, as it is added by the backend automatically. Listing it explicitly in the documentation was a historical mistake and has been fixed since, see: https://github.com/pypa/setuptools/commit/f7d30a9529378cf69054b5176249e5457aaf640a --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e4312c3..0c61e2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] build-backend = 'setuptools.build_meta' -requires = ['setuptools', 'setuptools-scm', 'wheel'] +requires = ['setuptools', 'setuptools-scm'] [project] name = 'portalocker' From 92e51301aa93b78c7672f17a3dbdc5d2ecc46bae Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Mon, 11 Mar 2024 20:05:28 +0000 Subject: [PATCH 157/225] Fixed try..except block losing stack trace --- portalocker/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 3b5682e..412879d 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -287,7 +287,7 @@ def try_close(): # pragma: no cover if exception: try_close() # We got a timeout... reraising - raise exceptions.LockException(exception) + raise exception # Prepare the filehandle (truncate if needed) fh = self._prepare_fh(fh) From 4d4ca654ad2a17ac296d41d90a456a8af531b06a Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Mon, 11 Mar 2024 20:07:50 +0000 Subject: [PATCH 158/225] Fixed try..except around fcntl.flock() accidentally catching non-timeout errors and hiding them until the timeout expires --- portalocker/portalocker.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 90307b7..5d51e62 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -88,11 +88,9 @@ def unlock(file_: typing.IO): elif os.name == 'posix': # pragma: no cover import fcntl + import errno def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): - locking_exceptions = (IOError,) - with contextlib.suppress(NameError): - locking_exceptions += (BlockingIOError,) # type: ignore # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled results # in an error if (flags & LockFlags.NON_BLOCKING) and not flags & ( @@ -105,10 +103,18 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): try: fcntl.flock(file_, flags) - except locking_exceptions as exc_value: - # The exception code varies on different systems so we'll catch - # every IO error - raise exceptions.LockException(exc_value, fh=file_) from exc_value + except OSError as exc_value: + # Python can use one of several different exception classes to represent + # timeout (most likely is BlockingIOError and IOError), but these errors + # may also represent other failures. On some systems, `IOError is OSError` + # which means checking for either IOError or OSError can mask other errors. + # The safest check is to catch OSError (from which the others inherit) + # and check the errno (which should be EACCESS or EAGAIN according to the + # spec). + if exc_value.errno in (errno.EACCES, errno.EAGAIN): + raise exceptions.LockException(exc_value, fh=file_) from exc_value + else: + raise def unlock(file_: typing.IO): fcntl.flock(file_.fileno(), LockFlags.UNBLOCK) From 75f023637596a389f33f89bd08e7b9571663c948 Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Mon, 11 Mar 2024 20:40:30 +0000 Subject: [PATCH 159/225] Added support for changing the locking implementation on linux (lockf vs flock) --- portalocker/portalocker.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 5d51e62..e1d2e26 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -90,6 +90,11 @@ def unlock(file_: typing.IO): import fcntl import errno + # The locking implementation. + # Expected values are either fcntl.flock() or fcntl.lockf(), + # but any callable that matches the syntax will be accepted. + LOCKER = fcntl.flock + def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled results # in an error @@ -102,7 +107,7 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): ) try: - fcntl.flock(file_, flags) + LOCKER(file_, flags) except OSError as exc_value: # Python can use one of several different exception classes to represent # timeout (most likely is BlockingIOError and IOError), but these errors @@ -117,7 +122,8 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): raise def unlock(file_: typing.IO): - fcntl.flock(file_.fileno(), LockFlags.UNBLOCK) + LOCKER(file_.fileno(), LockFlags.UNBLOCK) + else: # pragma: no cover raise RuntimeError('PortaLocker only defined for nt and posix platforms') From ed3efdb3afd201868e08ef1e45da500cb624ff10 Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Mon, 11 Mar 2024 22:12:54 +0000 Subject: [PATCH 160/225] Fixed opening files readonly and locking with EXCLUSIVE or vice versa in tests --- portalocker_tests/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index a6fe310..e0c2736 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -163,11 +163,11 @@ def test_exlusive(tmpfile): with open(tmpfile, 'w') as fh: fh.write('spam and eggs') - with open(tmpfile) as fh: + with open(tmpfile, "w") as fh: portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB) # Make sure we can't read the locked file - with pytest.raises(portalocker.LockException), open(tmpfile) as fh2: + with pytest.raises(portalocker.LockException), open(tmpfile, "r+") as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.read() @@ -211,10 +211,10 @@ def test_blocking_timeout(tmpfile): flags = LockFlags.SHARED with pytest.warns(UserWarning): - with portalocker.Lock(tmpfile, timeout=5, flags=flags): + with portalocker.Lock(tmpfile, "a+", timeout=5, flags=flags): pass - lock = portalocker.Lock(tmpfile, flags=flags) + lock = portalocker.Lock(tmpfile, "a+", flags=flags) with pytest.warns(UserWarning): lock.acquire(timeout=5) @@ -330,9 +330,9 @@ def test_exclusive_processes(tmpfile, fail_when_locked): reason='Locking on Windows requires a file object', ) def test_lock_fileno(tmpfile): - with open(tmpfile, 'a') as a: - with open(tmpfile, 'a') as b: - # Lock exclusive non-blocking + with open(tmpfile, 'a+') as a: + with open(tmpfile, 'a+') as b: + # Lock shared non-blocking flags = LockFlags.SHARED | LockFlags.NON_BLOCKING # First lock file a From 2335cd57894f2874bd651a34c0167020d9844dcd Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Mon, 11 Mar 2024 22:13:31 +0000 Subject: [PATCH 161/225] Added tests for both flock and lockf mechanisms --- portalocker_tests/tests.py | 75 +++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index e0c2736..ec97549 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -3,6 +3,8 @@ import os import time import typing +import fcntl +import portalocker.portalocker import pytest @@ -10,6 +12,16 @@ from portalocker import LockFlags, utils +@pytest.fixture +def locker(request): + # Setup + locker = portalocker.portalocker.LOCKER + portalocker.portalocker.LOCKER = request.param + yield request.param + # Teardown. + portalocker.portalocker.LOCKER = locker + + def test_exceptions(tmpfile): with open(tmpfile, 'a') as a, open(tmpfile, 'a') as b: # Lock exclusive non-blocking @@ -206,8 +218,16 @@ def test_shared(tmpfile): # Make sure we can explicitly unlock the file portalocker.unlock(f) - -def test_blocking_timeout(tmpfile): +@pytest.mark.parametrize( + 'locker', + [ + fcntl.flock, + pytest.param( + fcntl.lockf, marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) + ], + indirect=['locker'] +) +def test_blocking_timeout(tmpfile, locker): flags = LockFlags.SHARED with pytest.warns(UserWarning): @@ -223,7 +243,8 @@ def test_blocking_timeout(tmpfile): os.name == 'nt', reason='Windows uses an entirely different lockmechanism', ) -def test_nonblocking(tmpfile): +@pytest.mark.parametrize('locker', [fcntl.flock, fcntl.lockf], indirect=['locker']) +def test_nonblocking(tmpfile, locker): with open(tmpfile, 'w') as fh, pytest.raises(RuntimeError): portalocker.lock(fh, LockFlags.NON_BLOCKING) @@ -297,8 +318,17 @@ def lock( ) +@pytest.mark.parametrize( + 'locker', + [ + fcntl.flock, + pytest.param( + fcntl.lockf, marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) + ], + indirect=['locker'] +) @pytest.mark.parametrize('fail_when_locked', [True, False]) -def test_shared_processes(tmpfile, fail_when_locked): +def test_shared_processes(tmpfile, fail_when_locked, locker): flags = LockFlags.SHARED | LockFlags.NON_BLOCKING with multiprocessing.Pool(processes=2) as pool: @@ -309,8 +339,17 @@ def test_shared_processes(tmpfile, fail_when_locked): assert result == LockResult() +@pytest.mark.parametrize( + 'locker', + [ + fcntl.flock, + pytest.param( + fcntl.lockf, marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) + ], + indirect=['locker'] +) @pytest.mark.parametrize('fail_when_locked', [True, False]) -def test_exclusive_processes(tmpfile, fail_when_locked): +def test_exclusive_processes(tmpfile, fail_when_locked, locker): flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING with multiprocessing.Pool(processes=2) as pool: @@ -329,7 +368,8 @@ def test_exclusive_processes(tmpfile, fail_when_locked): os.name == 'nt', reason='Locking on Windows requires a file object', ) -def test_lock_fileno(tmpfile): +@pytest.mark.parametrize('locker', [fcntl.flock, fcntl.lockf], indirect=['locker']) +def test_lock_fileno(tmpfile, locker): with open(tmpfile, 'a+') as a: with open(tmpfile, 'a+') as b: # Lock shared non-blocking @@ -340,3 +380,26 @@ def test_lock_fileno(tmpfile): # Now see if we can lock using fileno() portalocker.lock(b.fileno(), flags) + + +@pytest.mark.skipif( + os.name == 'nt', + reason='Windows only has one locking mechanism', +) +@pytest.mark.parametrize('locker', [fcntl.flock, fcntl.lockf], indirect=['locker']) +def test_locker_mechanism(tmpfile, locker): + """Can we switch the locking mechanism?""" + # We can test for flock vs lockf based on their different behaviour re. locking + # the same file. + with portalocker.Lock(tmpfile, "a+", flags = LockFlags.EXCLUSIVE) as a: + # If we have flock(), we cannot get another lock on the same file. + if locker is fcntl.flock: + with pytest.raises(portalocker.LockException): + portalocker.Lock(tmpfile, "r+", flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING).acquire() + + elif locker is fcntl.lockf: + # But on lockf, we can! + portalocker.Lock(tmpfile, "r+", flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING).acquire() + + else: + raise Exception("Update test") From a0de45a6885e36373a7173ce1acfd6186c0062df Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Tue, 12 Mar 2024 09:03:54 +0000 Subject: [PATCH 162/225] Added some comments to explain the exception raising logic in portalocker.lock() --- portalocker/portalocker.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index e1d2e26..e6ad6b9 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -117,8 +117,13 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): # and check the errno (which should be EACCESS or EAGAIN according to the # spec). if exc_value.errno in (errno.EACCES, errno.EAGAIN): + # A timeout exception, wrap this so the outer code knows to try again + # (if it wants to). + # TODO: Would AlreadyLocked by a better error to raise here? Or perhaps + # a new exception class like TryAgain? raise exceptions.LockException(exc_value, fh=file_) from exc_value else: + # Something else went wrong; don't wrap this so we stop immediately. raise def unlock(file_: typing.IO): From bb31c06473db426d35eb8f7f5c2f7528b6d3b76d Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Tue, 12 Mar 2024 09:04:46 +0000 Subject: [PATCH 163/225] Added catch for non LockException classes to make sure the file handle isn't left open in case of failure --- portalocker/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index 412879d..890d2b9 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -281,8 +281,13 @@ def try_close(): # pragma: no cover if fail_when_locked: try_close() raise exceptions.AlreadyLocked(exception) from exc + except Exception as exc: + # Something went wrong with the locking mechanism. + # Wrap in a LockException and re-raise: + try_close() + raise exceptions.LockException(exc) from exc - # Wait a bit + # Wait a bit if exception: try_close() From c348de8e1d9cdc93aa82f46f14fe27e140ceb8b5 Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Fri, 26 Apr 2024 08:53:08 +0100 Subject: [PATCH 164/225] Removed fcntl import from tests so windows tests can pass --- portalocker_tests/tests.py | 43 +++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index ec97549..46454af 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -3,7 +3,6 @@ import os import time import typing -import fcntl import portalocker.portalocker import pytest @@ -15,11 +14,25 @@ @pytest.fixture def locker(request): # Setup - locker = portalocker.portalocker.LOCKER - portalocker.portalocker.LOCKER = request.param + if os.name == 'posix': + import fcntl + old_locker = portalocker.portalocker.LOCKER + if request.param == "flock": + new_locker = fcntl.flock + + elif request.param == 'lockf': + new_locker = fcntl.lockf + + else: + raise ValueError("Unrecognised locking mechanism '{}'".format(request.param)) + + portalocker.portalocker.LOCKER = new_locker + yield request.param + # Teardown. - portalocker.portalocker.LOCKER = locker + if os.name == 'posix': + portalocker.portalocker.LOCKER = old_locker def test_exceptions(tmpfile): @@ -221,9 +234,9 @@ def test_shared(tmpfile): @pytest.mark.parametrize( 'locker', [ - fcntl.flock, + 'flock', pytest.param( - fcntl.lockf, marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) + 'lockf', marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) ], indirect=['locker'] ) @@ -243,7 +256,7 @@ def test_blocking_timeout(tmpfile, locker): os.name == 'nt', reason='Windows uses an entirely different lockmechanism', ) -@pytest.mark.parametrize('locker', [fcntl.flock, fcntl.lockf], indirect=['locker']) +@pytest.mark.parametrize('locker', ['flock', 'lockf'], indirect=['locker']) def test_nonblocking(tmpfile, locker): with open(tmpfile, 'w') as fh, pytest.raises(RuntimeError): portalocker.lock(fh, LockFlags.NON_BLOCKING) @@ -321,9 +334,9 @@ def lock( @pytest.mark.parametrize( 'locker', [ - fcntl.flock, + 'flock', pytest.param( - fcntl.lockf, marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) + 'lockf', marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) ], indirect=['locker'] ) @@ -342,9 +355,9 @@ def test_shared_processes(tmpfile, fail_when_locked, locker): @pytest.mark.parametrize( 'locker', [ - fcntl.flock, + 'flock', pytest.param( - fcntl.lockf, marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) + 'lockf', marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) ], indirect=['locker'] ) @@ -368,7 +381,7 @@ def test_exclusive_processes(tmpfile, fail_when_locked, locker): os.name == 'nt', reason='Locking on Windows requires a file object', ) -@pytest.mark.parametrize('locker', [fcntl.flock, fcntl.lockf], indirect=['locker']) +@pytest.mark.parametrize('locker', ['flock', 'lockf'], indirect=['locker']) def test_lock_fileno(tmpfile, locker): with open(tmpfile, 'a+') as a: with open(tmpfile, 'a+') as b: @@ -386,18 +399,18 @@ def test_lock_fileno(tmpfile, locker): os.name == 'nt', reason='Windows only has one locking mechanism', ) -@pytest.mark.parametrize('locker', [fcntl.flock, fcntl.lockf], indirect=['locker']) +@pytest.mark.parametrize('locker', ['flock', 'lockf'], indirect=['locker']) def test_locker_mechanism(tmpfile, locker): """Can we switch the locking mechanism?""" # We can test for flock vs lockf based on their different behaviour re. locking # the same file. with portalocker.Lock(tmpfile, "a+", flags = LockFlags.EXCLUSIVE) as a: # If we have flock(), we cannot get another lock on the same file. - if locker is fcntl.flock: + if locker is 'flock': with pytest.raises(portalocker.LockException): portalocker.Lock(tmpfile, "r+", flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING).acquire() - elif locker is fcntl.lockf: + elif locker is 'lockf': # But on lockf, we can! portalocker.Lock(tmpfile, "r+", flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING).acquire() From 1a7fba498cb97b48bb15f30814150d422cf3dd24 Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Fri, 26 Apr 2024 10:53:47 +0100 Subject: [PATCH 165/225] Added test for missing coverage of exception logic --- portalocker_tests/tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 46454af..49ea5d6 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -4,6 +4,8 @@ import time import typing import portalocker.portalocker +from portalocker import exceptions +import math import pytest @@ -416,3 +418,11 @@ def test_locker_mechanism(tmpfile, locker): else: raise Exception("Update test") + +def test_exception(tmpfile): + """Do we stop immediately if the locking fails, even with a timeout?""" + # NON_BLOCKING is not allowed by itself + lock = portalocker.Lock(tmpfile, "w", timeout = math.inf, flags = LockFlags.NON_BLOCKING) + + with pytest.raises(exceptions.LockException): + lock.acquire() From 47a620b0187038df88f20c8b12dd018c324eae34 Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Fri, 26 Apr 2024 10:58:27 +0100 Subject: [PATCH 166/225] Fixed instance comparison for string literal --- portalocker_tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 49ea5d6..b85dd5d 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -408,11 +408,11 @@ def test_locker_mechanism(tmpfile, locker): # the same file. with portalocker.Lock(tmpfile, "a+", flags = LockFlags.EXCLUSIVE) as a: # If we have flock(), we cannot get another lock on the same file. - if locker is 'flock': + if locker == 'flock': with pytest.raises(portalocker.LockException): portalocker.Lock(tmpfile, "r+", flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING).acquire() - elif locker is 'lockf': + elif locker == 'lockf': # But on lockf, we can! portalocker.Lock(tmpfile, "r+", flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING).acquire() From c55199f39058b4828897ed49300dcd3c77ec9343 Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Fri, 26 Apr 2024 11:21:50 +0100 Subject: [PATCH 167/225] Added skip for exception test on windows --- portalocker_tests/tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index b85dd5d..7f4669c 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -418,7 +418,11 @@ def test_locker_mechanism(tmpfile, locker): else: raise Exception("Update test") - + +@pytest.mark.skipif( + os.name == 'nt', + reason='Windows has a different locking mechanism', +) def test_exception(tmpfile): """Do we stop immediately if the locking fails, even with a timeout?""" # NON_BLOCKING is not allowed by itself From 07e66c440c889b6a20b3a43614a5072ca8760ffe Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Fri, 26 Apr 2024 14:33:43 +0100 Subject: [PATCH 168/225] Added patch for test in a desperate attempt to satisfy code coverage for windows --- portalocker_tests/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 7f4669c..d97e35a 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -419,14 +419,14 @@ def test_locker_mechanism(tmpfile, locker): else: raise Exception("Update test") -@pytest.mark.skipif( - os.name == 'nt', - reason='Windows has a different locking mechanism', -) -def test_exception(tmpfile): + +def test_exception(monkeypatch, tmpfile): """Do we stop immediately if the locking fails, even with a timeout?""" - # NON_BLOCKING is not allowed by itself - lock = portalocker.Lock(tmpfile, "w", timeout = math.inf, flags = LockFlags.NON_BLOCKING) + def patched_lock(*args, **kwargs): + raise ValueError("Test exception") + + monkeypatch.setattr('portalocker.utils.portalocker.lock', patched_lock) + lock = portalocker.Lock(tmpfile, "w", timeout = math.inf) with pytest.raises(exceptions.LockException): lock.acquire() From 2261180d1853be8ba1c8561766a2bf6b31fa8338 Mon Sep 17 00:00:00 2001 From: oliver-s-lee Date: Sat, 11 May 2024 09:04:23 +0100 Subject: [PATCH 169/225] Added debugging to try and fix tests --- portalocker_tests/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index d97e35a..4e72799 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -351,6 +351,8 @@ def test_shared_processes(tmpfile, fail_when_locked, locker): results = pool.starmap_async(lock, 2 * [args]) for result in results.get(timeout=3): + if result.exception_class != None: + raise result.exception_class assert result == LockResult() From 96d0621a4b4dca9bb5ffaf0e06f50f5692c93f8d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 11 May 2024 11:52:46 +0200 Subject: [PATCH 170/225] fixed pyright errors --- portalocker/redis.py | 2 +- portalocker/utils.py | 9 +++++---- ruff.toml | 1 - 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/portalocker/redis.py b/portalocker/redis.py index 59ee5ff..2a84807 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -131,7 +131,7 @@ def acquire( timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = None, - ): + ) -> 'RedisLock': timeout = utils.coalesce(timeout, self.timeout, 0.0) check_interval = utils.coalesce( check_interval, diff --git a/portalocker/utils.py b/portalocker/utils.py index 3b5682e..701d113 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -132,8 +132,8 @@ def acquire( timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = None, - ): - return NotImplemented + ) -> typing.Union['LockBase', typing.IO[typing.AnyStr]]: + ... def _timeout_generator( self, @@ -157,7 +157,8 @@ def _timeout_generator( @abc.abstractmethod def release(self): - return NotImplemented + ... + def __enter__(self): return self.acquire() @@ -470,7 +471,7 @@ def get_filename(self, number) -> pathlib.Path: number=number, ) - def acquire( + def acquire( # type: ignore[reportGeneralTypeIssues] self, timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, diff --git a/ruff.toml b/ruff.toml index e484f82..886defd 100644 --- a/ruff.toml +++ b/ruff.toml @@ -6,7 +6,6 @@ target-version = 'py38' src = ['portalocker'] exclude = ['docs'] -format = 'grouped' ignore = [ 'A001', # Variable {name} is shadowing a Python builtin 'A002', # Argument {name} is shadowing a Python builtin From a3acc9a173527ba7b1ee092b21750799f58910d0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 11 May 2024 14:58:21 +0200 Subject: [PATCH 171/225] fixed several typing, linting and other small issues --- portalocker/__init__.py | 9 +-- portalocker/__main__.py | 14 +++-- portalocker/constants.py | 1 + portalocker/portalocker.py | 42 ++++++++----- portalocker/utils.py | 14 ++--- portalocker_tests/tests.py | 125 ++++++++++++++++++++++--------------- ruff.toml | 12 ++-- 7 files changed, 128 insertions(+), 89 deletions(-) diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 9170e33..b5026f4 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -1,4 +1,5 @@ -from . import __about__, constants, exceptions, portalocker, utils +from . import __about__, constants, exceptions, portalocker +from .utils import BoundedSemaphore, Lock, RLock, TemporaryFileLock, open_atomic try: # pragma: no cover from .redis import RedisLock @@ -52,11 +53,6 @@ #: Locking utility class to automatically handle opening with timeouts and #: context wrappers -Lock = utils.Lock -RLock = utils.RLock -BoundedSemaphore = utils.BoundedSemaphore -TemporaryFileLock = utils.TemporaryFileLock -open_atomic = utils.open_atomic __all__ = [ 'lock', @@ -71,6 +67,7 @@ 'RLock', 'AlreadyLocked', 'BoundedSemaphore', + 'TemporaryFileLock', 'open_atomic', 'RedisLock', ] diff --git a/portalocker/__main__.py b/portalocker/__main__.py index 658a3ec..25bdf23 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -9,7 +9,7 @@ dist_path = base_path / 'dist' _default_output_path = base_path / 'dist' / 'portalocker.py' -_RELATIVE_IMPORT_RE = re.compile(r'^from \. import (?P.+)$') +_RELATIVE_IMPORT_RE = re.compile(r'^from \.(?P.*?) import (?P.+)$') _USELESS_ASSIGNMENT_RE = re.compile(r'^(?P\w+) = \1\n$') _TEXT_TEMPLATE = """''' @@ -50,10 +50,14 @@ def _read_file(path, seen_files): seen_files.add(path) for line in path.open(): if match := _RELATIVE_IMPORT_RE.match(line): - for name in match.group('names').split(','): - name = name.strip() - names.add(name) - yield from _read_file(src_path / f'{name}.py', seen_files) + from_ = match.group('from') + if from_: + yield from _read_file(src_path / f'{from_}.py', seen_files) + else: + for name in match.group('names').split(','): + name = name.strip() + names.add(name) + yield from _read_file(src_path / f'{name}.py', seen_files) else: yield _clean_line(line, names) diff --git a/portalocker/constants.py b/portalocker/constants.py index 72733c8..2099f1f 100644 --- a/portalocker/constants.py +++ b/portalocker/constants.py @@ -14,6 +14,7 @@ - `UNBLOCK` unlock ''' + import enum import os diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index e6ad6b9..e7b0585 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -1,4 +1,3 @@ -import contextlib import os import typing @@ -87,8 +86,8 @@ def unlock(file_: typing.IO): ) from exc elif os.name == 'posix': # pragma: no cover - import fcntl import errno + import fcntl # The locking implementation. # Expected values are either fcntl.flock() or fcntl.lockf(), @@ -109,26 +108,35 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): try: LOCKER(file_, flags) except OSError as exc_value: - # Python can use one of several different exception classes to represent - # timeout (most likely is BlockingIOError and IOError), but these errors - # may also represent other failures. On some systems, `IOError is OSError` - # which means checking for either IOError or OSError can mask other errors. - # The safest check is to catch OSError (from which the others inherit) - # and check the errno (which should be EACCESS or EAGAIN according to the - # spec). + # Python can use one of several different exception classes to + # represent timeout (most likely is BlockingIOError and IOError), + # but these errors may also represent other failures. On some + # systems, `IOError is OSError` which means checking for either + # IOError or OSError can mask other errors. + # The safest check is to catch OSError (from which the others + # inherit) and check the errno (which should be EACCESS or EAGAIN + # according to the spec). if exc_value.errno in (errno.EACCES, errno.EAGAIN): - # A timeout exception, wrap this so the outer code knows to try again - # (if it wants to). - # TODO: Would AlreadyLocked by a better error to raise here? Or perhaps - # a new exception class like TryAgain? - raise exceptions.LockException(exc_value, fh=file_) from exc_value + # A timeout exception, wrap this so the outer code knows to try + # again (if it wants to). + raise exceptions.AlreadyLocked( + exc_value, fh=file_, + ) from exc_value else: - # Something else went wrong; don't wrap this so we stop immediately. - raise + # Something else went wrong; don't wrap this so we stop + # immediately. + raise exceptions.LockException( + exc_value, fh=file_, + ) from exc_value + except EOFError as exc_value: + # On NFS filesystems, flock can raise an EOFError + raise exceptions.LockException( + exc_value, + fh=file_, + ) from exc_value def unlock(file_: typing.IO): LOCKER(file_.fileno(), LockFlags.UNBLOCK) - else: # pragma: no cover raise RuntimeError('PortaLocker only defined for nt and posix platforms') diff --git a/portalocker/utils.py b/portalocker/utils.py index 29d9830..f2446ac 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -132,8 +132,7 @@ def acquire( timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = None, - ) -> typing.Union['LockBase', typing.IO[typing.AnyStr]]: - ... + ) -> typing.Union['LockBase', typing.IO[typing.AnyStr]]: ... def _timeout_generator( self, @@ -156,11 +155,9 @@ def _timeout_generator( time.sleep(max(0.001, (i * f_check_interval) - since_start_time)) @abc.abstractmethod - def release(self): - ... - + def release(self): ... - def __enter__(self): + def __enter__(self) -> typing.Union['LockBase', typing.IO[typing.AnyStr]]: return self.acquire() def __exit__( @@ -236,7 +233,7 @@ def acquire( timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = None, - ) -> typing.IO: + ) -> typing.IO[typing.AnyStr]: '''Acquire the locked filehandle''' fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) @@ -301,6 +298,9 @@ def try_close(): # pragma: no cover self.fh = fh return fh + def __enter__(self) -> typing.IO[typing.AnyStr]: + return self.acquire() + def release(self): '''Releases the currently locked file handle''' if self.fh: diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 4e72799..cb0a241 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -1,16 +1,15 @@ import dataclasses +import math import multiprocessing import os import time import typing -import portalocker.portalocker -from portalocker import exceptions -import math import pytest import portalocker -from portalocker import LockFlags, utils +import portalocker.portalocker +from portalocker import LockFlags, exceptions, utils @pytest.fixture @@ -18,15 +17,18 @@ def locker(request): # Setup if os.name == 'posix': import fcntl + old_locker = portalocker.portalocker.LOCKER - if request.param == "flock": + if request.param == 'flock': new_locker = fcntl.flock - + elif request.param == 'lockf': new_locker = fcntl.lockf - + else: - raise ValueError("Unrecognised locking mechanism '{}'".format(request.param)) + raise ValueError( + f'Unrecognised locking mechanism {request.param!r}', + ) portalocker.portalocker.LOCKER = new_locker @@ -187,16 +189,19 @@ def test_release_unacquired(tmpfile): def test_exlusive(tmpfile): + text_0 = 'spam and eggs' with open(tmpfile, 'w') as fh: - fh.write('spam and eggs') + fh.write(text_0) - with open(tmpfile, "w") as fh: + with open(tmpfile) as fh: portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB) # Make sure we can't read the locked file - with pytest.raises(portalocker.LockException), open(tmpfile, "r+") as fh2: + with pytest.raises(portalocker.LockException), open( + tmpfile, 'r+', + ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) - fh2.read() + assert fh2.read() == text_0 # Make sure we can't write the locked file with pytest.raises(portalocker.LockException), open( @@ -233,23 +238,28 @@ def test_shared(tmpfile): # Make sure we can explicitly unlock the file portalocker.unlock(f) + @pytest.mark.parametrize( - 'locker', - [ - 'flock', - pytest.param( - 'lockf', marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) - ], - indirect=['locker'] + 'locker', + [ + 'flock', + pytest.param( + 'lockf', + marks=pytest.mark.skipif( + os.name == 'nt', reason='lockf() is not available on windows', + ), + ), + ], + indirect=['locker'], ) def test_blocking_timeout(tmpfile, locker): flags = LockFlags.SHARED with pytest.warns(UserWarning): - with portalocker.Lock(tmpfile, "a+", timeout=5, flags=flags): + with portalocker.Lock(tmpfile, 'a+', timeout=5, flags=flags): pass - lock = portalocker.Lock(tmpfile, "a+", flags=flags) + lock = portalocker.Lock(tmpfile, 'a+', flags=flags) with pytest.warns(UserWarning): lock.acquire(timeout=5) @@ -334,13 +344,17 @@ def lock( @pytest.mark.parametrize( - 'locker', - [ - 'flock', - pytest.param( - 'lockf', marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) - ], - indirect=['locker'] + 'locker', + [ + 'flock', + pytest.param( + 'lockf', + marks=pytest.mark.skipif( + os.name == 'nt', reason='lockf() is not available on windows', + ), + ), + ], + indirect=['locker'], ) @pytest.mark.parametrize('fail_when_locked', [True, False]) def test_shared_processes(tmpfile, fail_when_locked, locker): @@ -351,19 +365,23 @@ def test_shared_processes(tmpfile, fail_when_locked, locker): results = pool.starmap_async(lock, 2 * [args]) for result in results.get(timeout=3): - if result.exception_class != None: + if result.exception_class is not None: raise result.exception_class assert result == LockResult() @pytest.mark.parametrize( - 'locker', - [ - 'flock', - pytest.param( - 'lockf', marks=pytest.mark.skipif(os.name == "nt", reason = "lockf() is not available on windows")) - ], - indirect=['locker'] + 'locker', + [ + 'flock', + pytest.param( + 'lockf', + marks=pytest.mark.skipif( + os.name == 'nt', reason='lockf() is not available on windows', + ), + ), + ], + indirect=['locker'], ) @pytest.mark.parametrize('fail_when_locked', [True, False]) def test_exclusive_processes(tmpfile, fail_when_locked, locker): @@ -405,30 +423,39 @@ def test_lock_fileno(tmpfile, locker): ) @pytest.mark.parametrize('locker', ['flock', 'lockf'], indirect=['locker']) def test_locker_mechanism(tmpfile, locker): - """Can we switch the locking mechanism?""" - # We can test for flock vs lockf based on their different behaviour re. locking - # the same file. - with portalocker.Lock(tmpfile, "a+", flags = LockFlags.EXCLUSIVE) as a: + '''Can we switch the locking mechanism?''' + # We can test for flock vs lockf based on their different behaviour re. + # locking the same file. + with portalocker.Lock(tmpfile, 'a+', flags=LockFlags.EXCLUSIVE): # If we have flock(), we cannot get another lock on the same file. if locker == 'flock': with pytest.raises(portalocker.LockException): - portalocker.Lock(tmpfile, "r+", flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING).acquire() - + portalocker.Lock( + tmpfile, + 'r+', + flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, + ).acquire() + elif locker == 'lockf': # But on lockf, we can! - portalocker.Lock(tmpfile, "r+", flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING).acquire() - + portalocker.Lock( + tmpfile, + 'r+', + flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, + ).acquire() + else: - raise Exception("Update test") - + raise RuntimeError('Update test') + def test_exception(monkeypatch, tmpfile): - """Do we stop immediately if the locking fails, even with a timeout?""" + '''Do we stop immediately if the locking fails, even with a timeout?''' + def patched_lock(*args, **kwargs): - raise ValueError("Test exception") + raise ValueError('Test exception') monkeypatch.setattr('portalocker.utils.portalocker.lock', patched_lock) - lock = portalocker.Lock(tmpfile, "w", timeout = math.inf) + lock = portalocker.Lock(tmpfile, 'w', timeout=math.inf) with pytest.raises(exceptions.LockException): lock.acquire() diff --git a/ruff.toml b/ruff.toml index 886defd..d99665b 100644 --- a/ruff.toml +++ b/ruff.toml @@ -6,6 +6,9 @@ target-version = 'py38' src = ['portalocker'] exclude = ['docs'] +line-length = 80 + +[lint] ignore = [ 'A001', # Variable {name} is shadowing a Python builtin 'A002', # Argument {name} is shadowing a Python builtin @@ -25,7 +28,6 @@ ignore = [ 'FA100', # Missing `from __future__ import annotations`, but uses `typing.Optional` ] -line-length = 80 select = [ 'A', # flake8-builtins 'ASYNC', # flake8 async checker @@ -59,19 +61,19 @@ select = [ 'UP', # pyupgrade ] -[per-file-ignores] +[lint.per-file-ignores] 'portalocker_tests/tests.py' = ['SIM115', 'SIM117'] -[pydocstyle] +[lint.pydocstyle] convention = 'google' ignore-decorators = ['typing.overload'] -[isort] +[lint.isort] case-sensitive = true combine-as-imports = true force-wrap-aliases = true -[flake8-quotes] +[lint.flake8-quotes] docstring-quotes = 'single' inline-quotes = 'single' multiline-quotes = 'single' From c584ef4a3babf3904538df799c30ca9aaf64661b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 11 May 2024 15:02:56 +0200 Subject: [PATCH 172/225] replaced format specifier with f-string to make ruff happy --- portalocker/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portalocker/utils.py b/portalocker/utils.py index f2446ac..33ff9a4 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -84,7 +84,7 @@ def open_atomic( # `pathlib.Path` cast in case `path` is a `str` path: pathlib.Path = pathlib.Path(filename) - assert not path.exists(), '%r exists' % path + assert not path.exists(), f'{path!r} exists' # Create the parent directory if it doesn't exist path.parent.mkdir(parents=True, exist_ok=True) From ef2c055ce4eb5418bad7a80f50a5fee6ef04d5c8 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 11 May 2024 15:10:55 +0200 Subject: [PATCH 173/225] made mypy happy --- mypy.ini | 1 + portalocker/__main__.py | 5 +++-- portalocker/redis.py | 2 +- portalocker/utils.py | 6 +++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mypy.ini b/mypy.ini index b66f60c..5c5b7e3 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,3 +5,4 @@ files = portalocker ignore_missing_imports = True +check_untyped_defs = True \ No newline at end of file diff --git a/portalocker/__main__.py b/portalocker/__main__.py index 25bdf23..0896ea1 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -3,6 +3,7 @@ import os import pathlib import re +import typing base_path = pathlib.Path(__file__).parent.parent src_path = base_path / 'portalocker' @@ -42,7 +43,7 @@ def main(argv=None): args.func(args) -def _read_file(path, seen_files): +def _read_file(path: pathlib.Path, seen_files: typing.Set[pathlib.Path]): if path in seen_files: return @@ -83,7 +84,7 @@ def combine(args): _TEXT_TEMPLATE.format((base_path / 'LICENSE').read_text()), ) - seen_files = set() + seen_files: typing.Set[pathlib.Path] = set() for line in _read_file(src_path / '__init__.py', seen_files): output_file.write(line) diff --git a/portalocker/redis.py b/portalocker/redis.py index 2a84807..11ee876 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -126,7 +126,7 @@ def channel_handler(self, message): def client_name(self): return f'{self.channel}-lock' - def acquire( + def acquire( # type: ignore[override] self, timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, diff --git a/portalocker/utils.py b/portalocker/utils.py index 33ff9a4..5115b0e 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -132,7 +132,7 @@ def acquire( timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = None, - ) -> typing.Union['LockBase', typing.IO[typing.AnyStr]]: ... + ) -> typing.IO[typing.AnyStr]: ... def _timeout_generator( self, @@ -157,7 +157,7 @@ def _timeout_generator( @abc.abstractmethod def release(self): ... - def __enter__(self) -> typing.Union['LockBase', typing.IO[typing.AnyStr]]: + def __enter__(self) -> typing.IO[typing.AnyStr]: return self.acquire() def __exit__( @@ -476,7 +476,7 @@ def get_filename(self, number) -> pathlib.Path: number=number, ) - def acquire( # type: ignore[reportGeneralTypeIssues] + def acquire( # type: ignore[override] self, timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, From becd9d10716df86aff34816ff62273ec3ec2143d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 11 May 2024 15:25:51 +0200 Subject: [PATCH 174/225] debugging combined file output --- portalocker_tests/test_combined.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/portalocker_tests/test_combined.py b/portalocker_tests/test_combined.py index bbd9eb2..fe0f1fa 100644 --- a/portalocker_tests/test_combined.py +++ b/portalocker_tests/test_combined.py @@ -6,6 +6,12 @@ def test_combined(tmpdir): output_file = tmpdir.join('combined.py') __main__.main(['combine', '--output-file', output_file.strpath]) + + print(output_file) + print('#################') + print(output_file.read()) + print('#################') + sys.path.append(output_file.dirname) # Combined is being generated above but linters won't understand that import combined # type: ignore From 5ee48d82c6211b010eaf8920ecece853c2e67630 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 11 May 2024 15:27:16 +0200 Subject: [PATCH 175/225] making ruff happy for tests --- portalocker_tests/test_combined.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/portalocker_tests/test_combined.py b/portalocker_tests/test_combined.py index fe0f1fa..dacda89 100644 --- a/portalocker_tests/test_combined.py +++ b/portalocker_tests/test_combined.py @@ -7,10 +7,10 @@ def test_combined(tmpdir): output_file = tmpdir.join('combined.py') __main__.main(['combine', '--output-file', output_file.strpath]) - print(output_file) - print('#################') - print(output_file.read()) - print('#################') + print(output_file) # noqa: T201 + print('#################') # noqa: T201 + print(output_file.read()) # noqa: T201 + print('#################') # noqa: T201 sys.path.append(output_file.dirname) # Combined is being generated above but linters won't understand that From f43234aa5262c5ca45920a1cf457a7e2d848c3eb Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 11 May 2024 15:33:58 +0200 Subject: [PATCH 176/225] perhaps a fix for the combined file on linux?! --- portalocker/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/portalocker/__main__.py b/portalocker/__main__.py index 0896ea1..1bedf52 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -53,6 +53,7 @@ def _read_file(path: pathlib.Path, seen_files: typing.Set[pathlib.Path]): if match := _RELATIVE_IMPORT_RE.match(line): from_ = match.group('from') if from_: + names.add(from_) yield from _read_file(src_path / f'{from_}.py', seen_files) else: for name in match.group('names').split(','): From 94414e4423f66a2e26d470c986ff30edbc51c47a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 20 May 2024 03:56:35 +0200 Subject: [PATCH 177/225] Fixed combine script to support multi-line import statements --- portalocker/__init__.py | 8 +++++++- portalocker/__main__.py | 26 ++++++++++++++++++++++---- portalocker/portalocker.py | 6 ++++-- portalocker_tests/tests.py | 12 ++++++++---- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/portalocker/__init__.py b/portalocker/__init__.py index b5026f4..0f5271d 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -1,5 +1,11 @@ from . import __about__, constants, exceptions, portalocker -from .utils import BoundedSemaphore, Lock, RLock, TemporaryFileLock, open_atomic +from .utils import ( + BoundedSemaphore, + Lock, + RLock, + TemporaryFileLock, + open_atomic, +) try: # pragma: no cover from .redis import RedisLock diff --git a/portalocker/__main__.py b/portalocker/__main__.py index 1bedf52..0051c2d 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -10,7 +10,10 @@ dist_path = base_path / 'dist' _default_output_path = base_path / 'dist' / 'portalocker.py' -_RELATIVE_IMPORT_RE = re.compile(r'^from \.(?P.*?) import (?P.+)$') +_NAMES_RE = re.compile(r'(?P[^()]+)$') +_RELATIVE_IMPORT_RE = re.compile( + r'^from \.(?P.*?) import (?P\(?)(?P[^()]+)$' +) _USELESS_ASSIGNMENT_RE = re.compile(r'^(?P\w+) = \1\n$') _TEXT_TEMPLATE = """''' @@ -29,7 +32,7 @@ def main(argv=None): combine_parser = subparsers.add_parser( 'combine', help='Combine all Python files into a single unified `portalocker.py` ' - 'file for easy distribution', + 'file for easy distribution', ) combine_parser.add_argument( '--output-file', @@ -49,9 +52,24 @@ def _read_file(path: pathlib.Path, seen_files: typing.Set[pathlib.Path]): names = set() seen_files.add(path) + paren = False + from_ = None for line in path.open(): - if match := _RELATIVE_IMPORT_RE.match(line): - from_ = match.group('from') + if paren: + if ')' in line: + line = line.split(')', 1)[1] + paren = False + continue + + match = _NAMES_RE.match(line) + else: + match = _RELATIVE_IMPORT_RE.match(line) + + if match: + if not paren: + paren = bool(match.group('paren')) + from_ = match.group('from') + if from_: names.add(from_) yield from _read_file(src_path / f'{from_}.py', seen_files) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index e7b0585..fbd82a0 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -120,13 +120,15 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): # A timeout exception, wrap this so the outer code knows to try # again (if it wants to). raise exceptions.AlreadyLocked( - exc_value, fh=file_, + exc_value, + fh=file_, ) from exc_value else: # Something else went wrong; don't wrap this so we stop # immediately. raise exceptions.LockException( - exc_value, fh=file_, + exc_value, + fh=file_, ) from exc_value except EOFError as exc_value: # On NFS filesystems, flock can raise an EOFError diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index cb0a241..9d4a9d0 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -198,7 +198,8 @@ def test_exlusive(tmpfile): # Make sure we can't read the locked file with pytest.raises(portalocker.LockException), open( - tmpfile, 'r+', + tmpfile, + 'r+', ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) assert fh2.read() == text_0 @@ -246,7 +247,8 @@ def test_shared(tmpfile): pytest.param( 'lockf', marks=pytest.mark.skipif( - os.name == 'nt', reason='lockf() is not available on windows', + os.name == 'nt', + reason='lockf() is not available on windows', ), ), ], @@ -350,7 +352,8 @@ def lock( pytest.param( 'lockf', marks=pytest.mark.skipif( - os.name == 'nt', reason='lockf() is not available on windows', + os.name == 'nt', + reason='lockf() is not available on windows', ), ), ], @@ -377,7 +380,8 @@ def test_shared_processes(tmpfile, fail_when_locked, locker): pytest.param( 'lockf', marks=pytest.mark.skipif( - os.name == 'nt', reason='lockf() is not available on windows', + os.name == 'nt', + reason='lockf() is not available on windows', ), ), ], From 6c22661e25f4922f83abc02f3a4aada1b87cef73 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 20 May 2024 03:57:33 +0200 Subject: [PATCH 178/225] tiny ruff fix --- portalocker/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portalocker/__main__.py b/portalocker/__main__.py index 0051c2d..e3ece97 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -12,7 +12,7 @@ _NAMES_RE = re.compile(r'(?P[^()]+)$') _RELATIVE_IMPORT_RE = re.compile( - r'^from \.(?P.*?) import (?P\(?)(?P[^()]+)$' + r'^from \.(?P.*?) import (?P\(?)(?P[^()]+)$', ) _USELESS_ASSIGNMENT_RE = re.compile(r'^(?P\w+) = \1\n$') From dcc023f33a3d1ca2da2388cde8a8b39365b960ae Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 21 May 2024 00:29:05 +0200 Subject: [PATCH 179/225] windows tests on github actions can be _really_ slow --- portalocker_tests/conftest.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/portalocker_tests/conftest.py b/portalocker_tests/conftest.py index 6c56a6a..c756227 100644 --- a/portalocker_tests/conftest.py +++ b/portalocker_tests/conftest.py @@ -5,12 +5,14 @@ import pytest +from portalocker import utils + logger = logging.getLogger(__name__) @pytest.fixture def tmpfile(tmp_path): - filename = tmp_path / str(random.random()) + filename = tmp_path / str(random.random())[2:] yield str(filename) with contextlib.suppress(PermissionError): filename.unlink(missing_ok=True) @@ -21,3 +23,10 @@ def pytest_sessionstart(session): # I'm not a 100% certain this will work correctly unfortunately... there # is some potential for breaking tests multiprocessing.set_start_method('spawn') + + +@pytest.fixture(autouse=True) +def reduce_timeouts(monkeypatch): + 'For faster testing we reduce the timeouts.' + monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.1) + monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.05) From a0c5c75262477e6f7167802ab26f5a489189c6de Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 21 May 2024 00:29:33 +0200 Subject: [PATCH 180/225] Simplified tests --- portalocker_tests/tests.py | 169 ++++++++++++++----------------------- 1 file changed, 63 insertions(+), 106 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 9d4a9d0..1d03a13 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -9,34 +9,22 @@ import portalocker import portalocker.portalocker -from portalocker import LockFlags, exceptions, utils +from portalocker import exceptions, LockFlags, utils +if os.name == 'posix': + import fcntl -@pytest.fixture -def locker(request): - # Setup - if os.name == 'posix': - import fcntl - - old_locker = portalocker.portalocker.LOCKER - if request.param == 'flock': - new_locker = fcntl.flock + LOCKERS = [ + fcntl.flock, + fcntl.lockf, + ] +else: + LOCKERS = [None] - elif request.param == 'lockf': - new_locker = fcntl.lockf - - else: - raise ValueError( - f'Unrecognised locking mechanism {request.param!r}', - ) - portalocker.portalocker.LOCKER = new_locker - - yield request.param - - # Teardown. - if os.name == 'posix': - portalocker.portalocker.LOCKER = old_locker +@pytest.fixture +def locker(request, monkeypatch): + monkeypatch.setattr(portalocker.portalocker, 'LOCKER', request.param) def test_exceptions(tmpfile): @@ -63,16 +51,16 @@ def test_with_timeout(tmpfile): with portalocker.Lock(tmpfile, timeout=0.1) as fh: print('writing some stuff to my cache...', file=fh) with portalocker.Lock( - tmpfile, - timeout=0.1, - mode='wb', - fail_when_locked=True, + tmpfile, + timeout=0.1, + mode='wb', + fail_when_locked=True, ): pass print('writing more stuff to my cache...', file=fh) -def test_without_timeout(tmpfile): +def test_without_timeout(tmpfile, monkeypatch): # Open the file 2 times with pytest.raises(portalocker.LockException): with portalocker.Lock(tmpfile, timeout=None) as fh: @@ -198,16 +186,16 @@ def test_exlusive(tmpfile): # Make sure we can't read the locked file with pytest.raises(portalocker.LockException), open( - tmpfile, - 'r+', + tmpfile, + 'r+', ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) assert fh2.read() == text_0 # Make sure we can't write the locked file with pytest.raises(portalocker.LockException), open( - tmpfile, - 'w+', + tmpfile, + 'w+', ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.write('surprise and fear') @@ -230,8 +218,8 @@ def test_shared(tmpfile): # Make sure we can't write the locked file with pytest.raises(portalocker.LockException), open( - tmpfile, - 'w+', + tmpfile, + 'w+', ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.write('surprise and fear') @@ -240,20 +228,7 @@ def test_shared(tmpfile): portalocker.unlock(f) -@pytest.mark.parametrize( - 'locker', - [ - 'flock', - pytest.param( - 'lockf', - marks=pytest.mark.skipif( - os.name == 'nt', - reason='lockf() is not available on windows', - ), - ), - ], - indirect=['locker'], -) +@pytest.mark.parametrize('locker', LOCKERS, indirect=True) def test_blocking_timeout(tmpfile, locker): flags = LockFlags.SHARED @@ -270,7 +245,7 @@ def test_blocking_timeout(tmpfile, locker): os.name == 'nt', reason='Windows uses an entirely different lockmechanism', ) -@pytest.mark.parametrize('locker', ['flock', 'lockf'], indirect=['locker']) +@pytest.mark.parametrize('locker', LOCKERS, indirect=True) def test_nonblocking(tmpfile, locker): with open(tmpfile, 'w') as fh, pytest.raises(RuntimeError): portalocker.lock(fh, LockFlags.NON_BLOCKING) @@ -278,10 +253,10 @@ def test_nonblocking(tmpfile, locker): def shared_lock(filename, **kwargs): with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=False, - flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, + filename, + timeout=0.1, + fail_when_locked=False, + flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, ): time.sleep(0.2) return True @@ -289,10 +264,10 @@ def shared_lock(filename, **kwargs): def shared_lock_fail(filename, **kwargs): with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=True, - flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, + filename, + timeout=0.1, + fail_when_locked=True, + flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, ): time.sleep(0.2) return True @@ -300,10 +275,10 @@ def shared_lock_fail(filename, **kwargs): def exclusive_lock(filename, **kwargs): with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=False, - flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, + filename, + timeout=0.1, + fail_when_locked=False, + flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, ): time.sleep(0.2) return True @@ -317,9 +292,11 @@ class LockResult: def lock( - filename: str, - fail_when_locked: bool, - flags: LockFlags, + filename: str, + fail_when_locked: bool, + flags: LockFlags, + timeout=0.1, + keep_locked=0.05, ) -> LockResult: # Returns a case of True, False or FileNotFound # https://thedailywtf.com/articles/what_is_truth_0x3f_ @@ -327,12 +304,12 @@ def lock( # only return string representations of the exception properties try: with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=fail_when_locked, - flags=flags, + filename, + timeout=timeout, + fail_when_locked=fail_when_locked, + flags=flags, ): - time.sleep(0.2) + time.sleep(keep_locked) return LockResult() except Exception as exception: @@ -345,56 +322,36 @@ def lock( ) -@pytest.mark.parametrize( - 'locker', - [ - 'flock', - pytest.param( - 'lockf', - marks=pytest.mark.skipif( - os.name == 'nt', - reason='lockf() is not available on windows', - ), - ), - ], - indirect=['locker'], -) @pytest.mark.parametrize('fail_when_locked', [True, False]) -def test_shared_processes(tmpfile, fail_when_locked, locker): +@pytest.mark.parametrize('locker', LOCKERS, indirect=True) +def test_shared_processes(tmpfile, locker, fail_when_locked): flags = LockFlags.SHARED | LockFlags.NON_BLOCKING + print() + print(f'{tmpfile=}, {fail_when_locked=}, {flags=}') with multiprocessing.Pool(processes=2) as pool: args = tmpfile, fail_when_locked, flags results = pool.starmap_async(lock, 2 * [args]) - for result in results.get(timeout=3): + # sourcery skip: no-loop-in-tests + for result in results.get(timeout=0.5): + print(f'{result=}') + # sourcery skip: no-conditionals-in-tests if result.exception_class is not None: raise result.exception_class assert result == LockResult() -@pytest.mark.parametrize( - 'locker', - [ - 'flock', - pytest.param( - 'lockf', - marks=pytest.mark.skipif( - os.name == 'nt', - reason='lockf() is not available on windows', - ), - ), - ], - indirect=['locker'], -) @pytest.mark.parametrize('fail_when_locked', [True, False]) -def test_exclusive_processes(tmpfile, fail_when_locked, locker): +@pytest.mark.parametrize('locker', LOCKERS, indirect=True) +def test_exclusive_processes(tmpfile: str, fail_when_locked: bool, locker): flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING + pool: multiprocessing.Pool with multiprocessing.Pool(processes=2) as pool: # filename, fail_when_locked, flags args = tmpfile, fail_when_locked, flags - a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=3) + a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=0.6) assert not a.exception_class or not b.exception_class assert issubclass( @@ -407,7 +364,7 @@ def test_exclusive_processes(tmpfile, fail_when_locked, locker): os.name == 'nt', reason='Locking on Windows requires a file object', ) -@pytest.mark.parametrize('locker', ['flock', 'lockf'], indirect=['locker']) +@pytest.mark.parametrize('locker', LOCKERS, indirect=True) def test_lock_fileno(tmpfile, locker): with open(tmpfile, 'a+') as a: with open(tmpfile, 'a+') as b: @@ -425,7 +382,7 @@ def test_lock_fileno(tmpfile, locker): os.name == 'nt', reason='Windows only has one locking mechanism', ) -@pytest.mark.parametrize('locker', ['flock', 'lockf'], indirect=['locker']) +@pytest.mark.parametrize('locker', LOCKERS, indirect=True) def test_locker_mechanism(tmpfile, locker): '''Can we switch the locking mechanism?''' # We can test for flock vs lockf based on their different behaviour re. @@ -438,7 +395,7 @@ def test_locker_mechanism(tmpfile, locker): tmpfile, 'r+', flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, - ).acquire() + ).acquire(timeout=0.1) elif locker == 'lockf': # But on lockf, we can! @@ -446,7 +403,7 @@ def test_locker_mechanism(tmpfile, locker): tmpfile, 'r+', flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, - ).acquire() + ).acquire(timeout=0.1) else: raise RuntimeError('Update test') From b775cf0e9c811c4de1c494fcd0ab54f3bb9c829f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 21 Jun 2024 09:07:59 +0200 Subject: [PATCH 181/225] fixed tests for linux --- portalocker_tests/conftest.py | 2 +- portalocker_tests/tests.py | 73 ++++++++++++++++++++++------------- 2 files changed, 47 insertions(+), 28 deletions(-) diff --git a/portalocker_tests/conftest.py b/portalocker_tests/conftest.py index c756227..5650288 100644 --- a/portalocker_tests/conftest.py +++ b/portalocker_tests/conftest.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) -@pytest.fixture +@pytest.fixture(scope='function') def tmpfile(tmp_path): filename = tmp_path / str(random.random())[2:] yield str(filename) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 1d03a13..25f1c44 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -25,6 +25,7 @@ @pytest.fixture def locker(request, monkeypatch): monkeypatch.setattr(portalocker.portalocker, 'LOCKER', request.param) + return request.param def test_exceptions(tmpfile): @@ -323,11 +324,10 @@ def lock( @pytest.mark.parametrize('fail_when_locked', [True, False]) -@pytest.mark.parametrize('locker', LOCKERS, indirect=True) -def test_shared_processes(tmpfile, locker, fail_when_locked): +def test_shared_processes(tmpfile, fail_when_locked): flags = LockFlags.SHARED | LockFlags.NON_BLOCKING print() - print(f'{tmpfile=}, {fail_when_locked=}, {flags=}') + print(f'{fail_when_locked=}, {flags=}, {os.name=}, {LOCKERS=}') with multiprocessing.Pool(processes=2) as pool: args = tmpfile, fail_when_locked, flags @@ -348,16 +348,39 @@ def test_exclusive_processes(tmpfile: str, fail_when_locked: bool, locker): flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING pool: multiprocessing.Pool + print('Locking', tmpfile, fail_when_locked, locker) with multiprocessing.Pool(processes=2) as pool: - # filename, fail_when_locked, flags - args = tmpfile, fail_when_locked, flags - a, b = pool.starmap_async(lock, 2 * [args]).get(timeout=0.6) - - assert not a.exception_class or not b.exception_class - assert issubclass( - a.exception_class or b.exception_class, # type: ignore - portalocker.LockException, - ) + # Submit tasks individually + result_a = pool.apply_async(lock, [tmpfile, fail_when_locked, flags]) + result_b = pool.apply_async(lock, [tmpfile, fail_when_locked, flags]) + + try: + a = result_a.get(timeout=0.6) # Wait for 'a' with timeout + except multiprocessing.TimeoutError: + a = None + + try: + # Lower timeout since we already waited with `a` + b = result_b.get(timeout=0.1) # Wait for 'b' with timeout + except multiprocessing.TimeoutError: + b = None + + assert a or b + # Make sure a is always filled + if b: + b, a = b, a + + print(f'{a=}') + print(f'{b=}') + + if b: + assert not a.exception_class or not b.exception_class + assert issubclass( + a.exception_class or b.exception_class, # type: ignore + portalocker.LockException, + ) + else: + assert not a.exception_class @pytest.mark.skipif( @@ -379,8 +402,8 @@ def test_lock_fileno(tmpfile, locker): @pytest.mark.skipif( - os.name == 'nt', - reason='Windows only has one locking mechanism', + os.name != 'posix', + reason='Only posix systems have different lockf behaviour', ) @pytest.mark.parametrize('locker', LOCKERS, indirect=True) def test_locker_mechanism(tmpfile, locker): @@ -388,25 +411,21 @@ def test_locker_mechanism(tmpfile, locker): # We can test for flock vs lockf based on their different behaviour re. # locking the same file. with portalocker.Lock(tmpfile, 'a+', flags=LockFlags.EXCLUSIVE): - # If we have flock(), we cannot get another lock on the same file. - if locker == 'flock': - with pytest.raises(portalocker.LockException): - portalocker.Lock( - tmpfile, - 'r+', - flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, - ).acquire(timeout=0.1) - - elif locker == 'lockf': - # But on lockf, we can! + # If we have lockf(), we cannot get another lock on the same file. + if locker is fcntl.lockf: portalocker.Lock( tmpfile, 'r+', flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, ).acquire(timeout=0.1) - + # But with other lock methods we can't else: - raise RuntimeError('Update test') + with pytest.raises(portalocker.LockException): + portalocker.Lock( + tmpfile, + 'r+', + flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, + ).acquire(timeout=0.1) def test_exception(monkeypatch, tmpfile): From 561f3c803449148884ee685ca7474091b7f998f4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 21 Jun 2024 10:32:01 +0200 Subject: [PATCH 182/225] increased timeouts for slower platforms --- portalocker_tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 25f1c44..be5abf6 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -334,7 +334,7 @@ def test_shared_processes(tmpfile, fail_when_locked): results = pool.starmap_async(lock, 2 * [args]) # sourcery skip: no-loop-in-tests - for result in results.get(timeout=0.5): + for result in results.get(timeout=1.0): print(f'{result=}') # sourcery skip: no-conditionals-in-tests if result.exception_class is not None: @@ -355,7 +355,7 @@ def test_exclusive_processes(tmpfile: str, fail_when_locked: bool, locker): result_b = pool.apply_async(lock, [tmpfile, fail_when_locked, flags]) try: - a = result_a.get(timeout=0.6) # Wait for 'a' with timeout + a = result_a.get(timeout=1.0) # Wait for 'a' with timeout except multiprocessing.TimeoutError: a = None From d7315a5bc4afbdb50da26cb68fec07c229f14fbd Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 21 Jun 2024 11:20:53 +0200 Subject: [PATCH 183/225] pyright fixes --- portalocker_tests/tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index be5abf6..415b57d 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -347,7 +347,6 @@ def test_shared_processes(tmpfile, fail_when_locked): def test_exclusive_processes(tmpfile: str, fail_when_locked: bool, locker): flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING - pool: multiprocessing.Pool print('Locking', tmpfile, fail_when_locked, locker) with multiprocessing.Pool(processes=2) as pool: # Submit tasks individually @@ -373,7 +372,13 @@ def test_exclusive_processes(tmpfile: str, fail_when_locked: bool, locker): print(f'{a=}') print(f'{b=}') + # make pyright happy + assert a is not None + if b: + # make pyright happy + assert b is not None + assert not a.exception_class or not b.exception_class assert issubclass( a.exception_class or b.exception_class, # type: ignore From 80d40bf5bfcc86bc85b096d7a1531935a7a206e7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 21 Jun 2024 11:30:19 +0200 Subject: [PATCH 184/225] ruff fixes --- portalocker_tests/tests.py | 2 +- ruff.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 415b57d..a1ffd47 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -9,7 +9,7 @@ import portalocker import portalocker.portalocker -from portalocker import exceptions, LockFlags, utils +from portalocker import LockFlags, exceptions, utils if os.name == 'posix': import fcntl diff --git a/ruff.toml b/ruff.toml index d99665b..e4180ff 100644 --- a/ruff.toml +++ b/ruff.toml @@ -62,7 +62,7 @@ select = [ ] [lint.per-file-ignores] -'portalocker_tests/tests.py' = ['SIM115', 'SIM117'] +'portalocker_tests/tests.py' = ['SIM115', 'SIM117', 'T201'] [lint.pydocstyle] convention = 'google' From 5cc3aad961bab8353282dfa5659346966ffbe09d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 21 Jun 2024 11:38:46 +0200 Subject: [PATCH 185/225] made mypy happy --- portalocker/__main__.py | 2 +- portalocker_tests/tests.py | 64 +++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/portalocker/__main__.py b/portalocker/__main__.py index e3ece97..ecac207 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -32,7 +32,7 @@ def main(argv=None): combine_parser = subparsers.add_parser( 'combine', help='Combine all Python files into a single unified `portalocker.py` ' - 'file for easy distribution', + 'file for easy distribution', ) combine_parser.add_argument( '--output-file', diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index a1ffd47..3675112 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -19,7 +19,7 @@ fcntl.lockf, ] else: - LOCKERS = [None] + LOCKERS = [None] # type: ignore @pytest.fixture @@ -52,10 +52,10 @@ def test_with_timeout(tmpfile): with portalocker.Lock(tmpfile, timeout=0.1) as fh: print('writing some stuff to my cache...', file=fh) with portalocker.Lock( - tmpfile, - timeout=0.1, - mode='wb', - fail_when_locked=True, + tmpfile, + timeout=0.1, + mode='wb', + fail_when_locked=True, ): pass print('writing more stuff to my cache...', file=fh) @@ -187,16 +187,16 @@ def test_exlusive(tmpfile): # Make sure we can't read the locked file with pytest.raises(portalocker.LockException), open( - tmpfile, - 'r+', + tmpfile, + 'r+', ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) assert fh2.read() == text_0 # Make sure we can't write the locked file with pytest.raises(portalocker.LockException), open( - tmpfile, - 'w+', + tmpfile, + 'w+', ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.write('surprise and fear') @@ -219,8 +219,8 @@ def test_shared(tmpfile): # Make sure we can't write the locked file with pytest.raises(portalocker.LockException), open( - tmpfile, - 'w+', + tmpfile, + 'w+', ) as fh2: portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.write('surprise and fear') @@ -254,10 +254,10 @@ def test_nonblocking(tmpfile, locker): def shared_lock(filename, **kwargs): with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=False, - flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, + filename, + timeout=0.1, + fail_when_locked=False, + flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, ): time.sleep(0.2) return True @@ -265,10 +265,10 @@ def shared_lock(filename, **kwargs): def shared_lock_fail(filename, **kwargs): with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=True, - flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, + filename, + timeout=0.1, + fail_when_locked=True, + flags=LockFlags.SHARED | LockFlags.NON_BLOCKING, ): time.sleep(0.2) return True @@ -276,10 +276,10 @@ def shared_lock_fail(filename, **kwargs): def exclusive_lock(filename, **kwargs): with portalocker.Lock( - filename, - timeout=0.1, - fail_when_locked=False, - flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, + filename, + timeout=0.1, + fail_when_locked=False, + flags=LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING, ): time.sleep(0.2) return True @@ -293,11 +293,11 @@ class LockResult: def lock( - filename: str, - fail_when_locked: bool, - flags: LockFlags, - timeout=0.1, - keep_locked=0.05, + filename: str, + fail_when_locked: bool, + flags: LockFlags, + timeout=0.1, + keep_locked=0.05, ) -> LockResult: # Returns a case of True, False or FileNotFound # https://thedailywtf.com/articles/what_is_truth_0x3f_ @@ -305,10 +305,10 @@ def lock( # only return string representations of the exception properties try: with portalocker.Lock( - filename, - timeout=timeout, - fail_when_locked=fail_when_locked, - flags=flags, + filename, + timeout=timeout, + fail_when_locked=fail_when_locked, + flags=flags, ): time.sleep(keep_locked) return LockResult() From f3d91afa1923bf77d9b28d1a25bf95da3ea545d2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 22 Jun 2024 20:35:19 +0200 Subject: [PATCH 186/225] type hinting improvements --- portalocker/portalocker.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index fbd82a0..1f9ccc7 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -8,6 +8,14 @@ LockFlags = constants.LockFlags +class HasFileno(typing.Protocol): + def fileno(self) -> int: ... + + +LOCKER: typing.Optional[typing.Callable[ + [typing.Union[int, HasFileno], int], typing.Any]] = None + + if os.name == 'nt': # pragma: no cover import msvcrt @@ -95,8 +103,9 @@ def unlock(file_: typing.IO): LOCKER = fcntl.flock def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): - # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled results - # in an error + assert LOCKER is not None, 'We need a locing function in `LOCKER` ' + # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled + # results in an error if (flags & LockFlags.NON_BLOCKING) and not flags & ( LockFlags.SHARED | LockFlags.EXCLUSIVE ): @@ -138,6 +147,7 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): ) from exc_value def unlock(file_: typing.IO): + assert LOCKER is not None, 'We need a locing function in `LOCKER` ' LOCKER(file_.fileno(), LockFlags.UNBLOCK) else: # pragma: no cover From 57ae4415408cbadde1020ec221325f6b67cbc22c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 22 Jun 2024 20:39:54 +0200 Subject: [PATCH 187/225] typing protocols cannot be covered by tests --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 2611ef0..a033179 100644 --- a/.coveragerc +++ b/.coveragerc @@ -10,6 +10,7 @@ exclude_lines = raise NotImplementedError if 0: if __name__ == .__main__.: + typing.Protocol omit = portalocker/redis.py From eedc656dfa703f395e016b5b44c62239dc79f0c0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 23 Jun 2024 00:46:33 +0200 Subject: [PATCH 188/225] Incrementing version to v2.9.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index e45c443..4f6b0f3 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,6 +1,6 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.8.2' +__version__ = '2.9.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 0f5271d..bb3bc80 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -20,7 +20,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.8.2' +__version__ = '2.9.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From c78d0558814b1c09f0e65692f9b46892b3296164 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 23 Jun 2024 00:48:25 +0200 Subject: [PATCH 189/225] Incrementing version to v2.10.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 4f6b0f3..daff9d9 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,6 +1,6 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.9.0' +__version__ = '2.10.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index bb3bc80..754b527 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -20,7 +20,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.9.0' +__version__ = '2.10.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From d73b90c9673fd9704c178b870bf13bb22ca07839 Mon Sep 17 00:00:00 2001 From: Tyler Harms Date: Tue, 25 Jun 2024 11:20:52 -0500 Subject: [PATCH 190/225] Fix spelling error Fix a simple spelling error - s/locing/locking --- portalocker/portalocker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 1f9ccc7..ceceeaa 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -103,7 +103,7 @@ def unlock(file_: typing.IO): LOCKER = fcntl.flock def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): - assert LOCKER is not None, 'We need a locing function in `LOCKER` ' + assert LOCKER is not None, 'We need a locking function in `LOCKER` ' # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled # results in an error if (flags & LockFlags.NON_BLOCKING) and not flags & ( @@ -147,7 +147,7 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): ) from exc_value def unlock(file_: typing.IO): - assert LOCKER is not None, 'We need a locing function in `LOCKER` ' + assert LOCKER is not None, 'We need a locking function in `LOCKER` ' LOCKER(file_.fileno(), LockFlags.UNBLOCK) else: # pragma: no cover From 11cb7a84ea7fbd6130c15984db804e496912a7ca Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 14 Jul 2024 00:57:22 +0200 Subject: [PATCH 191/225] increasing the timeouts to make the osx tests less flaky --- portalocker_tests/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 3675112..ee0d91b 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -354,13 +354,13 @@ def test_exclusive_processes(tmpfile: str, fail_when_locked: bool, locker): result_b = pool.apply_async(lock, [tmpfile, fail_when_locked, flags]) try: - a = result_a.get(timeout=1.0) # Wait for 'a' with timeout + a = result_a.get(timeout=1.1) # Wait for 'a' with timeout except multiprocessing.TimeoutError: a = None try: # Lower timeout since we already waited with `a` - b = result_b.get(timeout=0.1) # Wait for 'b' with timeout + b = result_b.get(timeout=0.2) # Wait for 'b' with timeout except multiprocessing.TimeoutError: b = None From c2c433d2af4f398ba1e218ba4668efa06f5e45e4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 14 Jul 2024 01:15:12 +0200 Subject: [PATCH 192/225] Incrementing version to v2.10.1 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index daff9d9..a0b817a 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,6 +1,6 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.10.0' +__version__ = '2.10.1' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 754b527..7e757ef 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -20,7 +20,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.10.0' +__version__ = '2.10.1' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From d42529f437efa4a39a4ad7afadf788a0f3bf6c12 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 18 Nov 2024 05:23:45 +0100 Subject: [PATCH 193/225] testing new release with CI --- portalocker/__about__.py | 2 +- portalocker/__main__.py | 17 ++-- portalocker/constants.py | 4 +- portalocker/exceptions.py | 4 +- portalocker/portalocker.py | 26 +++--- portalocker/redis.py | 42 ++++++---- portalocker/types.py | 113 ++++++++++++++++++++++++++ portalocker/utils.py | 140 ++++++++++++++++++-------------- portalocker_tests/conftest.py | 2 +- portalocker_tests/test_redis.py | 2 +- portalocker_tests/tests.py | 4 +- pyproject.toml | 1 + ruff.toml | 4 + tox.ini | 1 + 14 files changed, 260 insertions(+), 102 deletions(-) create mode 100644 portalocker/types.py diff --git a/portalocker/__about__.py b/portalocker/__about__.py index a0b817a..8353d60 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -2,5 +2,5 @@ __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' __version__ = '2.10.1' -__description__ = '''Wraps the portalocker recipe for easy usage''' +__description__ = """Wraps the portalocker recipe for easy usage""" __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__main__.py b/portalocker/__main__.py index ecac207..21eca48 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import logging import os @@ -25,7 +27,7 @@ logger = logging.getLogger(__name__) -def main(argv=None): +def main(argv: typing.Sequence[str] | None = None) -> None: parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(required=True) @@ -46,11 +48,14 @@ def main(argv=None): args.func(args) -def _read_file(path: pathlib.Path, seen_files: typing.Set[pathlib.Path]): +def _read_file( + path: pathlib.Path, + seen_files: typing.Set[pathlib.Path], +) -> typing.Iterator[str]: if path in seen_files: return - names = set() + names: set[str] = set() seen_files.add(path) paren = False from_ = None @@ -82,17 +87,17 @@ def _read_file(path: pathlib.Path, seen_files: typing.Set[pathlib.Path]): yield _clean_line(line, names) -def _clean_line(line, names): +def _clean_line(line: str, names: set[str]): # Replace `some_import.spam` with `spam` if names: joined_names = '|'.join(names) - line = re.sub(fr'\b({joined_names})\.', '', line) + line = re.sub(rf'\b({joined_names})\.', '', line) # Replace useless assignments (e.g. `spam = spam`) return _USELESS_ASSIGNMENT_RE.sub('', line) -def combine(args): +def combine(args: argparse.Namespace): output_file = args.output_file pathlib.Path(output_file.name).parent.mkdir(parents=True, exist_ok=True) diff --git a/portalocker/constants.py b/portalocker/constants.py index 2099f1f..198571f 100644 --- a/portalocker/constants.py +++ b/portalocker/constants.py @@ -1,4 +1,4 @@ -''' +""" Locking constants Lock types: @@ -13,7 +13,7 @@ Manually unlock, only needed internally - `UNBLOCK` unlock -''' +""" import enum import os diff --git a/portalocker/exceptions.py b/portalocker/exceptions.py index e871d13..54d1bfa 100644 --- a/portalocker/exceptions.py +++ b/portalocker/exceptions.py @@ -1,5 +1,7 @@ import typing +from portalocker import types + class BaseLockException(Exception): # noqa: N818 # Error codes: @@ -8,7 +10,7 @@ class BaseLockException(Exception): # noqa: N818 def __init__( self, *args: typing.Any, - fh: typing.Union[typing.IO, None, int] = None, + fh: typing.Union[types.IO, None, int] = None, **kwargs: typing.Any, ) -> None: self.fh = fh diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index ceceeaa..32519fc 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import os import typing -from . import constants, exceptions +from . import constants, exceptions, types # Alias for readability. Due to import recursion issues we cannot do: # from .constants import LockFlags @@ -12,9 +14,9 @@ class HasFileno(typing.Protocol): def fileno(self) -> int: ... -LOCKER: typing.Optional[typing.Callable[ - [typing.Union[int, HasFileno], int], typing.Any]] = None - +LOCKER: typing.Optional[ + typing.Callable[[typing.Union[int, HasFileno], int], typing.Any] +] = None if os.name == 'nt': # pragma: no cover import msvcrt @@ -100,9 +102,9 @@ def unlock(file_: typing.IO): # The locking implementation. # Expected values are either fcntl.flock() or fcntl.lockf(), # but any callable that matches the syntax will be accepted. - LOCKER = fcntl.flock + LOCKER = fcntl.flock # pyright: ignore[reportConstantRedefinition] - def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): + def lock(file: int | types.IO, flags: LockFlags): assert LOCKER is not None, 'We need a locking function in `LOCKER` ' # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled # results in an error @@ -115,7 +117,7 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): ) try: - LOCKER(file_, flags) + LOCKER(file, flags) except OSError as exc_value: # Python can use one of several different exception classes to # represent timeout (most likely is BlockingIOError and IOError), @@ -130,25 +132,25 @@ def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): # again (if it wants to). raise exceptions.AlreadyLocked( exc_value, - fh=file_, + fh=file, ) from exc_value else: # Something else went wrong; don't wrap this so we stop # immediately. raise exceptions.LockException( exc_value, - fh=file_, + fh=file, ) from exc_value except EOFError as exc_value: # On NFS filesystems, flock can raise an EOFError raise exceptions.LockException( exc_value, - fh=file_, + fh=file, ) from exc_value - def unlock(file_: typing.IO): + def unlock(file: types.IO): assert LOCKER is not None, 'We need a locking function in `LOCKER` ' - LOCKER(file_.fileno(), LockFlags.UNBLOCK) + LOCKER(file.fileno(), LockFlags.UNBLOCK) else: # pragma: no cover raise RuntimeError('PortaLocker only defined for nt and posix platforms') diff --git a/portalocker/redis.py b/portalocker/redis.py index 11ee876..523f9d7 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -1,3 +1,4 @@ +# pyright: reportUnknownMemberType=false import _thread import json import logging @@ -5,7 +6,7 @@ import time import typing -from redis import client +import redis from . import exceptions, utils @@ -15,8 +16,8 @@ DEFAULT_THREAD_SLEEP_TIME = 0.1 -class PubSubWorkerThread(client.PubSubWorkerThread): # type: ignore - def run(self): +class PubSubWorkerThread(redis.client.PubSubWorkerThread): # type: ignore + def run(self) -> None: try: super().run() except Exception: # pragma: no cover @@ -25,7 +26,7 @@ def run(self): class RedisLock(utils.LockBase): - ''' + """ An extremely reliable Redis lock based on pubsub with a keep-alive thread As opposed to most Redis locking systems based on key/value pairs, @@ -59,30 +60,31 @@ class RedisLock(utils.LockBase): to override these you need to explicitly specify a value (e.g. `health_check_interval=0`) - ''' + """ redis_kwargs: typing.Dict[str, typing.Any] thread: typing.Optional[PubSubWorkerThread] channel: str timeout: float - connection: typing.Optional[client.Redis] - pubsub: typing.Optional[client.PubSub] = None + connection: typing.Optional[redis.client.Redis[str]] + pubsub: typing.Optional[redis.client.PubSub] = None close_connection: bool DEFAULT_REDIS_KWARGS: typing.ClassVar[typing.Dict[str, typing.Any]] = dict( health_check_interval=10, + decode_responses=True, ) def __init__( self, channel: str, - connection: typing.Optional[client.Redis] = None, + connection: typing.Optional[redis.client.Redis[str]] = None, timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = False, thread_sleep_time: float = DEFAULT_THREAD_SLEEP_TIME, unavailable_timeout: float = DEFAULT_UNAVAILABLE_TIMEOUT, - redis_kwargs: typing.Optional[typing.Dict] = None, + redis_kwargs: typing.Optional[typing.Dict[str, typing.Any]] = None, ): # We don't want to close connections given as an argument self.close_connection = not connection @@ -103,18 +105,22 @@ def __init__( fail_when_locked=fail_when_locked, ) - def get_connection(self) -> client.Redis: + def get_connection(self) -> redis.client.Redis[str]: if not self.connection: - self.connection = client.Redis(**self.redis_kwargs) + self.connection = redis.client.Redis(**self.redis_kwargs) return self.connection - def channel_handler(self, message): + def channel_handler(self, message: typing.Dict[str, str]) -> None: if message.get('type') != 'message': # pragma: no cover return + raw_data = message.get('data') + if not raw_data: + return + try: - data = json.loads(message.get('data')) + data = json.loads(raw_data) except TypeError: # pragma: no cover logger.debug('TypeError while parsing: %r', message) return @@ -189,7 +195,11 @@ def acquire( # type: ignore[override] raise exceptions.AlreadyLocked(exceptions) - def check_or_kill_lock(self, connection, timeout): + def check_or_kill_lock( + self, + connection: redis.client.Redis[str], + timeout: float, + ): # Random channel name to get messages back from the lock response_channel = f'{self.channel}-{random.random()}' @@ -217,7 +227,9 @@ def check_or_kill_lock(self, connection, timeout): for client_ in connection.client_list('pubsub'): # pragma: no cover if client_.get('name') == self.client_name: logger.warning('Killing unavailable redis client: %r', client_) - connection.client_kill_filter(client_.get('id')) + connection.client_kill_filter( # pyright: ignore + client_.get('id'), + ) return None def release(self): diff --git a/portalocker/types.py b/portalocker/types.py new file mode 100644 index 0000000..8abcbe7 --- /dev/null +++ b/portalocker/types.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import pathlib +import typing +from typing import Union + +Mode = typing.Literal[ + # Text modes + 'r', + 'rt', + 'tr', # Read text + 'w', + 'wt', + 'tw', # Write text + 'a', + 'at', + 'ta', # Append text + 'x', + 'xt', + 'tx', # Exclusive creation text + 'r+', + '+r', + 'rt+', + 'r+t', + '+rt', + 'tr+', + 't+r', + '+tr', # Read and write text + 'w+', + '+w', + 'wt+', + 'w+t', + '+wt', + 'tw+', + 't+w', + '+tw', # Write and read text + 'a+', + '+a', + 'at+', + 'a+t', + '+at', + 'ta+', + 't+a', + '+ta', # Append and read text + 'x+', + '+x', + 'xt+', + 'x+t', + '+xt', + 'tx+', + 't+x', + '+tx', # Exclusive creation and read text + 'U', + 'rU', + 'Ur', + 'rtU', + 'rUt', + 'Urt', + 'trU', + 'tUr', + 'Utr', # Universal newline support + # Binary modes + 'rb', + 'br', # Read binary + 'wb', + 'bw', # Write binary + 'ab', + 'ba', # Append binary + 'xb', + 'bx', # Exclusive creation binary + 'rb+', + 'r+b', + '+rb', + 'br+', + 'b+r', + '+br', # Read and write binary + 'wb+', + 'w+b', + '+wb', + 'bw+', + 'b+w', + '+bw', # Write and read binary + 'ab+', + 'a+b', + '+ab', + 'ba+', + 'b+a', + '+ba', # Append and read binary + 'xb+', + 'x+b', + '+xb', + 'bx+', + 'b+x', + '+bx', # Exclusive creation and read binary + 'rbU', + 'rUb', + 'Urb', + 'brU', + 'bUr', + 'Ubr', + # Universal newline support in binary mode +] +Filename = Union[str, pathlib.Path] +IO: typing.TypeAlias = Union[typing.IO[str], typing.IO[bytes]] + + +class FileOpenKwargs(typing.TypedDict): + buffering: int | None + encoding: str | None + errors: str | None + newline: str | None + closefd: bool | None + opener: typing.Callable[[str, int], int] | None diff --git a/portalocker/utils.py b/portalocker/utils.py index 5115b0e..ab0b7b5 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import abc import atexit import contextlib @@ -10,7 +12,8 @@ import typing import warnings -from . import constants, exceptions, portalocker +from . import constants, exceptions, portalocker, types +from .types import Filename, Mode logger = logging.getLogger(__name__) @@ -24,11 +27,9 @@ 'open_atomic', ] -Filename = typing.Union[str, pathlib.Path] - def coalesce(*args: typing.Any, test_value: typing.Any = None) -> typing.Any: - '''Simple coalescing function that returns the first value that is not + """Simple coalescing function that returns the first value that is not equal to the `test_value`. Or `None` if no value is valid. Usually this means that the last given value is the default value. @@ -48,7 +49,7 @@ def coalesce(*args: typing.Any, test_value: typing.Any = None) -> typing.Any: # This won't work because of the `is not test_value` type testing: >>> coalesce([], dict(spam='eggs'), test_value=[]) [] - ''' + """ return next((arg for arg in args if arg is not test_value), None) @@ -56,8 +57,8 @@ def coalesce(*args: typing.Any, test_value: typing.Any = None) -> typing.Any: def open_atomic( filename: Filename, binary: bool = True, -) -> typing.Iterator[typing.IO]: - '''Open a file for atomic writing. Instead of locking this method allows +) -> typing.Iterator[types.IO]: + """Open a file for atomic writing. Instead of locking this method allows you to write the entire file and move it to the actual location. Note that this makes the assumption that a rename is atomic on your platform which is generally the case but not a guarantee. @@ -80,9 +81,13 @@ def open_atomic( ... written = fh.write(b'test') >>> assert path_filename.exists() >>> path_filename.unlink() - ''' + """ # `pathlib.Path` cast in case `path` is a `str` - path: pathlib.Path = pathlib.Path(filename) + path: pathlib.Path + if isinstance(filename, pathlib.Path): + path = filename + else: + path = pathlib.Path(filename) assert not path.exists(), f'{path!r} exists' @@ -169,12 +174,12 @@ def __exit__( self.release() return None - def __delete__(self, instance): + def __delete__(self, instance: LockBase): instance.release() class Lock(LockBase): - '''Lock manager with built-in timeout + """Lock manager with built-in timeout Args: filename: filename @@ -192,21 +197,31 @@ class Lock(LockBase): Note that the file is opened first and locked later. So using 'w' as mode will result in truncate _BEFORE_ the lock is checked. - ''' + """ + + fh: types.IO | None + filename: str + mode: str + truncate: bool + timeout: float + check_interval: float + fail_when_locked: bool + flags: constants.LockFlags + file_open_kwargs: dict[str, typing.Any] def __init__( self, filename: Filename, - mode: str = 'a', - timeout: typing.Optional[float] = None, + mode: Mode = 'a', + timeout: float | None = None, check_interval: float = DEFAULT_CHECK_INTERVAL, fail_when_locked: bool = DEFAULT_FAIL_WHEN_LOCKED, flags: constants.LockFlags = LOCK_METHOD, - **file_open_kwargs, + **file_open_kwargs: typing.Any, ): if 'w' in mode: truncate = True - mode = mode.replace('w', 'a') + mode = typing.cast(Mode, mode.replace('w', 'a')) else: truncate = False @@ -218,15 +233,13 @@ def __init__( stacklevel=1, ) - self.fh: typing.Optional[typing.IO] = None - self.filename: str = str(filename) - self.mode: str = mode - self.truncate: bool = truncate - self.timeout: float = timeout - self.check_interval: float = check_interval - self.fail_when_locked: bool = fail_when_locked - self.flags: constants.LockFlags = flags + self.fh = None + self.filename = str(filename) + self.mode = mode + self.truncate = truncate + self.flags = flags self.file_open_kwargs = file_open_kwargs + super().__init__(timeout, check_interval, fail_when_locked) def acquire( self, @@ -234,7 +247,7 @@ def acquire( check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = None, ) -> typing.IO[typing.AnyStr]: - '''Acquire the locked filehandle''' + """Acquire the locked filehandle""" fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) @@ -248,9 +261,10 @@ def acquire( ) # If we already have a filehandle, return it - fh: typing.Optional[typing.IO] = self.fh + fh = self.fh if fh: - return fh + # Due to type invariance we need to cast the type + return typing.cast(typing.IO[typing.AnyStr], fh) # Get a new filehandler fh = self._get_fh() @@ -296,41 +310,44 @@ def try_close(): # pragma: no cover fh = self._prepare_fh(fh) self.fh = fh - return fh + return typing.cast(typing.IO[typing.AnyStr], fh) def __enter__(self) -> typing.IO[typing.AnyStr]: return self.acquire() def release(self): - '''Releases the currently locked file handle''' + """Releases the currently locked file handle""" if self.fh: portalocker.unlock(self.fh) self.fh.close() self.fh = None - def _get_fh(self) -> typing.IO: - '''Get a new filehandle''' - return open( # noqa: SIM115 - self.filename, - self.mode, - **self.file_open_kwargs, + def _get_fh(self) -> types.IO: + """Get a new filehandle""" + return typing.cast( + types.IO, + open( # noqa: SIM115 + self.filename, + self.mode, + **self.file_open_kwargs, + ), ) - def _get_lock(self, fh: typing.IO) -> typing.IO: - ''' + def _get_lock(self, fh: types.IO) -> types.IO: + """ Try to lock the given filehandle - returns LockException if it fails''' + returns LockException if it fails""" portalocker.lock(fh, self.flags) return fh - def _prepare_fh(self, fh: typing.IO) -> typing.IO: - ''' + def _prepare_fh(self, fh: types.IO) -> types.IO: + """ Prepare the filehandle for usage If truncate is a number, the file will be truncated to that amount of bytes - ''' + """ if self.truncate: fh.seek(0) fh.truncate(0) @@ -339,20 +356,20 @@ def _prepare_fh(self, fh: typing.IO) -> typing.IO: class RLock(Lock): - ''' + """ A reentrant lock, functions in a similar way to threading.RLock in that it can be acquired multiple times. When the corresponding number of release() calls are made the lock will finally release the underlying file lock. - ''' + """ def __init__( self, - filename, - mode='a', - timeout=DEFAULT_TIMEOUT, - check_interval=DEFAULT_CHECK_INTERVAL, - fail_when_locked=False, - flags=LOCK_METHOD, + filename: Filename, + mode: Mode = 'a', + timeout: float = DEFAULT_TIMEOUT, + check_interval: float = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool = False, + flags: constants.LockFlags = LOCK_METHOD, ): super().__init__( filename, @@ -369,9 +386,10 @@ def acquire( timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = None, - ) -> typing.IO: + ) -> typing.IO[typing.AnyStr]: + fh: typing.IO[typing.AnyStr] if self._acquire_count >= 1: - fh = self.fh + fh = typing.cast(typing.IO[typing.AnyStr], self.fh) else: fh = super().acquire(timeout, check_interval, fail_when_locked) self._acquire_count += 1 @@ -392,11 +410,11 @@ def release(self): class TemporaryFileLock(Lock): def __init__( self, - filename='.lock', - timeout=DEFAULT_TIMEOUT, - check_interval=DEFAULT_CHECK_INTERVAL, - fail_when_locked=True, - flags=LOCK_METHOD, + filename: str = '.lock', + timeout: float = DEFAULT_TIMEOUT, + check_interval: float = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool = True, + flags: constants.LockFlags = LOCK_METHOD, ): Lock.__init__( self, @@ -416,7 +434,7 @@ def release(self): class BoundedSemaphore(LockBase): - ''' + """ Bounded semaphore to prevent too many parallel processes from running This method is deprecated because multiple processes that are completely @@ -429,7 +447,7 @@ class BoundedSemaphore(LockBase): 'bounded_semaphore.00.lock' >>> str(sorted(semaphore.get_random_filenames())[1]) 'bounded_semaphore.01.lock' - ''' + """ lock: typing.Optional[Lock] @@ -470,7 +488,7 @@ def get_random_filenames(self) -> typing.Sequence[pathlib.Path]: random.shuffle(filenames) return filenames - def get_filename(self, number) -> pathlib.Path: + def get_filename(self, number: int) -> pathlib.Path: return pathlib.Path(self.directory) / self.filename_pattern.format( name=self.name, number=number, @@ -522,7 +540,7 @@ def release(self): # pragma: no cover class NamedBoundedSemaphore(BoundedSemaphore): - ''' + """ Bounded semaphore to prevent too many parallel processes from running It's also possible to specify a timeout when acquiring the lock to wait @@ -544,7 +562,7 @@ class NamedBoundedSemaphore(BoundedSemaphore): >>> 'bounded_semaphore' in str(semaphore.get_filenames()[0]) True - ''' + """ def __init__( self, diff --git a/portalocker_tests/conftest.py b/portalocker_tests/conftest.py index 5650288..ad2dc23 100644 --- a/portalocker_tests/conftest.py +++ b/portalocker_tests/conftest.py @@ -27,6 +27,6 @@ def pytest_sessionstart(session): @pytest.fixture(autouse=True) def reduce_timeouts(monkeypatch): - 'For faster testing we reduce the timeouts.' + "For faster testing we reduce the timeouts." monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.1) monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.05) diff --git a/portalocker_tests/test_redis.py b/portalocker_tests/test_redis.py index e9bec02..2f274a1 100644 --- a/portalocker_tests/test_redis.py +++ b/portalocker_tests/test_redis.py @@ -46,7 +46,7 @@ def test_redis_lock(): @pytest.mark.parametrize('timeout', [None, 0, 0.001]) @pytest.mark.parametrize('check_interval', [None, 0, 0.0005]) def test_redis_lock_timeout(timeout, check_interval): - connection = client.Redis() + connection: client.Redis[str] = client.Redis(decode_responses=True) channel = str(random.random()) lock_a = redis.RedisLock(channel) lock_a.acquire(timeout=timeout, check_interval=check_interval) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index ee0d91b..d3b57cd 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -412,7 +412,7 @@ def test_lock_fileno(tmpfile, locker): ) @pytest.mark.parametrize('locker', LOCKERS, indirect=True) def test_locker_mechanism(tmpfile, locker): - '''Can we switch the locking mechanism?''' + """Can we switch the locking mechanism?""" # We can test for flock vs lockf based on their different behaviour re. # locking the same file. with portalocker.Lock(tmpfile, 'a+', flags=LockFlags.EXCLUSIVE): @@ -434,7 +434,7 @@ def test_locker_mechanism(tmpfile, locker): def test_exception(monkeypatch, tmpfile): - '''Do we stop immediately if the locking fails, even with a timeout?''' + """Do we stop immediately if the locking fails, even with a timeout?""" def patched_lock(*args, **kwargs): raise ValueError('Test exception') diff --git a/pyproject.toml b/pyproject.toml index 0c61e2b..08ea8ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,3 +98,4 @@ skip = '*/htmlcov,./docs/_build,*.asc' [tool.pyright] include = ['portalocker', 'portalocker_tests'] exclude = ['dist/*'] +strict = ['portalocker'] diff --git a/ruff.toml b/ruff.toml index e4180ff..194a455 100644 --- a/ruff.toml +++ b/ruff.toml @@ -8,6 +8,10 @@ exclude = ['docs'] line-length = 80 +[format] +quote-style = 'single' + + [lint] ignore = [ 'A001', # Variable {name} is shadowing a Python builtin diff --git a/tox.ini b/tox.ini index 29bf765..371099c 100644 --- a/tox.ini +++ b/tox.ini @@ -42,6 +42,7 @@ changedir = basepython = python3 deps = pyright + types-redis -e{toxinidir}[tests,redis] commands = pyright {toxinidir}/portalocker {toxinidir}/portalocker_tests From cada500eda2b40f23b487fb4dd8de5015bdbc728 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 18 Nov 2024 12:08:43 +0100 Subject: [PATCH 194/225] Fixed several bugs with the testing suite. --- portalocker/__about__.py | 2 +- portalocker/__main__.py | 6 ++++++ portalocker/constants.py | 4 ++-- portalocker/redis.py | 8 +++++--- portalocker/utils.py | 18 +++++++++--------- portalocker_tests/conftest.py | 2 +- portalocker_tests/tests.py | 4 ++-- 7 files changed, 26 insertions(+), 18 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index 8353d60..a0b817a 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -2,5 +2,5 @@ __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' __version__ = '2.10.1' -__description__ = """Wraps the portalocker recipe for easy usage""" +__description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__main__.py b/portalocker/__main__.py index 21eca48..e777e50 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -60,6 +60,9 @@ def _read_file( paren = False from_ = None for line in path.open(): + if '__future__' in line: + continue + if paren: if ')' in line: line = line.split(')', 1)[1] @@ -101,6 +104,9 @@ def combine(args: argparse.Namespace): output_file = args.output_file pathlib.Path(output_file.name).parent.mkdir(parents=True, exist_ok=True) + # We're handling this separately because it has to be the first import. + output_file.write('from __future__ import annotations\n') + output_file.write( _TEXT_TEMPLATE.format((base_path / 'README.rst').read_text()), ) diff --git a/portalocker/constants.py b/portalocker/constants.py index 198571f..2099f1f 100644 --- a/portalocker/constants.py +++ b/portalocker/constants.py @@ -1,4 +1,4 @@ -""" +''' Locking constants Lock types: @@ -13,7 +13,7 @@ Manually unlock, only needed internally - `UNBLOCK` unlock -""" +''' import enum import os diff --git a/portalocker/redis.py b/portalocker/redis.py index 523f9d7..cb94adf 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -1,4 +1,6 @@ # pyright: reportUnknownMemberType=false +from __future__ import annotations + import _thread import json import logging @@ -26,7 +28,7 @@ def run(self) -> None: class RedisLock(utils.LockBase): - """ + ''' An extremely reliable Redis lock based on pubsub with a keep-alive thread As opposed to most Redis locking systems based on key/value pairs, @@ -60,7 +62,7 @@ class RedisLock(utils.LockBase): to override these you need to explicitly specify a value (e.g. `health_check_interval=0`) - """ + ''' redis_kwargs: typing.Dict[str, typing.Any] thread: typing.Optional[PubSubWorkerThread] @@ -137,7 +139,7 @@ def acquire( # type: ignore[override] timeout: typing.Optional[float] = None, check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = None, - ) -> 'RedisLock': + ) -> RedisLock: timeout = utils.coalesce(timeout, self.timeout, 0.0) check_interval = utils.coalesce( check_interval, diff --git a/portalocker/utils.py b/portalocker/utils.py index ab0b7b5..ab065b0 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -247,7 +247,7 @@ def acquire( check_interval: typing.Optional[float] = None, fail_when_locked: typing.Optional[bool] = None, ) -> typing.IO[typing.AnyStr]: - """Acquire the locked filehandle""" + '''Acquire the locked filehandle''' fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) @@ -316,14 +316,14 @@ def __enter__(self) -> typing.IO[typing.AnyStr]: return self.acquire() def release(self): - """Releases the currently locked file handle""" + '''Releases the currently locked file handle''' if self.fh: portalocker.unlock(self.fh) self.fh.close() self.fh = None def _get_fh(self) -> types.IO: - """Get a new filehandle""" + '''Get a new filehandle''' return typing.cast( types.IO, open( # noqa: SIM115 @@ -334,20 +334,20 @@ def _get_fh(self) -> types.IO: ) def _get_lock(self, fh: types.IO) -> types.IO: - """ + ''' Try to lock the given filehandle - returns LockException if it fails""" + returns LockException if it fails''' portalocker.lock(fh, self.flags) return fh def _prepare_fh(self, fh: types.IO) -> types.IO: - """ + ''' Prepare the filehandle for usage If truncate is a number, the file will be truncated to that amount of bytes - """ + ''' if self.truncate: fh.seek(0) fh.truncate(0) @@ -356,11 +356,11 @@ def _prepare_fh(self, fh: types.IO) -> types.IO: class RLock(Lock): - """ + ''' A reentrant lock, functions in a similar way to threading.RLock in that it can be acquired multiple times. When the corresponding number of release() calls are made the lock will finally release the underlying file lock. - """ + ''' def __init__( self, diff --git a/portalocker_tests/conftest.py b/portalocker_tests/conftest.py index ad2dc23..5650288 100644 --- a/portalocker_tests/conftest.py +++ b/portalocker_tests/conftest.py @@ -27,6 +27,6 @@ def pytest_sessionstart(session): @pytest.fixture(autouse=True) def reduce_timeouts(monkeypatch): - "For faster testing we reduce the timeouts." + 'For faster testing we reduce the timeouts.' monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.1) monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.05) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index d3b57cd..ee0d91b 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -412,7 +412,7 @@ def test_lock_fileno(tmpfile, locker): ) @pytest.mark.parametrize('locker', LOCKERS, indirect=True) def test_locker_mechanism(tmpfile, locker): - """Can we switch the locking mechanism?""" + '''Can we switch the locking mechanism?''' # We can test for flock vs lockf based on their different behaviour re. # locking the same file. with portalocker.Lock(tmpfile, 'a+', flags=LockFlags.EXCLUSIVE): @@ -434,7 +434,7 @@ def test_locker_mechanism(tmpfile, locker): def test_exception(monkeypatch, tmpfile): - """Do we stop immediately if the locking fails, even with a timeout?""" + '''Do we stop immediately if the locking fails, even with a timeout?''' def patched_lock(*args, **kwargs): raise ValueError('Test exception') From db8719379a6ac0394dd6710438767d3eaafeb93f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 18 Nov 2024 12:14:01 +0100 Subject: [PATCH 195/225] made ruff happy, hopefully no issues with these updates. --- portalocker/__main__.py | 4 +-- portalocker/portalocker.py | 6 ++-- portalocker/redis.py | 28 +++++++-------- portalocker/utils.py | 72 +++++++++++++++++++------------------- portalocker_tests/tests.py | 2 +- ruff.toml | 2 +- 6 files changed, 56 insertions(+), 58 deletions(-) diff --git a/portalocker/__main__.py b/portalocker/__main__.py index e777e50..ed8cf4e 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -50,7 +50,7 @@ def main(argv: typing.Sequence[str] | None = None) -> None: def _read_file( path: pathlib.Path, - seen_files: typing.Set[pathlib.Path], + seen_files: set[pathlib.Path], ) -> typing.Iterator[str]: if path in seen_files: return @@ -114,7 +114,7 @@ def combine(args: argparse.Namespace): _TEXT_TEMPLATE.format((base_path / 'LICENSE').read_text()), ) - seen_files: typing.Set[pathlib.Path] = set() + seen_files: set[pathlib.Path] = set() for line in _read_file(src_path / '__init__.py', seen_files): output_file.write(line) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 32519fc..9bdfa52 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -14,9 +14,7 @@ class HasFileno(typing.Protocol): def fileno(self) -> int: ... -LOCKER: typing.Optional[ - typing.Callable[[typing.Union[int, HasFileno], int], typing.Any] -] = None +LOCKER: typing.Callable[[int | HasFileno, int], typing.Any] | None = None if os.name == 'nt': # pragma: no cover import msvcrt @@ -28,7 +26,7 @@ def fileno(self) -> int: ... __overlapped = pywintypes.OVERLAPPED() - def lock(file_: typing.Union[typing.IO, int], flags: LockFlags): + def lock(file_: typing.IO | int, flags: LockFlags): # Windows locking does not support locking through `fh.fileno()` so # we cast it to make mypy and pyright happy file_ = typing.cast(typing.IO, file_) diff --git a/portalocker/redis.py b/portalocker/redis.py index cb94adf..f58c492 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -64,15 +64,15 @@ class RedisLock(utils.LockBase): ''' - redis_kwargs: typing.Dict[str, typing.Any] - thread: typing.Optional[PubSubWorkerThread] + redis_kwargs: dict[str, typing.Any] + thread: PubSubWorkerThread | None channel: str timeout: float - connection: typing.Optional[redis.client.Redis[str]] - pubsub: typing.Optional[redis.client.PubSub] = None + connection: redis.client.Redis[str] | None + pubsub: redis.client.PubSub | None = None close_connection: bool - DEFAULT_REDIS_KWARGS: typing.ClassVar[typing.Dict[str, typing.Any]] = dict( + DEFAULT_REDIS_KWARGS: typing.ClassVar[dict[str, typing.Any]] = dict( health_check_interval=10, decode_responses=True, ) @@ -80,13 +80,13 @@ class RedisLock(utils.LockBase): def __init__( self, channel: str, - connection: typing.Optional[redis.client.Redis[str]] = None, - timeout: typing.Optional[float] = None, - check_interval: typing.Optional[float] = None, - fail_when_locked: typing.Optional[bool] = False, + connection: redis.client.Redis[str] | None = None, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = False, thread_sleep_time: float = DEFAULT_THREAD_SLEEP_TIME, unavailable_timeout: float = DEFAULT_UNAVAILABLE_TIMEOUT, - redis_kwargs: typing.Optional[typing.Dict[str, typing.Any]] = None, + redis_kwargs: dict[str, typing.Any] | None = None, ): # We don't want to close connections given as an argument self.close_connection = not connection @@ -113,7 +113,7 @@ def get_connection(self) -> redis.client.Redis[str]: return self.connection - def channel_handler(self, message: typing.Dict[str, str]) -> None: + def channel_handler(self, message: dict[str, str]) -> None: if message.get('type') != 'message': # pragma: no cover return @@ -136,9 +136,9 @@ def client_name(self): def acquire( # type: ignore[override] self, - timeout: typing.Optional[float] = None, - check_interval: typing.Optional[float] = None, - fail_when_locked: typing.Optional[bool] = None, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, ) -> RedisLock: timeout = utils.coalesce(timeout, self.timeout, 0.0) check_interval = utils.coalesce( diff --git a/portalocker/utils.py b/portalocker/utils.py index ab065b0..f2c630e 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -94,15 +94,15 @@ def open_atomic( # Create the parent directory if it doesn't exist path.parent.mkdir(parents=True, exist_ok=True) - temp_fh = tempfile.NamedTemporaryFile( + with tempfile.NamedTemporaryFile( mode=binary and 'wb' or 'w', dir=str(path.parent), delete=False, - ) - yield temp_fh - temp_fh.flush() - os.fsync(temp_fh.fileno()) - temp_fh.close() + ) as temp_fh: + yield temp_fh + temp_fh.flush() + os.fsync(temp_fh.fileno()) + try: os.rename(temp_fh.name, path) finally: @@ -120,9 +120,9 @@ class LockBase(abc.ABC): # pragma: no cover def __init__( self, - timeout: typing.Optional[float] = None, - check_interval: typing.Optional[float] = None, - fail_when_locked: typing.Optional[bool] = None, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, ): self.timeout = coalesce(timeout, DEFAULT_TIMEOUT) self.check_interval = coalesce(check_interval, DEFAULT_CHECK_INTERVAL) @@ -134,15 +134,15 @@ def __init__( @abc.abstractmethod def acquire( self, - timeout: typing.Optional[float] = None, - check_interval: typing.Optional[float] = None, - fail_when_locked: typing.Optional[bool] = None, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, ) -> typing.IO[typing.AnyStr]: ... def _timeout_generator( self, - timeout: typing.Optional[float], - check_interval: typing.Optional[float], + timeout: float | None, + check_interval: float | None, ) -> typing.Iterator[int]: f_timeout = coalesce(timeout, self.timeout, 0.0) f_check_interval = coalesce(check_interval, self.check_interval, 0.0) @@ -167,10 +167,10 @@ def __enter__(self) -> typing.IO[typing.AnyStr]: def __exit__( self, - exc_type: typing.Optional[typing.Type[BaseException]], - exc_value: typing.Optional[BaseException], + exc_type: type[BaseException] | None, + exc_value: BaseException | None, traceback: typing.Any, # Should be typing.TracebackType - ) -> typing.Optional[bool]: + ) -> bool | None: self.release() return None @@ -243,9 +243,9 @@ def __init__( def acquire( self, - timeout: typing.Optional[float] = None, - check_interval: typing.Optional[float] = None, - fail_when_locked: typing.Optional[bool] = None, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, ) -> typing.IO[typing.AnyStr]: '''Acquire the locked filehandle''' @@ -383,9 +383,9 @@ def __init__( def acquire( self, - timeout: typing.Optional[float] = None, - check_interval: typing.Optional[float] = None, - fail_when_locked: typing.Optional[bool] = None, + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, ) -> typing.IO[typing.AnyStr]: fh: typing.IO[typing.AnyStr] if self._acquire_count >= 1: @@ -449,7 +449,7 @@ class BoundedSemaphore(LockBase): 'bounded_semaphore.01.lock' """ - lock: typing.Optional[Lock] + lock: Lock | None def __init__( self, @@ -457,15 +457,15 @@ def __init__( name: str = 'bounded_semaphore', filename_pattern: str = '{name}.{number:02d}.lock', directory: str = tempfile.gettempdir(), - timeout: typing.Optional[float] = DEFAULT_TIMEOUT, - check_interval: typing.Optional[float] = DEFAULT_CHECK_INTERVAL, - fail_when_locked: typing.Optional[bool] = True, + timeout: float | None = DEFAULT_TIMEOUT, + check_interval: float | None = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool | None = True, ): self.maximum = maximum self.name = name self.filename_pattern = filename_pattern self.directory = directory - self.lock: typing.Optional[Lock] = None + self.lock: Lock | None = None super().__init__( timeout=timeout, check_interval=check_interval, @@ -496,10 +496,10 @@ def get_filename(self, number: int) -> pathlib.Path: def acquire( # type: ignore[override] self, - timeout: typing.Optional[float] = None, - check_interval: typing.Optional[float] = None, - fail_when_locked: typing.Optional[bool] = None, - ) -> typing.Optional[Lock]: + timeout: float | None = None, + check_interval: float | None = None, + fail_when_locked: bool | None = None, + ) -> Lock | None: assert not self.lock, 'Already locked' filenames = self.get_filenames() @@ -567,12 +567,12 @@ class NamedBoundedSemaphore(BoundedSemaphore): def __init__( self, maximum: int, - name: typing.Optional[str] = None, + name: str | None = None, filename_pattern: str = '{name}.{number:02d}.lock', directory: str = tempfile.gettempdir(), - timeout: typing.Optional[float] = DEFAULT_TIMEOUT, - check_interval: typing.Optional[float] = DEFAULT_CHECK_INTERVAL, - fail_when_locked: typing.Optional[bool] = True, + timeout: float | None = DEFAULT_TIMEOUT, + check_interval: float | None = DEFAULT_CHECK_INTERVAL, + fail_when_locked: bool | None = True, ): if name is None: name = 'bounded_semaphore.%d' % random.randint(0, 1000000) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index ee0d91b..f6ddb36 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -287,7 +287,7 @@ def exclusive_lock(filename, **kwargs): @dataclasses.dataclass(order=True) class LockResult: - exception_class: typing.Union[typing.Type[Exception], None] = None + exception_class: typing.Union[type[Exception], None] = None exception_message: typing.Union[str, None] = None exception_repr: typing.Union[str, None] = None diff --git a/ruff.toml b/ruff.toml index 194a455..c62d40f 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,7 +1,7 @@ # We keep the ruff configuration separate so it can easily be shared across # all projects -target-version = 'py38' +target-version = 'py39' src = ['portalocker'] exclude = ['docs'] From fdfd9cf6376ac61ba883a6230cf11dcc0db7db43 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 18 Nov 2024 12:16:59 +0100 Subject: [PATCH 196/225] made ruff happy, hopefully no issues with these updates. --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index c62d40f..065da24 100644 --- a/ruff.toml +++ b/ruff.toml @@ -30,6 +30,7 @@ ignore = [ 'SIM114', # Combine `if` branches using logical `or` operator 'RET506', # Unnecessary `else` after `raise` statement 'FA100', # Missing `from __future__ import annotations`, but uses `typing.Optional` + 'RUF100', # Unused noqa directives. Due to multiple Python versions, we need to keep them ] select = [ From 446d7a2d63d5c1dc5c4dc74da01d0a936ad4377b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 18 Nov 2024 12:45:56 +0100 Subject: [PATCH 197/225] made mypy happy-ish --- portalocker/portalocker.py | 4 +- portalocker/types.py | 130 +++++++++++-------------------------- portalocker_tests/tests.py | 33 ++++++---- ruff.toml | 7 +- 4 files changed, 67 insertions(+), 107 deletions(-) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 9bdfa52..e9184d6 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -102,7 +102,7 @@ def unlock(file_: typing.IO): # but any callable that matches the syntax will be accepted. LOCKER = fcntl.flock # pyright: ignore[reportConstantRedefinition] - def lock(file: int | types.IO, flags: LockFlags): + def lock(file: int | types.IO, flags: LockFlags): # type: ignore[misc] assert LOCKER is not None, 'We need a locking function in `LOCKER` ' # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled # results in an error @@ -146,7 +146,7 @@ def lock(file: int | types.IO, flags: LockFlags): fh=file, ) from exc_value - def unlock(file: types.IO): + def unlock(file: types.IO): # type: ignore[misc] assert LOCKER is not None, 'We need a locking function in `LOCKER` ' LOCKER(file.fileno(), LockFlags.UNBLOCK) diff --git a/portalocker/types.py b/portalocker/types.py index 8abcbe7..81814ae 100644 --- a/portalocker/types.py +++ b/portalocker/types.py @@ -4,104 +4,50 @@ import typing from typing import Union +# fmt: off Mode = typing.Literal[ # Text modes - 'r', - 'rt', - 'tr', # Read text - 'w', - 'wt', - 'tw', # Write text - 'a', - 'at', - 'ta', # Append text - 'x', - 'xt', - 'tx', # Exclusive creation text - 'r+', - '+r', - 'rt+', - 'r+t', - '+rt', - 'tr+', - 't+r', - '+tr', # Read and write text - 'w+', - '+w', - 'wt+', - 'w+t', - '+wt', - 'tw+', - 't+w', - '+tw', # Write and read text - 'a+', - '+a', - 'at+', - 'a+t', - '+at', - 'ta+', - 't+a', - '+ta', # Append and read text - 'x+', - '+x', - 'xt+', - 'x+t', - '+xt', - 'tx+', - 't+x', - '+tx', # Exclusive creation and read text - 'U', - 'rU', - 'Ur', - 'rtU', - 'rUt', - 'Urt', - 'trU', - 'tUr', - 'Utr', # Universal newline support + # Read text + 'r', 'rt', 'tr', + # Write text + 'w', 'wt', 'tw', + # Append text + 'a', 'at', 'ta', + # Exclusive creation text + 'x', 'xt', 'tx', + # Read and write text + 'r+', '+r', 'rt+', 'r+t', '+rt', 'tr+', 't+r', '+tr', + # Write and read text + 'w+', '+w', 'wt+', 'w+t', '+wt', 'tw+', 't+w', '+tw', + # Append and read text + 'a+', '+a', 'at+', 'a+t', '+at', 'ta+', 't+a', '+ta', + # Exclusive creation and read text + 'x+', '+x', 'xt+', 'x+t', '+xt', 'tx+', 't+x', '+tx', + # Universal newline support + 'U', 'rU', 'Ur', 'rtU', 'rUt', 'Urt', 'trU', 'tUr', 'Utr', + # Binary modes - 'rb', - 'br', # Read binary - 'wb', - 'bw', # Write binary - 'ab', - 'ba', # Append binary - 'xb', - 'bx', # Exclusive creation binary - 'rb+', - 'r+b', - '+rb', - 'br+', - 'b+r', - '+br', # Read and write binary - 'wb+', - 'w+b', - '+wb', - 'bw+', - 'b+w', - '+bw', # Write and read binary - 'ab+', - 'a+b', - '+ab', - 'ba+', - 'b+a', - '+ba', # Append and read binary - 'xb+', - 'x+b', - '+xb', - 'bx+', - 'b+x', - '+bx', # Exclusive creation and read binary - 'rbU', - 'rUb', - 'Urb', - 'brU', - 'bUr', - 'Ubr', + # Read binary + 'rb', 'br', + # Write binary + 'wb', 'bw', + # Append binary + 'ab', 'ba', + # Exclusive creation binary + 'xb', 'bx', + # Read and write binary + 'rb+', 'r+b', '+rb', 'br+', 'b+r', '+br', + # Write and read binary + 'wb+', 'w+b', '+wb', 'bw+', 'b+w', '+bw', + # Append and read binary + 'ab+', 'a+b', '+ab', 'ba+', 'b+a', '+ba', + # Exclusive creation and read binary + 'xb+', 'x+b', '+xb', 'bx+', 'b+x', '+bx', # Universal newline support in binary mode + 'rbU', 'rUb', 'Urb', 'brU', 'bUr', 'Ubr', ] Filename = Union[str, pathlib.Path] -IO: typing.TypeAlias = Union[typing.IO[str], typing.IO[bytes]] +IO: typing.TypeAlias = Union[typing.IO[str], typing.IO[bytes]] # type: ignore[name-defined] class FileOpenKwargs(typing.TypedDict): diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index f6ddb36..a875d4f 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -186,18 +186,24 @@ def test_exlusive(tmpfile): portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB) # Make sure we can't read the locked file - with pytest.raises(portalocker.LockException), open( - tmpfile, - 'r+', - ) as fh2: + with ( + pytest.raises(portalocker.LockException), + open( + tmpfile, + 'r+', + ) as fh2, + ): portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) assert fh2.read() == text_0 # Make sure we can't write the locked file - with pytest.raises(portalocker.LockException), open( - tmpfile, - 'w+', - ) as fh2: + with ( + pytest.raises(portalocker.LockException), + open( + tmpfile, + 'w+', + ) as fh2, + ): portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.write('surprise and fear') @@ -218,10 +224,13 @@ def test_shared(tmpfile): assert fh2.read() == 'spam and eggs' # Make sure we can't write the locked file - with pytest.raises(portalocker.LockException), open( - tmpfile, - 'w+', - ) as fh2: + with ( + pytest.raises(portalocker.LockException), + open( + tmpfile, + 'w+', + ) as fh2, + ): portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB) fh2.write('surprise and fear') diff --git a/ruff.toml b/ruff.toml index 065da24..0853930 100644 --- a/ruff.toml +++ b/ruff.toml @@ -4,7 +4,12 @@ target-version = 'py39' src = ['portalocker'] -exclude = ['docs'] +exclude = [ + 'docs', + # Ignore local test files/directories/old-stuff + 'test.py', + '*_old.py', +] line-length = 80 From 7a8f74694fd066734d8ed269a5da216eef58d2c5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 18 Nov 2024 12:53:49 +0100 Subject: [PATCH 198/225] updated github actions --- .github/workflows/lint.yml | 2 +- .github/workflows/python-package.yml | 4 ++-- .github/workflows/stale.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4c92829..108877c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.8', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 71209cc..f0acbb8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12'] os: ['macos-latest', 'windows-latest'] steps: @@ -41,7 +41,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 7101b3f..b0efa53 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -12,6 +12,6 @@ jobs: - uses: actions/stale@v8 with: days-before-stale: 30 + days-before-pr-stale: -1 exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement exempt-all-pr-assignees: true - From 5f9061f1a8d4969839cee12c2a423d1f74f9862d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 18 Nov 2024 12:59:30 +0100 Subject: [PATCH 199/225] fixed sphinx config to new format --- docs/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 10570a6..3dd0173 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -350,4 +350,6 @@ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/3/': None} +intersphinx_mapping = dict( + python=('http://docs.python.org/3/', None), +) From eb885f24c64fcd5ad336db176ab0c6d4273fae87 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 18 Nov 2024 13:05:24 +0100 Subject: [PATCH 200/225] made flake8 happy too --- portalocker/types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/portalocker/types.py b/portalocker/types.py index 81814ae..c0ee5cc 100644 --- a/portalocker/types.py +++ b/portalocker/types.py @@ -47,7 +47,10 @@ 'rbU', 'rUb', 'Urb', 'brU', 'bUr', 'Ubr', ] Filename = Union[str, pathlib.Path] -IO: typing.TypeAlias = Union[typing.IO[str], typing.IO[bytes]] # type: ignore[name-defined] +IO: typing.TypeAlias = Union[ # type: ignore[name-defined] + typing.IO[str], + typing.IO[bytes], +] class FileOpenKwargs(typing.TypedDict): From 9d705ee88eb0b6cb975a4380a6ab34f32373a84e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 18 Nov 2024 13:11:55 +0100 Subject: [PATCH 201/225] Dropped support for deprecated Python versions, only 3.9 and up are supported now. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 08ea8ab..b537120 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,11 +37,11 @@ classifiers = [ 'Operating System :: Unix', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: IronPython', 'Programming Language :: Python :: Implementation :: PyPy', From 928b29d789bab337b4594e118e2d36f1079c79d0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 18 Nov 2024 13:12:06 +0100 Subject: [PATCH 202/225] Incrementing version to v3.0.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index a0b817a..c4a0806 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,6 +1,6 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '2.10.1' +__version__ = '3.0.0' __description__ = '''Wraps the portalocker recipe for easy usage''' __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 7e757ef..ce03e83 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -20,7 +20,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '2.10.1' +__version__ = '3.0.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 8d4ea12ceb81d607ddeb5ca009479f0b1ce6b113 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Tue, 17 Dec 2024 15:09:24 +0100 Subject: [PATCH 203/225] Include missing typehints (mostly function return value typehints) --- portalocker/__main__.py | 4 ++-- portalocker/portalocker.py | 8 ++++---- portalocker/redis.py | 8 ++++---- portalocker/utils.py | 24 ++++++++++++------------ 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/portalocker/__main__.py b/portalocker/__main__.py index ed8cf4e..d99ffcf 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -90,7 +90,7 @@ def _read_file( yield _clean_line(line, names) -def _clean_line(line: str, names: set[str]): +def _clean_line(line: str, names: set[str]) -> str: # Replace `some_import.spam` with `spam` if names: joined_names = '|'.join(names) @@ -100,7 +100,7 @@ def _clean_line(line: str, names: set[str]): return _USELESS_ASSIGNMENT_RE.sub('', line) -def combine(args: argparse.Namespace): +def combine(args: argparse.Namespace) -> None: output_file = args.output_file pathlib.Path(output_file.name).parent.mkdir(parents=True, exist_ok=True) diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index e9184d6..dcc1d87 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -26,7 +26,7 @@ def fileno(self) -> int: ... __overlapped = pywintypes.OVERLAPPED() - def lock(file_: typing.IO | int, flags: LockFlags): + def lock(file_: typing.IO | int, flags: LockFlags) -> None: # Windows locking does not support locking through `fh.fileno()` so # we cast it to make mypy and pyright happy file_ = typing.cast(typing.IO, file_) @@ -64,7 +64,7 @@ def lock(file_: typing.IO | int, flags: LockFlags): if savepos: file_.seek(savepos) - def unlock(file_: typing.IO): + def unlock(file_: typing.IO) -> None: try: savepos = file_.tell() if savepos: @@ -102,7 +102,7 @@ def unlock(file_: typing.IO): # but any callable that matches the syntax will be accepted. LOCKER = fcntl.flock # pyright: ignore[reportConstantRedefinition] - def lock(file: int | types.IO, flags: LockFlags): # type: ignore[misc] + def lock(file: int | types.IO, flags: LockFlags) -> None: # type: ignore[misc] assert LOCKER is not None, 'We need a locking function in `LOCKER` ' # Locking with NON_BLOCKING without EXCLUSIVE or SHARED enabled # results in an error @@ -146,7 +146,7 @@ def lock(file: int | types.IO, flags: LockFlags): # type: ignore[misc] fh=file, ) from exc_value - def unlock(file: types.IO): # type: ignore[misc] + def unlock(file: types.IO) -> None: # type: ignore[misc] assert LOCKER is not None, 'We need a locking function in `LOCKER` ' LOCKER(file.fileno(), LockFlags.UNBLOCK) diff --git a/portalocker/redis.py b/portalocker/redis.py index f58c492..bce3c89 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -87,7 +87,7 @@ def __init__( thread_sleep_time: float = DEFAULT_THREAD_SLEEP_TIME, unavailable_timeout: float = DEFAULT_UNAVAILABLE_TIMEOUT, redis_kwargs: dict[str, typing.Any] | None = None, - ): + ) -> None: # We don't want to close connections given as an argument self.close_connection = not connection @@ -131,7 +131,7 @@ def channel_handler(self, message: dict[str, str]) -> None: self.connection.publish(data['response_channel'], str(time.time())) @property - def client_name(self): + def client_name(self) -> str: return f'{self.channel}-lock' def acquire( # type: ignore[override] @@ -201,7 +201,7 @@ def check_or_kill_lock( self, connection: redis.client.Redis[str], timeout: float, - ): + ) -> bool | None: # Random channel name to get messages back from the lock response_channel = f'{self.channel}-{random.random()}' @@ -234,7 +234,7 @@ def check_or_kill_lock( ) return None - def release(self): + def release(self) -> None: if self.thread: # pragma: no branch self.thread.stop() self.thread.join() diff --git a/portalocker/utils.py b/portalocker/utils.py index f2c630e..f5cf9d6 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -123,7 +123,7 @@ def __init__( timeout: float | None = None, check_interval: float | None = None, fail_when_locked: bool | None = None, - ): + ) -> None: self.timeout = coalesce(timeout, DEFAULT_TIMEOUT) self.check_interval = coalesce(check_interval, DEFAULT_CHECK_INTERVAL) self.fail_when_locked = coalesce( @@ -160,7 +160,7 @@ def _timeout_generator( time.sleep(max(0.001, (i * f_check_interval) - since_start_time)) @abc.abstractmethod - def release(self): ... + def release(self) -> None: ... def __enter__(self) -> typing.IO[typing.AnyStr]: return self.acquire() @@ -174,7 +174,7 @@ def __exit__( self.release() return None - def __delete__(self, instance: LockBase): + def __delete__(self, instance: LockBase) -> None: instance.release() @@ -218,7 +218,7 @@ def __init__( fail_when_locked: bool = DEFAULT_FAIL_WHEN_LOCKED, flags: constants.LockFlags = LOCK_METHOD, **file_open_kwargs: typing.Any, - ): + ) -> None: if 'w' in mode: truncate = True mode = typing.cast(Mode, mode.replace('w', 'a')) @@ -315,7 +315,7 @@ def try_close(): # pragma: no cover def __enter__(self) -> typing.IO[typing.AnyStr]: return self.acquire() - def release(self): + def release(self) -> None: '''Releases the currently locked file handle''' if self.fh: portalocker.unlock(self.fh) @@ -370,7 +370,7 @@ def __init__( check_interval: float = DEFAULT_CHECK_INTERVAL, fail_when_locked: bool = False, flags: constants.LockFlags = LOCK_METHOD, - ): + ) -> None: super().__init__( filename, mode, @@ -396,7 +396,7 @@ def acquire( assert fh return fh - def release(self): + def release(self) -> None: if self._acquire_count == 0: raise exceptions.LockException( 'Cannot release more times than acquired', @@ -415,7 +415,7 @@ def __init__( check_interval: float = DEFAULT_CHECK_INTERVAL, fail_when_locked: bool = True, flags: constants.LockFlags = LOCK_METHOD, - ): + ) -> None: Lock.__init__( self, filename=filename, @@ -427,7 +427,7 @@ def __init__( ) atexit.register(self.release) - def release(self): + def release(self) -> None: Lock.release(self) if os.path.isfile(self.filename): # pragma: no branch os.unlink(self.filename) @@ -460,7 +460,7 @@ def __init__( timeout: float | None = DEFAULT_TIMEOUT, check_interval: float | None = DEFAULT_CHECK_INTERVAL, fail_when_locked: bool | None = True, - ): + ) -> None: self.maximum = maximum self.name = name self.filename_pattern = filename_pattern @@ -533,7 +533,7 @@ def try_lock(self, filenames: typing.Sequence[Filename]) -> bool: return False - def release(self): # pragma: no cover + def release(self) -> None: # pragma: no cover if self.lock is not None: self.lock.release() self.lock = None @@ -573,7 +573,7 @@ def __init__( timeout: float | None = DEFAULT_TIMEOUT, check_interval: float | None = DEFAULT_CHECK_INTERVAL, fail_when_locked: bool | None = True, - ): + ) -> None: if name is None: name = 'bounded_semaphore.%d' % random.randint(0, 1000000) super().__init__( From d3f04dff6d3e8cf65c682e2d87d4bd39e00cc685 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Dec 2024 23:24:26 +0100 Subject: [PATCH 204/225] aded ruff config --- ruff.toml | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/ruff.toml b/ruff.toml index 0853930..df8b08b 100644 --- a/ruff.toml +++ b/ruff.toml @@ -6,12 +6,13 @@ target-version = 'py39' src = ['portalocker'] exclude = [ 'docs', + '.tox', # Ignore local test files/directories/old-stuff 'test.py', '*_old.py', ] -line-length = 80 +line-length = 79 [format] quote-style = 'single' @@ -34,7 +35,12 @@ ignore = [ 'C408', # Unnecessary {obj_type} call (rewrite as a literal) 'SIM114', # Combine `if` branches using logical `or` operator 'RET506', # Unnecessary `else` after `raise` statement + 'Q001', # Remove bad quotes + 'Q002', # Remove bad quotes 'FA100', # Missing `from __future__ import annotations`, but uses `typing.Optional` + 'COM812', # Missing trailing comma in a list + 'ISC001', # String concatenation with implicit str conversion + 'SIM108', # Ternary operators are not always more readable 'RUF100', # Unused noqa directives. Due to multiple Python versions, we need to keep them ] @@ -76,7 +82,10 @@ select = [ [lint.pydocstyle] convention = 'google' -ignore-decorators = ['typing.overload'] +ignore-decorators = [ + 'typing.overload', + 'typing.override', +] [lint.isort] case-sensitive = true @@ -87,3 +96,20 @@ force-wrap-aliases = true docstring-quotes = 'single' inline-quotes = 'single' multiline-quotes = 'single' + +[format] +line-ending = 'lf' +indent-style = 'space' +quote-style = 'single' +docstring-code-format = true +skip-magic-trailing-comma = false +exclude = [ + '__init__.py', +] + +[lint.pycodestyle] +max-line-length = 79 + +[lint.flake8-pytest-style] +mark-parentheses = true + From a9c384722f52f8ce76f2ff496a8586f3feb417b4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Dec 2024 23:24:38 +0100 Subject: [PATCH 205/225] modernized mypy config --- mypy.ini | 8 -------- pyproject.toml | 9 +++++++++ 2 files changed, 9 insertions(+), 8 deletions(-) delete mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 5c5b7e3..0000000 --- a/mypy.ini +++ /dev/null @@ -1,8 +0,0 @@ -[mypy] -warn_return_any = True -warn_unused_configs = True -files = portalocker - -ignore_missing_imports = True - -check_untyped_defs = True \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b537120..d694039 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,3 +99,12 @@ skip = '*/htmlcov,./docs/_build,*.asc' include = ['portalocker', 'portalocker_tests'] exclude = ['dist/*'] strict = ['portalocker'] + +[tool.mypy] +python_version = '3.9' +strict = true +warn_return_any = true +warn_unused_configs = true +files = ['portalocker'] +ignore_missing_imports = true +check_untyped_defs = true From 422f6308d0732b2ed0bcff102f336560e83ee27b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Dec 2024 05:49:11 +0100 Subject: [PATCH 206/225] mypy strict --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 16 ++++++++-------- portalocker/constants.py | 4 ++-- portalocker/portalocker.py | 6 +++--- portalocker/redis.py | 6 +++--- portalocker/utils.py | 24 ++++++++++++------------ portalocker_tests/conftest.py | 2 +- portalocker_tests/test_redis.py | 6 +++--- portalocker_tests/tests.py | 14 +++++++++----- pyproject.toml | 22 ++++++++++------------ ruff.toml | 3 --- 11 files changed, 52 insertions(+), 53 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index c4a0806..dd6cbfc 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -2,5 +2,5 @@ __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' __version__ = '3.0.0' -__description__ = '''Wraps the portalocker recipe for easy usage''' +__description__ = """Wraps the portalocker recipe for easy usage""" __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index ce03e83..3b26a3a 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -61,19 +61,19 @@ #: context wrappers __all__ = [ - 'lock', - 'unlock', 'LOCK_EX', - 'LOCK_SH', 'LOCK_NB', + 'LOCK_SH', 'LOCK_UN', - 'LockFlags', - 'LockException', - 'Lock', - 'RLock', 'AlreadyLocked', 'BoundedSemaphore', + 'Lock', + 'LockException', + 'LockFlags', + 'RLock', + 'RedisLock', 'TemporaryFileLock', + 'lock', 'open_atomic', - 'RedisLock', + 'unlock', ] diff --git a/portalocker/constants.py b/portalocker/constants.py index 2099f1f..198571f 100644 --- a/portalocker/constants.py +++ b/portalocker/constants.py @@ -1,4 +1,4 @@ -''' +""" Locking constants Lock types: @@ -13,7 +13,7 @@ Manually unlock, only needed internally - `UNBLOCK` unlock -''' +""" import enum import os diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index dcc1d87..94f66e1 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -26,10 +26,10 @@ def fileno(self) -> int: ... __overlapped = pywintypes.OVERLAPPED() - def lock(file_: typing.IO | int, flags: LockFlags) -> None: + def lock(file_: types.IO | int, flags: LockFlags) -> None: # Windows locking does not support locking through `fh.fileno()` so # we cast it to make mypy and pyright happy - file_ = typing.cast(typing.IO, file_) + file_ = typing.cast(types.IO, file_) mode = 0 if flags & LockFlags.NON_BLOCKING: @@ -64,7 +64,7 @@ def lock(file_: typing.IO | int, flags: LockFlags) -> None: if savepos: file_.seek(savepos) - def unlock(file_: typing.IO) -> None: + def unlock(file_: types.IO) -> None: try: savepos = file_.tell() if savepos: diff --git a/portalocker/redis.py b/portalocker/redis.py index bce3c89..2336213 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -28,7 +28,7 @@ def run(self) -> None: class RedisLock(utils.LockBase): - ''' + """ An extremely reliable Redis lock based on pubsub with a keep-alive thread As opposed to most Redis locking systems based on key/value pairs, @@ -62,7 +62,7 @@ class RedisLock(utils.LockBase): to override these you need to explicitly specify a value (e.g. `health_check_interval=0`) - ''' + """ redis_kwargs: dict[str, typing.Any] thread: PubSubWorkerThread | None @@ -246,5 +246,5 @@ def release(self) -> None: self.pubsub.close() self.pubsub = None - def __del__(self): + def __del__(self) -> None: self.release() diff --git a/portalocker/utils.py b/portalocker/utils.py index f5cf9d6..86128cb 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -95,7 +95,7 @@ def open_atomic( path.parent.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile( - mode=binary and 'wb' or 'w', + mode=(binary and 'wb') or 'w', dir=str(path.parent), delete=False, ) as temp_fh: @@ -247,7 +247,7 @@ def acquire( check_interval: float | None = None, fail_when_locked: bool | None = None, ) -> typing.IO[typing.AnyStr]: - '''Acquire the locked filehandle''' + """Acquire the locked filehandle""" fail_when_locked = coalesce(fail_when_locked, self.fail_when_locked) @@ -269,7 +269,7 @@ def acquire( # Get a new filehandler fh = self._get_fh() - def try_close(): # pragma: no cover + def try_close() -> None: # pragma: no cover # Silently try to close the handle if possible, ignore all issues if fh is not None: with contextlib.suppress(Exception): @@ -316,14 +316,14 @@ def __enter__(self) -> typing.IO[typing.AnyStr]: return self.acquire() def release(self) -> None: - '''Releases the currently locked file handle''' + """Releases the currently locked file handle""" if self.fh: portalocker.unlock(self.fh) self.fh.close() self.fh = None def _get_fh(self) -> types.IO: - '''Get a new filehandle''' + """Get a new filehandle""" return typing.cast( types.IO, open( # noqa: SIM115 @@ -334,20 +334,20 @@ def _get_fh(self) -> types.IO: ) def _get_lock(self, fh: types.IO) -> types.IO: - ''' + """ Try to lock the given filehandle - returns LockException if it fails''' + returns LockException if it fails""" portalocker.lock(fh, self.flags) return fh def _prepare_fh(self, fh: types.IO) -> types.IO: - ''' + """ Prepare the filehandle for usage If truncate is a number, the file will be truncated to that amount of bytes - ''' + """ if self.truncate: fh.seek(0) fh.truncate(0) @@ -356,11 +356,11 @@ def _prepare_fh(self, fh: types.IO) -> types.IO: class RLock(Lock): - ''' + """ A reentrant lock, functions in a similar way to threading.RLock in that it can be acquired multiple times. When the corresponding number of release() calls are made the lock will finally release the underlying file lock. - ''' + """ def __init__( self, @@ -575,7 +575,7 @@ def __init__( fail_when_locked: bool | None = True, ) -> None: if name is None: - name = 'bounded_semaphore.%d' % random.randint(0, 1000000) + name = f'bounded_semaphore.{random.randint(0, 1000000):d}' super().__init__( maximum, name, diff --git a/portalocker_tests/conftest.py b/portalocker_tests/conftest.py index 5650288..ad2dc23 100644 --- a/portalocker_tests/conftest.py +++ b/portalocker_tests/conftest.py @@ -27,6 +27,6 @@ def pytest_sessionstart(session): @pytest.fixture(autouse=True) def reduce_timeouts(monkeypatch): - 'For faster testing we reduce the timeouts.' + "For faster testing we reduce the timeouts." monkeypatch.setattr(utils, 'DEFAULT_TIMEOUT', 0.1) monkeypatch.setattr(utils, 'DEFAULT_CHECK_INTERVAL', 0.05) diff --git a/portalocker_tests/test_redis.py b/portalocker_tests/test_redis.py index 2f274a1..798a9f8 100644 --- a/portalocker_tests/test_redis.py +++ b/portalocker_tests/test_redis.py @@ -26,7 +26,7 @@ def set_redis_timeouts(monkeypatch): monkeypatch.setattr(_thread, 'interrupt_main', lambda: None) -def test_redis_lock(): +def test_redis_lock() -> None: channel = str(random.random()) lock_a: redis.RedisLock = redis.RedisLock(channel) @@ -61,7 +61,7 @@ def test_redis_lock_timeout(timeout, check_interval): lock_a.connection.close() -def test_redis_lock_context(): +def test_redis_lock_context() -> None: channel = str(random.random()) lock_a = redis.RedisLock(channel, fail_when_locked=True) @@ -72,7 +72,7 @@ def test_redis_lock_context(): pass -def test_redis_relock(): +def test_redis_relock() -> None: channel = str(random.random()) lock_a = redis.RedisLock(channel, fail_when_locked=True) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index a875d4f..910c7da 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -305,8 +305,8 @@ def lock( filename: str, fail_when_locked: bool, flags: LockFlags, - timeout=0.1, - keep_locked=0.05, + timeout: float = 0.1, + keep_locked: float = 0.05, ) -> LockResult: # Returns a case of True, False or FileNotFound # https://thedailywtf.com/articles/what_is_truth_0x3f_ @@ -353,7 +353,11 @@ def test_shared_processes(tmpfile, fail_when_locked): @pytest.mark.parametrize('fail_when_locked', [True, False]) @pytest.mark.parametrize('locker', LOCKERS, indirect=True) -def test_exclusive_processes(tmpfile: str, fail_when_locked: bool, locker): +def test_exclusive_processes( + tmpfile: str, + fail_when_locked: bool, + locker: typing.Callable[..., typing.Any], +) -> None: flags = LockFlags.EXCLUSIVE | LockFlags.NON_BLOCKING print('Locking', tmpfile, fail_when_locked, locker) @@ -421,7 +425,7 @@ def test_lock_fileno(tmpfile, locker): ) @pytest.mark.parametrize('locker', LOCKERS, indirect=True) def test_locker_mechanism(tmpfile, locker): - '''Can we switch the locking mechanism?''' + """Can we switch the locking mechanism?""" # We can test for flock vs lockf based on their different behaviour re. # locking the same file. with portalocker.Lock(tmpfile, 'a+', flags=LockFlags.EXCLUSIVE): @@ -443,7 +447,7 @@ def test_locker_mechanism(tmpfile, locker): def test_exception(monkeypatch, tmpfile): - '''Do we stop immediately if the locking fails, even with a timeout?''' + """Do we stop immediately if the locking fails, even with a timeout?""" def patched_lock(*args, **kwargs): raise ValueError('Test exception') diff --git a/pyproject.toml b/pyproject.toml index d694039..ca98b57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,18 +5,10 @@ requires = ['setuptools', 'setuptools-scm'] [project] name = 'portalocker' dynamic = ['version'] -authors = [{name = 'Rick van Hattem', email = 'wolph@wol.ph'}] -license = {text = 'BSD-3-Clause'} +authors = [{ name = 'Rick van Hattem', email = 'wolph@wol.ph' }] +license = { text = 'BSD-3-Clause' } description = 'Wraps the portalocker recipe for easy usage' -keywords = [ - 'locking', - 'locks', - 'with', - 'statement', - 'windows', - 'linux', - 'unix', -] +keywords = ['locking', 'locks', 'with', 'statement', 'windows', 'linux', 'unix'] readme = 'README.rst' classifiers = [ 'Development Status :: 5 - Production/Stable', @@ -53,6 +45,7 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Software Development :: Libraries', 'Topic :: System :: Monitoring', + 'Typing :: Typed', ] requires-python = '>=3.8' dependencies = ['pywin32>=226; platform_system == "Windows"'] @@ -105,6 +98,11 @@ python_version = '3.9' strict = true warn_return_any = true warn_unused_configs = true -files = ['portalocker'] +warn_unused_ignores = false +packages = ['portalocker', 'portalocker_tests'] ignore_missing_imports = true check_untyped_defs = true +exclude = ['dist', 'docs', '.venv', 'venv'] +[[tool.mypy.overrides]] +module = ['portalocker_tests.*'] +disallow_untyped_defs = false diff --git a/ruff.toml b/ruff.toml index df8b08b..04d227f 100644 --- a/ruff.toml +++ b/ruff.toml @@ -14,9 +14,6 @@ exclude = [ line-length = 79 -[format] -quote-style = 'single' - [lint] ignore = [ From a993d64e5f2155b9d27fb825a9b5a3df627a01d7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 01:15:01 +0100 Subject: [PATCH 207/225] fixed tests, pyright, ruff, etc... --- .github/dependabot.yml | 25 ++++++++ .github/workflows/lint.yml | 5 ++ .github/workflows/python-package.yml | 4 ++ .pre-commit-config.yaml | 73 ++++++++++++++++++++++++ docs/portalocker.types.rst | 7 +++ portalocker/__init__.py | 2 +- portalocker/constants.py | 2 +- portalocker/portalocker.py | 4 +- portalocker/redis.py | 2 +- portalocker/utils.py | 2 +- portalocker_tests/test_combined.py | 2 +- portalocker_tests/tests.py | 45 +++++++++++---- pyproject.toml | 40 ++++++++++++- tox.ini | 85 ---------------------------- 14 files changed, 194 insertions(+), 104 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .pre-commit-config.yaml create mode 100644 docs/portalocker.types.rst delete mode 100644 tox.ini diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..532bf89 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,25 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + target-branch: master + labels: + - "meta: CI" + schedule: + interval: monthly + groups: + actions: + patterns: + - "*" + + - package-ecosystem: pip + directory: / + target-branch: master + labels: + - "meta: deps" + schedule: + interval: monthly + groups: + actions: + patterns: + - "*" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 108877c..cd8cd90 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,9 +7,14 @@ on: env: FORCE_COLOR: 1 +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: lint: runs-on: ubuntu-latest + strategy: fail-fast: false matrix: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f0acbb8..cc6a814 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -9,6 +9,10 @@ on: env: FORCE_COLOR: 1 +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: # Run os specific tests on the slower OS X/Windows machines windows_osx: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..27ac480 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,73 @@ +ci: + autoupdate_branch: "master" + autoupdate_commit_msg: "⬆️ update pre-commit hooks" + skip: + - basedpyright + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-illegal-windows-names + - id: check-json + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-symlinks + - id: check-toml + - id: check-vcs-permalinks + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: destroyed-symlinks + - id: detect-aws-credentials + args: [--allow-missing-credentials] + - id: detect-private-key + - id: fix-byte-order-marker + - id: forbid-submodules + - id: name-tests-test + args: [--pytest-test-first] + - id: no-commit-to-branch + args: [--branch, master] + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.43.0 + hooks: + - id: markdownlint + + - repo: https://github.com/executablebooks/mdformat + rev: 0.7.21 + hooks: + - id: mdformat + additional_dependencies: + - mdformat-gfm + - mdformat-gfm-alerts + + - repo: https://github.com/crate-ci/typos + rev: v1.28.4 + hooks: + - id: typos + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.4 + hooks: + - id: ruff + args: [--fix, --show-fixes] + types_or: [python, pyi] + + - id: ruff-format + types_or: [python, pyi] + + - repo: local + hooks: + - id: basedpyright + name: basedpyright + entry: uv run --no-sync --locked basedpyright + language: system + types_or: [python, pyi] diff --git a/docs/portalocker.types.rst b/docs/portalocker.types.rst new file mode 100644 index 0000000..ae402ba --- /dev/null +++ b/docs/portalocker.types.rst @@ -0,0 +1,7 @@ +portalocker.types module +======================== + +.. automodule:: portalocker.types + :members: + :undoc-members: + :show-inheritance: diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 3b26a3a..1e8ded1 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -10,7 +10,7 @@ try: # pragma: no cover from .redis import RedisLock except ImportError: # pragma: no cover - RedisLock = None # type: ignore + RedisLock = None # type: ignore[assignment,misc] #: The package name on Pypi diff --git a/portalocker/constants.py b/portalocker/constants.py index 198571f..895fccc 100644 --- a/portalocker/constants.py +++ b/portalocker/constants.py @@ -30,7 +30,7 @@ #: non-blocking LOCK_NB = 0x4 #: unlock - LOCK_UN = msvcrt.LK_UNLCK # type: ignore + LOCK_UN = msvcrt.LK_UNLCK # type: ignore[attr-defined] elif os.name == 'posix': # pragma: no cover import fcntl diff --git a/portalocker/portalocker.py b/portalocker/portalocker.py index 94f66e1..dcd7bca 100644 --- a/portalocker/portalocker.py +++ b/portalocker/portalocker.py @@ -44,7 +44,7 @@ def lock(file_: types.IO | int, flags: LockFlags) -> None: if savepos: file_.seek(0) - os_fh = msvcrt.get_osfhandle(file_.fileno()) # type: ignore + os_fh = msvcrt.get_osfhandle(file_.fileno()) # type: ignore[attr-defined] try: win32file.LockFileEx(os_fh, mode, 0, -0x10000, __overlapped) except pywintypes.error as exc_value: @@ -70,7 +70,7 @@ def unlock(file_: types.IO) -> None: if savepos: file_.seek(0) - os_fh = msvcrt.get_osfhandle(file_.fileno()) # type: ignore + os_fh = msvcrt.get_osfhandle(file_.fileno()) # type: ignore[attr-defined] try: win32file.UnlockFileEx( os_fh, diff --git a/portalocker/redis.py b/portalocker/redis.py index 2336213..b8e6814 100644 --- a/portalocker/redis.py +++ b/portalocker/redis.py @@ -18,7 +18,7 @@ DEFAULT_THREAD_SLEEP_TIME = 0.1 -class PubSubWorkerThread(redis.client.PubSubWorkerThread): # type: ignore +class PubSubWorkerThread(redis.client.PubSubWorkerThread): def run(self) -> None: try: super().run() diff --git a/portalocker/utils.py b/portalocker/utils.py index 86128cb..e2d6b11 100644 --- a/portalocker/utils.py +++ b/portalocker/utils.py @@ -393,7 +393,7 @@ def acquire( else: fh = super().acquire(timeout, check_interval, fail_when_locked) self._acquire_count += 1 - assert fh + assert fh is not None return fh def release(self) -> None: diff --git a/portalocker_tests/test_combined.py b/portalocker_tests/test_combined.py index dacda89..700611a 100644 --- a/portalocker_tests/test_combined.py +++ b/portalocker_tests/test_combined.py @@ -14,6 +14,6 @@ def test_combined(tmpdir): sys.path.append(output_file.dirname) # Combined is being generated above but linters won't understand that - import combined # type: ignore + import combined # pyright: ignore[reportMissingImports] assert combined diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 910c7da..d82e331 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -2,6 +2,7 @@ import math import multiprocessing import os +import sys import time import typing @@ -19,7 +20,7 @@ fcntl.lockf, ] else: - LOCKERS = [None] # type: ignore + LOCKERS = [None] # type: ignore[list-item] @pytest.fixture @@ -61,7 +62,7 @@ def test_with_timeout(tmpfile): print('writing more stuff to my cache...', file=fh) -def test_without_timeout(tmpfile, monkeypatch): +def test_without_timeout(tmpfile): # Open the file 2 times with pytest.raises(portalocker.LockException): with portalocker.Lock(tmpfile, timeout=None) as fh: @@ -333,6 +334,10 @@ def lock( @pytest.mark.parametrize('fail_when_locked', [True, False]) +@pytest.mark.skipif( + 'pypy' in sys.version.lower(), + reason='pypy3 does not support the multiprocessing test', + ) def test_shared_processes(tmpfile, fail_when_locked): flags = LockFlags.SHARED | LockFlags.NON_BLOCKING print() @@ -351,8 +356,25 @@ def test_shared_processes(tmpfile, fail_when_locked): assert result == LockResult() +def _lock_and_sleep( + filename: str, + fail_when_locked: bool, + flags: LockFlags, + keep_locked: float = 0.1, +) -> LockResult: + result = lock(filename, fail_when_locked, flags) + print(f'{result=}') + time.sleep(keep_locked) + return result + + @pytest.mark.parametrize('fail_when_locked', [True, False]) @pytest.mark.parametrize('locker', LOCKERS, indirect=True) +# Skip pypy3 +@pytest.mark.skipif( + 'pypy' in sys.version.lower(), + reason='pypy3 does not support the multiprocessing test', +) def test_exclusive_processes( tmpfile: str, fail_when_locked: bool, @@ -367,23 +389,26 @@ def test_exclusive_processes( result_b = pool.apply_async(lock, [tmpfile, fail_when_locked, flags]) try: - a = result_a.get(timeout=1.1) # Wait for 'a' with timeout + a = result_a.get(timeout=1) # Wait for 'a' with timeout except multiprocessing.TimeoutError: a = None + print(f'{a=}') + print(repr(a)) + try: # Lower timeout since we already waited with `a` - b = result_b.get(timeout=0.2) # Wait for 'b' with timeout + b = result_b.get(timeout=0.1) # Wait for 'b' with timeout except multiprocessing.TimeoutError: b = None + print(f'{b=}') + print(repr(b)) + assert a or b # Make sure a is always filled - if b: - b, a = b, a - - print(f'{a=}') - print(f'{b=}') + if a is None: + b, a = a, b # make pyright happy assert a is not None @@ -394,7 +419,7 @@ def test_exclusive_processes( assert not a.exception_class or not b.exception_class assert issubclass( - a.exception_class or b.exception_class, # type: ignore + a.exception_class or b.exception_class, # type: ignore[arg-type] portalocker.LockException, ) else: diff --git a/pyproject.toml b/pyproject.toml index ca98b57..0d614b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,12 @@ classifiers = [ 'Topic :: System :: Monitoring', 'Typing :: Typed', ] -requires-python = '>=3.8' -dependencies = ['pywin32>=226; platform_system == "Windows"'] +requires-python = '>=3.9' +dependencies = [ + 'pywin32>=226; platform_system == "Windows"', + 'tomli-w>=1.0.0', + 'tomlkit>=0.13.2', +] [project.urls] bugs = 'https://github.com/wolph/portalocker/issues' @@ -103,6 +107,38 @@ packages = ['portalocker', 'portalocker_tests'] ignore_missing_imports = true check_untyped_defs = true exclude = ['dist', 'docs', '.venv', 'venv'] +enable_error_code = ['ignore-without-code', 'truthy-bool', 'redundant-expr'] +warn_unreachable = true + [[tool.mypy.overrides]] module = ['portalocker_tests.*'] disallow_untyped_defs = false + +[dependency-groups] +dev = [ + 'pytest-doc>=0.0.1', + 'pytest>=8.3.4', + 'pytest-docs>=0.1.0', + 'pytest-mypy>=0.10.3', + 'pytest-timeout>=2.3.1', + 'redis>=5.2.1', + 'sphinx>=7.1.2', + 'types-redis>=4.6.0.20241004', + 'mypy>=1.14.0', + 'pytest-cov>=5.0.0', +] + +[tool.ruff] +src = ['portalocker', 'portalocker_tests'] +include = ['portalocker/**/*.py', 'portalocker_tests/**/*.py'] + +[tool.repo-review] +ignore = [ + 'PY004', # no /docs + 'PY007', # tox configured in tox.toml + 'PP301', # pytest is irrelevant + 'PC111', # no blacken-docs because markdown has no code + 'PC140', # manual typecheck pre-commit hooks + 'PC170', # no pygrep-hooks because no rST + 'RTD', # no RTD +] diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 371099c..0000000 --- a/tox.ini +++ /dev/null @@ -1,85 +0,0 @@ -[tox] -envlist = - py38 - py39 - py310 - py311 - py312 - pypy3 - flake8 - docs - mypy - pyright - ruff - codespell - black - -skip_missing_interpreters = True - -[testenv] -pass_env = - FORCE_COLOR -basepython = - py38: python3.8 - py39: python3.9 - py310: python3.10 - py311: python3.11 - py312: python3.12 - pypy3: pypy3 - -deps = -e{toxinidir}[tests,redis] -commands = python -m pytest {posargs} - -[testenv:mypy] -basepython = python3 -deps = mypy -commands = - mypy --install-types --non-interactive - mypy - -[testenv:pyright] -changedir = -basepython = python3 -deps = - pyright - types-redis - -e{toxinidir}[tests,redis] -commands = pyright {toxinidir}/portalocker {toxinidir}/portalocker_tests - -[testenv:flake8] -basepython = python3 -deps = flake8>=6.0.0 -commands = flake8 {toxinidir}/portalocker {toxinidir}/portalocker_tests - -[testenv:black] -basepython = python3 -deps = black -commands = black {toxinidir}/portalocker {toxinidir}/portalocker_tests - -[testenv:docs] -basepython = python3 -deps = -r{toxinidir}/docs/requirements.txt -allowlist_externals = - rm - mkdir -whitelist_externals = - rm - cd - mkdir -commands = - rm -f docs/modules.rst - mkdir -p docs/_static - sphinx-apidoc -e -o docs/ portalocker - rm -f docs/modules.rst - sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html {posargs} - -[testenv:ruff] -commands = ruff check {toxinidir}/portalocker {toxinidir}/portalocker_tests -deps = ruff -skip_install = true - -[testenv:codespell] -commands = codespell . -deps = codespell -skip_install = true -command = codespell From 96bd076394e913743d1d7570745ef1c94cd16038 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Dec 2024 00:18:35 +0000 Subject: [PATCH 208/225] Bump the actions group with 4 updates Bumps the actions group with 4 updates: [actions/setup-python](https://github.com/actions/setup-python), [jakebailey/pyright-action](https://github.com/jakebailey/pyright-action), [supercharge/redis-github-action](https://github.com/supercharge/redis-github-action) and [actions/stale](https://github.com/actions/stale). Updates `actions/setup-python` from 4 to 5 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) Updates `jakebailey/pyright-action` from 1 to 2 - [Release notes](https://github.com/jakebailey/pyright-action/releases) - [Commits](https://github.com/jakebailey/pyright-action/compare/v1...v2) Updates `supercharge/redis-github-action` from 1.7.0 to 1.8.0 - [Release notes](https://github.com/supercharge/redis-github-action/releases) - [Changelog](https://github.com/supercharge/redis-github-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/supercharge/redis-github-action/compare/1.7.0...1.8.0) Updates `actions/stale` from 8 to 9 - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v8...v9) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: jakebailey/pyright-action dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: supercharge/redis-github-action dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions ... Signed-off-by: dependabot[bot] --- .github/workflows/lint.yml | 4 ++-- .github/workflows/python-package.yml | 6 +++--- .github/workflows/stale.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 108877c..d086f7c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -31,7 +31,7 @@ jobs: python -m pip install -e '.[tests]' - name: Linting with pyright - uses: jakebailey/pyright-action@v1 + uses: jakebailey/pyright-action@v2 with: path: portalocker portalocker_tests diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f0acbb8..3f74c4c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -22,7 +22,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -46,9 +46,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Start Redis - uses: supercharge/redis-github-action@1.7.0 + uses: supercharge/redis-github-action@1.8.0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index b0efa53..936d0d9 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -9,7 +9,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@v9 with: days-before-stale: 30 days-before-pr-stale: -1 From 46140ec23ae28c1acab6f1eb58ffe167d2fdbe84 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 01:23:26 +0100 Subject: [PATCH 209/225] added retries to flaky tests --- portalocker_tests/tests.py | 7 ++++--- pyproject.toml | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index d82e331..ee1b3e9 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -337,7 +337,7 @@ def lock( @pytest.mark.skipif( 'pypy' in sys.version.lower(), reason='pypy3 does not support the multiprocessing test', - ) +) def test_shared_processes(tmpfile, fail_when_locked): flags = LockFlags.SHARED | LockFlags.NON_BLOCKING print() @@ -375,6 +375,7 @@ def _lock_and_sleep( 'pypy' in sys.version.lower(), reason='pypy3 does not support the multiprocessing test', ) +@pytest.mark.flaky(reruns=5, reruns_delay=1) def test_exclusive_processes( tmpfile: str, fail_when_locked: bool, @@ -389,7 +390,7 @@ def test_exclusive_processes( result_b = pool.apply_async(lock, [tmpfile, fail_when_locked, flags]) try: - a = result_a.get(timeout=1) # Wait for 'a' with timeout + a = result_a.get(timeout=1.2) # Wait for 'a' with timeout except multiprocessing.TimeoutError: a = None @@ -398,7 +399,7 @@ def test_exclusive_processes( try: # Lower timeout since we already waited with `a` - b = result_b.get(timeout=0.1) # Wait for 'b' with timeout + b = result_b.get(timeout=0.6) # Wait for 'b' with timeout except multiprocessing.TimeoutError: b = None diff --git a/pyproject.toml b/pyproject.toml index 0d614b7..93ea4ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,7 @@ dev = [ 'types-redis>=4.6.0.20241004', 'mypy>=1.14.0', 'pytest-cov>=5.0.0', + 'pytest-rerunfailures>=15.0', ] [tool.ruff] From fbd50060a775a2553da702676b44ab0410c5f5c1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 04:06:47 +0100 Subject: [PATCH 210/225] testing appveyor fix --- appveyor.yml | 7 +++---- portalocker_tests/tests.py | 1 + pyproject.toml | 19 ++++--------------- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0b57035..a48bd93 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,16 +6,15 @@ image: environment: matrix: - - TOXENV: py38 - TOXENV: py39 - TOXENV: py310 - TOXENV: py311 + - TOXENV: py312 install: - - py -m pip install -U tox setuptools wheel - - py -m pip install -Ue ".[tests]" + - powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" build: false # Not a C# project, build stuff at the test step instead. test_script: - - py -m tox" + - uv run tox diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index ee1b3e9..737b9d0 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -338,6 +338,7 @@ def lock( 'pypy' in sys.version.lower(), reason='pypy3 does not support the multiprocessing test', ) +@pytest.mark.flaky(reruns=5, reruns_delay=1) def test_shared_processes(tmpfile, fail_when_locked): flags = LockFlags.SHARED | LockFlags.NON_BLOCKING print() diff --git a/pyproject.toml b/pyproject.toml index 93ea4ef..e8374cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ tests = [ 'pytest-mypy>=0.8.0', 'types-redis', 'redis', + 'pytest-rerunfailures>=15.0', ] redis = ['redis'] @@ -114,21 +115,6 @@ warn_unreachable = true module = ['portalocker_tests.*'] disallow_untyped_defs = false -[dependency-groups] -dev = [ - 'pytest-doc>=0.0.1', - 'pytest>=8.3.4', - 'pytest-docs>=0.1.0', - 'pytest-mypy>=0.10.3', - 'pytest-timeout>=2.3.1', - 'redis>=5.2.1', - 'sphinx>=7.1.2', - 'types-redis>=4.6.0.20241004', - 'mypy>=1.14.0', - 'pytest-cov>=5.0.0', - 'pytest-rerunfailures>=15.0', -] - [tool.ruff] src = ['portalocker', 'portalocker_tests'] include = ['portalocker/**/*.py', 'portalocker_tests/**/*.py'] @@ -143,3 +129,6 @@ ignore = [ 'PC170', # no pygrep-hooks because no rST 'RTD', # no RTD ] + +[tool.uv.pip] +all-extras = true From 6fdd2170c6ccf2d957e3b81efbe2fe186ea69a2b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 04:12:09 +0100 Subject: [PATCH 211/225] fixed uv sync dependencies --- pyproject.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e8374cf..a4446e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,11 @@ warn_unreachable = true module = ['portalocker_tests.*'] disallow_untyped_defs = false +[dependency-groups] +dev = [ + 'portalocker[tests]', +] + [tool.ruff] src = ['portalocker', 'portalocker_tests'] include = ['portalocker/**/*.py', 'portalocker_tests/**/*.py'] @@ -129,6 +134,3 @@ ignore = [ 'PC170', # no pygrep-hooks because no rST 'RTD', # no RTD ] - -[tool.uv.pip] -all-extras = true From 1ea2915ff284d7236acf4f82a801f7e8287877db Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 04:15:36 +0100 Subject: [PATCH 212/225] fixed uv sync dependencies --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index a48bd93..74ac847 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,4 +17,5 @@ install: build: false # Not a C# project, build stuff at the test step instead. test_script: + - $env:Path = "C:\Users\appveyor\.local\bin;$env:Path" - uv run tox From 9a02788f60464ea268aa13f5999bdc3070ac2d65 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 04:21:54 +0100 Subject: [PATCH 213/225] testing appveyor fix --- appveyor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 74ac847..db4fb91 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,5 +17,4 @@ install: build: false # Not a C# project, build stuff at the test step instead. test_script: - - $env:Path = "C:\Users\appveyor\.local\bin;$env:Path" - - uv run tox + - C:\Users\appveyor\.local\bin\uv.exe run tox From f8f257e813479ffdeaa5aa5ba70cc68b5290c631 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 04:25:19 +0100 Subject: [PATCH 214/225] testing appveyor fix --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index db4fb91..e25bcd8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,4 +17,4 @@ install: build: false # Not a C# project, build stuff at the test step instead. test_script: - - C:\Users\appveyor\.local\bin\uv.exe run tox + - C:\Users\appveyor\.local\bin\uvx.exe tox From 818cda9dfe2153415e020704681edb5a22148e46 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 04:32:22 +0100 Subject: [PATCH 215/225] testing appveyor fix --- appveyor.yml | 4 +++ tox.toml | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 tox.toml diff --git a/appveyor.yml b/appveyor.yml index e25bcd8..f3c7320 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,11 +10,15 @@ environment: - TOXENV: py310 - TOXENV: py311 - TOXENV: py312 + - TOXENV: py313 install: - powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" +# - py -m pip install -U tox setuptools wheel +# - py -m pip install -Ue ".[tests]" build: false # Not a C# project, build stuff at the test step instead. test_script: + - dir - C:\Users\appveyor\.local\bin\uvx.exe tox diff --git a/tox.toml b/tox.toml new file mode 100644 index 0000000..d1d08a4 --- /dev/null +++ b/tox.toml @@ -0,0 +1,78 @@ +env_list = [ + 'py39', + 'py310', + 'py311', + 'py312', + 'pypy3', + 'docs', + 'mypy', + 'pyright', + 'ruff', + 'repo-review', + 'codespell', +] +skip_missing_interpreters = true + +[env_run_base] +pass_env = ['FORCE_COLOR'] +commands = [['pytest', '{posargs}']] +extras = ['tests', 'redis'] + +[env.mypy] +commands = [['mypy']] + +[env.pyright] +deps = ['pyright'] +commands = [['pyright']] + +[env.ruff] +deps = ['ruff'] +commands = [['ruff', 'check'], ['ruff', 'format', '--check']] + +[env.docs] +extras = ['docs'] +allowlist_externals = ['rm', 'cd', 'mkdir'] +commands = [ + [ + 'rm', + '-f', + 'docs/modules.rst', + ], + [ + 'mkdir', + '-p', + 'docs/_static', + ], + [ + 'sphinx-apidoc', + '-e', + '-o', + 'docs/', + 'portalocker', + ], + [ + 'rm', + '-f', + 'docs/modules.rst', + ], + [ + 'sphinx-build', + '-b', + 'html', + '-d', + 'docs/_build/doctrees', + 'docs', + 'docs/_build/html', + '{posargs}', + ], +] + +[env.repo-review] +deps = ['sp-repo-review[cli]', 'validate-pyproject'] +commands = [['repo-review']] + +[env.codespell] +commands = [['codespell']] +deps = ['codespell', 'tomli'] +skip_install = true +command = 'codespell' From 43f2f59b5e8975d9dbfe6fec55e4154a0d5acbf5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 04:44:53 +0100 Subject: [PATCH 216/225] testing appveyor fix --- tox.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.toml b/tox.toml index d1d08a4..d6b3813 100644 --- a/tox.toml +++ b/tox.toml @@ -68,6 +68,7 @@ commands = [ ] [env.repo-review] +basepython = 'py3.12' deps = ['sp-repo-review[cli]', 'validate-pyproject'] commands = [['repo-review']] From 433fbff4eb2f4763544a14bbae8de6460a6218ff Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 04:52:01 +0100 Subject: [PATCH 217/225] testing tox github actions fix --- tox.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.toml b/tox.toml index d6b3813..13cb369 100644 --- a/tox.toml +++ b/tox.toml @@ -68,7 +68,7 @@ commands = [ ] [env.repo-review] -basepython = 'py3.12' +basepython = 'py312' deps = ['sp-repo-review[cli]', 'validate-pyproject'] commands = [['repo-review']] From a104dd3e3fc87bb6aa130c0cadb93ec4d14821d7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 05:11:58 +0100 Subject: [PATCH 218/225] testing tox github actions fix --- tox.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.toml b/tox.toml index 13cb369..69a0599 100644 --- a/tox.toml +++ b/tox.toml @@ -68,7 +68,7 @@ commands = [ ] [env.repo-review] -basepython = 'py312' +basepython = ['py312'] deps = ['sp-repo-review[cli]', 'validate-pyproject'] commands = [['repo-review']] From 63c543a44154ac2d661f40f3f0096191bb2a42b8 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 05:15:16 +0100 Subject: [PATCH 219/225] increased timeout for lses flaky test --- portalocker_tests/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index 737b9d0..c9b38cc 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -349,7 +349,7 @@ def test_shared_processes(tmpfile, fail_when_locked): results = pool.starmap_async(lock, 2 * [args]) # sourcery skip: no-loop-in-tests - for result in results.get(timeout=1.0): + for result in results.get(timeout=1.2): print(f'{result=}') # sourcery skip: no-conditionals-in-tests if result.exception_class is not None: From f066f0532b95fd50a2bc02d07bfe9493d0a758da Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 12:33:23 +0100 Subject: [PATCH 220/225] last attempt for appveyor --- portalocker_tests/tests.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/portalocker_tests/tests.py b/portalocker_tests/tests.py index c9b38cc..bbc5f57 100644 --- a/portalocker_tests/tests.py +++ b/portalocker_tests/tests.py @@ -349,7 +349,7 @@ def test_shared_processes(tmpfile, fail_when_locked): results = pool.starmap_async(lock, 2 * [args]) # sourcery skip: no-loop-in-tests - for result in results.get(timeout=1.2): + for result in results.get(timeout=1.5): print(f'{result=}') # sourcery skip: no-conditionals-in-tests if result.exception_class is not None: @@ -357,18 +357,6 @@ def test_shared_processes(tmpfile, fail_when_locked): assert result == LockResult() -def _lock_and_sleep( - filename: str, - fail_when_locked: bool, - flags: LockFlags, - keep_locked: float = 0.1, -) -> LockResult: - result = lock(filename, fail_when_locked, flags) - print(f'{result=}') - time.sleep(keep_locked) - return result - - @pytest.mark.parametrize('fail_when_locked', [True, False]) @pytest.mark.parametrize('locker', LOCKERS, indirect=True) # Skip pypy3 From fe481b688c892d00902ba841f82f8aa6472db018 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 12:42:26 +0100 Subject: [PATCH 221/225] Incrementing version to v3.1.0 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index dd6cbfc..a31a87e 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,6 +1,6 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '3.0.0' +__version__ = '3.1.0' __description__ = """Wraps the portalocker recipe for easy usage""" __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 1e8ded1..03c83d2 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -20,7 +20,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '3.0.0' +__version__ = '3.1.0' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 69cb82423ee7c7312e34aa4f92323bf2cd9f703d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 13:20:21 +0100 Subject: [PATCH 222/225] improved combine script --- portalocker/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/portalocker/__main__.py b/portalocker/__main__.py index d99ffcf..f31930a 100644 --- a/portalocker/__main__.py +++ b/portalocker/__main__.py @@ -124,7 +124,8 @@ def combine(args: argparse.Namespace) -> None: logger.info(f'Wrote combined file to {output_file.name}') # Run black and ruff if available. If not then just run the file. os.system(f'black {output_file.name}') - os.system(f'ruff --fix {output_file.name}') + os.system(f'ruff format {output_file.name}') + os.system(f'ruff check --fix --fix-only {output_file.name}') os.system(f'python3 {output_file.name}') From d5086e13d2dfab582e03327aaab37d2b2c680d14 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 15:22:08 +0100 Subject: [PATCH 223/225] removed accidental toml requirements --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a4446e3..71a2808 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,8 +50,6 @@ classifiers = [ requires-python = '>=3.9' dependencies = [ 'pywin32>=226; platform_system == "Windows"', - 'tomli-w>=1.0.0', - 'tomlkit>=0.13.2', ] [project.urls] From 22a68d0fd22fb73e5c96de656e6df2a3c6be2cf5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 Dec 2024 15:22:23 +0100 Subject: [PATCH 224/225] Incrementing version to v3.1.1 --- portalocker/__about__.py | 2 +- portalocker/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/portalocker/__about__.py b/portalocker/__about__.py index a31a87e..e734840 100644 --- a/portalocker/__about__.py +++ b/portalocker/__about__.py @@ -1,6 +1,6 @@ __package_name__ = 'portalocker' __author__ = 'Rick van Hattem' __email__ = 'wolph@wol.ph' -__version__ = '3.1.0' +__version__ = '3.1.1' __description__ = """Wraps the portalocker recipe for easy usage""" __url__ = 'https://github.com/WoLpH/portalocker' diff --git a/portalocker/__init__.py b/portalocker/__init__.py index 03c83d2..b461e9c 100644 --- a/portalocker/__init__.py +++ b/portalocker/__init__.py @@ -20,7 +20,7 @@ #: Current author's email address __email__ = __about__.__email__ #: Version number -__version__ = '3.1.0' +__version__ = '3.1.1' #: Package description for Pypi __description__ = __about__.__description__ #: Package homepage From 78104bdab4bbe6f5203329b44da4963dbde9d725 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 6 Jan 2025 03:10:13 +0100 Subject: [PATCH 225/225] Update stale.yml --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 936d0d9..c3b6dc2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -14,4 +14,4 @@ jobs: days-before-stale: 30 days-before-pr-stale: -1 exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement - exempt-all-pr-assignees: true + remove-issue-stale-when-updated: true