diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml deleted file mode 100644 index a65ae28..0000000 --- a/.github/workflows/black.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: Black - -on: [push, pull_request] - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: psf/black@stable \ No newline at end of file diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml deleted file mode 100644 index ef7818a..0000000 --- a/.github/workflows/flake8.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: flake8 - -on: [push, pull_request] - -jobs: - flake8-lint: - runs-on: ubuntu-latest - name: Lint - steps: - - name: Check out source repository - uses: actions/checkout@v3 - - name: Set up Python environment - uses: actions/setup-python@v4 - with: - python-version: "3.12" - - name: flake8 Lint - uses: py-actions/flake8@v2 - with: - max-line-length: "88" \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..7810955 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,28 @@ +name: Lints + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: flake8 + run: | + pip install `egrep -o 'flake8==\S+' requirements.txt` # install our version of flake8 + flake8 web/ --config wikimedia/setup.cfg + + - name: black + run: | + pip install `egrep -o 'black==\S+' requirements.txt` # install our version of black + black --check --diff . # Uses pyproject.toml + + - name: mypy + run: | + pip install -r requirements.txt # install a full environment for mypy + cd wikimedia + PYTHONPATH=. mypy \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 5df3615..e344dab 100644 --- a/poetry.lock +++ b/poetry.lock @@ -198,6 +198,27 @@ redshift = ["redshift-connector (>=2.0.0,<3.0.0)"] sparql = ["SPARQLWrapper (>=2.0.0,<3.0.0)", "requests (>=2.0.0,<3.0.0)"] sqlserver = ["pyodbc (>=4,<6)"] +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + [[package]] name = "black" version = "24.4.2" @@ -886,6 +907,52 @@ files = [ {file = "mwparserfromhell-0.6.6.tar.gz", hash = "sha256:71afec1e9784ba576e95d6f34845582d3c733a3a52ba770dd8a9c3a40e5b649f"}, ] +[[package]] +name = "mypy" +version = "1.10.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2"}, + {file = "mypy-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99"}, + {file = "mypy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2"}, + {file = "mypy-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9"}, + {file = "mypy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051"}, + {file = "mypy-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1"}, + {file = "mypy-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee"}, + {file = "mypy-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de"}, + {file = "mypy-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7"}, + {file = "mypy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53"}, + {file = "mypy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b"}, + {file = "mypy-1.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30"}, + {file = "mypy-1.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e"}, + {file = "mypy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5"}, + {file = "mypy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda"}, + {file = "mypy-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0"}, + {file = "mypy-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727"}, + {file = "mypy-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4"}, + {file = "mypy-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061"}, + {file = "mypy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f"}, + {file = "mypy-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976"}, + {file = "mypy-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec"}, + {file = "mypy-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821"}, + {file = "mypy-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746"}, + {file = "mypy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a"}, + {file = "mypy-1.10.0-py3-none-any.whl", hash = "sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee"}, + {file = "mypy-1.10.0.tar.gz", hash = "sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -1273,6 +1340,17 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "soupsieve" +version = "2.5" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + [[package]] name = "typing-extensions" version = "4.12.2" @@ -1508,4 +1586,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.12,<4.0" -content-hash = "727a6653750005a7d3417d697aae9261b9f72f6059884628b42b7ab39352094b" +content-hash = "2fff7cf751f07ea79a28212a16f8ff55a4fa81acbdb9d74864771b86cf874c82" diff --git a/pyproject.toml b/pyproject.toml index 7793cb0..1ccf73c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,10 +17,16 @@ s3fs = "^2024.6.0" mwparserfromhell = "^0.6.6" validators = "^0.28.3" requests = "^2.32.3" +beautifulsoup4 = "^4.12.3" -[tool.poetry.dev-dependencies] +[tool.black] +line-length = 88 +extend-exclude = "sdc-sync.py" + +[tool.poetry.group.dev.dependencies] black = "^24.4.2" flake8 = "^7.1.0" +mypy = "^1.10.0" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/requirements.txt b/requirements.txt index c9cb234..e8bd807 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,43 +1,845 @@ -aiobotocore==2.5.2 -aiohttp==3.8.4 -aioitertools==0.11.0 -aiosignal==1.3.1 -async-timeout==4.0.2 -attrs==23.1.0 -awswrangler==3.2.1 -boto3==1.28.2 -botocore==1.29.161 -certifi==2023.5.7 -cffi==1.15.1 -charset-normalizer==3.2.0 -cleo==2.0.1 -cramjam==2.6.2 -crashtest==0.4.1 -docutils==0.20.1 -fastparquet==2023.7.0 -frozenlist==1.3.3 -fsspec==2023.6.0 -idna==3.4 -jmespath==1.0.1 -multidict==6.0.4 -mwparserfromhell==0.6.4 -numpy==1.25.1 -packaging==23.1 -pandas==2.0.3 -pyarrow==12.0.1 -pycparser==2.21 -python-dateutil==2.8.2 -python-magic==0.4.27 -pytz==2023.3 -pywikibot==8.2.0 -PyYAML==6.0 -rapidfuzz==2.15.1 -requests==2.31.0 -s3fs==2023.6.0 -s3transfer==0.6.1 -six==1.16.0 -typing_extensions==4.7.1 -tzdata==2023.3 -urllib3==1.26.16 -wrapt==1.15.0 -yarl==1.9.2 +aiobotocore==2.13.1 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:134f9606c2f91abde38cbc61c3241113e26ff244633e0c31abb7e09da3581c9b \ + --hash=sha256:1bef121b99841ee3cc788e4ed97c332ba32353b1f00e886d1beb3aae95520858 +aiohttp==3.9.5 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:0605cc2c0088fcaae79f01c913a38611ad09ba68ff482402d3410bf59039bfb8 \ + --hash=sha256:0a158704edf0abcac8ac371fbb54044f3270bdbc93e254a82b6c82be1ef08f3c \ + --hash=sha256:0cbf56238f4bbf49dab8c2dc2e6b1b68502b1e88d335bea59b3f5b9f4c001475 \ + --hash=sha256:1732102949ff6087589408d76cd6dea656b93c896b011ecafff418c9661dc4ed \ + --hash=sha256:18f634d540dd099c262e9f887c8bbacc959847cfe5da7a0e2e1cf3f14dbf2daf \ + --hash=sha256:239f975589a944eeb1bad26b8b140a59a3a320067fb3cd10b75c3092405a1372 \ + --hash=sha256:2faa61a904b83142747fc6a6d7ad8fccff898c849123030f8e75d5d967fd4a81 \ + --hash=sha256:320e8618eda64e19d11bdb3bd04ccc0a816c17eaecb7e4945d01deee2a22f95f \ + --hash=sha256:38d80498e2e169bc61418ff36170e0aad0cd268da8b38a17c4cf29d254a8b3f1 \ + --hash=sha256:3916c8692dbd9d55c523374a3b8213e628424d19116ac4308e434dbf6d95bbdd \ + --hash=sha256:393c7aba2b55559ef7ab791c94b44f7482a07bf7640d17b341b79081f5e5cd1a \ + --hash=sha256:3b7b30258348082826d274504fbc7c849959f1989d86c29bc355107accec6cfb \ + --hash=sha256:3fcb4046d2904378e3aeea1df51f697b0467f2aac55d232c87ba162709478c46 \ + --hash=sha256:4109adee842b90671f1b689901b948f347325045c15f46b39797ae1bf17019de \ + --hash=sha256:4558e5012ee03d2638c681e156461d37b7a113fe13970d438d95d10173d25f78 \ + --hash=sha256:45731330e754f5811c314901cebdf19dd776a44b31927fa4b4dbecab9e457b0c \ + --hash=sha256:4715a9b778f4293b9f8ae7a0a7cef9829f02ff8d6277a39d7f40565c737d3771 \ + --hash=sha256:471f0ef53ccedec9995287f02caf0c068732f026455f07db3f01a46e49d76bbb \ + --hash=sha256:4d3ebb9e1316ec74277d19c5f482f98cc65a73ccd5430540d6d11682cd857430 \ + --hash=sha256:4ff550491f5492ab5ed3533e76b8567f4b37bd2995e780a1f46bca2024223233 \ + --hash=sha256:52c27110f3862a1afbcb2af4281fc9fdc40327fa286c4625dfee247c3ba90156 \ + --hash=sha256:55b39c8684a46e56ef8c8d24faf02de4a2b2ac60d26cee93bc595651ff545de9 \ + --hash=sha256:5a7ee16aab26e76add4afc45e8f8206c95d1d75540f1039b84a03c3b3800dd59 \ + --hash=sha256:5ca51eadbd67045396bc92a4345d1790b7301c14d1848feaac1d6a6c9289e888 \ + --hash=sha256:5d6b3f1fabe465e819aed2c421a6743d8debbde79b6a8600739300630a01bf2c \ + --hash=sha256:60cdbd56f4cad9f69c35eaac0fbbdf1f77b0ff9456cebd4902f3dd1cf096464c \ + --hash=sha256:6380c039ec52866c06d69b5c7aad5478b24ed11696f0e72f6b807cfb261453da \ + --hash=sha256:639d0042b7670222f33b0028de6b4e2fad6451462ce7df2af8aee37dcac55424 \ + --hash=sha256:66331d00fb28dc90aa606d9a54304af76b335ae204d1836f65797d6fe27f1ca2 \ + --hash=sha256:67c3119f5ddc7261d47163ed86d760ddf0e625cd6246b4ed852e82159617b5fb \ + --hash=sha256:694d828b5c41255e54bc2dddb51a9f5150b4eefa9886e38b52605a05d96566e8 \ + --hash=sha256:6ae79c1bc12c34082d92bf9422764f799aee4746fd7a392db46b7fd357d4a17a \ + --hash=sha256:702e2c7c187c1a498a4e2b03155d52658fdd6fda882d3d7fbb891a5cf108bb10 \ + --hash=sha256:714d4e5231fed4ba2762ed489b4aec07b2b9953cf4ee31e9871caac895a839c0 \ + --hash=sha256:7b179eea70833c8dee51ec42f3b4097bd6370892fa93f510f76762105568cf09 \ + --hash=sha256:7f64cbd44443e80094309875d4f9c71d0401e966d191c3d469cde4642bc2e031 \ + --hash=sha256:82a6a97d9771cb48ae16979c3a3a9a18b600a8505b1115cfe354dfb2054468b4 \ + --hash=sha256:84dabd95154f43a2ea80deffec9cb44d2e301e38a0c9d331cc4aa0166fe28ae3 \ + --hash=sha256:8676e8fd73141ded15ea586de0b7cda1542960a7b9ad89b2b06428e97125d4fa \ + --hash=sha256:88e311d98cc0bf45b62fc46c66753a83445f5ab20038bcc1b8a1cc05666f428a \ + --hash=sha256:8b4f72fbb66279624bfe83fd5eb6aea0022dad8eec62b71e7bf63ee1caadeafe \ + --hash=sha256:8c64a6dc3fe5db7b1b4d2b5cb84c4f677768bdc340611eca673afb7cf416ef5a \ + --hash=sha256:8cf142aa6c1a751fcb364158fd710b8a9be874b81889c2bd13aa8893197455e2 \ + --hash=sha256:8d1964eb7617907c792ca00b341b5ec3e01ae8c280825deadbbd678447b127e1 \ + --hash=sha256:93e22add827447d2e26d67c9ac0161756007f152fdc5210277d00a85f6c92323 \ + --hash=sha256:9c69e77370cce2d6df5d12b4e12bdcca60c47ba13d1cbbc8645dd005a20b738b \ + --hash=sha256:9dbc053ac75ccc63dc3a3cc547b98c7258ec35a215a92bd9f983e0aac95d3d5b \ + --hash=sha256:9e3a1ae66e3d0c17cf65c08968a5ee3180c5a95920ec2731f53343fac9bad106 \ + --hash=sha256:a6ea1a5b409a85477fd8e5ee6ad8f0e40bf2844c270955e09360418cfd09abac \ + --hash=sha256:a81b1143d42b66ffc40a441379387076243ef7b51019204fd3ec36b9f69e77d6 \ + --hash=sha256:ad7f2919d7dac062f24d6f5fe95d401597fbb015a25771f85e692d043c9d7832 \ + --hash=sha256:afc52b8d969eff14e069a710057d15ab9ac17cd4b6753042c407dcea0e40bf75 \ + --hash=sha256:b3df71da99c98534be076196791adca8819761f0bf6e08e07fd7da25127150d6 \ + --hash=sha256:c088c4d70d21f8ca5c0b8b5403fe84a7bc8e024161febdd4ef04575ef35d474d \ + --hash=sha256:c26959ca7b75ff768e2776d8055bf9582a6267e24556bb7f7bd29e677932be72 \ + --hash=sha256:c413016880e03e69d166efb5a1a95d40f83d5a3a648d16486592c49ffb76d0db \ + --hash=sha256:c6021d296318cb6f9414b48e6a439a7f5d1f665464da507e8ff640848ee2a58a \ + --hash=sha256:c671dc117c2c21a1ca10c116cfcd6e3e44da7fcde37bf83b2be485ab377b25da \ + --hash=sha256:c7a4b7a6cf5b6eb11e109a9755fd4fda7d57395f8c575e166d363b9fc3ec4678 \ + --hash=sha256:c8a02fbeca6f63cb1f0475c799679057fc9268b77075ab7cf3f1c600e81dd46b \ + --hash=sha256:cd2adf5c87ff6d8b277814a28a535b59e20bfea40a101db6b3bdca7e9926bc24 \ + --hash=sha256:d1469f228cd9ffddd396d9948b8c9cd8022b6d1bf1e40c6f25b0fb90b4f893ed \ + --hash=sha256:d153f652a687a8e95ad367a86a61e8d53d528b0530ef382ec5aaf533140ed00f \ + --hash=sha256:d5ab8e1f6bee051a4bf6195e38a5c13e5e161cb7bad83d8854524798bd9fcd6e \ + --hash=sha256:da00da442a0e31f1c69d26d224e1efd3a1ca5bcbf210978a2ca7426dfcae9f58 \ + --hash=sha256:da22dab31d7180f8c3ac7c7635f3bcd53808f374f6aa333fe0b0b9e14b01f91a \ + --hash=sha256:e0ae53e33ee7476dd3d1132f932eeb39bf6125083820049d06edcdca4381f342 \ + --hash=sha256:e7a6a8354f1b62e15d48e04350f13e726fa08b62c3d7b8401c0a1314f02e3558 \ + --hash=sha256:e9a3d838441bebcf5cf442700e3963f58b5c33f015341f9ea86dcd7d503c07e2 \ + --hash=sha256:edea7d15772ceeb29db4aff55e482d4bcfb6ae160ce144f2682de02f6d693551 \ + --hash=sha256:f22eb3a6c1080d862befa0a89c380b4dafce29dc6cd56083f630073d102eb595 \ + --hash=sha256:f26383adb94da5e7fb388d441bf09c61e5e35f455a3217bfd790c6b6bc64b2ee \ + --hash=sha256:f3c2890ca8c59ee683fd09adf32321a40fe1cf164e3387799efb2acebf090c11 \ + --hash=sha256:f64fd07515dad67f24b6ea4a66ae2876c01031de91c93075b8093f07c0a2d93d \ + --hash=sha256:fcde4c397f673fdec23e6b05ebf8d4751314fa7c24f93334bf1f1364c1c69ac7 \ + --hash=sha256:ff84aeb864e0fac81f676be9f4685f0527b660f1efdc40dcede3c251ef1e867f +aioitertools==0.11.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394 \ + --hash=sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831 +aiosignal==1.3.1 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ + --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 +attrs==23.2.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ + --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 +awswrangler==3.8.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:5f61ded812ad0c9ee0b0156ae940292d204089e9be5b2c2ba32c6908c0fb498b \ + --hash=sha256:a1d72cff0d8f1ad380b25661775d1bd4616b3966fb9d9e6298d025ec0072014f +beautifulsoup4==4.12.3 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051 \ + --hash=sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed +boto3==1.34.131 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:05e388cb937e82be70bfd7eb0c84cf8011ff35cf582a593873ac21675268683b \ + --hash=sha256:dab8f72a6c4e62b4fd70da09e08a6b2a65ea2115b27dd63737142005776ef216 +botocore==1.34.131 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:13b011d7b206ce00727dcee26548fa3b550db9046d5a0e90ac25a6e6c8fde6ef \ + --hash=sha256:502ddafe1d627fcf1e4c007c86454e5dd011dba7c58bd8e8a5368a79f3e387dc +certifi==2024.6.2 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516 \ + --hash=sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56 +charset-normalizer==3.3.2 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \ + --hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \ + --hash=sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786 \ + --hash=sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8 \ + --hash=sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09 \ + --hash=sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185 \ + --hash=sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574 \ + --hash=sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e \ + --hash=sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519 \ + --hash=sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898 \ + --hash=sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269 \ + --hash=sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3 \ + --hash=sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f \ + --hash=sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6 \ + --hash=sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8 \ + --hash=sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a \ + --hash=sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73 \ + --hash=sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc \ + --hash=sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714 \ + --hash=sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2 \ + --hash=sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc \ + --hash=sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce \ + --hash=sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d \ + --hash=sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e \ + --hash=sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6 \ + --hash=sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269 \ + --hash=sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96 \ + --hash=sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d \ + --hash=sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a \ + --hash=sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4 \ + --hash=sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77 \ + --hash=sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d \ + --hash=sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0 \ + --hash=sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed \ + --hash=sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068 \ + --hash=sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac \ + --hash=sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25 \ + --hash=sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8 \ + --hash=sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab \ + --hash=sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26 \ + --hash=sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2 \ + --hash=sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db \ + --hash=sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f \ + --hash=sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5 \ + --hash=sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99 \ + --hash=sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c \ + --hash=sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d \ + --hash=sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811 \ + --hash=sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa \ + --hash=sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a \ + --hash=sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03 \ + --hash=sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b \ + --hash=sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04 \ + --hash=sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c \ + --hash=sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001 \ + --hash=sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458 \ + --hash=sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389 \ + --hash=sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99 \ + --hash=sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985 \ + --hash=sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537 \ + --hash=sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238 \ + --hash=sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f \ + --hash=sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d \ + --hash=sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796 \ + --hash=sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a \ + --hash=sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143 \ + --hash=sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8 \ + --hash=sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c \ + --hash=sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5 \ + --hash=sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5 \ + --hash=sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711 \ + --hash=sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4 \ + --hash=sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6 \ + --hash=sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c \ + --hash=sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7 \ + --hash=sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4 \ + --hash=sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b \ + --hash=sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae \ + --hash=sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12 \ + --hash=sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c \ + --hash=sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae \ + --hash=sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8 \ + --hash=sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887 \ + --hash=sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b \ + --hash=sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4 \ + --hash=sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f \ + --hash=sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5 \ + --hash=sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33 \ + --hash=sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519 \ + --hash=sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561 +cramjam==2.8.3 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:00524bb23f4abb3a3bfff08aa32b9274843170c5b43855807e0f59670e2ac98c \ + --hash=sha256:007aa9444cb27b8691baae73ca907133cd939987438f874774011b4c740732dd \ + --hash=sha256:04b31d427a8902e5c2eec4b8f29873de7a3ade202e3d68e7f2354b9f0aa00bc7 \ + --hash=sha256:07af94191f6a245226dc8a8bc6c94808e382ce9dfcca4bab0e8015fbc7fc3322 \ + --hash=sha256:080f3eb7b648f5ba9d35084d8dddc68246a8f365df239792f6712908f0aa568e \ + --hash=sha256:11c9d30bc53892c57a3b296756c23659323ab1419a2b4bf22bbafc07b247bb67 \ + --hash=sha256:11d2e9eebc7d202eda0ae09fb56a2cdbeb5a1563e89d2118bf18cf0030f35f77 \ + --hash=sha256:1374fe9a4431e546bb4501a16b84875d0bf80fc4e6c8942f0d5608ae48474267 \ + --hash=sha256:19b8c97350c8d65daea26267dd1becb59073569aac2ae5743952d7f48da5d37a \ + --hash=sha256:1ba1a8ff855b30b4069a9b45ea9e7f2b5d882c7953bdfccda8d4b275fa7057ce \ + --hash=sha256:1fd25201f1278dc6faa2ae35e67b7a5bb352b7fc6ed1ee939637414ca8115863 \ + --hash=sha256:246f1f7d32cac2b64617d2dddba11a82851e73cdcf9d1abb799b08dcd9d2ea49 \ + --hash=sha256:2476828dea4089aa3cb9160391f8b36f793ca651afdcba80de1e341373928397 \ + --hash=sha256:24990be4010b2185dcecc67133cd727657036e7b132d7de598148f5b1eb8e452 \ + --hash=sha256:24c2b426dd8fafb894f93a88f42e2827e14199d66836cb100582037e5371c724 \ + --hash=sha256:269f94d2efe6b6a97624782cd3b541e60535dd5874f4a8d5d0ba66ef59424ae3 \ + --hash=sha256:274878883e7fadf95a6b5bc58f9c1dd39fef2c31d68e18a0fb8594226457fba7 \ + --hash=sha256:28c30078effc100739d3f9b227276a8360c1b32aac65efb4f641630552213548 \ + --hash=sha256:29987b54e31efed66738e8f236c597c4c9a91ec9d57bcb74307712e07505b4bb \ + --hash=sha256:2be92c6f0bcffaf8ea6a8164fe0388a188fec2fa9eff1828e8b64dc3a83740f9 \ + --hash=sha256:2cb64a97e625ca029b55e37769b8c354e64cbea042c75471915dc385935d30ed \ + --hash=sha256:30e2d745cd4d244b7973d15aaebeedb537b980f9d3da80e6dea75ee1a872f9fa \ + --hash=sha256:3277fd42399755d6d3730edec4a192174ee64d219e0ffbc90613f15cbabf711f \ + --hash=sha256:345a952c5d4b922830efaa67dc0b42d21e18c182c1a1bda6d20bb78235f31d6f \ + --hash=sha256:35647a0e37a4dfec85a44c7966ae476b7db0e6cd65d91c08f1fb3007ed774d92 \ + --hash=sha256:3850dac9a2f6dcb3249d23f9d505117643b967bdc1c572ed0cc492a48fd69daf \ + --hash=sha256:3f6303c8cc583dfe5054cf84717674f75b18bca4ae8e576dc863958d5494dc4b \ + --hash=sha256:440a18fd4ae42e06dbbd7aee91d8248b61da9fef7610ffbd553d1ba93931394b \ + --hash=sha256:476890974229713fc7b4c16fb050b756ba926c67e4d1200b3e03c5c051e9b552 \ + --hash=sha256:4822eb5fe6839cd3d0439e5431e766ad010b2a388ca9617aa6372b6030897782 \ + --hash=sha256:4a554bcfd068e831affd64a4f067c7c9b00b359742597c4fdadd18ff673baf30 \ + --hash=sha256:4c1d2d39c2193a77c5e5b327944f90e6ecf2caa1b55e7176cc83d80706ea15de \ + --hash=sha256:4f7c16d358df366e308137411125a2bb50d1b19924fced3a390898fa8c9a074d \ + --hash=sha256:5023a737d8d9cf5d123e6d87d088929c3cfb2aae90e0f584204427f74882150a \ + --hash=sha256:51e847dcfe74fba379fed2bc2b45f5c2f11c3ece5e9eebcf63f39a9594184588 \ + --hash=sha256:572cb9a8dc5a189691d6e03a9bf9b4305fd9a9f36bb0f9fde55fc36837c2e6b3 \ + --hash=sha256:594477faff7f4380fa123cfbcf10ab8ee5af1a28b95750b66931ffafcb11ab5c \ + --hash=sha256:5d1ac94e00c64258330105473c641441db02b4dc3e9e9f2963d204e53ed93025 \ + --hash=sha256:6379b92912f7569e126bd48d10e7087ddd20ea88a939532e3c4a85c2fa05d600 \ + --hash=sha256:65bfd41aa92c0025f32ba09214b48e9367a81122586b2617439b4327c4bd179c \ + --hash=sha256:6653c262ad71e6c0ae08eeca3af2ee89ad47483b6312f2c6094518cb77872406 \ + --hash=sha256:6721edd8f911ad84db83ee4902b7579fc01c55849062f3f1f4171b58fccf98eb \ + --hash=sha256:67e09b42e744efd08b93ac56f6100a859a31617d7146725516f3f2c744149d97 \ + --hash=sha256:6b1fa0a6ea8183831d04572597c182bd6cece62d583a36cde1e6a86e72ce2389 \ + --hash=sha256:6c04f363cb4b316719421724521432b6e7f6490e5baaaf7692af961c28d0279b \ + --hash=sha256:6eec7e985f35708c234542721863d82781d0f7f6a71b45e14ce6d2625d4b131d \ + --hash=sha256:6f838d06d06709b9ce8b1ceae36aea4e1c7e613365185a91edcbeb5884f5e606 \ + --hash=sha256:7337bd8218bd8508f35904274a38cce843a237fe6e23104238bbeb2f337107ed \ + --hash=sha256:771b44e549f90b5532508782e25d1c40b8054dd83d52253d05945fc05836b252 \ + --hash=sha256:77346ac669f5445d14b74476a4e8f3a259fd22681bd73790e92b8956d7e225fc \ + --hash=sha256:7871e1fd3ee8ca16799ba22d49fc1e52e78976fa8c659be41630eeb2914475a7 \ + --hash=sha256:80b088d15866b37851fd53e2b471becc9ec487257dceca1878621072a18e833e \ + --hash=sha256:832224f52fa1e601e0ab678dba9bdfde3686fc4cd1a9f2ed4748f29eaf1cb553 \ + --hash=sha256:86e238b6de79e045f5197df2c9dfaf8d10b37a6517ff4ffc4775fe5a3cf4d4a4 \ + --hash=sha256:87e26e3e1d5fed1cac5b41be648d0daf0793f94cf4a7aebefce1f4f6656e2d21 \ + --hash=sha256:8c8aa6d08c135ae7f0da01e6559a332c5d8fe4989a594db401040e385d04dffd \ + --hash=sha256:8ea1dc11538842ff20d9872a17214994f5913cbf3be5594b54aad2422becdf19 \ + --hash=sha256:8ed658f36a2bf667d5b8c7c6690103ad99f81cc62a1b64891b69298447329d4b \ + --hash=sha256:962b7106287bcc463150766b5b8c69f32dcc69713a8dbce00e0ca6936f95c55b \ + --hash=sha256:9728861bc0390681824961778b36f7f0b95039e8b90d46f1b67f51232f1ee159 \ + --hash=sha256:97c706c520c3f8b0184278cc86187528458350216c6e4fa85d3f16bcad0d365d \ + --hash=sha256:990e65c2bf1c155a9ddec5ecabf431cf77596432f697d3c6e0831b5174c51c40 \ + --hash=sha256:9a1943f2cc0deee037ddcf92beff6049e12d4e6d557f568ddf59fb3b848f2152 \ + --hash=sha256:9d08f1bab949ffd6dd6f25a89e4f7062d147aeea9c067e4dd155bdb190e5a519 \ + --hash=sha256:9efe6915aa7ef176f3a7f42a4e46504573215953331b139abefd20d07d8aba82 \ + --hash=sha256:a2fededed05a042f093dbf1b11d69afb1874a2c9197fcf1d58c142ba9111db5a \ + --hash=sha256:ab67f29094165f0771acad8dd16e840259cfedcc94067af229530496dbf1a24c \ + --hash=sha256:ab8146faa5d8c52edf23724843c36469fc32ff2c4a174eba72f4da6de5016688 \ + --hash=sha256:afa065bab70e27565695441f69f493af3d379b8723030f2c3d2547d2e312a4be \ + --hash=sha256:b188e750b95172c01defcfcfbba629cad797718b34402ec61b3bc9ff99403599 \ + --hash=sha256:ba1e45074757ab0482ac544e60613b6b8658100ac9985c91868a4598cdfb63ba \ + --hash=sha256:ba542f07fe3f41475d78626973533539e6cf2d5b6af37923fe6c7e7f0f74b9b2 \ + --hash=sha256:bc8f24c32124bb47536882c6b941cdb88cc16e4fa64d5bf347cb8dd72a193fc3 \ + --hash=sha256:bd8c601fe8717e52517a2f2eef78217086acf449627bfdda97e3f53fd79c92af \ + --hash=sha256:be6fb5dd5bf1c89c717a73a1057505959f35c08e0e97a76d4cc6391b90d2263b \ + --hash=sha256:bec9ca5431c32ba94996b7c1c56695b37d48713b97ee1d2a456f4046f009e82f \ + --hash=sha256:c14728e3360cd212d5b606ca703c3bd1c8912efcdbc1aa032c81c2882509ebd5 \ + --hash=sha256:c28830ecf76501356d678dac4f37563554ec1c651a53a990cdf595f7ed75c651 \ + --hash=sha256:c7e8329cde48740df8d332dade2f52b74612b8ea86005341c99bb192c82a5ce7 \ + --hash=sha256:c9d54a4aa475d5e902f2ee518bdaa02f26c089e9f72950d00d1643c090f0deb3 \ + --hash=sha256:cb5f4d061e9abdc6663551446c332a58c101efb31fd1746229872600274c2b20 \ + --hash=sha256:d5a0a2fe240c97587df07f3d5e1027673d599b3a6a7a0ab540aea69f09e9ff7a \ + --hash=sha256:d824fd98364bc946c38ed324a3ec7befba055285aaf2c1ca61894bb7616226e8 \ + --hash=sha256:d93b42d22bf3e17290c5e4cf58e715a419330bb5255c35933c14db82ecf3872c \ + --hash=sha256:d9b244d04cef82872d12c227a2f202f080a454d664c05db351626e6ad4aaa307 \ + --hash=sha256:dac42b2b4c3950e7eda9b5551e0e904784ed0c0428accc29171c230fb919ec72 \ + --hash=sha256:dcf7791e1cedb982ccc873ec9392c6cfb9c714a64ebf1ed4e8310b9cb44655f2 \ + --hash=sha256:e23e323ad28ed3e4e3a24ceffdab0ff235954109a88b536ea7b3b7886bd0a536 \ + --hash=sha256:e30f1f00de913b440baa36647817b9b7120a69b04eca05f3354aaf5b40f95ee5 \ + --hash=sha256:e8ec1d4f27eb9d0412f0c567e7ffd14fbeb2b318a1ac394d5de4047c431fe94c \ + --hash=sha256:e954599c6369f429a868852eff453b894d88866acba439b65131ea93f5400b47 \ + --hash=sha256:eea606b01b43b91626e3aafd463bd19b6ed739bdb8b2b309e5d7ff72afc0e89d \ + --hash=sha256:ef0173fb457f73cf9c2553092419db0eba4d582890db95e542a4d93e11340421 \ + --hash=sha256:ef29fb916fe74be65d0ab8871ab8d964b0f5eb8028bb84b325be43675a59d6e7 \ + --hash=sha256:f667843e7a8fca208eecfe44e04088242f8ca60d74d4950fac3722043538d700 \ + --hash=sha256:fb5d7739e2bc573ade12327ef7717b1ac5876c62938fab20eb54d762da23cae2 \ + --hash=sha256:fc0c6eb8185c68f79a25bb298825e345cc09b826f5828bd8146e3600ca6e9981 \ + --hash=sha256:fc9c45469914099897c47bfc501616fb377f28a865adebf90ea6f3c8ae6dd4e6 \ + --hash=sha256:fe6434d3ee0899bc9396801d1abbc5d1fe77662bd3d1f1c1573fac6708459138 \ + --hash=sha256:fe84440100e7045190da7f80219be9989b0b6db6acadb3ae9cfe0935d93ebf8c +fastparquet==2024.5.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:0034d1b5af3a71cc2fb29c590f442c0b514f710d6d6996794ae375dcfe050c05 \ + --hash=sha256:07fc5a45450a39cd07c6ef0e0219ac4b1879f8b27c825ee4ba5d87a3ae505f11 \ + --hash=sha256:08358d99278c5d3fb523d819fff5c74d572d8f67ebbe2215a2c7bfca7e3664cf \ + --hash=sha256:1496d83d7a77c19abae796e3b582539884fc893d75a3ad4f90df12f8f23a902a \ + --hash=sha256:2a951106782d51e5ab110beaad29c4aa0537f045711bb0bf146f65aeaed14174 \ + --hash=sha256:2b3cf7b4eb1b06e87b97a3a5c9124e4b1c08a8903ba017052c5fe2c482414a3d \ + --hash=sha256:3a60f7b0b308d6b9f12c642cf5237a05d754926fb31ce865ff7072bceab19fbb \ + --hash=sha256:42149929b71d9122bd501aa695681f40a04a9fa3f5b802cf0fb6aa4e95ccf2dd \ + --hash=sha256:47695037fdc534ef4247f25ccf17dcbd8825be6ecb70c54ca54d588a794f4a6d \ + --hash=sha256:4a2045c21f90358541286f26f0735bfb2265b075413fbced3b876fc8848eda52 \ + --hash=sha256:4e6ac308a2f391ce589c99b8376e7cdfe4241ef5770ac4cf4c1c93f940bda83c \ + --hash=sha256:5626fc72204001b7e82fedb4b02174ecb4e2d4143b38b4ea8d2f9eb65f6b000e \ + --hash=sha256:56d03b0a291d6a575ab365516c53b4da8e040347f8d43af79be25893c591b38c \ + --hash=sha256:611da9043f9dab1c63e6c90a6b124e3d2789c34fefa00d45356517f1e8a09c83 \ + --hash=sha256:691348cc85890663dd3c0bb02544d38d4c07a0c3d68837324dc01007301150b5 \ + --hash=sha256:784989ee2c251960b8f00dc38c6c730f784712c8e3d08cc7e0ce842055476af1 \ + --hash=sha256:9dfbed87b4b58b0794b2cb3aa4abcb43fc01480a10c7779a323d2dd1599f6acd \ + --hash=sha256:b562be0f43a007493014512602ab6b0207d13ea4ae85e0d94d61febf08efa1ee \ + --hash=sha256:ba251231b005c0f3f7e56f6e9cd1939be99b2d810ab5b05039271e260c0196c6 \ + --hash=sha256:c8b2e86fe6488cce0e3d41263bb0296ef9bbb875a2fca09d67d7685640017a66 \ + --hash=sha256:cb93e8951f46943c8567c9a555cb3d24d2c78efdf78e95fd72177d80da73a10f \ + --hash=sha256:cc99d7c0f1816394d53aadd47919bba70bb81355259d8788d28e35913816aee0 \ + --hash=sha256:d20bba5c39139a88d8d6931764b830ba14042742d802238d9edf86d4d765ad7a \ + --hash=sha256:dfdc8aaec67edd30814c2c2f0e291eb3c3044525d18c87e835ef8793d6e2ea2d \ + --hash=sha256:dffd1d0ac6e89e31c5b6dacf67a8d299d4afbbcf0bf8b797373904c819c48f51 \ + --hash=sha256:e1fa068ef1826bff6d4a9106a6f9e9d6fd20b8b516da4b82d87840cb5fd3947c \ + --hash=sha256:e5b1ed889f4ac7ea059ff95f4a01f5c07c825c50c2e1bc9e2b64c814df94c243 \ + --hash=sha256:e9de270e17a6ae2f02c716421d60e18d35d4718037f561b3e359989db19f700a \ + --hash=sha256:ea3796c4a38ef8b372a3056b5cef52ca8182fa554fa51c7637c2421e69ee56e5 \ + --hash=sha256:f411056152b5d3cc82b6624d9da80535d10d9277d921fdb2e9516e93c8c227e8 \ + --hash=sha256:f5c3cabcfa2f534e4b23343c1ab84c37d336da73770005e608d1894ab1084600 \ + --hash=sha256:fc3d35ff8341cd65baecac71062e9d73393d7afda207b3421709c1d3f4baa194 +frozenlist==1.4.1 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7 \ + --hash=sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98 \ + --hash=sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad \ + --hash=sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5 \ + --hash=sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae \ + --hash=sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e \ + --hash=sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a \ + --hash=sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701 \ + --hash=sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d \ + --hash=sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6 \ + --hash=sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6 \ + --hash=sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106 \ + --hash=sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75 \ + --hash=sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868 \ + --hash=sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a \ + --hash=sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0 \ + --hash=sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1 \ + --hash=sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826 \ + --hash=sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec \ + --hash=sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6 \ + --hash=sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950 \ + --hash=sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19 \ + --hash=sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0 \ + --hash=sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8 \ + --hash=sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a \ + --hash=sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09 \ + --hash=sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86 \ + --hash=sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c \ + --hash=sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5 \ + --hash=sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b \ + --hash=sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b \ + --hash=sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d \ + --hash=sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0 \ + --hash=sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea \ + --hash=sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776 \ + --hash=sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a \ + --hash=sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897 \ + --hash=sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7 \ + --hash=sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09 \ + --hash=sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9 \ + --hash=sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe \ + --hash=sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd \ + --hash=sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742 \ + --hash=sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09 \ + --hash=sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0 \ + --hash=sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932 \ + --hash=sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1 \ + --hash=sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a \ + --hash=sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49 \ + --hash=sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d \ + --hash=sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7 \ + --hash=sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480 \ + --hash=sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89 \ + --hash=sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e \ + --hash=sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b \ + --hash=sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82 \ + --hash=sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb \ + --hash=sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068 \ + --hash=sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8 \ + --hash=sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b \ + --hash=sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb \ + --hash=sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2 \ + --hash=sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11 \ + --hash=sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b \ + --hash=sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc \ + --hash=sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0 \ + --hash=sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497 \ + --hash=sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17 \ + --hash=sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0 \ + --hash=sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2 \ + --hash=sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439 \ + --hash=sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5 \ + --hash=sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac \ + --hash=sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825 \ + --hash=sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887 \ + --hash=sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced \ + --hash=sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74 +fsspec==2024.6.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:58d7122eb8a1a46f7f13453187bfea4972d66bf01618d37366521b1998034cee \ + --hash=sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2 +idna==3.7 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \ + --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0 +jmespath==1.0.1 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ + --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe +multidict==6.0.5 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556 \ + --hash=sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c \ + --hash=sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29 \ + --hash=sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b \ + --hash=sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8 \ + --hash=sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7 \ + --hash=sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd \ + --hash=sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40 \ + --hash=sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6 \ + --hash=sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3 \ + --hash=sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c \ + --hash=sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9 \ + --hash=sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5 \ + --hash=sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae \ + --hash=sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442 \ + --hash=sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9 \ + --hash=sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc \ + --hash=sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c \ + --hash=sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea \ + --hash=sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5 \ + --hash=sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50 \ + --hash=sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182 \ + --hash=sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453 \ + --hash=sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e \ + --hash=sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600 \ + --hash=sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733 \ + --hash=sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda \ + --hash=sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241 \ + --hash=sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461 \ + --hash=sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e \ + --hash=sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e \ + --hash=sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b \ + --hash=sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e \ + --hash=sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7 \ + --hash=sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386 \ + --hash=sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd \ + --hash=sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9 \ + --hash=sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf \ + --hash=sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee \ + --hash=sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5 \ + --hash=sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a \ + --hash=sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271 \ + --hash=sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54 \ + --hash=sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4 \ + --hash=sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496 \ + --hash=sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb \ + --hash=sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319 \ + --hash=sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3 \ + --hash=sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f \ + --hash=sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527 \ + --hash=sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed \ + --hash=sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604 \ + --hash=sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef \ + --hash=sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8 \ + --hash=sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5 \ + --hash=sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5 \ + --hash=sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626 \ + --hash=sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c \ + --hash=sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d \ + --hash=sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c \ + --hash=sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc \ + --hash=sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc \ + --hash=sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b \ + --hash=sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38 \ + --hash=sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450 \ + --hash=sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1 \ + --hash=sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f \ + --hash=sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3 \ + --hash=sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755 \ + --hash=sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226 \ + --hash=sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a \ + --hash=sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046 \ + --hash=sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf \ + --hash=sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479 \ + --hash=sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e \ + --hash=sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1 \ + --hash=sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a \ + --hash=sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83 \ + --hash=sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929 \ + --hash=sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93 \ + --hash=sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a \ + --hash=sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c \ + --hash=sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44 \ + --hash=sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89 \ + --hash=sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba \ + --hash=sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e \ + --hash=sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da \ + --hash=sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24 \ + --hash=sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423 \ + --hash=sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef +mwparserfromhell==0.6.6 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:007d0859e5467241b73c6e974df039a074609ce4e2b9df8c2263a8920554d032 \ + --hash=sha256:03e03b8bec729af850457d045b04d0c9d3e296ff8bf66b455f754cccb29c3bea \ + --hash=sha256:063c1e79befd1f55d77c358e0f5006f5ecf88ddf218ff6af55188d686139330e \ + --hash=sha256:1915fe4f5e5ae34f16242d4cd98da2adc81a810ab94105ec2af3dc95d7ce74aa \ + --hash=sha256:1960bcc5115ea57427df130150edf1dbfc2fb03465e548e630bb6eb37976d793 \ + --hash=sha256:19e9a4bcd85707c83172405eb2a9a046eff9d38dd7f1a56a5e5ecbbfef4a640a \ + --hash=sha256:1d2422659abb29191a0fa096cf8bead837ac3ecd343065569b2acc7a84ecf866 \ + --hash=sha256:2b75fae6d01c8fda19dbf127175122d7aa2964ef6454690e6868bbc3d80a7bc1 \ + --hash=sha256:50c482e703d2d51401f7e36a71ae9493901f170225940196292f97398713dde5 \ + --hash=sha256:54e2dd30edc1a358408d14343b30dcca0b4613227781e4bbee968bd4395d94ff \ + --hash=sha256:59633d3cc09993af75ced8dfbd6800e1e38e64620851a095575621548448875c \ + --hash=sha256:6a89edf53f15877223d923e122e9a97f3f7b85f56dc56d91a3d77b89c9dd4126 \ + --hash=sha256:6b11dea3bcdebe4554933169eade815e9d6b898175faa5a20a744524fd99210f \ + --hash=sha256:71afec1e9784ba576e95d6f34845582d3c733a3a52ba770dd8a9c3a40e5b649f \ + --hash=sha256:746bad799179684994ecee72a26352e0bbe2b697f6a7e35dc5ad151606bcb8ab \ + --hash=sha256:910d36bc70e8bea758380e75c12fd47626b295abec9f73a6099d8f937a649e77 \ + --hash=sha256:9136696d6b29838adcf8f428e3f7028b2c6e788fc05fe1beeb4b135429c356df \ + --hash=sha256:a58251a5d5c77abdfd061624dc05667c2774e93e8178a2fbd1a3b45f8673f1a9 \ + --hash=sha256:cdc46c115b2495d4025920b7b30a6885a96d2b797ccc4009bf3cc02940ae55d3 \ + --hash=sha256:d2febd92a55a3f19b461833267726cb81429c3d6cb0006ad1691dfa849789e5d \ + --hash=sha256:d6995b9cfe6ec79556db0232a39210ac11aa69ee304cfc95b29c51be381e202b \ + --hash=sha256:dbe5976b1b524e26aa2eb71b6219960f2578f56b536c68e0a79deb63e3b7f710 \ + --hash=sha256:e28ffa9a7e0748ec64002a84234201ef69c2d4a710508baf9cc25f4ee274c6bd \ + --hash=sha256:ebc70f8a24aa60e54728be740f1c12a4acb1b12d1cc947d87b067cc1c83339fd \ + --hash=sha256:fd05481adc0806f4b8f8f8cb309ec56924b17ce386cb1c2f73919d8a012e6b16 \ + --hash=sha256:fff66e97f7c02aa0fd57ff8f702977a9c5a1d72ef55b64ee9b146291e4c41057 +numpy==1.26.4 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ + --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ + --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ + --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ + --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ + --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ + --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ + --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ + --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ + --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ + --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ + --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ + --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ + --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ + --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ + --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ + --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ + --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ + --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ + --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ + --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ + --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ + --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ + --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ + --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ + --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ + --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ + --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ + --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ + --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ + --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ + --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ + --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ + --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ + --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ + --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f +packaging==24.1 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ + --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 +pandas==2.2.2 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863 \ + --hash=sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2 \ + --hash=sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1 \ + --hash=sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad \ + --hash=sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db \ + --hash=sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76 \ + --hash=sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51 \ + --hash=sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32 \ + --hash=sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08 \ + --hash=sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b \ + --hash=sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4 \ + --hash=sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921 \ + --hash=sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288 \ + --hash=sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee \ + --hash=sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0 \ + --hash=sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24 \ + --hash=sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99 \ + --hash=sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151 \ + --hash=sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd \ + --hash=sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce \ + --hash=sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57 \ + --hash=sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef \ + --hash=sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54 \ + --hash=sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a \ + --hash=sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238 \ + --hash=sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23 \ + --hash=sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772 \ + --hash=sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce \ + --hash=sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad +pyarrow==16.1.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a \ + --hash=sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2 \ + --hash=sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f \ + --hash=sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2 \ + --hash=sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315 \ + --hash=sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9 \ + --hash=sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b \ + --hash=sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55 \ + --hash=sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15 \ + --hash=sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e \ + --hash=sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f \ + --hash=sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c \ + --hash=sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a \ + --hash=sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa \ + --hash=sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a \ + --hash=sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd \ + --hash=sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628 \ + --hash=sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef \ + --hash=sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e \ + --hash=sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff \ + --hash=sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b \ + --hash=sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c \ + --hash=sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c \ + --hash=sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f \ + --hash=sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3 \ + --hash=sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6 \ + --hash=sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c \ + --hash=sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147 \ + --hash=sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5 \ + --hash=sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7 \ + --hash=sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710 \ + --hash=sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4 \ + --hash=sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed \ + --hash=sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848 \ + --hash=sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83 \ + --hash=sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444 +python-dateutil==2.9.0.post0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 +python-magic==0.4.27 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b \ + --hash=sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3 +pytz==2024.1 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812 \ + --hash=sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319 +pywikibot==9.2.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:3df116778b108207a9e03fdfbdea71eb2f41144fcc9247d8797a4b7a8417aa18 \ + --hash=sha256:fd659c3111eebfb3704cddc4e2dcb7afac15f418a8c4cdd4ef680d7658237e60 +requests==2.32.3 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 +s3fs==2024.6.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:8d5f591956a61c7d64097eff4847598826f09d60b4ce9a16202565693569f6d4 \ + --hash=sha256:a59020ededc61e9666f1e473ce4aa28764e5f7b3c97414beb15cd9be522a87b6 +s3transfer==0.10.1 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19 \ + --hash=sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d +setuptools==70.1.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5 \ + --hash=sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267 +six==1.16.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 +soupsieve==2.5 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690 \ + --hash=sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7 +typing-extensions==4.12.2 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ + --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 +tzdata==2024.1 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd \ + --hash=sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252 +urllib3==2.2.2 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472 \ + --hash=sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168 +validators==0.28.3 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:53cafa854f13850156259d9cc479b864ee901f6a96e6b109e6fc33f98f37d99f \ + --hash=sha256:c6c79840bcde9ba77b19f6218f7738188115e27830cbaff43264bc4ed24c429d +wrapt==1.16.0 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc \ + --hash=sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81 \ + --hash=sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09 \ + --hash=sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e \ + --hash=sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca \ + --hash=sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0 \ + --hash=sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb \ + --hash=sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487 \ + --hash=sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40 \ + --hash=sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c \ + --hash=sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060 \ + --hash=sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202 \ + --hash=sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41 \ + --hash=sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9 \ + --hash=sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b \ + --hash=sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664 \ + --hash=sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d \ + --hash=sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362 \ + --hash=sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00 \ + --hash=sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc \ + --hash=sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1 \ + --hash=sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267 \ + --hash=sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956 \ + --hash=sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966 \ + --hash=sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1 \ + --hash=sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228 \ + --hash=sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72 \ + --hash=sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d \ + --hash=sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292 \ + --hash=sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0 \ + --hash=sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0 \ + --hash=sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36 \ + --hash=sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c \ + --hash=sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5 \ + --hash=sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f \ + --hash=sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73 \ + --hash=sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b \ + --hash=sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2 \ + --hash=sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593 \ + --hash=sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39 \ + --hash=sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389 \ + --hash=sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf \ + --hash=sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf \ + --hash=sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89 \ + --hash=sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c \ + --hash=sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c \ + --hash=sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f \ + --hash=sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440 \ + --hash=sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465 \ + --hash=sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136 \ + --hash=sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b \ + --hash=sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8 \ + --hash=sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3 \ + --hash=sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8 \ + --hash=sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6 \ + --hash=sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e \ + --hash=sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f \ + --hash=sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c \ + --hash=sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e \ + --hash=sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8 \ + --hash=sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2 \ + --hash=sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020 \ + --hash=sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35 \ + --hash=sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d \ + --hash=sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3 \ + --hash=sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537 \ + --hash=sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809 \ + --hash=sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d \ + --hash=sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a \ + --hash=sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4 +yarl==1.9.4 ; python_version >= "3.12" and python_version < "4.0" \ + --hash=sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51 \ + --hash=sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce \ + --hash=sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559 \ + --hash=sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0 \ + --hash=sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81 \ + --hash=sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc \ + --hash=sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4 \ + --hash=sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c \ + --hash=sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130 \ + --hash=sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136 \ + --hash=sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e \ + --hash=sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec \ + --hash=sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7 \ + --hash=sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1 \ + --hash=sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455 \ + --hash=sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099 \ + --hash=sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129 \ + --hash=sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10 \ + --hash=sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142 \ + --hash=sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98 \ + --hash=sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa \ + --hash=sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7 \ + --hash=sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525 \ + --hash=sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c \ + --hash=sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9 \ + --hash=sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c \ + --hash=sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8 \ + --hash=sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b \ + --hash=sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf \ + --hash=sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23 \ + --hash=sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd \ + --hash=sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27 \ + --hash=sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f \ + --hash=sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece \ + --hash=sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434 \ + --hash=sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec \ + --hash=sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff \ + --hash=sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78 \ + --hash=sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d \ + --hash=sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863 \ + --hash=sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53 \ + --hash=sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31 \ + --hash=sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15 \ + --hash=sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5 \ + --hash=sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b \ + --hash=sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57 \ + --hash=sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3 \ + --hash=sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1 \ + --hash=sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f \ + --hash=sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad \ + --hash=sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c \ + --hash=sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7 \ + --hash=sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2 \ + --hash=sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b \ + --hash=sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2 \ + --hash=sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b \ + --hash=sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9 \ + --hash=sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be \ + --hash=sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e \ + --hash=sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984 \ + --hash=sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4 \ + --hash=sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074 \ + --hash=sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2 \ + --hash=sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392 \ + --hash=sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91 \ + --hash=sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541 \ + --hash=sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf \ + --hash=sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572 \ + --hash=sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66 \ + --hash=sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575 \ + --hash=sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14 \ + --hash=sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5 \ + --hash=sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1 \ + --hash=sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e \ + --hash=sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551 \ + --hash=sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17 \ + --hash=sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead \ + --hash=sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0 \ + --hash=sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe \ + --hash=sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234 \ + --hash=sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0 \ + --hash=sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7 \ + --hash=sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34 \ + --hash=sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42 \ + --hash=sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385 \ + --hash=sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78 \ + --hash=sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be \ + --hash=sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958 \ + --hash=sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749 \ + --hash=sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec diff --git a/sdc-sync.py b/sdc-sync.py index d6f60d1..b7e04fa 100644 --- a/sdc-sync.py +++ b/sdc-sync.py @@ -7,229 +7,324 @@ # DPLA API key -key = 'XXX' +key = "XXX" # When running manually, sometimes it is helpful to specify the category to work on in the command line, using --cat "". parser = argparse.ArgumentParser() -parser.add_argument('--cat', dest='cat', metavar='CAT', - action='store') -parser.add_argument('--method', dest='method', metavar='METHOD', - action='store') -parser.add_argument('--lists', dest='lists', metavar='LISTS', - action='store') +parser.add_argument("--cat", dest="cat", metavar="CAT", action="store") +parser.add_argument("--method", dest="method", metavar="METHOD", action="store") +parser.add_argument("--lists", dest="lists", metavar="LISTS", action="store") args = parser.parse_args() -method = 'livecat' +method = "livecat" if args.method: method = args.method -hubs = json.loads(requests.get('https://raw.githubusercontent.com/dpla/ingestion3/develop/src/main/resources/wiki/institutions_v2.json').text) +hubs = json.loads( + requests.get( + "https://raw.githubusercontent.com/dpla/ingestion3/develop/src/main/resources/wiki/institutions_v2.json" + ).text +) -rights = json.load(open('rights.json')) -subject_ids = json.loads(requests.get('https://raw.githubusercontent.com/DominicBM/ingestion3/develop/src/main/resources/subjects.json').text) +rights = json.load(open("rights.json")) +subject_ids = json.loads( + requests.get( + "https://raw.githubusercontent.com/DominicBM/ingestion3/develop/src/main/resources/subjects.json" + ).text +) # This is the JSON used for formatting a claim. The P459 -> Q61848113 (determination method) qualifier is hardcoded in for everything DPLA adds. Not all data types have the same format for value, so this is formatted in the function for each property added. + def formattedclaim(prop, value, type, dpla_id): claim = { "mainsnak": { "snaktype": "value", "property": prop, - "datavalue": { - "value": value, - "type": type - } + "datavalue": {"value": value, "type": type}, }, "type": "statement", - "rank":"normal", + "rank": "normal", "qualifiers": { "P459": [ { "snaktype": "value", "property": "P459", "datavalue": { - "value": { - "entity-type": "item", - "numeric-id": 61848113 - }, - "type": "wikibase-entityid" + "value": {"entity-type": "item", "numeric-id": 61848113}, + "type": "wikibase-entityid", }, - "datatype": "wikibase-item" + "datatype": "wikibase-item", } ] }, - "references": [{ - "snaks": { - "P854": [ - { - "snaktype": "value", - "property": "P854", - "datavalue": { - "value": "https://dp.la/item/" + dpla_id, - "type": "string" + "references": [ + { + "snaks": { + "P854": [ + { + "snaktype": "value", + "property": "P854", + "datavalue": { + "value": "https://dp.la/item/" + dpla_id, + "type": "string", + }, } - } - ], - "P123": [ - { - "snaktype": "value", - "property": "P123", - "datavalue": { - "value": { - "entity-type": "item", - "numeric-id": 2944483, + ], + "P123": [ + { + "snaktype": "value", + "property": "P123", + "datavalue": { + "value": { + "entity-type": "item", + "numeric-id": 2944483, + }, + "type": "wikibase-entityid", }, - "type": "wikibase-entityid" } - } - ], - "P813": [ - { - "snaktype": "value", - "property": "P813", - "datavalue": { - "value": { - "time": "+" + str(datetime.date.today()) + "T00:00:00Z", - "timezone": 0, - "before": 0, - "after": 0, - "precision": 11, - "calendarmodel": "http://www.wikidata.org/entity/Q1985727" + ], + "P813": [ + { + "snaktype": "value", + "property": "P813", + "datavalue": { + "value": { + "time": "+" + + str(datetime.date.today()) + + "T00:00:00Z", + "timezone": 0, + "before": 0, + "after": 0, + "precision": 11, + "calendarmodel": "http://www.wikidata.org/entity/Q1985727", + }, + "type": "time", }, - "type": "time" } - } - ] + ], + } } - }] + ], } - if value == 'somevalue': - claim['mainsnak'].pop('datavalue') - claim['mainsnak']['snaktype'] = 'somevalue' + if value == "somevalue": + claim["mainsnak"].pop("datavalue") + claim["mainsnak"]["snaktype"] = "somevalue" return claim + # This is the function that will perform the POST to the Wikimedia Commons API, when passed all necessary parameters, to add a qualifier if it is missing for anexisting claim. Currently, this is only used for P459. It creates a JSON object to send in the body of the request with the data to post and the login token. -def postqual(claimid, prop, value): - summary = u'Adding [[:d:Property:' + prop + ']] to ' + claimid + '.' +def postqual(claimid, prop, value): - postdata = {u'action' : u'wbsetqualifier', - u'format' : u'json', - u'claim' : claimid, - u'property' : prop, - u'snaktype' : u'value', - u'value' : value, - u'token' : token, - u'bot' : True, - } + summary = "Adding [[:d:Property:" + prop + "]] to " + claimid + "." + + postdata = { + "action": "wbsetqualifier", + "format": "json", + "claim": claimid, + "property": prop, + "snaktype": "value", + "value": value, + "token": token, + "bot": True, + } try: - qual = json.loads(http.fetch(u'https://commons.wikimedia.org/w/api.php', method='POST', data=postdata).text) + qual = json.loads( + http.fetch( + "https://commons.wikimedia.org/w/api.php", method="POST", data=postdata + ).text + ) pywikibot.output(summary) except Exception as e: print(repr(e)) # print(qual) - print(site.tokens['csrf']) - site.get_tokens(['csrf']) - site.tokens.load_tokens(['csrf']) - print(site.tokens['csrf']) + print(site.tokens["csrf"]) + site.get_tokens(["csrf"]) + site.tokens.load_tokens(["csrf"]) + print(site.tokens["csrf"]) + # This function performs an initial GET request on the given Wikimedia file to check if the statement we will be adding is already in the page. It returns a boolean, with True if the statement is not found and can be added. "qid" is passed as a tuple with both the value and the data type, so this check can handle the formatting for different data types. If statements are found in the entity with the prop and value, but no qualifiers, we return the statement id instead, so that the qualifier can be added to that statement instead of creating a new one using postqual(). + def check(mediaid, qid, prop): - request = site.simple_request(action='wbgetentities', ids=mediaid) + request = site.simple_request(action="wbgetentities", ids=mediaid) bool = False - ref = '' + ref = "" ret = bool, ref raw = request.submit() - if raw.get('entities').get(mediaid).get('pageid'): - existing_data = raw.get('entities').get(mediaid) + if raw.get("entities").get(mediaid).get("pageid"): + existing_data = raw.get("entities").get(mediaid) else: - return True, '' + return True, "" exit() try: - if existing_data.get('statements').get(prop): - statements = existing_data.get('statements').get(prop) + if existing_data.get("statements").get(prop): + statements = existing_data.get("statements").get(prop) else: - return True, '' + return True, "" exit() except: - return True, '' + return True, "" exit() # The following code is used to check the existing statements that match the prop. If any statement matches the prop and qid but has no qualifiers, the statement id is returned. If there is a matching statement with qualifiers, return False. Otherwise (statements with matching prop have no matching qid) return True. This logic is not complete: it will return a statement id for a statement with no qualifier, even if another statement already has the desired qualifier. Also, it would return False even in cases where the qualifier value is different from the desired qualifier, in cases where there there is a matching qid and prop with qualifiers. # statement['mainsnak']['datavalue']['value'] == qid - if qid[0] == 'item': - if any(statement['mainsnak']['datavalue']['value']['id'] == qid[1] and not statement.get('references') for statement in statements): + if qid[0] == "item": + if any( + statement["mainsnak"]["datavalue"]["value"]["id"] == qid[1] + and not statement.get("references") + for statement in statements + ): for statement in statements: - if statement['mainsnak']['datavalue']['value']['id'] == qid[1] and not statement.get('references'): - ref = statement['id'] - if any(statement['mainsnak']['datavalue']['value']['id'] == qid[1] and not statement.get('qualifiers') for statement in statements): + if statement["mainsnak"]["datavalue"]["value"]["id"] == qid[ + 1 + ] and not statement.get("references"): + ref = statement["id"] + if any( + statement["mainsnak"]["datavalue"]["value"]["id"] == qid[1] + and not statement.get("qualifiers") + for statement in statements + ): for statement in statements: - if statement['mainsnak']['datavalue']['value']['id'] == qid[1] and not statement.get('qualifiers'): - return add_det(statement['id']), ref + if statement["mainsnak"]["datavalue"]["value"]["id"] == qid[ + 1 + ] and not statement.get("qualifiers"): + return add_det(statement["id"]), ref exit() - elif any(statement['mainsnak']['datavalue']['value']['id'] == qid[1] for statement in statements): - print(' -- There already exists a statement with a ' + prop + ' > ' + qid[1] + ' claim for ' + mediaid + '.') + elif any( + statement["mainsnak"]["datavalue"]["value"]["id"] == qid[1] + for statement in statements + ): + print( + " -- There already exists a statement with a " + + prop + + " > " + + qid[1] + + " claim for " + + mediaid + + "." + ) # print(existing_data.get('statements')) return False, ref else: return True, ref - if qid[0] == 'string': - if any(statement['mainsnak']['datavalue']['value'] == qid[1] and not statement.get('references') for statement in statements): + if qid[0] == "string": + if any( + statement["mainsnak"]["datavalue"]["value"] == qid[1] + and not statement.get("references") + for statement in statements + ): for statement in statements: - if statement['mainsnak']['datavalue']['value'] == qid[1] and not statement.get('references'): - ref = statement['id'] - if any(statement['mainsnak']['datavalue']['value'] == qid[1] and not statement.get('qualifiers') for statement in statements): + if statement["mainsnak"]["datavalue"]["value"] == qid[ + 1 + ] and not statement.get("references"): + ref = statement["id"] + if any( + statement["mainsnak"]["datavalue"]["value"] == qid[1] + and not statement.get("qualifiers") + for statement in statements + ): for statement in statements: - if statement['mainsnak']['datavalue']['value'] == qid[1] and not statement.get('qualifiers'): - return add_det(statement['id']), ref + if statement["mainsnak"]["datavalue"]["value"] == qid[ + 1 + ] and not statement.get("qualifiers"): + return add_det(statement["id"]), ref exit() - elif any(statement['mainsnak']['datavalue']['value'] == qid[1] for statement in statements): - print(' -- There already exists a statement with a ' + prop + ' > ' + qid[1] + ' claim for ' + mediaid + '.') + elif any( + statement["mainsnak"]["datavalue"]["value"] == qid[1] + for statement in statements + ): + print( + " -- There already exists a statement with a " + + prop + + " > " + + qid[1] + + " claim for " + + mediaid + + "." + ) # print(existing_data.get('statements')) return False, ref else: return True, ref - if qid[0] == 'monolingualtext': - if any(statement['mainsnak']['datavalue']['value']['text'] == qid[1] and not statement.get('references') for statement in statements): + if qid[0] == "monolingualtext": + if any( + statement["mainsnak"]["datavalue"]["value"]["text"] == qid[1] + and not statement.get("references") + for statement in statements + ): for statement in statements: - if statement['mainsnak']['datavalue']['value']['text'] == qid[1] and not statement.get('references'): - ref = statement['id'] - if any(statement['mainsnak']['datavalue']['value']['text'] == qid[1] and not statement.get('qualifiers') for statement in statements): + if statement["mainsnak"]["datavalue"]["value"]["text"] == qid[ + 1 + ] and not statement.get("references"): + ref = statement["id"] + if any( + statement["mainsnak"]["datavalue"]["value"]["text"] == qid[1] + and not statement.get("qualifiers") + for statement in statements + ): for statement in statements: - if statement['mainsnak']['datavalue']['value']['text'] == qid[1] and not statement.get('qualifiers'): - return add_det(statement['id']), ref + if statement["mainsnak"]["datavalue"]["value"]["text"] == qid[ + 1 + ] and not statement.get("qualifiers"): + return add_det(statement["id"]), ref exit() - elif any(statement['mainsnak']['datavalue']['value']['text'] == qid[1] for statement in statements): - print(' -- There already exists a statement with a ' + prop + ' > ' + qid[1] + ' claim for ' + mediaid + '.') + elif any( + statement["mainsnak"]["datavalue"]["value"]["text"] == qid[1] + for statement in statements + ): + print( + " -- There already exists a statement with a " + + prop + + " > " + + qid[1] + + " claim for " + + mediaid + + "." + ) # print(existing_data.get('statements')) return False, ref else: return True, ref - if qid[0] == 'somevalue': + if qid[0] == "somevalue": try: - if any(statement['qualifiers'].get('P2093') for statement in statements): + if any(statement["qualifiers"].get("P2093") for statement in statements): for statement in statements: - if statement['qualifiers']['P2093'][0]['datavalue']['value'] == qid[1] and not statement.get('references'): - ref = statement['id'] + if statement["qualifiers"]["P2093"][0]["datavalue"]["value"] == qid[ + 1 + ] and not statement.get("references"): + ref = statement["id"] for statement in statements: try: - if statement['qualifiers']['P2093'][0]['datavalue']['value'] == qid[1]: - print(' -- There already exists a statement with a ' + prop + ' > ' + qid[1] + ' claim for ' + mediaid + '.') + if ( + statement["qualifiers"]["P2093"][0]["datavalue"]["value"] + == qid[1] + ): + print( + " -- There already exists a statement with a " + + prop + + " > " + + qid[1] + + " claim for " + + mediaid + + "." + ) # print(existing_data.get('statements')) return False, ref @@ -241,16 +336,29 @@ def check(mediaid, qid, prop): return True, ref except KeyError: return True, ref - if qid[0] == 'source': + if qid[0] == "source": try: - if any(statement['qualifiers'].get('P973') for statement in statements): + if any(statement["qualifiers"].get("P973") for statement in statements): for statement in statements: - if statement['qualifiers']['P973'][0]['datavalue']['value'] == qid[1] and not statement.get('references'): - ref = statement['id'] + if statement["qualifiers"]["P973"][0]["datavalue"]["value"] == qid[ + 1 + ] and not statement.get("references"): + ref = statement["id"] for statement in statements: try: - if statement['qualifiers']['P973'][0]['datavalue']['value'] == qid[1]: - print(' -- There already exists a statement with a ' + prop + ' > ' + qid[1] + ' claim for ' + mediaid + '.') + if ( + statement["qualifiers"]["P973"][0]["datavalue"]["value"] + == qid[1] + ): + print( + " -- There already exists a statement with a " + + prop + + " > " + + qid[1] + + " claim for " + + mediaid + + "." + ) # print(existing_data.get('statements')) return False, ref @@ -263,333 +371,534 @@ def check(mediaid, qid, prop): except KeyError: return True, ref + # The following functions define specific statements to add, and uses formattedclaim() to append them to the "claims" array. It first uses the check() to check if the statement is not yet in the item, and appends it the list of statements to add in the edit if not. For now, we are just hardcoding actual values which are the same for all edits. check() returns True, False, or the string value of a statement id. + def add_rs(mediaid, rs, dpla_id): if rights.get(rs): prop = list(rights[rs].keys())[0] qid = rights[rs][prop] - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - checkclaim = check(mediaid, ('item', qid), prop) + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) # return claim - if prop == 'P275' and not qid == 'Q6938433': - prop = 'P6216' - qid = 'Q50423863' + if prop == "P275" and not qid == "Q6938433": + prop = "P6216" + qid = "Q50423863" - if prop == 'P6426': - prop = 'P6216' - qid = 'Q19652' + if prop == "P6426": + prop = "P6216" + qid = "Q19652" - if qid == 'Q6938433': - prop = 'P6216' - qid = 'Q88088423' + if qid == "Q6938433": + prop = "P6216" + qid = "Q88088423" - if rs == 'http://creativecommons.org/publicdomain/mark/1.0/': - prop = 'P6216' - qid = 'Q19652' + if rs == "http://creativecommons.org/publicdomain/mark/1.0/": + prop = "P6216" + qid = "Q19652" if rs: - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - checkclaim = check(mediaid, ('item', qid), prop) + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_collection(mediaid, hub, institution, dpla_id): - if hub == 'Q518155': + if hub == "Q518155": institution = hub if institution: qid = institution - prop = u'P195' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - checkclaim = check(mediaid, ('item', qid), prop) + prop = "P195" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_access(mediaid, access, dpla_id): if access: qid = access - prop = u'P7228' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - checkclaim = check(mediaid, ('item', qid), prop) + prop = "P7228" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_level(mediaid, level, dpla_id): if level: qid = level - prop = u'P6224' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - checkclaim = check(mediaid, ('item', qid), prop) + prop = "P6224" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_parent(mediaid, parent, dpla_id): if institution: qid = institution - prop = u'P195' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - checkclaim = check(mediaid, ('item', qid), prop) + prop = "P195" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_id(mediaid, id): - prop = u'P760' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, id, 'string', id) - checkclaim = check(mediaid, ('string', id), prop) + prop = "P760" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim(prop, id, "string", id) + checkclaim = check(mediaid, ("string", id), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_naid(mediaid, naid, dpla_id): - prop = u'P1225' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, naid, 'string', dpla_id) - checkclaim = check(mediaid, ('string', naid), prop) + prop = "P1225" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim(prop, naid, "string", dpla_id) + checkclaim = check(mediaid, ("string", naid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_subject(mediaid, subject, dpla_id): - prop = u'P4272' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, subject, 'string', dpla_id) - checkclaim = check(mediaid, ('string', subject), prop) + prop = "P4272" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim(prop, subject, "string", dpla_id) + checkclaim = check(mediaid, ("string", subject), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim -def add_subject_entity(mediaid, qid, dpla_id): - prop = u'P921' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - checkclaim = check(mediaid, ('item', qid), prop) +def add_subject_entity(mediaid, qid, dpla_id): + prop = "P921" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_title(mediaid, title, dpla_id): if title: title = title - prop = u'P1476' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, {"text": title[:1499].rstrip(), "language": "en"}, 'monolingualtext', dpla_id) - checkclaim = check(mediaid, ('monolingualtext', title), prop) + prop = "P1476" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim( + prop, + {"text": title[:1499].rstrip(), "language": "en"}, + "monolingualtext", + dpla_id, + ) + checkclaim = check(mediaid, ("monolingualtext", title), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_desc(mediaid, desc, dpla_id): if desc: desc = desc - prop = u'P10358' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, {"text": desc[:1499].rstrip(), "language": "en"}, 'monolingualtext', dpla_id) - checkclaim = check(mediaid, ('monolingualtext', desc), prop) + prop = "P10358" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim( + prop, + {"text": desc[:1499].rstrip(), "language": "en"}, + "monolingualtext", + dpla_id, + ) + checkclaim = check(mediaid, ("monolingualtext", desc), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_creator(mediaid, creator, dpla_id): if creator: - prop = u'P170' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, 'somevalue', 'wikibase-entityid', dpla_id) - claim['qualifiers']['P2093'] = [{"snaktype": "value", "property": "P2093", "datavalue": { "value": creator[:1499].rstrip(), "type": "string"}}] - checkclaim = check(mediaid, ('somevalue', creator), prop) + prop = "P170" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim(prop, "somevalue", "wikibase-entityid", dpla_id) + claim["qualifiers"]["P2093"] = [ + { + "snaktype": "value", + "property": "P2093", + "datavalue": {"value": creator[:1499].rstrip(), "type": "string"}, + } + ] + checkclaim = check(mediaid, ("somevalue", creator), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + # This will catch when displayDate is a single year or a date. def add_date(mediaid, date, dpla_id): - prop = u'P170' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - if re.match('^[0-9]{1,4}$', str(date)): + prop = "P170" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + if re.match("^[0-9]{1,4}$", str(date)): if int(date) < 2025: - datetime.datetime.strptime('%Y') - claim = formattedclaim(prop, date, 'time', dpla_id) - checkclaim = check(mediaid, ('time', date), prop) + datetime.datetime.strptime("%Y") + claim = formattedclaim(prop, date, "time", dpla_id) + checkclaim = check(mediaid, ("time", date), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_contributed(mediaid, hub, institution, dpla_id): - prop = u'P9126' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - qid = u'Q2944483' - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - claim['qualifiers']['P3831'] = [{"snaktype": "value", "property": "P3831", "datavalue": { "value": { "entity-type": "item", "numeric-id": 393351 }, "type": "wikibase-entityid" }}] - checkclaim = check(mediaid, ('item', qid), prop) + prop = "P9126" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + qid = "Q2944483" + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + claim["qualifiers"]["P3831"] = [ + { + "snaktype": "value", + "property": "P3831", + "datavalue": { + "value": {"entity-type": "item", "numeric-id": 393351}, + "type": "wikibase-entityid", + }, + } + ] + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) - if hub == u'Q518155': - qid = u'Q518155' - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - claim['qualifiers']['P3831'] = [{"snaktype": "value", "property": "P3831", "datavalue": { "value": { "entity-type": "item", "numeric-id": 108296843 }, "type": "wikibase-entityid" }}] - checkclaim = check(mediaid, ('item', qid), prop) + claims["claims"].append(claim) + if hub == "Q518155": + qid = "Q518155" + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + claim["qualifiers"]["P3831"] = [ + { + "snaktype": "value", + "property": "P3831", + "datavalue": { + "value": {"entity-type": "item", "numeric-id": 108296843}, + "type": "wikibase-entityid", + }, + } + ] + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) qid = institution - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - claim['qualifiers']['P3831'] = [{"snaktype": "value", "property": "P3831", "datavalue": { "value": { "entity-type": "item", "numeric-id": 108296919 }, "type": "wikibase-entityid" }}] - checkclaim = check(mediaid, ('item', qid), prop) + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + claim["qualifiers"]["P3831"] = [ + { + "snaktype": "value", + "property": "P3831", + "datavalue": { + "value": {"entity-type": "item", "numeric-id": 108296919}, + "type": "wikibase-entityid", + }, + } + ] + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) else: qid = hub - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - claim['qualifiers']['P3831'] = [{"snaktype": "value", "property": "P3831", "datavalue": { "value": { "entity-type": "item", "numeric-id": 393351 }, "type": "wikibase-entityid" }}] - checkclaim = check(mediaid, ('item', qid), prop) + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + claim["qualifiers"]["P3831"] = [ + { + "snaktype": "value", + "property": "P3831", + "datavalue": { + "value": {"entity-type": "item", "numeric-id": 393351}, + "type": "wikibase-entityid", + }, + } + ] + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) qid = institution - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - claim['qualifiers']['P3831'] = [{"snaktype": "value", "property": "P3831", "datavalue": { "value": { "entity-type": "item", "numeric-id": 108296843 }, "type": "wikibase-entityid" }}] - checkclaim = check(mediaid, ('item', qid), prop) + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + claim["qualifiers"]["P3831"] = [ + { + "snaktype": "value", + "property": "P3831", + "datavalue": { + "value": {"entity-type": "item", "numeric-id": 108296843}, + "type": "wikibase-entityid", + }, + } + ] + checkclaim = check(mediaid, ("item", qid), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) + def add_local_id(mediaid, id, institution, dpla_id): if id: - prop = u'P217' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, id, 'string', dpla_id) - checkclaim = check(mediaid, ('string', id), prop) - claim['qualifiers']['P195'] = [{"snaktype": "value", "property": "P195", "datavalue": { "value": { "entity-type": "item", "numeric-id": int(institution.replace(u'Q', u'')) }, "type": "wikibase-entityid" }}] + prop = "P217" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim(prop, id, "string", dpla_id) + checkclaim = check(mediaid, ("string", id), prop) + claim["qualifiers"]["P195"] = [ + { + "snaktype": "value", + "property": "P195", + "datavalue": { + "value": { + "entity-type": "item", + "numeric-id": int(institution.replace("Q", "")), + }, + "type": "wikibase-entityid", + }, + } + ] if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_source(mediaid, hub, url, dpla_id): - qid = u'Q74228490' - prop = u'P7482' - summary = u' -- Adding [[:d:Property:' + prop + ']] to ' + mediaid + '.' - claim = formattedclaim(prop, { "entity-type": "item", "numeric-id": int(qid.replace(u'Q', u'')) }, 'wikibase-entityid', dpla_id) - claim['qualifiers']['P973'] = [{"snaktype": "value", "property": "P973", "datavalue": { "value": url, "type": "string"}, "datatype": "url"}] - claim['qualifiers']['P137'] = [{"snaktype": "value", "property": "P137", "datavalue": { "value": { "entity-type": "item", "numeric-id": int(hub.replace(u'Q', u'')) }, "type": "wikibase-entityid" }}] - checkclaim = check(mediaid, ('source', url), prop) + qid = "Q74228490" + prop = "P7482" + summary = " -- Adding [[:d:Property:" + prop + "]] to " + mediaid + "." + claim = formattedclaim( + prop, + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))}, + "wikibase-entityid", + dpla_id, + ) + claim["qualifiers"]["P973"] = [ + { + "snaktype": "value", + "property": "P973", + "datavalue": {"value": url, "type": "string"}, + "datatype": "url", + } + ] + claim["qualifiers"]["P137"] = [ + { + "snaktype": "value", + "property": "P137", + "datavalue": { + "value": { + "entity-type": "item", + "numeric-id": int(hub.replace("Q", "")), + }, + "type": "wikibase-entityid", + }, + } + ] + checkclaim = check(mediaid, ("source", url), prop) if checkclaim[1]: add_ref(checkclaim[1], claim) if checkclaim[0] == True: pywikibot.output(summary) - claims['claims'].append(claim) + claims["claims"].append(claim) return claim + def add_det(claimid): if claimid: - qid = u'Q61848113' - prop = u'P459' - value = json.dumps({"entity-type":"item","numeric-id": int(qid.replace(u'Q', u'')) }) + qid = "Q61848113" + prop = "P459" + value = json.dumps( + {"entity-type": "item", "numeric-id": int(qid.replace("Q", ""))} + ) postqual(claimid, prop, value) + def add_ref(claimid, claim): if claimid: - claim['id'] = claimid - refclaims['claims'].append(claim) - print(' -- Adding reference for ' + claimid + '.') - -def dpla_claims(mediaid, dpla_id, url, descs, dates, titles, hub, local_ids, institution, rs, creators, subjects, naids, access, level): - print(' -- Accessing Commons ID ' + mediaid) + claim["id"] = claimid + refclaims["claims"].append(claim) + print(" -- Adding reference for " + claimid + ".") + + +def dpla_claims( + mediaid, + dpla_id, + url, + descs, + dates, + titles, + hub, + local_ids, + institution, + rs, + creators, + subjects, + naids, + access, + level, +): + print(" -- Accessing Commons ID " + mediaid) try: - file_claims = json.loads(requests.get('https://commons.wikimedia.org/wiki/Special:EntityData/' + mediaid + '.json').text) + file_claims = json.loads( + requests.get( + "https://commons.wikimedia.org/wiki/Special:EntityData/" + + mediaid + + ".json" + ).text + ) except: file_claims = {} - file_claims['entities'] = {mediaid: {'statements': {}}} - print(' -- Accessed Commons ID ' + mediaid) + file_claims["entities"] = {mediaid: {"statements": {}}} + print(" -- Accessed Commons ID " + mediaid) dpla_claims = [] removals = [] if rights.get(rs): rightsprop = list(rights[rs].keys())[0] rightsvalue = rights[rs][rightsprop] - if rightsprop == 'P275': - statusvalue = 'Q50423863' + if rightsprop == "P275": + statusvalue = "Q50423863" - if rightsprop == 'P6426': - statusvalue = 'Q19652' + if rightsprop == "P6426": + statusvalue = "Q19652" - if rightsvalue == 'Q6938433': - statusvalue = 'Q88088423' + if rightsvalue == "Q6938433": + statusvalue = "Q88088423" - if rs == 'http://creativecommons.org/publicdomain/mark/1.0/': - statusvalue = 'Q19652' + if rs == "http://creativecommons.org/publicdomain/mark/1.0/": + statusvalue = "Q19652" parsesubjects = [] parsetitles = [] @@ -610,139 +919,258 @@ def dpla_claims(mediaid, dpla_id, url, descs, dates, titles, hub, local_ids, ins creators = parsecreators descs = parsedescs subjects = parsesubjects - props = ['P6216', 'P275', 'P217', 'P6426', 'P760', 'P1476', 'P195', 'P170', 'P9126', 'P7482', 'P4272', 'P571', 'P10358', 'P1225', 'P7228', 'P6224', 'P921'] - claims = {'P6216': statusvalue, rightsprop: rightsvalue, 'P217': local_ids, 'P760': [dpla_id], 'P1476': titles, 'P195': ['Q518155' if hub == 'Q518155' else institution], 'P170': creators, 'P9126': ['Q2944483', hub, institution], 'P7482': [url], 'P4272': subjects, 'P571': dates, 'P10358': descs, 'P1225': naids, 'P6224': [level], 'P7228': [access], 'P921': parsesubjectentities} - for prop in file_claims['entities'][mediaid]['statements'].keys(): - for stmt in file_claims['entities'][mediaid]['statements'][prop]: - if stmt.get('references'): - if any(pub['snaks']['P123'][0]['datavalue']['value']['id'] == 'Q2944483' for pub in stmt['references']): - if stmt['mainsnak']['snaktype'] == 'value': - type = stmt['mainsnak']['datavalue']['type'] - if stmt['mainsnak']['property'] == 'P7482': + props = [ + "P6216", + "P275", + "P217", + "P6426", + "P760", + "P1476", + "P195", + "P170", + "P9126", + "P7482", + "P4272", + "P571", + "P10358", + "P1225", + "P7228", + "P6224", + "P921", + ] + claims = { + "P6216": statusvalue, + rightsprop: rightsvalue, + "P217": local_ids, + "P760": [dpla_id], + "P1476": titles, + "P195": ["Q518155" if hub == "Q518155" else institution], + "P170": creators, + "P9126": ["Q2944483", hub, institution], + "P7482": [url], + "P4272": subjects, + "P571": dates, + "P10358": descs, + "P1225": naids, + "P6224": [level], + "P7228": [access], + "P921": parsesubjectentities, + } + for prop in file_claims["entities"][mediaid]["statements"].keys(): + for stmt in file_claims["entities"][mediaid]["statements"][prop]: + if stmt.get("references"): + if any( + pub["snaks"]["P123"][0]["datavalue"]["value"]["id"] == "Q2944483" + for pub in stmt["references"] + ): + if stmt["mainsnak"]["snaktype"] == "value": + type = stmt["mainsnak"]["datavalue"]["type"] + if stmt["mainsnak"]["property"] == "P7482": try: - dpla_claims.append({stmt['mainsnak']['property']: {"id": stmt['id'], "value": stmt['qualifiers']['P973'][0]['datavalue']['value']} }) + dpla_claims.append( + { + stmt["mainsnak"]["property"]: { + "id": stmt["id"], + "value": stmt["qualifiers"]["P973"][0][ + "datavalue" + ]["value"], + } + } + ) except: pass - elif type == 'wikibase-entityid': - dpla_claims.append({stmt['mainsnak']['property']: {"id": stmt['id'], "value": stmt['mainsnak']['datavalue']['value']['id']} }) - elif type == 'string': - dpla_claims.append({stmt['mainsnak']['property']: {"id": stmt['id'], "value": stmt['mainsnak']['datavalue']['value']} }) - elif type == 'monolingualtext': - dpla_claims.append({stmt['mainsnak']['property']: {"id": stmt['id'], "value": stmt['mainsnak']['datavalue']['value']['text']} }) - if stmt['mainsnak']['snaktype'] == 'somevalue': + elif type == "wikibase-entityid": + dpla_claims.append( + { + stmt["mainsnak"]["property"]: { + "id": stmt["id"], + "value": stmt["mainsnak"]["datavalue"]["value"][ + "id" + ], + } + } + ) + elif type == "string": + dpla_claims.append( + { + stmt["mainsnak"]["property"]: { + "id": stmt["id"], + "value": stmt["mainsnak"]["datavalue"]["value"], + } + } + ) + elif type == "monolingualtext": + dpla_claims.append( + { + stmt["mainsnak"]["property"]: { + "id": stmt["id"], + "value": stmt["mainsnak"]["datavalue"]["value"][ + "text" + ], + } + } + ) + if stmt["mainsnak"]["snaktype"] == "somevalue": try: - dpla_claims.append({stmt['mainsnak']['property']: {"id": stmt['id'], "value": stmt['qualifiers']['P2093'][0]['datavalue']['value']} }) + dpla_claims.append( + { + stmt["mainsnak"]["property"]: { + "id": stmt["id"], + "value": stmt["qualifiers"]["P2093"][0][ + "datavalue" + ]["value"], + } + } + ) except: - removals.append(stmt['id']) + removals.append(stmt["id"]) for claim in dpla_claims: for prop in claim.keys(): if prop not in claims.keys(): - removals.append(claim[prop]['id']) - elif claim[prop]['value'] not in claims[prop]: + removals.append(claim[prop]["id"]) + elif claim[prop]["value"] not in claims[prop]: # if any(clm[:1499].replace('\xa0',' ').rstrip() == claim[prop]['value'] for clm in claims[prop]): - removals.append(claim[prop]['id']) + removals.append(claim[prop]["id"]) if len(removals) > 0: - rmdata = {u'action' : u'wbremoveclaims', - u'format' : u'json', - u'id' : mediaid, - u'claim' : '|'.join(removals), - u'token' : token, - u'bot' : True, - u'summary': u'Changing structured data claims from [[COM:DPLA|DPLA]] item \'[[dpla:' + dpla_id + '|' + dpla_id + ']]\'. [[COM:DPLA/MOD|Leave feedback]]!' - } - - save = http.fetch(u'https://commons.wikimedia.org/w/api.php', method='POST', data=rmdata) - print(' --- Saved removals!') + rmdata = { + "action": "wbremoveclaims", + "format": "json", + "id": mediaid, + "claim": "|".join(removals), + "token": token, + "bot": True, + "summary": "Changing structured data claims from [[COM:DPLA|DPLA]] item '[[dpla:" + + dpla_id + + "|" + + dpla_id + + "]]'. [[COM:DPLA/MOD|Leave feedback]]!", + } + + save = http.fetch( + "https://commons.wikimedia.org/w/api.php", method="POST", data=rmdata + ) + print(" --- Saved removals!") + def parsed(dpla_id): - print(' -- Accessing DPLA ID ' + dpla_id) + print(" -- Accessing DPLA ID " + dpla_id) try: - dpla = json.loads(requests.get('https://api.dp.la/v2/items/' + dpla_id + '?api_key=4e956679e5d18f3acdf8edb6c35f2462', timeout=15).text) + dpla = json.loads( + requests.get( + "https://api.dp.la/v2/items/" + + dpla_id + + "?api_key=4e956679e5d18f3acdf8edb6c35f2462", + timeout=15, + ).text + ) except: - print(' -- Sleeping 30 seconds and retrying...') + print(" -- Sleeping 30 seconds and retrying...") time.sleep(30) - dpla = json.loads(requests.get('https://api.dp.la/v2/items/' + dpla_id + '?api_key=4e956679e5d18f3acdf8edb6c35f2462').text) - print(' -- Accessed DPLA ID ' + dpla_id) + dpla = json.loads( + requests.get( + "https://api.dp.la/v2/items/" + + dpla_id + + "?api_key=4e956679e5d18f3acdf8edb6c35f2462" + ).text + ) + print(" -- Accessed DPLA ID " + dpla_id) try: - dpla = dpla['docs'][0] + dpla = dpla["docs"][0] except: print(dpla) - print('DPLA API returned error.') + print("DPLA API returned error.") return False - hub = hubs[dpla['provider']['name']]['Wikidata'] - institution = hubs[dpla['provider']['name']]['institutions'][dpla['dataProvider']['name']]['Wikidata'] - titles = dpla['sourceResource']['title'] - rs = dpla['rights'] - url = dpla['isShownAt'] + hub = hubs[dpla["provider"]["name"]]["Wikidata"] + institution = hubs[dpla["provider"]["name"]]["institutions"][ + dpla["dataProvider"]["name"] + ]["Wikidata"] + titles = dpla["sourceResource"]["title"] + rs = dpla["rights"] + url = dpla["isShownAt"] try: - dates = dpla['sourceResource']['date'] + dates = dpla["sourceResource"]["date"] except: - dates = '' + dates = "" try: - local_ids = dpla['sourceResource']['identifier'] + local_ids = dpla["sourceResource"]["identifier"] except: - local_ids = '' + local_ids = "" try: - descs = dpla['sourceResource']['description'] + descs = dpla["sourceResource"]["description"] except: - descs = '' + descs = "" try: subjects = [] - for subject in dpla['sourceResource']['subject']: + for subject in dpla["sourceResource"]["subject"]: added = False # print(subject.get('name')) - if subject.get('name') in subject_ids: - for subjqid in subject_ids[subject.get('name')]['id']: + if subject.get("name") in subject_ids: + for subjqid in subject_ids[subject.get("name")]["id"]: # print((str(subject.get('name')), subjqid)) if not (any(subjqid in i for i in subjects)): # print((str(subject.get('name')), subjqid)) - subjects.append((str(subject.get('name')), subjqid)) + subjects.append((str(subject.get("name")), subjqid)) added = True - if not (any(subject.get('name') in i for i in subjects)): - subjects.append((str(subject.get('name') or ''), '')) + if not (any(subject.get("name") in i for i in subjects)): + subjects.append((str(subject.get("name") or ""), "")) # print((str(subject.get('name') or ''), '')) added = True - elif subject.get('exactMatch'): - subjqid = '' - naid = subject.get('exactMatch')[0].replace('https://catalog.archives.gov/id/', '') - h = requests.get('https://wikidata.reconci.link/en/api?queries=%7B%0A%20%20%22q1%22%3A%20%7B%0A%20%20%20%20%22query%22%3A%20%22' + str(subject.get('name') or '') + '%22%2C%0A%20%20%20%20%20%20%22limit%22%3A%205%2C%0A%20%20%20%20%20%20%22properties%22%3A%20%5B%0A%20%20%20%20%20%20%20%20%7B%0A%20%20%20%20%20%20%20%20%20%20%22pid%22%3A%20%22P1225%22%2C%0A%20%20%20%20%20%20%20%20%20%20%22v%22%3A%20%22' + naid + '%22%0A%20%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%20%5D%2C%0A%20%20%20%20%20%20%22type_strict%22%3A%20%22should%22%0A%20%20%20%20%7D%0A%7D') + elif subject.get("exactMatch"): + subjqid = "" + naid = subject.get("exactMatch")[0].replace( + "https://catalog.archives.gov/id/", "" + ) + h = requests.get( + "https://wikidata.reconci.link/en/api?queries=%7B%0A%20%20%22q1%22%3A%20%7B%0A%20%20%20%20%22query%22%3A%20%22" + + str(subject.get("name") or "") + + "%22%2C%0A%20%20%20%20%20%20%22limit%22%3A%205%2C%0A%20%20%20%20%20%20%22properties%22%3A%20%5B%0A%20%20%20%20%20%20%20%20%7B%0A%20%20%20%20%20%20%20%20%20%20%22pid%22%3A%20%22P1225%22%2C%0A%20%20%20%20%20%20%20%20%20%20%22v%22%3A%20%22" + + naid + + "%22%0A%20%20%20%20%20%20%20%20%7D%0A%20%20%20%20%20%20%5D%2C%0A%20%20%20%20%20%20%22type_strict%22%3A%20%22should%22%0A%20%20%20%20%7D%0A%7D" + ) subjectresults = json.loads(h.text) - if subjectresults['q1']['result']: - subjqid = subjectresults['q1']['result'][0]['id'] - subjects.append((str(subject.get('name') or ''), subjqid)) + if subjectresults["q1"]["result"]: + subjqid = subjectresults["q1"]["result"][0]["id"] + subjects.append((str(subject.get("name") or ""), subjqid)) added = True if added == False: # print((str(subject.get('name') or ''), '')) - subjects.append((str(subject.get('name') or ''), '')) + subjects.append((str(subject.get("name") or ""), "")) # print(subjects) except: - subjects = '' + subjects = "" try: - creators = dpla['sourceResource']['creator'] + creators = dpla["sourceResource"]["creator"] except: - creators = '' - if dpla['provider']['name'] == "National Archives and Records Administration": - naids = dpla['sourceResource']['identifier'] - codes = {"10031403": "Q66739888", "10031402": "Q24238356", "10031399": "Q66739729", "10031400": "Q66739849", "10031401": "Q66739875"} + creators = "" + if dpla["provider"]["name"] == "National Archives and Records Administration": + naids = dpla["sourceResource"]["identifier"] + codes = { + "10031403": "Q66739888", + "10031402": "Q24238356", + "10031399": "Q66739729", + "10031400": "Q66739849", + "10031401": "Q66739875", + } levels = {"item": "Q11723795", "itemAv": "Q11723795", "fileUnit": "Q59221146"} - xml = BeautifulSoup(dpla['originalRecord']['stringValue'], 'xml') + xml = BeautifulSoup(dpla["originalRecord"]["stringValue"], "xml") try: - acccess_naid = str(xml.find("accessRestriction").find("status").find("naId").text) + acccess_naid = str( + xml.find("accessRestriction").find("status").find("naId").text + ) access = codes[acccess_naid] except: - access = '' + access = "" for key in levels.keys(): if xml.find(key): level = levels[key] - local_ids = '' + local_ids = "" else: - naids = '' - access = '' - level = '' - + naids = "" + access = "" + level = "" # language # type @@ -751,71 +1179,119 @@ def parsed(dpla_id): # contributor # publisher - return url, descs, dates, titles, hub, local_ids, institution, rs, creators, subjects, naids, access, level + return ( + url, + descs, + dates, + titles, + hub, + local_ids, + institution, + rs, + creators, + subjects, + naids, + access, + level, + ) # Only make a post request if the claims array has accumulated at least one claim to add to the file. If any results set finds at least one edit to make, then 'posted' remains True, and the search will be tried again to pick up any more edits to make. If a whole results set is checked and no edits remain, we assume the edits to files with those search parameters are all complete. - if len(claims['claims']) > 0: + if len(claims["claims"]) > 0: posted = True try: - post = json.loads(http.fetch(u'https://commons.wikimedia.org/w/api.php', method='POST', data=postdata).text) - print(' --- Saved new claims!') + post = json.loads( + http.fetch( + "https://commons.wikimedia.org/w/api.php", + method="POST", + data=postdata, + ).text + ) + print(" --- Saved new claims!") except: - print(' --- Error encountered. 1') + print(" --- Error encountered. 1") sys.exit() return count + # Since we are posting directly to the API, we must explicitly request a login token that will be sent with the POSTs. site = pywikibot.Site() + def login(): - tokenrequest = http.fetch(u'https://commons.wikimedia.org/w/api.php?action=query&meta=tokens&type=csrf&format=json') + tokenrequest = http.fetch( + "https://commons.wikimedia.org/w/api.php?action=query&meta=tokens&type=csrf&format=json" + ) tokendata = json.loads(tokenrequest.text) - token = tokendata.get(u'query').get(u'tokens').get(u'csrftoken') + token = tokendata.get("query").get("tokens").get("csrftoken") return token + token = login() # We can use a PWB generator to programatically make the list of files we are working on based on a set of criteria. Here, we are generating the page titles from a Wikimedia Commons search and categories. For other types of available page generators, see . As an additional step, we take the pageid provided by the generator and prepend "M" for the mediaid needed for posting SDC statements. If the list of claims generated is greater than zero, then we send the post using wbeditentity to the Wikimedia Commons API. count = 0 -if method == 'list': +if method == "list": - ltotal = [i for i in os.listdir(args.lists) if '.txt' in i] - lists = [i for i in ltotal if not ('COMPLETE' in i) and not ('WORKING' in i)] - percent = 100 * ( len(ltotal) - len(lists) ) / len(ltotal) + ltotal = [i for i in os.listdir(args.lists) if ".txt" in i] + lists = [i for i in ltotal if not ("COMPLETE" in i) and not ("WORKING" in i)] + percent = 100 * (len(ltotal) - len(lists)) / len(ltotal) while len(lists) > 0: if len(lists) > 1: - x = random.choice(range(0, len(lists)-1)) + x = random.choice(range(0, len(lists) - 1)) elif len(lists) == 1: x = 0 - working_file = args.lists + '/WORKING-' + lists[x] + working_file = args.lists + "/WORKING-" + lists[x] print(working_file) - os.rename(args.lists + '/' + lists[x], working_file) + os.rename(args.lists + "/" + lists[x], working_file) files = pywikibot.pagegenerators.TextIOPageGenerator(working_file) for file in files: - print('\n' + str(file).replace('\"\"', '"')) - mediaid = 'M' + str(file.pageid) - dpla_id = re.sub(r'^.*DPLA - (.*?)[\.| ].*$', r'\1', str(file)) + print("\n" + str(file).replace('""', '"')) + mediaid = "M" + str(file.pageid) + dpla_id = re.sub(r"^.*DPLA - (.*?)[\.| ].*$", r"\1", str(file)) try: - url, descs, dates, titles, hub, local_ids, institution, rs, creators, subjects, naids, access, level = parsed(dpla_id) + ( + url, + descs, + dates, + titles, + hub, + local_ids, + institution, + rs, + creators, + subjects, + naids, + access, + level, + ) = parsed(dpla_id) except TypeError: - with open('Missing ids.txt', 'a') as missing: - missing.write(dpla_id + '\n') - print(' -- Missing ID recorded.') + with open("Missing ids.txt", "a") as missing: + missing.write(dpla_id + "\n") + print(" -- Missing ID recorded.") continue - claims = {"claims":[]} - refclaims = {"claims":[]} + claims = {"claims": []} + refclaims = {"claims": []} count = count + 1 - print(str(count) + ':\n - ' + args.lists + '/' + lists[x] + ' (' + str('{:.2f}'.format(percent)) + '% done)') + print( + str(count) + + ":\n - " + + args.lists + + "/" + + lists[x] + + " (" + + str("{:.2f}".format(percent)) + + "% done)" + ) add_rs(mediaid, rs, dpla_id) add_id(mediaid, dpla_id) for title in titles: @@ -839,106 +1315,166 @@ def login(): add_naid(mediaid, naid, dpla_id) add_access(mediaid, access, dpla_id) add_level(mediaid, level, dpla_id) - if len(refclaims['claims']) > 0: - postrefs = {u'action' : u'wbeditentity', - u'format' : u'json', - u'id' : mediaid, - u'data' : json.dumps(refclaims), - u'token' : token, - u'bot' : True, - u'summary': u'Added structured data references from [[COM:DPLA|DPLA]] item \'[[dpla:' + dpla_id + '|' + dpla_id + ']]\'. [[COM:DPLA/MOD|Leave feedback]]!' - } - save = http.fetch(u'https://commons.wikimedia.org/w/api.php', method='POST', data=postrefs) + if len(refclaims["claims"]) > 0: + postrefs = { + "action": "wbeditentity", + "format": "json", + "id": mediaid, + "data": json.dumps(refclaims), + "token": token, + "bot": True, + "summary": "Added structured data references from [[COM:DPLA|DPLA]] item '[[dpla:" + + dpla_id + + "|" + + dpla_id + + "]]'. [[COM:DPLA/MOD|Leave feedback]]!", + } + save = http.fetch( + "https://commons.wikimedia.org/w/api.php", + method="POST", + data=postrefs, + ) try: post = json.loads(save.text) - if post['success'] == 1: - print(' --- Saved new refs!') + if post["success"] == 1: + print(" --- Saved new refs!") else: print(post) - print(' --- Error encountered on save.') + print(" --- Error encountered on save.") sys.exit() except: try: token = login() - postrefs = {u'action' : u'wbeditentity', - u'format' : u'json', - u'id' : mediaid, - u'data' : json.dumps(refclaims), - u'token' : token, - u'bot' : True, - u'summary': u'Added structured data references from [[COM:DPLA|DPLA]] item \'[[dpla:' + dpla_id + '|' + dpla_id + ']]\'. [[COM:DPLA/MOD|Leave feedback]]!' - } - save = http.fetch(u'https://commons.wikimedia.org/w/api.php', method='POST', data=postrefs) + postrefs = { + "action": "wbeditentity", + "format": "json", + "id": mediaid, + "data": json.dumps(refclaims), + "token": token, + "bot": True, + "summary": "Added structured data references from [[COM:DPLA|DPLA]] item '[[dpla:" + + dpla_id + + "|" + + dpla_id + + "]]'. [[COM:DPLA/MOD|Leave feedback]]!", + } + save = http.fetch( + "https://commons.wikimedia.org/w/api.php", + method="POST", + data=postrefs, + ) post = json.loads(save.text) - if post['success'] == 1: - print(' --- Saved new refs!') + if post["success"] == 1: + print(" --- Saved new refs!") else: print(post) - print(' --- Error encountered on save.') + print(" --- Error encountered on save.") sys.exit() except: - print(' --- Error encountered. 2') + print(" --- Error encountered. 2") sys.exit() - postdata = {u'action' : u'wbeditentity', - u'format' : u'json', - u'id' : mediaid, - u'data' : json.dumps(claims), - u'token' : token, - u'bot' : True, - u'summary': u'Added structured data claims from [[COM:DPLA|DPLA]] item \'[[dpla:' + dpla_id + '|' + dpla_id + ']]\'. [[COM:DPLA/MOD|Leave feedback]]!' - } + postdata = { + "action": "wbeditentity", + "format": "json", + "id": mediaid, + "data": json.dumps(claims), + "token": token, + "bot": True, + "summary": "Added structured data claims from [[COM:DPLA|DPLA]] item '[[dpla:" + + dpla_id + + "|" + + dpla_id + + "]]'. [[COM:DPLA/MOD|Leave feedback]]!", + } - if len(claims['claims']) > 0: + if len(claims["claims"]) > 0: try: - save = http.fetch(u'https://commons.wikimedia.org/w/api.php', method='POST', data=postdata) + save = http.fetch( + "https://commons.wikimedia.org/w/api.php", + method="POST", + data=postdata, + ) except requests.exceptions.ConnectionError: try: - save = http.fetch(u'https://commons.wikimedia.org/w/api.php', method='POST', data=postdata) + save = http.fetch( + "https://commons.wikimedia.org/w/api.php", + method="POST", + data=postdata, + ) except requests.exceptions.ConnectionError: - save = http.fetch(u'https://commons.wikimedia.org/w/api.php', method='POST', data=postdata) + save = http.fetch( + "https://commons.wikimedia.org/w/api.php", + method="POST", + data=postdata, + ) try: post = json.loads(save.text) - if post['success'] == 1: - print(' --- Saved new claims!') + if post["success"] == 1: + print(" --- Saved new claims!") else: print(post) - print(' --- Error encountered on save.') + print(" --- Error encountered on save.") sys.exit() except: try: token = login() - postdata = {u'action' : u'wbeditentity', - u'format' : u'json', - u'id' : mediaid, - u'data' : json.dumps(claims), - u'token' : token, - u'bot' : True, - u'summary': u'Added structured data claims from [[COM:DPLA|DPLA]] item \'[[dpla:' + dpla_id + '|' + dpla_id + ']]\'. [[COM:DPLA/MOD|Leave feedback]]!' - } - save = http.fetch(u'https://commons.wikimedia.org/w/api.php', method='POST', data=postdata) + postdata = { + "action": "wbeditentity", + "format": "json", + "id": mediaid, + "data": json.dumps(claims), + "token": token, + "bot": True, + "summary": "Added structured data claims from [[COM:DPLA|DPLA]] item '[[dpla:" + + dpla_id + + "|" + + dpla_id + + "]]'. [[COM:DPLA/MOD|Leave feedback]]!", + } + save = http.fetch( + "https://commons.wikimedia.org/w/api.php", + method="POST", + data=postdata, + ) post = json.loads(save.text) - if post['success'] == 1: - print(' --- Saved new claims!') + if post["success"] == 1: + print(" --- Saved new claims!") else: print(post) - print(' --- Error encountered on save.') + print(" --- Error encountered on save.") sys.exit() except: - print(str(post) + '\n' + str(postdata)) - print(' --- Error encountered. 3') + print(str(post) + "\n" + str(postdata)) + print(" --- Error encountered. 3") sys.exit() - dpla_claims(mediaid, dpla_id, url, descs, dates, titles, hub, local_ids, institution, rs, creators, subjects, naids, access, level) - - os.rename(working_file, args.lists + '/COMPLETE-' + lists[x]) - - ltotal = [i for i in os.listdir(args.lists) if '.txt' in i] - lists = [i for i in ltotal if not ('COMPLETE' in i) and not ('WORKING' in i)] + dpla_claims( + mediaid, + dpla_id, + url, + descs, + dates, + titles, + hub, + local_ids, + institution, + rs, + creators, + subjects, + naids, + access, + level, + ) + + os.rename(working_file, args.lists + "/COMPLETE-" + lists[x]) + + ltotal = [i for i in os.listdir(args.lists) if ".txt" in i] + lists = [i for i in ltotal if not ("COMPLETE" in i) and not ("WORKING" in i)] # De-dupe missing file: duduped = set() - for line in open('Missing ids.txt', "r"): + for line in open("Missing ids.txt", "r"): duduped.add(line) - with open('Missing ids.txt', "w") as f: - f.write(''.join(duduped)) \ No newline at end of file + with open("Missing ids.txt", "w") as f: + f.write("".join(duduped))