From 731e55e0d634de3931495f83f6e41d2cd6bbe1f0 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Tue, 6 Feb 2024 16:44:07 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fake-useragent=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit related to #319 --------- Co-authored-by: Justin Flannery --- camply/providers/base_provider.py | 2 +- camply/providers/going_to_camp/going_to_camp_provider.py | 4 +--- camply/providers/recreation_dot_gov/recdotgov_provider.py | 6 ++---- camply/providers/usedirect/usedirect.py | 3 +++ camply/providers/xanterra/yellowstone_lodging.py | 4 +--- pyproject.toml | 2 +- requirements.txt | 4 ++-- requirements/requirements-all.py3.10.txt | 4 ++-- requirements/requirements-all.py3.11.txt | 4 ++-- requirements/requirements-all.py3.12.txt | 4 ++-- requirements/requirements-all.py3.8.txt | 4 ++-- requirements/requirements-all.py3.9.txt | 8 +++----- requirements/requirements-docs.txt | 6 +++--- requirements/requirements-test.txt | 6 +++--- 14 files changed, 28 insertions(+), 33 deletions(-) diff --git a/camply/providers/base_provider.py b/camply/providers/base_provider.py index 4cf7749c..613234c2 100644 --- a/camply/providers/base_provider.py +++ b/camply/providers/base_provider.py @@ -82,7 +82,7 @@ def __init__(self): """ Initialize with a session """ - _user_agent = UserAgent(use_external_data=False, browsers=["chrome"]).chrome + _user_agent = UserAgent(browsers=["chrome"]).random self.session = requests.Session() self.headers = {"User-Agent": _user_agent} self.session.headers = self.headers diff --git a/camply/providers/going_to_camp/going_to_camp_provider.py b/camply/providers/going_to_camp/going_to_camp_provider.py index b908f51c..c395595e 100644 --- a/camply/providers/going_to_camp/going_to_camp_provider.py +++ b/camply/providers/going_to_camp/going_to_camp_provider.py @@ -335,9 +335,7 @@ def _api_request( url = None if endpoint: url = endpoint.format(hostname) - user_agent = { - "User-Agent": UserAgent(use_external_data=False, browsers=["chrome"]).chrome - } + user_agent = {"User-Agent": UserAgent(browsers=["chrome"]).random} response = requests.get(url=url, headers=user_agent, params=params, timeout=30) if response.ok is False: error_message = f"Receiving bad data from GoingToCamp API: status_code: {response.status_code}: {response.text}" diff --git a/camply/providers/recreation_dot_gov/recdotgov_provider.py b/camply/providers/recreation_dot_gov/recdotgov_provider.py index 942988bc..0fbee5df 100644 --- a/camply/providers/recreation_dot_gov/recdotgov_provider.py +++ b/camply/providers/recreation_dot_gov/recdotgov_provider.py @@ -58,7 +58,7 @@ def __init__(self, api_key: Optional[str] = None): "accept": "application/json", "apikey": _api_key, } - _user_agent = UserAgent(use_external_data=False, browsers=["chrome"]).chrome + _user_agent = UserAgent(browsers=["chrome"]).random self._user_agent = {"User-Agent": _user_agent} @property @@ -579,9 +579,7 @@ def make_recdotgov_request( requests.Response """ # BUILD THE HEADERS EXPECTED FROM THE API - user_agent = { - "User-Agent": UserAgent(use_external_data=False, browsers=["chrome"]).chrome - } + user_agent = {"User-Agent": UserAgent(browsers=["chrome"]).random} headers = STANDARD_HEADERS.copy() headers.update(user_agent) headers.update(RecreationBookingConfig.API_REFERRERS) diff --git a/camply/providers/usedirect/usedirect.py b/camply/providers/usedirect/usedirect.py index 212718e4..29008a46 100644 --- a/camply/providers/usedirect/usedirect.py +++ b/camply/providers/usedirect/usedirect.py @@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Union import ratelimit +from fake_useragent import UserAgent from pydantic import ValidationError from camply.config import FileConfig @@ -322,6 +323,8 @@ def get_campsites_response( key: value for key, value in data.items() if value not in [None, [], ""] } url = f"{self.base_url}/{self.rdr_path}/{UseDirectConfig.AVAILABILITY_ENDPOINT}" + random_ua = UserAgent(browsers=["chrome"]).random + self.json_headers["User-Agent"] = random_ua response = self.make_http_request_retry( url=url, method="POST", diff --git a/camply/providers/xanterra/yellowstone_lodging.py b/camply/providers/xanterra/yellowstone_lodging.py index 46bda0e0..b09022d1 100755 --- a/camply/providers/xanterra/yellowstone_lodging.py +++ b/camply/providers/xanterra/yellowstone_lodging.py @@ -88,9 +88,7 @@ def _try_retry_get_data(endpoint: str, params: Optional[dict] = None) -> dict: dict """ yellowstone_headers = {} - user_agent = { - "User-Agent": UserAgent(use_external_data=False, browsers=["chrome"]).chrome - } + user_agent = {"User-Agent": UserAgent(browsers=["chrome"]).random} yellowstone_headers.update(user_agent) yellowstone_headers.update(STANDARD_HEADERS) yellowstone_headers.update(YellowstoneConfig.API_REFERRERS) diff --git a/pyproject.toml b/pyproject.toml index d65c6413..fa9a2755 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ classifiers = [ ] dependencies = [ "click~=8.1.3", - "fake-useragent~=1.1.3", + "fake-useragent~=1.4.0", "pandas>=2,<3", "pydantic~=1.10.13", "python-dotenv~=1.0.0", diff --git a/requirements.txt b/requirements.txt index 310218fc..b69a7818 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by hatch-pip-compile with Python 3.11 # # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -32,7 +32,7 @@ click==8.1.7 # apprise # rich-click # trogon -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.default idna==3.6 # via requests diff --git a/requirements/requirements-all.py3.10.txt b/requirements/requirements-all.py3.10.txt index ca0b5426..0170fcee 100644 --- a/requirements/requirements-all.py3.10.txt +++ b/requirements/requirements-all.py3.10.txt @@ -8,7 +8,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -46,7 +46,7 @@ exceptiongroup==1.2.0 # via pytest execnet==2.0.2 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.all.py3.10 freezegun==1.2.2 # via hatch.envs.all.py3.10 diff --git a/requirements/requirements-all.py3.11.txt b/requirements/requirements-all.py3.11.txt index e877dde5..70706116 100644 --- a/requirements/requirements-all.py3.11.txt +++ b/requirements/requirements-all.py3.11.txt @@ -8,7 +8,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -44,7 +44,7 @@ coverage==7.4.0 # pytest-cov execnet==2.0.2 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.all.py3.11 freezegun==1.2.2 # via hatch.envs.all.py3.11 diff --git a/requirements/requirements-all.py3.12.txt b/requirements/requirements-all.py3.12.txt index b0445e01..50018f65 100644 --- a/requirements/requirements-all.py3.12.txt +++ b/requirements/requirements-all.py3.12.txt @@ -8,7 +8,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -44,7 +44,7 @@ coverage==7.4.0 # pytest-cov execnet==2.0.2 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.all.py3.12 freezegun==1.2.2 # via hatch.envs.all.py3.12 diff --git a/requirements/requirements-all.py3.8.txt b/requirements/requirements-all.py3.8.txt index c4d6ea8a..fd2c7d09 100644 --- a/requirements/requirements-all.py3.8.txt +++ b/requirements/requirements-all.py3.8.txt @@ -8,7 +8,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -46,7 +46,7 @@ exceptiongroup==1.2.0 # via pytest execnet==2.0.2 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.all.py3.8 freezegun==1.2.2 # via hatch.envs.all.py3.8 diff --git a/requirements/requirements-all.py3.9.txt b/requirements/requirements-all.py3.9.txt index 45661f98..931a1097 100644 --- a/requirements/requirements-all.py3.9.txt +++ b/requirements/requirements-all.py3.9.txt @@ -8,7 +8,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -39,14 +39,12 @@ click==8.1.7 # rich-click # trogon coverage==7.4.0 - # via - # coverage - # pytest-cov + # via pytest-cov exceptiongroup==1.2.0 # via pytest execnet==2.0.2 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.all.py3.9 freezegun==1.2.2 # via hatch.envs.all.py3.9 diff --git a/requirements/requirements-docs.txt b/requirements/requirements-docs.txt index 408f0b1a..9b429926 100644 --- a/requirements/requirements-docs.txt +++ b/requirements/requirements-docs.txt @@ -1,7 +1,7 @@ # # This file is autogenerated by hatch-pip-compile with Python 3.11 # -# [constraints] requirements.txt (SHA256: 7d2ce7d5ed6d5060713188a3a42ce8fdfb3aa0da2ebcb6a000c552ab9fb0cf33) +# [constraints] requirements.txt (SHA256: 2a92e1c6e109ef4c9ce314b9a60b303169d2ddf6cb3b4c4c9df4288cec4ac2ef) # # - markdown-callouts # - markdown-exec @@ -16,7 +16,7 @@ # - mkdocstrings-python # - pymdown-extensions # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -51,7 +51,7 @@ colorama==0.4.6 # via # griffe # mkdocs-material -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via # -c requirements.txt # hatch.envs.docs diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt index 6e5788a8..bff0d2c0 100644 --- a/requirements/requirements-test.txt +++ b/requirements/requirements-test.txt @@ -1,7 +1,7 @@ # # This file is autogenerated by hatch-pip-compile with Python 3.11 # -# [constraints] requirements.txt (SHA256: 7d2ce7d5ed6d5060713188a3a42ce8fdfb3aa0da2ebcb6a000c552ab9fb0cf33) +# [constraints] requirements.txt (SHA256: 2a92e1c6e109ef4c9ce314b9a60b303169d2ddf6cb3b4c4c9df4288cec4ac2ef) # # - pytest~=7.3.1 # - pytest-cov~=4.0.0 @@ -10,7 +10,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -52,7 +52,7 @@ coverage==7.2.7 # pytest-cov execnet==1.9.0 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via # -c requirements.txt # hatch.envs.test