From 3d34e9913529683bb7d4295b240f1bc0bf529f7a Mon Sep 17 00:00:00 2001 From: shengchenyang <15538221825@163.com> Date: Sun, 29 Dec 2024 02:20:17 +0800 Subject: [PATCH] chore: deprecation reminder --- ayugespidertools/common/typevars.py | 5 ++++- ayugespidertools/scraper/http/request/aiohttp.py | 8 ++++++++ ayugespidertools/scraper/middlewares/netlib/aiohttplib.py | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ayugespidertools/common/typevars.py b/ayugespidertools/common/typevars.py index f374fb6..fe5bf01 100644 --- a/ayugespidertools/common/typevars.py +++ b/ayugespidertools/common/typevars.py @@ -2,6 +2,7 @@ from __future__ import annotations import threading +from enum import Enum from typing import TYPE_CHECKING, Any, Literal, NamedTuple, TypeVar, Union from sqlalchemy import create_engine @@ -18,8 +19,10 @@ NoneType = type(None) I_Str = TypeVar("I_Str", int, str) B_Str = TypeVar("B_Str", bytes, str) -I_Str_N = TypeVar("I_Str_N", int, str, NoneType) +I_Str_N = TypeVar("I_Str_N", int, str, None) Str_Lstr = TypeVar("Str_Lstr", str, list[str]) +_SENTINEL = Enum("_SENTINEL", "sentinel") +sentinel = _SENTINEL.sentinel AiohttpRequestMethodStr = Literal[ "GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH" diff --git a/ayugespidertools/scraper/http/request/aiohttp.py b/ayugespidertools/scraper/http/request/aiohttp.py index c26cdcd..9e53bdb 100644 --- a/ayugespidertools/scraper/http/request/aiohttp.py +++ b/ayugespidertools/scraper/http/request/aiohttp.py @@ -7,6 +7,9 @@ from scrapy import Request +from ayugespidertools.common.typevars import _SENTINEL, sentinel +from ayugespidertools.config import logger + __all__ = [ "AiohttpRequest", ] @@ -45,6 +48,7 @@ def __init__( callback: CallbackT | None = None, method: str = "GET", headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, + body: _SENTINEL = sentinel, cookies: CookiesT | None = None, meta: dict[str, Any] | None = None, encoding: str = "utf-8", @@ -82,6 +86,10 @@ def __init__( max_field_size: int | None = None, timeout: ClientTimeout | None = None, ) -> None: + if body is not sentinel: + logger.error( + "parameter 'body' is deprecated, use 'json' or 'data' argument instead" + ) aiohttp_req_args = { "method": method, diff --git a/ayugespidertools/scraper/middlewares/netlib/aiohttplib.py b/ayugespidertools/scraper/middlewares/netlib/aiohttplib.py index c220313..bd94538 100644 --- a/ayugespidertools/scraper/middlewares/netlib/aiohttplib.py +++ b/ayugespidertools/scraper/middlewares/netlib/aiohttplib.py @@ -44,7 +44,7 @@ class AiohttpDownloaderMiddleware: def _retry( self, request: AyuRequest, - reason: int, + reason: str | int, spider: AyuSpider, ) -> AyuRequest | None: """重试请求 @@ -70,7 +70,7 @@ def _retry_with_limit( self, request: AyuRequest, retries: int, - reason: str, + reason: str | int, stats: StatsCollector, ): logger.debug(f"Retrying {request} (failed {retries} times): {reason}")