diff --git a/ayugespidertools/scraper/pipelines/mysql/__init__.py b/ayugespidertools/scraper/pipelines/mysql/__init__.py index 6d8b681..e0b0d4e 100644 --- a/ayugespidertools/scraper/pipelines/mysql/__init__.py +++ b/ayugespidertools/scraper/pipelines/mysql/__init__.py @@ -43,7 +43,7 @@ def process_item(self, item: Any, spider: "AyuSpider") -> Any: self.insert_item(alter_item) return item - def insert_item(self, alter_item: "AlterItem"): + def insert_item(self, alter_item: "AlterItem") -> None: """通用插入数据 Args: diff --git a/ayugespidertools/scraper/pipelines/mysql/twisted.py b/ayugespidertools/scraper/pipelines/mysql/twisted.py index b3cb0f0..b19f201 100644 --- a/ayugespidertools/scraper/pipelines/mysql/twisted.py +++ b/ayugespidertools/scraper/pipelines/mysql/twisted.py @@ -12,6 +12,8 @@ ] if TYPE_CHECKING: + from twisted.python.failure import Failure + from ayugespidertools.common.typevars import MysqlConf, slogT from ayugespidertools.spiders import AyuSpider @@ -40,10 +42,10 @@ def open_spider(self, spider: "AyuSpider") -> None: query = self.dbpool.runInteraction(self.db_create) query.addErrback(self.db_create_err) - def db_create(self, cursor): + def db_create(self, cursor: Any) -> None: pass - def db_create_err(self, failure): + def db_create_err(self, failure: "Failure") -> None: self.slog.error(f"创建数据表失败: {failure}") def process_item(self, item: Any, spider: "AyuSpider") -> Any: @@ -52,7 +54,7 @@ def process_item(self, item: Any, spider: "AyuSpider") -> Any: query.addErrback(self.handle_error, item) return item - def db_insert(self, cursor, item): + def db_insert(self, cursor: Any, item: Any) -> Any: alter_item = ReuseOperation.reshape_item(item) if not (new_item := alter_item.new_item): return @@ -84,5 +86,5 @@ def db_insert(self, cursor, item): return self.db_insert(cursor, item) return item - def handle_error(self, failure, item): + def handle_error(self, failure: "Failure", item: Any) -> None: self.slog.error(f"插入数据失败:{failure}, item: {item}") diff --git a/ayugespidertools/scraper/pipelines/oracle/twisted.py b/ayugespidertools/scraper/pipelines/oracle/twisted.py index 1c5df47..84d86f5 100644 --- a/ayugespidertools/scraper/pipelines/oracle/twisted.py +++ b/ayugespidertools/scraper/pipelines/oracle/twisted.py @@ -11,6 +11,7 @@ if TYPE_CHECKING: from oracledb.connection import Connection + from twisted.python.failure import Failure from ayugespidertools.common.typevars import OracleConf, slogT from ayugespidertools.spiders import AyuSpider @@ -42,10 +43,10 @@ def open_spider(self, spider: "AyuSpider") -> None: query = self.dbpool.runInteraction(self.db_create) query.addErrback(self.db_create_err) - def db_create(self, cursor): + def db_create(self, cursor: Any) -> None: pass - def db_create_err(self, failure): + def db_create_err(self, failure: "Failure") -> None: self.slog.error(f"创建数据表失败: {failure}") def process_item(self, item: Any, spider: "AyuSpider") -> Any: @@ -54,7 +55,7 @@ def process_item(self, item: Any, spider: "AyuSpider") -> Any: query.addErrback(self.handle_error, item) return item - def db_insert(self, cursor, item): + def db_insert(self, cursor: Any, item: Any) -> Any: alter_item = ReuseOperation.reshape_item(item) if not (new_item := alter_item.new_item): return @@ -63,5 +64,5 @@ def db_insert(self, cursor, item): cursor.execute(sql, tuple(new_item.values())) return item - def handle_error(self, failure, item): + def handle_error(self, failure: "Failure", item: Any) -> None: self.slog.error(f"插入数据失败:{failure}, item: {item}") diff --git a/ayugespidertools/scraper/pipelines/postgres/twisted.py b/ayugespidertools/scraper/pipelines/postgres/twisted.py index 34fd620..0d6c82d 100644 --- a/ayugespidertools/scraper/pipelines/postgres/twisted.py +++ b/ayugespidertools/scraper/pipelines/postgres/twisted.py @@ -12,6 +12,8 @@ __all__ = ["AyuTwistedPostgresPipeline"] if TYPE_CHECKING: + from twisted.python.failure import Failure + from ayugespidertools.common.typevars import PostgreSQLConf, slogT from ayugespidertools.spiders import AyuSpider @@ -40,10 +42,10 @@ def open_spider(self, spider: "AyuSpider") -> None: query = self.dbpool.runInteraction(self.db_create) query.addErrback(self.db_create_err) - def db_create(self, cursor): + def db_create(self, cursor: Any) -> None: pass - def db_create_err(self, failure): + def db_create_err(self, failure: "Failure") -> None: self.slog.error(f"创建数据表失败: {failure}") def process_item(self, item: Any, spider: "AyuSpider") -> Any: @@ -52,7 +54,7 @@ def process_item(self, item: Any, spider: "AyuSpider") -> Any: query.addErrback(self.handle_error, item) return item - def db_insert(self, cursor, item): + def db_insert(self, cursor: Any, item: Any) -> Any: alter_item = ReuseOperation.reshape_item(item) if not (new_item := alter_item.new_item): return @@ -81,5 +83,5 @@ def db_insert(self, cursor, item): return self.db_insert(cursor, item) return item - def handle_error(self, failure, item): + def handle_error(self, failure: "Failure", item: Any) -> None: self.slog.error(f"插入数据失败:{failure}, item: {item}")