|
|
# -*- coding: utf-8 -*-
|
|
|
# @Time : 2026/1/19 10:01
|
|
|
# @Author : zhaoxiangpeng
|
|
|
# @File : crawl_article_by_id.py
|
|
|
|
|
|
import time
|
|
|
import logging
|
|
|
from typing import List
|
|
|
import pymysql
|
|
|
from pymysql import cursors
|
|
|
from twisted.internet import defer
|
|
|
from scrapy.crawler import CrawlerProcess
|
|
|
from scrapy.utils.project import get_project_settings
|
|
|
from science_article_wos.spiders.wos_article_download_by_id import WosArticleDownloadByIdSpider
|
|
|
from science_article_wos.utils import tools
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
def starter_forever():
|
|
|
def check_task() -> bool:
|
|
|
from pymongo import MongoClient
|
|
|
cli = MongoClient(settings.get("MONGO_URI"))
|
|
|
db = cli[settings.get("MONGO_DATABASE")]
|
|
|
r = db['todo_ids_wos'].find_one(filter={"state": 0})
|
|
|
if r:
|
|
|
return True
|
|
|
return False
|
|
|
|
|
|
def check_session() -> bool:
|
|
|
from redis import Redis
|
|
|
cli = Redis.from_url(settings.get("REDIS_URL"), decode_responses=True)
|
|
|
keys = cli.keys('cookie_pool:wos_sid:*')
|
|
|
for key in keys:
|
|
|
# 获取所有的信息
|
|
|
status = cli.hget(key, "status")
|
|
|
if status == "normal":
|
|
|
real_sid = key.rsplit(':', maxsplit=1)[-1]
|
|
|
return True
|
|
|
return False
|
|
|
|
|
|
@defer.inlineCallbacks
|
|
|
def f(running: bool = True):
|
|
|
while running:
|
|
|
# 连接到mongodb查询是否有未执行的任务
|
|
|
if not check_task():
|
|
|
logger.info("没有可下载的任务，即将结束")
|
|
|
running = False
|
|
|
continue
|
|
|
# 查询redis中是否有可用的cookie
|
|
|
if not check_session():
|
|
|
logger.info("没有有可用的cookie，等待")
|
|
|
time.sleep(60 * 5)
|
|
|
continue
|
|
|
yield process.crawl(WosArticleDownloadByIdSpider)
|
|
|
time.sleep(60 * 2)
|
|
|
|
|
|
settings = get_project_settings()
|
|
|
process = CrawlerProcess(settings)
|
|
|
f(True)
|
|
|
process.start()
|
|
|
|
|
|
|
|
|
def starter():
|
|
|
process = CrawlerProcess(get_project_settings())
|
|
|
process.crawl(WosArticleDownloadByIdSpider)
|
|
|
process.start()
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
starter_forever()
|