|
|
|
@ -109,6 +109,9 @@ class ScienceArticleWosDownloaderMiddleware:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class WosStarterApiXkeyDownloaderMiddleware:
|
|
|
|
class WosStarterApiXkeyDownloaderMiddleware:
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
https://api.clarivate.com/swagger-ui/?apikey=none&url=https%3A%2F%2Fdeveloper.clarivate.com%2Fapis%2Fwos-starter%2Fswagger
|
|
|
|
|
|
|
|
"""
|
|
|
|
async def process_request(self, request, spider):
|
|
|
|
async def process_request(self, request, spider):
|
|
|
|
key_param = {
|
|
|
|
key_param = {
|
|
|
|
'X-ApiKey': '53b8164e7543ccebe489988287e8b871bc2c0880'
|
|
|
|
'X-ApiKey': '53b8164e7543ccebe489988287e8b871bc2c0880'
|
|
|
|
@ -185,10 +188,13 @@ class WosCookieMiddleware:
|
|
|
|
def get_sid_from_redis(self):
|
|
|
|
def get_sid_from_redis(self):
|
|
|
|
val = self.redis_cli.rpoplpush(self.cookiepool_cache_key, self.cookiepool_cache_key)
|
|
|
|
val = self.redis_cli.rpoplpush(self.cookiepool_cache_key, self.cookiepool_cache_key)
|
|
|
|
if val:
|
|
|
|
if val:
|
|
|
|
self.redis_cli.hincrby(f'{self.redis_key_prefix}:{val}', 'used_times', 1)
|
|
|
|
self.inc_used_times(val)
|
|
|
|
return val
|
|
|
|
return val
|
|
|
|
return None
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def inc_used_times(self, val: str = None):
|
|
|
|
|
|
|
|
self.redis_cli.hincrby(f'{self.redis_key_prefix}:{val}', 'used_times', 1)
|
|
|
|
|
|
|
|
|
|
|
|
def mark_sid_status(self, sid: str, status: str = 'validate'):
|
|
|
|
def mark_sid_status(self, sid: str, status: str = 'validate'):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
:param sid:
|
|
|
|
:param sid:
|
|
|
|
@ -221,6 +227,11 @@ class WosCookieMiddleware:
|
|
|
|
self.redis_cli.delete(f'{self.cookiepool_cache_key}')
|
|
|
|
self.redis_cli.delete(f'{self.cookiepool_cache_key}')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class WosSessionMiddleware:
|
|
|
|
|
|
|
|
def process_request(self, request: Request, spider: Spider):
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class A:
|
|
|
|
class A:
|
|
|
|
def __init__(self, redis_cli):
|
|
|
|
def __init__(self, redis_cli):
|
|
|
|
self.redis_cli = redis_cli
|
|
|
|
self.redis_cli = redis_cli
|
|
|
|
|