You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
54 lines
1.6 KiB
Python
54 lines
1.6 KiB
Python
# pipelines.py
|
|
import logging
|
|
import pymongo
|
|
from itemadapter import ItemAdapter
|
|
|
|
from science_article_add.items.wos import WosArticleItem, WosCitedNumberItem, WosIdRelationItem
|
|
from science_article_add.pipelines.verify_data import VerifyDataIntegrity
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class MongoDBPipeline:
|
|
def __init__(self, mongo_uri, mongo_db):
|
|
self.mongo_uri = mongo_uri
|
|
self.mongo_db = mongo_db
|
|
|
|
@classmethod
|
|
def from_crawler(cls, crawler):
|
|
return cls(
|
|
mongo_uri=crawler.settings.get('MONGO_URI'),
|
|
mongo_db=crawler.settings.get('MONGO_DATABASE', 'scrapy_data')
|
|
)
|
|
|
|
def open_spider(self, spider):
|
|
self.client = pymongo.MongoClient(self.mongo_uri)
|
|
self.db = self.client[self.mongo_db]
|
|
|
|
def close_spider(self, spider):
|
|
self.client.close()
|
|
|
|
def process_item(self, item, spider):
|
|
adapter = ItemAdapter(item)
|
|
|
|
# 根据Item类型存储到不同的集合
|
|
if isinstance(item, WosIdRelationItem):
|
|
collection_name = 'relation_school_wos'
|
|
elif isinstance(item, WosCitedNumberItem):
|
|
collection_name = 'relation_cited_number_wos'
|
|
else:
|
|
collection_name = 'data_other'
|
|
|
|
# 插入数据
|
|
self.db[collection_name].insert_one(dict(adapter))
|
|
|
|
return item
|
|
|
|
|
|
class WosVerifyDataIntegrity(VerifyDataIntegrity):
|
|
|
|
def open_spider(self, spider):
|
|
spider_batch_ids = spider.get_batch_ids()
|
|
for batch in spider_batch_ids:
|
|
if batch.get("field") == "UT":
|
|
self.batch_ids.add(batch.get("third_id"))
|