zhaoxiangpeng
/
science-spider2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wos:用科学平台的wos_raw_data当去重库

Browse Source
main
zhaoxiangpeng 3 weeks ago
parent 1bf0703dba
commit 9e0f6b8b55
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File

2
science_article_wos/science_article_wos/pipelines.py
Unescape Escape View File

@ -195,7 +195,7 @@ class DupTodoBySciencePipeline(DupTodoPipeline):
settings = spider.settings settings = spider.settings
dup_uri = settings.get("MONGO_URI_SCIENCE") dup_uri = settings.get("MONGO_URI_SCIENCE")
cli = MongoClient(dup_uri) cli = MongoClient(dup_uri)
dup_db = cli.get_database("MONGO_DATABASE_SCIENCE") dup_db = cli.get_database(settings.get("MONGO_DATABASE_SCIENCE"))
self.dup_collection = dup_db.get_collection('wos_raw_data') self.dup_collection = dup_db.get_collection('wos_raw_data')
def is_exists(self, item, filter_key) -> bool: def is_exists(self, item, filter_key) -> bool:

Write Preview
Loading…
Cancel
Save