Pipeline.py

class MongoDBPipeline:
def __init__(self,conn,database):
self.conn = conn
self.database = database
@classmethod
def from_crawler(cls, crawler):
return cls(
conn=crawler.settings.get(‘MONGODB_CONNECTION_STRING’),
database=crawler.settings.get(‘MONGODB_DATABASE’)
)
def open_spider(self,spider):
# 創建連接對象
self.client = pymongo.MongoClient(self.conn)
# 連接要用的使用的數據庫
self.db = self.client[self.database]
def process_item(self,item,spider):
# 設置使用表名稱
name = item.__class__.__name__
# 默認item類名
self.db[name].insert_one(dict(item))
return item
def close_spider(self,spider):
self.client.close()