spider/elabSpider/pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymongo
from scrapy.conf import settings
from elabSpider.items import *
import urllib.parse


class ElabspiderPipeline(object):

    def __init__(self):
        # self.client = pymongo.MongoClient(host='139.196.5.59', port=27017)
        # self.client.admin.authenticate(name='dbuser', password='elab@123', mechanism='SCRAM-SHA-1')
        # username = urllib.parse.quote_plus('dbuser')
        # password = urllib.parse.quote_plus('elab@123')
        # uri = "mongodb://dbuser:elab@123@example.com/?authSource=the_database&authMechanism=SCRAM-SHA-1"
        # self.client = pymongo.MongoClient('101.132.106.154', authSource='logdb')

        self.client = pymongo.MongoClient(settings['MONGO_HOST'], authSource='logdb')

        self.db = self.client[settings['MONGO_DB']]
        self.coll = self.db[settings['MONGO_COLL']]

    def process_item(self, item, spider):
        if isinstance(item, CommunityItem):
            self.coll = self.db['departmengprice']
        elif isinstance(item, FTXCommunityItem):
            self.coll = self.db['ftxcommunity']
        elif isinstance(item, ResoldApartmentItem):
            self.coll = self.db[settings['MONGO_COLL']]
        elif isinstance(item, RentalHouseItem):
            if spider.name == 'lfsrentalHouse':
                self.coll = self.db['lfs_rental_house']
            elif spider.name == 'sjkrentalHouse':
                self.coll = self.db['sjk_rental_house']
            else:
                self.coll = self.db['rental_house']
        elif isinstance(item, FTXRentalHouseItem):
            self.coll = self.db['ftx_rental_house']
        elif isinstance(item, ResoldHouseItem):
            if spider.name == 'sjkresoldHouse':
                self.coll = self.db['sjk_resold_house']
            elif spider.name == 'lfsresoldHouse':
                self.coll = self.db['lfs_resold_house']
            else:
                self.coll = self.db['nb_resold_house']
        elif isinstance(item, LfsAveragePriceItem):
            self.coll = self.db['lfs_average_price']
        self.coll.insert_one(item)
        return item