【已解决】PySpider中把结果保存到MongoDB数据库中

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-10-10 14:50:08
# Project: ScholasticStorybook

from pyspider.libs.base_handler import *

import re
import json
# import html
import lxml
from bs4 import BeautifulSoup

from urllib.parse import quote_plus
from pymongo import MongoClient

HostUrl = "
https://www.scholastic.com
"
# RowsPerPage = 24
RowsPerPage = 72

MONGODB_DB_NAME = "Scholastic"
MONGODB_COLLECTION_NAME = "Storybook"


######################################################################
# Common Util Functions
######################################################################

def generateMongoUri(host=None,
                port=None,
                isUseAuth=False,
                username=None,
                password=None,
                authSource=None,
                authMechanism=None):
    """"generate mongodb uri"""
    mongodbUri = ""

    if not host:
        # host = "127.0.0.0"
        host = "localhost"

    if not port:
        port = 27017

    mongodbUri = "mongodb://%s:%s" % (
        host, \
        port
    )
    # '
mongodb://localhost:27017
'
    # '
mongodb://xxx:27017
'

    if isUseAuth:
        mongodbUri = "mongodb://%s:%s@%s:%s" % (
            quote_plus(username), \
            quote_plus(password), \
            host, \
            port \
        )
        print(mongodbUri)

        if authSource:
            mongodbUri = mongodbUri + ("/%s" % authSource)
            print("mongodbUri=%s" % mongodbUri)

        if authMechanism:
            mongodbUri = mongodbUri + ("?authMechanism=%s" % authMechanism)
            print("mongodbUri=%s" % mongodbUri)

    print("return mongodbUri=%s" % mongodbUri)
    #
mongodb://username:quoted_password@host:port/authSource?authMechanism=authMechanism
    #
mongodb://localhost:27017

    return mongodbUri


def createMongoClient():
    mongoUri = generateMongoUri()
    print("mongoUri=%s" % mongoUri)
    client = MongoClient(mongoUri)
    print("client=%s" % client) # client=MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True)
    return client

。。。

######################################################################
# Project Related
######################################################################

class ResultMongo(object):

    def __init__(self):
        print("ResultMongo __init__")
        self.client = createMongoClient()
        print("self.client=%s" % self.client)

        self.db = self.client[MONGODB_DB_NAME]
        print("self.db=%s" % self.db) # self.db=Database(MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), 'Scholastic')

        self.collection = self.db[MONGODB_COLLECTION_NAME]
        print("self.collection=%s" % self.collection) # self.collection=Collection(Database(MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), 'Scholastic'), 'Storybook')

    def __del__(self):
        print("ResultMongo __del__")
        self.client.close()

    def on_result(self, result):
        """save result to mongodb"""
        print("ResultMongo on_result: result=%s" % result)

        respResult = None

        if result:
            respResult = self.collection.insert(result)
            print("respResult=%s" % respResult) # respResult=5bc45fad7f4d3847b78e8c69

        return respResult

def debugMongoResult():
    mongo = ResultMongo()
    print("mongo=%s" % mongo) # mongo=<__main__.ResultMongo object at 0x10b24dcf8>
    dataDict = {
        'authors': ['Sharon Creech'],
        'coverImgUrl': '
https://www.scholastic.com/content5/media/products/66/9780439569866_mres.jpg
',
        'description': "I guess it does\nlook like a poem\nwhen you see it\ntyped up\nlike that.\n\nJack hates poetry. Only girls write it and every time he tries to, his brain feels empty. But his teacher, Ms. Stretchberry, won't stop giving her class poetry assignments, and Jack can't avoid them. But then something amazing happens. The more he writes, the more he learns he does have something to say.\n\nWith a fresh and deceptively simple style, acclaimed author Sharon Creech tells a story with enormous heart. Written as a series of free-verse poems from Jack's point of view, Love That Dog shows how one boy finds his own voice with the help of a teacher, a writer, a pencil, some yellow paper, and of course, a dog.",
        'draLevel': '50',
        'genre': 'Fiction',
        'gradeLevelEquivalent': '',
        'grades': ['6-8'],
        'guidedReading': 'T',
        'illustrators': [],
        'isbn13': '9780439569866',
        'lexileMeasure': '1010L',
        'originUrl': '
https://www.scholastic.com/content/scholastic/books2/love-that-dog-by-sharon-creech
',
        'pages': 112,
        'recommendations': [{
            'title': "Girls' Life Ultimate Guide to Surviving Middle School",
            'url': '
https://www.scholastic.com/content/scholastic/books2/girls-rsquo-life-ultimate-guide-to-surviving-middle-school-by-b
'
            },
            {
            'title': "Girls' Life Ultimate Guide To Surviving Middle School",
            'url': '
https://www.scholastic.com/content/scholastic/books2/girls-rsquo-life-ultimate-guide-to-surviving-middle-school-by-b
'
            },
            {
            'title': 'The Date to Save',
            'url': '
https://www.scholastic.com/content/scholastic/books2/date-to-save-the-by-stephanie-kate-strohm
'
            }
        ],
        'seriesName': '',
        'seriesNumber': 0,
        'tags': ['Poetry Writing', 'School Life'],
        'title': 'Love That Dog',
        'url': '
https://www.scholastic.com/teachers/books/love-that-dog-by-sharon-creech/
'
    }

    respResult = mongo.on_result(dataDict)
    print("respResult=%s" % respResult)

if __name__ == "__main__":
    debugMongoResult()


######################################################################
# PySpider Main Part
######################################################################

class Handler(BaseHandler):
    mongo = ResultMongo()
    print("mongo=%s" % mongo)

    crawl_config = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
        ...
    }

    # @every(minutes=24 * 60)
    def on_start(self):
        ...

    # @config(priority=2)
    def singleBookCallback(self, response):
        ...

        respDict = {
            "originUrl": originUrl,
            "url": url,
            "title": title,
            "coverImgUrl": coverImgUrl,
            "authors": authors,
            "illustrators": illustrators,
            "seriesName": seriesName,
            "seriesNumber": seriesNumber,
            "grades": grades,
            "guidedReading": guidedReading,
            "lexileMeasure": lexileMeasure,
            "gradeLevelEquivalent": gradeLevelEquivalent,
            "draLevel": draLevel,
            "genre": genre,
            "description": description,
            "pages": pages,
            "isbn13": isbn13,
            "tags": tags,
            "recommendations": recommendations,
        }
        print("respDict=%s" % respDict)
        return respDict

    def on_result(self, result):
        print("PySpider on_result: result=%s" % result)
        self.mongo.on_result(result) # 执行插入数据的操作
        super(Handler, self).on_result(result) # 调用原有的数据存储

respDict={'originUrl': ...
PySpider on_result: result={'originUrl':...
ResultMongo on_result: result={'originUrl': ...
respResult=5bc464a1bfaa444ec0607173
{'_id': ObjectId('5bc464a1bfaa444ec0607173'),
 'authors': ['Cynthia Lord'],
 'coverImgUrl': 'https://www.scholastic.com/content5/media/products/38/9780439443838_mres.jpg',
 'description': 'This 2007 Newbery Honor Book is a humorous and heartwarming debut about feeling different and finding acceptance.\nTwelve-year-old Catherine just wants a normal life. Which is near impossible when you have a brother with autism and a family that revolves around his disability. She\'s spent years trying to teach David the rules - from "a peach is not a funny-looking apple" to "keep your pants on in public" - in order to stop his embarrassing behaviors. But the summer Catherine meets Jason, a paraplegic boy, and Kristi, the next-door friend she\'s always wished for, it\'s her own shocking behavior that turns everything upside down and forces her to ask: What is normal?\n"A heartwarming first novel" - Booklist',
 'draLevel': '40',
 'genre': 'Fiction',
 'gradeLevelEquivalent': '',
 'grades': ['3-5', '6-8'],
 'guidedReading': 'R',
 'illustrators': [],
 'isbn13': '9780439443838',
 'lexileMeasure': '670L',
 'originUrl': 'https://www.scholastic.com/content/scholastic/books2/rules-by-cynthia-lord',
 'pages': 224,
 'recommendations': [{'title': 'Braced',
                      'url': 'https://www.scholastic.com/content/scholastic/books2/braced-by-alyson-gerber'},
                     {'title': 'Chasing the Milky Way',
                      'url': 'https://www.scholastic.com/content/scholastic/books2/chasing-the-milky-way-by-erin-e-moulton'},
                     {'title': 'I Even Funnier',
                      'url': 'https://www.scholastic.com/content/scholastic/books2/i-even-funnier-by-chris-grabenstein'}],
 'seriesName': '',
 'seriesNumber': 0,
 'tags': ['Autism',
          'Friends and Friendship',
          'Living with Illness and Disabilities',
          'Physical Challenges',
          'Siblings',
          'Tolerance and Acceptance'],
 'title': 'Rules',
 'url': 'https://www.scholastic.com/teachers/books/rules-by-cynthia-lord/'}

转载请注明：在路上 » 【已解决】PySpider中把结果保存到MongoDB数据库中