概述
利用Scrapy爬取豆瓣电影Top250排行榜电影信息,并保存到MongoDB数据库中
使用pymongo库操作MOngodb数据库
没有进行数据清洗
源码
items.py
class Douban250Item(scrapy.Item):"""豆瓣电影Top250爬虫 Item"""# 电影名称title=scrapy.Field()# 信息info=scrapy.Field()# 评分star=scrapy.Field()# 评价人数num=scrapy.Field()# 一句话点评quote=scrapy.Field()
spider/douban250_spider.py
# !/usr/bin/env python# -*- coding:utf-8 -*-import scrapyfrom myscrapy.items import Douban250Itemclass Douban250Spider(scrapy.Spider):"""豆瓣电影Top250爬虫Spider"""name = 'douban250'allowed_domains=['',]base_url='/top250?start='offset=0start_urls=[base_url+str(offset),]def parse(self, response):# 包含本页所有电影的SelectorListmovies=response.xpath('//ol/li/div[@class="item"]')for movie in movies:title=movie.xpath('./div[@class="info"]//a/span[1]/text()')[0].extract()info=movie.xpath('./div[@class="info"]//p[1]')[0].extract()star=movie.xpath('.//span[@class="rating_num"]/text()')[0].extract()num=movie.xpath('.//div[@class="star"]/span[last()]/text()')[0].extract()quote=movie.xpath('./div[@class="info"]//p[2]')[0].extract()item=Douban250Item()item['title'] = titleitem['info'] = infoitem['star'] = staritem['num'] = numitem['quote'] = quoteyield itemif self.offset<225:self.offset+=25yield scrapy.Request(url=self.base_url+str(self.offset),callback=self.parse)
pipelines.py
import pymongoclass Douban250Pipeline(object):"""豆瓣电影Top250爬虫Item Pipeline"""def __init__(self):# 连接数据库服务器mongo_client=pymongo.MongoClient(host='127.0.0.1',port=27017)# 指定数据库mydb=mongo_client['mydb']# 指定集合self.douban250_col=mydb['douban250']def process_item(self,item,spider):# 插入数据self.douban250_col.insert(dict(item))return itemdef close_spider(self,spider):pass
settings.py
ITEM_PIPELINES = {'myscrapy.pipelines.Douban250Pipeline': 300,}
