44.scrapy爬取链家网站二手房信息-2

全面采集二手房数据：

网站二手房总数据量为27650条，但有的参数字段会出现一些问题，因为只给返回100页数据，具体查看就需要去细分请求url参数去请求网站数据。
我这里大概的获取了一下筛选条件参数，一些存在问题也没做细化处理，大致的采集数据量为21096，实际19794条。

看一下执行完成结果：

{'downloader/exception_count': 199,
'downloader/exception_type_count/twisted.internet.error.NoRouteError': 192,
'downloader/exception_type_count/twisted.web._newclient.ResponseNeverReceived': 7,
'downloader/request_bytes': 9878800,
'downloader/request_count': 21096,
'downloader/request_method_count/GET': 21096,
'downloader/response_bytes': 677177525,
'downloader/response_count': 20897,
'downloader/response_status_count/200': 20832,
'downloader/response_status_count/301': 49,
'downloader/response_status_count/302': 11,
'downloader/response_status_count/404': 5,
'dupefilter/filtered': 53,
'finish_reason': 'finished',
'finish_time': datetime.datetime(2018, 11, 12, 8, 49, 42, 371235),
'httperror/response_ignored_count': 5,
'httperror/response_ignored_status_count/404': 5,
'log_count/DEBUG': 21098,
'log_count/ERROR': 298,
'log_count/INFO': 61,
'request_depth_max': 3,
'response_received_count': 20837,
'retry/count': 199,
'retry/reason_count/twisted.internet.error.NoRouteError': 192,
'retry/reason_count/twisted.web._newclient.ResponseNeverReceived': 7,
'scheduler/dequeued': 21096,
'scheduler/dequeued/memory': 21096,
'scheduler/enqueued': 21096,
'scheduler/enqueued/memory': 21096,
'spider_exceptions/TypeError': 298,
'start_time': datetime.datetime(2018, 11, 12, 7, 59, 52, 608383)}
2018-11-12 16:49:42 [scrapy.core.engine] INFO: Spider closed (finished)

采集数据如图：

num = 296910/15=19794条

 

2. lianjia.py# -*- coding: utf-8 -*-
import scrapyclass LianjiaSpider(scrapy.Spider):name = 'lianjia'allowed_domains = ['gz.lianjia.com']start_urls = ['https://gz.lianjia.com/ershoufang/pg1/']def parse(self, response):for i in range(1,8):for j in range(1,8):url = 'https://gz.lianjia.com/ershoufang/p{}a{}pg1'.format(i,j)yield scrapy.Request(url=url,callback=self.parse_detail)def parse_detail(self,response):# 符合筛选条件的个数counts = response.xpath("//h2[@class='total fl']/span/text()").extract_first().strip()# print(counts)if int(counts)%30 >0:p_num = int(counts)//30+1# print(p_num)# 拼接首页urlfor k in  range(1,p_num+1):url = response.urllink_url = url.split('pg')[0]+'pg{}/'.format(k)# print(link_url)yield scrapy.Request(url=link_url,callback=self.parse_detail2)def parse_detail2(self,response):#获取当前页面urllink_urls = response.xpath("//div[@class='info clear']/div[@class='title']/a/@href").extract()for link_url in link_urls:# print(link_url)yield scrapy.Request(url=link_url,callback=self.parse_detail3)# print('*'*100)def parse_detail3(self,response):title = response.xpath("//div[@class='title']/h1[@class='main']/text()").extract_first()print('标题: '+ title)dist = response.xpath("//div[@class='areaName']/span[@class='info']/a/text()").extract_first()print('所在区域: '+ dist)contents = response.xpath("//div[@class='introContent']/div[@class='base']")# print(contents)house_type = contents.xpath("./div[@class='content']/ul/li[1]/text()").extract_first()print('房屋户型: '+ house_type)floor = contents.xpath("./div[@class='content']/ul/li[2]/text()").extract_first()print('所在楼层: '+ floor)built_area = contents.xpath("./div[@class='content']/ul/li[3]/text()").extract_first()print('建筑面积: '+ built_area)family_structure = contents.xpath("./div[@class='content']/ul/li[4]/text()").extract_first()print('户型结构: '+ family_structure)inner_area = contents.xpath("./div[@class='content']/ul/li[5]/text()").extract_first()print('套内面积: '+ inner_area)architectural_type = contents.xpath("./div[@class='content']/ul/li[6]/text()").extract_first()print('建筑类型: '+ architectural_type)house_orientation = contents.xpath("./div[@class='content']/ul/li[7]/text()").extract_first()print('房屋朝向: '+ house_orientation)building_structure = contents.xpath("./div[@class='content']/ul/li[8]/text()").extract_first()print('建筑结构: '+ building_structure)decoration_condition = contents.xpath("./div[@class='content']/ul/li[9]/text()").extract_first()print('装修状况: '+ decoration_condition)proportion = contents.xpath("./div[@class='content']/ul/li[10]/text()").extract_first()print('梯户比例: '+ proportion)elevator = contents.xpath("./div[@class='content']/ul/li[11]/text()").extract_first()print('配备电梯: '+ elevator)age_limit =contents.xpath("./div[@class='content']/ul/li[12]/text()").extract_first()print('产权年限: '+ age_limit)# try:#     house_label = response.xpath("//div[@class='content']/a/text()").extract_first()# except:#     house_label = ''# print('房源标签: ' + house_label)with open('text2', 'a', encoding='utf-8')as f:f.write('\n'.join([title,dist,house_type,floor,built_area,family_structure,inner_area,architectural_type,house_orientation,building_structure,decoration_condition,proportion,elevator,age_limit]))f.write('\n' + '=' * 50 + '\n')print('-'*100)

3.代码还需要细分的话，就多配置url的请求参数，缩小筛选范围，获取页面就更精准，就能避免筛选到过3000的数据类型，可以再去细分。