xpath爬好大学网站

from lxml import etree
import pymongo
from config import *
import requests# 连接mongoDB数据库
client = pymongo.MongoClient(MONGO_URL,connect=False)
db = client[MONGO_DB]def get_page(year):url = "http://www.zuihaodaxue.cn/zuihaodaxuepaiming" + year + ".html"response = requests.get(url)response.encoding = 'UTF-8'return response.textdef parse_page_detail(html, year):# 使用xpth爬取网页etree_html = etree.HTML(html)排名 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[1]/text()')学校 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[2]/div/text()')省市 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[3]/text()')总分 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[4]/text()')生源质量 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[5]/text()')培养结果 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[6]/text()')科研规模 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[7]/text()')科研质量 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[8]/text()')顶尖成果 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[9]/text()')顶尖人才 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[10]/text()')科技服务 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[11]/text()')成果转化 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[12]/text()')学生国际化 = etree_html.xpath('//tbody[@class="hidden_zhpm"]/tr/td[13]/text()')# 将数据压缩data = zip(排名, 学校, 省市, 总分, 生源质量, 培养结果, 科研规模, 科研质量, 顶尖成果, 顶尖人才, 科技服务, 成果转化, 学生国际化)# 构建字典for it in data:yield {"year": year,"排名": it[0],"学校": it[1],"省市": it[2],"总分": it[3],"生源质量": it[4],"培养结果": it[5],"科研规模": it[6],"科研质量": it[7],"顶尖成果": it[8],"顶尖人才": it[9],"科技服务": it[10],"成果转化": it[11],"学生国际化": it[12]}# 存到mongoDB数据库
def save_to_mongoDB(data):if db[MONGO_TABLE].insert(data):print('存储成功:', data)return Trueprint('存储失败')return Falseif __name__ == '__main__':year = "2020"html = get_page(year)for data in parse_page_detail(html, year+"年"):save_to_mongoDB(data)