使用多线程获取某招聘网站的信息，使用环境py3，话不多说直接上代码
该导的包你可就不能少了
import threading
import requests
from pyquery import PyQuery as pq
import json
from pymongo import MongoClient
from fake_useragent import UserAgent
import time
from config import *
我这里用到的是Mongodb储存的，fake_useragent是一个‘User-Agent’一个共享库，看着用，我这边写的是一个配置文件，大家可以看将**文件写自己的文件。
client = MongoClient(().mogodb()["IP"],().mogodb()["port"])
u = UserAgent()
class data():
def __init__(self,startpage,page,citynum,hangyenum):self.startpage=startpageself.page=pageself.citynum=citynumself.hangyenum=hangyenum
def datature(self):if self.startpage==0:self.startpage=1for i in range(self.startpage,self.page):try:headers = {'User-Agent': u.random,}yu = requests.get('这里是一个代理url').textip = yu.split(':')[0]port = yu.split(':')[1]proxy_http = "http://{0}:{1}".format(ip, port)proxy_dict = {"http": proxy_http}print(proxy_dict)url='https://fe-api.zhaopin.com/c/i/sou?start={0}&pageSize=60&cityId={1}&industry={2}&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kt=3&_v=0.64455588&x-zp-page-request-id=2fb2f4e53284471aa21894bf291db027-1543801568582-396367'.format(str(i*60),self.citynum,self.hangyenum)print(self.startpage,self.page)r=requests.get(url,headers=headers,proxies=proxy_dict)dict1=json.loads(r.text)for j in dict1['data']['results']:href='https://jobs.zhaopin.com/'+j['number']+'.htm'href=j['positionURL']print(href)r1 = requests.get(href, headers=headers,proxies=proxy_dict)doc = pq(r1.text)title = doc('.new-info h1.l.info-h3').text()xinzhi = doc('.new-info div.l.info-money strong').text()company = doc('li.clearfix div.company.l a').text()jianjie = doc('li.clearfix div.info-three.l').text()zhiweixinxi = doc('div.l.pos-info-in div.pos-info-tit p.r ').text()zhiweimiaoshu = doc('div.responsibility.pos-common div.pos-ul').text()gongsigaikuang = doc('div.intro-content div[align="left"]').text()lianxifangshi = doc('div.intro-content div[style="font-family: 宋体;"]').text()list1=jianjie.split(' ')try:list2=zhiweixinxi.split(' ')[0]except:list2=''try:jianjie1=list1[0]jianjie2=list1[1]jianjie3=list1[2]jianjie4=list1[3]except:passif title!=""and len(list1)==4:db = client.自己的数据库名1# 创建数据库名post = {'title': title,'xinzhi': xinzhi,'conmpany': company,'dizhi': jianjie1,'jingyan':jianjie2,'xueli':jianjie3,'zhaopinrenshu':jianjie4,'zhiweixinxi': list2,'zhiweimiaoshu': zhiweimiaoshu,'gongsigaikuang': gongsigaikuang,'lianxifangshi': lianxifangshi,'html':href}posts = db.zhaopin # 创建表名post_id = posts.insert_one(post).inserted_id # 创建id将数据提交else:passprint(i)self.startpage+=1except:print("请求次数过多或程序出错错误页：",self.startpage)self.startpage += 1self.datature()下面是使用多线程执行上面这段请求解析
def action(a,b,c,d):
start = data(int(a),int(b),c,d)
start.datature()
if __name__=="__main__":
while True:print('88888888此程序启动为默认程序8888888888')for i in range(26):citynum = zhilianzhaopin().sheng(i)["citynum"]for j in range(12):hangyenum = zhilianzhaopin().hangye(j)["hangyenum"]print('点击查看多少页,默认程序是101页：', end='')print('https://sou.zhaopin.com/?jl={0}&in={1}'.format(citynum, hangyenum))page = zhilianzhaopin().zongyeshu(101)[1]time.sleep(5)fenyepage = int(int(page) / 10)duandianpage = fenyepage * 10if fenyepage!=0:for i in range(11):if i <= 9:t = threading.Thread(target=action, args=(i * fenyepage, (i + 1) * fenyepage,citynum,hangyenum))else:t = threading.Thread(target=action, args=(duandianpage, page,citynum,hangyenum))t.start()elif fenyepage==0:action(1, page,citynum,hangyenum)
这下面是配置文件，可供参考：
class zhilianzhaopin():
def mogodb(self):a={}#mongodbIpIP=''#mongodb端口号port=a["IP"]=IPa["port"]=portif a["IP"]!="" and a["port"]!="" :return aelse:print("配置文件mongodb参数缺失")
#省会名
def sheng(self,cityxuhao):list1={0: '北京', 1: '上海', 2: '深圳', 3: '广州', 4: '天津', 5: '成都', 6: '杭州', 7: '武汉', 8: '大连', 9: '长春', 10: '南京', 11: '济南',12: '青岛', 13: '苏州', 14: '沈阳', 15: '西安', 16: '郑州', 17: '长沙', 18: '重庆', 19: '哈尔滨', 20: '无锡', 21: '宁波', 22: '福州',23: '厦门', 24: '石家庄', 25: '合肥', 26: '惠州'}a='''0: '北京', 1: '上海', 2: '深圳', 3: '广州', 4: '天津', 5: '成都', 6: '杭州', 7: '武汉', 8: '大连', 9: '长春' '''b=''' 10: '南京', 11: '济南',12: '青岛', 13: '苏州', 14: '沈阳', 15: '西安', 16: '郑州', 17: '长沙', 18: '重庆', '''c='''19: '哈尔滨', 20: '无锡', 21: '宁波', 22: '福州',23: '厦门', 24: '石家庄', 25: '合肥', 26: '惠州' '''# print('%s\n%s\n%s'%(a,b,c))# try:# cityxuhao=int(input('输入序号，整数：'))# except:# print('输入错误！！！')# self.sheng()#城市名city=list1[cityxuhao]print(city)#城市编码dict1={}citynum={'北京':'530','上海':'538','深圳':'765','广州':'763','天津':'537','成都':'801','杭州':'653','武汉':'736','大连':'600','长春':'631','南京':'635','济南':'702','青岛':'703','苏州':'639','沈阳':'599','西安':'854','郑州':'719','长沙':'749','重庆':'551','哈尔滨':'622','无锡':'636','宁波':'654','福州':'681','厦门':'682','石家庄':'515','合肥':'664','惠州':'773'}dict1['citynum']=citynum[city]if dict1['citynum']!="":return dict1else:print("配置文件省会名参数出错")#行业
def hangye(self,hangyexuhao):#参考行业list1={0: '互联网/IT', 1: '金融', 2: '房地产/建筑', 3: '商业服务', 4: '贸易/批发/零售', 5: '教育/艺术', 6: '服务业', 7: '文化/传媒/娱乐', 8: '制造业', 9: '物流运输', 10: '能源/环保', 11: '政府/非盈利', 12: '农林牧渔'}a=''' 0: '互联网/IT', 1: '金融', 2: '房地产/建筑', 3: '商业服务', 4: '贸易/批发/零售', 5: '教育/艺术', 6: '服务业', 7: '文化/传媒/娱乐', 8: '制造业', 9: '物流运输' '''b=''' 10: '能源/环保', 11: '政府/非盈利', 12: '农林牧渔' '''# print('%s\n%s'%(a,b))# hanyexuhao=hanyexuhao# try:# hangyexuhao=int(input('输入对应序号：'))# except:# print("输入错误!!!")# self.hangye()dict1={}hangye = list1[hangyexuhao]print(hangye)# 行业编码hangyenum = {"互联网/IT": "10100", "金融": "10200", "房地产/建筑": "10800", "商业服务": "10900", "贸易/批发/零售": "10300","教育/艺术": "10400", "服务业": "10000", "文化/传媒/娱乐": "11300", "制造业": "10500", "物流运输": "11500","能源/环保": "11600", "政府/非盈利": "11100", "农林牧渔": "11400"}dict1["hangyenum"] = hangyenum[hangye]if dict1["hangyenum"]!="":return dict1else:print("配置文件行业中参数缺失")#总页数
def zongyeshu(self,page):startpage=1try:# page=int(input('输入总页数：'))page=int(page)except:print('输入有误！！！')self.zongyeshu()if page!="":return startpage,pageelse:print("配置文件总页数缺失")
有啥可以帮助的可以联系1307761253
