# 目标url: https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false
# post请求要提交的表单数据
from urllib import request,parse
import json,pymysql,time
def lagouspider(url,formdata):
# 发起请求返回响应结果
response_data = load_page_data(url,formdata)
# 得到一个json数据,需返回一个python类型的数据
data = json.loads(response_data)
if data['success']:
print('请求成功')
# 拿到职位信息
postionJobs = data['content']['positionResult']['result']
for jobinfo in postionJobs:
jobdata = {}
jobdata['positionName'] = jobinfo['positionName']
jobdata['publishTime'] = jobinfo['formatCreateTime']
jobdata['companyname'] = jobinfo['companyShortName']
jobdata['salary'] = jobinfo['salary']
jobdata['worYear'] = jobinfo['workYear']
jobdata['education'] = jobinfo['education']
jobdata['industry'] = jobinfo['industryField']
jobdata['stage'] = jobinfo['financeStage']
jobdata['companySize'] = jobinfo['companySize']
jobdata['fuli'] = ','.join(jobinfo['companyLabelList'])
jobdata['positionAdvantage'] = ','.join(jobinfo['positionAdvantage'])
# 存数据
save_data_to_db(jobdata)
# 判断是否需要发起下一次请求
# 取出当前页码
cur_page = int(data['content']['pageNo'])
# 每页多少条
page_size = int(data['content']['pageSize'])
#职位总数
totalcount = int(data['content']['positionResult']['totalCount'])
if cur_page*page_size < totalcount:
next_page = cur_page +1
print('继续发起请求第'+str(next_page)+'页')
formdata['pn'] = next_page
lagouspider(url,formdata)
else:
print('请求不成功,请稍后尝试')
time.sleep(10)
print('重新发起第'+formdata['pn']+'页请求')
lagouspider(url,formdata)
def load_page_data(url,formdata):
"""
发起请求
:param url:
:param formdata:
:return:
"""
# 将表单数据转为web服务器可以识别的url编码格式的bytes类型的数据
form_data = parse.urlencode(formdata).encode('utf-8')
req_header = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:64.0) Gecko/20100101 Firefox/64.0',
'Referer':'https://www.lagou.com/jobs/list_c%2B%2B?labelWords=&fromSearch=true&suginput=',
}
# 构建一个request对象
req = request.Request(url, headers=req_header,data=formdata)
# 根据Request对象发起请求
response = request.urlopen(req)
if response.status == 200:
return response.read().decode('utf-8')
def save_data_to_db(jobdata):
"""
存储数据
:param jobdata:
:return:
"""
sql = """
INSERT INTO lagou(%s)
VALUE (%s)
"""%(','.join(jobdata.keys()),','.join(['%s']*len(jobdata)))
try:
cursor.execute(sql,list(jobdata.values()))
mysql_client.commit()
except Exception as err:
print(err)
mysql_client.rollback()
if __name__ == '__main__':
# 创建数据库链接
"""
host=None, user=None, password="",
database=None, port=0, unix_socket=None,
charset=''
"""
mysql_client = pymysql.Connect('127.0.0.1','root','18603503110','1712B',3306,charset='utf8')
# 创建游标(执行mysql语句)
cursor = mysql_client.cursor()
# 目标
url = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false'
# 要提交的参数
formdata = {
'first':'true',
'pn':1,
'kd':'c++',
}
lagouspider(url,formdata)
网友评论