parese的简单用法
- urlencode()函数:帮我们将key:value这样的键值对转换**"key=value"这样的字符串（将字典键值对按URL编码转换，从而能被web服务器接受）
- parse的unquote():把 URL编码字符串，转换回原先字符串
- parse.quote():把字符串（不是字典）转乘url编码
from urllib import request,parse
wd = {
"kw":"你好"
}
aa = "你好"
a = parse.quote(aa)
b = parse.unquote(a)
c = parse.urlencode(wd)
d = parse.unquote(c)
print(a)
print(b)
print(c)
print(type(d))
print(d)
"""
%E4%BD%A0%E5%A5%BD
你好
kw=%E4%BD%A0%E5%A5%BD
<class 'str'>
"""
一般HTTP请求提交数据，需要编码成 URL编码格式，然后做为url的一部分，或者作为参数传到Request对象中。
Get方式
GET请求一般用于我们向服务器获取数据，比如说，我们用百度搜索美女：https://www.baidu.com/s?wd=美女
最终中文被转换为十六进制的数据
https://www.baidu.com/s?wd = %E7%BE%8E%E5%A5%B3
在其中我们可以看到在请求部分里,http://www.baidu.com/s? 之后出现一个长长的字符串，其实就是我们要查询的关键词美女，于是我们可以尝试用默认的Get方式来发送请求。
小例子
百度搜索
"""
https://www.baidu.com/s?wd=%E9%A9%AC%E4%BA%91&pn=10
"""
from urllib import parse,request
from fake_useragent import UserAgent
def searchSpider(kw,start_page,end_page):
for page in range(start_page,end_page+1):
# 将字典类型的参数转url编码
zidian = {
"wd": kw,
"pn": (page-1)*10
}
resule = parse.urlencode(zidian)
print(resule)
full = "https://www.baidu.com/s?"+resule
print(full)
html = load_page(full)
filename = "第"+str(page)+"页"+kw+".html"
save_page_html(html,filename)
def load_page(url):
req_heard = {
"User-Agent":ua.random
}
print(req_heard)
req = request.Request(url,headers=req_heard)
response = request.urlopen(req)
if response.status == 200:
print("请求成功")
return response.read().decode()
def save_page_html(html,filename):
with open("test/"+filename,"w",encoding="utf-8") as file:
file.write(html)
if __name__ == "__main__":
ua = UserAgent()
kw = input("请输入关键字")
#起始页和截至页
start_page = int(input("输入起始页"))
end_page = int(input("输入截止页"))
searchSpider(kw, start_page, end_page)
百度贴吧
#百度贴吧帖子图片，下载到本地
#分析帖子贴吧中分页的url地址规律
'''
https://tieba.baidu.com/f?ie=utf-8&kw=%E7%BE%8E%E5%A5%B3&fr=search
https://tieba.baidu.com/f?kw=%E7%BE%8E%E5%A5%B3&ie=utf-8&pn=50
https://tieba.baidu.com/f?kw=%E7%BE%8E%E5%A5%B3&ie=utf-8&pn=100
'''
import re
from fake_useragent import UserAgent
from urllib import parse,request
def tiebaSpider(name,start_page,end_page):
for page in range(start_page,end_page+1):
parmars = {
"kw":name,
"ie":"utf-8",
"pn":(page-1)*50
}
a = parse.urlencode(parmars)
full_url = "https://tieba.baidu.com/f?"+a
html = load_data(full_url)
#从页面源码中匹配出详情地址
tiezi_urlinfo = parse_page_detail_url(html)
for note in tiezi_urlinfo:
detail_url = "https://tieba.baidu.com"+note[0]
title = note[1]
# print("正在获取"+title+"的帖子详情")
html = load_data(detail_url)
images = parse_detail_imageurl(html)
download_image(images)
def load_data(url):
req_header = {
"User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
}
req = request.Request(url,headers=req_header)
response = request.urlopen(req)
if response.status == 200:
return response.read().decode()
#使用正则从每个分页的html页面源码中，提取帖子的详情url地址
def parse_page_detail_url(html):
pattern = re.compile('<div.*?class="threadlist_title pull_left j_th_tit ">' +
'.*?<a.*?href="(.*?)".*?>(.*?)</a>.*?</div>', re.S)
result = re.findall(pattern,html)
return result
def parse_detail_imageurl(html):
pattern = re.compile('<img.*?class="BDE_Image".*?src="(.*?)".*?>', re.S)
result = re.findall(pattern, html)
print("图片链接",result)
return result
def download_image(image):
for i in image:
req_header = {
"User-Agent": ua.random
}
req = request.Request(i, headers=req_header)
response = request.urlopen(req)
if response.status == 200:
flilename = response.url[-20:]
with open("img/"+flilename,"wb") as filr:
filr.write(response.read())
print(flilename,"下载完成")
if __name__== "__main__":
ua = UserAgent()
name = input("输入关键字")
start_page = int(input("输入起始页"))
end_page = int(input("输入结束页"))
tiebaSpider(name, start_page, end_page)
Post方式
上面我们说了Request请求对象的里有data参数，它就是用在POST里的，我们要传送的数据就是这个参数data，data是一个字典，里面要匹配键值对
发送post请求
#测试接口：https://httpbin.org/post
from urllib import parse,request
url = 'https://httpbin.org/post'
#表单数据
fordata = {
'name':'红红火火',
'age':18,
'gender':'男'
}
#先使用urlencode将参数转为url编码格式的字符串,
# 然后在使用encode()方法将字符串转为bytes类型的参数
formdata = parse.urlencode(fordata).encode('utf-8')
#不需要添加请求头
response = request.urlopen(url,data=formdata)
print(response.status)
print(response.read().decode('utf-8'))
#需要设置请求头
red_header = {
'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}
req = request.Request(url,headers=red_header,data=formdata)
response = request.urlopen(req)
print(response.status)
print(response.read().decode('utf-8'))
有道
from urllib import parse,request
import json
# POST请求的目标URL（这个代码是之前的链接，方便我们使用，不用传递sign参数，新版中该参数是加密的）
url = "http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule&smartresult=ugc&sessionFrom=null"
formdata = {
'i': '我的祖国',
'from': 'AUTO',
'to': 'AUTO',
'smartresult': 'dict',
'client': 'fanyideskweb',
'doctype': 'json',
'version': '2.1',
'keyfrom': 'fanyi.web',
'action': 'FY_BY_CLICKBUTTION',
'typoResult': 'false',
}
formdata = parse.urlencode(formdata).encode('utf-8')
req_header = {
'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}
req = request.Request(url,headers=req_header,data=formdata)
response = request.urlopen(req)
print(response.status)
# print(response.read().decode('utf-8'))
json_str = response.read().decode('utf-8')
print(json_str)
# json.loads():将json字符串,转换为python的数据类型；
# json类型的数据（对象和数组）对象－＞dict 数组->list
# json.load()
# json.dumps()
# json.dump()
data = json.loads(json_str)
print(type(data))
result = data['translateResult'][0][0]['tgt']
print(result)
拉钩
#
#目标url:
# https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false
# post请求要提交的表单数据
# first: true
# pn: 1 （页码）
# kd: c++ (关键字)
from urllib import request,parse
import json,pymysql,time
#pip3 install pymysql
def lagouspider(url,formdata):
#发起请求返回响应结果
response_data = load_page_data(url,formdata)
#得到的数据是一个json字符串,需要转为python类型的数据
data = json.loads(response_data)
print(data)
if data['success']:
print('请求成功')
#拿到职位信息
postionJobs = data['content']['positionResult']['result']
for jobinfo in postionJobs:
jobdata = {}
#职位标题
jobdata["companyId"] = jobinfo["companyId"]
jobdata['positionName'] = jobinfo['positionName']
#职位的发布时间
jobdata['publishTime'] = jobinfo['formatCreateTime']
#公司名称
jobdata['companyName'] = jobinfo['companyShortName']
#薪资
# jobdata['salary'] = jobinfo['salary']
# #工作经验
# jobdata['workYear'] = jobinfo['workYear']
# #学历
# jobdata['education'] = jobinfo['education']
# #公司类型
# jobdata['industry'] = jobinfo['industryField']
# #融资
# jobdata['stage'] = jobinfo['financeStage']
# #人数
# jobdata['companySize'] = jobinfo['companySize']
# #福利
# jobdata['fuli'] = ','.join(jobinfo['companyLabelList'])
# #招聘诱惑
# jobdata['positionAdvantage'] = jobinfo['positionAdvantage']
save_data_to_db(jobdata)
#print(jobdata)
#判断是否需要发起下一次请求
#取出当前页码
cur_page = int(data['content']['pageNo'])
#每页返回多少条数据
page_size = int(data['content']['pageSize'])
#职位总数
totalcount = int(data['content']['positionResult']['totalCount'])
if cur_page*page_size < totalcount:
#下一页的页码
next_page = cur_page+1
print('继续发起请求第'+str(next_page)+'页')
formdata['pn'] = next_page
time.sleep(1)
lagouspider(url,formdata)
else:
print('请求不成功,休息一会继续发起请求')
time.sleep(10)
print('重新发起第'+str(formdata['pn'])+'页请求')
lagouspider(url,formdata)
def load_page_data(url,formdata):
"""
发起请求(下载器)
:param url:
:param formdata:
:return:
"""
#将表单数据转为web服务器能识别的url编码格式的bytes类型的数据
form_data = parse.urlencode(formdata).encode('utf-8')
#设置请求头
req_headers = {
'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
'Referer':'https://www.lagou.com/jobs/list_c%2B%2B?labelWords=&fromSearch=true&suginput=',
}
#构建一个request对象
req = request.Request(url,headers=req_headers,data=form_data)
#发起请求
response = request.urlopen(req)
if response.status == 200:
return response.read().decode('utf-8')
def save_data_to_db(jobdata):
"""
存储数据
:param jobdata: 对应的是一个字典（存放的是职位信息）
:return:
"""
sql = """
INSERT INTO list_lagou(%s)
VALUES (%s)
""" % (
','.join(jobdata.keys()),
','.join(["%s"]*len(jobdata))
)
try:
cursor.execute(sql,list(jobdata.values()))
mysql_client.commit()
except Exception as err:
print(err)
mysql_client.rollback()
if __name__ == '__main__':
#数据库连接
"""
host=None, user=None, password="",
database=None, port=0,
charset=''
"""
mysql_client = pymysql.Connect(
'127.0.0.1','root','123456',
'test01',3306,charset='utf8',
)
#创建游标(执行sql语句)
cursor = mysql_client.cursor()
#目标url
url = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false'
#要提交的参数
formdata = {
'first': 'true',
'pn': 1,
'kd': 'c++',
}
lagouspider(url,formdata)
-
GET方式是直接以链接形式访问，链接中包含了所有的参数，服务器端用Request.QueryString获取变量的值。如果包含了密码的话是一种不安全的选择，不过你可以直观地看到自己提交了什么内容。
-
POST则不会在网址上显示所有的参数，服务器端用Request.Form获取提交的数据，在Form提交的时候。但是HTML代码里如果不指定 method 属性，则默认为GET请求，Form中提交的数据将会附加在url之后，以?分开与url分开。
网友评论