网站为:https://blog.csdn.net/
源码为:
header 在这里:
结果如下:
可复制代码:
import re
import urllib.request
url="https://blog.csdn.net/"
headers=("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36")
opener=urllib.request.build_opener()
opener.addheaders=[headers]
urllib.request.install_opener(opener) ##安装成为全局
data=opener.open(url).read().decode("utf-8","ignore")
pat='href="(https://blog.csdn.net/.*?)"'
allurl=re.compile(pat).findall(data)
for i in range(0,len(allurl)):
try:
print("第"+str(i)+"次爬取")
thisurl=allurl[i]
file="E:/py/csdn/"+str(i)+".html"
urllib.request.urlretrieve(thisurl,file)
print("----成功----")
except urllib.error.URLError as e:
if hasattr(e,"code"):
print(e.code)
if hasattr(e,"reason"):
print(e.reason)
网友评论