很简单地一个爬取程序,适合初学者
源码如下:
from urllib.request import urlopen
from bs4 import BeautifulSoup
import pymysql
html = urlopen("http://www.jianshu.com")
bsobj = BeautifulSoup(html,"html.parser")
# print(bsobj.findAll("h4",{"class":"title"}))#打印获取的对象
SqlConnect = pymysql.connect(host = 'localhost',user = 'root',password = '123456',db = 'liusenTestURL',charset = 'utf8mb4')
cur = SqlConnect.cursor()#获取一个游标
#写入数据库函数
def writeDataBase(title,content,textURL):
cur.execute("INSERT INTO jianshuTEXT (title,content,URL) VALUES (%s,%s,%s)", (title, content,textURL))
cur.connection.commit()
#获取内容函数
def gainContent(contentHtml):
contenthtml = urlopen(contentHtml)
contentBsObj = BeautifulSoup(contenthtml,"html.parser")
textTitle = contentBsObj.find('title').get_text()
print('title : '+textTitle)
print('----------------------')
textContent = contentBsObj.find("div",{"class":"show-content"}).get_text()
# print(textContent)
writeDataBase(textTitle,textContent,contentHtml)
try:
for title in bsobj.find("ul", {"class": "article-list thumbnails"}).findAll("h4", {"class": "title"}):
# print(title.find("a"))
if 'href' in title.find("a").attrs:
contenthtml = 'http://www.jianshu.com' + title.find("a").attrs['href']
print(contenthtml)
gainContent(contenthtml)
finally:
cur.close()
SqlConnect.close()
欢迎一起交流学习
有时候网页编码不是utf-8,这就不太好弄了.假如现在第三方请求库用的是requests,那么请求下来的数据要做一个转化过程,针对gb2312网页编码,现在要做如下处理,否则会中文乱码
detailURL = "http://xxx.xxx.xxxxxx.com/"
html = requests.session().get(detailURL, headers=headers)
jieshouText = html.text.encode('ISO-8859-1',"ignore").decode(requests.utils.get_encodings_from_content(html.text)[0],"ignore")
参考:python的requests类抓取中文页面出现乱码
http://www.zhetenga.com/view/python的requests类抓取中文页面出现乱码-0abbaa140.html
解释很详细
网友评论