贴出代码以供大家共享，其实爬虫很简单，难的是突破反爬虫，今天看了一些反爬虫的资料，明天或后天整理一下，写上一篇文章。
代码如下：
import requests
from bs4 import BeautifulSoup
import os,time,random
from selenium import webdriver
user_agent_list = [
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) App leWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53"
]
baseurl='http://www.66bb.org'
href_list=[]
name_list=[]
UA=random.choice(user_agent_list)#获取随机的User_Agent
headers={'User_Agent':UA}
proxies = {"http": "http://119.5.0.58:808"}
proxies = {"http": "http://219.217.80.182:1080"}
html=requests.get(baseurl,headers=headers)
html.encoding='gb2312'
Soup=BeautifulSoup(html.text,'lxml')
href_all=Soup.find('ul',class_='topline_3b').findAll('li')
for i in range(0,len(href_all)):
lanmu_href=href_all[i].find('a')['href']
lanmu_name=href_all[i].find('a').get_text()
href_list.append(lanmu_href)
name_list.append(lanmu_name)
for lanmu_num in range(0,len(href_list)):
baseurl=baseurl+href_list[lanmu_num]
start_num=0
start_page=int(start_num/20)+1
start_image=int(start_num%20)
#看了看，一共有6页
for i in range(start_page,15):
#获取每页的链接和名称
if i != 1:
urllink=baseurl+'list4'+str(i)+'.html'
else:
urllink=baseurl
#获取第X页的第XX个链接地址和文件夹名称
UA=random.choice(user_agent_list)#获取随机的User_Agent
headers={'User_Agent':UA}
#proxies = {"http": "http://119.5.0.58:808"}
#proxies = {"http": "http://219.217.80.182:1080"}
html=requests.get(urllink,headers=headers)
html.encoding='gb2312'
Soup=BeautifulSoup(html.text,'lxml')
all_url=Soup.find(class_="fzltp").findAll('li')
#所有的dd标签都是图册，而dt是栏目头
print(u'第',i,u'页共包含有',len(all_url),u'个图册，开始下载')
print(all_url[0])
print(all_url[0].find('a'))
#开始提取图册的链接地址和帖子名称
for num in range(start_image,len(all_url)):
#获取地址，连接后解析html，主要目的是为了获取图片总数
href=all_url[num].find('a')['href']
href1='http://www.66bb.org'+href
#http://www.66xx.org/ArtDD/
UA=random.choice(user_agent_list)#获取随机的User_Agent
headers={'User_Agent':UA}
page1html=requests.get(href1,headers=headers)
page1html.encoding='gb2312'
page1soup=BeautifulSoup(page1html.text,'lxml')
total_pages_1=page1soup.find('div',class_='tpm01').find('font',color='blue').get_text()
total_pages_1=int(total_pages_1[-3:-1])
print(u'共有',total_pages_1,u'页图片需要保存')
#获取文件夹名字
dirname=all_url[num].find('img')['alt']
#建立文件夹，打开子文件夹
os.chdir("D:\\")
path = name_list[lanmu_num]+'\\'+dirname+'('+str(total_pages_1)+'P)'
isExists = os.path.exists(os.path.join("D:\\66人体艺术图片\\",path))
if not isExists:
print(u'建了一个名字叫做', path, u'的文件夹！')
os.makedirs(os.path.join("D:\\66人体艺术图片\\", path))
else:
print(u'名字叫做', path, u'的文件夹已经存在了！')
continue
#跳出本次循环
os.chdir("D:\\66人体艺术图片\\"+path)
#basename=imgsrc1[-14:-5]
#寻找所有的图片地址，打开图片网页，保存图片
#最后一个页面的网页打不开，不是个例，应该是个普遍现象
for i in range(1,total_pages_1+1):
baseurl_2=href1+str(i) +'.html'
html_2=requests.get(baseurl_2,headers=headers)
html_2.encoding='gb2312'
soup_2=BeautifulSoup(html_2.text,'lxml')
#提取主图的地址和名字
image_href=soup_2.find('div',class_='imgbox').find('img')['src']
image_href='http://www.66bb.org'+image_href
#image_name=soup_2.find('div',class_='imgbox').find('img')['alt']
#image_name=image_name[-3:]+'.jpg'
image_name=str(i)+'.jpg'
#这里如果普通的请求总是中断，可以使用selenium来操纵chrome和PhantomJS
#browser=webdriver.Chrome()
#browser = webdriver.PhantomJS(executable_path="phantomjs.exe")
#browser.get(image_href)
time.sleep(1)
#imghtml=browser.page_source #这个函数获取页面的html
UA=random.choice(user_agent_list)#获取随机的User_Agent
headers={'User_Agent':UA}
try:
imgrealhtml=requests.get(image_href,headers=headers)
except:
time.sleep(5)
print(u'请稍等，5秒后重连')
UA=random.choice(user_agent_list)#获取随机的User_Agent
headers={'User_Agent':UA}
imgrealhtml=requests.get(image_href,headers=headers)
f = open(image_name,'ab')
f.write(imgrealhtml.content)
f.close()
print(u'第',i,u'张图片保存完毕')
time.sleep(1)
#browser.quit()
print(u'共有',total_pages_1,u'张图片保存完毕')
网友评论