先上图:
scrapy框架爬取某表情网站表情图【源码+GIF表情包下载】
python源代码
mport scrapy
import os,sys
import requests
import re
class scrapyone(scrapy.Spider):
name = "stackone"
start_urls = ["http://qq.yh31.com/ql/bd/"]
def parse(self,response):
hrf=response.xpath('//*[@id="main_bblm"]/div[2]/dl/dd/li')
for li in hrf:
item={}
href=li.xpath('a/@href').extract()
hreftext=li.xpath('a/text()').extract()
full_url = 'http://qq.yh31.com'+ ''.join(list(href))
hreftext=''.join(list(hreftext))
#文件夹名称
if hreftext=='>更多>':
continue
path = 'C:\GIF'
if not os.path.exists(path):
os.makedirs(path)
item['dirname']=hreftext
yield scrapy.Request(url=full_url,meta={'key':item},callback = self.parse1)
def parse1(self,response):
ite={}
full_url=[]
url1 = response.xpath('//*[@id="pe100_page_infolist"]/a[2]/@href').extract()
url2 = response.xpath('//*[@id="pe100_page_infolist"]/a[2]/@href').re('\d+')
url1 = ''.join(url1)
url1 = url1.split('_')
url2 = ''.join(url2)
ite['dirn']=response.meta['key']['dirname']
for i in range(1,int(url2)+1):
full_url='http://qq.yh31.com'+url1[0]+'_'+str(i)+'.html'
#print(full_url)
yield scrapy.Request(url=full_url,meta={'key1':ite},callback = self.parse2)
def parse2(self,response):
p1=response.meta['key1']['dirn']
resp = response.xpath('//*[@id="main_bblm"]/div[1]/li/dt/a')
path = 'C:\GIF\\'+''.join(p1)
for lst in resp:
alt = lst.xpath('img/@alt').extract()
src = lst.xpath('img/@src').extract()
src = 'http://qq.yh31.com'+ ''.join(list(src))
alt = ''.join(list(alt))
html=requests.get(src)
with open(path+'\\'+alt+'.gif', 'wb') as file:
file.write(html.content)
脚本执行方式:cmd-->切换到脚本所在目录-->scrapy runspider xxxx.py
执行后会自动根据GIF分类在c:\gif文件夹下建立相应文件夹存储gif图片。python学习交流裙588+090+942
这里还是要推荐下小编的Python学习裙:【五 八 八,零 九 零,九 四 二】不管你是小白还是大牛,小编我都欢迎,不定期分享干货,包括小编自己整理的一份2018最新的Python资料和0基础入门教程,欢迎初学和进阶中的小伙伴。在不忙的时间我会给大家解惑。
网友评论