python爬虫之百度贴吧爬去图片

目标获取百度贴吧中帖子详情里面的图片,并下载到本地

"""

  step1：分析贴吧中分页的url地址规律,要根据url构造请求
  https://tieba.baidu.com/f?ie=utf-8
  &kw=%E7%BE%8E%E5%A5%B3&pn=0

  https://tieba.baidu.com/f?kw=%E7%BE%8E%E5%A5%B3
  &ie=utf-8&pn=50

  https://tieba.baidu.com/f?kw=%E7%BE%8E%E5%A5%B3
  &ie=utf-8&pn=100

  https://tieba.baidu.com/f?kw=%E7%BE%8E%E5%A5%B3
  &ie=utf-8&pn=150

  step2:获取分页中帖子详情的url地址,要根据url构造请求

  step3:从帖子情页面中总获取图片地址,要根据url构造请求

"""

  from urllib import parse,request
  import re

  def tiebaSpider(name,start_page,end_page):

      for page in range(start_page,end_page+1):
          #https://tieba.baidu.com/f?      kw=%E7%BE%8E%E5%A5%B3&ie=utf-8&pn=150
    parmars = {
        'kw':name,
        'ie':'utf-8',
        'pn':(page-1)*50
    }
    #将字典类型的参数,转换为url编码格式的字符串
    result = parse.urlencode(parmars)
    #拼接完整的url地址
    full_url = 'https://tieba.baidu.com/f?'+result
    #根据分页的url地址发起请求,得到响应结果,提取html页面源码
    html = load_data(full_url)
    #从页面源码中匹配出帖子详情的url地址
    tiezi_urlinfo = parse_page_detail_url(html)
    for note in tiezi_urlinfo:
        #https://tieba.baidu.com/p/5981722687
        #帖子的详情地址
        detail_url = 'https://tieba.baidu.com'+note[0]
        #帖子的标题
        title = note[1]
        print('正在获取'+title+'的帖子详情')
        #根据梯子详情的url地址发起请求,获取到页面源码
        html = load_data(detail_url)
        #从帖子详情中,提取图片的url地址
        images = parse_detail_imageurl(html)
        #下载图片
        download_image(images)

  def load_data(url):
#设置请求头
      req_header = {
          'User-Agent':'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0',
      }
      #构造一个request对象
      req = request.Request(url,headers=req_header)
      #发起请求
      response = request.urlopen(req)

      if response.status == 200:

          return response.read().decode('utf-8','ignore')

  def download_image(images):
      """
      根据图片的url地址发起请求,获取图片的二进制数据,进行本地存储
      :param images:
      :return:
      """
      for image_url in images:
          # 设置请求头
          req_header = {
              'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0',
          }
          # 构造一个request对象
          req = request.Request(image_url, headers=req_header)
          # 发起请求
         response = request.urlopen(req)

          if response.status == 200:

              filename = response.url[-20:]

              with open('tiebaprcture/'+filename,'wb') as file:

                  file.write(response.read())

                  print(filename,'下载完成')

  def parse_page_detail_url(html):
      """
      使用正则，从每一个分页的html页面源码中,提取帖子详情的url地址
      :param html: 每一个分页页面源码
      :return:
      """
      # pattern = re.compile(
      #     '<div\sclass="threadlist_title pull_left j_th_tit ">'+
      #     '.*?<a.*?href="(.*?)".*?</div>',re.S
      # )
      pattern = re.compile(
          '<div\sclass="threadlist_title pull_left j_th_tit ">' +
          '.*?<a.*?href="(.*?)".*?>(.*?)</a>.*?</div>', re.S
      )
      result = re.findall(pattern,html)
      #print(result)
      return result

  def parse_detail_imageurl(html):
      """
      根据正则从帖子详情的html页面源码中,提取图片地址
      :param html:
      :return:
      """
      pattern = re.compile('<img.*?class="BDE_Image".*?src="(.*?)".*?>',re.S)
      result = re.findall(pattern,html)
      print('图片链接',result)
      return result

  if __name__ == '__main__':
      # 输入贴吧的名称
      name = input('输入贴吧名称:')
      # 起始页
      start_page = int(input('输入起始页:'))
      # 截止页
      end_page = int(input('输入截止页:'))

      tiebaSpider(name,start_page,end_page)