在进行接口测试等工作中，很多页面访问需要权限，这些权限管理字段一般存储于header里面，已cookie、token等形式存在，因此经常需要在url1中发送请求获得这些许可证，在后面的接口测试中利用许可证构造header继续进行请求以下是一个举例，是构造header在爬虫中的应用。

coding=utf-8

http://699pic.com/download/getDownloadUrl----获取下载URL地址的接口接收pid:500472407

返回包含图片链接URL=http://down.699pic.com/photo/50047/2407.jpg?_upt=63305cd11514965673&_upd=500472407.jpg

upd分解成两块填入 _upt为实时生成，具有时间使用限制 #图片id

如何获得_upd?

http://699pic.com/sousuo-61847-0-1-0-0-0.html第一页#keyword 如何转化为61847---页面代码中可以找到

http://699pic.com/sousuo-61847-0-2-0-0-0.html第二页

http://699pic.com/sousuo-61847-0-3-0-0-0.html第三页

多页面获取只要改变 这个数字，数字在html代码中可寻找到

_upd在网页元素中html代码中可以找到拼接即可

问题转化为keyword---五位数字的对应关系如何生成的

import requests
import time
import multiprocessing#多进程
from bs4 import BeautifulSoup#用于处理html文本，可以树状解析，方便查找和拆分
import sys
import io
from urllib import request#用于模拟登陆请求，携带登陆cookie信息进行访问
import json
import os
import random

sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')#改变标准输出的默认编码

class SpiderForPicture(object):
author = 'Blokks'

def __init__(self, keyword):
    self.keyword = keyword

def saving_folder_making(self):
    folder_path = 'F:\\test_auto\\spider\\pictures\\' + self.keyword
    if not os.path.exists(folder_path):
        os.mkdir(folder_path)
        print('创建名字为%s的目录存放图片' % self.keyword)
    return folder_path

def get_page_count(self):
    try:
        keyword = self.keyword
        url = 'http://699pic.com/tupian/' + keyword + '.html'
        html = requests.get(url)
        content = (html.content).decode('utf-8')
        re_1 = BeautifulSoup(content, "lxml")
        re_2 = re_1.find_all(name='div', attrs={'class': 'pager-linkPage'})
        re_3 = re_2[0].find_all(name='a')
        list_ = []
        list_result = []
        result_dict = {}
        for item in re_3:
            ls = (item.get('href')).split('-')
            list_.append(ls)
            list_result.append(int(ls[3]))
        page_count = str(max(list_result))
        key_number = str(list_[0][1])
        result_dict[key_number] = page_count
        return result_dict#取得对应数字kw和页码数
    except:
        print('搜索关键字未找到图片...')
        exit(1)

def get_pic_id(self):
    pic_id_list = []
    kw_dict = self.get_page_count()
    list_ = []
    for i in kw_dict:
        list_.append(i)
        list_.append(kw_dict[i])
    page_count = list_[1]
    print('根据关键字%s一共搜索到图片%s张' % (self.keyword, page_count))
    key_number = list_[0]
    for num in range(1, int(page_count)+1):
        url = 'http://699pic.com/sousuo-'+key_number+'-0-'+str(num)+'-0-0-0.html'
        html = requests.get(url)
        content = (html.content).decode('utf-8')
        re_1 = BeautifulSoup(content, "lxml")
        re_2 = re_1.find_all(name='div', attrs={'class': 'list'})
        for item in re_2:
            pic_id_list.append(item.get('data-id'))
    for i in pic_id_list:
        if len(str(i)) < 9:
            pic_id_list.remove(i)
    return pic_id_list

def get_download_url(self):
    pic_id_list = self.get_pic_id()
    url_pool = []
    for pic_id in pic_id_list:
        url = 'http://699pic.com/download/getDownloadUrl?pid=' + pic_id
        cookie_str = r'2017endalert=1; uniqid=5a4c7bd11a363; bargain_popup=1; uv_cookie=c610bdc8d6965b2e7abec5d93' \
                     r'd07ad59; is_click_activity=1; from_data=YTo1OntzOjQ6Imhvc3QiO3M6MTA6IjY5OXBpYy5jb20iO3M6Mzoi' \
                     r'c2VtIjtiOjA7czoxMDoic291cmNlZnJvbSI7aTowO3M6NDoid29yZCI7TjtzOjM6ImtpZCI7aTowO30%3D; isVip=0; ' \
                     r'isPay=0; is_qy_vip=1; is_join_2017_end_18454014=0; isSearch=0; s_token=03e987b8c9b7912d89e77b' \
                     r'b7fd9b62e8; PHPSESSID=kt1v9k8sid51kg0ej6e127cvkvgmpc7q; Qs_lvt_135734=1513923395%2C1513923542' \
                     r'%2C1514961873%2C1515026629%2C1515031146; mediav=%7B%22eid%22%3A%22278616%22%2C%22ep%22%3A' \
                     r'%22%22%2C%22vid%22%3A%22%5EySs)9Ku%25D%3A*qX%24(Pe%3FD%22%2C%22ctn%22%3A%22%22%7D; ' \
                     r'Hm_lvt_1154154465e0978ab181e2fd9a9b9057=1515026630,1515026702,1515031028,1515031147; ' \
                     r'Hm_lvt_ddcd8445645e86f06e172516cac60b6a=1515026629,1515026702,1515031028,1515031147; ' \
                     r'recentlysearch=YTo0OntpOjA7YToyOntzOjI6Imt3IjtzOjc6ImRpYW5uYW8iO3M6NjoicGlueWluIjtzOjY6IjMx' \
                     r'MTExMCI7fWk6MTthOjI6e3M6Mjoia3ciO3M6Njoi55S16ISRIjtzOjY6InBpbnlpbiI7czo3OiJkaWFubmFvIjt9aTo' \
                     r'yO2E6Mjp7czoyOiJrdyI7czoxMjoi5pm66IO95a625bGFIjtzOjY6InBpbnlpbiI7czoxMjoiemhpbmVuZ2ppYWp1Ij' \
                     r't9aTozO2E6Mjp7czoyOiJrdyI7czo2OiLlpKfmtbciO3M6NjoicGlueWluIjtzOjU6ImRhaGFpIjt9fQ%3D%3D; ' \
                     r'search_Kw=%22diannao%22; is_join_2017_end_533435=0; Qs_pv_135734=144824772440290620%2C38906' \
                     r'64247893633500%2C3737559667568741000%2C2243149228815513300%2C1985644855545767200; ' \
                     r'Hm_lpvt_1154154465e0978ab181e2fd9a9b9057=1515034556; Hm_lpvt_ddcd8445645e86f06e172516cac60' \
                     r'b6a=1515034556; redirect=http%3A%2F%2F699pic.com%2Ftupian-500472175.html; session_data=YTo1' \
                     r'OntzOjM6InVpZCI7czo2OiI1MzM0MzUiO3M6NToidG9rZW4iO3M6MzI6ImZkZDIyZWY5NDJlMjY3NjViYTdhMGE2NmY' \
                     r'4NzVmMTE3IjtzOjM6InV1dCI7czozMjoiMWM0Y2E4ZDZmMDRhYTdhYmJiNTNkNTkwZmI4MGJiMWMiO3M6NDoiZGF0YS' \
                     r'I7YToxOntzOjg6InVzZXJuYW1lIjtzOjEyOiLku5nlpbPlprnlprkiO31zOjY6ImV4dGltZSI7aToxNTE1NjM5MzgzO' \
                     r'30%3D; uid=533435; username=%E4%BB%99%E5%A5%B3%E5%A6%B9%E5%A6%B9; head_pic=http%3A%2F%2' \
                     r'Fq.qlogo.cn%2Fqqapp%2F101268598%2FD2C2DF0668D1C9B957ADD345B9B7A420%2F40; login_user=1'
        req = request.Request(url)
        req.add_header('Cookie', cookie_str)
        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36')
        resp = request.urlopen(req)
        result_ = resp.read().decode('utf-8')
        result_dict = json.loads(result_)
        if not 'url' in result_dict:
            print('cookies失败o(╥﹏╥)o')
            exit(1)
        download_url = result_dict['url']
        url_pool.append(download_url)
    return url_pool

def download_picture(self, url):
    file_name = self.keyword + str(random.randint(100000, 999999)) + '.jpg'
    folder_path = self.saving_folder_making()
    file_path = folder_path + '\\' + file_name
    resp = requests.get(url)
    content = resp.content
    with open(file_path, 'wb') as f:
        f.write(content)

def main():
start_time = time.time()
keyword = input('请输入需要搜索的关键字(拼音)：')
spider = SpiderForPicture(keyword)
middle_time = time.time()
time_cost = middle_time - start_time
url_pool = spider.get_download_url()
print('下载地址解析完毕---用时%s---现在开始下载....' % time_cost)
p = multiprocessing.Pool(processes=4)
p.map(spider.download_picture, url_pool)
p.close()
p.join()
end_time = time.time()
time_used = end_time - start_time
print('全部下载完毕，用时%s' % time_used)

if name == 'main':
main()

可以看到上面代码中cookie贼长，在本例子中使用add_header进行请求头构造完成后面的请求需要。