2019-06-04 淘宝评论爬取

直接附上代码：

import urllib.request
import urllib.parse
import jsonpath
import json
import re
import time

'''
ajax接口
https://rate.taobao.com/feedRateList.htm?auctionNumId=593014213362&userNumId=903511695&currentPageNum=2&pageSize=20

'''

items = []
def handle_request(url, page):
    url = url.format(page)
    # print(url)

    # 必须加COOKIE,不然json返回不回来
    headers = {

        'Accept': '*/*',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Referer': 'https://item.taobao.com/item.htm?spm=a230r.1.14.31.1a9df9eaZ2TtQ2&id=593014213362&ns=1&abbucket=4',

        'Cookie': 'miid=1034753504511266755;cna=dx1nFLT5mQcCAXWaR2QxwmBW;'

                  'thw=cn;t=325ff4e457909046382b22a33c83eadb;tracknick=acc%5Cu80E1%5Cu4F20;'

                  'lgc=acc%5Cu80E1%5Cu4F20;tg=0;v=0;'

                  'cookie2=128cea00303f12de470490c850cc00b1;_tb_token_=fdb4b335af83e;'

                  'unb=2701405241;sg=%E4%BC%A01a;_l_g_=Ug%3D%3D;skt=051d1e62c9ab8dc5;'

                  'cookie1=BxMH3NLGjomw6rJWdejLCTcGCFEkYwy%2BoQnBejFVBZI%3D;csg=93fdcb74;'

                  'uc3=vt3=F8dBy3jYcBxWmWeUeDY%3D;id2=UU8IPT1oZbApWA%3D%3D;nk2=AnIsGxSURQ%3D%3D;lg2=Vq8l%2BKCLz3%2F65A%3D%3D;'

                  'existShop=MTU1OTY1MDAxNw%3D%3D;_cc_=U%2BGCWk%2F7og%3D%3D;dnk=acc%5Cu80E1%5Cu4F20;_nk_=acc%5Cu80E1%5Cu4F20;'

                  'cookie17=UU8IPT1oZbApWA%3D%3D;mt=ci=6_1&np=;'

                  'enc=UFEUuOrXKTfwHDdKWP%2F9JacRMgPfTTQxKHxhkY8FExWK0%2FOKPqyryg048bj3QRjP%2Blwvl2%2BmU0N0pIBcrlD2PA%3D%3D;'
                  'hng=CN%7Czh-CN%7CCNY%7C156;'
                  'uc1=cookie15=VT5L2FSpMGV7TQ%3D%3D&cookie14=UoTZ7Yy7WSTGKg%3D%3D;'

                  '_m_h5_tk=3627342e3b09ae88a2c724b51c503d85_1559657695270;'
                  '_m_h5_tk_enc=c88d8e3f4ba68bc083f853e956656b6c;'
                  'x=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0%26__ll%3D-1%26_ato%3D0;'
                  'x5sec=7b22726174656d616e616765723b32223a22623034623263306164386231666234336634303330663463646430663064653743504c4f32656346454e364e6b736a586d664c7373774561444449334d4445304d4455794e4445374d513d3d227d;'
                  'whl=-1%260%260%261559652412550;'
                  'l=bB_eivGRvlAmYuqwBOfgiuI8a17TwIdfhsPzw4_G2ICP_bfDwBcPWZTPLZTkC3GVa6GXR3oyuHLBBW8iLyUCh;'
                  'isg=BGNjWHeLO7xJIPCeqwvJFNPV8qfN8Pf9ajjwFJXBBkM21IT2HSu26rnCyuT_9E-S',

        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',

        'Connection': 'keep-alive',
        'Host': 'rate.taobao.com',
    }
    request = urllib.request.Request(url=url, headers=headers)
    return request


def json_parse(json_content):
    # 将json格式两边的小括号干掉,使用正则替换方式
    json_content = re.sub(r'\(', '', json_content)
    json_content = re.sub(r'\)', '', json_content)

    # 方式二，使用字符串方式去掉小括号,将json格式两边的小括号和非字符干掉
    # json_content=json_content.strip('() \n\t\r')
    # print(json_content)

    # 将json格式字符串转化为python对象
    py_json = json.loads(json_content)
    comments_list = py_json['comments']
    for comment in comments_list:
        # 使用jsonpath方式查找
        user = jsonpath.jsonpath(comment, '$..user')[0]
        # 用户头像
        user_face = 'http;' + user['avatar']
        # print(user_face)
        #用户名
        user_name=user['nick']
        # print(user_name)
        #评论内容
        user_text=comment['content']
        #评论日期
        user_date=comment['date']
        #手机信息
        info=jsonpath.jsonpath(comment, '$..sku')[0]
    # up_date=comment['date']
    # up_content=comment['content']
    # up_modol=comment['auction']['sku']
    # up_author=comment['user']['nick']
        item={
        '评论人': user_name,
        '评论图像':user_face,
        '评论时间': user_date,
        '评论内容': user_text,
        '购买手机类型': info,
        }

        items.append(item)


def main():
    url = 'https://rate.taobao.com/feedRateList.htm?auctionNumId=593014213362&userNumId=903511695&currentPageNum={}&pageSize=20'
    start_page = int(input('请输入爬取开始页面：'))
    end_page = int(input('请输入爬取结束页面：'))
    for page in range(start_page, end_page + 1):
        request = handle_request(url, page)
        json_content = urllib.request.urlopen(request).read().decode()
        # print(json_content)
        time.sleep(2)
        json_parse(json_content)
    string = json.dumps(items, ensure_ascii=False)
    # 以TXT格式保存到本地
    with open('taobao.txt', 'w', encoding='utf8') as fp:
        fp.write(string)


if __name__ == '__main__':
    main()