爬取京东商品实例

import random
uas = ["Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
        "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", 
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", 
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", 
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", 
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", 
        "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", 
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", 
        "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", 
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", 
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", 
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", 
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", 
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", 
        "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", 
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", 
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", 
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
    ]

UserAgent=random.choice(uas)
print(UserAgent)


import requests
import random
url='https://list.jd.com/list.html?cat=9987,653,655'
this_ua=random.choice(uas)
head={"user-agent":this_ua}
res=requests.get(url,headers=head)
res.encoding='utf-8'
print(res)

# 爬取常规数据:商品名称 商家名称
root=etree.HTML(res.text)

# 商品名称
name=root.xpath('//div[@class="p-name"]/a/em/text()')
for i in range(len(name)):
    name[i]=name[i].strip()
    
print(name)

# 获取商家名称
shop=root.xpath('//div[@class="p-shop"]/@data-shop_name')
print(shop)


# 获取全部商品的sku
sku=root.xpath('//li[@class="gl-item"]/div/@data-sku')
print(sku)


# 获取价格
price=[]
for i in range(len(sku)):
    this_sku=sku[i]
    price_url="https://p.3.cn/prices/mgets?callback=jQuery8966120&ext=11101100&pin=&type=1&area=1_72_4137_0&skuIds=J_"+sku[i]+"&pdbp=0&pdtk=&pdpin=&pduid=825425048&source=list_pc_front&_=1558338909272"
    price_res=requests.get(price_url)
    pat='"p":"(.*?)"'
    this_price=re.findall(pat,price_res.text)
    price=price+this_price
    
print(price)

# 评论数
commentcount=[]
for i in range(len(sku)):
    this_sku=sku[i]
    commenturl="https://club.jd.com/comment/productCommentSummaries.action?referenceIds="+this_sku+"&callback=jQuery345302&_=1558338910413"
    commentres=requests.get(commenturl)
    commentpat='"ShowCount":(.*?),'
    this_commentcount=re.findall(commentpat,commentres.text)
    commentcount=commentcount+this_commentcount
print(commentcount)

image.png image.png
image.png
image.png

# 利用循环爬取多页数据
import requests
from lxml import etree
import re
from fake_useragent import UserAgent
from pandas import DataFrame
import pandas as pd
import time

pInfoAll=DataFrame()

for i in range(1,3):
    url="https://list.jd.com/list.html?cat=9987,653,655&page="+str(i)
    
    this_ua=random.choice(uas)
    head={"user-agent":this_ua}
    res=requests.get(url,headers=head)
    res.encoding='utf-8'
    root=etree.HTML(res.text)

    # 商品名称
    name=root.xpath('//div[@class="p-name"]/a/em/text()')
    for i in range(len(name)):
        name[i]=name[i].strip()

    # 获取商家名称
    shop=root.xpath('//div[@class="p-shop"]/@data-shop_name')

    # 获取全部商品的sku
    sku=root.xpath('//li[@class="gl-item"]/div/@data-sku')

    # 获取价格
    price=[]
    for i in range(len(sku)):
        this_sku=sku[i]
        price_url="https://p.3.cn/prices/mgets?callback=jQuery8966120&ext=11101100&pin=&type=1&area=1_72_4137_0&skuIds=J_"+sku[i]+"&pdbp=0&pdtk=&pdpin=&pduid=825425048&source=list_pc_front&_=1558338909272"
        price_res=requests.get(price_url)
        pat='"p":"(.*?)"'
        this_price=re.findall(pat,price_res.text)
        price=price+this_price
    
    # 评论数
    commentcount=[]
    for i in range(len(sku)):
        this_sku=sku[i]
        commenturl="https://club.jd.com/comment/productCommentSummaries.action?referenceIds="+this_sku+"&callback=jQuery345302&_=1558338910413"
        commentres=requests.get(commenturl)
        commentpat='"ShowCount":(.*?),'
        this_commentcount=re.findall(commentpat,commentres.text)
        commentcount=commentcount+this_commentcount

    pInfo=DataFrame([sku,name,shop,price,commentcount]).T
    pInfoAll=pd.concat([pInfoAll,pInfo])
    pInfo.columns=['sku','商品名称','商家','价格','评论数']
    time.sleep(2)

print(pInfoAll.head())
print(pInfoAll.describe())