爬取appstore应用信息

#!/usr/bin/env python
# encoding=utf8  
import sys  

import bs4
from bs4 import BeautifulSoup
import requests
import json
import re

gIgnoreGenreList = ['报刊杂志', '贴纸', '商品指南']
    
# https://www.apple.com/cn/itunes/charts/paid-apps/
# https://www.apple.com/cn/itunes/charts/free-apps/
def parse_appstore_page(cate_url, out_file):
    # section apps grid
    html = requests.get(cate_url).content  
    soup = BeautifulSoup(html, 'html.parser', from_encoding='utf-8')

    result = soup.find("div", id="main")

    result = result.find("section", class_="section apps grid")
    result = result.find("ul")
    app_list = result.children
    app_result = []
    for child in app_list:
        if isinstance(child, bs4.element.Tag):
            app_info = child.find("h3").find("a")

            # print child
            app_name = app_info.string
            app_itunes_url = app_info.get("href")

            # print(repr(app_name).decode('unicode-escape'))
            # print(repr(app_detail_url).decode('unicode-escape'))
            one_app = {}
            one_app["app_name"] = app_name
            one_app["app_detail_url"] = app_itunes_url
            app_result.append(one_app)

    if len(out_file) > 0:
        print "save result to file :%s" % out_file
        result_string = json.dumps(app_result, ensure_ascii=False)
        f=open(out_file,"w")
        f.write(result_string)
        f.write('\n')
        f.close()

# https://itunes.apple.com/cn/genre/ios/id36?mt=8
def parse_genre_page(genre_url, limit = 10, out_file = "genre_result.txt"):
    html = requests.get(genre_url).content  
    soup = BeautifulSoup(html, 'html.parser', from_encoding='utf-8')

    result = soup.find("div", id="main")

    result = result.find("div", id="content")
    result = result.find("div", id="genre-nav")
    result = result.find("div", class_="grid3-column")

    # ul list
    ul_list = result.find_all("ul", recursive=False)
    cate_result = []

    for one_ul in ul_list:
        if isinstance(one_ul, bs4.element.Tag):

            cate_list = one_ul.children

            for cate in cate_list:
                cate_info = cate.find("a")
                cate_name = cate_info.string

                if cate_name in gIgnoreGenreList:
                    print "ingore cate :" + cate_name
                    continue

                cate_url = cate_info.get("href")

                one_cate = {}
                one_cate["name"] = cate_name
                one_cate["url"] = cate_url

                print "processing genre %s." % cate_name
                cate_app = parse_genre_content(cate_url, limit)
                one_cate["app_list"] = cate_app

                cate_result.append(one_cate)
                break

    if len(out_file) > 0:
        print "save result to file :%s" % out_file
        result_string = json.dumps(cate_result, ensure_ascii=False)
        f=open(out_file,"w")
        f.write(result_string)
        f.write('\n')
        f.close()

# https://itunes.apple.com/cn/genre/ios-导航/id6010?mt=8 
# 返回一个数组 [{app_name:xxxx, app_detail_url:xxxxx}]
# limit 为app返回数量，-1 为不限制
def parse_genre_content(content_url, limit = -1):
    html = requests.get(content_url).content  
    soup = BeautifulSoup(html, 'html.parser', from_encoding='utf-8')

    result = soup.find("div", id="main")

    result = result.find("div", id="content")
    result = result.find("div", id="selectedgenre")
    result = result.find("div", class_="grid3-column")

    # ul list
    ul_list = result.find_all("div")
    app_result = []

    count = 0
    for one_ul in ul_list:
        if isinstance(one_ul, bs4.element.Tag):

            app_list = one_ul.find("ul").children

            for app in app_list:
                if isinstance(app, bs4.element.Tag):
                    

                    if limit > -1 and count >= limit:
                        break   
                    # print count
                    app_info = app.find("a")
                    app_name = app_info.string
                    app_url = app_info.get("href")

                    one_app = {}
                    one_app["app_name"] = app_name
                    one_app["app_detail_url"] = app_url
                    one_app["app_id"] = parse_appid(app_url)

                    app_detail = parse_detail_page(app_url)

                    if app_detail != None:
                        print app_detail
                        one_app.update(app_detail)
                    
                    app_result.append(one_app)
                    count += 1
                        
    return app_result


# https://itunes.apple.com/cn/app/高德地图-精准导航-出行必备/id461703208?mt=8
# {latest_version:x.x.x, update_date:xxxxxxx, system_version:iOS 7.0}
def parse_detail_page(detail_url):
    app_detail = {}

    html = requests.get(detail_url).content  
    soup = BeautifulSoup(html, 'html.parser', from_encoding='utf-8')
    result = soup.find("main", class_="is-app-theme")

    # print result
    # result_string = soup
    # f=open("out_file.txt","w")
    # print >> f, result
    # f.write(result_string)
    # f.close()

    # return

    whats_new = result.find("section", class_="l-content-width section section--bordered whats-new")

# 获取版本号和最后更新时间
    update_date = whats_new.find("time").string
    latest_version = parse_appver(whats_new.find("p").string)

    # print update_date
    # print latest_version
    # APP 信息
    information = result.find("dl", class_=["information-list", "information-list--app", "medium-columns"])

# 获取app支持的最低iOS系统版本号
    version_label = information.select("div")[3].select("div > span")[0].string
    system_version = parse_systver(version_label)
    
    # print system_version
    # return

    app_detail["update_date"] = update_date
    app_detail["latest_version"] = latest_version
    app_detail["system_version"] = system_version
    
    return app_detail

def parse_appid(detail_url):
    # [^/]+(?!.*/)(?=[\?]+)
    return re.search("[^/]+(?!.*/)(?=[\?]+)", detail_url).group(0)

def parse_systver(version_label):
    # return version_label
    return re.search("iOS.(\d+\.\d+)", version_label).group(1)

def parse_appver(version_label):
    # "版本 10.5.0"
    # return version_label
    return re.search(u"版本\s(.+)", version_label).group(1)

if __name__ == '__main__':

    reload(sys)  
    sys.setdefaultencoding('utf8')   
    # parse_appstore_page('https://www.apple.com/cn/itunes/charts/free-apps/', 'free-apps.txt')
    # parse_appstore_page('https://www.apple.com/cn/itunes/charts/paid-apps/', 'paid-apps.txt')

    parse_genre_page('https://itunes.apple.com/cn/genre/ios/id36?mt=8', 1, 'appstore-genre.txt')

    # version = re.search("iOS.(\d+\.\d+)", "xxx iOS 8.0 asdasdadasd").group(1)
    # print version