作业思路

要被考试给急疯了...背书背得脑壳疼，具体思路在代码里有，分析由于时间关系，就不详述了。
选取了一个另一个方向来爬取答案，就是爬取搜索结果，所涉及到的知识点稍微多了一些，有提交表单，以html形式保存的json，然后尝试了一下如何保存到数据库中。
还有一个要注意的是，就是有的用户名是不规则的，它和个人简介合起来是一个超链接内容，这个还是比较难处理的，所以就做了下简化处理。

爬取的页面

作业结果

image.png

作业代码

# -*- coding: utf-8 -*-

"""
项目目标
1.问题的标题
2.点赞数
3.回答的简略说明
4.回答的作者（要区分于匿名用户）
5.评论数

"""
import requests
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
import json
from bs4 import BeautifulSoup
import MySQLdb

def getdata():

    headers = {"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"}
    cookies = {
        'd_c0': '"AHACIj6lFguPTkEZTVmz8MPJ8UkcuI03hag=|1483278273"',
        '_zap': '108369f2-93e6-407e-8928-b57f44168838',
        'q_c1': '0d3d690c67bc4b12a163beb462e17d67|1492571575000|1483278273000',
        'cap_id': '"NmFlNDgyZmMxNWQ0NGFiYmFhZThmMGM2MDVkZDM1MWM=|1493684435|1958b2bb9fb6f09116056cc4bc5af76e00dbaff7"',
        'l_cap_id': '"ZjViYmQ0MzBkNTFjNGJjOWE5YTExZWRjMjlkNjM2YjQ=|1493684435|1204e5c054805d8bebf3b49a8a2fc09e2f1bd870"',
        'auth_type': '"c2luYQ==|1493684515|245a0f575f151c486829c687ba3d3c90fc5fa558"',
        'token': '"Mi4wMGljWjNYRkVBNzIyRGJjZDZkMjMyZmMwM2JtMWs=|1493684515|a76491c048b7e825a77983ef0a272bdc9e721853"',
        'client_i': '"NTA3MzY2MzQwNA==|1493684515|9a08a2c9a6c02e86da4810cb5bc4437f6d7b1028"',
        'aliyungf_tc': 'AQAAAFOsE04F5QQAHVcc2i2qHTGdX1x0',
        'acw_tc': 'AQAAAAnnBnQS0AcAHVcc2omDblHld+wt',
        '_xsrf': '398e0b041f39355dadfc06b76ba18cdb',
        's-q': 'python',
        's-i': '1',
        'sid': 'ob2ttqkg',
        's-t': 'autocomplete',
        'z_c0': 'Mi4wQUFBQ3hmbTFzUXNBY0FJaVBxVVdDeGNBQUFCaEFsVk5OVjR2V1FETW9QS19PRlExWGVjUHNOcFVoTG5mYWg1LUtR|1493701061|4258e8f2ddc800c42c8b7d94385f578ca97f34d5',
        '__utma': '51854390.213172690.1484833633.1493684449.1493701029.4',
        '__utmb': '51854390.0.10.1493701029',
        '__utmc': '51854390',
        '__utmz': '51854390.1493701029.4.4.utmcsr=baidu|utmccn=(organic)|utmcmd=organic',
        '__utmv': '51854390.100--|2=registration_date=20170502=1^3=entry_date=20170101=1'
    }
    total = []
    for i in range(0,17):
        page = 10 * i

        #构造请求的表单，可以定义很多查询内容，如把sort去除，就是默认为时间排序
        data = {"q":"python",
        "sort":"upvote",
        "correction":"0",
        "type":"content",
        "offset":page}
        r = requests.get("https://www.zhihu.com/r/search?", headers=headers, data=data, cookies=cookies)

        #请求所得的是一个json，但是这个json中还包含了html标签，所以还需要用beautifulsoup给提取出来
        totaldata = json.loads(r.text)


        for one in totaldata["htmls"]:
            item = {}
            soup = BeautifulSoup(one, "lxml")

            item["title"] = soup.find("div", {"class": "title"}).get_text() #标题
            try:
                item["author"] = soup.find("span", {"class": "author-link-line"}).get_text() #作者名
            except:
                item["author"] = "匿名用户"

            try:
                item["bio"] = soup.find("span", {"class": "bio"}).get_text() #标签
            except:
                item["bio"] = "没有介绍"

            try:
                item["like"] = soup.find("span", {"class": "count"}).get_text() #点赞数
            except:
                item["like"] = 0

            try:
                item["comment"] = soup.find("a", {"class": "action-item js-toggleCommentBox"}).get_text() #评论数
            except:
                item["comment"] = 0

            try:
                item["zaiyao"] = soup.find("div", {"class": "summary hidden-expanded"}).get_text()
            except:
                item["zaiyao"] = "没有回答"

            total.append(item)
    return total

# 创建一个数据库表来保存信息
    #连接数据库
def mysql_conn():
    conn=MySQLdb.connect(
        host='127.0.0.1',
        user = "root",
        passwd = "882645",
        charset = "utf8",
        db='Lagou',
        port = 3306
    )
    return conn

#建表
def create_table():
    create_sql='''
    CREATE TABLE `zhihu`(
    id INT(11) NOT NULL AUTO_INCREMENT,
    TITLE VARCHAR(855),
    LIKES INT(255),
    AUTHOR VARCHAR(255),
    BIO VARCHAR(255),
    ZAIYAO VARCHAR(855),
    COMMENTS VARCHAR(255),
    PRIMARY KEY (`id`)
    )ENGINE=INNODB DEFAULT CHARSET =utf8
    '''
    conn=mysql_conn()
    with conn:
        cursor = conn.cursor()
        cursor.execute(create_sql)
        conn.commit()

#插入记录
def insert(item):
    sql='insert into zhihu (TITLE,LIKES,AUTHOR,BIO,ZAIYAO,COMMENTS) values(%s,%s,%s,%s,%s,%s)'
    conn = mysql_conn()
    with conn:
        cursor = conn.cursor()
        try:
            cursor.execute(sql,(item['title'],item['like'],item['author'],item['bio'],item['zaiyao'],item['comment']))

            cursor.connection.commit()

        except BaseException as e:
            print u"错误在这里>>>>", e, u"<<<<错误在这里"

        conn.commit()

if __name__ == '__main__':
    create_table()
    a = getdata()
    for i in a:
        insert(i)
    print u"处理完毕"