# -*- coding: utf-8 -*-
import requests
import json
import time
import random
import pandas as pd
from flask import Flask, jsonify
from flask import abort,request
import pymysql
from sqlalchemy import create_engine
from datetime import datetime, date
app = Flask(__name__)
@app.route('/getdata',methods=['GET'])
def get_data():
df = save_to_dataframe()
# force_ascii保证了返回的数据是中文的,不是unicode编码的
return jsonify(df.to_json(orient='records',force_ascii=False))
#下载第一页数据
def get_one_page(url):
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36'
}
response = requests.get(url,headers=headers)
if response.status_code == 200: #页面正常响应
return response.text # 返回页面源代码
return None
#解析第一页数据
def parse_one_page(html):
data = json.loads(html)['cmts'] #评论以json形式存储,故以json形式截取
for item in data:
yield{ #该方法返回一个字典
'comment':item['content'],
'date':item['time'].split(' ')[0],
'rate':item['score'],
'city':item['cityName'],
'nickname':item['nickName']
}
#保存数据到dataframe
def save_to_dataframe():
items = []
for i in range(1, 5):
url = 'http://m.maoyan.com/mmdb/comments/movie/341516.json?_v_=yes&offset=' + str(i)
html = get_one_page(url)
print('正在保存第%d页.'% i)
for item in parse_one_page(html):
items.append(item)
data = pd.DataFrame(items)
data.drop_duplicates(inplace=True)
# 保存到数据库中,设置charset=utf8mb4
conn = create_engine('mysql+pymysql://root:mysql@127.0.0.1:3306/test?charset=utf8mb4')
data.to_sql('test_db', con=conn, if_exists='replace', index=False)
#反扒
time.sleep(float(random.randint(1,100)) /20)
return data
if __name__ =='__main__':
app.run(host="127.0.0.1", port=8000, debug=True)
get_one_page(url)返回的数据格式为:
{"approve":0,"approved":false,"authInfo":"","avatarurl":"https://img.meituan.net/avatar/1b506b5147bc501e20523ad2801830ad9934.jpg",
"cityName":"招远","content":"不错,挺好看的","filmView":false,"id":1035077242,"isMajor":false,"juryLevel":0,
"majorType":0,"movieId":341516,"nick":"nIP482975932","nickName":"nIP482975932","oppose":0,"pro":false,
"reply":0,"score":5,"spoiler":0,"startTime":"2018-08-13 09:08:32","supportComment":true,"supportLike":true,
"sureViewed":0,"tagList":{"fixed":[{"id":1,"name":"好评"},{"id":4,"name":"购票"}]},
"time":"2018-08-13 09:08","userId":503247692,"userLevel":3,"videoDuration":0,"vipInfo":"","vipType":0},
postman返回成功界面:
数据库成功插入数据界面:
注意点:
-
to_json()不设置force_ascii=False时,postman返回页面为unicode格式,不是中文页面,但是数据库可以正常插入。
postman返回unicode界面:
数据库插入数据成功: - 数据库的charset=utf8时,插入数据不成功,数据库为空。
- 数据库的字符集也要设置为utf8mb4。
总结
- pandas.DataFrame.to_json(force_ascii=False)保证了返回的数据是中文的,不是unicode编码的。
- 数据库的字符集设置为utf8mb4,并且在设置create_engine参数的时候,也要设置字符集charset=utf8mb4,这样就可保证数据库的内容都是中文的。
网友评论