最近公司业务要求，需要找一批疑似涉诈骗的数据，百度贴吧是一个好的平台。
# -*- coding: utf-8 -*-
import requests
import time
from bs4 import BeautifulSoup
import io
import sys
#sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='gbk') #改变标准输出的默认编码
#生活大爆炸吧
'''
# 标题&帖子链接：
<a rel="noreferrer" href="/p/4788526595" title="我的人物设计和制作" target="_blank" class="j_th_tit ">我的人物设计和制作</a>
#发帖人：
<span class="tb_icon_author " title="主题作者: 新日落" data-field="{"user_id":2137596235}"><i class="icon_author"></i><span class="frs-author-name-wrap"><a rel="noreferrer" data-field="{"un":"\u65b0\u65e5\u843d"}" class="frs-author-name j_user_card " href="/home/main/?un=%E6%96%B0%E6%97%A5%E8%90%BD&ie=utf-8&fr=frs" target="_blank">新日落</a></span><span class="icon_wrap icon_wrap_theme1 frs_bright_icons "></span> </span>
#发帖日期：
<span class="pull-right is_show_create_time" title="创建时间">2016-09</span>
#回复数量：
<div class="col2_left j_threadlist_li_left">
<span class="threadlist_rep_num center_text" title="回复">73</span>
</div>
'''
#抓取网页的通用框架,获取页面的内容
def getHtml(url):
try:
r= requests.get(url,timeout=30)
#状态码不是200就发出httpError的异常
r.raise_for_status()
#获取正确的编码格式
# r.encoding=r.apparent_encoding
r.encoding="utf-8"
#打印内容
return r.text
except:
return "wrong!"
#分析网页的html文件，整理信息，保存问列表文件中
def get_content(url):
#初始化一个列表来保存所有的帖子信息
contents=[]
#获取网页的内容
html=getHtml(url)
#将网页内容格式化利用bs4库
soup = BeautifulSoup(html, 'lxml')
#获取所有的li标签属性为 j_thread_list clearfix，用列表接收
liTags = soup.find_all('li', attrs={'class': ' j_thread_list clearfix'})
print (len(liTags))
#循环这个内容li集合
for li in liTags:
#将爬取到了每一条信息。保存到字典里
content={}
#将异样抛出，避免无数据时，停止运
try:
#开始筛选信息
content['title']=li.find('a',attrs={"class":"j_th_tit"}).text.strip()#.strip() 翻译为中文
print (li.find('a',attrs={"class":"j_th_tit"}).text.strip())
#获取a标签的内部属性
content['link'] ="http://tieba.baidu.com/"+li.find('a', attrs={"class": "j_th_tit"})["href"]
print("http://tieba.baidu.com/"+li.find('a', attrs={"class": "j_th_tit"})["href"])
# 获取第一条内容（标题 + 第一条内容结合内容会比较丰富）
content['content'] = li.find('div', attrs={"class":'threadlist_abs threadlist_abs_onlyline '}).text.strip()
print(li.find('div', attrs={"class": 'threadlist_abs threadlist_abs_onlyline '}).text.strip())
content['author']=li.find('span',attrs={"class":'tb_icon_author '}).text.strip()
print (li.find('span',attrs={"class":'tb_icon_author '}).text.strip())
content['responseNum']=li.find('span',attrs={'class': 'threadlist_rep_num center_text'}).text.strip()
print(li.find(
'span', attrs={'class': 'threadlist_rep_num center_text'}).text.strip())
content['creatTime']=li.find('span',attrs={"class":'pull-right is_show_create_time'}).text.strip()
print (li.find(
'span', attrs={'class': 'pull-right is_show_create_time'}).text.strip())
#将字典加入到列表中
contents.append(content)
except:
print('出问题')
#返回数据
return contents
def writeTxt(object_dir, content):
#这里不能写成 f=open("data.txt",'a+'）否则会乱码，设置沉utf-8的格式，与getHtml(url):中的encoding一致
f=open(object_dir, 'a+',encoding='utf-8')
for c in content:
# f.write('标题： {} \t 链接：{} \t 发帖人：{} \t 发帖时间：{} \t 回复数量： {} \n'.format(
# c['title'], c['link'], c['author'], c['creatTime'], c['responseNum']))
# f.write('标题： {} \t 内容： {} \t \n'.format(
# c['title'], c['content']))
f.write('{}。{}\n链接：{}\n'.format(
c['title'], c['content'], c['link']))
def main(url, page, object_dir='data.txt'):
url_list=[]
#将所需要爬去的url放到列表中
for i in range(0,page):
url_list.append(url+'&pn='+str(i*50))
for u in url_list:
content=get_content(u)
writeTxt(object_dir, content)
if __name__=="__main__":
# url = "https://tieba.baidu.com/f?ie=utf-8&kw" \
# "=%E6%B8%B8%E6%88%8F%E5%8F%B7&fr=search"
# page = 20
# object_dir = '游戏账号.txt'
# url = "https://tieba.baidu.com/f?ie=utf-8&kw=%E7%82%" \
# "AB%E8%88%9E%E8%B4%A6%E5%8F%B7%E4%BA%A4%E6%98%93"
# page = 20
# object_dir = '炫舞账号交易.txt'
# url = "https://tieba.baidu.com/f?ie=utf-8&kw=" \
# "lol%E8%B4%A6%E5%8F%B7%E4%BA%A4%E6%98%93"
# page = 20
# object_dir = '英雄联盟账号交易.txt'
url = "http://tieba.baidu.com/f?" \
"kw=%E8%99%9A%E6%8B%9F%E5%B8%81&ie=utf-8&pn=0"
page = 200
object_dir = '虚拟币.txt'
main(url,page, object_dir)
# get_content("https://tieba.baidu.com/f?ie=utf-8&kw=%E6%B8%B8%E6%88%8F%E5%8F%B7&fr=search")
另一个
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# author: ShidongDu time2019/9/6
# -*- coding:utf-8 -*-
import os
import codecs
import json
import urllib
import urllib.request
from lxml import etree
class Spider:
def __init__(self, pages, url, dir):
# self.pages = int(input('请输入需要爬取的页数(请输入50的倍数数字):'))
self.pages = pages
# self.url = 'http://tieba.baidu.com/f?kw=%E6%95%B4%E5%AE%B9&ie=utf-8&pn='
self.dir = dir
self.url = url
self.ua_header = {"User-Agent" : "Mozilla/5.0(compatible;MSIE 9.0;Windows NT 6.1; Trident/5.0;"}
def tiebaSpider(self):
for page in range(50, self.pages + 1, 50):
url = self.url + str(page)
# 并且获取页面所有帖子链接,
links = self.loadPage(url)
#读取页面内容
def loadPage(self, url):
req = urllib.request.Request(url, headers=self.ua_header)
html = urllib.request.urlopen(req).read().decode('utf8',errors='replace')
#解析html 为 HTML
selector = etree.HTML(html)
# print(selector)
links = selector.xpath('//div[@class="threadlist_lz clearfix"]/div/a/@href')
for link in links:
link = "http://tieba.baidu.com" + link
self.loadImages(link)
# 获取
def loadImages(self, link):
req = urllib.request.Request(link, headers= self.ua_header)
html = urllib.request.urlopen(req).read().decode('utf8',errors='replace')
selector = etree.HTML(html)
#获取这个帖子里所有回复人，回复内容，和帖子标题
title = selector.xpath('//div[@class="left_section"]//div/h1/text()')[0]
# 获取每个内容块
content_field = selector.xpath('//div[@class="l_post j_l_post l_post_bright "]')
reply = {}
reply['reply_title'] = title
for each_content in content_field:
reply_info = json.loads(each_content.xpath('@data-field')[0])
author = reply_info['author']['user_name']
reply_time = reply_info['content']['date']
content = each_content.xpath('div[@class="d_post_content_main"]/div/cc/div[starts-with(@id, "post_content") \
and contains(@class,"d_post_content j_d_post_content clearfix")]')
reply['reply_author'] = author
reply['reply_content_time'] = reply_time
reply['reply_content'] = content[0].xpath('string(.)').replace(' ', '')
self.writeImages(self.dir, reply)
#按帖子title来建立文件名
def writeImages(self, dir, reply):
s_path = './'+ dir+ '/'
if not os.path.isdir(s_path):
os.mkdir(s_path)
else:
pass
try:
file = codecs.open(s_path + str(reply['reply_title']) + '.txt', 'a', encoding='utf-8')
# file.write(reply['reply_author'] + ":" + reply['reply_content'] + '\n')
file.write(reply['reply_content'] + '\n')
file.close()
except:
print("oops~")
# 5000条，即100页
pages = 5000
# url:爬取的页面
url = "http://tieba.baidu.com/f?kw=%E7%88%B1%E5%A5%87%E8%89%BAvip%E4%BC%9A%E5%91%98&ie=utf-8"
dir = "爱奇艺"
Spider = Spider(pages, url, dir)
Spider.tiebaSpider()
print("OK!")
网友评论