最终效果:
最终效果.png我的代码:
from bs4 import BeautifulSoup
import requests
import time
import pymongo
client = pymongo.MongoClient('localhost',27017)
homework = client['homework']
work2_1 = homework['work2_1']
urls = []
#抓取列表页中的内容页链接
#data:列表页链接
def conUrlFun(data):
wb_data = requests.get(data)
Soup = BeautifulSoup(wb_data.text,'lxml')
conUrls= Soup.select('.pic_list > li > a.resule_img_a')
for conUrl in conUrls:
#将爬取到的链接存入urls列表中
urls.append(conUrl.get('href'))
# 获取指定数量的列表页中内容页链接
# data1,data2:初始页至结束页
def urlFun(data1,data2):
urls = ["http://bj.xiaozhu.com/search-duanzufang-p{}-0/".format(i) for i in range(data1, data2)]
for url in urls:
print(url)
# 打印页面中的内容页链接
conUrlFun(url)
time.sleep(0.5)
# 获取内容页中的标题、地址、价格、图片、昵称、性别、头像
# data1:内容页链接
def conFun(data):
wb_data = requests.get(data)
Soup = BeautifulSoup(wb_data.text,'lxml')
title = Soup.select('.pho_info > h4 > em')[0].get_text()
addr = Soup.select('.pho_info > p > span')[0].get_text().strip()
price = Soup.select('.day_l > span')[0].get_text()
image = Soup.select('.pho_show_big > div > img')[0].get('src')
avartar = Soup.select('.member_pic > a > img')[0].get('src')
name = Soup.select('.lorder_name')[0].get_text()
sexs = Soup.select('.member_pic > div')[0]['class'][0]
#男 = member_ico 女 = member_ico1 无 = 空
if sexs == str('member_ico'):
sex = '男'
elif sexs == str('member_ico1'):
sex = '女'
else:
sex = '不明'
datas = {
"title":title,
"addr":addr,
"price":int(float(price)),
"image":image,
"avartar":avartar,
"name":name,
"sex":sex
}
work2_1.insert_one(datas)
print(datas,'已写入数据库')
#获取前4页的租房信息,并写入数据库中
# urlFun(1,4)
# for url in urls:
# conFun(url)
# time.sleep(0.5)
#筛选出所有价可靠大于等于500元的房源信息
for item in work2_1.find({'price':{'$gte':500}}):
print(item)
学到的知识:
-
连接服务器:client = pymongo.MongoClient('localhost',27017)
-
连接数据库:walden = client['walden']
-
连接数据库中的表:sheet_tab = walden['sheet_tab']
-
将数据存入数据表中:sheet_tab.insert_one(data)
-
查询数据库中的数据:find()函数
$lt/$lte/$gt/$gte/$ne,依次等价于</<=/>/>=/!=。(l表示less g表示greater e表示equal n表示not )
例:
筛选出所有价可靠大于等于500元的房源信息
for item in work2_1.find({'price':{'$gte':400}}):
print(item)
网友评论