pyhton实战作业2_2

写了两个爬虫
本质上其实是跟一个爬虫一样的

爬虫1：获取所有手机号的链接，并且存在数据库里
一共116页

import pymongo
import requests
from bs4 import BeautifulSoup

client=pymongo.MongoClient('localhost',27017)
walden=client['walden']
phone_num_link=walden['phone_num_link']


def get_info_frompage(startpage,endpage):
    infos=[]
    for i in range(startpage,endpage+1):
        # http://bj.58.com/shoujihao/pn1/
        oriurl="http://bj.58.com/shoujihao/pn"+str(i)+"/"
        wb_data_ori = requests.get(oriurl)
        soup_ori = BeautifulSoup(wb_data_ori.text,'lxml')
        titles=soup_ori.select("strong.number")
        links=soup_ori.select("#infolist > div > ul > div.boxlist > ul > li > a.t")
        j=0
        for title,link in zip(titles,links) :
            if link.get('href').find("jump.zhineng")<0 :#小于0代表找不到“jump.zhineng”
                data={
                    "title":title.text,
                    "link":link.get('href')
                }
                phone_num_link.insert_one(data)
                j+=1
            else:
                continue
        print(j)

    return infos

get_info_frompage(1,116)

爬虫2：从数据库里将链接取出来，根据链接逐一爬取手机号信息
并且将信息存到数据库中。
做了一下404页面校验，不过好像没遇到404
一共3480条数据

import pymongo
import requests
from bs4 import BeautifulSoup

client=pymongo.MongoClient('localhost',27017)
walden=client['walden']
phone_num_link=walden['phone_num_link']
num_info=walden['num_info']


def get_one_info(url):
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text,'lxml')
    if soup.select("head > script")[0].get('src').find("//j1.58cdn.com.cn/js/404/topbar404.js")>0:#如果是404页面（即找到了404.js）
        return 0
    else:
        title=soup.select("#main > div.col.detailPrimary.mb15 > div.col_sub.mainTitle > h1")[0].text
        price=soup.select("#main > div.col.detailPrimary.mb15 > div.col_sub.sumary > ul > li:nth-of-type(1) > div.su_con > span")[0].text
        host=soup.select("#divContacter1 > ul > ul > li > a")[0].text
        # number_address=soup.select("span.c_999.f12")[0].text if soup.find_all('span','c_999') else None

        data={
            'title':title.strip()[0:11],
            'price':price.strip(),
            'host':host.strip(),
            # 'number_address':number_address.replace(" ","").replace("\t","").replace("\n","")[:-10][7:],#三个replace分别删除了空格、制表符、换行符，后面两个分别是删去末尾10个字符，删去开头7个字符

        }
        # print(data)
        return data

# get_one_info("http://bj.58.com/shoujihao/26120773122616x.shtml")
for item in phone_num_link.find():
    info=get_one_info(item['link'])
    # print(info)
    num_info.insert_one(info)

结果如下

Paste_Image.png