Judge_end.py

函数功能：判断是否已读到最后一页

import requests
from bs4 import BeautifulSoup


def judge_end(url):
    html=requests.get(url)
    bs_html=BeautifulSoup(html.text,'lxml')
    snumber=bs_html.select('#infocont > span > b')
    number=eval(snumber[0].get_text())
    if number!=0:
        return True
    else:
        return False



'''
#infolist > div > ul > div.boxlist > ul > li:nth-child(1)
http://bj.58.com/shoujihao/pn3
http://bj.58.com/shoujihao/pn200/
'''

Main.py

函数功能：获取所有的手机链接

from PhoneInf import get_Phone_Href

urls=['http://bj.58.com/shoujihao/pn{}'.format(str(i)) for i in range(1,130)]


def main():
    for url in urls:
        get_Phone_Href(url)


main()

PhoneInf.py

函数功能：获取手机号码，具体页面链接地址的信息等

from bs4 import BeautifulSoup
import requests
import time
from Judge_end import judge_end


import pymongo

client=pymongo.MongoClient('localhost',27017)
TC58_phone=client['TC58_phone0']
phoneHref=TC58_phone['phoneHref']

def get_Phone_Href(url):
    if judge_end(url):
        html=requests.get(url)
        bs_html=BeautifulSoup(html.text,'lxml')

        phone_numbers=bs_html.select('#infolist > div > ul > div.boxlist > ul > li > a.t > strong[class="number"]')
        phone_href=bs_html.select('#infolist > div > ul > div.boxlist > ul > li > a.t')
        for number,href in zip(phone_numbers,phone_href):
            phone_href_data={
                'number':number.get_text(),
                'href':href.get('href'),
                'read':'none'
            }

            phoneHref.insert_one(phone_href_data)
        time.sleep(2)
        print(phoneHref.count())
    else:
        pass