【Python爬虫】-第四期课后练习15

from bs4 import BeautifulSoup
import requests
import csv
import os

def main(url_temp):
    rep = requests.get(url_temp)
    rep.encoding = 'gb2312'
    soup = BeautifulSoup(rep.text, 'lxml')
    root_divs = soup.find_all("div", class_="fkt")
    # print(root_divs)
    writeCsv('城市', '地区', '号段', '号段说明', '号码')
    for root_div in root_divs:
        # print(root_div)

        fkbj_div = root_div.find("div", class_="fkbj")

        if (fkbj_div is not None ):
            city_str = fkbj_div.find('a').text.strip()
            fklk_div = root_div.find("div", class_="fklk")
            area_as = fklk_div.find_all('a')
            for area_a in area_as:
                # print(type(area_a))
                area_str = area_a.text.strip()
                area_href = area_a['href']
                # print(city_str,area_str,area_href)
                rep2 = requests.get(area_href)
                rep2.encoding = 'gb2312'
                soup2 = BeautifulSoup(rep2.text, 'lxml')
                hd_divs = soup2.find_all("div", class_="num_bg")
                hd_uls = soup2.find_all("ul")
                print(len(hd_divs))
                print(len(hd_uls))
                for num in range(0,len(hd_divs)):
                    # print(hd_divs[num])
                    # print(hd_uls[num])
                    hd_div = hd_divs[num]
                    hd_ul = hd_uls[num]
                    hd_str = hd_div.find("span").text.strip()
                    num_bg_str = hd_div.text.strip()
                    print(hd_str,num_bg_str)
                    li_as = hd_ul.find_all("a")
                    for li_a in li_as:
                        li_str = li_a.text.strip()
                        print(city_str,area_str,hd_str,num_bg_str,li_str)
                        writeCsv(city_str,area_str,hd_str,num_bg_str,li_str)

def writeCsv(city_str,area_str,hd_str,num_bg_str,li_str):
    base_dir = os.path.abspath(__file__)
    parent_dir = os.path.dirname(base_dir)
    menu_dir = os.path.join(parent_dir, 'CSVTEST')
    if os.path.isdir(menu_dir):
        pass
    else:
        os.mkdir(menu_dir)
    os.chdir(menu_dir)
    file_name = os.path.join(menu_dir, 'csvtest.csv')
    with open(file_name, 'a', encoding='gb2312') as file:
        writer = csv.writer(file)
        writer.writerow((city_str.strip(),area_str.strip(),hd_str.strip(),num_bg_str.strip(),li_str.strip()))

if __name__ == '__main__':
    n_url = "http://www.51hao.cc/"
    main(n_url)