import requests
from bs4 import BeautifulSoup
import os
import re
def get_Soup(Fiction_url):
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0',
'Connection': 'keep - alive'
}
html_doc = requests.get(Fiction_url,headers=header)
html_doc.encoding = 'utf-8' # 笔趣阁网页编码为gbk 所以需要修改 防止中文乱码。
soup = BeautifulSoup(html_doc.text, 'html.parser')
return soup
#返回soup
def get_FictionName(soup):
fictionName = soup.find('h1').text
fiction = {'名称':fictionName}
return fiction
def mkdir(fiction):
save_path = 'D:\\'+ fiction['名称']
isExists = os.path.exists(save_path) #判断目录是否存在
if not isExists:
os.makedirs(save_path) #创建目录
print('{}目录创建成功'.find(save_path))
return save_path
else:
print("目录已经存在,请修改文件名")
save_path = 'D:\\'+ input('请输入新的文件名:')
os.makedirs(save_path)
print('{}目录创建成功'.find(save_path))
return save_path
def get_Fiction_AddAndName(soup):
i = 1 #用来个章节计数用
chapter_add = [] #章节地址 list
chapter_name = [] #章节名 list
rstr = r"[\=\(\)\,\/\\\:\*\?\"\<\>\|\' ']" #为去掉章节名中的特殊字符做准备
for chapter in soup.select('td'):
href = chapter.select('a')[0]['href']
chapter_add.append(href)
name = re.sub(rstr, "", chapter.select('a')[0].text) #去掉章节中的特殊字符
text_name = '第' + str(i) + '章' + name
chapter_name.append(text_name)
i += 1
return chapter_add,chapter_name
def get_Fiction(chapter_add,chapter_name,save_path):
sum_chapter = len(chapter_add)
i = 0
for chapter in chapter_add:
chapter_content = requests.get(chapter)
chapter_content.encoding = 'utf-8'
content_soup = BeautifulSoup(chapter_content.text,'html')
try:
text = content_soup.find('dd', id='contents').text
except:
print("第{}章节为空".format(i))
text = '空'
create_txt(text,save_path,chapter_name[i])
i += 1
print('共{}章,正在写入第{}章.....'.format(sum_chapter,i))
def create_txt(text,save_path,chapter_name):
f = open(save_path+'\\'+ chapter_name+'.txt','w+',encoding='utf-8') #创建txt文件 w 是写入 encoding 修改编码方式
f.write(text)
f.close()
Fiction_url = 'https://www.23us.so/files/article/html/15/15497/'
soup = get_Soup(Fiction_url) #获取soup
fiction = get_FictionName(soup) #获取小说名字
save_path = mkdir(fiction) #创建目录
chapter_add,chapter_name = get_Fiction_AddAndName(soup)
get_Fiction(chapter_add,chapter_name,save_path)
网友评论