下面是代码，关键部分都有注释。
# coding=utf-8
from lxml import etree
import requests
import re
import time
import xlwt
headers = {
'authority': 'www.xforange.com',
'method': 'GET',
'path': '/article/3304.html',
'scheme': 'https',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'zh-CN,zh;q=0.9',
'cache-control': 'max-age=0',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'
}
headerss = {
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
}
url_1 = "https://www.xforange.com/news/index"
url_2 = ".html"
# 创建一个workbook 设置编码
workbook = xlwt.Workbook(encoding='utf-8')
# 创建一个worksheet
worksheet = workbook.add_sheet('My Worksheet', cell_overwrite_ok=True)
worksheet.write(0, 0, "标题")
worksheet.write(0, 1, "副标题")
worksheet.write(0, 2, "时间")
worksheet.write(0, 3, "来源")
worksheet.write(0, 4, "内容")
d=1
for i in range(1,8):
i = str(i)
url = url_1 + i + url_2
print(url)#输出列表的地址
html = requests.get(url, headers=headerss).text
harf = re.findall('https://www.xforange.com/news/[0-9]{4}.html', html)
for x in range(len(harf)):
if x<10:
url = harf[x]
time.sleep(1)
#print(url)
response = requests.get(url, headers=headers)
html = etree.HTML(response.text)
article = html.xpath('//div[contains(@class,"article_show")]')[0]
title = article.xpath('.//div[@class="left"]/h1/text()')[0]
Subtitle = article.xpath('.//div[@class="content"]/p[1]/text()')
date = article.xpath(
'.//div[@class="info"]/span[1]/text()')[0].split('：')[1]
source = article.xpath(
'.//div[@class="info"]/span[2]/text()')[0].split('：')[1]
#content1 = ''.join([text for text in article.xpath('.//div[@class="content"]//text()') if '\t' not in text])
a = response.text
content = re.search(
r'<div class=\"content\">.*?</div>', a).group(0)
# print(content)
# 写入excel
# 参数对应 行, 列, 值
print(Subtitle)
worksheet.write(d, 0, title)#标题
worksheet.write(d, 1, Subtitle)#副标题
worksheet.write(d, 2, date)#时间
worksheet.write(d, 3, "转自"+source)#来源
worksheet.write(d, 4, content)#内容：富文本
d+=1
# 保存
workbook.save('信丰脐橙果农网行业动态数据.xls')
网友评论