前言
最近购买了《Python3 爬虫、数据清洗与可视化实战》，刚好适逢暑假，就尝试从携程页面对广州的周边游产品进行爬虫数据捕捉。
因为才学Python不够一个星期，python的命名规范还是不太了解，只能套用之前iOS开发的命名规范，有不足之处请多多指点
一、前期
1.主要用到的库
from bs4 import BeautifulSoup
import time
import re #正则表达式
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains #浏览器操作
import xlrd
import xlwt
from xlutils.copy import copy
import os
BeautifulSoup：用于对标签等数据进行定位和抓取
selenium：用于启动浏览器和对页面进行自动操作
time：暂停等待操作
xlrd、xlwt、xlutils：对数据结果进行Excel读写保存操作
2.核心思路
1，跳进出发点的周边游页面（广州）
2，在首页捕捉推荐的热门目的地和热点景点，进行保存
3，针对目的地地点进行遍历搜索所展示的旅游产品
4，产品数据参数抓取
5，数据保存
6，退出浏览器
二、代码
1.启动浏览器
def setupDriverSetting():
global driver
# url = 'http://m.ctrip.com/restapi/soa2/10290/createclientid?systemcode=09&createtype=3&conte'#获取cookieID
# 手机端
# url = 'https://m.ctrip.com/webapp/vacations/tour/list?tab=64&kwd=%E7%8F%A0%E6%B5%B7&salecity=32&searchtype=tour&sctiy=32'
# 电脑端
url = 'https://weekend.ctrip.com/around/'
# 设置用chrome启动
driver = webdriver.Chrome()
# #设置fireFox请求头参数
# profile = webdriver.FirefoxProfile()
# user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:58.0) Gecko/20100101 Firefox/58.0"
# profile.set_preference("general.useragent.override",user_agent)
#
# driver = webdriver.Firefox(profile)
driver.get(url)
用webdriver启动Chrome或者fireFox，并跳进首页URL
2.选择出发点城市
def select_StartPlace(startPlace):
#点击出发点view
driver.find_element_by_xpath("//*[@id='CitySelect']").click()
#选择出发点
cityList = driver.find_elements_by_xpath("//*[@id='CitySelect']/dd/ul")
for link in cityList:
links = link.find_elements(By.TAG_NAME,"a")
for eachCity in links:
cityStr = eachCity.text
if cityStr == startPlace:
print("找到目标城市:"+eachCity.get_attribute('href'))
driver.get(eachCity.get_attribute('href'))
time.sleep(2)
try:
WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//*[@id='SearchText']")))
except:
print('出发地页面加载不成功')
break
主要是用find_element_by_xpath寻找目标城市进行选择筛选，然后跳到城市专页
3.搜索目的地
def finAllDestinationPage():
#查找总数组
destType = driver.find_element_by_id("J_sub_circum")#id 决定产品范围（周边游，境外游）
print(destType.text)
destType1 = destType.find_element_by_class_name("side_jmp_dest")
destTypeItem = destType1.get_attribute('innerHTML')
item = BeautifulSoup(destTypeItem,'lxml')
destTypeList = item.find_all('li')
allDestinationListDic = {}
for each in destTypeList:
typeName = each.h4.string
typeList = each.find_all('a')
list = []
for i in typeList:
list.append(i.string)
allDestinationListDic[typeName] = list
return allDestinationListDic
搜索所有可推荐目的地和景点，并用字典保存
4.旅游产品列表页
def jump_destinationPage(startPlace,destination):
#定位搜索栏
try:
WebDriverWait(driver,5).until(EC.presence_of_element_located((By.XPATH,"//*[@id='SearchText']")))
except:
print('查找不到搜索栏')
finally:
print('本地页面加载完毕')
driver.find_element_by_xpath("//input[@id='SearchText']").send_keys(destination)
print("输入目的地："+destination)
driver.find_element_by_xpath("//*[@id='SearchBtn']").click()
print("点击搜索按钮结束")
time.sleep(2)
try:
WebDriverWait(driver,5).until(EC.presence_of_element_located((By.XPATH,"//*[@id='js-dpSearcher']")))
except:
print('产品列表页加载不成功')
finally:
print('产品列表页加载完毕')
#再选一次出发地，以防出错
reSelect_StartPlace(startPlace)
#搜索页数
pageHtml = driver.find_element_by_xpath("//*[@id='_sort']/div/span")
print(pageHtml.text)
pageNumStr = pageHtml.text
pageNumStr = pageNumStr[:-1]
print("获取的num:" + pageNumStr)
#正则表达式 查找页数
pageNumS = re.findall(r'\d+',pageNumStr)
pageNum = int(pageNumS[1])
print(pageNum)
tourProductList = []
for i in range(0,pageNum):
itemList = showCurrentPageAllData()
#收集数据
for j in range(0,len(itemList)):
eachItem = collectCurrentPageEachData(j)
tourProductList.append(eachItem)
#点击下一页
driver.find_element_by_xpath("//input[@id='ipt_page_txt']").clear()
driver.find_element_by_xpath("//input[@id='ipt_page_txt']").send_keys(str(i+2))
driver.find_element_by_xpath("//*[@id='ipt_page_btn']").click()
print("点击下一页结束->"+str(i+2)+"页")
time.sleep(2)
return driver
跳进产品页，并根据标签，抓取总页数，在遍历所有旅游产品后，再跳到下一页进行循环遍历
5.产品数据抓取
def collectCurrentPageEachData(itemNum):
itemList = driver.find_elements_by_class_name("product_box")
str = itemList[itemNum].get_attribute('innerHTML')#转换成字符串
# item = BeautifulSoup(str,"html.parser")#获取item的soup对象
item = BeautifulSoup(str, "lxml") # 获取item的soup对象
# print("+++++++"+item.prettify())
# 解析
#产品名称
titleNameHtml = item.find('h2',class_= 'product_title')
print("-------"+titleNameHtml.get_text())
productName = titleNameHtml.get_text()
#产品链接
productLink = titleNameHtml.a['href']
productLink = productLink[2:]
productLink = "https://"+productLink
print("link:" + productLink)
#产品类型
productType = item.find('em')
print("type:"+productType.get_text())
productTypeStr = productType.get_text()
#产品价格
priceHtml = item.find('span',class_='sr_price')
priceStr = priceHtml.strong.get_text()
#判断是否为数字
if priceStr.isdigit() == True :
priceStr = "%.2f"%float(priceStr)
print("price:"+priceStr)
#产品供应商
productRetail = item.find('p',class_='product_retail')
productRetailStr = productRetail['title']
if "供应商" in productRetailStr:
productRetailStr = productRetailStr[4:]
print("retail:" + productRetailStr)
#产品评分
try :
gradeHtml = item.find('p', class_='grade')
gradeStr = gradeHtml.strong.get_text()
print("grade:" + gradeStr)
except:
print('查找不到评分')
gradeStr = ''
# 产品人数
try:
commentHtml = item.find('div', class_='comment')
commentStr = commentHtml.em.get_text()
commentNumS = re.findall(r'\d+', commentStr)
commentNum = int(commentNumS[0])
print("comment:",commentNum)
except:
print('查找不到出游人数')
commentNum = ''
return {
'名称':productName,
'链接':productLink,
'类型':productTypeStr,
'价格':priceStr,
'供应商':productRetailStr,
'评分':gradeStr,
'人数':commentNum,
}
在产品页面上获取所有可见信息，并返回
6.数据保存
class ExcelFileManager:
def creatExcelFile(fileName,sheetName,headRowList):
# 获取项目所在目录
filePath = os.getcwd() + '/' + fileName + '.xls'
#如果不存在就新增
try:
oldFile = xlrd.open_workbook(filePath)
file = copy(oldFile)
except:
file = xlwt.Workbook()
print("新建文件")
#如果不存在就新增
try:
sheet1 = file.add_sheet(sheetName,cell_overwrite_ok=True)
except:
sheet1 = file.get_sheet(sheetName)
#设置style样式
head_style = xlwt.easyxf('font: name Times New Roman, color-index red, bold on',num_format_str='#,##0.00')
row0 = headRowList
for i in range(0,len(row0)):
sheet1.write(0,i,row0[i],head_style)
print(filePath)
file.save(filePath)
def addDataToExcelFile(fileName,sheetName,dataList):
filePath = os.getcwd()+'/'+fileName+'.xls'
file = xlrd.open_workbook(filePath)
#已存在的行数
newRows = file.sheet_by_name(sheetName).nrows
new_File = copy(file)
sheet = new_File.get_sheet(sheetName)
try:
for i in range(0,len(dataList)):
for j in range(0,len(dataList[i])):
sheet.write(i+newRows,j,dataList[i][j])
except Exception as e:
print(e)
new_File.save(filePath)
Excel文件创建与保存数据，不得不说，python对Excel支持不是很友好，xlrd和xlwt仅支持读和写，不支持增加sheet或者在原有Excel文件上添加数据等操作，需要用到第三方库
三、抓取结果：
1530848043475.jpg
网友评论