爬虫之携程网旅游信息爬取

最近手又有点闲，闲着无聊就爬取了携程的信息，这个爬虫是我随手写的，各位可以看看。

# -*- encoding=utf-8 -*-
#爬取携程旅游信息
#2017/8/14

from bs4 import BeautifulSoup

from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
import os
import csv
import time

driver = webdriver.Chrome()

#打开网页
def gethtml(place):
    try:
        driver.get('http://vacations.ctrip.com/')
        welcome = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "body > div.jewel_pop_box > div.jewel_pop > span")))
        welcome.click()
        seaech = driver.find_element_by_css_selector('#searchpanel > div.search_wrap > div.new_search_content > div > input')
        check = driver.find_element_by_css_selector('#searchpanel > div.search_wrap > a.main_search_btn')
        seaech.send_keys(place)
        check.click()
        return driver.page_source
    except TimeoutException:
        gethtml(place)

#换页
def changepage(page):
    try:
        time.sleep(4)
        js="var q=document.documentElement.scrollTop=10000"
        driver.execute_script(js)
        p =  WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR,'#ipt_page_txt')))
        submit = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR,'#ipt_page_btn')))
        p.clear()
        p.send_keys(page)
        submit.click()
        return driver.page_source
    except:
        return changepage(page)

#解析页面
def checkpage(html):
    soup = BeautifulSoup(html,'lxml')
    links = soup.find_all('div',class_='main_mod product_box flag_product ')
    for link in links:
        name =link.h2.get_text()
        href = 'http:' + link.h2.a['href']
        price = link.find('span',class_='sr_price').get_text()[:-1]
        try:
            agree =  link.find('p',class_='grade').get_text()[:-1]
        except:
            agree = ''
        try:
            alp = link.find('div',class_='comment')
            people = alp.em.string[:-3]
            about = alp.a.get_text()[:-3]
        except:
            people = ''
            about = ''
        yield {
            '产品':name,
            '链接':href,
            '价格':price,
            '评分':agree,
            '人数':people,
            '点评':about,
        }

    
#创建文件夹
def make(place):
    path = 'D:/数据/'
    if not os.path.exists:
        os.makedirs(path)
    with open(path + place +'旅游信息.csv','w') as f:
        writer = csv.writer(f)
        writer.writerow(['产品','链接','价格','评分','人数','点评'])
        f.close()
    def save_to_csv(i):
        with open(path + place +'旅游信息.csv','a') as f:
            writer = csv.writer(f)
            try:
                writer.writerow([ i['产品'],i['链接'],i['价格'],i['评分'], i['人数'],i['点评'] ])
            except:
                pass
            f.close()
    return save_to_csv

#主函数
def main(place):
    save = make(place)
    html = gethtml(place)
    for i in checkpage(html):
        save(i)
        print(i)
    for i in range(2,101):
        
        html = changepage(i)
        for fil in checkpage(html):
            save(fil)
            print(fil)
    driver.quit() 

#执行程序
if __name__ == '__main__':
    place = input('请输入要查询的地点：')
    main(place)

u=4098028651,4134268318&fm=202.jpg