selenium动态网页与请求

安装selenium

1. 进入虚拟环境：workon 虚拟环境名

2. pip install selenium

selenium文档

http://selenium-python-zh.readthedocs.io/en/latest/api.html

第一个selenium实例

from seleniumimport webdriver

driver = webdriver.Chrome(executable_path='D:\学习资料\python\chromedriver.exe')

driver.get("http://www.python.org")

print(driver.page_source)

driver.quit()

---

selenium得到页面结合Scrapy 的Selector选择器

from seleniumimport webdriver

from scrapy.selectorimport Selector

driver = webdriver.Chrome(executable_path='D:\学习资料\python\chromedriver.exe')

driver.get("https://detail.tmall.com/item.htm?spm=a220m.1000858.1000725.1.7a257dc20iSLSE&id=569308450159&skuId=3650260320758&user_id=2012762407&cat_id=53636001&is_b=1&rn=59df58578e268de4c0d5a925730ff828")

print(driver.page_source)

t_selectot = Selector(text=driver.page_source)

price = t_selectot.css('.tm-promo-price .tm-price::text').extract()

driver.quit()

---

selenium 完成知乎模拟登陆

from seleniumimport webdriver

from scrapy.selectorimport Selector

driver = webdriver.Chrome(executable_path='D:\学习资料\python\chromedriver.exe')

driver.get("https://www.zhihu.com/signup")

driver.find_element_by_css_selector('.SignContainer-switch span').click()# 点击切换到登陆页面

driver.find_element_by_css_selector('.Input-wrapper input[name="username"]').send_keys('****')# 输入账号

driver.find_element_by_css_selector('.Input-wrapper input[name="password"]').send_keys('*****')# 输入密码

driver.find_element_by_css_selector('.Login-content button[type="submit"]').click()# 登陆按钮

---

selenium 完成微博模拟登陆

from seleniumimport webdriver

from scrapy.selectorimport Selector

driver = webdriver.Chrome(executable_path='D:\学习资料\python\chromedriver.exe')

driver.get("https://www.weibo.com")

import time

time.sleep(15)

driver.find_element_by_css_selector('#loginname').send_keys('****')# 输入账号

driver.find_element_by_css_selector('.input_wrap input[type="password"]').send_keys('****')# 输入密码

driver.find_element_by_css_selector('.info_list.login_btn span[node-type="submitStates"]').click()# 登陆按钮

---

selenium 实现下拉条功能

from seleniumimport webdriver

from scrapy.selectorimport Selector

driver = webdriver.Chrome(executable_path='D:\学习资料\python\chromedriver.exe')

driver.get("http://smart.blogchina.com/")

import time

# 实现鼠标下拉功能,执行javascript代码

for iin range(3):

    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);"

    " var lenOfPage=document.body.scrollHeight;"

                      "return lenOfPage;")

    time.sleep(3)