basis_spider

• 获取html文本，取出回应结果

import requests
res = requests.get('http://news.sina.com.cn/c/2018-08-27/doc-ihiixzkm1477063.shtml')
res.encoding = 'utf-8'
print(res.text)

• 剖析网页元素，取出里边的文字内容

import requests
from bs4 import BeautifulSoup
res = requests.get('http://news.sina.com.cn/c/2018-08-27/doc-ihiixzkm1477063.shtml')
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text,"html.parser")
print(soup.text)

• 取出特殊标签和结点中的资料

import requests
from bs4 import BeautifulSoup
res = requests.get('http://news.sina.com.cn/c/2018-08-27/doc-ihiixzkm1477063.shtml')
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text,"html.parser")

header = soup.select('h1')
print(header)
print(header[1].text)

alink = soup.select('a')
print(alink)
for link in alink:
    print(link.text)

• 取出含有特定css属性的元素,id前带#，class前带点

import requests
from bs4 import BeautifulSoup

res = requests.get('http://news.sina.com.cn/c/2018-08-27/doc-ihiixzkm1477063.shtml')
res.encoding = 'utf-8'

soup = BeautifulSoup(res.text,"html.parser")

alink = soup.select('.main-title')
print(alink[0].text)