Urllib库相关操作

1.什么是Urllib

• Python内置的HTTP请求库
• urllib.request 请求模块
• urllib.error 异常处理模块
• urllib.parse url解析模块
• urllib.robotparser robots.txt解析模块

2.用法讲解

2.1urlopen

# get
import urllib.request

response = urllib.request.urlopen('http://www.zhihu.com')
print(response.read().decode('utf-8'))

#post
import urllib.parse
import urllib.request

data = bytes(urllib.parse.urlencode({'word':'hello'}), encoding='utf-8')
response = urllib.request.urlopen('http://httpbin.org/post', data=data)
print(response.read().decode('utf-8'))

#timeout
import socket
import urllib.request
import urllib.error

try:
    response = urllib.request.urlopen('http://httpbin.org/get', timeout=0.1)
except urllib.error.URLError as e:
    if isinstance(e.reason, socket.timeout):
        print("Time Out!")

2.2响应

响应类型

import urllib.request

response = urllib.request.urlopen('http://www.baidu.com')
print(type(response))

状态码、响应头

import urllib.request

response = urllib.request.urlopen('http://www.python.org')
print(response.status)
print(response.getheaders())
print(response.getheader('Server'))

Request

import urllib.request

request = urllib.request.Request('http://www.python.org')
response = urllib.request.urlopen(request)
print(response.read().decode('utf-8'))

import urllib.parse

url = 'http://httpbin.org/post'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36',
    'Host':'httpbin.org'
}
dict = {
    'name':'Yuanshuo'
}
data = bytes(urllib.parse.urlencode(dict), encoding='utf-8')
request = urllib.request.Request(url=url, data=data, headers=headers, method='POST')
response = urllib.request.urlopen(request)
print(response.read().decode('utf-8'))

Handler

代理

# 
import urllib.request

proxy_handler = urllib.request.ProxyHandler({
    'http':'http://104.156.225.251:7777',
    'https':'https://104.156.225.251:7777'
})
opener = urllib.request.build_opener(proxy_handler)
response = opener.open('http://www.httpbin.org/get')
print(response.read().decode('utf-8'))

Cookie

import http.cookiejar, urllib.request

cookie = http.cookiejar.CookieJar()
handle = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handle)
response = opener.open('http://www.baidu.com')
for item in cookie:
    print(item.name+'='+item.value)

#本地存储cookie
import http.cookiejar, urllib.request

filename = 'cookie.txt'
cookie = http.cookiejar.LWPCookieJar(filename)
handle = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handle)
response = opener.open('http://www.baidu.com')
cookie.save(ignore_discard=True, ignore_expires=True)

#读取本地cookie
import http.cookiejar, urllib.request

cookie = http.cookiejar.LWPCookieJar()
cookie.load('cookie.txt', ignore_discard=True, ignore_expires=True)
handle = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handle)
response = opener.open('http://www.baidu.com')
print(response.read().decode('utf-8'))

异常处理

import urllib
import socket

try:
    response = urllib.request.urlopen('http://www.yuanshuo.com')
except urllib.error.HTTPError as e:
    print(e.reason, e.code, e.headers, sep='\n')
except urllib.error.URLError as e:
    print(e.reason)
    if isinstance(e.reason, socket.timeout):
        print('Time Out')
else:
    print('Request Successfully')

URL解析

urlparse

from urllib.parse import urlparse

result = urlparse('https://www.bilibili.com/video/av52151245?t=1273&p=8')
print(type(result), result)

urlencode

from urllib.parse import urlencode

params = {
    'name':'Yuanshuo',
    'age':'57'
}
base_url = 'http://www.com?'
url = base_url + urlencode(params)
print(url)