python的requests库

requests各种请求方式

#版本号
In [46]: requests.__version__
Out[46]: '2.19.1'

In [6]: import requests

In [7]: r = requests.get('https://api.github.com/events')

In [8]: r = requests.post('http://httpbin.org/post', data = {'key':'value'})

In [9]: r = requests.put('http://httpbin.org/put', data = {'key':'value'})

In [10]: r = requests.delete('http://httpbin.org/delete')

In [11]: r = requests.head('http://httpbin.org/get')

In [12]: r = requests.options('http://httpbin.org/get')

url拼接

In [13]: payload = {'key1':'value1','v2':'value2'}

In [14]: r = requests.get("http://httpbin.org/get", params=payload)

In [15]: r.url
Out[15]: 'http://httpbin.org/get?key1=value1&v2=value2'

requests响应内容

In [1]: import requests

In [2]: r = requests.get("http://bj.meituan.com/meishi/")

In [3]: r.status_code
Out[3]: 200

In [4]: r.encoding
Out[4]: 'utf-8'

In [5]: r.headers
Out[5]: {'Server': 'Tengine', 'Date': 'Thu, 12 Jul 2018 09:50:36 GMT', 'Content-Type': 'text/html; charset=utf-8', 'Transfer-Encoding': 'chunked', 'Connection': 'keep-alive', 'X-Powered-By': 'dx-web-meishife-pcweb04.dx.sankuai.com_production', 'Set-Cookie': 'client-id=2247b634-52ff-4a27-a0a9-01b1810e6d9f; path=/; expires=Fri, 13 Jul 2018 09:50:36 GMT; httponly, uuid=9b6cad4c-90fa-4f51-a59c-5f4f37cd4464; path=/; domain=meituan.com; httponly', 'Cache-Control': 'no-cache', 'Vary': 'Accept-Encoding', 'Content-Encoding': 'gzip'}
#请求头大小写不敏感可以不区分大小写调用,例:r.headers.get('server')

In [6]: r.text
#内容太多

In [47]: r = requests.get('https://api.github.com/events')
In [48]: r.json()
#示例返回2条内容
[{'id': '7959832918',
  'type': 'PushEvent',
  'actor': {'id': 7868421,
   'login': 'whitestone8214',
   'display_login': 'whitestone8214',
   'gravatar_id': '',
   'url': 'https://api.github.com/users/whitestone8214',
   'avatar_url': 'https://avatars.githubusercontent.com/u/7868421?'},
  'repo': {'id': 139670300,
   'name': 'whitestone8214/Crawler',
   'url': 'https://api.github.com/repos/whitestone8214/Crawler'},
  'payload': {'push_id': 2716420119,
   'size': 1,
   'distinct_size': 1,
   'ref': 'refs/heads/master',
   'head': 'd178c8282b826cf360e3e4656d0258aedaaace36',
   'before': '6c08e2c6f66243c1003df07a80af10e337def37b',
   'commits': [{'sha': 'd178c8282b826cf360e3e4656d0258aedaaace36',
     'author': {'email': 'whitestone8214@openmailbox.org', 'name': 'Minho Jo'},
     'message': "#2: whiteline.js 0.2.1 (과 그에 따른 마이그레이션); 입력란 크기 조절; 다시 그린 툴바 아이콘(일단은 툴바 쪽만); 컨텍스트 메뉴 '텍스트 에디터로 열기'; 기타 등등",
     'distinct': True,
     'url': 'https://api.github.com/repos/whitestone8214/Crawler/commits/d178c8282b826cf360e3e4656d0258aedaaace36'}]},
  'public': True,
  'created_at': '2018-07-13T03:27:17Z'},
 {'id': '7959832917',
  'type': 'WatchEvent',
  'actor': {'id': 5679265,
   'login': 'LFH104',
   'display_login': 'LFH104',
   'gravatar_id': '',
   'url': 'https://api.github.com/users/LFH104',
   'avatar_url': 'https://avatars.githubusercontent.com/u/5679265?'},
  'repo': {'id': 129860458,
   'name': 'phobal/ivideo',
   'url': 'https://api.github.com/repos/phobal/ivideo'},
  'payload': {'action': 'started'},
  'public': True,
  'created_at': '2018-07-13T03:27:17Z'}]

In [48]: r = requests.get('https://api.github.com/events', stream=True)
ln [50]: r.raw
Out[50]: <urllib3.response.HTTPResponse at 0x7fe0a51a4278>

In [51]: r.raw.read(10)
Out[51]: b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03'

In [74]: r = requests.get('http://github.com')
In [75]: r.url
Out[75]: 'https://github.com/'

In [76]: r.history
Out[76]: [<Response [301]>]

post请求form表单

#字典形式
In [13]: payload = {'key1':'value1','v2':'value2'}
In [16]: r = requests.post("http://httpbin.org/post", data=payload)

In [17]: r.text
Out[17]: '{"args":{},"data":"","files":{},"form":{"key1":"value1","v2":"value2"},"headers":{"Accept":"*/*","Accept-Encoding":"gzip, deflate","Connection":"close","Content-Length":"21","Content-Type":"application/x-www-form-urlencoded","Host":"httpbin.org","User-Agent":"python-requests/2.18.4"},"json":null,"origin":"47.90.49.89","url":"http://httpbin.org/post"}\n'

#元组形式
In [18]: payload = (('key1', 'value1'), ('key1', 'value2'))
In [19]: r = requests.post("http://httpbin.org/post", data=payload)

In [20]: r.text
Out[20]: '{"args":{},"data":"","files":{},"form":{"key1":["value1","value2"]},"headers":{"Accept":"*/*","Accept-Encoding":"gzip, deflate","Connection":"close","Content-Length":"23","Content-Type":"application/x-www-form-urlencoded","Host":"httpbin.org","User-Agent":"python-requests/2.18.4"},"json":null,"origin":"47.90.49.89","url":"http://httpbin.org/post"}\n'

自定义请求头

In [52]: url = 'https://api.github.com/some/endpoint'

In [53]: headers = {'user-agent': 'my-app/0.0.1'}

In [54]: r = requests.get(url, headers=headers)
栗子:
import requests
import json

from datetime import date,datetime
import time


# 发送群消息
url = "https://api.telegram.org/bot"
headers = {'Content-Type': 'application/json'}
data =  {
  "chat_id": -12345,       
  "text": "**hi**",            
  "reply_markup": {             
  "inline_keyboard": [[{      
      "text":"google",              
      "url":"www.google.com"    
  }]]
  },
  "parse_mode":"Markdown"      
}
content = requests.post(url,headers=headers,data=json.dumps(data))
data =json.loads(content.text)
print(data)

自定义cookie

In [61]: url = 'http://httpbin.org/cookies'

In [62]: cookies = dict(cookies_are='working')

In [63]: r = requests.get(url, cookies=cookies)

In [64]: r.text
Out[64]: '{"cookies":{"cookies_are":"working"}}\n'

In [65]: jar = requests.cookies.RequestsCookieJar()

In [66]: jar.set('tasty_cookie', 'yum', domain='httpbin.org', path='/cookies')
Out[66]: Cookie(version=0, name='tasty_cookie', value='yum', port=None, port_specified=False, domain='httpbin.org', domain_specified=True, domain_initial_dot=False, path='/cookies', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)

In [67]: jar.set('gross_cookie', 'blech', domain='httpbin.org', path='/elsewhere ')
Out[67]: Cookie(version=0, name='gross_cookie', value='blech', port=None, port_specified=False, domain='httpbin.org', domain_specified=True, domain_initial_dot=False, path='/elsewhere', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)

In [68]: url = 'http://httpbin.org/cookies'

In [69]: r = requests.get(url, cookies=jar)

In [70]: r.text
Out[70]: '{"cookies":{"tasty_cookie":"yum"}}\n'

In [71]: url = 'http://httpbin.org/elsewhere'

In [72]: r = requests.get(url, cookies=jar)

In [73]: r.text
Out[73]: '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">\n<title>404 Not Found</title>\n<h1>Not Found</h1>\n<p>The requested URL was not found on the server.  If you entered the URL manually please check your spelling and try again.</p>\n'

requests爬虫小栗子

import requests
import json
import redis
from datetime import date,datetime
import time

r = redis.StrictRedis('localhost',6379,decode_responses=True,db = 3)


class JsonExtendEncoder(json.JSONEncoder):
    """
        This class provide an extension to json serialization for datetime/date.
    """
    def default(self, obj):
        """
            provide a interface for datetime/date
        """
        if isinstance(obj, datetime):
            return obj.strftime('%Y-%m-%d %H:%M:%S')
        elif isinstance(obj, date):
            return obj.strftime('%Y-%m-%d')
        else:
            return json.JSONEncoder.default(self, obj)
i = 0

while True:
    p = r.pipeline()
    url = "https://api.schail.com/v3/ticker/summary?type=0&sort=1&offset=0&limit=100&top=0"
    time.sleep(2)
    content = requests.get(url)
    data =json.loads(content.text)
    data_dict = {}
    data = data.get('data')
    data_dict['data'] = data
    current_time = datetime.now()
    data_dict['current_time'] = current_time
    # print(data_dict)
    exchange_list = []
    for tid_dict in data.get('summaryList'):

        pair_url = "https://api.schail.com/v1/ticker/exchange?tickerId=%s&offset=0&limit=50"%tid_dict['tickerId']
        pair_content = requests.get(pair_url)
        pair_data = json.loads(pair_content.text)
        exchange_dict = {}
        
        exchange_dict['data'] = pair_data['data']
        current_time = datetime.now()
        exchange_dict['current_time'] = current_time
        
        exchange_list.append(exchange_dict)
    
    data_dict['exchange_list'] = exchange_list 
    print(data_dict)   
    i += 1
    p.execute()   
    r.zadd("tokenclubdata", i,json.dumps(data_dict,ensure_ascii=False,cls=JsonExtendEncoder))



# https://api.schail.com/v1/ticker/exchange?tickerId=bitcoin&offset=0&limit=50
# https://api.schail.com/v1/ticker/exchange?tickerId=ethereum&offset=0&limit=50