Python爬虫集

这里是最近几年写过的爬虫,做一个分享

1. NASA Mars图片爬虫

NASA的图片是可以通过接口获取到,这就很简单,我们只需要访问接口去获取就行了

程序运行截图
爬取的图片

import requests
import json

headers = {
    "user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
}

params = {
    'size':100,
    'from':100    ,
    'sort':'promo-date-time:desc',
    'q':'((ubernode-type:image) AND (topics:3152))',
    '_source_include':'promo-date-time,master-image,nid,title,topics,missions,collections,other-tags,ubernode-type,primary-tag,secondary-tag,cardfeed-title,type,collection-asset-link,link-or-attachment,pr-leader-sentence,image-feature-caption,attachments,uri'
}
# https://www.nasa.gov/sites/default/files/thumbnails/image/pia23454.jpg

public_path = "https://www.nasa.gov/sites/default/files/"

rsp=requests.get('https://www.nasa.gov/api/2/ubernode/_search',headers=headers,params=params)
# print(rsp.text)
b = json.loads  (rsp.text)
hits = b['hits']['hits']
for h in hits:
    _source = h['_source']
    img_path = _source['master-image']['uri']
    title = _source['title'].replace(' ',"_").replace("'","").replace("/","-")
    img_path = img_path.replace('public://',public_path)
    print(img_path)
    try:

        r=requests.get(img_path,stream=True,headers=headers,timeout=10)
        f=open('./img/%s.jpg'%title,'wb')
        for a in r.iter_content(chunk_size=1024):
            f.write(a)
        f.close()
        print("Downloaded: "+title)
    except:
        print("Download failed: "+title)