小试爬虫

由于种种原因，简书等第三方平台博客不再保证能够同步更新，欢迎移步 GitHub：https://github.com/kingcos/Perspective/。谢谢！

• Info:
• macOS 10.12 beta
• Python 2.7.10
• PyCharm 2016.2

安装 Python

macOS 自带了 2.7 版本的 Python，因此无需安装。

注意：
在 PyCharm 中开发，有时候会默认为 2.6.9 的 Python，在 Preferences-Project: xxx-Project Interpreter 中，将 Python 版本切换为 2.7.10。（我仅在本文的 Demo Pro 中采用了 PyCharm IDE 开发）

Terminal 输入 python，即可查看 Python 版本。

Python 2.7.10 (default, Aug  1 2016, 20:48:38) 
[GCC 4.2.1 Compatible Apple LLVM 8.0.0 (clang-800.0.34)] on darwin
Type "help", "copyright", "credits" or "license" for more information.

众所周知，Python 分为两大版本，2.7.x & 3.x，两者是不兼容的，暂时这里以 2.7.x 版本为例。稍后可能更新 3.x 版甚至 Java 版的爬虫 Demo。

安装 pip

pip 即 Python 的包管理工具，我们需要包管理工具来下载所需要的第三方包。

macOS 虽然有 Python 环境，但是没有 pip 插件。

Terminal 输入 sudo easy_install pip，再输入用户密码即可安装。

Terminal 输入 pip，即可验证安装成功。

Usage:   
  pip <command> [options]

...

使用 pip 安装 Beautiful Soup 4

Beautiful Soup 是 Python 的一个第三方包，用于从 HTML 或 XML 文档中提取数据。安装 BS4 的方法有很多，这里使用 pip 安装。

在 Terminal 输入 sudo pip install beautifulsoup4，再输入用户密码即可安装。

Demo

以下 Demo 均可以爬取到相关网页的内容，但由于各种原因，以下网站的链接均删去，参考时请根据具体网站具体分析。有些网站的反爬虫机制比较健全，导致返回错误码，可以查阅相关模拟浏览器或模拟登陆的资料。

爬取学校的某个网站的照片

# coding: utf8

import urllib, urllib2, os, os.path
from bs4 import BeautifulSoup

def getSoup(url):
    if url is None:
        return None
    response = urllib2.urlopen(url)

    if response.getcode() != 200:
        return None
    
    soup = BeautifulSoup(response, 'html.parser')
    return soup

def getImage(url, prefix):
    soup = getSoup(url)
    images = soup.find(name='div', attrs={'class':'six alpha'}).find_all('img')
    i = 0
    for img in images:
        image = 'http://www.ngsying.cn' + images[i]['src']
        i += 1
        urllib.urlretrieve(image, prefix + ' %d.jpg' % i)

def downloadImages(urls, marks):
    i = 0
    for url in urls:
        if not os.path.exists(marks[i]):
            os.mkdir(marks[i])
        prefix = os.path.join(marks[i], marks[i])
        getImage(url, prefix)
        i += 1

if __name__ == '__main__':
    print 'Welcome to kingcos\' PicDownloader!'

    urls = set()
    marks = []
    marks.append('20160910')
    marks.append('20160911')
    marks.append('20160912')
    marks.append('20160913')
    marks.append('20160914')
    marks.append('20160915')
    marks.append('20160916')

    urls.add('http://www.***.cn/show.asp?id=67')
    urls.add('http://www.***.cn/show.asp?id=60')
    urls.add('http://www.***.cn/show.asp?id=61')
    urls.add('http://www.***.cn/show.asp?id=62')
    urls.add('http://www.***.cn/show.asp?id=63')
    urls.add('http://www.***.cn/show.asp?id=64')
    urls.add('http://www.***.cn/show.asp?id=65')

    downloadImages(urls, marks)
    print 'Downloading has been finished.'

爬取某图

# coding: utf8

import urllib, urllib2, os, os.path, random
from bs4 import BeautifulSoup

global n
n = 1

def getSoup(url):
    if url is None:
        return None

    response = urllib2.urlopen(url)

    if response.getcode() != 200:
        return None
    
    soup = BeautifulSoup(response, 'html.parser')
    return soup

def getImage(url, prefix):
    soup = getSoup(url)
    images = soup.find(name='div', attrs={'id':'srcPic'}).find_all('img')
    i = 0
    for img in images:
        image = images[i]['src']
        i += 1
        global n
        urllib.urlretrieve(image, prefix + ' %d.jpg' % n)
        n += 1

def downloadImages(urls, marks):
    i = 0
    for url in urls:
        if not os.path.exists(marks[i]):
            os.mkdir(marks[i])
        prefix = os.path.join(marks[i], marks[i])
        getImage(url, prefix)
        # i += 1

if __name__ == '__main__':
    print 'Welcome to kingcos\' PicDownloader!'

    urls = set()
    marks = []
    marks.append('24974')

    for i in range(1, 10):
        url = 'http://www.***.com/meimei/24974-' + str(i) + '.html'
        urls.add(url)

    downloadImages(urls, marks)
    print 'Downloading has been finished.'

爬取某站图片

该网站有一定的反爬机制，通过设置 header 绕过。

# coding: utf8

import urllib, urllib2, os, os.path, random
from bs4 import BeautifulSoup

def getSoup(url):
    if url is None:
        return None

    header = { "User-Agent": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36' }
    req = urllib2.Request(url, headers=header)
    response = urllib2.urlopen(req)

    if response.getcode() != 200:
        return None
    
    soup = BeautifulSoup(response, 'html.parser')
    return soup

def getImage(url, prefix):
    soup = getSoup(url)
    images = soup.find(name='div', attrs={'class':'main-body'}).find_all('a')
    i = 0
    for img in images:
        image = img['href']
        i += 1
        urllib.urlretrieve(image, prefix + ' %d.jpg' % i)

def downloadImages(urls, marks):
    i = 0
    for url in urls:
        if not os.path.exists(marks[i]):
            os.mkdir(marks[i])
        prefix = os.path.join(marks[i], marks[i])
        getImage(url, prefix)
        i += 1

if __name__ == '__main__':
    print 'Welcome to kingcos\' PicDownloader!'

    urls = set()
    marks = ['14007', '13998', '13987']

    for m in marks:
        url = 'http://www.***.***/' + m + '/'
        urls.add(url)

    downloadImages(urls, marks)
    print 'Downloading has been finished.'

Demo Pro

上面的 Demo 仅仅是爬去某个单独的网页，下面是可以爬取相关链接指定图片的 Demo。即指定一个根 URL，其可以爬取到其他符合条件的 URL，进而爬取其中的内容，而不局限于传入的指定 URL。测试爬取了 1000+ 张图片。

爬去某站的相关图片

crawler_main.py

入口主函数：
负责统一调用其他类的方法

# coding: utf8

from Crawler import url_manager, html_downloader, html_parser, html_outputer

class CrawlerMain(object):

    def __init__(self):
        self.urls = url_manager.UrlManager()
        self.downloader = html_downloader.HtmlDownloader()
        self.parser = html_parser.HtmlParser()
        self.outputer = html_outputer.HtmlOutputer()

    def craw(self, root_url):
        count = 1
        self.urls.add_new_url(root_url)
        while self.urls.has_new_url():
            try:
                new_url = self.urls.get_new_url()
                print 'craw %d: %s' % (count, new_url)
                html_cont = self.downloader.download(new_url)
                new_urls, new_data = self.parser.parse(new_url, html_cont)
                self.urls.add_new_urls(new_urls)
                self.outputer.collect_data(new_data)

                if count == 100:
                    break
                count += 1
            except:
                print 'Crawl finished!'

if __name__ == "__main__":
    print 'Welcome to kingcos\' PicDownloader!'
    root_url = 'http://www.***.***/'
    obj_spider = CrawlerMain()
    obj_spider.craw(root_url)

url_manager.py

URL 管理器：
负责管理 URL，包括添加新 URL，判断是非存在未爬取链接等方法

# coding: utf8

class UrlManager():
    def __init__(self):
        self.new_urls = set()
        self.old_urls = set()

    def add_new_url(self, url):
        if url is None:
            return
        if url not in self.new_urls and url not in self.old_urls:
            self.new_urls.add(url)

    def add_new_urls(self, urls):
        if urls is None or len(urls) == 0:
            return
        for url in urls:
            self.add_new_url(url)

    def has_new_url(self):
        return len(self.new_urls) != 0

    def get_new_url(self):
        new_url =  self.new_urls.pop()
        self.old_urls.add(new_url)
        return new_url

html_downloader.py

HTML 下载器：
负责传回指定 URL 的响应

# coding: utf8

import urllib2

class HtmlDownloader():

    def download(self, url):
        if url is None:
            return None

        header = {
            "User-Agent": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36'}
        req = urllib2.Request(url, headers=header)
        response = urllib2.urlopen(req)

        if response.getcode() != 200:
            return None

        return response

html_parser.py

HTML 转换器：
对新的 URL 对应的 HTML 分析，提取所需内容，并提取符合需求的 URL，加入到待爬取 URL 集合。

# coding: utf8

import re, urlparse
from bs4 import BeautifulSoup

class HtmlParser():

    def _get_new_urls(self, page_url, soup):
        new_urls = set()
        links = soup.find_all('a', href = re.compile(r"http://www.***.***/\d+/"))
        for link in links:
            new_url = link['href']
            new_full_url = urlparse.urljoin(page_url, new_url)
            new_urls.add(new_full_url)
        return new_urls
    
    def _get_new_data(self, page_url, soup):
        res_data = {}
        res_data['url'] = page_url
        images = soup.find(name='div', attrs={'class':'main-body'}).find_all('a')
        return images

    def parse(self, page_url, html_cont):

        if page_url is None or html_cont is None:
            return
        
        soup = BeautifulSoup(html_cont, 'html.parser')
        new_urls = self._get_new_urls(page_url, soup)
        new_data = self._get_new_data(page_url, soup)
        return new_urls, new_data

html_outputer.py

HTML 输出器：
对收集到数据进行取回，放置在本地文件夹内

# coding: utf8

import urllib, random, os

class HtmlOutputer():

    def collect_data(self, images):
        i = 0
        for img in images:
            image = img['href']
            i += 1
            if not os.path.exists('img'):
                os.mkdir('img')
            prefix = os.path.join('img', 'img')
            urllib.urlretrieve(image, prefix + '%d' % i + str(random.randint(1, 100)) + '.jpg')

参考资料

Python 入门
Python 开发简单爬虫
Beautiful Soup 官方网站
Beatiful Soup 4 Docs