爬虫

import requests

from lxml import etree

from threading import Thread

import time

from queue import Queue

#单线程

base_url='https://movie.douban.com/top250?filter=&start='

movie_list=[]

for i in range(10):

    url1=base_url+str((i*25))

    response=requests.get(url=url1)

    content=response.content

    #接收数据

    content=content.decode('utf-8')

    html1 = etree.HTML(content)

    item_list=html1.xpath('//div[@class="item"]')

    for item in item_list:

        movie = {}

        movie['编号'] = item.xpath('./div[1]/em/text()')[0]

        movie['名称'] = item.xpath('./div[2]//a/span[1]/text()')[0]

        movie['评分'] = item.xpath('./div[2]//span[@class="rating_num"]/text()')[0]

        movie_list.append(movie)

print(len(movie_list))

#多线程

def douban(url1):

    response = requests.get(url=url1)

    content=response.content

    #接收数据

    content=content.decode('utf-8')

    html1 = etree.HTML(content)

    item_list=html1.xpath('//div[@class="item"]')

    for item in item_list:

        movie = {}

        movie['编号'] = item.xpath('./div[1]/em/text()')[0]

        movie['名称'] = item.xpath('./div[2]//a/span[1]/text()')[0]

        movie['评分'] = item.xpath('./div[2]//span[@class="rating_num"]/text()')[0]

        #自动创建锁防止多线程执行造成存储数据时发生混乱

        movie_list.put(movie)

if __name__ == '__main__':

    start_time=time.time()

    movie_list = Queue()

    base_url = 'https://movie.douban.com/top250?filter=&start='

    movie_list=[]

    thread_list=[]

    for i in range(10):

        url1=base_url+str((i*25))

        thread_spider=Thread(target=douban,args=[url1])

        thread_list.append(thread_spider)

        thread_spider.start()

    for thread_one in thread_list:

        thread_one.join()

    #防止主进程先运行到这句话,所以需要让住主进程等待子进程

    end_time = time.time()

    while not movie_list.empty():

        print(movie_list.get())

    print('爬取时间总结：%s' % (end_time - start_time))

#利用协程爬取数据

import requests

import json

import time

from lxml import etree

from threading import Thread

from queue import Queue

from gevent import  monkey

import gevent

import random

# gevent 让我们可以按同步的方式来写异步程序

# monkey.patch_all() 会在Python程序执行时动态的将网络库（socket,select,thread)

# 替换掉，变成异步的库，让我们的程序可以异步的方式处理网络相关的任务

monkey.patch_all()

def spider(url):

    response = requests.get(url=url,headers=headers,proxies = {'http':random.choice(proxies_ips)})

    content = response.content

    content = content.decode('utf-8')

    '''数据处理'''

    html = etree.HTML(content)

    item_list = html.xpath('//div[@class="item"]')

    for item in item_list:

        movie = {}

        movie['编号'] = item.xpath('./div[1]/em/text()')[0]

        print(movie['编号'])

        movie['名称'] = item.xpath('./div[2]//a/span[1]/text()')[0]

        movie['评分'] = item.xpath('./div[2]//span[@class="rating_num"]/text()')[0]

        movie_list.put(movie)

if __name__ == '__main__':

    proxies_ips = [

        'http://122.114.31.177:808',

        'http://61.135.217.7:80',

        'http://14.20.180.209:8118',

    ]

    headers={

         'Host':'movie.douban.com',

        'Referer':'https://accounts.douban.com/login?redir=https://movie.douban.com/top250?filter=&source=None',

        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',

    }

    start_time = time.time()

    # 存储数据

    movie_list = Queue()

    '''发送请求'''

    base_url = 'https://movie.douban.com/top250?filter=&start='

    gevent_spiders = []

    for i in range(10):

        url = base_url + str(i * 25)

        print(url)

        gevent_spider = gevent.spawn(spider,url)

        gevent_spiders.append(gevent_spider)

    gevent.joinall(gevent_spiders)

    end_time = time.time()

    print('爬取时间总结：%s' % (end_time - start_time))