python3下scrapy调用浏览器登入browsercook

Middleware.py 构建中间件

from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
import browser_cookie3     #pip install browser_cookie3
class BrowserCookiesMiddleware(CookiesMiddleware):
    def __init__(self,debug=False):
            super().__init__(debug)
            self.load_browser_cookies()
 
    def load_browser_cookies(self):
 
        #jar=self.jars['chrome']
        #chrome_cookies=browser_cookie3.chrome()
        #for cookie in chrome_cookies:
        #    jar.set_cookie(cookie)

        jar=self.jars['firefox']
        firefox_cookeis=browser_cookie3.firefox()
        for cookie in firefox_cookeis:
            jar.set_cookie(cookie)

在settings中禁用内置的CookiesMiddleware，启用自定义的DownloaderMiddleware。

DOWNLOADER_MIDDLEWARES = {
    'xxx.middlewares.BrowserCookiesDownloaderMiddleware':910，
        'scrapy.downloadermiddlewares.cookies.CookiesMiddleware':None,
}
COOKIES_ENABLED = True

构造Request对象时，需加上cookiejar对象

#scrapy.Request(url, meta={'cookiejar':'chrome'}）
scrapy.Request(url, meta={'cookiejar':'firefox'}）