# -*- coding: utf-8 -*-
# Scrapy settings for TxScrapy project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# http://doc.scrapy.org/en/latest/topics/settings.html
# http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
# http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
# 爬虫名
BOT_NAME = 'TxScrapy'
# 爬虫目录
SPIDER_MODULES = ['TxScrapy.spiders']
# 新爬虫目录
NEWSPIDER_MODULE = 'TxScrapy.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
# 设置自己的user-agent用来伪装爬虫行为
# USER_AGENT = 'TxScrapy (+http://www.yourdomain.com)'
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"
# Obey robots.txt rules
# 设置robots.txt规则(默认情况下会优先检查网站是否存在rebots.txt规则文件,存在则根据网站规则运行,无视就False)
ROBOTSTXT_OBEY = False
# 设置是否过滤重复请求,默认情况下False,只记录第一个重复请求
DUPEFILTER_DEBUG = False
# 遇到失去响应或超时的页面是否重试
RETRY_ENABLED = True
# 重试次数
RETRY_TIMES = 10
# Configure maximum concurrent requests performed by Scrapy (default: 16)
# 配置同时启动线程数量
# CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# 对于同一网站的请求间隔延迟限制,默认为0
DOWNLOAD_DELAY = 0.25
# 配置日志记录等级
LOG_LEVEL = 'DEBUG'
# The download delay setting will honor only one of: 下载延迟设置,仅一个有效(二选一设置)
# 对任何单个域名/单个IP执行的并发请求数
CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
# 是否开启COOKIES携带
COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
# 是否启用Telnet控制台?
# TELNETCONSOLE_ENABLED = False
# Override the default request headers:
# 配置默认的request headers ,请求头
# DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
# }
DEFAULT_REQUEST_HEADERS = {
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language":"zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
"Accept-Encoding":"gzip, deflate",
"Connection":"keep-alive",
}
# Enable or disable spider middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
# 启用或关闭爬虫中间件(以及配置自定义中间件)
SPIDER_MIDDLEWARES = {
'TxScrapy.middlewares.TxscrapySpiderMiddleware': 543,
}
# Enable or disable downloader middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
# 启用或关闭下载器中间件(以及配置自定义中间件)
DOWNLOADER_MIDDLEWARES = {
'TxScrapy.middlewares.MyCustomDownloaderMiddleware': 543,
}
# Enable or disable extensions
# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
# 包含项目中启用的扩展名及其顺序的字典
# EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
# }
# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
# 启用或关闭ITEM中间件(以及配置自定义中间件)
ITEM_PIPELINES = {
'TxScrapy.pipelines.TxscrapyPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See http://doc.scrapy.org/en/latest/topics/autothrottle.html
# 启用或关闭AutoThrottle扩展
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
# AutoThrottle初始下载延迟
AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
# AutoThrottle高延迟情况下的最大下载延迟(秒)
AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
# 并行发送给服务器的请求平均数
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
# 是否显示每个响应的调试信息
# AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# 启用和配置HTTP缓存(默认禁用)
# HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
# MySQL数据库配置
MYSQL_HOST = '192.168.30.161'
MYSQL_DBNAME = 'gds_argidata20170324'
MYSQL_USER = 'root'
MYSQL_PASSWD = ''
# 设置文件保存路径
FILES_STORE = '/var/spiderDownloads/'
# 文件过期时间 天
FILES_EXPIRES = 90
网友评论