Scrapy相关

输出日志文件，Settings配置：

LOG_FILE = 'DEBUG_%s.txt' % (datetime.now().strftime('%Y%m%d_%H%M_%S'))

定时任务调度：

# -*- coding: utf-8 -*- 
# @Description  : 调度程序，每天执行一次

import os
import datetime
from apscheduler.schedulers.blocking import BlockingScheduler


def job_yingyongbao():
    print('Start job yingyongbao ', datetime.datetime.now())
    os.system('scrapy crawl yingyongbao_spider')


def job_wandoujia():
    print('Start job wandoujia ', datetime.datetime.now())
    os.system('scrapy crawl wandoujia_spider')


if __name__ == '__main__':
    scheduler = BlockingScheduler()

    job1_time = datetime.datetime.now() + datetime.timedelta(seconds=5)
    job2_time = job1_time + datetime.timedelta(seconds=10)
    scheduler.add_job(job_yingyongbao, 'interval', start_date=job1_time, hours=24)
    scheduler.add_job(job_wandoujia, 'interval', start_date=job2_time, hours=24)

    scheduler.start()