window下是 dir 命令, linux是 ls
提到定时任务调度的时候，相信很多人会想到芹菜celery，要么就写个脚本塞到crontab中。不过，一个小的定时脚本，要用celery的话太“重”了。所以，我找到了一个轻量级的定时任务调度的库：schedule。库的安装还是最简单的pip install schedule，使用起来也是很容易理解的。
from scrapy import cmdline
import schedule, time, os, subprocess, logging, datetime
from subprocess import run
from multiprocessing import Process
schedule.every(10).minutes.do(job)
schedule.every().hour.do(job)
schedule.every().day.at("10:30").do(job)
schedule.every(5).to(10).days.do(job)
schedule.every().monday.do(job)
schedule.every().wednesday.at("13:15").do(job)
# -*- coding: utf-8 -*-
import subprocess, schedule, time, datetime, logging
from multiprocessing import Process
from scrapy import cmdline
def crawl_work():
print("I'm working...")
## cmd = "scrapy crawl NanPing"
## subprocess.Popen(cmd)
## run(cmd, shell=True)
## pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout
## print(pipe.read())
print('-'*100)
args = ["scrapy", "crawl", 'it']
while True:
start = time.time()
p = Process(target=cmdline.execute, args=(args,))
p.start()
p.join()
logging.debug("### use time: %s" % (time.time() - start))
if __name__=='__main__':
print('*'*10+'开始执行定时爬虫'+'*'*10)
schedule.every(1).minutes.do(crawl_work)
print('当前时间为{}'.format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
print('*' * 10 + '定时爬虫开始运行' + '*' * 10)
while True:
schedule.run_pending()
time.sleep(10)
第二种定时：
from multiprocessing import Process
from scrapy import cmdline
import time,logging
# 配置参数即可, 爬虫名称，运行频率
confs = [
{
"spider_name": "it",
"frequency": 2,
},
]
def start_spider(spider_name, frequency):
args = ["scrapy", "crawl", spider_name]
while True:
start = time.time()
p = Process(target=cmdline.execute, args=(args,))
p.start()
p.join()
logging.debug("### use time: %s" % (time.time() - start))
time.sleep(frequency)
if __name__ == '__main__':
for conf in confs:
process = Process(target=start_spider,args=(conf["spider_name"], conf["frequency"]))
process.start()
time.sleep(86400)
网友评论