scrapy爬虫框架采集完成后进行暂停延时关闭方法

一、scrapy延时关闭

import re
import time
import uuid
from datetime import datetime
import scrapy
from scrapy.signalmanager import SignalManager
from scrapy import signals
from pydispatch import dispatcherclass TestSpider(scrapy.Spider):name = "test_spider"start_time = time.time()def start_requests(self):SignalManager(dispatcher.Any).connect(self.closed_handler, signal=signals.spider_closed)def closed_handler(self):total_scrapy_time = time.time() - self.start_timeremain_time = self.settings['CLOSESPIDER_TIMEOUT'] - total_scrapy_timeif remain_time > 0:time.sleep(remain_time-10)