From 9db0c1975e3143930e5f100751465076a2db2d57 Mon Sep 17 00:00:00 2001 From: Braulio Bhavamitra Date: Tue, 13 Aug 2013 13:56:39 -0300 Subject: [PATCH] Rescue and notify exceptions in feed-updater and delayed_job --- lib/feed_updater.rb | 14 ++++++++++++++ script/delayed_job | 21 +++++++++++++++++++++ vendor/plugins/delayed_job/lib/delayed/worker.rb | 23 +++++++++++++---------- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/lib/feed_updater.rb b/lib/feed_updater.rb index 7a9f7ec..0d579d0 100644 --- a/lib/feed_updater.rb +++ b/lib/feed_updater.rb @@ -19,6 +19,18 @@ end # stops the process. class FeedUpdater + class ExceptionNotification < ActionMailer::Base + def mail error + environment = Environment.default + + recipients NOOSFERO_CONF['exception_recipients'] + from environment.contact_email + reply_to environment.contact_email + subject "[#{environment.name}] Feed-updater: #{error.message}" + body render(:text => error.backtrace.join("\n")) + end + end + # indicates how much time one feed will be left without updates # (ActiveSupport::Duration). Default: 4.hours cattr_accessor :update_interval @@ -79,6 +91,8 @@ class FeedUpdater feed_handler.process(container) end end + rescue Exception => e + FeedUpdater::ExceptionNotification.deliver_mail e if NOOSFERO_CONF['exception_recipients'].present? end end diff --git a/script/delayed_job b/script/delayed_job index 513d609..87ee978 100755 --- a/script/delayed_job +++ b/script/delayed_job @@ -10,4 +10,25 @@ require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'envi require 'daemons' require 'delayed/command' +# based on https://groups.google.com/forum/#!topic/delayed_job/ZGMUFFppNgs +class Delayed::Worker + class ExceptionNotification < ActionMailer::Base + def mail error + environment = Environment.default + + recipients NOOSFERO_CONF['exception_recipients'] + from environment.contact_email + reply_to environment.contact_email + subject "[#{environment.name}] DelayedJob: #{error.message}" + body render(:text => error.backtrace.join("\n")) + end + end + + def handle_failed_job_with_notification job, error + Delayed::Worker::ExceptionNotification.deliver_mail error if NOOSFERO_CONF['exception_recipients'].present? + handle_failed_job_without_notification job, error + end + alias_method_chain :handle_failed_job, :notification +end + Delayed::Command.new(ARGV).daemonize diff --git a/vendor/plugins/delayed_job/lib/delayed/worker.rb b/vendor/plugins/delayed_job/lib/delayed/worker.rb index aeb11c7..53e6607 100644 --- a/vendor/plugins/delayed_job/lib/delayed/worker.rb +++ b/vendor/plugins/delayed_job/lib/delayed/worker.rb @@ -10,12 +10,12 @@ module Delayed self.max_attempts = 25 self.max_run_time = 4.hours self.default_priority = 0 - + # By default failed jobs are destroyed after too many attempts. If you want to keep them around # (perhaps to inspect the reason for the failure), set this to false. cattr_accessor :destroy_failed_jobs self.destroy_failed_jobs = true - + self.logger = if defined?(Merb::Logger) Merb.logger elsif defined?(RAILS_DEFAULT_LOGGER) @@ -24,9 +24,9 @@ module Delayed # name_prefix is ignored if name is set directly attr_accessor :name_prefix - + cattr_reader :backend - + def self.backend=(backend) if backend.is_a? Symbol require "delayed/backend/#{backend}" @@ -35,7 +35,7 @@ module Delayed @@backend = backend silence_warnings { ::Delayed.const_set(:Job, backend) } end - + def self.guess_backend self.backend ||= if defined?(ActiveRecord) :active_record @@ -97,7 +97,7 @@ module Delayed ensure Delayed::Job.clear_locks!(name) end - + # Do num jobs and return stats on success/failure. # Exit early if interrupted. def work_off(num = 100) @@ -117,7 +117,7 @@ module Delayed return [success, failure] end - + def run(job) runtime = Benchmark.realtime do Timeout.timeout(self.class.max_run_time.to_i) { job.invoke_job } @@ -129,7 +129,7 @@ module Delayed handle_failed_job(job, e) return false # work failed end - + # Reschedule the job in the future (when a job fails). # Uses an exponential scale depending on the number of failed attempts. def reschedule(job, time = nil) @@ -162,13 +162,14 @@ module Delayed end protected - + def handle_failed_job(job, error) job.last_error = error.message + "\n" + error.backtrace.join("\n") say "#{job.name} failed with #{error.class.name}: #{error.message} - #{job.attempts} failed attempts", Logger::ERROR reschedule(job) + rescue => e # don't crash here end - + # Run the next job we can get an exclusive lock on. # If no jobs are left we return nil def reserve_and_run_one_job @@ -186,6 +187,8 @@ module Delayed end run(job) if job + rescue => e + handle_failed_job(job, e) end end end -- libgit2 0.21.2