Rescue and notify exceptions in feed-updater and delayed_job

Braulio Bhavamitra
1 parent 1b7dcab6
Showing 3 changed files with 48 additions and 10 deletions Show diff stats
lib/feed_updater.rb
script/delayed_job
vendor/plugins/delayed_job/lib/delayed/worker.rb
@@ -19,6 +19,18 @@ end
 # stops the process.
 class FeedUpdater
+  class ExceptionNotification < ActionMailer::Base
+    def mail error
+      environment = Environment.default
+
+      recipients NOOSFERO_CONF['exception_recipients']
+      from       environment.contact_email
+      reply_to   environment.contact_email
+      subject    "[#{environment.name}] Feed-updater: #{error.message}"
+      body       render(:text => error.backtrace.join("\n"))
+    end
+  end
+
   # indicates how much time one feed will be left without updates
   # (ActiveSupport::Duration). Default: <tt>4.hours</tt>
   cattr_accessor :update_interval
@@ -79,6 +91,8 @@ class FeedUpdater
         feed_handler.process(container)
       end
     end
+  rescue Exception => e
+    FeedUpdater::ExceptionNotification.deliver_mail e if NOOSFERO_CONF['exception_recipients'].present?
   end
 end
@@ -10,4 +10,25 @@ require File.expand_path(File.join(File.dirname(__FILE__), &#39;..&#39;, &#39;config&#39;, &#39;envi
 require 'daemons'
 require 'delayed/command'
+# based on https://groups.google.com/forum/#!topic/delayed_job/ZGMUFFppNgs
+class Delayed::Worker
+  class ExceptionNotification < ActionMailer::Base
+    def mail error
+      environment = Environment.default
+
+      recipients NOOSFERO_CONF['exception_recipients']
+      from       environment.contact_email
+      reply_to   environment.contact_email
+      subject    "[#{environment.name}] DelayedJob: #{error.message}"
+      body       render(:text => error.backtrace.join("\n"))
+    end
+  end
+
+  def handle_failed_job_with_notification job, error
+    Delayed::Worker::ExceptionNotification.deliver_mail error if NOOSFERO_CONF['exception_recipients'].present?
+    handle_failed_job_without_notification job, error 
+  end
+  alias_method_chain :handle_failed_job, :notification
+end
+
 Delayed::Command.new(ARGV).daemonize
@@ -10,12 +10,12 @@ module Delayed
     self.max_attempts = 25
     self.max_run_time = 4.hours
     self.default_priority = 0
-    
+
     # By default failed jobs are destroyed after too many attempts. If you want to keep them around
     # (perhaps to inspect the reason for the failure), set this to false.
     cattr_accessor :destroy_failed_jobs
     self.destroy_failed_jobs = true
-    
+
     self.logger = if defined?(Merb::Logger)
       Merb.logger
     elsif defined?(RAILS_DEFAULT_LOGGER)
@@ -24,9 +24,9 @@ module Delayed
     # name_prefix is ignored if name is set directly
     attr_accessor :name_prefix
-    
+
     cattr_reader :backend
-    
+
     def self.backend=(backend)
       if backend.is_a? Symbol
         require "delayed/backend/#{backend}"
@@ -35,7 +35,7 @@ module Delayed
       @@backend = backend
       silence_warnings { ::Delayed.const_set(:Job, backend) }
     end
-    
+
     def self.guess_backend
       self.backend ||= if defined?(ActiveRecord)
         :active_record
@@ -97,7 +97,7 @@ module Delayed
     ensure
       Delayed::Job.clear_locks!(name)
     end
-    
+
     # Do num jobs and return stats on success/failure.
     # Exit early if interrupted.
     def work_off(num = 100)
@@ -117,7 +117,7 @@ module Delayed
       return [success, failure]
     end
-    
+
     def run(job)
       runtime =  Benchmark.realtime do
         Timeout.timeout(self.class.max_run_time.to_i) { job.invoke_job }
@@ -129,7 +129,7 @@ module Delayed
       handle_failed_job(job, e)
       return false  # work failed
     end
-    
+
     # Reschedule the job in the future (when a job fails).
     # Uses an exponential scale depending on the number of failed attempts.
     def reschedule(job, time = nil)
@@ -162,13 +162,14 @@ module Delayed
     end
   protected
-    
+
     def handle_failed_job(job, error)
       job.last_error = error.message + "\n" + error.backtrace.join("\n")
       say "#{job.name} failed with #{error.class.name}: #{error.message} - #{job.attempts} failed attempts", Logger::ERROR
       reschedule(job)
+    rescue => e # don't crash here
     end
-    
+
     # Run the next job we can get an exclusive lock on.
     # If no jobs are left we return nil
     def reserve_and_run_one_job
@@ -186,6 +187,8 @@ module Delayed
       end
       run(job) if job
+    rescue => e
+      handle_failed_job(job, e)
     end
   end
 end
	@@ -19,6 +19,18 @@ end		@@ -19,6 +19,18 @@ end
19	# stops the process.	19	# stops the process.
20	class FeedUpdater	20	class FeedUpdater
21		21
		22	+ class ExceptionNotification < ActionMailer::Base
		23	+ def mail error
		24	+ environment = Environment.default
		25	+
		26	+ recipients NOOSFERO_CONF['exception_recipients']
		27	+ from environment.contact_email
		28	+ reply_to environment.contact_email
		29	+ subject "[#{environment.name}] Feed-updater: #{error.message}"
		30	+ body render(:text => error.backtrace.join("\n"))
		31	+ end
		32	+ end
		33	+
22	# indicates how much time one feed will be left without updates	34	# indicates how much time one feed will be left without updates
23	# (ActiveSupport::Duration). Default: <tt>4.hours</tt>	35	# (ActiveSupport::Duration). Default: <tt>4.hours</tt>
24	cattr_accessor :update_interval	36	cattr_accessor :update_interval
	@@ -79,6 +91,8 @@ class FeedUpdater		@@ -79,6 +91,8 @@ class FeedUpdater
79	feed_handler.process(container)	91	feed_handler.process(container)
80	end	92	end
81	end	93	end
		94	+ rescue Exception => e
		95	+ FeedUpdater::ExceptionNotification.deliver_mail e if NOOSFERO_CONF['exception_recipients'].present?
82	end	96	end
83	end	97	end
84		98
	@@ -10,4 +10,25 @@ require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'envi		@@ -10,4 +10,25 @@ require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'envi
10	require 'daemons'	10	require 'daemons'
11	require 'delayed/command'	11	require 'delayed/command'
12		12
		13	+# based on https://groups.google.com/forum/#!topic/delayed_job/ZGMUFFppNgs
		14	+class Delayed::Worker
		15	+ class ExceptionNotification < ActionMailer::Base
		16	+ def mail error
		17	+ environment = Environment.default
		18	+
		19	+ recipients NOOSFERO_CONF['exception_recipients']
		20	+ from environment.contact_email
		21	+ reply_to environment.contact_email
		22	+ subject "[#{environment.name}] DelayedJob: #{error.message}"
		23	+ body render(:text => error.backtrace.join("\n"))
		24	+ end
		25	+ end
		26	+
		27	+ def handle_failed_job_with_notification job, error
		28	+ Delayed::Worker::ExceptionNotification.deliver_mail error if NOOSFERO_CONF['exception_recipients'].present?
		29	+ handle_failed_job_without_notification job, error
		30	+ end
		31	+ alias_method_chain :handle_failed_job, :notification
		32	+end
		33	+
13	Delayed::Command.new(ARGV).daemonize	34	Delayed::Command.new(ARGV).daemonize