rework the problem cache and refingerprinting code

Now that it is easy to change fingerprinting logic, it is important that users have a way to regenerate fingerprints on existing notices. This commit reorganizes and unifies problem refingerprinting between both merge/unmerge and the global refingerprinting rake task.

rework the problem cache and refingerprinting code
Now that it is easy to change fingerprinting logic, it is important that users have a way to regenerate fingerprints on existing notices. This commit reorganizes and unifies problem refingerprinting between both merge/unmerge and the global refingerprinting rake task.
Stephen Crosby
1 parent 9c40727d
Showing 15 changed files with 255 additions and 341 deletions Show diff stats
app/interactors/notice_refingerprinter.rb
app/interactors/problem_merge.rb
app/interactors/problem_recacher.rb
app/interactors/problem_updater_cache.rb
app/models/notice.rb
app/models/notice_fingerprinter.rb
app/models/problem.rb
lib/tasks/errbit/database.rake
spec/fabricators/backtrace_fabricator.rb
spec/fabricators/err_fabricator.rb
spec/fabricators/notice_fabricator.rb
spec/interactors/notice_refingerprinter_spec.rb
spec/interactors/problem_merge_spec.rb
spec/interactors/problem_updater_cache_spec.rb
spec/models/problem_spec.rb
@@ -0,0 +1,36 @@
+class NoticeRefingerprinter
+  LOG_EVERY = 100
+  LOG_ITR = '%.1f%% complete, %i notice(s) remaining'
+  LOG_START = 'Regenerating notice fingerprints for %i notices'
+
+  def self.run
+    count = Notice.count
+    puts format(LOG_START, count)
+
+    Notice.no_timeout.each_with_index do |notice, index|
+      refingerprint(notice)
+
+      next unless (index + 1) % LOG_EVERY == 0
+      puts format(LOG_ITR, (index * 100 / count), count - index)
+    end
+
+    puts 'Finished generating notice fingerprints'
+  end
+
+  def self.apps
+    @apps ||= App.all.index_by(&:id)
+  end
+
+  def self.refingerprint(notice)
+    app = apps[notice.try(:err).try(:problem).try(:app_id)]
+    app.find_or_create_err!(
+      error_class: notice.error_class,
+      environment: notice.environment_name,
+      fingerprint: app.notice_fingerprinter.generate(app.api_key, notice, notice.backtrace)
+    )
+  end
+
+  def self.puts(*args)
+    Rails.logger.info(*args)
+  end
+end
@@ -16,13 +16,7 @@ class ProblemMerge
       problem.reload # deference all associate objet to avoid delete him after
       ProblemDestroy.execute(problem)
     end
-    reset_cached_attributes
+    merged_problem.recache
     merged_problem
   end
-
-  private
-
-  def reset_cached_attributes
-    ProblemUpdaterCache.new(merged_problem).update
-  end
 end
@@ -0,0 +1,24 @@
+class ProblemRecacher
+  LOG_EVERY = 100
+  LOG_ITR = '%.1f%% complete, %i problem(s) remaining'
+  LOG_START = 'Re-caching problem attributes for %i problems'
+
+  def self.run
+    count = Problem.count
+    puts format(LOG_START, count)
+
+    Problem.no_timeout.each_with_index do |problem, index|
+      problem.recache
+      problem.destroy if problem.notices_count == 0
+
+      next unless (index + 1) % LOG_EVERY == 0
+      puts format(LOG_ITR, (index * 100 / count), count - index)
+    end
+
+    puts "Finished re-caching problem attributes"
+  end
+
+  def self.puts(*args)
+    Rails.logger.info(*args)
+  end
+end
@@ -1,88 +0,0 @@
-class ProblemUpdaterCache
-  def initialize(problem, notice=nil)
-    @problem = problem
-    @notice = notice
-  end
-  attr_reader :problem
-
-  ##
-  # Update cache information about child associate to this problem
-  #
-  # update the notices count, and some notice informations
-  #
-  # @return [ Problem ] the problem with this update
-  #
-  def update
-    update_notices_count
-    update_notices_cache
-    problem
-  end
-
-  private
-
-  def update_notices_count
-    if @notice
-      problem.inc(notices_count: 1)
-    else
-      problem.update_attribute(
-        :notices_count, problem.notices.count
-      )
-    end
-  end
-
-  ##
-  # Update problem statistique from some notice information
-  #
-  def update_notices_cache
-    first_notice = notices.first
-    last_notice = notices.last
-    notice ||= @notice || first_notice
-
-    attrs = {}
-    attrs[:first_notice_at] = first_notice.created_at if first_notice
-    attrs[:last_notice_at] = last_notice.created_at if last_notice
-    attrs.merge!(
-      :message     => notice.message,
-      :where       => notice.where,
-      :messages    => attribute_count(:message, messages),
-      :hosts       => attribute_count(:host, hosts),
-      :user_agents => attribute_count(:user_agent_string, user_agents)
-    ) if notice
-    problem.update_attributes!(attrs)
-  end
-
-  def notices
-    @notices ||= @notice ? [@notice].sort(&:created_at) : problem.notices.order_by([:created_at, :asc])
-  end
-
-  def messages
-    @notice ? problem.messages : {}
-  end
-
-  def hosts
-    @notice ? problem.hosts : {}
-  end
-
-  def user_agents
-    @notice ? problem.user_agents : {}
-  end
-
-  private
-
-    def attribute_count(value, init)
-      init.tap do |counts|
-        notices.each do |notice|
-          counts[attribute_index(notice.send(value))] ||= {
-            'value' => notice.send(value),
-            'count' => 0
-          }
-          counts[attribute_index(notice.send(value))]['count'] += 1
-        end
-      end
-    end
-
-    def attribute_index(value)
-      @attributes_index ||= {}
-      @attributes_index[value.to_s] ||= Digest::MD5.hexdigest(value.to_s)
-    end
-end
@@ -48,7 +48,8 @@ class Notice
   end
   def environment_name
-    server_environment['server-environment'] || server_environment['environment-name']
+    n = server_environment['server-environment'] || server_environment['environment-name']
+    n.blank? ? 'development' : n
   end
   def component
@@ -119,6 +120,12 @@ class Notice
     end
   end
+  # filter memory addresses out of object strings
+  # example: "#<Object:0x007fa2b33d9458>" becomes "#<Object>"
+  def filtered_message
+    message.gsub(/(#<.+?):[0-9a-f]x[0-9a-f]+(>)/, '\1\2')
+  end
+
   protected
   def problem_recache
@@ -16,7 +16,7 @@ class NoticeFingerprinter
   def generate(api_key, notice, backtrace)
     material = []
     material << notice.error_class if error_class
-    material << notice.message if message
+    material << notice.filtered_message if message
     material << notice.component if component
     material << notice.action if action
     material << notice.environment_name if environment_name
@@ -142,16 +142,22 @@ class Problem
         { "$match" => { err_id: { "$in" => err_ids } } },
         { "$group" => { _id: "$#{v}", count: {"$sum" => 1} } }
       ]).each do |agg|
-        next if agg[:_id] == nil
-
-        send(k)[Digest::MD5.hexdigest(agg[:_id])] = {
-          value: agg[:_id],
-          count: agg[:count]
+        send(k)[Digest::MD5.hexdigest(agg[:_id] || 'N/A')] = {
+          'value' => agg[:_id] || 'N/A',
+          'count' => agg[:count]
         }
       end
     end
-    self.notices_count = Notice.where({ err_id: { "$in" => err_ids }}).count
+    first_notice = notices.order_by([:created_at, :asc]).first
+    last_notice = notices.order_by([:created_at, :desc]).first
+
+    self.notices_count = notices.count
+    self.first_notice_at = first_notice.created_at if first_notice
+    self.message = first_notice.message if first_notice
+    self.where = first_notice.where if first_notice
+    self.last_notice_at = last_notice.created_at if last_notice
+
     save
   end
-require 'digest/sha1'
-
 namespace :errbit do
-  namespace :db do
-
-    desc "Updates cached attributes on Problem"
-    task :update_problem_attrs => :environment do
-      puts "Updating problems"
-      Problem.no_timeout.all.each do |problem|
-        ProblemUpdaterCache.new(problem).update
-      end
-    end
-
-    desc "Updates Problem#notices_count"
-    task :update_notices_count => :environment do
-      puts "Updating problem.notices_count"
-      Problem.no_timeout.all.each do |pr|
-        pr.update_attributes(:notices_count => pr.notices.count)
-      end
-    end
-
-    desc "Delete resolved errors from the database. (Useful for limited heroku databases)"
-    task :clear_resolved => :environment do
-      require 'resolved_problem_clearer'
-      puts "=== Cleared #{ResolvedProblemClearer.new.execute} resolved errors from the database."
-    end
-
-    desc "Regenerate fingerprints"
-    task :regenerate_fingerprints => :environment do
-
-      def normalize_backtrace(backtrace)
-        backtrace[0...3].map do |trace|
-          trace.merge 'method' => trace['method'].to_s.gsub(/[0-9_]{10,}+/, "__FRAGMENT__")
-        end
-      end
+  desc "Updates cached attributes on Problem"
+  task :problem_recache => :environment do
+    ProblemRecacher.run
+  end
-      def fingerprint(source)
-        Digest::SHA1.hexdigest(source.to_s)
-      end
+  desc "Delete resolved errors from the database. (Useful for limited heroku databases)"
+  task :clear_resolved => :environment do
+    require 'resolved_problem_clearer'
+    puts "=== Cleared #{ResolvedProblemClearer.new.execute} resolved errors from the database."
+  end
-      puts "Regenerating Err fingerprints"
-      Err.create_indexes
-      Err.all.each do |err|
-        next if err.notices.count == 0
-        source = {
-          :backtrace => normalize_backtrace(err.notices.first.backtrace).to_s,
-          :error_class => err.error_class,
-          :component => err.component,
-          :action => err.action,
-          :environment => err.environment,
-          :api_key => err.app.api_key
-        }
-        err.update_attributes(:fingerprint => fingerprint(source))
-      end
-    end
+  desc "Regenerate fingerprints"
+  task :notice_refingerprint => :environment do
+    NoticeRefingerprinter.run
+    ProblemRecacher.run
+  end
-    desc "Remove notices in batch"
-    task :notices_delete, [ :problem_id ] => [ :environment ] do
-      BATCH_SIZE = 1000
-      if args[:problem_id]
-        item_count = Problem.find(args[:problem_id]).notices.count
-        removed_count = 0
-        puts "Notices to remove: #{item_count}"
-        while item_count > 0
-          Problem.find(args[:problem_id]).notices.limit(BATCH_SIZE).each do |notice|
-            notice.remove
-            removed_count += 1
-          end
-          item_count -= BATCH_SIZE
-          puts "Removed #{removed_count} notices"
+  desc "Remove notices in batch"
+  task :notices_delete, [ :problem_id ] => [ :environment ] do
+    BATCH_SIZE = 1000
+    if args[:problem_id]
+      item_count = Problem.find(args[:problem_id]).notices.count
+      removed_count = 0
+      puts "Notices to remove: #{item_count}"
+      while item_count > 0
+        Problem.find(args[:problem_id]).notices.limit(BATCH_SIZE).each do |notice|
+          notice.remove
+          removed_count += 1
         end
+        item_count -= BATCH_SIZE
+        puts "Removed #{removed_count} notices"
       end
     end
   end
@@ -0,0 +1,19 @@
+Fabricator :backtrace do
+  lines(:count => 99) do
+    {
+      number: rand(999),
+      file: "/path/to/file/#{SecureRandom.hex(4)}.rb",
+      method: ActiveSupport.methods.shuffle.first
+    }
+  end
+end
+
+Fabricator :short_backtrace do
+  lines(:count => 5) do
+    {
+      number: rand(999),
+      file: "/path/to/file/#{SecureRandom.hex(4)}.rb",
+      method: ActiveSupport.methods.shuffle.first
+    }
+  end
+end
@@ -2,29 +2,3 @@ Fabricator :err do
   problem
   fingerprint 'some-finger-print'
 end
-
-Fabricator :notice do
-  app
-  err
-  error_class         'FooError'
-  message             'Too Much Bar'
-  backtrace
-  server_environment  { {'environment-name' => 'production'} }
-  request             {{ 'component' => 'foo', 'action' => 'bar' }}
-  notifier            {{ 'name' => 'Notifier', 'version' => '1', 'url' => 'http://toad.com' }}
-
-  after_create do
-    Problem.cache_notice(err.problem_id, self)
-    problem.reload
-  end
-end
-
-Fabricator :backtrace do
-  lines(:count => 99) do
-    {
-      number: rand(999),
-      file: "/path/to/file/#{SecureRandom.hex(4)}.rb",
-      method: ActiveSupport.methods.shuffle.first
-    }
-  end
-end
@@ -0,0 +1,14 @@
+Fabricator :notice do
+  err
+  error_class         'FooError'
+  message             'Too Much Bar'
+  backtrace
+  server_environment  {{ 'environment-name' => 'production' }}
+  request             {{ 'component' => 'foo', 'action' => 'bar' }}
+  notifier            {{ 'name' => 'Notifier', 'version' => '1', 'url' => 'http://toad.com' }}
+
+  after_create do
+    Problem.cache_notice(err.problem_id, self)
+    problem.reload
+  end
+end
@@ -0,0 +1,16 @@
+describe NoticeRefingerprinter do
+  let(:backtrace) do
+    Fabricate(:backtrace)
+  end
+
+  let(:notices) do
+    5.times.map do
+      Fabricate(:notice, backtrace: backtrace)
+    end
+  end
+
+  it 'shits' do
+    binding.pry
+    1
+  end
+end
@@ -47,7 +47,7 @@ describe ProblemMerge do
     end
     it 'update problem cache' do
-      expect(ProblemUpdaterCache).to receive(:new).with(problem).and_return(double(:update => true))
+      expect(problem).to receive(:recache)
       problem_merge.merge
     end
@@ -1,145 +0,0 @@
-describe ProblemUpdaterCache do
-  let(:problem) { Fabricate(:problem_with_errs) }
-  let(:first_errs) { problem.errs }
-  let!(:notice) { Fabricate(:notice, :err => first_errs.first) }
-
-  describe "#update" do
-    context "without notice pass args" do
-      before do
-        problem.update_attribute(:notices_count, 0)
-      end
-
-      it 'update the notice_count' do
-        expect {
-          ProblemUpdaterCache.new(problem).update
-        }.to change{
-          problem.notices_count
-        }.from(0).to(1)
-      end
-
-      context "with only one notice" do
-        before do
-          problem.update_attributes!(:messages => {})
-          ProblemUpdaterCache.new(problem).update
-        end
-
-        it 'update information about this notice' do
-          expect(problem.message).to eq notice.message
-          expect(problem.where).to eq notice.where
-        end
-
-        it 'update first_notice_at' do
-          expect(problem.first_notice_at).to eq notice.reload.created_at
-        end
-
-        it 'update last_notice_at' do
-          expect(problem.last_notice_at).to eq notice.reload.created_at
-        end
-
-        it 'update stats messages' do
-          expect(problem.messages).to eq({
-            Digest::MD5.hexdigest(notice.message) => {'value' => notice.message, 'count' => 1}
-          })
-        end
-
-        it 'update stats hosts' do
-          expect(problem.hosts).to eq({
-            Digest::MD5.hexdigest(notice.host) => {'value' => notice.host, 'count' => 1}
-          })
-        end
-
-        it 'update stats user_agents' do
-          expect(problem.user_agents).to eq({
-            Digest::MD5.hexdigest(notice.user_agent_string) => {'value' => notice.user_agent_string, 'count' => 1}
-          })
-        end
-      end
-
-      context "with several notices" do
-        let!(:notice_2) { Fabricate(:notice, :err => first_errs.first) }
-        let!(:notice_3) { Fabricate(:notice, :err => first_errs.first) }
-        before do
-          problem.update_attributes!(:messages => {})
-          ProblemUpdaterCache.new(problem).update
-        end
-        it 'update information about this notice' do
-          expect(problem.message).to eq notice.message
-          expect(problem.where).to eq notice.where
-        end
-
-        it 'update first_notice_at' do
-          expect(problem.first_notice_at.to_i).to be_within(2).of(notice.created_at.to_i)
-        end
-
-        it 'update last_notice_at' do
-          expect(problem.last_notice_at.to_i).to be_within(2).of(notice.created_at.to_i)
-        end
-
-        it 'update stats messages' do
-          expect(problem.messages).to eq({
-            Digest::MD5.hexdigest(notice.message) => {'value' => notice.message, 'count' => 3}
-          })
-        end
-
-        it 'update stats hosts' do
-          expect(problem.hosts).to eq({
-            Digest::MD5.hexdigest(notice.host) => {'value' => notice.host, 'count' => 3}
-          })
-        end
-
-        it 'update stats user_agents' do
-          expect(problem.user_agents).to eq({
-            Digest::MD5.hexdigest(notice.user_agent_string) => {'value' => notice.user_agent_string, 'count' => 3}
-          })
-        end
-
-      end
-    end
-
-    context "with notice pass in args" do
-
-      before do
-        ProblemUpdaterCache.new(problem, notice).update
-      end
-
-      it 'increase notices_count by 1' do
-        expect {
-          ProblemUpdaterCache.new(problem, notice).update
-        }.to change{
-          problem.notices_count
-        }.by(1)
-      end
-
-      it 'update information about this notice' do
-        expect(problem.message).to eq notice.message
-        expect(problem.where).to eq notice.where
-      end
-
-      it 'update first_notice_at' do
-        expect(problem.first_notice_at).to eq notice.created_at
-      end
-
-      it 'update last_notice_at' do
-        expect(problem.last_notice_at).to eq notice.created_at
-      end
-
-      it 'inc stats messages' do
-        expect(problem.messages).to eq({
-          Digest::MD5.hexdigest(notice.message) => {'value' => notice.message, 'count' => 2}
-        })
-      end
-
-      it 'inc stats hosts' do
-        expect(problem.hosts).to eq({
-          Digest::MD5.hexdigest(notice.host) => {'value' => notice.host, 'count' => 2}
-        })
-      end
-
-      it 'inc stats user_agents' do
-        expect(problem.user_agents).to eq({
-          Digest::MD5.hexdigest(notice.user_agent_string) => {'value' => notice.user_agent_string, 'count' => 2}
-        })
-      end
-    end
-  end
-end
@@ -397,4 +397,100 @@ describe Problem, type: &#39;model&#39; do
       end
     end
   end
+
+  describe '#recache' do
+    let(:problem) { Fabricate(:problem_with_errs) }
+    let(:first_errs) { problem.errs }
+    let!(:notice) { Fabricate(:notice, :err => first_errs.first) }
+
+    before do
+      problem.update_attribute(:notices_count, 0)
+    end
+
+    it 'update the notice_count' do
+      expect {
+        problem.recache
+      }.to change{
+        problem.notices_count
+      }.from(0).to(1)
+    end
+
+    context "with only one notice" do
+      before do
+        problem.update_attributes!(:messages => {})
+        problem.recache
+      end
+
+      it 'update information about this notice' do
+        expect(problem.message).to eq notice.message
+        expect(problem.where).to eq notice.where
+      end
+
+      it 'update first_notice_at' do
+        expect(problem.first_notice_at).to eq notice.reload.created_at
+      end
+
+      it 'update last_notice_at' do
+        expect(problem.last_notice_at).to eq notice.reload.created_at
+      end
+
+      it 'update stats messages' do
+        expect(problem.messages).to eq({
+          Digest::MD5.hexdigest(notice.message) => {'value' => notice.message, 'count' => 1}
+        })
+      end
+
+      it 'update stats hosts' do
+        expect(problem.hosts).to eq({
+          Digest::MD5.hexdigest(notice.host) => {'value' => notice.host, 'count' => 1}
+        })
+      end
+
+      it 'update stats user_agents' do
+        expect(problem.user_agents).to eq({
+          Digest::MD5.hexdigest(notice.user_agent_string) => {'value' => notice.user_agent_string, 'count' => 1}
+        })
+      end
+    end
+
+    context "with several notices" do
+      let!(:notice_2) { Fabricate(:notice, :err => first_errs.first) }
+      let!(:notice_3) { Fabricate(:notice, :err => first_errs.first) }
+      before do
+        problem.update_attributes!(:messages => {})
+        problem.recache
+      end
+
+      it 'update information about this notice' do
+        expect(problem.message).to eq notice.message
+        expect(problem.where).to eq notice.where
+      end
+
+      it 'update first_notice_at' do
+        expect(problem.first_notice_at.to_i).to be_within(2).of(notice.created_at.to_i)
+      end
+
+      it 'update last_notice_at' do
+        expect(problem.last_notice_at.to_i).to be_within(2).of(notice.created_at.to_i)
+      end
+
+      it 'update stats messages' do
+        expect(problem.messages).to eq({
+          Digest::MD5.hexdigest(notice.message) => {'value' => notice.message, 'count' => 3}
+        })
+      end
+
+      it 'update stats hosts' do
+        expect(problem.hosts).to eq({
+          Digest::MD5.hexdigest(notice.host) => {'value' => notice.host, 'count' => 3}
+        })
+      end
+
+      it 'update stats user_agents' do
+        expect(problem.user_agents).to eq({
+          Digest::MD5.hexdigest(notice.user_agent_string) => {'value' => notice.user_agent_string, 'count' => 3}
+        })
+      end
+    end
+  end
 end
@@ -0,0 +1,36 @@		@@ -0,0 +1,36 @@
	1	+class NoticeRefingerprinter
	2	+ LOG_EVERY = 100
	3	+ LOG_ITR = '%.1f%% complete, %i notice(s) remaining'
	4	+ LOG_START = 'Regenerating notice fingerprints for %i notices'
	5	+
	6	+ def self.run
	7	+ count = Notice.count
	8	+ puts format(LOG_START, count)
	9	+
	10	+ Notice.no_timeout.each_with_index do \|notice, index\|
	11	+ refingerprint(notice)
	12	+
	13	+ next unless (index + 1) % LOG_EVERY == 0
	14	+ puts format(LOG_ITR, (index * 100 / count), count - index)
	15	+ end
	16	+
	17	+ puts 'Finished generating notice fingerprints'
	18	+ end
	19	+
	20	+ def self.apps
	21	+ @apps \|\|= App.all.index_by(&:id)
	22	+ end
	23	+
	24	+ def self.refingerprint(notice)
	25	+ app = apps[notice.try(:err).try(:problem).try(:app_id)]
	26	+ app.find_or_create_err!(
	27	+ error_class: notice.error_class,
	28	+ environment: notice.environment_name,
	29	+ fingerprint: app.notice_fingerprinter.generate(app.api_key, notice, notice.backtrace)
	30	+ )
	31	+ end
	32	+
	33	+ def self.puts(*args)
	34	+ Rails.logger.info(*args)
	35	+ end
	36	+end
@@ -0,0 +1,24 @@		@@ -0,0 +1,24 @@
	1	+class ProblemRecacher
	2	+ LOG_EVERY = 100
	3	+ LOG_ITR = '%.1f%% complete, %i problem(s) remaining'
	4	+ LOG_START = 'Re-caching problem attributes for %i problems'
	5	+
	6	+ def self.run
	7	+ count = Problem.count
	8	+ puts format(LOG_START, count)
	9	+
	10	+ Problem.no_timeout.each_with_index do \|problem, index\|
	11	+ problem.recache
	12	+ problem.destroy if problem.notices_count == 0
	13	+
	14	+ next unless (index + 1) % LOG_EVERY == 0
	15	+ puts format(LOG_ITR, (index * 100 / count), count - index)
	16	+ end
	17	+
	18	+ puts "Finished re-caching problem attributes"
	19	+ end
	20	+
	21	+ def self.puts(*args)
	22	+ Rails.logger.info(*args)
	23	+ end
	24	+end
	@@ -1,88 +0,0 @@	@@ -1,88 +0,0 @@
1	-class ProblemUpdaterCache
2	- def initialize(problem, notice=nil)
3	- @problem = problem
4	- @notice = notice
5	- end
6	- attr_reader :problem
7	-
8	- ##
9	- # Update cache information about child associate to this problem
10	- #
11	- # update the notices count, and some notice informations
12	- #
13	- # @return [ Problem ] the problem with this update
14	- #
15	- def update
16	- update_notices_count
17	- update_notices_cache
18	- problem
19	- end
20	-
21	- private
22	-
23	- def update_notices_count
24	- if @notice
25	- problem.inc(notices_count: 1)
26	- else
27	- problem.update_attribute(
28	- :notices_count, problem.notices.count
29	- )
30	- end
31	- end
32	-
33	- ##
34	- # Update problem statistique from some notice information
35	- #
36	- def update_notices_cache
37	- first_notice = notices.first
38	- last_notice = notices.last
39	- notice \|\|= @notice \|\| first_notice
40	-
41	- attrs = {}
42	- attrs[:first_notice_at] = first_notice.created_at if first_notice
43	- attrs[:last_notice_at] = last_notice.created_at if last_notice
44	- attrs.merge!(
45	- :message => notice.message,
46	- :where => notice.where,
47	- :messages => attribute_count(:message, messages),
48	- :hosts => attribute_count(:host, hosts),
49	- :user_agents => attribute_count(:user_agent_string, user_agents)
50	- ) if notice
51	- problem.update_attributes!(attrs)
52	- end
53	-
54	- def notices
55	- @notices \|\|= @notice ? [@notice].sort(&:created_at) : problem.notices.order_by([:created_at, :asc])
56	- end
57	-
58	- def messages
59	- @notice ? problem.messages : {}
60	- end
61	-
62	- def hosts
63	- @notice ? problem.hosts : {}
64	- end
65	-
66	- def user_agents
67	- @notice ? problem.user_agents : {}
68	- end
69	-
70	- private
71	-
72	- def attribute_count(value, init)
73	- init.tap do \|counts\|
74	- notices.each do \|notice\|
75	- counts[attribute_index(notice.send(value))] \|\|= {
76	- 'value' => notice.send(value),
77	- 'count' => 0
78	- }
79	- counts[attribute_index(notice.send(value))]['count'] += 1
80	- end
81	- end
82	- end
83	-
84	- def attribute_index(value)
85	- @attributes_index \|\|= {}
86	- @attributes_index[value.to_s] \|\|= Digest::MD5.hexdigest(value.to_s)
87	- end
88	-end
	@@ -16,7 +16,7 @@ class NoticeFingerprinter		@@ -16,7 +16,7 @@ class NoticeFingerprinter
16	def generate(api_key, notice, backtrace)	16	def generate(api_key, notice, backtrace)
17	material = []	17	material = []
18	material << notice.error_class if error_class	18	material << notice.error_class if error_class
19	- material << notice.message if message	19	+ material << notice.filtered_message if message
20	material << notice.component if component	20	material << notice.component if component
21	material << notice.action if action	21	material << notice.action if action
22	material << notice.environment_name if environment_name	22	material << notice.environment_name if environment_name
1	-require 'digest/sha1'
2	-
3	namespace :errbit do	1	namespace :errbit do
4	- namespace :db do
5	-
6	- desc "Updates cached attributes on Problem"
7	- task :update_problem_attrs => :environment do
8	- puts "Updating problems"
9	- Problem.no_timeout.all.each do \|problem\|
10	- ProblemUpdaterCache.new(problem).update
11	- end
12	- end
13	-
14	- desc "Updates Problem#notices_count"
15	- task :update_notices_count => :environment do
16	- puts "Updating problem.notices_count"
17	- Problem.no_timeout.all.each do \|pr\|
18	- pr.update_attributes(:notices_count => pr.notices.count)
19	- end
20	- end
21	-
22	- desc "Delete resolved errors from the database. (Useful for limited heroku databases)"
23	- task :clear_resolved => :environment do
24	- require 'resolved_problem_clearer'
25	- puts "=== Cleared #{ResolvedProblemClearer.new.execute} resolved errors from the database."
26	- end
27	-
28	- desc "Regenerate fingerprints"
29	- task :regenerate_fingerprints => :environment do
30	-
31	- def normalize_backtrace(backtrace)
32	- backtrace[0...3].map do \|trace\|
33	- trace.merge 'method' => trace['method'].to_s.gsub(/[0-9_]{10,}+/, "__FRAGMENT__")
34	- end
35	- end	2	+ desc "Updates cached attributes on Problem"
		3	+ task :problem_recache => :environment do
		4	+ ProblemRecacher.run
		5	+ end
36		6
37	- def fingerprint(source)
38	- Digest::SHA1.hexdigest(source.to_s)
39	- end	7	+ desc "Delete resolved errors from the database. (Useful for limited heroku databases)"
		8	+ task :clear_resolved => :environment do
		9	+ require 'resolved_problem_clearer'
		10	+ puts "=== Cleared #{ResolvedProblemClearer.new.execute} resolved errors from the database."
		11	+ end
40		12
41	- puts "Regenerating Err fingerprints"
42	- Err.create_indexes
43	- Err.all.each do \|err\|
44	- next if err.notices.count == 0
45	- source = {
46	- :backtrace => normalize_backtrace(err.notices.first.backtrace).to_s,
47	- :error_class => err.error_class,
48	- :component => err.component,
49	- :action => err.action,
50	- :environment => err.environment,
51	- :api_key => err.app.api_key
52	- }
53	- err.update_attributes(:fingerprint => fingerprint(source))
54	- end
55	- end	13	+ desc "Regenerate fingerprints"
		14	+ task :notice_refingerprint => :environment do
		15	+ NoticeRefingerprinter.run
		16	+ ProblemRecacher.run
		17	+ end
56		18
57	- desc "Remove notices in batch"
58	- task :notices_delete, [ :problem_id ] => [ :environment ] do
59	- BATCH_SIZE = 1000
60	- if args[:problem_id]
61	- item_count = Problem.find(args[:problem_id]).notices.count
62	- removed_count = 0
63	- puts "Notices to remove: #{item_count}"
64	- while item_count > 0
65	- Problem.find(args[:problem_id]).notices.limit(BATCH_SIZE).each do \|notice\|
66	- notice.remove
67	- removed_count += 1
68	- end
69	- item_count -= BATCH_SIZE
70	- puts "Removed #{removed_count} notices"	19	+ desc "Remove notices in batch"
		20	+ task :notices_delete, [ :problem_id ] => [ :environment ] do
		21	+ BATCH_SIZE = 1000
		22	+ if args[:problem_id]
		23	+ item_count = Problem.find(args[:problem_id]).notices.count
		24	+ removed_count = 0
		25	+ puts "Notices to remove: #{item_count}"
		26	+ while item_count > 0
		27	+ Problem.find(args[:problem_id]).notices.limit(BATCH_SIZE).each do \|notice\|
		28	+ notice.remove
		29	+ removed_count += 1
71	end	30	end
		31	+ item_count -= BATCH_SIZE
		32	+ puts "Removed #{removed_count} notices"
72	end	33	end
73	end	34	end
74	end	35	end
@@ -0,0 +1,19 @@		@@ -0,0 +1,19 @@
	1	+Fabricator :backtrace do
	2	+ lines(:count => 99) do
	3	+ {
	4	+ number: rand(999),
	5	+ file: "/path/to/file/#{SecureRandom.hex(4)}.rb",
	6	+ method: ActiveSupport.methods.shuffle.first
	7	+ }
	8	+ end
	9	+end
	10	+
	11	+Fabricator :short_backtrace do
	12	+ lines(:count => 5) do
	13	+ {
	14	+ number: rand(999),
	15	+ file: "/path/to/file/#{SecureRandom.hex(4)}.rb",
	16	+ method: ActiveSupport.methods.shuffle.first
	17	+ }
	18	+ end
	19	+end
	@@ -2,29 +2,3 @@ Fabricator :err do		@@ -2,29 +2,3 @@ Fabricator :err do
2	problem	2	problem
3	fingerprint 'some-finger-print'	3	fingerprint 'some-finger-print'
4	end	4	end
5	-
6	-Fabricator :notice do
7	- app
8	- err
9	- error_class 'FooError'
10	- message 'Too Much Bar'
11	- backtrace
12	- server_environment { {'environment-name' => 'production'} }
13	- request {{ 'component' => 'foo', 'action' => 'bar' }}
14	- notifier {{ 'name' => 'Notifier', 'version' => '1', 'url' => 'http://toad.com' }}
15	-
16	- after_create do
17	- Problem.cache_notice(err.problem_id, self)
18	- problem.reload
19	- end
20	-end
21	-
22	-Fabricator :backtrace do
23	- lines(:count => 99) do
24	- {
25	- number: rand(999),
26	- file: "/path/to/file/#{SecureRandom.hex(4)}.rb",
27	- method: ActiveSupport.methods.shuffle.first
28	- }
29	- end
30	-end
@@ -0,0 +1,14 @@		@@ -0,0 +1,14 @@
	1	+Fabricator :notice do
	2	+ err
	3	+ error_class 'FooError'
	4	+ message 'Too Much Bar'
	5	+ backtrace
	6	+ server_environment {{ 'environment-name' => 'production' }}
	7	+ request {{ 'component' => 'foo', 'action' => 'bar' }}
	8	+ notifier {{ 'name' => 'Notifier', 'version' => '1', 'url' => 'http://toad.com' }}
	9	+
	10	+ after_create do
	11	+ Problem.cache_notice(err.problem_id, self)
	12	+ problem.reload
	13	+ end
	14	+end
@@ -0,0 +1,16 @@		@@ -0,0 +1,16 @@
	1	+describe NoticeRefingerprinter do
	2	+ let(:backtrace) do
	3	+ Fabricate(:backtrace)
	4	+ end
	5	+
	6	+ let(:notices) do
	7	+ 5.times.map do
	8	+ Fabricate(:notice, backtrace: backtrace)
	9	+ end
	10	+ end
	11	+
	12	+ it 'shits' do
	13	+ binding.pry
	14	+ 1
	15	+ end
	16	+end
	@@ -47,7 +47,7 @@ describe ProblemMerge do		@@ -47,7 +47,7 @@ describe ProblemMerge do
47	end	47	end
48		48
49	it 'update problem cache' do	49	it 'update problem cache' do
50	- expect(ProblemUpdaterCache).to receive(:new).with(problem).and_return(double(:update => true))	50	+ expect(problem).to receive(:recache)
51	problem_merge.merge	51	problem_merge.merge
52	end	52	end
53		53