From db3fc55258dfaf5b13982b2395865d599573a473 Mon Sep 17 00:00:00 2001 From: Marcin Ciunelis Date: Thu, 3 Nov 2011 00:09:21 +0100 Subject: [PATCH] Problem notices cache is more space effective --- app/helpers/application_helper.rb | 4 ++-- app/models/problem.rb | 46 ++++++++++++++++++++++++++++++++++++---------- db/migrate/20111102173347_cache_problem_statistics_fix.rb | 33 +++++++++++++++++++++++++++++++++ spec/models/problem_spec.rb | 24 ++++++++++++------------ 4 files changed, 83 insertions(+), 24 deletions(-) create mode 100644 db/migrate/20111102173347_cache_problem_statistics_fix.rb diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 440cb84..07503fe 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -47,8 +47,8 @@ module ApplicationHelper end def tally(collection) - collection.inject({}) do |tallies, value| - tallies[value] = (tallies[value] || 0) + 1 + collection.values.inject({}) do |tallies, tally| + tallies[tally['value']] = tally['count'] tallies end end diff --git a/app/models/problem.rb b/app/models/problem.rb index 66e5b02..c474a34 100644 --- a/app/models/problem.rb +++ b/app/models/problem.rb @@ -18,9 +18,9 @@ class Problem field :environment field :klass field :where - field :user_agents, :type => Array, :default => [] - field :messages, :type => Array, :default => [] - field :hosts, :type => Array, :default => [] + field :user_agents, :type => Hash, :default => {} + field :messages, :type => Hash, :default => {} + field :hosts, :type => Hash, :default => {} field :comments_count, :type => Integer, :default => 0 index :app_id @@ -128,19 +128,45 @@ class Problem :environment => notice.environment_name, :klass => notice.klass, :where => notice.where, - :messages => messages.push(notice.message), - :hosts => hosts.push(notice.host), - :user_agents => user_agents.push(notice.user_agent_string) + :messages => attribute_count_increase(:messages, notice.message), + :hosts => attribute_count_increase(:hosts, notice.host), + :user_agents => attribute_count_increase(:user_agents, notice.user_agent_string) ) if notice update_attributes!(attrs) end def remove_cached_notice_attribures(notice) - messages.delete_at(messages.index(notice.message)) - hosts.delete_at(hosts.index(notice.host)) - user_agents.delete_at(user_agents.index(notice.user_agent_string)) - save! + update_attributes!( + :messages => attribute_count_descrease(:messages, notice.message), + :hosts => attribute_count_descrease(:hosts, notice.host), + :user_agents => attribute_count_descrease(:user_agents, notice.user_agent_string) + ) end + private + def attribute_count_increase(name, value) + counter, index = send(name), attribute_index(value) + if counter[index].nil? + counter[index] = {'value' => value, 'count' => 1} + else + counter[index]['count'] += 1 + end + counter + end + + def attribute_count_descrease(name, value) + counter, index = send(name), attribute_index(value) + if counter[index]['count'] > 1 + counter[index]['count'] -= 1 + else + counter.delete(index) + end + counter + end + + def attribute_index(value) + Digest::MD5.hexdigest(value.to_s) + end + end diff --git a/db/migrate/20111102173347_cache_problem_statistics_fix.rb b/db/migrate/20111102173347_cache_problem_statistics_fix.rb new file mode 100644 index 0000000..9d461bd --- /dev/null +++ b/db/migrate/20111102173347_cache_problem_statistics_fix.rb @@ -0,0 +1,33 @@ +class CacheProblemStatisticsFix < Mongoid::Migration + def self.up + Problem.all.each do |problem| + messages = {} + hosts = {} + user_agents = {} + problem.notices.each do |notice| + messages = count_attribute(messages, notice.message) + hosts = count_attribute(hosts, notice.host) + user_agents = count_attribute(user_agents, notice.user_agent_string) + end + problem.update_attributes(:messages => messages, :hosts => hosts, :user_agents => user_agents) + end + end + + def self.down + Problem.all.each do |problem| + problem.update_attributes(:messages => {}, :hosts => {}, :user_agents => {}) + end + end + + private + def self.count_attribute(counter, value) + index = Digest::MD5.hexdigest(value.to_s) + if counter[index].nil? + counter[index] = {'value' => value, 'count' => 1} + else + counter[index]['count'] += 1 + end + counter + end + +end \ No newline at end of file diff --git a/spec/models/problem_spec.rb b/spec/models/problem_spec.rb index 096d39f..3cd9137 100644 --- a/spec/models/problem_spec.rb +++ b/spec/models/problem_spec.rb @@ -205,14 +205,14 @@ describe Problem do @err = Factory(:err, :problem => @problem) end - it "#messages returns [] by default" do - @problem.messages.should == [] + it "#messages should be empty by default" do + @problem.messages.should == {} end it "adding a notice adds a string to #messages" do lambda { Factory(:notice, :err => @err, :message => 'ERR 1') - }.should change(@problem, :messages).from([]).to(['ERR 1']) + }.should change(@problem, :messages).from({}).to({Digest::MD5.hexdigest('ERR 1') => {'value' => 'ERR 1', 'count' => 1}}) end it "removing a notice removes string from #messages" do @@ -220,7 +220,7 @@ describe Problem do lambda { @err.notices.first.destroy @problem.reload - }.should change(@problem, :messages).from(['ERR 1']).to([]) + }.should change(@problem, :messages).from({Digest::MD5.hexdigest('ERR 1') => {'value' => 'ERR 1', 'count' => 1}}).to({}) end end @@ -231,14 +231,14 @@ describe Problem do @err = Factory(:err, :problem => @problem) end - it "#hosts returns [] by default" do - @problem.hosts.should == [] + it "#hosts should be empty by default" do + @problem.hosts.should == {} end it "adding a notice adds a string to #hosts" do lambda { Factory(:notice, :err => @err, :request => {'url' => "http://example.com/resource/12"}) - }.should change(@problem, :hosts).from([]).to(['example.com']) + }.should change(@problem, :hosts).from({}).to({Digest::MD5.hexdigest('example.com') => {'value' => 'example.com', 'count' => 1}}) end it "removing a notice removes string from #hosts" do @@ -246,7 +246,7 @@ describe Problem do lambda { @err.notices.first.destroy @problem.reload - }.should change(@problem, :hosts).from(['example.com']).to([]) + }.should change(@problem, :hosts).from({Digest::MD5.hexdigest('example.com') => {'value' => 'example.com', 'count' => 1}}).to({}) end end @@ -257,14 +257,14 @@ describe Problem do @err = Factory(:err, :problem => @problem) end - it "#user_agents returns [] by default" do - @problem.user_agents.should == [] + it "#user_agents should be empty by default" do + @problem.user_agents.should == {} end it "adding a notice adds a string to #user_agents" do lambda { Factory(:notice, :err => @err, :request => {'cgi-data' => {'HTTP_USER_AGENT' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16'}}) - }.should change(@problem, :user_agents).from([]).to(['Chrome 10.0.648.204']) + }.should change(@problem, :user_agents).from({}).to({Digest::MD5.hexdigest('Chrome 10.0.648.204') => {'value' => 'Chrome 10.0.648.204', 'count' => 1}}) end it "removing a notice removes string from #user_agents" do @@ -272,7 +272,7 @@ describe Problem do lambda { @err.notices.first.destroy @problem.reload - }.should change(@problem, :user_agents).from(['Chrome 10.0.648.204']).to([]) + }.should change(@problem, :user_agents).from({Digest::MD5.hexdigest('Chrome 10.0.648.204') => {'value' => 'Chrome 10.0.648.204', 'count' => 1}}).to({}) end end -- libgit2 0.21.2