Problem notices cache is more space effective

Marcin Ciunelis
1 parent 7dd78504
Showing 4 changed files with 83 additions and 24 deletions Show diff stats
app/helpers/application_helper.rb
app/models/problem.rb
db/migrate/20111102173347_cache_problem_statistics_fix.rb
spec/models/problem_spec.rb
@@ -47,8 +47,8 @@ module ApplicationHelper
   end
  
   def tally(collection)
-    collection.inject({}) do |tallies, value|
-      tallies[value] = (tallies[value] || 0) + 1
+    collection.values.inject({}) do |tallies, tally|
+      tallies[tally['value']] = tally['count']
       tallies
     end
   end
@@ -18,9 +18,9 @@ class Problem
   field :environment
   field :klass
   field :where
-  field :user_agents, :type => Array, :default => []
-  field :messages, :type => Array, :default => []
-  field :hosts, :type => Array, :default => []
+  field :user_agents, :type => Hash, :default => {}
+  field :messages,    :type => Hash, :default => {}
+  field :hosts,       :type => Hash, :default => {}
   field :comments_count, :type => Integer, :default => 0
  
   index :app_id
@@ -128,19 +128,45 @@ class Problem
       :environment => notice.environment_name,
       :klass => notice.klass,
       :where => notice.where,
-      :messages => messages.push(notice.message),
-      :hosts => hosts.push(notice.host),
-      :user_agents => user_agents.push(notice.user_agent_string)
+      :messages    => attribute_count_increase(:messages, notice.message),
+      :hosts       => attribute_count_increase(:hosts, notice.host),
+      :user_agents => attribute_count_increase(:user_agents, notice.user_agent_string)
       ) if notice
     update_attributes!(attrs)
   end
  
   def remove_cached_notice_attribures(notice)
-    messages.delete_at(messages.index(notice.message))
-    hosts.delete_at(hosts.index(notice.host))
-    user_agents.delete_at(user_agents.index(notice.user_agent_string))
-    save!
+    update_attributes!(
+      :messages    => attribute_count_descrease(:messages, notice.message),
+      :hosts       => attribute_count_descrease(:hosts, notice.host),
+      :user_agents => attribute_count_descrease(:user_agents, notice.user_agent_string)
+    )
   end
  
+  private
+    def attribute_count_increase(name, value)
+      counter, index = send(name), attribute_index(value)
+      if counter[index].nil?
+        counter[index] = {'value' => value, 'count' => 1}
+      else
+        counter[index]['count'] += 1
+      end
+      counter
+    end
+
+    def attribute_count_descrease(name, value)
+      counter, index = send(name), attribute_index(value)
+      if counter[index]['count'] > 1
+        counter[index]['count'] -= 1
+      else
+        counter.delete(index)
+      end
+      counter
+    end
+
+    def attribute_index(value)
+      Digest::MD5.hexdigest(value.to_s)
+    end
+
 end
  
@@ -0,0 +1,33 @@
+class CacheProblemStatisticsFix < Mongoid::Migration
+  def self.up
+    Problem.all.each do |problem|
+      messages = {}
+      hosts = {}
+      user_agents = {}
+      problem.notices.each do |notice|
+      	messages    = count_attribute(messages, notice.message)
+        hosts       = count_attribute(hosts, notice.host)
+        user_agents = count_attribute(user_agents, notice.user_agent_string)
+      end
+      problem.update_attributes(:messages => messages, :hosts => hosts, :user_agents => user_agents)
+    end
+  end
+
+  def self.down
+    Problem.all.each do |problem|
+      problem.update_attributes(:messages => {}, :hosts => {}, :user_agents => {})
+    end
+  end
+
+  private
+    def self.count_attribute(counter, value)
+      index = Digest::MD5.hexdigest(value.to_s)
+      if counter[index].nil?
+        counter[index] = {'value' => value, 'count' => 1}
+      else
+        counter[index]['count'] += 1
+      end
+  	  counter
+  	end
+
+end
 \ No newline at end of file
@@ -205,14 +205,14 @@ describe Problem do
       @err = Factory(:err, :problem => @problem)
     end
  
-    it "#messages returns [] by default" do
-      @problem.messages.should == []
+    it "#messages should be empty by default" do
+      @problem.messages.should == {}
     end
  
     it "adding a notice adds a string to #messages" do
       lambda {
         Factory(:notice, :err => @err, :message => 'ERR 1')
-      }.should change(@problem, :messages).from([]).to(['ERR 1'])
+      }.should change(@problem, :messages).from({}).to({Digest::MD5.hexdigest('ERR 1') => {'value' => 'ERR 1', 'count' => 1}})
     end
  
     it "removing a notice removes string from #messages" do
@@ -220,7 +220,7 @@ describe Problem do
       lambda {
         @err.notices.first.destroy
         @problem.reload
-      }.should change(@problem, :messages).from(['ERR 1']).to([])
+      }.should change(@problem, :messages).from({Digest::MD5.hexdigest('ERR 1') => {'value' => 'ERR 1', 'count' => 1}}).to({})
     end
   end
  
@@ -231,14 +231,14 @@ describe Problem do
       @err = Factory(:err, :problem => @problem)
     end
  
-    it "#hosts returns [] by default" do
-      @problem.hosts.should == []
+    it "#hosts should be empty by default" do
+      @problem.hosts.should == {}
     end
  
     it "adding a notice adds a string to #hosts" do
       lambda {
         Factory(:notice, :err => @err, :request => {'url' => "http://example.com/resource/12"})
-      }.should change(@problem, :hosts).from([]).to(['example.com'])
+      }.should change(@problem, :hosts).from({}).to({Digest::MD5.hexdigest('example.com') => {'value' => 'example.com', 'count' => 1}})
     end
  
     it "removing a notice removes string from #hosts" do
@@ -246,7 +246,7 @@ describe Problem do
       lambda {
         @err.notices.first.destroy
         @problem.reload
-      }.should change(@problem, :hosts).from(['example.com']).to([])
+      }.should change(@problem, :hosts).from({Digest::MD5.hexdigest('example.com') => {'value' => 'example.com', 'count' => 1}}).to({})
     end
   end
  
@@ -257,14 +257,14 @@ describe Problem do
       @err = Factory(:err, :problem => @problem)
     end
  
-    it "#user_agents returns [] by default" do
-      @problem.user_agents.should == []
+    it "#user_agents should be empty by default" do
+      @problem.user_agents.should == {}
     end
  
     it "adding a notice adds a string to #user_agents" do
       lambda {
         Factory(:notice, :err => @err, :request => {'cgi-data' => {'HTTP_USER_AGENT' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16'}})
-      }.should change(@problem, :user_agents).from([]).to(['Chrome 10.0.648.204'])
+      }.should change(@problem, :user_agents).from({}).to({Digest::MD5.hexdigest('Chrome 10.0.648.204') => {'value' => 'Chrome 10.0.648.204', 'count' => 1}})
     end
  
     it "removing a notice removes string from #user_agents" do
@@ -272,7 +272,7 @@ describe Problem do
       lambda {
         @err.notices.first.destroy
         @problem.reload
-      }.should change(@problem, :user_agents).from(['Chrome 10.0.648.204']).to([])
+      }.should change(@problem, :user_agents).from({Digest::MD5.hexdigest('Chrome 10.0.648.204') => {'value' => 'Chrome 10.0.648.204', 'count' => 1}}).to({})
     end
   end
...	...	@@ -47,8 +47,8 @@ module ApplicationHelper
47	47	end
48	48
49	49	def tally(collection)
50		- collection.inject({}) do \|tallies, value\|
51		- tallies[value] = (tallies[value] \|\| 0) + 1
	50	+ collection.values.inject({}) do \|tallies, tally\|
	51	+ tallies[tally['value']] = tally['count']
52	52	tallies
53	53	end
54	54	end
...	...
...	...	@@ -18,9 +18,9 @@ class Problem
18	18	field :environment
19	19	field :klass
20	20	field :where
21		- field :user_agents, :type => Array, :default => []
22		- field :messages, :type => Array, :default => []
23		- field :hosts, :type => Array, :default => []
	21	+ field :user_agents, :type => Hash, :default => {}
	22	+ field :messages, :type => Hash, :default => {}
	23	+ field :hosts, :type => Hash, :default => {}
24	24	field :comments_count, :type => Integer, :default => 0
25	25
26	26	index :app_id
...	...	@@ -128,19 +128,45 @@ class Problem
128	128	:environment => notice.environment_name,
129	129	:klass => notice.klass,
130	130	:where => notice.where,
131		- :messages => messages.push(notice.message),
132		- :hosts => hosts.push(notice.host),
133		- :user_agents => user_agents.push(notice.user_agent_string)
	131	+ :messages => attribute_count_increase(:messages, notice.message),
	132	+ :hosts => attribute_count_increase(:hosts, notice.host),
	133	+ :user_agents => attribute_count_increase(:user_agents, notice.user_agent_string)
134	134	) if notice
135	135	update_attributes!(attrs)
136	136	end
137	137
138	138	def remove_cached_notice_attribures(notice)
139		- messages.delete_at(messages.index(notice.message))
140		- hosts.delete_at(hosts.index(notice.host))
141		- user_agents.delete_at(user_agents.index(notice.user_agent_string))
142		- save!
	139	+ update_attributes!(
	140	+ :messages => attribute_count_descrease(:messages, notice.message),
	141	+ :hosts => attribute_count_descrease(:hosts, notice.host),
	142	+ :user_agents => attribute_count_descrease(:user_agents, notice.user_agent_string)
	143	+ )
143	144	end
144	145
	146	+ private
	147	+ def attribute_count_increase(name, value)
	148	+ counter, index = send(name), attribute_index(value)
	149	+ if counter[index].nil?
	150	+ counter[index] = {'value' => value, 'count' => 1}
	151	+ else
	152	+ counter[index]['count'] += 1
	153	+ end
	154	+ counter
	155	+ end
	156	+
	157	+ def attribute_count_descrease(name, value)
	158	+ counter, index = send(name), attribute_index(value)
	159	+ if counter[index]['count'] > 1
	160	+ counter[index]['count'] -= 1
	161	+ else
	162	+ counter.delete(index)
	163	+ end
	164	+ counter
	165	+ end
	166	+
	167	+ def attribute_index(value)
	168	+ Digest::MD5.hexdigest(value.to_s)
	169	+ end
	170	+
145	171	end
146	172
...	...
...	...	@@ -0,0 +1,33 @@
	1	+class CacheProblemStatisticsFix < Mongoid::Migration
	2	+ def self.up
	3	+ Problem.all.each do \|problem\|
	4	+ messages = {}
	5	+ hosts = {}
	6	+ user_agents = {}
	7	+ problem.notices.each do \|notice\|
	8	+ messages = count_attribute(messages, notice.message)
	9	+ hosts = count_attribute(hosts, notice.host)
	10	+ user_agents = count_attribute(user_agents, notice.user_agent_string)
	11	+ end
	12	+ problem.update_attributes(:messages => messages, :hosts => hosts, :user_agents => user_agents)
	13	+ end
	14	+ end
	15	+
	16	+ def self.down
	17	+ Problem.all.each do \|problem\|
	18	+ problem.update_attributes(:messages => {}, :hosts => {}, :user_agents => {})
	19	+ end
	20	+ end
	21	+
	22	+ private
	23	+ def self.count_attribute(counter, value)
	24	+ index = Digest::MD5.hexdigest(value.to_s)
	25	+ if counter[index].nil?
	26	+ counter[index] = {'value' => value, 'count' => 1}
	27	+ else
	28	+ counter[index]['count'] += 1
	29	+ end
	30	+ counter
	31	+ end
	32	+
	33	+end
0	34	\ No newline at end of file
...	...
...	...	@@ -205,14 +205,14 @@ describe Problem do
205	205	@err = Factory(:err, :problem => @problem)
206	206	end
207	207
208		- it "#messages returns [] by default" do
209		- @problem.messages.should == []
	208	+ it "#messages should be empty by default" do
	209	+ @problem.messages.should == {}
210	210	end
211	211
212	212	it "adding a notice adds a string to #messages" do
213	213	lambda {
214	214	Factory(:notice, :err => @err, :message => 'ERR 1')
215		- }.should change(@problem, :messages).from([]).to(['ERR 1'])
	215	+ }.should change(@problem, :messages).from({}).to({Digest::MD5.hexdigest('ERR 1') => {'value' => 'ERR 1', 'count' => 1}})
216	216	end
217	217
218	218	it "removing a notice removes string from #messages" do
...	...	@@ -220,7 +220,7 @@ describe Problem do
220	220	lambda {
221	221	@err.notices.first.destroy
222	222	@problem.reload
223		- }.should change(@problem, :messages).from(['ERR 1']).to([])
	223	+ }.should change(@problem, :messages).from({Digest::MD5.hexdigest('ERR 1') => {'value' => 'ERR 1', 'count' => 1}}).to({})
224	224	end
225	225	end
226	226
...	...	@@ -231,14 +231,14 @@ describe Problem do
231	231	@err = Factory(:err, :problem => @problem)
232	232	end
233	233
234		- it "#hosts returns [] by default" do
235		- @problem.hosts.should == []
	234	+ it "#hosts should be empty by default" do
	235	+ @problem.hosts.should == {}
236	236	end
237	237
238	238	it "adding a notice adds a string to #hosts" do
239	239	lambda {
240	240	Factory(:notice, :err => @err, :request => {'url' => "http://example.com/resource/12"})
241		- }.should change(@problem, :hosts).from([]).to(['example.com'])
	241	+ }.should change(@problem, :hosts).from({}).to({Digest::MD5.hexdigest('example.com') => {'value' => 'example.com', 'count' => 1}})
242	242	end
243	243
244	244	it "removing a notice removes string from #hosts" do
...	...	@@ -246,7 +246,7 @@ describe Problem do
246	246	lambda {
247	247	@err.notices.first.destroy
248	248	@problem.reload
249		- }.should change(@problem, :hosts).from(['example.com']).to([])
	249	+ }.should change(@problem, :hosts).from({Digest::MD5.hexdigest('example.com') => {'value' => 'example.com', 'count' => 1}}).to({})
250	250	end
251	251	end
252	252
...	...	@@ -257,14 +257,14 @@ describe Problem do
257	257	@err = Factory(:err, :problem => @problem)
258	258	end
259	259
260		- it "#user_agents returns [] by default" do
261		- @problem.user_agents.should == []
	260	+ it "#user_agents should be empty by default" do
	261	+ @problem.user_agents.should == {}
262	262	end
263	263
264	264	it "adding a notice adds a string to #user_agents" do
265	265	lambda {
266	266	Factory(:notice, :err => @err, :request => {'cgi-data' => {'HTTP_USER_AGENT' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16'}})
267		- }.should change(@problem, :user_agents).from([]).to(['Chrome 10.0.648.204'])
	267	+ }.should change(@problem, :user_agents).from({}).to({Digest::MD5.hexdigest('Chrome 10.0.648.204') => {'value' => 'Chrome 10.0.648.204', 'count' => 1}})
268	268	end
269	269
270	270	it "removing a notice removes string from #user_agents" do
...	...	@@ -272,7 +272,7 @@ describe Problem do
272	272	lambda {
273	273	@err.notices.first.destroy
274	274	@problem.reload
275		- }.should change(@problem, :user_agents).from(['Chrome 10.0.648.204']).to([])
	275	+ }.should change(@problem, :user_agents).from({Digest::MD5.hexdigest('Chrome 10.0.648.204') => {'value' => 'Chrome 10.0.648.204', 'count' => 1}}).to({})
276	276	end
277	277	end
278	278
...	...