Commit ad8ee6ad4acbeed08e1156df9e80853cfc956130
1 parent
a1fa6c61
Exists in
master
and in
1 other branch
Prompt cache statistics to check for cache hit rate
Showing
4 changed files
with
46 additions
and
2 deletions
Show diff stats
app/controllers/prompts_controller.rb
@@ -144,8 +144,11 @@ class PromptsController < InheritedResources::Base | @@ -144,8 +144,11 @@ class PromptsController < InheritedResources::Base | ||
144 | logger.info("Question #{@question.id} is using catchup algorithm!") | 144 | logger.info("Question #{@question.id} is using catchup algorithm!") |
145 | @next_prompt = @question.pop_prompt_queue | 145 | @next_prompt = @question.pop_prompt_queue |
146 | if @next_prompt.nil? | 146 | if @next_prompt.nil? |
147 | + @question.record_prompt_cache_miss | ||
147 | logger.info("Catchup prompt cache miss! Nothing in prompt_queue") | 148 | logger.info("Catchup prompt cache miss! Nothing in prompt_queue") |
148 | @next_prompt = @question.catchup_choose_prompt | 149 | @next_prompt = @question.catchup_choose_prompt |
150 | + else | ||
151 | + @question.record_prompt_cache_hit | ||
149 | end | 152 | end |
150 | @question.send_later :add_prompt_to_queue | 153 | @question.send_later :add_prompt_to_queue |
151 | else | 154 | else |
app/controllers/questions_controller.rb
@@ -64,6 +64,9 @@ class QuestionsController < InheritedResources::Base | @@ -64,6 +64,9 @@ class QuestionsController < InheritedResources::Base | ||
64 | if @p.nil? | 64 | if @p.nil? |
65 | logger.info("Catchup prompt cache miss! Nothing in prompt_queue") | 65 | logger.info("Catchup prompt cache miss! Nothing in prompt_queue") |
66 | @p = @question.catchup_choose_prompt | 66 | @p = @question.catchup_choose_prompt |
67 | + @question.record_prompt_cache_miss | ||
68 | + else | ||
69 | + @question.record_prompt_cache_hit | ||
67 | end | 70 | end |
68 | @question.send_later :add_prompt_to_queue | 71 | @question.send_later :add_prompt_to_queue |
69 | 72 |
app/models/question.rb
@@ -240,7 +240,6 @@ class Question < ActiveRecord::Base | @@ -240,7 +240,6 @@ class Question < ActiveRecord::Base | ||
240 | 240 | ||
241 | def density | 241 | def density |
242 | # slow code, only to be run by cron job once at night | 242 | # slow code, only to be run by cron job once at night |
243 | - the_prompts = prompts.find(:all, :include => ['left_choice', 'right_choice']) | ||
244 | 243 | ||
245 | seed_seed_sum = 0 | 244 | seed_seed_sum = 0 |
246 | seed_seed_total = 0 | 245 | seed_seed_total = 0 |
@@ -254,7 +253,8 @@ class Question < ActiveRecord::Base | @@ -254,7 +253,8 @@ class Question < ActiveRecord::Base | ||
254 | nonseed_nonseed_sum= 0 | 253 | nonseed_nonseed_sum= 0 |
255 | nonseed_nonseed_total= 0 | 254 | nonseed_nonseed_total= 0 |
256 | 255 | ||
257 | - the_prompts.each do |p| | 256 | + #the_prompts = prompts.find(:all, :include => ['left_choice', 'right_choice']) |
257 | + prompts.find_each(:include => ['left_choice', 'right_choice']) do |p| | ||
258 | if p.left_choice.user_created == false && p.right_choice.user_created == false | 258 | if p.left_choice.user_created == false && p.right_choice.user_created == false |
259 | seed_seed_sum += p.appearances.size | 259 | seed_seed_sum += p.appearances.size |
260 | seed_seed_total +=1 | 260 | seed_seed_total +=1 |
@@ -317,7 +317,22 @@ class Question < ActiveRecord::Base | @@ -317,7 +317,22 @@ class Question < ActiveRecord::Base | ||
317 | prompt = prompt_id.nil? ? nil : Prompt.find(prompt_id.to_i) | 317 | prompt = prompt_id.nil? ? nil : Prompt.find(prompt_id.to_i) |
318 | end | 318 | end |
319 | 319 | ||
320 | + def record_prompt_cache_miss | ||
321 | + $redis.incr(self.pq_key + "_" + Time.now.to_date.to_s + "_"+ "misses") | ||
322 | + $redis.expire(self.pq_key, 24*60*60 * 3) #Expire in three days | ||
323 | + end | ||
324 | + | ||
325 | + def record_prompt_cache_hit | ||
326 | + $redis.incr(self.pq_key + "_" + Time.now.to_date.to_s + "_"+ "hits") | ||
327 | + $redis.expire(self.pq_key, 24*60*60 * 3) #Expire in three days | ||
328 | + end | ||
320 | 329 | ||
330 | + def get_prompt_cache_misses(date) | ||
331 | + $redis.get(self.pq_key + "_" + date.to_s + "_"+ "misses") | ||
332 | + end | ||
333 | + def get_prompt_cache_hits(date) | ||
334 | + $redis.get(self.pq_key + "_" + date.to_s + "_"+ "hits") | ||
335 | + end | ||
321 | 336 | ||
322 | 337 | ||
323 | 338 |
lib/tasks/test_api.rake
@@ -352,6 +352,25 @@ namespace :test_api do | @@ -352,6 +352,25 @@ namespace :test_api do | ||
352 | end | 352 | end |
353 | 353 | ||
354 | 354 | ||
355 | + #catchup specific | ||
356 | + if question.uses_catchup? | ||
357 | + misses = question.get_prompt_cache_misses(Date.yesterday).to_i | ||
358 | + hits = question.get_prompt_cache_hits(Date.yesterday).to_i | ||
359 | + | ||
360 | + | ||
361 | + yesterday_votes = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday]) | ||
362 | + | ||
363 | + if misses + hits != yesterday_votes | ||
364 | + error_msg += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_votes}, Actual: #{misses+hits}\n" | ||
365 | + end | ||
366 | + | ||
367 | + miss_rate = misses.to_f / yesterday_votes.to_f | ||
368 | + if miss_rate > 0.1 | ||
369 | + error_msg += "Error! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}\n" | ||
370 | + end | ||
371 | + end | ||
372 | + | ||
373 | + | ||
355 | if error_bool | 374 | if error_bool |
356 | error_msg += "Question #{question.id}: 2*wins = #{2*total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n" | 375 | error_msg += "Question #{question.id}: 2*wins = #{2*total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n" |
357 | end | 376 | end |
@@ -416,6 +435,7 @@ namespace :test_api do | @@ -416,6 +435,7 @@ namespace :test_api do | ||
416 | " The prompt count matches the expected number of prompts ( num_choices ^2 - num choices) for each question\n" + | 435 | " The prompt count matches the expected number of prompts ( num_choices ^2 - num choices) for each question\n" + |
417 | " All Vote objects have an associated appearance object\n" + | 436 | " All Vote objects have an associated appearance object\n" + |
418 | " All Vote objects have an client response time < calculated server roundtrip time\n" | 437 | " All Vote objects have an client response time < calculated server roundtrip time\n" |
438 | + " More than 90% of prompts on catchup algorithm questions were served from cache\n" | ||
419 | 439 | ||
420 | print success_msg | 440 | print success_msg |
421 | 441 | ||
@@ -423,6 +443,9 @@ namespace :test_api do | @@ -423,6 +443,9 @@ namespace :test_api do | ||
423 | else | 443 | else |
424 | CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg) | 444 | CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg) |
425 | 445 | ||
446 | + puts "There were errors: " | ||
447 | + puts error_msg | ||
448 | + | ||
426 | unless bad_choices.blank? | 449 | unless bad_choices.blank? |
427 | 450 | ||
428 | puts "Here's a list of choice ids that you may want to modify: #{bad_choices.uniq.inspect}" | 451 | puts "Here's a list of choice ids that you may want to modify: #{bad_choices.uniq.inspect}" |