Download as
Browse Code »

Commit ad8ee6ad4acbeed08e1156df9e80853cfc956130

Authored by Dhruv Kapadia
1 parent a1fa6c61
Exists in master and in 1 other branch production-vendorized

Prompt cache statistics to check for cache hit rate

Showing 4 changed files with 46 additions and 2 deletions   Show diff stats
app/controllers/prompts_controller.rb
  Diff comments   View file @ad8ee6a
@@ -144,8 +144,11 @@ class PromptsController < InheritedResources::Base @@ -144,8 +144,11 @@ class PromptsController < InheritedResources::Base
144 logger.info("Question #{@question.id} is using catchup algorithm!") 144 logger.info("Question #{@question.id} is using catchup algorithm!")
145 @next_prompt = @question.pop_prompt_queue 145 @next_prompt = @question.pop_prompt_queue
146 if @next_prompt.nil? 146 if @next_prompt.nil?
  147 + @question.record_prompt_cache_miss
147 logger.info("Catchup prompt cache miss! Nothing in prompt_queue") 148 logger.info("Catchup prompt cache miss! Nothing in prompt_queue")
148 @next_prompt = @question.catchup_choose_prompt 149 @next_prompt = @question.catchup_choose_prompt
  150 + else
  151 + @question.record_prompt_cache_hit
149 end 152 end
150 @question.send_later :add_prompt_to_queue 153 @question.send_later :add_prompt_to_queue
151 else 154 else
app/controllers/questions_controller.rb
  Diff comments   View file @ad8ee6a
@@ -64,6 +64,9 @@ class QuestionsController < InheritedResources::Base @@ -64,6 +64,9 @@ class QuestionsController < InheritedResources::Base
64 if @p.nil? 64 if @p.nil?
65 logger.info("Catchup prompt cache miss! Nothing in prompt_queue") 65 logger.info("Catchup prompt cache miss! Nothing in prompt_queue")
66 @p = @question.catchup_choose_prompt 66 @p = @question.catchup_choose_prompt
  67 + @question.record_prompt_cache_miss
  68 + else
  69 + @question.record_prompt_cache_hit
67 end 70 end
68 @question.send_later :add_prompt_to_queue 71 @question.send_later :add_prompt_to_queue
69 72
app/models/question.rb
  Diff comments   View file @ad8ee6a
@@ -240,7 +240,6 @@ class Question < ActiveRecord::Base @@ -240,7 +240,6 @@ class Question < ActiveRecord::Base
240 240
241 def density 241 def density
242 # slow code, only to be run by cron job once at night 242 # slow code, only to be run by cron job once at night
243 - the_prompts = prompts.find(:all, :include => ['left_choice', 'right_choice'])  
244 243
245 seed_seed_sum = 0 244 seed_seed_sum = 0
246 seed_seed_total = 0 245 seed_seed_total = 0
@@ -254,7 +253,8 @@ class Question < ActiveRecord::Base @@ -254,7 +253,8 @@ class Question < ActiveRecord::Base
254 nonseed_nonseed_sum= 0 253 nonseed_nonseed_sum= 0
255 nonseed_nonseed_total= 0 254 nonseed_nonseed_total= 0
256 255
257 - the_prompts.each do |p| 256 + #the_prompts = prompts.find(:all, :include => ['left_choice', 'right_choice'])
  257 + prompts.find_each(:include => ['left_choice', 'right_choice']) do |p|
258 if p.left_choice.user_created == false && p.right_choice.user_created == false 258 if p.left_choice.user_created == false && p.right_choice.user_created == false
259 seed_seed_sum += p.appearances.size 259 seed_seed_sum += p.appearances.size
260 seed_seed_total +=1 260 seed_seed_total +=1
@@ -317,7 +317,22 @@ class Question < ActiveRecord::Base @@ -317,7 +317,22 @@ class Question < ActiveRecord::Base
317 prompt = prompt_id.nil? ? nil : Prompt.find(prompt_id.to_i) 317 prompt = prompt_id.nil? ? nil : Prompt.find(prompt_id.to_i)
318 end 318 end
319 319
  320 + def record_prompt_cache_miss
  321 + $redis.incr(self.pq_key + "_" + Time.now.to_date.to_s + "_"+ "misses")
  322 + $redis.expire(self.pq_key, 24*60*60 * 3) #Expire in three days
  323 + end
  324 +
  325 + def record_prompt_cache_hit
  326 + $redis.incr(self.pq_key + "_" + Time.now.to_date.to_s + "_"+ "hits")
  327 + $redis.expire(self.pq_key, 24*60*60 * 3) #Expire in three days
  328 + end
320 329
  330 + def get_prompt_cache_misses(date)
  331 + $redis.get(self.pq_key + "_" + date.to_s + "_"+ "misses")
  332 + end
  333 + def get_prompt_cache_hits(date)
  334 + $redis.get(self.pq_key + "_" + date.to_s + "_"+ "hits")
  335 + end
321 336
322 337
323 338
lib/tasks/test_api.rake
  Diff comments   View file @ad8ee6a
@@ -352,6 +352,25 @@ namespace :test_api do @@ -352,6 +352,25 @@ namespace :test_api do
352 end 352 end
353 353
354 354
  355 + #catchup specific
  356 + if question.uses_catchup?
  357 + misses = question.get_prompt_cache_misses(Date.yesterday).to_i
  358 + hits = question.get_prompt_cache_hits(Date.yesterday).to_i
  359 +
  360 +
  361 + yesterday_votes = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
  362 +
  363 + if misses + hits != yesterday_votes
  364 + error_msg += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_votes}, Actual: #{misses+hits}\n"
  365 + end
  366 +
  367 + miss_rate = misses.to_f / yesterday_votes.to_f
  368 + if miss_rate > 0.1
  369 + error_msg += "Error! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}\n"
  370 + end
  371 + end
  372 +
  373 +
355 if error_bool 374 if error_bool
356 error_msg += "Question #{question.id}: 2*wins = #{2*total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n" 375 error_msg += "Question #{question.id}: 2*wins = #{2*total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
357 end 376 end
@@ -416,6 +435,7 @@ namespace :test_api do @@ -416,6 +435,7 @@ namespace :test_api do
416 " The prompt count matches the expected number of prompts ( num_choices ^2 - num choices) for each question\n" + 435 " The prompt count matches the expected number of prompts ( num_choices ^2 - num choices) for each question\n" +
417 " All Vote objects have an associated appearance object\n" + 436 " All Vote objects have an associated appearance object\n" +
418 " All Vote objects have an client response time < calculated server roundtrip time\n" 437 " All Vote objects have an client response time < calculated server roundtrip time\n"
  438 + " More than 90% of prompts on catchup algorithm questions were served from cache\n"
419 439
420 print success_msg 440 print success_msg
421 441
@@ -423,6 +443,9 @@ namespace :test_api do @@ -423,6 +443,9 @@ namespace :test_api do
423 else 443 else
424 CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg) 444 CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg)
425 445
  446 + puts "There were errors: "
  447 + puts error_msg
  448 +
426 unless bad_choices.blank? 449 unless bad_choices.blank?
427 450
428 puts "Here's a list of choice ids that you may want to modify: #{bad_choices.uniq.inspect}" 451 puts "Here's a list of choice ids that you may want to modify: #{bad_choices.uniq.inspect}"