rearrange question methods and dynamically create rake tasks

Luke Baker
1 parent 8b086743
Showing 1 changed file with 133 additions and 119 deletions Show diff stats
lib/tasks/test_api.rake
@@ -125,14 +125,11 @@ namespace :test_api do
       end
  
  
-      #catchup specific 
-      if question.uses_catchup?
-        message, error_occurred = check_prompt_cache_hit_rate(question)
-        if error_occurred
-          errors << message
-        else
-          successes << message
-        end
+      message, error_occurred = check_prompt_cache_hit_rate(question)
+      if error_occurred
+        errors << message
+      else
+        successes << message
       end
  
     end
@@ -303,155 +300,172 @@ namespace :test_api do
     return error_message.blank? ? [success_message, false] : [error_message, true] 
   end
  
-  def check_each_choice_appears_within_n_stddevs(question)
-    error_message =""
-    success_message = "Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question " +
-      "(Note: this applies only to seed choices (not user submitted) and choices currently marked active)"
  
-    wins_by_choice_id = question.votes.active.count(:group => :choice_id, :conditions => ["creator_id = ?", question.creator_id])
-    losses_by_choice_id= question.votes.active_loser.count(:group => :loser_choice_id, :conditions => ["creator_id = ?", question.creator_id])
+  namespace :question do
  
-    #Rails returns an ordered hash, which doesn't allow for blocks to change merging logic.
-    #A little hack to create a normal hash
-    wins_hash = {}
-    wins_hash.merge!(wins_by_choice_id)
-    losses_hash = {}
-    losses_hash.merge!(losses_by_choice_id)
+    # use this to dynamically create rake task for each question test
+    question_tasks = {
+      :answered_appearances_equals_votes_and_skips => "Ensure that a question has: answered_appearances == votes + skips",
+      :check_each_choice_appears_within_n_stddevs => "Ensure each choice appears within 6 standard deviations",
+      :check_each_choice_equally_likely_to_appear_left_or_right => "Ensure each choice is equally likely to appear on left or right",
+      :check_prompt_cache_hit_rate => "Check prompt cache hit rate",
+      :check_object_counter_cache_values_match_actual_values => "Check that object counter cache values match actual values"
+    }
+
+    # dynamically create tasks for each question task
+    question_tasks.each do |taskname, description|
+      desc description
+      task taskname, [:question_id] => :environment do |t, args|
+        a = cleanup_args(args)
+        questions = Question.find(a[:question_id])
+        questions.each do |question|
+          # call task
+          puts send(taskname, question).inspect
+        end
+      end
+    end
  
+    def answered_appearances_equals_votes_and_skips(question)
+      error_message = ""
+      success_message = "All vote and skip objects have an associated appearance object"
+      skip_appearances_count = Appearance.count(
+        :conditions => ["skips.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL AND answerable_type = 'Skip'", question.id],
+        :joins => "LEFT JOIN skips ON (skips.id = appearances.answerable_id)")
+      vote_appearances_count = Appearance.count(
+        :conditions => ["votes.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL and answerable_type = 'Vote'", question.id],
+        :joins => "LEFT JOIN votes ON (votes.id = appearances.answerable_id)")
+      total_answered_appearances = skip_appearances_count + vote_appearances_count
+      total_votes = question.votes.count
+      total_skips = question.skips.count
+      if (total_answered_appearances != total_votes + total_skips)
+        error_message += "Question #{question.id}: answered_appearances = #{total_answered_appearances}, votes = #{total_votes}, skips = #{total_skips}"
+      end
  
  
-    appearances_by_choice_id = wins_hash.merge(losses_hash) do |key, oldval, newval| oldval + newval end
+      return error_message.blank? ? [success_message, false] : [error_message, true] 
+    end
  
-    sum = total_appearances = appearances_by_choice_id.values.inject(0) {|sum, x| sum +=x}
-    mean = average_appearances = total_appearances.to_f / appearances_by_choice_id.size.to_f
+    def check_each_choice_appears_within_n_stddevs(question)
+      error_message =""
+      success_message = "Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question " +
+        "(Note: this applies only to seed choices (not user submitted) and choices currently marked active)"
  
-    if sum > 0:
-      stddev = Math.sqrt( appearances_by_choice_id.values.inject(0) { |sum, e| sum + (e - mean) ** 2 } / appearances_by_choice_id.size.to_f )
+      wins_by_choice_id = question.votes.active.count(:group => :choice_id, :conditions => ["creator_id = ?", question.creator_id])
+      losses_by_choice_id= question.votes.active_loser.count(:group => :loser_choice_id, :conditions => ["creator_id = ?", question.creator_id])
  
-      # this choice appears to have been deactivated then reactivated after
-      # a period of voting
-      ignore_choices = [133189]
-      appearances_by_choice_id.each do |choice_id, n_i| 
-        if ((n_i < (mean - 6*stddev)) || (n_i > mean + 6 *stddev)) && !ignore_choices.include?(choice_id) && Choice.find(choice_id).active?
-          error_message += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question\n"
-        end
-      end
-    end
+      #Rails returns an ordered hash, which doesn't allow for blocks to change merging logic.
+      #A little hack to create a normal hash
+      wins_hash = {}
+      wins_hash.merge!(wins_by_choice_id)
+      losses_hash = {}
+      losses_hash.merge!(losses_by_choice_id)
  
-    return error_message.blank? ? [success_message, false] : [error_message, true] 
-  end
  
-  def check_each_choice_equally_likely_to_appear_left_or_right(question)
-    error_message = ""
-    success_message = "All choices have equal probability of appearing on left or right (within error params)"
-    question.choices.each do |c|
-      left_prompts_ids = c.prompts_on_the_left.ids_only
-      right_prompts_ids = c.prompts_on_the_right.ids_only
  
-      left_appearances = question.appearances.count(:conditions => {:prompt_id => left_prompts_ids})
-      right_appearances = question.appearances.count(:conditions => {:prompt_id => right_prompts_ids})
+      appearances_by_choice_id = wins_hash.merge(losses_hash) do |key, oldval, newval| oldval + newval end
  
-      n = left_appearances + right_appearances
+      sum = total_appearances = appearances_by_choice_id.values.inject(0) {|sum, x| sum +=x}
+      mean = average_appearances = total_appearances.to_f / appearances_by_choice_id.size.to_f
  
-      if n == 0
-        next
-      end
-      est_p = right_appearances.to_f / n.to_f
-      z = (est_p - 0.5).abs / Math.sqrt((0.5 * 0.5) / n.to_f)
+      if sum > 0:
+        stddev = Math.sqrt( appearances_by_choice_id.values.inject(0) { |sum, e| sum + (e - mean) ** 2 } / appearances_by_choice_id.size.to_f )
  
-      if z > 6 
-        error_message += "Error: Choice ID #{c.id} seems to favor one side: Left Appearances #{left_appearances}, Right Appearances: #{right_appearances}, z = #{z}\n"
+        # this choice appears to have been deactivated then reactivated after
+        # a period of voting
+        ignore_choices = [133189]
+        appearances_by_choice_id.each do |choice_id, n_i| 
+          if ((n_i < (mean - 6*stddev)) || (n_i > mean + 6 *stddev)) && !ignore_choices.include?(choice_id) && Choice.find(choice_id).active?
+            error_message += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question\n"
+          end
+        end
       end
+
+      return error_message.blank? ? [success_message, false] : [error_message, true] 
     end
-    return error_message.blank? ? [success_message, false] : [error_message, true] 
-  end
-  def check_prompt_cache_hit_rate(question)
-    error_message = ""
-    success_message = "At least 90% of prompts on catchup algorithm questions were served from cache\n" 
  
-    misses = question.get_prompt_cache_misses(Date.yesterday).to_i
-    hits = question.get_prompt_cache_hits(Date.yesterday).to_i
+    def check_each_choice_equally_likely_to_appear_left_or_right(question)
+      error_message = ""
+      success_message = "All choices have equal probability of appearing on left or right (within error params)"
+      question.choices.each do |c|
+        left_prompts_ids = c.prompts_on_the_left.ids_only
+        right_prompts_ids = c.prompts_on_the_right.ids_only
  
-    question.expire_prompt_cache_tracking_keys(Date.yesterday)
+        left_appearances = question.appearances.count(:conditions => {:prompt_id => left_prompts_ids})
+        right_appearances = question.appearances.count(:conditions => {:prompt_id => right_prompts_ids})
  
-    yesterday_appearances = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
+        n = left_appearances + right_appearances
  
-    if misses + hits != yesterday_appearances
-      error_message += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_appearances}, Actual: #{misses+hits}, Hits: #{hits}, Misses: #{misses}\n"
-    end
+        if n == 0
+          next
+        end
+        est_p = right_appearances.to_f / n.to_f
+        z = (est_p - 0.5).abs / Math.sqrt((0.5 * 0.5) / n.to_f)
  
-    if yesterday_appearances > 5 # this test isn't worthwhile for small numbers of appearances
-      miss_rate = misses.to_f / yesterday_appearances.to_f
-      if miss_rate > 0.1
-        error_message += "Warning! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}, total appearances yesterday: #{yesterday_appearances}\n"
+        if z > 6 
+          error_message += "Error: Choice ID #{c.id} seems to favor one side: Left Appearances #{left_appearances}, Right Appearances: #{right_appearances}, z = #{z}\n"
+        end
       end
+      return error_message.blank? ? [success_message, false] : [error_message, true] 
     end
-    return error_message.blank? ? [success_message, false] : [error_message, true] 
-  end
-
-  def check_object_counter_cache_values_match_actual_values(question)
-    error_message = ""
-    success_message = "All cached object values match actual values within database"
-    # Checks that counter_cache is working as expected
-    cached_prompts_size = question.prompts.size
-    actual_prompts_size = question.prompts.count
+    def check_prompt_cache_hit_rate(question)
+      error_message = ""
+      success_message = "At least 90% of prompts on catchup algorithm questions were served from cache\n" 
+      return [success_message, false] unless question.uses_catchup?
  
-    if cached_prompts_size != actual_prompts_size
-      error_message += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n"
-    end
+      misses = question.get_prompt_cache_misses(Date.yesterday).to_i
+      hits = question.get_prompt_cache_hits(Date.yesterday).to_i
  
-    cached_votes_size = question.votes.size
-    actual_votes_size = question.votes.count
+      question.expire_prompt_cache_tracking_keys(Date.yesterday)
  
-    if cached_votes_size != actual_votes_size
-      error_message += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n"
-    end
+      yesterday_appearances = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
  
-    cached_choices_size = question.choices.size
-    actual_choices_size = question.choices.count
+      if misses + hits != yesterday_appearances
+        error_message += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_appearances}, Actual: #{misses+hits}, Hits: #{hits}, Misses: #{misses}\n"
+      end
  
-    if cached_choices_size != actual_choices_size
-      error_message+= "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n"
+      if yesterday_appearances > 5 # this test isn't worthwhile for small numbers of appearances
+        miss_rate = misses.to_f / yesterday_appearances.to_f
+        if miss_rate > 0.1
+          error_message += "Warning! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}, total appearances yesterday: #{yesterday_appearances}\n"
+        end
+      end
+      return error_message.blank? ? [success_message, false] : [error_message, true] 
     end
  
-    #if cached_prompts_size != question.choices.size **2 - question.choices.size 
-    # error_message += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n"
-    #end
-    return error_message.blank? ? [success_message, false] : [error_message, true] 
-  end
+    def check_object_counter_cache_values_match_actual_values(question)
+      error_message = ""
+      success_message = "All cached object values match actual values within database"
+      # Checks that counter_cache is working as expected
+      cached_prompts_size = question.prompts.size
+      actual_prompts_size = question.prompts.count
  
-  namespace :question do
+      if cached_prompts_size != actual_prompts_size
+        error_message += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n"
+      end
  
-    desc "Ensure that a question has: answered_appearances == votes + skips"
-    task :answered_appearances_equals_votes_and_skips, [:question_id] => :environment do |t, args|
-      a = cleanup_args(args)
-      questions = Question.find(a[:question_id])
-      questions.each do |question|
-        puts answered_appearances_equals_votes_and_skips(question).inspect
+      cached_votes_size = question.votes.size
+      actual_votes_size = question.votes.count
+
+      if cached_votes_size != actual_votes_size
+        error_message += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n"
       end
-    end
  
-  end
+      cached_choices_size = question.choices.size
+      actual_choices_size = question.choices.count
  
-  def answered_appearances_equals_votes_and_skips(question)
-    error_message = ""
-    success_message = "All vote and skip objects have an associated appearance object"
-    skip_appearances_count = Appearance.count(
-      :conditions => ["skips.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL AND answerable_type = 'Skip'", question.id],
-      :joins => "LEFT JOIN skips ON (skips.id = appearances.answerable_id)")
-    vote_appearances_count = Appearance.count(
-      :conditions => ["votes.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL and answerable_type = 'Vote'", question.id],
-      :joins => "LEFT JOIN votes ON (votes.id = appearances.answerable_id)")
-    total_answered_appearances = skip_appearances_count + vote_appearances_count
-    total_votes = question.votes.count
-    total_skips = question.skips.count
-    if (total_answered_appearances != total_votes + total_skips)
-      error_message += "Question #{question.id}: answered_appearances = #{total_answered_appearances}, votes = #{total_votes}, skips = #{total_skips}"
+      if cached_choices_size != actual_choices_size
+        error_message+= "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n"
+      end
+
+      #if cached_prompts_size != question.choices.size **2 - question.choices.size 
+      # error_message += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n"
+      #end
+      return error_message.blank? ? [success_message, false] : [error_message, true] 
     end
+  end
  
+  # END OF QUESTION NAMESPACE
  
-    return error_message.blank? ? [success_message, false] : [error_message, true] 
-  end
  
   def response_time_tests
     error_message = ""
...	...	@@ -125,14 +125,11 @@ namespace :test_api do
125	125	end
126	126
127	127
128		- #catchup specific
129		- if question.uses_catchup?
130		- message, error_occurred = check_prompt_cache_hit_rate(question)
131		- if error_occurred
132		- errors << message
133		- else
134		- successes << message
135		- end
	128	+ message, error_occurred = check_prompt_cache_hit_rate(question)
	129	+ if error_occurred
	130	+ errors << message
	131	+ else
	132	+ successes << message
136	133	end
137	134
138	135	end
...	...	@@ -303,155 +300,172 @@ namespace :test_api do
303	300	return error_message.blank? ? [success_message, false] : [error_message, true]
304	301	end
305	302
306		- def check_each_choice_appears_within_n_stddevs(question)
307		- error_message =""
308		- success_message = "Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question " +
309		- "(Note: this applies only to seed choices (not user submitted) and choices currently marked active)"
310	303
311		- wins_by_choice_id = question.votes.active.count(:group => :choice_id, :conditions => ["creator_id = ?", question.creator_id])
312		- losses_by_choice_id= question.votes.active_loser.count(:group => :loser_choice_id, :conditions => ["creator_id = ?", question.creator_id])
	304	+ namespace :question do
313	305
314		- #Rails returns an ordered hash, which doesn't allow for blocks to change merging logic.
315		- #A little hack to create a normal hash
316		- wins_hash = {}
317		- wins_hash.merge!(wins_by_choice_id)
318		- losses_hash = {}
319		- losses_hash.merge!(losses_by_choice_id)
	306	+ # use this to dynamically create rake task for each question test
	307	+ question_tasks = {
	308	+ :answered_appearances_equals_votes_and_skips => "Ensure that a question has: answered_appearances == votes + skips",
	309	+ :check_each_choice_appears_within_n_stddevs => "Ensure each choice appears within 6 standard deviations",
	310	+ :check_each_choice_equally_likely_to_appear_left_or_right => "Ensure each choice is equally likely to appear on left or right",
	311	+ :check_prompt_cache_hit_rate => "Check prompt cache hit rate",
	312	+ :check_object_counter_cache_values_match_actual_values => "Check that object counter cache values match actual values"
	313	+ }
	314	+
	315	+ # dynamically create tasks for each question task
	316	+ question_tasks.each do \|taskname, description\|
	317	+ desc description
	318	+ task taskname, [:question_id] => :environment do \|t, args\|
	319	+ a = cleanup_args(args)
	320	+ questions = Question.find(a[:question_id])
	321	+ questions.each do \|question\|
	322	+ # call task
	323	+ puts send(taskname, question).inspect
	324	+ end
	325	+ end
	326	+ end
320	327
	328	+ def answered_appearances_equals_votes_and_skips(question)
	329	+ error_message = ""
	330	+ success_message = "All vote and skip objects have an associated appearance object"
	331	+ skip_appearances_count = Appearance.count(
	332	+ :conditions => ["skips.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL AND answerable_type = 'Skip'", question.id],
	333	+ :joins => "LEFT JOIN skips ON (skips.id = appearances.answerable_id)")
	334	+ vote_appearances_count = Appearance.count(
	335	+ :conditions => ["votes.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL and answerable_type = 'Vote'", question.id],
	336	+ :joins => "LEFT JOIN votes ON (votes.id = appearances.answerable_id)")
	337	+ total_answered_appearances = skip_appearances_count + vote_appearances_count
	338	+ total_votes = question.votes.count
	339	+ total_skips = question.skips.count
	340	+ if (total_answered_appearances != total_votes + total_skips)
	341	+ error_message += "Question #{question.id}: answered_appearances = #{total_answered_appearances}, votes = #{total_votes}, skips = #{total_skips}"
	342	+ end
321	343
322	344
323		- appearances_by_choice_id = wins_hash.merge(losses_hash) do \|key, oldval, newval\| oldval + newval end
	345	+ return error_message.blank? ? [success_message, false] : [error_message, true]
	346	+ end
324	347
325		- sum = total_appearances = appearances_by_choice_id.values.inject(0) {\|sum, x\| sum +=x}
326		- mean = average_appearances = total_appearances.to_f / appearances_by_choice_id.size.to_f
	348	+ def check_each_choice_appears_within_n_stddevs(question)
	349	+ error_message =""
	350	+ success_message = "Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question " +
	351	+ "(Note: this applies only to seed choices (not user submitted) and choices currently marked active)"
327	352
328		- if sum > 0:
329		- stddev = Math.sqrt( appearances_by_choice_id.values.inject(0) { \|sum, e\| sum + (e - mean) ** 2 } / appearances_by_choice_id.size.to_f )
	353	+ wins_by_choice_id = question.votes.active.count(:group => :choice_id, :conditions => ["creator_id = ?", question.creator_id])
	354	+ losses_by_choice_id= question.votes.active_loser.count(:group => :loser_choice_id, :conditions => ["creator_id = ?", question.creator_id])
330	355
331		- # this choice appears to have been deactivated then reactivated after
332		- # a period of voting
333		- ignore_choices = [133189]
334		- appearances_by_choice_id.each do \|choice_id, n_i\|
335		- if ((n_i < (mean - 6stddev)) \|\| (n_i > mean + 6 stddev)) && !ignore_choices.include?(choice_id) && Choice.find(choice_id).active?
336		- error_message += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question\n"
337		- end
338		- end
339		- end
	356	+ #Rails returns an ordered hash, which doesn't allow for blocks to change merging logic.
	357	+ #A little hack to create a normal hash
	358	+ wins_hash = {}
	359	+ wins_hash.merge!(wins_by_choice_id)
	360	+ losses_hash = {}
	361	+ losses_hash.merge!(losses_by_choice_id)
340	362
341		- return error_message.blank? ? [success_message, false] : [error_message, true]
342		- end
343	363
344		- def check_each_choice_equally_likely_to_appear_left_or_right(question)
345		- error_message = ""
346		- success_message = "All choices have equal probability of appearing on left or right (within error params)"
347		- question.choices.each do \|c\|
348		- left_prompts_ids = c.prompts_on_the_left.ids_only
349		- right_prompts_ids = c.prompts_on_the_right.ids_only
350	364
351		- left_appearances = question.appearances.count(:conditions => {:prompt_id => left_prompts_ids})
352		- right_appearances = question.appearances.count(:conditions => {:prompt_id => right_prompts_ids})
	365	+ appearances_by_choice_id = wins_hash.merge(losses_hash) do \|key, oldval, newval\| oldval + newval end
353	366
354		- n = left_appearances + right_appearances
	367	+ sum = total_appearances = appearances_by_choice_id.values.inject(0) {\|sum, x\| sum +=x}
	368	+ mean = average_appearances = total_appearances.to_f / appearances_by_choice_id.size.to_f
355	369
356		- if n == 0
357		- next
358		- end
359		- est_p = right_appearances.to_f / n.to_f
360		- z = (est_p - 0.5).abs / Math.sqrt((0.5 * 0.5) / n.to_f)
	370	+ if sum > 0:
	371	+ stddev = Math.sqrt( appearances_by_choice_id.values.inject(0) { \|sum, e\| sum + (e - mean) ** 2 } / appearances_by_choice_id.size.to_f )
361	372
362		- if z > 6
363		- error_message += "Error: Choice ID #{c.id} seems to favor one side: Left Appearances #{left_appearances}, Right Appearances: #{right_appearances}, z = #{z}\n"
	373	+ # this choice appears to have been deactivated then reactivated after
	374	+ # a period of voting
	375	+ ignore_choices = [133189]
	376	+ appearances_by_choice_id.each do \|choice_id, n_i\|
	377	+ if ((n_i < (mean - 6stddev)) \|\| (n_i > mean + 6 stddev)) && !ignore_choices.include?(choice_id) && Choice.find(choice_id).active?
	378	+ error_message += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question\n"
	379	+ end
	380	+ end
364	381	end
	382	+
	383	+ return error_message.blank? ? [success_message, false] : [error_message, true]
365	384	end
366		- return error_message.blank? ? [success_message, false] : [error_message, true]
367		- end
368		- def check_prompt_cache_hit_rate(question)
369		- error_message = ""
370		- success_message = "At least 90% of prompts on catchup algorithm questions were served from cache\n"
371	385
372		- misses = question.get_prompt_cache_misses(Date.yesterday).to_i
373		- hits = question.get_prompt_cache_hits(Date.yesterday).to_i
	386	+ def check_each_choice_equally_likely_to_appear_left_or_right(question)
	387	+ error_message = ""
	388	+ success_message = "All choices have equal probability of appearing on left or right (within error params)"
	389	+ question.choices.each do \|c\|
	390	+ left_prompts_ids = c.prompts_on_the_left.ids_only
	391	+ right_prompts_ids = c.prompts_on_the_right.ids_only
374	392
375		- question.expire_prompt_cache_tracking_keys(Date.yesterday)
	393	+ left_appearances = question.appearances.count(:conditions => {:prompt_id => left_prompts_ids})
	394	+ right_appearances = question.appearances.count(:conditions => {:prompt_id => right_prompts_ids})
376	395
377		- yesterday_appearances = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
	396	+ n = left_appearances + right_appearances
378	397
379		- if misses + hits != yesterday_appearances
380		- error_message += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_appearances}, Actual: #{misses+hits}, Hits: #{hits}, Misses: #{misses}\n"
381		- end
	398	+ if n == 0
	399	+ next
	400	+ end
	401	+ est_p = right_appearances.to_f / n.to_f
	402	+ z = (est_p - 0.5).abs / Math.sqrt((0.5 * 0.5) / n.to_f)
382	403
383		- if yesterday_appearances > 5 # this test isn't worthwhile for small numbers of appearances
384		- miss_rate = misses.to_f / yesterday_appearances.to_f
385		- if miss_rate > 0.1
386		- error_message += "Warning! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}, total appearances yesterday: #{yesterday_appearances}\n"
	404	+ if z > 6
	405	+ error_message += "Error: Choice ID #{c.id} seems to favor one side: Left Appearances #{left_appearances}, Right Appearances: #{right_appearances}, z = #{z}\n"
	406	+ end
387	407	end
	408	+ return error_message.blank? ? [success_message, false] : [error_message, true]
388	409	end
389		- return error_message.blank? ? [success_message, false] : [error_message, true]
390		- end
391		-
392		- def check_object_counter_cache_values_match_actual_values(question)
393		- error_message = ""
394		- success_message = "All cached object values match actual values within database"
395		- # Checks that counter_cache is working as expected
396		- cached_prompts_size = question.prompts.size
397		- actual_prompts_size = question.prompts.count
	410	+ def check_prompt_cache_hit_rate(question)
	411	+ error_message = ""
	412	+ success_message = "At least 90% of prompts on catchup algorithm questions were served from cache\n"
	413	+ return [success_message, false] unless question.uses_catchup?
398	414
399		- if cached_prompts_size != actual_prompts_size
400		- error_message += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n"
401		- end
	415	+ misses = question.get_prompt_cache_misses(Date.yesterday).to_i
	416	+ hits = question.get_prompt_cache_hits(Date.yesterday).to_i
402	417
403		- cached_votes_size = question.votes.size
404		- actual_votes_size = question.votes.count
	418	+ question.expire_prompt_cache_tracking_keys(Date.yesterday)
405	419
406		- if cached_votes_size != actual_votes_size
407		- error_message += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n"
408		- end
	420	+ yesterday_appearances = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
409	421
410		- cached_choices_size = question.choices.size
411		- actual_choices_size = question.choices.count
	422	+ if misses + hits != yesterday_appearances
	423	+ error_message += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_appearances}, Actual: #{misses+hits}, Hits: #{hits}, Misses: #{misses}\n"
	424	+ end
412	425
413		- if cached_choices_size != actual_choices_size
414		- error_message+= "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n"
	426	+ if yesterday_appearances > 5 # this test isn't worthwhile for small numbers of appearances
	427	+ miss_rate = misses.to_f / yesterday_appearances.to_f
	428	+ if miss_rate > 0.1
	429	+ error_message += "Warning! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}, total appearances yesterday: #{yesterday_appearances}\n"
	430	+ end
	431	+ end
	432	+ return error_message.blank? ? [success_message, false] : [error_message, true]
415	433	end
416	434
417		- #if cached_prompts_size != question.choices.size **2 - question.choices.size
418		- # error_message += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n"
419		- #end
420		- return error_message.blank? ? [success_message, false] : [error_message, true]
421		- end
	435	+ def check_object_counter_cache_values_match_actual_values(question)
	436	+ error_message = ""
	437	+ success_message = "All cached object values match actual values within database"
	438	+ # Checks that counter_cache is working as expected
	439	+ cached_prompts_size = question.prompts.size
	440	+ actual_prompts_size = question.prompts.count
422	441
423		- namespace :question do
	442	+ if cached_prompts_size != actual_prompts_size
	443	+ error_message += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n"
	444	+ end
424	445
425		- desc "Ensure that a question has: answered_appearances == votes + skips"
426		- task :answered_appearances_equals_votes_and_skips, [:question_id] => :environment do \|t, args\|
427		- a = cleanup_args(args)
428		- questions = Question.find(a[:question_id])
429		- questions.each do \|question\|
430		- puts answered_appearances_equals_votes_and_skips(question).inspect
	446	+ cached_votes_size = question.votes.size
	447	+ actual_votes_size = question.votes.count
	448	+
	449	+ if cached_votes_size != actual_votes_size
	450	+ error_message += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n"
431	451	end
432		- end
433	452
434		- end
	453	+ cached_choices_size = question.choices.size
	454	+ actual_choices_size = question.choices.count
435	455
436		- def answered_appearances_equals_votes_and_skips(question)
437		- error_message = ""
438		- success_message = "All vote and skip objects have an associated appearance object"
439		- skip_appearances_count = Appearance.count(
440		- :conditions => ["skips.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL AND answerable_type = 'Skip'", question.id],
441		- :joins => "LEFT JOIN skips ON (skips.id = appearances.answerable_id)")
442		- vote_appearances_count = Appearance.count(
443		- :conditions => ["votes.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL and answerable_type = 'Vote'", question.id],
444		- :joins => "LEFT JOIN votes ON (votes.id = appearances.answerable_id)")
445		- total_answered_appearances = skip_appearances_count + vote_appearances_count
446		- total_votes = question.votes.count
447		- total_skips = question.skips.count
448		- if (total_answered_appearances != total_votes + total_skips)
449		- error_message += "Question #{question.id}: answered_appearances = #{total_answered_appearances}, votes = #{total_votes}, skips = #{total_skips}"
	456	+ if cached_choices_size != actual_choices_size
	457	+ error_message+= "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n"
	458	+ end
	459	+
	460	+ #if cached_prompts_size != question.choices.size **2 - question.choices.size
	461	+ # error_message += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n"
	462	+ #end
	463	+ return error_message.blank? ? [success_message, false] : [error_message, true]
450	464	end
	465	+ end
451	466
	467	+ # END OF QUESTION NAMESPACE
452	468
453		- return error_message.blank? ? [success_message, false] : [error_message, true]
454		- end
455	469
456	470	def response_time_tests
457	471	error_message = ""
...	...