diff --git a/lib/tasks/test_api.rake b/lib/tasks/test_api.rake index b84538e..dec2dc6 100644 --- a/lib/tasks/test_api.rake +++ b/lib/tasks/test_api.rake @@ -125,14 +125,11 @@ namespace :test_api do end - #catchup specific - if question.uses_catchup? - message, error_occurred = check_prompt_cache_hit_rate(question) - if error_occurred - errors << message - else - successes << message - end + message, error_occurred = check_prompt_cache_hit_rate(question) + if error_occurred + errors << message + else + successes << message end end @@ -303,155 +300,172 @@ namespace :test_api do return error_message.blank? ? [success_message, false] : [error_message, true] end - def check_each_choice_appears_within_n_stddevs(question) - error_message ="" - success_message = "Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question " + - "(Note: this applies only to seed choices (not user submitted) and choices currently marked active)" - wins_by_choice_id = question.votes.active.count(:group => :choice_id, :conditions => ["creator_id = ?", question.creator_id]) - losses_by_choice_id= question.votes.active_loser.count(:group => :loser_choice_id, :conditions => ["creator_id = ?", question.creator_id]) + namespace :question do - #Rails returns an ordered hash, which doesn't allow for blocks to change merging logic. - #A little hack to create a normal hash - wins_hash = {} - wins_hash.merge!(wins_by_choice_id) - losses_hash = {} - losses_hash.merge!(losses_by_choice_id) + # use this to dynamically create rake task for each question test + question_tasks = { + :answered_appearances_equals_votes_and_skips => "Ensure that a question has: answered_appearances == votes + skips", + :check_each_choice_appears_within_n_stddevs => "Ensure each choice appears within 6 standard deviations", + :check_each_choice_equally_likely_to_appear_left_or_right => "Ensure each choice is equally likely to appear on left or right", + :check_prompt_cache_hit_rate => "Check prompt cache hit rate", + :check_object_counter_cache_values_match_actual_values => "Check that object counter cache values match actual values" + } + + # dynamically create tasks for each question task + question_tasks.each do |taskname, description| + desc description + task taskname, [:question_id] => :environment do |t, args| + a = cleanup_args(args) + questions = Question.find(a[:question_id]) + questions.each do |question| + # call task + puts send(taskname, question).inspect + end + end + end + def answered_appearances_equals_votes_and_skips(question) + error_message = "" + success_message = "All vote and skip objects have an associated appearance object" + skip_appearances_count = Appearance.count( + :conditions => ["skips.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL AND answerable_type = 'Skip'", question.id], + :joins => "LEFT JOIN skips ON (skips.id = appearances.answerable_id)") + vote_appearances_count = Appearance.count( + :conditions => ["votes.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL and answerable_type = 'Vote'", question.id], + :joins => "LEFT JOIN votes ON (votes.id = appearances.answerable_id)") + total_answered_appearances = skip_appearances_count + vote_appearances_count + total_votes = question.votes.count + total_skips = question.skips.count + if (total_answered_appearances != total_votes + total_skips) + error_message += "Question #{question.id}: answered_appearances = #{total_answered_appearances}, votes = #{total_votes}, skips = #{total_skips}" + end - appearances_by_choice_id = wins_hash.merge(losses_hash) do |key, oldval, newval| oldval + newval end + return error_message.blank? ? [success_message, false] : [error_message, true] + end - sum = total_appearances = appearances_by_choice_id.values.inject(0) {|sum, x| sum +=x} - mean = average_appearances = total_appearances.to_f / appearances_by_choice_id.size.to_f + def check_each_choice_appears_within_n_stddevs(question) + error_message ="" + success_message = "Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question " + + "(Note: this applies only to seed choices (not user submitted) and choices currently marked active)" - if sum > 0: - stddev = Math.sqrt( appearances_by_choice_id.values.inject(0) { |sum, e| sum + (e - mean) ** 2 } / appearances_by_choice_id.size.to_f ) + wins_by_choice_id = question.votes.active.count(:group => :choice_id, :conditions => ["creator_id = ?", question.creator_id]) + losses_by_choice_id= question.votes.active_loser.count(:group => :loser_choice_id, :conditions => ["creator_id = ?", question.creator_id]) - # this choice appears to have been deactivated then reactivated after - # a period of voting - ignore_choices = [133189] - appearances_by_choice_id.each do |choice_id, n_i| - if ((n_i < (mean - 6*stddev)) || (n_i > mean + 6 *stddev)) && !ignore_choices.include?(choice_id) && Choice.find(choice_id).active? - error_message += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question\n" - end - end - end + #Rails returns an ordered hash, which doesn't allow for blocks to change merging logic. + #A little hack to create a normal hash + wins_hash = {} + wins_hash.merge!(wins_by_choice_id) + losses_hash = {} + losses_hash.merge!(losses_by_choice_id) - return error_message.blank? ? [success_message, false] : [error_message, true] - end - def check_each_choice_equally_likely_to_appear_left_or_right(question) - error_message = "" - success_message = "All choices have equal probability of appearing on left or right (within error params)" - question.choices.each do |c| - left_prompts_ids = c.prompts_on_the_left.ids_only - right_prompts_ids = c.prompts_on_the_right.ids_only - left_appearances = question.appearances.count(:conditions => {:prompt_id => left_prompts_ids}) - right_appearances = question.appearances.count(:conditions => {:prompt_id => right_prompts_ids}) + appearances_by_choice_id = wins_hash.merge(losses_hash) do |key, oldval, newval| oldval + newval end - n = left_appearances + right_appearances + sum = total_appearances = appearances_by_choice_id.values.inject(0) {|sum, x| sum +=x} + mean = average_appearances = total_appearances.to_f / appearances_by_choice_id.size.to_f - if n == 0 - next - end - est_p = right_appearances.to_f / n.to_f - z = (est_p - 0.5).abs / Math.sqrt((0.5 * 0.5) / n.to_f) + if sum > 0: + stddev = Math.sqrt( appearances_by_choice_id.values.inject(0) { |sum, e| sum + (e - mean) ** 2 } / appearances_by_choice_id.size.to_f ) - if z > 6 - error_message += "Error: Choice ID #{c.id} seems to favor one side: Left Appearances #{left_appearances}, Right Appearances: #{right_appearances}, z = #{z}\n" + # this choice appears to have been deactivated then reactivated after + # a period of voting + ignore_choices = [133189] + appearances_by_choice_id.each do |choice_id, n_i| + if ((n_i < (mean - 6*stddev)) || (n_i > mean + 6 *stddev)) && !ignore_choices.include?(choice_id) && Choice.find(choice_id).active? + error_message += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question\n" + end + end end + + return error_message.blank? ? [success_message, false] : [error_message, true] end - return error_message.blank? ? [success_message, false] : [error_message, true] - end - def check_prompt_cache_hit_rate(question) - error_message = "" - success_message = "At least 90% of prompts on catchup algorithm questions were served from cache\n" - misses = question.get_prompt_cache_misses(Date.yesterday).to_i - hits = question.get_prompt_cache_hits(Date.yesterday).to_i + def check_each_choice_equally_likely_to_appear_left_or_right(question) + error_message = "" + success_message = "All choices have equal probability of appearing on left or right (within error params)" + question.choices.each do |c| + left_prompts_ids = c.prompts_on_the_left.ids_only + right_prompts_ids = c.prompts_on_the_right.ids_only - question.expire_prompt_cache_tracking_keys(Date.yesterday) + left_appearances = question.appearances.count(:conditions => {:prompt_id => left_prompts_ids}) + right_appearances = question.appearances.count(:conditions => {:prompt_id => right_prompts_ids}) - yesterday_appearances = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday]) + n = left_appearances + right_appearances - if misses + hits != yesterday_appearances - error_message += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_appearances}, Actual: #{misses+hits}, Hits: #{hits}, Misses: #{misses}\n" - end + if n == 0 + next + end + est_p = right_appearances.to_f / n.to_f + z = (est_p - 0.5).abs / Math.sqrt((0.5 * 0.5) / n.to_f) - if yesterday_appearances > 5 # this test isn't worthwhile for small numbers of appearances - miss_rate = misses.to_f / yesterday_appearances.to_f - if miss_rate > 0.1 - error_message += "Warning! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}, total appearances yesterday: #{yesterday_appearances}\n" + if z > 6 + error_message += "Error: Choice ID #{c.id} seems to favor one side: Left Appearances #{left_appearances}, Right Appearances: #{right_appearances}, z = #{z}\n" + end end + return error_message.blank? ? [success_message, false] : [error_message, true] end - return error_message.blank? ? [success_message, false] : [error_message, true] - end - - def check_object_counter_cache_values_match_actual_values(question) - error_message = "" - success_message = "All cached object values match actual values within database" - # Checks that counter_cache is working as expected - cached_prompts_size = question.prompts.size - actual_prompts_size = question.prompts.count + def check_prompt_cache_hit_rate(question) + error_message = "" + success_message = "At least 90% of prompts on catchup algorithm questions were served from cache\n" + return [success_message, false] unless question.uses_catchup? - if cached_prompts_size != actual_prompts_size - error_message += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n" - end + misses = question.get_prompt_cache_misses(Date.yesterday).to_i + hits = question.get_prompt_cache_hits(Date.yesterday).to_i - cached_votes_size = question.votes.size - actual_votes_size = question.votes.count + question.expire_prompt_cache_tracking_keys(Date.yesterday) - if cached_votes_size != actual_votes_size - error_message += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n" - end + yesterday_appearances = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday]) - cached_choices_size = question.choices.size - actual_choices_size = question.choices.count + if misses + hits != yesterday_appearances + error_message += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_appearances}, Actual: #{misses+hits}, Hits: #{hits}, Misses: #{misses}\n" + end - if cached_choices_size != actual_choices_size - error_message+= "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n" + if yesterday_appearances > 5 # this test isn't worthwhile for small numbers of appearances + miss_rate = misses.to_f / yesterday_appearances.to_f + if miss_rate > 0.1 + error_message += "Warning! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}, total appearances yesterday: #{yesterday_appearances}\n" + end + end + return error_message.blank? ? [success_message, false] : [error_message, true] end - #if cached_prompts_size != question.choices.size **2 - question.choices.size - # error_message += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n" - #end - return error_message.blank? ? [success_message, false] : [error_message, true] - end + def check_object_counter_cache_values_match_actual_values(question) + error_message = "" + success_message = "All cached object values match actual values within database" + # Checks that counter_cache is working as expected + cached_prompts_size = question.prompts.size + actual_prompts_size = question.prompts.count - namespace :question do + if cached_prompts_size != actual_prompts_size + error_message += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n" + end - desc "Ensure that a question has: answered_appearances == votes + skips" - task :answered_appearances_equals_votes_and_skips, [:question_id] => :environment do |t, args| - a = cleanup_args(args) - questions = Question.find(a[:question_id]) - questions.each do |question| - puts answered_appearances_equals_votes_and_skips(question).inspect + cached_votes_size = question.votes.size + actual_votes_size = question.votes.count + + if cached_votes_size != actual_votes_size + error_message += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n" end - end - end + cached_choices_size = question.choices.size + actual_choices_size = question.choices.count - def answered_appearances_equals_votes_and_skips(question) - error_message = "" - success_message = "All vote and skip objects have an associated appearance object" - skip_appearances_count = Appearance.count( - :conditions => ["skips.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL AND answerable_type = 'Skip'", question.id], - :joins => "LEFT JOIN skips ON (skips.id = appearances.answerable_id)") - vote_appearances_count = Appearance.count( - :conditions => ["votes.valid_record = 1 and appearances.question_id = ? AND answerable_id IS NOT NULL and answerable_type = 'Vote'", question.id], - :joins => "LEFT JOIN votes ON (votes.id = appearances.answerable_id)") - total_answered_appearances = skip_appearances_count + vote_appearances_count - total_votes = question.votes.count - total_skips = question.skips.count - if (total_answered_appearances != total_votes + total_skips) - error_message += "Question #{question.id}: answered_appearances = #{total_answered_appearances}, votes = #{total_votes}, skips = #{total_skips}" + if cached_choices_size != actual_choices_size + error_message+= "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n" + end + + #if cached_prompts_size != question.choices.size **2 - question.choices.size + # error_message += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n" + #end + return error_message.blank? ? [success_message, false] : [error_message, true] end + end + # END OF QUESTION NAMESPACE - return error_message.blank? ? [success_message, false] : [error_message, true] - end def response_time_tests error_message = "" -- libgit2 0.21.2