Refactored nightly api test. Added api test for l/r probabilty

Dhruv Kapadia
1 parent 26ecf5ed
Showing 1 changed file with 188 additions and 109 deletions Show diff stats
lib/tasks/test_api.rake
@@ -243,13 +243,98 @@ namespace :test_api do
    desc "Description here"
    task(:question_vote_consistency => :environment) do
       questions = Question.find(:all)
+      errors = []
+      successes = []
  
-      error_msg = ""
-
-      bad_choices = []
-      bad_votes = []
       questions.each do |question|
  
+	message, error_occurred = check_basic_balanced_stats(question)
+        #hack for now, get around to doing this with block /yield to get rid of duplication
+        if error_occurred
+	   errors << message
+        else
+	   successes << message
+        end
+
+
+        message, error_occurred = check_each_choice_appears_within_n_stddevs(question)
+        if error_occurred
+	   errors << message
+        else
+	   successes << message
+        end
+        
+	message, error_occurred = check_each_choice_equally_likely_to_appear_left_or_right(question)
+        if error_occurred
+	   errors << message
+        else
+	   successes << message
+        end
+	
+	
+			
+	message, error_occurred = check_object_counter_cache_values_match_actual_values(question)
+        if error_occurred
+	   errors << message
+        else
+	   successes << message
+        end
+
+
+	#catchup specific 
+	if question.uses_catchup?
+	     message, error_occurred = check_prompt_cache_hit_rate(question)
+             if error_occurred
+	        errors << message
+             else
+	        successes << message
+             end
+	end
+     end
+
+     message, error_occurred = ensure_all_votes_and_skips_have_unique_appearance
+     
+     if error_occurred
+	errors << message
+     else
+	successes << message
+     end
+
+     message, error_occurred = response_time_tests	
+
+     if error_occurred
+	errors << message
+     else
+	successes << message
+     end
+
+     email_text = "Conducted the following tests on API data and found the following results\n" +
+			 "For each of the #{questions.length} questions in the database: \n"
+     errors.each do |e|
+	email_text += "     Test FAILED: " + e + "\n"
+     end
+
+     successes.uniq.each do |s|
+	s.split("\n").each do |m| # some successes have several lines
+	   email_text += "     Test Passed: " + m + "\n"
+	end
+     end
+	 
+     puts email_text
+     
+     if errors.empty?
+	CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", email_text)
+     else
+     	CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , email_text)
+     end
+
+   end
+   def check_basic_balanced_stats(question)
+	error_message = ""
+	success_message = "2 x Total Wins = Total Votes\n" +
+			 "Total Votes (wins + losses) is Even\n" +
+			 "Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +
+		         "Total generated prompts on left = Total generated prompts on right"
 	total_wins =0
 	total_votes =0
 	total_generated_prompts_on_left = 0
@@ -257,6 +342,7 @@ namespace :test_api do
 	total_scores_gte_fifty= 0
 	total_scores_lte_fifty= 0
 	error_bool = false
+
 	question.choices.each do |choice|
  
 	    if choice.wins
@@ -277,20 +363,19 @@ namespace :test_api do
 	    delta = 0.001
  
 	    if (cached_score - generated_score).abs >= delta
-		 error_msg += "Error! The cached_score is not equal to the calculated score for choice #{choice.id}"
+		 error_message += "Error! The cached_score is not equal to the calculated score for choice #{choice.id}"
  
 		 print "This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
  
-		 bad_choices << choice.id
  
 	    end
  
 	    if cached_score == 0.0 || cached_score == 100.0 || cached_score.nil?
-		 error_msg += "Error! The cached_score for choice #{choice.id} is exactly 0 or 100, the value: #{cached_score}"
+		 error_message += "Error! The cached_score for choice #{choice.id} is exactly 0 or 100, the value: #{cached_score}"
 		 print "Either 0 or 100 This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
-		 bad_choices << choice.id
 	    end
  
+
 	    if cached_score >= 50
 		    total_scores_gte_fifty +=1
 	    end
@@ -302,31 +387,40 @@ namespace :test_api do
         end
  
 	if (2*total_wins != total_votes)
-		 error_msg += "Error 1: 2 x Total Wins != Total votes"
+		 error_message += "Error 1: 2 x Total Wins != Total votes"
 		 error_bool= true
 	end
  
 	if(total_votes % 2 != 0)
-		error_msg += "Error 2: Total votes is not Even!"
+		error_message += "Error 2: Total votes is not Even!"
 		error_bool= true
 	end
  
 	if(total_votes != 2* question.votes_count)
-		error_msg += "Error 3: Total votes != 2 x # vote objects"
+		error_message += "Error 3: Total votes != 2 x # vote objects"
 		error_bool = true
 	end
  
 	if(total_generated_prompts_on_right != total_generated_prompts_on_right)
-		error_msg += "Error 4: Total generated prompts on left != Total generated prompts on right"
+		error_message += "Error 4: Total generated prompts on left != Total generated prompts on right"
 		error_bool = true
 	end
  
 	if(total_scores_lte_fifty == question.choices.size || total_scores_gte_fifty == question.choices.size) && (total_scores_lte_fifty != total_scores_gte_fifty)
-		error_msg += "Error: The scores of all choices are either all above 50, or all below 50. This is probably wrong"
+		error_message += "Error: The scores of all choices are either all above 50, or all below 50. This is probably wrong"
 		error_bool = true
 		puts "Error score fifty: #{question.id}"
 	end
-
+	
+	if error_bool
+	   error_message += "Question #{question.id}: 2*wins = #{2*total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
+	end
+        return error_message.blank? ? [success_message, false] : [error_message, true] 
+   end
+   def check_each_choice_appears_within_n_stddevs(question)
+	error_message =""
+	success_message = "Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question " +
+			 "(Note: this applies only to seed choices (not user submitted) and choices currently marked active)"
  
 	wins_by_choice_id = question.votes.active.count(:group => :choice_id)
 	losses_by_choice_id= question.votes.active.count(:conditions => "loser_choice_id IS NOT NULL", :group => :loser_choice_id)
@@ -339,6 +433,7 @@ namespace :test_api do
 	losses_hash.merge!(losses_by_choice_id)
  
  
+
 	appearances_by_choice_id = wins_hash.merge(losses_hash) do |key, oldval, newval| oldval + newval end
  
 	sum = total_appearances = appearances_by_choice_id.values.inject(0) {|sum, x| sum +=x}
@@ -349,157 +444,141 @@ namespace :test_api do
  
            appearances_by_choice_id.each do |choice_id, n_i| 
 		if (n_i < (mean - 6*stddev)) || (n_i > mean + 6 *stddev)
-			error_msg += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question"
-			error_bool = true
+			error_message += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question"
 		end
 	   end
 	end
-			
+
+        return error_message.blank? ? [success_message, false] : [error_message, true] 
+   end
+   def check_each_choice_equally_likely_to_appear_left_or_right(question)
+   	error_message = ""
+	success_message = "All choices have equal probability of appearing on left or right (within error params)"
+	question.choices.each do |c|
+	         left_prompts_ids = c.prompts_on_the_left.ids_only
+	         right_prompts_ids = c.prompts_on_the_right.ids_only
+
+	         left_appearances = question.appearances.count(:conditions => {:prompt_id => left_prompts_ids})
+	         right_appearances = question.appearances.count(:conditions => {:prompt_id => right_prompts_ids})
+
+		 n = left_appearances + right_appearances
+
+		 if n == 0
+		    next
+		 end
+		 est_p = right_appearances.to_f / n.to_f
+		 z = (est_p - 0.5).abs / Math.sqrt((0.5 * 0.5) / n.to_f)
+
+		 if z > 6 
+		 	error_message += "Error: Choice ID #{c.id} seems to favor one side: Left Appearances #{left_appearances}, Right Appearances: #{right_appearances}, z = #{z}\n"
+		 end
+	end
+        return error_message.blank? ? [success_message, false] : [error_message, true] 
+   end
+   def check_prompt_cache_hit_rate(question)
+	error_message = ""
+	success_message = "At least 90% of prompts on catchup algorithm questions were served from cache\n" 
+
+	misses = question.get_prompt_cache_misses(Date.yesterday).to_i
+	hits = question.get_prompt_cache_hits(Date.yesterday).to_i
+
+	question.expire_prompt_cache_tracking_keys(Date.yesterday)
+		
+	yesterday_votes = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
+
+	if misses + hits != yesterday_votes
+	     error_message += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_votes}, Actual: #{misses+hits}\n"
+	end
+
+	miss_rate = misses.to_f / yesterday_votes.to_f
+	if miss_rate > 0.1
+	     error_message += "Error! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}\n"
+	end
+        return error_message.blank? ? [success_message, false] : [error_message, true] 
+   end
+
+   def check_object_counter_cache_values_match_actual_values(question)
+        error_message = ""
+        success_message = "All cached object values match actual values within database"
         # Checks that counter_cache is working as expected
 	cached_prompts_size = question.prompts.size
 	actual_prompts_size = question.prompts.count
  
 	if cached_prompts_size != actual_prompts_size
-		error_msg += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n"
+		error_message += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n"
 	end
  
 	cached_votes_size = question.votes.size
 	actual_votes_size = question.votes.count
  
 	if cached_votes_size != actual_votes_size
-		error_msg += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n"
+		error_message += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n"
 	end
  
 	cached_choices_size = question.choices.size
 	actual_choices_size = question.choices.count
  
 	if cached_choices_size != actual_choices_size
-		error_msg += "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n"
+		error_message+= "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n"
 	end
  
 	if cached_prompts_size != question.choices.size **2 - question.choices.size 
-		error_msg += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n"
+		error_message += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n"
 	end
-
-
-	#catchup specific 
-	if question.uses_catchup?
-		misses = question.get_prompt_cache_misses(Date.yesterday).to_i
-		hits = question.get_prompt_cache_hits(Date.yesterday).to_i
-
-		question.expire_prompt_cache_tracking_keys(Date.yesterday)
-
-
-		
-		yesterday_votes = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
-
-		if misses + hits != yesterday_votes
-		     error_msg += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_votes}, Actual: #{misses+hits}\n"
-		end
-
-		miss_rate = misses.to_f / yesterday_votes.to_f
-		if miss_rate > 0.1
-		     error_msg += "Error! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}\n"
-		end
-	end
-
-
-	if error_bool
-	   error_msg += "Question #{question.id}: 2*wins = #{2*total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
-	end
-
-	error_bool = false
-     end
-
+        return error_message.blank? ? [success_message, false] : [error_message, true] 
+   end
+   
+   def ensure_all_votes_and_skips_have_unique_appearance
+     error_message = ""
+     success_message = "All vote and skip objects have an associated appearance object"
      votes_without_appearances= Vote.count(:conditions => {:appearance_id => nil})
      if (votes_without_appearances > 0)
-	     error_msg += "Error! There are #{votes_without_appearances} votes without associated appearance objects."
+	     error_message += "Error! There are #{votes_without_appearances} votes without associated appearance objects."
      end
  
      skips_without_appearances= Skip.count(:conditions => {:appearance_id => nil})
      if (skips_without_appearances > 0)
-	     error_msg += "Error! There are #{skips_without_appearances} skips without associated appearance objects."
+	     error_message += "Error! There are #{skips_without_appearances} skips without associated appearance objects."
      end
+     
+     return error_message.blank? ? [success_message, false] : [error_message, true] 
+   end
  
+   def response_time_tests
+     error_message = ""
+     success_message = "All Vote objects have an client response time < calculated server roundtrip time\n" 
  
      recording_client_time_start_date = Vote.find(:all, :conditions => 'time_viewed IS NOT NULL', :order => 'created_at', :limit => 1).first.created_at
  
      Vote.find_each(:batch_size => 1000, :include => :appearance) do |v|
  
-
 	     # Subtracting DateTime objects results in the difference in days
 	     server_response_time = v.created_at.to_f - v.appearance.created_at.to_f
 	     if server_response_time < 0
 	        the_error_msg = "Error! Vote #{v.id} was created before the appearance associated with it: Appearance id: #{v.appearance.id}, Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}\n"
  
-
-		error_msg += the_error_msg
-	        print the_error_msg
-
-		print "Error!"
+		error_message += the_error_msg
+		print "Error!" + the_error_msg
 	     end
  
 	     if v.time_viewed && v.time_viewed/1000 > server_response_time 
 		     the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, has a longer client response time than is possible. Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
  
-		     error_msg += the_error_msg
-		     print the_error_msg
-
-		     bad_votes << v.id
+		     error_message += the_error_msg
+	             print the_error_msg
  
              elsif v.time_viewed.nil?
 		     if v.created_at > recording_client_time_start_date && v.missing_response_time_exp != 'invalid'
 	                the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, does not have a client response, even though it should! Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
-			error_msg += the_error_msg
+		        error_message += the_error_msg
 	                print the_error_msg
 		     end
  
 	     end
  
-
-     end
-            
-     if error_msg.blank?
-        
-	success_msg = "Conducted the following tests on API data and found no inconsistencies.\n" +
-			 "For each of the #{questions.length} questions in the database: \n" +
-			 "     2 x Total Wins = Total Votes\n" +
-			 "     Total Votes (wins + losses) is Even\n" +
-			 "     Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +
-		         "     Total generated prompts on left = Total generated prompts on right\n" + 
-		         "     Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question\n" +
-			 "             Note: this applies only to seed choices (not user submitted) and choices currently marked active\n" + 
-			 "     The cached score value matches the calculated score value for each choice\n" +
-			 "     The cached vote count matches the actual number of votes for each question\n" +
-			 "     The cached choices count matches the actual number of choices for each question\n" +
-			 "     The cached prompt count matches the actual number of prompts for each question\n" +
-			 "     The prompt count matches the expected number of prompts ( num_choices ^2 - num choices) for each question\n" +
-			 "     All Vote objects have an associated appearance object\n" +
-			 "     All Vote objects have an client response time < calculated server roundtrip time\n" 
-			 "     More than 90% of prompts on catchup algorithm questions were served from cache\n" 
-
-	print success_msg
-
-	CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", success_msg)
-     else
-     	CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg)
-
-	puts "There were errors: "
-	puts error_msg
-
-	unless bad_choices.blank?
-
-		puts "Here's a list of choice ids that you may want to modify: #{bad_choices.uniq.inspect}"
-
-	end
-	unless bad_votes.blank?
-
-		puts "Here's a list of vote ids that you may want to modify: #{bad_votes.uniq.inspect}"
-
-	end
-	print error_msg
+       end
+	
+        return error_message.blank? ? [success_message, false] : [error_message, true] 
      end
-
-   end
 end
...	...	@@ -243,13 +243,98 @@ namespace :test_api do
243	243	desc "Description here"
244	244	task(:question_vote_consistency => :environment) do
245	245	questions = Question.find(:all)
	246	+ errors = []
	247	+ successes = []
246	248
247		- error_msg = ""
248		-
249		- bad_choices = []
250		- bad_votes = []
251	249	questions.each do \|question\|
252	250
	251	+ message, error_occurred = check_basic_balanced_stats(question)
	252	+ #hack for now, get around to doing this with block /yield to get rid of duplication
	253	+ if error_occurred
	254	+ errors << message
	255	+ else
	256	+ successes << message
	257	+ end
	258	+
	259	+
	260	+ message, error_occurred = check_each_choice_appears_within_n_stddevs(question)
	261	+ if error_occurred
	262	+ errors << message
	263	+ else
	264	+ successes << message
	265	+ end
	266	+
	267	+ message, error_occurred = check_each_choice_equally_likely_to_appear_left_or_right(question)
	268	+ if error_occurred
	269	+ errors << message
	270	+ else
	271	+ successes << message
	272	+ end
	273	+
	274	+
	275	+
	276	+ message, error_occurred = check_object_counter_cache_values_match_actual_values(question)
	277	+ if error_occurred
	278	+ errors << message
	279	+ else
	280	+ successes << message
	281	+ end
	282	+
	283	+
	284	+ #catchup specific
	285	+ if question.uses_catchup?
	286	+ message, error_occurred = check_prompt_cache_hit_rate(question)
	287	+ if error_occurred
	288	+ errors << message
	289	+ else
	290	+ successes << message
	291	+ end
	292	+ end
	293	+ end
	294	+
	295	+ message, error_occurred = ensure_all_votes_and_skips_have_unique_appearance
	296	+
	297	+ if error_occurred
	298	+ errors << message
	299	+ else
	300	+ successes << message
	301	+ end
	302	+
	303	+ message, error_occurred = response_time_tests
	304	+
	305	+ if error_occurred
	306	+ errors << message
	307	+ else
	308	+ successes << message
	309	+ end
	310	+
	311	+ email_text = "Conducted the following tests on API data and found the following results\n" +
	312	+ "For each of the #{questions.length} questions in the database: \n"
	313	+ errors.each do \|e\|
	314	+ email_text += " Test FAILED: " + e + "\n"
	315	+ end
	316	+
	317	+ successes.uniq.each do \|s\|
	318	+ s.split("\n").each do \|m\| # some successes have several lines
	319	+ email_text += " Test Passed: " + m + "\n"
	320	+ end
	321	+ end
	322	+
	323	+ puts email_text
	324	+
	325	+ if errors.empty?
	326	+ CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", email_text)
	327	+ else
	328	+ CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , email_text)
	329	+ end
	330	+
	331	+ end
	332	+ def check_basic_balanced_stats(question)
	333	+ error_message = ""
	334	+ success_message = "2 x Total Wins = Total Votes\n" +
	335	+ "Total Votes (wins + losses) is Even\n" +
	336	+ "Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +
	337	+ "Total generated prompts on left = Total generated prompts on right"
253	338	total_wins =0
254	339	total_votes =0
255	340	total_generated_prompts_on_left = 0
...	...	@@ -257,6 +342,7 @@ namespace :test_api do
257	342	total_scores_gte_fifty= 0
258	343	total_scores_lte_fifty= 0
259	344	error_bool = false
	345	+
260	346	question.choices.each do \|choice\|
261	347
262	348	if choice.wins
...	...	@@ -277,20 +363,19 @@ namespace :test_api do
277	363	delta = 0.001
278	364
279	365	if (cached_score - generated_score).abs >= delta
280		- error_msg += "Error! The cached_score is not equal to the calculated score for choice #{choice.id}"
	366	+ error_message += "Error! The cached_score is not equal to the calculated score for choice #{choice.id}"
281	367
282	368	print "This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
283	369
284		- bad_choices << choice.id
285	370
286	371	end
287	372
288	373	if cached_score == 0.0 \|\| cached_score == 100.0 \|\| cached_score.nil?
289		- error_msg += "Error! The cached_score for choice #{choice.id} is exactly 0 or 100, the value: #{cached_score}"
	374	+ error_message += "Error! The cached_score for choice #{choice.id} is exactly 0 or 100, the value: #{cached_score}"
290	375	print "Either 0 or 100 This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
291		- bad_choices << choice.id
292	376	end
293	377
	378	+
294	379	if cached_score >= 50
295	380	total_scores_gte_fifty +=1
296	381	end
...	...	@@ -302,31 +387,40 @@ namespace :test_api do
302	387	end
303	388
304	389	if (2*total_wins != total_votes)
305		- error_msg += "Error 1: 2 x Total Wins != Total votes"
	390	+ error_message += "Error 1: 2 x Total Wins != Total votes"
306	391	error_bool= true
307	392	end
308	393
309	394	if(total_votes % 2 != 0)
310		- error_msg += "Error 2: Total votes is not Even!"
	395	+ error_message += "Error 2: Total votes is not Even!"
311	396	error_bool= true
312	397	end
313	398
314	399	if(total_votes != 2* question.votes_count)
315		- error_msg += "Error 3: Total votes != 2 x # vote objects"
	400	+ error_message += "Error 3: Total votes != 2 x # vote objects"
316	401	error_bool = true
317	402	end
318	403
319	404	if(total_generated_prompts_on_right != total_generated_prompts_on_right)
320		- error_msg += "Error 4: Total generated prompts on left != Total generated prompts on right"
	405	+ error_message += "Error 4: Total generated prompts on left != Total generated prompts on right"
321	406	error_bool = true
322	407	end
323	408
324	409	if(total_scores_lte_fifty == question.choices.size \|\| total_scores_gte_fifty == question.choices.size) && (total_scores_lte_fifty != total_scores_gte_fifty)
325		- error_msg += "Error: The scores of all choices are either all above 50, or all below 50. This is probably wrong"
	410	+ error_message += "Error: The scores of all choices are either all above 50, or all below 50. This is probably wrong"
326	411	error_bool = true
327	412	puts "Error score fifty: #{question.id}"
328	413	end
329		-
	414	+
	415	+ if error_bool
	416	+ error_message += "Question #{question.id}: 2wins = #{2total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
	417	+ end
	418	+ return error_message.blank? ? [success_message, false] : [error_message, true]
	419	+ end
	420	+ def check_each_choice_appears_within_n_stddevs(question)
	421	+ error_message =""
	422	+ success_message = "Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question " +
	423	+ "(Note: this applies only to seed choices (not user submitted) and choices currently marked active)"
330	424
331	425	wins_by_choice_id = question.votes.active.count(:group => :choice_id)
332	426	losses_by_choice_id= question.votes.active.count(:conditions => "loser_choice_id IS NOT NULL", :group => :loser_choice_id)
...	...	@@ -339,6 +433,7 @@ namespace :test_api do
339	433	losses_hash.merge!(losses_by_choice_id)
340	434
341	435
	436	+
342	437	appearances_by_choice_id = wins_hash.merge(losses_hash) do \|key, oldval, newval\| oldval + newval end
343	438
344	439	sum = total_appearances = appearances_by_choice_id.values.inject(0) {\|sum, x\| sum +=x}
...	...	@@ -349,157 +444,141 @@ namespace :test_api do
349	444
350	445	appearances_by_choice_id.each do \|choice_id, n_i\|
351	446	if (n_i < (mean - 6stddev)) \|\| (n_i > mean + 6 stddev)
352		- error_msg += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question"
353		- error_bool = true
	447	+ error_message += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question"
354	448	end
355	449	end
356	450	end
357		-
	451	+
	452	+ return error_message.blank? ? [success_message, false] : [error_message, true]
	453	+ end
	454	+ def check_each_choice_equally_likely_to_appear_left_or_right(question)
	455	+ error_message = ""
	456	+ success_message = "All choices have equal probability of appearing on left or right (within error params)"
	457	+ question.choices.each do \|c\|
	458	+ left_prompts_ids = c.prompts_on_the_left.ids_only
	459	+ right_prompts_ids = c.prompts_on_the_right.ids_only
	460	+
	461	+ left_appearances = question.appearances.count(:conditions => {:prompt_id => left_prompts_ids})
	462	+ right_appearances = question.appearances.count(:conditions => {:prompt_id => right_prompts_ids})
	463	+
	464	+ n = left_appearances + right_appearances
	465	+
	466	+ if n == 0
	467	+ next
	468	+ end
	469	+ est_p = right_appearances.to_f / n.to_f
	470	+ z = (est_p - 0.5).abs / Math.sqrt((0.5 * 0.5) / n.to_f)
	471	+
	472	+ if z > 6
	473	+ error_message += "Error: Choice ID #{c.id} seems to favor one side: Left Appearances #{left_appearances}, Right Appearances: #{right_appearances}, z = #{z}\n"
	474	+ end
	475	+ end
	476	+ return error_message.blank? ? [success_message, false] : [error_message, true]
	477	+ end
	478	+ def check_prompt_cache_hit_rate(question)
	479	+ error_message = ""
	480	+ success_message = "At least 90% of prompts on catchup algorithm questions were served from cache\n"
	481	+
	482	+ misses = question.get_prompt_cache_misses(Date.yesterday).to_i
	483	+ hits = question.get_prompt_cache_hits(Date.yesterday).to_i
	484	+
	485	+ question.expire_prompt_cache_tracking_keys(Date.yesterday)
	486	+
	487	+ yesterday_votes = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
	488	+
	489	+ if misses + hits != yesterday_votes
	490	+ error_message += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_votes}, Actual: #{misses+hits}\n"
	491	+ end
	492	+
	493	+ miss_rate = misses.to_f / yesterday_votes.to_f
	494	+ if miss_rate > 0.1
	495	+ error_message += "Error! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}\n"
	496	+ end
	497	+ return error_message.blank? ? [success_message, false] : [error_message, true]
	498	+ end
	499	+
	500	+ def check_object_counter_cache_values_match_actual_values(question)
	501	+ error_message = ""
	502	+ success_message = "All cached object values match actual values within database"
358	503	# Checks that counter_cache is working as expected
359	504	cached_prompts_size = question.prompts.size
360	505	actual_prompts_size = question.prompts.count
361	506
362	507	if cached_prompts_size != actual_prompts_size
363		- error_msg += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n"
	508	+ error_message += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n"
364	509	end
365	510
366	511	cached_votes_size = question.votes.size
367	512	actual_votes_size = question.votes.count
368	513
369	514	if cached_votes_size != actual_votes_size
370		- error_msg += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n"
	515	+ error_message += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n"
371	516	end
372	517
373	518	cached_choices_size = question.choices.size
374	519	actual_choices_size = question.choices.count
375	520
376	521	if cached_choices_size != actual_choices_size
377		- error_msg += "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n"
	522	+ error_message+= "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n"
378	523	end
379	524
380	525	if cached_prompts_size != question.choices.size **2 - question.choices.size
381		- error_msg += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n"
	526	+ error_message += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n"
382	527	end
383		-
384		-
385		- #catchup specific
386		- if question.uses_catchup?
387		- misses = question.get_prompt_cache_misses(Date.yesterday).to_i
388		- hits = question.get_prompt_cache_hits(Date.yesterday).to_i
389		-
390		- question.expire_prompt_cache_tracking_keys(Date.yesterday)
391		-
392		-
393		-
394		- yesterday_votes = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
395		-
396		- if misses + hits != yesterday_votes
397		- error_msg += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_votes}, Actual: #{misses+hits}\n"
398		- end
399		-
400		- miss_rate = misses.to_f / yesterday_votes.to_f
401		- if miss_rate > 0.1
402		- error_msg += "Error! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}\n"
403		- end
404		- end
405		-
406		-
407		- if error_bool
408		- error_msg += "Question #{question.id}: 2wins = #{2total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
409		- end
410		-
411		- error_bool = false
412		- end
413		-
	528	+ return error_message.blank? ? [success_message, false] : [error_message, true]
	529	+ end
	530	+
	531	+ def ensure_all_votes_and_skips_have_unique_appearance
	532	+ error_message = ""
	533	+ success_message = "All vote and skip objects have an associated appearance object"
414	534	votes_without_appearances= Vote.count(:conditions => {:appearance_id => nil})
415	535	if (votes_without_appearances > 0)
416		- error_msg += "Error! There are #{votes_without_appearances} votes without associated appearance objects."
	536	+ error_message += "Error! There are #{votes_without_appearances} votes without associated appearance objects."
417	537	end
418	538
419	539	skips_without_appearances= Skip.count(:conditions => {:appearance_id => nil})
420	540	if (skips_without_appearances > 0)
421		- error_msg += "Error! There are #{skips_without_appearances} skips without associated appearance objects."
	541	+ error_message += "Error! There are #{skips_without_appearances} skips without associated appearance objects."
422	542	end
	543	+
	544	+ return error_message.blank? ? [success_message, false] : [error_message, true]
	545	+ end
423	546
	547	+ def response_time_tests
	548	+ error_message = ""
	549	+ success_message = "All Vote objects have an client response time < calculated server roundtrip time\n"
424	550
425	551	recording_client_time_start_date = Vote.find(:all, :conditions => 'time_viewed IS NOT NULL', :order => 'created_at', :limit => 1).first.created_at
426	552
427	553	Vote.find_each(:batch_size => 1000, :include => :appearance) do \|v\|
428	554
429		-
430	555	# Subtracting DateTime objects results in the difference in days
431	556	server_response_time = v.created_at.to_f - v.appearance.created_at.to_f
432	557	if server_response_time < 0
433	558	the_error_msg = "Error! Vote #{v.id} was created before the appearance associated with it: Appearance id: #{v.appearance.id}, Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}\n"
434	559
435		-
436		- error_msg += the_error_msg
437		- print the_error_msg
438		-
439		- print "Error!"
	560	+ error_message += the_error_msg
	561	+ print "Error!" + the_error_msg
440	562	end
441	563
442	564	if v.time_viewed && v.time_viewed/1000 > server_response_time
443	565	the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, has a longer client response time than is possible. Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
444	566
445		- error_msg += the_error_msg
446		- print the_error_msg
447		-
448		- bad_votes << v.id
	567	+ error_message += the_error_msg
	568	+ print the_error_msg
449	569
450	570	elsif v.time_viewed.nil?
451	571	if v.created_at > recording_client_time_start_date && v.missing_response_time_exp != 'invalid'
452	572	the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, does not have a client response, even though it should! Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
453		- error_msg += the_error_msg
	573	+ error_message += the_error_msg
454	574	print the_error_msg
455	575	end
456	576
457	577	end
458	578
459		-
460		- end
461		-
462		- if error_msg.blank?
463		-
464		- success_msg = "Conducted the following tests on API data and found no inconsistencies.\n" +
465		- "For each of the #{questions.length} questions in the database: \n" +
466		- " 2 x Total Wins = Total Votes\n" +
467		- " Total Votes (wins + losses) is Even\n" +
468		- " Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +
469		- " Total generated prompts on left = Total generated prompts on right\n" +
470		- " Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question\n" +
471		- " Note: this applies only to seed choices (not user submitted) and choices currently marked active\n" +
472		- " The cached score value matches the calculated score value for each choice\n" +
473		- " The cached vote count matches the actual number of votes for each question\n" +
474		- " The cached choices count matches the actual number of choices for each question\n" +
475		- " The cached prompt count matches the actual number of prompts for each question\n" +
476		- " The prompt count matches the expected number of prompts ( num_choices ^2 - num choices) for each question\n" +
477		- " All Vote objects have an associated appearance object\n" +
478		- " All Vote objects have an client response time < calculated server roundtrip time\n"
479		- " More than 90% of prompts on catchup algorithm questions were served from cache\n"
480		-
481		- print success_msg
482		-
483		- CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", success_msg)
484		- else
485		- CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg)
486		-
487		- puts "There were errors: "
488		- puts error_msg
489		-
490		- unless bad_choices.blank?
491		-
492		- puts "Here's a list of choice ids that you may want to modify: #{bad_choices.uniq.inspect}"
493		-
494		- end
495		- unless bad_votes.blank?
496		-
497		- puts "Here's a list of vote ids that you may want to modify: #{bad_votes.uniq.inspect}"
498		-
499		- end
500		- print error_msg
	579	+ end
	580	+
	581	+ return error_message.blank? ? [success_message, false] : [error_message, true]
501	582	end
502		-
503		- end
504	583	end
505	584
...	...