Additional api data consistency checks

Dhruv Kapadia
1 parent 93444bab
Showing 1 changed file with 89 additions and 3 deletions Show diff stats
lib/tasks/test_api.rake
@@ -117,7 +117,6 @@ namespace :test_api do
    end
-   task(:generate_historical_density_data)
    desc "Should only need to be run once"
    task(:generate_all_possible_prompts => :environment) do
@@ -219,12 +218,15 @@ namespace :test_api do
       error_msg = ""
+      bad_choices = []
       questions.each do |question|
 	total_wins =0
 	total_votes =0
 	total_generated_prompts_on_left = 0
 	total_generated_prompts_on_right = 0
+	total_scores_gte_fifty= 0
+	total_scores_lte_fifty= 0
 	error_bool = false
 	question.choices.each do |choice|
@@ -239,6 +241,32 @@ namespace :test_api do
 	    total_generated_prompts_on_left += choice.prompts_on_the_left.size
 	    total_generated_prompts_on_right += choice.prompts_on_the_right.size
+
+	    cached_score = choice.score
+	    generated_score = choice.compute_score
+
+	    if cached_score.round != generated_score.round
+		 error_msg += "Error! The cached_score is not equal to the calculated score for choice #{choice.id}"
+
+		 print "This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
+
+		 bad_choices << choice.id
+
+	    end
+
+	    if cached_score == 0.0 || cached_score == 100.0 || cached_score.nil?
+		 error_msg += "Error! The cached_score for choice #{choice.id} is exactly 0 or 100, the value: #{cached_score}"
+		 print "Either 0 or 100 This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
+		 bad_choices << choice.id
+	    end
+
+	    if cached_score >= 50
+		    total_scores_gte_fifty +=1
+	    end
+	    if cached_score <= 50
+		    total_scores_lte_fifty +=1
+	    end
+
         end
 	if (2*total_wins != total_votes)
@@ -261,6 +289,13 @@ namespace :test_api do
 		error_bool = true
 	end
+	if(total_scores_lte_fifty == question.choices.size || total_scores_gte_fifty == question.choices.size) && (total_scores_lte_fifty != total_scores_gte_fifty)
+		error_msg += "Error: The scores of all choices are either all above 50, or all below 50. This is probably wrong"
+		error_bool = true
+		puts "Error score fifty: #{question.id}"
+	end
+
+
 	wins_by_choice_id = question.votes.active.count(:group => :choice_id)
 	losses_by_choice_id= question.votes.active.count(:conditions => "loser_choice_id IS NOT NULL", :group => :loser_choice_id)
@@ -293,9 +328,50 @@ namespace :test_api do
 	if error_bool
 	   error_msg += "Question #{question.id}: 2*wins = #{2*total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
 	end
+
 	error_bool = false
      end
-     
+
+     votes_without_appearances= Vote.count(:conditions => {:appearance_id => nil})
+     if (votes_without_appearances > 0)
+	     error_msg += "Error! There are #{votes_without_appearances} votes without associated appearance objects."
+     end
+
+     recording_client_time_start_date = Vote.find(:all, :conditions => 'time_viewed IS NOT NULL', :order => 'created_at', :limit => 1).first.created_at
+
+     Vote.find_each(:batch_size => 1000, :include => :appearance) do |v|
+
+
+	     # Subtracting DateTime objects results in the difference in days
+	     server_response_time = v.created_at.to_f - v.appearance.created_at.to_f
+	     if server_response_time < 0
+	        the_error_msg = "Error! Vote #{v.id} was created before the appearance associated with it: Appearance id: #{v.appearance.id}, Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}\n"
+
+
+		error_msg += the_error_msg
+	        print the_error_msg
+
+		print "Error!"
+	     end
+
+	     if v.time_viewed && v.time_viewed/1000 > server_response_time 
+		     the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, has a longer client response time than is possible. Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
+
+		     error_msg += the_error_msg
+		     print the_error_msg
+
+             elsif v.time_viewed.nil?
+		     if v.created_at > recording_client_time_start_date 
+	                the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, does not have a client response, even though it should! Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
+			error_msg += the_error_msg
+	                print the_error_msg
+		     end
+
+	     end
+
+
+     end
+            
      if error_msg.blank?
 	success_msg = "Conducted the following tests on API data and found no inconsistencies.\n" +
@@ -305,13 +381,23 @@ namespace :test_api do
 			 "     Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +
 		         "     Total generated prompts on left = Total generated prompts on right\n" + 
 		         "     Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question\n" +
-			 "             Note: this applies only to seed choices (not user submitted) and choices currently marked active\n"
+			 "             Note: this applies only to seed choices (not user submitted) and choices currently marked active\n" + 
+			 "     The cached score value matches the calculated score value for each choice\n" +
+			 "     All Vote objects have an associated appearance object\n" +
+			 "     All Vote objects have an client response time < calculated server roundtrip time\n" 
 	print success_msg
 	CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", success_msg)
      else
      	CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg)
+
+	unless bad_choices.blank?
+
+		puts "Here's a list of choice ids that you may want to modify: #{bad_choices.uniq.inspect}"
+
+	end
+	print error_msg
      end
    end
	@@ -117,7 +117,6 @@ namespace :test_api do		@@ -117,7 +117,6 @@ namespace :test_api do
117		117
118	end	118	end
119		119
120	- task(:generate_historical_density_data)
121		120
122	desc "Should only need to be run once"	121	desc "Should only need to be run once"
123	task(:generate_all_possible_prompts => :environment) do	122	task(:generate_all_possible_prompts => :environment) do
	@@ -219,12 +218,15 @@ namespace :test_api do		@@ -219,12 +218,15 @@ namespace :test_api do
219		218
220	error_msg = ""	219	error_msg = ""
221		220
		221	+ bad_choices = []
222	questions.each do \|question\|	222	questions.each do \|question\|
223		223
224	total_wins =0	224	total_wins =0
225	total_votes =0	225	total_votes =0
226	total_generated_prompts_on_left = 0	226	total_generated_prompts_on_left = 0
227	total_generated_prompts_on_right = 0	227	total_generated_prompts_on_right = 0
		228	+ total_scores_gte_fifty= 0
		229	+ total_scores_lte_fifty= 0
228	error_bool = false	230	error_bool = false
229	question.choices.each do \|choice\|	231	question.choices.each do \|choice\|
230		232
	@@ -239,6 +241,32 @@ namespace :test_api do		@@ -239,6 +241,32 @@ namespace :test_api do
239		241
240	total_generated_prompts_on_left += choice.prompts_on_the_left.size	242	total_generated_prompts_on_left += choice.prompts_on_the_left.size
241	total_generated_prompts_on_right += choice.prompts_on_the_right.size	243	total_generated_prompts_on_right += choice.prompts_on_the_right.size
		244	+
		245	+ cached_score = choice.score
		246	+ generated_score = choice.compute_score
		247	+
		248	+ if cached_score.round != generated_score.round
		249	+ error_msg += "Error! The cached_score is not equal to the calculated score for choice #{choice.id}"
		250	+
		251	+ print "This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
		252	+
		253	+ bad_choices << choice.id
		254	+
		255	+ end
		256	+
		257	+ if cached_score == 0.0 \|\| cached_score == 100.0 \|\| cached_score.nil?
		258	+ error_msg += "Error! The cached_score for choice #{choice.id} is exactly 0 or 100, the value: #{cached_score}"
		259	+ print "Either 0 or 100 This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
		260	+ bad_choices << choice.id
		261	+ end
		262	+
		263	+ if cached_score >= 50
		264	+ total_scores_gte_fifty +=1
		265	+ end
		266	+ if cached_score <= 50
		267	+ total_scores_lte_fifty +=1
		268	+ end
		269	+
242	end	270	end
243		271
244	if (2*total_wins != total_votes)	272	if (2*total_wins != total_votes)
	@@ -261,6 +289,13 @@ namespace :test_api do		@@ -261,6 +289,13 @@ namespace :test_api do
261	error_bool = true	289	error_bool = true
262	end	290	end
263		291
		292	+ if(total_scores_lte_fifty == question.choices.size \|\| total_scores_gte_fifty == question.choices.size) && (total_scores_lte_fifty != total_scores_gte_fifty)
		293	+ error_msg += "Error: The scores of all choices are either all above 50, or all below 50. This is probably wrong"
		294	+ error_bool = true
		295	+ puts "Error score fifty: #{question.id}"
		296	+ end
		297	+
		298	+
264	wins_by_choice_id = question.votes.active.count(:group => :choice_id)	299	wins_by_choice_id = question.votes.active.count(:group => :choice_id)
265	losses_by_choice_id= question.votes.active.count(:conditions => "loser_choice_id IS NOT NULL", :group => :loser_choice_id)	300	losses_by_choice_id= question.votes.active.count(:conditions => "loser_choice_id IS NOT NULL", :group => :loser_choice_id)
266		301
	@@ -293,9 +328,50 @@ namespace :test_api do		@@ -293,9 +328,50 @@ namespace :test_api do
293	if error_bool	328	if error_bool
294	error_msg += "Question #{question.id}: 2wins = #{2total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"	329	error_msg += "Question #{question.id}: 2wins = #{2total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
295	end	330	end
		331	+
296	error_bool = false	332	error_bool = false
297	end	333	end
298	-	334	+
		335	+ votes_without_appearances= Vote.count(:conditions => {:appearance_id => nil})
		336	+ if (votes_without_appearances > 0)
		337	+ error_msg += "Error! There are #{votes_without_appearances} votes without associated appearance objects."
		338	+ end
		339	+
		340	+ recording_client_time_start_date = Vote.find(:all, :conditions => 'time_viewed IS NOT NULL', :order => 'created_at', :limit => 1).first.created_at
		341	+
		342	+ Vote.find_each(:batch_size => 1000, :include => :appearance) do \|v\|
		343	+
		344	+
		345	+ # Subtracting DateTime objects results in the difference in days
		346	+ server_response_time = v.created_at.to_f - v.appearance.created_at.to_f
		347	+ if server_response_time < 0
		348	+ the_error_msg = "Error! Vote #{v.id} was created before the appearance associated with it: Appearance id: #{v.appearance.id}, Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}\n"
		349	+
		350	+
		351	+ error_msg += the_error_msg
		352	+ print the_error_msg
		353	+
		354	+ print "Error!"
		355	+ end
		356	+
		357	+ if v.time_viewed && v.time_viewed/1000 > server_response_time
		358	+ the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, has a longer client response time than is possible. Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
		359	+
		360	+ error_msg += the_error_msg
		361	+ print the_error_msg
		362	+
		363	+ elsif v.time_viewed.nil?
		364	+ if v.created_at > recording_client_time_start_date
		365	+ the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, does not have a client response, even though it should! Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
		366	+ error_msg += the_error_msg
		367	+ print the_error_msg
		368	+ end
		369	+
		370	+ end
		371	+
		372	+
		373	+ end
		374	+
299	if error_msg.blank?	375	if error_msg.blank?
300		376
301	success_msg = "Conducted the following tests on API data and found no inconsistencies.\n" +	377	success_msg = "Conducted the following tests on API data and found no inconsistencies.\n" +
	@@ -305,13 +381,23 @@ namespace :test_api do		@@ -305,13 +381,23 @@ namespace :test_api do
305	" Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +	381	" Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +
306	" Total generated prompts on left = Total generated prompts on right\n" +	382	" Total generated prompts on left = Total generated prompts on right\n" +
307	" Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question\n" +	383	" Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question\n" +
308	- " Note: this applies only to seed choices (not user submitted) and choices currently marked active\n"	384	+ " Note: this applies only to seed choices (not user submitted) and choices currently marked active\n" +
		385	+ " The cached score value matches the calculated score value for each choice\n" +
		386	+ " All Vote objects have an associated appearance object\n" +
		387	+ " All Vote objects have an client response time < calculated server roundtrip time\n"
309		388
310	print success_msg	389	print success_msg
311		390
312	CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", success_msg)	391	CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", success_msg)
313	else	392	else
314	CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg)	393	CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg)
		394	+
		395	+ unless bad_choices.blank?
		396	+
		397	+ puts "Here's a list of choice ids that you may want to modify: #{bad_choices.uniq.inspect}"
		398	+
		399	+ end
		400	+ print error_msg
315	end	401	end
316		402
317	end	403	end