Additional api data consistency checks

Dhruv Kapadia
1 parent 93444bab
Showing 1 changed file with 89 additions and 3 deletions Show diff stats
lib/tasks/test_api.rake
@@ -117,7 +117,6 @@ namespace :test_api do
  
    end
  
-   task(:generate_historical_density_data)
  
    desc "Should only need to be run once"
    task(:generate_all_possible_prompts => :environment) do
@@ -219,12 +218,15 @@ namespace :test_api do
  
       error_msg = ""
  
+      bad_choices = []
       questions.each do |question|
  
 	total_wins =0
 	total_votes =0
 	total_generated_prompts_on_left = 0
 	total_generated_prompts_on_right = 0
+	total_scores_gte_fifty= 0
+	total_scores_lte_fifty= 0
 	error_bool = false
 	question.choices.each do |choice|
  
@@ -239,6 +241,32 @@ namespace :test_api do
  
 	    total_generated_prompts_on_left += choice.prompts_on_the_left.size
 	    total_generated_prompts_on_right += choice.prompts_on_the_right.size
+
+	    cached_score = choice.score
+	    generated_score = choice.compute_score
+
+	    if cached_score.round != generated_score.round
+		 error_msg += "Error! The cached_score is not equal to the calculated score for choice #{choice.id}"
+
+		 print "This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
+
+		 bad_choices << choice.id
+
+	    end
+
+	    if cached_score == 0.0 || cached_score == 100.0 || cached_score.nil?
+		 error_msg += "Error! The cached_score for choice #{choice.id} is exactly 0 or 100, the value: #{cached_score}"
+		 print "Either 0 or 100 This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
+		 bad_choices << choice.id
+	    end
+
+	    if cached_score >= 50
+		    total_scores_gte_fifty +=1
+	    end
+	    if cached_score <= 50
+		    total_scores_lte_fifty +=1
+	    end
+
         end
  
 	if (2*total_wins != total_votes)
@@ -261,6 +289,13 @@ namespace :test_api do
 		error_bool = true
 	end
  
+	if(total_scores_lte_fifty == question.choices.size || total_scores_gte_fifty == question.choices.size) && (total_scores_lte_fifty != total_scores_gte_fifty)
+		error_msg += "Error: The scores of all choices are either all above 50, or all below 50. This is probably wrong"
+		error_bool = true
+		puts "Error score fifty: #{question.id}"
+	end
+
+
 	wins_by_choice_id = question.votes.active.count(:group => :choice_id)
 	losses_by_choice_id= question.votes.active.count(:conditions => "loser_choice_id IS NOT NULL", :group => :loser_choice_id)
  
@@ -293,9 +328,50 @@ namespace :test_api do
 	if error_bool
 	   error_msg += "Question #{question.id}: 2*wins = #{2*total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
 	end
+
 	error_bool = false
      end
-     
+
+     votes_without_appearances= Vote.count(:conditions => {:appearance_id => nil})
+     if (votes_without_appearances > 0)
+	     error_msg += "Error! There are #{votes_without_appearances} votes without associated appearance objects."
+     end
+
+     recording_client_time_start_date = Vote.find(:all, :conditions => 'time_viewed IS NOT NULL', :order => 'created_at', :limit => 1).first.created_at
+
+     Vote.find_each(:batch_size => 1000, :include => :appearance) do |v|
+
+
+	     # Subtracting DateTime objects results in the difference in days
+	     server_response_time = v.created_at.to_f - v.appearance.created_at.to_f
+	     if server_response_time < 0
+	        the_error_msg = "Error! Vote #{v.id} was created before the appearance associated with it: Appearance id: #{v.appearance.id}, Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}\n"
+
+
+		error_msg += the_error_msg
+	        print the_error_msg
+
+		print "Error!"
+	     end
+
+	     if v.time_viewed && v.time_viewed/1000 > server_response_time 
+		     the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, has a longer client response time than is possible. Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
+
+		     error_msg += the_error_msg
+		     print the_error_msg
+
+             elsif v.time_viewed.nil?
+		     if v.created_at > recording_client_time_start_date 
+	                the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, does not have a client response, even though it should! Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
+			error_msg += the_error_msg
+	                print the_error_msg
+		     end
+
+	     end
+
+
+     end
+            
      if error_msg.blank?
  
 	success_msg = "Conducted the following tests on API data and found no inconsistencies.\n" +
@@ -305,13 +381,23 @@ namespace :test_api do
 			 "     Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +
 		         "     Total generated prompts on left = Total generated prompts on right\n" + 
 		         "     Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question\n" +
-			 "             Note: this applies only to seed choices (not user submitted) and choices currently marked active\n"
+			 "             Note: this applies only to seed choices (not user submitted) and choices currently marked active\n" + 
+			 "     The cached score value matches the calculated score value for each choice\n" +
+			 "     All Vote objects have an associated appearance object\n" +
+			 "     All Vote objects have an client response time < calculated server roundtrip time\n" 
  
 	print success_msg
  
 	CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", success_msg)
      else
      	CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg)
+
+	unless bad_choices.blank?
+
+		puts "Here's a list of choice ids that you may want to modify: #{bad_choices.uniq.inspect}"
+
+	end
+	print error_msg
      end
  
    end
...	...	@@ -117,7 +117,6 @@ namespace :test_api do
117	117
118	118	end
119	119
120		- task(:generate_historical_density_data)
121	120
122	121	desc "Should only need to be run once"
123	122	task(:generate_all_possible_prompts => :environment) do
...	...	@@ -219,12 +218,15 @@ namespace :test_api do
219	218
220	219	error_msg = ""
221	220
	221	+ bad_choices = []
222	222	questions.each do \|question\|
223	223
224	224	total_wins =0
225	225	total_votes =0
226	226	total_generated_prompts_on_left = 0
227	227	total_generated_prompts_on_right = 0
	228	+ total_scores_gte_fifty= 0
	229	+ total_scores_lte_fifty= 0
228	230	error_bool = false
229	231	question.choices.each do \|choice\|
230	232
...	...	@@ -239,6 +241,32 @@ namespace :test_api do
239	241
240	242	total_generated_prompts_on_left += choice.prompts_on_the_left.size
241	243	total_generated_prompts_on_right += choice.prompts_on_the_right.size
	244	+
	245	+ cached_score = choice.score
	246	+ generated_score = choice.compute_score
	247	+
	248	+ if cached_score.round != generated_score.round
	249	+ error_msg += "Error! The cached_score is not equal to the calculated score for choice #{choice.id}"
	250	+
	251	+ print "This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
	252	+
	253	+ bad_choices << choice.id
	254	+
	255	+ end
	256	+
	257	+ if cached_score == 0.0 \|\| cached_score == 100.0 \|\| cached_score.nil?
	258	+ error_msg += "Error! The cached_score for choice #{choice.id} is exactly 0 or 100, the value: #{cached_score}"
	259	+ print "Either 0 or 100 This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
	260	+ bad_choices << choice.id
	261	+ end
	262	+
	263	+ if cached_score >= 50
	264	+ total_scores_gte_fifty +=1
	265	+ end
	266	+ if cached_score <= 50
	267	+ total_scores_lte_fifty +=1
	268	+ end
	269	+
242	270	end
243	271
244	272	if (2*total_wins != total_votes)
...	...	@@ -261,6 +289,13 @@ namespace :test_api do
261	289	error_bool = true
262	290	end
263	291
	292	+ if(total_scores_lte_fifty == question.choices.size \|\| total_scores_gte_fifty == question.choices.size) && (total_scores_lte_fifty != total_scores_gte_fifty)
	293	+ error_msg += "Error: The scores of all choices are either all above 50, or all below 50. This is probably wrong"
	294	+ error_bool = true
	295	+ puts "Error score fifty: #{question.id}"
	296	+ end
	297	+
	298	+
264	299	wins_by_choice_id = question.votes.active.count(:group => :choice_id)
265	300	losses_by_choice_id= question.votes.active.count(:conditions => "loser_choice_id IS NOT NULL", :group => :loser_choice_id)
266	301
...	...	@@ -293,9 +328,50 @@ namespace :test_api do
293	328	if error_bool
294	329	error_msg += "Question #{question.id}: 2wins = #{2total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
295	330	end
	331	+
296	332	error_bool = false
297	333	end
298		-
	334	+
	335	+ votes_without_appearances= Vote.count(:conditions => {:appearance_id => nil})
	336	+ if (votes_without_appearances > 0)
	337	+ error_msg += "Error! There are #{votes_without_appearances} votes without associated appearance objects."
	338	+ end
	339	+
	340	+ recording_client_time_start_date = Vote.find(:all, :conditions => 'time_viewed IS NOT NULL', :order => 'created_at', :limit => 1).first.created_at
	341	+
	342	+ Vote.find_each(:batch_size => 1000, :include => :appearance) do \|v\|
	343	+
	344	+
	345	+ # Subtracting DateTime objects results in the difference in days
	346	+ server_response_time = v.created_at.to_f - v.appearance.created_at.to_f
	347	+ if server_response_time < 0
	348	+ the_error_msg = "Error! Vote #{v.id} was created before the appearance associated with it: Appearance id: #{v.appearance.id}, Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}\n"
	349	+
	350	+
	351	+ error_msg += the_error_msg
	352	+ print the_error_msg
	353	+
	354	+ print "Error!"
	355	+ end
	356	+
	357	+ if v.time_viewed && v.time_viewed/1000 > server_response_time
	358	+ the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, has a longer client response time than is possible. Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
	359	+
	360	+ error_msg += the_error_msg
	361	+ print the_error_msg
	362	+
	363	+ elsif v.time_viewed.nil?
	364	+ if v.created_at > recording_client_time_start_date
	365	+ the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, does not have a client response, even though it should! Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
	366	+ error_msg += the_error_msg
	367	+ print the_error_msg
	368	+ end
	369	+
	370	+ end
	371	+
	372	+
	373	+ end
	374	+
299	375	if error_msg.blank?
300	376
301	377	success_msg = "Conducted the following tests on API data and found no inconsistencies.\n" +
...	...	@@ -305,13 +381,23 @@ namespace :test_api do
305	381	" Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +
306	382	" Total generated prompts on left = Total generated prompts on right\n" +
307	383	" Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question\n" +
308		- " Note: this applies only to seed choices (not user submitted) and choices currently marked active\n"
	384	+ " Note: this applies only to seed choices (not user submitted) and choices currently marked active\n" +
	385	+ " The cached score value matches the calculated score value for each choice\n" +
	386	+ " All Vote objects have an associated appearance object\n" +
	387	+ " All Vote objects have an client response time < calculated server roundtrip time\n"
309	388
310	389	print success_msg
311	390
312	391	CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", success_msg)
313	392	else
314	393	CronMailer.deliver_info_message("#{CRON_EMAIL},#{ERRORS_EMAIL}", "Error! Failure of API Vote Consistency " , error_msg)
	394	+
	395	+ unless bad_choices.blank?
	396	+
	397	+ puts "Here's a list of choice ids that you may want to modify: #{bad_choices.uniq.inspect}"
	398	+
	399	+ end
	400	+ print error_msg
315	401	end
316	402
317	403	end
...	...