Download as
Browse Code »

Commit 8716cedc034f94a2771478b1e49a3c269af636d6

Authored by Luke Baker
1 parent 3bd8a171
Exists in master and in 1 other branch production-vendorized

update spacing in test_api.rake

Showing 1 changed file with 447 additions and 445 deletions   Show diff stats
lib/tasks/test_api.rake
  Diff comments   View file @8716ced
... ... @@ -5,120 +5,120 @@ namespace :test_api do
5 5 desc "Don't run unless you know what you are doing"
6 6 task(:generate_lots_of_votes => :environment) do
7 7 if Rails.env.production?
8   - print "You probably don't want to run this in production as it will falsify a bunch of random votes"
  8 + print "You probably don't want to run this in production as it will falsify a bunch of random votes"
9 9 end
10 10  
11 11  
12 12 current_user = User.first
13 13 1000.times do |n|
14   - puts "#{n} votes completed" if n % 100 == 0
  14 + puts "#{n} votes completed" if n % 100 == 0
15 15 question = Question.find(214) # test question change as needed
16   - @prompt = question.catchup_choose_prompt
  16 + @prompt = question.catchup_choose_prompt
17 17 @appearance = current_user.record_appearance(current_user.default_visitor, @prompt)
18 18  
19   - direction = (rand(2) == 0) ? "left" : "right"
20   - current_user.record_vote(:prompt => @prompt, :direction => direction, :appearance_lookup => @appearance.lookup)
  19 + direction = (rand(2) == 0) ? "left" : "right"
  20 + current_user.record_vote(:prompt => @prompt, :direction => direction, :appearance_lookup => @appearance.lookup)
21 21 end
22 22  
23 23 end
24 24  
25 25 desc "Generate appearances for any votes that have no current appearance, should only need to be run once"
26 26 task(:generate_appearances_for_existing_votes => :environment) do
27   - votes = Vote.all
  27 + votes = Vote.all
28 28  
29   - count = 0
30   - votes.each do |v|
31   - if v.appearance.nil?
32   - print "."
33   - a = Appearance.create(:voter_id => v.voter_id, :site_id => v.site_id, :prompt_id => v.prompt_id, :question_id => v.question_id, :created_at => v.created_at, :updated_at => v.updated_at)
34   - v.appearance = a
35   - v.save
  29 + count = 0
  30 + votes.each do |v|
  31 + if v.appearance.nil?
  32 + print "."
  33 + a = Appearance.create(:voter_id => v.voter_id, :site_id => v.site_id, :prompt_id => v.prompt_id, :question_id => v.question_id, :created_at => v.created_at, :updated_at => v.updated_at)
  34 + v.appearance = a
  35 + v.save
36 36  
37   - count += 1
38   - end
39   - end
  37 + count += 1
  38 + end
  39 + end
40 40  
41   - print count
  41 + print count
42 42 end
43 43  
44 44  
45 45 desc "Generate past density information"
46 46 task(:generate_past_densities => :environment) do
47   - #this is not elegant, but should only be run once, so quick and dirty wins
48   -
49   - start_date = Vote.find(:all, :conditions => 'loser_choice_id IS NOT NULL', :order => :created_at, :limit => 1).first.created_at.to_date
50   - start_date.upto(Date.today) do |the_date|
51   - questions = Question.find(:all, :conditions => ['created_at < ?', the_date])
52   -
53   - print the_date.to_s
54   - questions.each do |q|
55   - puts q.id
56   - relevant_choices = q.choices.find(:all, :conditions => ['created_at < ?', the_date])
57   -
58   - seed_choices = 0
59   -
60   - if relevant_choices == 0
61   - next
62   - #this question had not been created yet
63   - end
64   -
65   - relevant_choices.each do |c|
66   - if !c.user_created
67   - seed_choices+=1
68   - end
69   -
70   - end
71   -
72   - nonseed_choices = relevant_choices.size - seed_choices
73   -
74   - seed_seed_total = seed_choices **2 - seed_choices
75   - nonseed_nonseed_total = nonseed_choices **2 - nonseed_choices
76   - seed_nonseed_total = seed_choices * nonseed_choices
77   - nonseed_seed_total = seed_choices * nonseed_choices
78   -
79   - seed_seed_sum = 0
80   - seed_nonseed_sum= 0
81   - nonseed_seed_sum= 0
82   - nonseed_nonseed_sum= 0
83   -
84   - q.appearances.find_each(:conditions => ['prompt_id IS NOT NULL AND created_at < ?', the_date]) do |a|
85   -
86   - p = a.prompt
87   - if p.left_choice.user_created == false && p.right_choice.user_created == false
88   - seed_seed_sum += 1
89   - elsif p.left_choice.user_created == false && p.right_choice.user_created == true
90   - seed_nonseed_sum += 1
91   - elsif p.left_choice.user_created == true && p.right_choice.user_created == false
92   - nonseed_seed_sum += 1
93   - elsif p.left_choice.user_created == true && p.right_choice.user_created == true
94   - nonseed_nonseed_sum += 1
95   - end
96   - end
97   -
98   - densities = {}
99   - densities[:seed_seed] = seed_seed_sum.to_f / seed_seed_total.to_f
100   - densities[:seed_nonseed] = seed_nonseed_sum.to_f / seed_nonseed_total.to_f
101   - densities[:nonseed_seed] = nonseed_seed_sum.to_f / nonseed_seed_total.to_f
102   - densities[:nonseed_nonseed] = nonseed_nonseed_sum.to_f / nonseed_nonseed_total.to_f
103   -
104   - densities.each do |type, average|
105   - d = Density.new
106   - d.created_at = the_date
107   - d.question_id = q.id
108   - d.prompt_type = type.to_s
109   - d.value = average.nan? ? nil : average
110   - d.save!
111   - end
112   -
113   - puts "Seed_seed sum: #{seed_seed_sum}, seed_seed total num: #{seed_seed_total}"
114   - puts "Seed_nonseed sum: #{seed_nonseed_sum}, seed_nonseed total num: #{seed_nonseed_total}"
115   - puts "Nonseed_seed sum: #{nonseed_seed_sum}, nonseed_seed total num: #{nonseed_seed_total}"
116   - puts "Nonseed_nonseed sum: #{nonseed_nonseed_sum}, nonseed_nonseed total num: #{nonseed_nonseed_total}"
117   -
118   -
119   - end
120   -
121   - end
  47 + #this is not elegant, but should only be run once, so quick and dirty wins
  48 +
  49 + start_date = Vote.find(:all, :conditions => 'loser_choice_id IS NOT NULL', :order => :created_at, :limit => 1).first.created_at.to_date
  50 + start_date.upto(Date.today) do |the_date|
  51 + questions = Question.find(:all, :conditions => ['created_at < ?', the_date])
  52 +
  53 + print the_date.to_s
  54 + questions.each do |q|
  55 + puts q.id
  56 + relevant_choices = q.choices.find(:all, :conditions => ['created_at < ?', the_date])
  57 +
  58 + seed_choices = 0
  59 +
  60 + if relevant_choices == 0
  61 + next
  62 + #this question had not been created yet
  63 + end
  64 +
  65 + relevant_choices.each do |c|
  66 + if !c.user_created
  67 + seed_choices+=1
  68 + end
  69 +
  70 + end
  71 +
  72 + nonseed_choices = relevant_choices.size - seed_choices
  73 +
  74 + seed_seed_total = seed_choices **2 - seed_choices
  75 + nonseed_nonseed_total = nonseed_choices **2 - nonseed_choices
  76 + seed_nonseed_total = seed_choices * nonseed_choices
  77 + nonseed_seed_total = seed_choices * nonseed_choices
  78 +
  79 + seed_seed_sum = 0
  80 + seed_nonseed_sum= 0
  81 + nonseed_seed_sum= 0
  82 + nonseed_nonseed_sum= 0
  83 +
  84 + q.appearances.find_each(:conditions => ['prompt_id IS NOT NULL AND created_at < ?', the_date]) do |a|
  85 +
  86 + p = a.prompt
  87 + if p.left_choice.user_created == false && p.right_choice.user_created == false
  88 + seed_seed_sum += 1
  89 + elsif p.left_choice.user_created == false && p.right_choice.user_created == true
  90 + seed_nonseed_sum += 1
  91 + elsif p.left_choice.user_created == true && p.right_choice.user_created == false
  92 + nonseed_seed_sum += 1
  93 + elsif p.left_choice.user_created == true && p.right_choice.user_created == true
  94 + nonseed_nonseed_sum += 1
  95 + end
  96 + end
  97 +
  98 + densities = {}
  99 + densities[:seed_seed] = seed_seed_sum.to_f / seed_seed_total.to_f
  100 + densities[:seed_nonseed] = seed_nonseed_sum.to_f / seed_nonseed_total.to_f
  101 + densities[:nonseed_seed] = nonseed_seed_sum.to_f / nonseed_seed_total.to_f
  102 + densities[:nonseed_nonseed] = nonseed_nonseed_sum.to_f / nonseed_nonseed_total.to_f
  103 +
  104 + densities.each do |type, average|
  105 + d = Density.new
  106 + d.created_at = the_date
  107 + d.question_id = q.id
  108 + d.prompt_type = type.to_s
  109 + d.value = average.nan? ? nil : average
  110 + d.save!
  111 + end
  112 +
  113 + puts "Seed_seed sum: #{seed_seed_sum}, seed_seed total num: #{seed_seed_total}"
  114 + puts "Seed_nonseed sum: #{seed_nonseed_sum}, seed_nonseed total num: #{seed_nonseed_total}"
  115 + puts "Nonseed_seed sum: #{nonseed_seed_sum}, nonseed_seed total num: #{nonseed_seed_total}"
  116 + puts "Nonseed_nonseed sum: #{nonseed_nonseed_sum}, nonseed_nonseed total num: #{nonseed_nonseed_total}"
  117 +
  118 +
  119 + end
  120 +
  121 + end
122 122  
123 123 end
124 124  
... ... @@ -126,53 +126,53 @@ namespace :test_api do
126 126 desc "Should only need to be run once"
127 127 task(:generate_all_possible_prompts => :environment) do
128 128 Question.find(:all).each do |q|
129   - choices = q.choices
130   - if q.prompts.size > choices.size**2 - choices.size
131   - print "ERROR: #{q.id}\n"
132   - next
133   - elsif q.prompts.size == choices.size**2 - choices.size
134   - print "#{q.id} has enough prompts, skipping...\n"
135   - next
136   - else
137   - print "#{q.id} should add #{(choices.size ** 2 - choices.size) - q.prompts.size}\n"
138   -
139   - end
  129 + choices = q.choices
  130 + if q.prompts.size > choices.size**2 - choices.size
  131 + print "ERROR: #{q.id}\n"
  132 + next
  133 + elsif q.prompts.size == choices.size**2 - choices.size
  134 + print "#{q.id} has enough prompts, skipping...\n"
  135 + next
  136 + else
  137 + print "#{q.id} should add #{(choices.size ** 2 - choices.size) - q.prompts.size}\n"
  138 +
  139 + end
140 140 created_timestring = q.created_at.to_s(:db)
141   - updated_timestring = Time.now.to_s(:db) #isn't rails awesome?
142   - promptscount=0
  141 + updated_timestring = Time.now.to_s(:db) #isn't rails awesome?
  142 + promptscount=0
143 143 inserts = []
144   - the_prompts = Prompt.find(:all, :select => 'id, left_choice_id, right_choice_id', :conditions => {:question_id => q.id})
  144 + the_prompts = Prompt.find(:all, :select => 'id, left_choice_id, right_choice_id', :conditions => {:question_id => q.id})
145 145  
146   - the_prompts_hash = {}
147   - the_prompts.each do |p|
148   - the_prompts_hash["#{p.left_choice_id},#{p.right_choice_id}"] = 1
149   - end
  146 + the_prompts_hash = {}
  147 + the_prompts.each do |p|
  148 + the_prompts_hash["#{p.left_choice_id},#{p.right_choice_id}"] = 1
  149 + end
150 150  
151 151 choices.each do |l|
152   - choices.each do |r|
153   - if l.id == r.id
154   - next
155   - else
156   - #p = the_prompts.find{|o| o.left_choice_id == l.id && o.right_choice_id == r.id}
157   - keystring = "#{l.id},#{r.id}"
158   - p = the_prompts_hash[keystring]
159   - if p.nil?
160   - inserts.push("(NULL, #{q.id}, NULL, #{l.id}, '#{created_timestring}', '#{updated_timestring}', NULL, 0, #{r.id}, NULL, NULL)")
161   - promptscount+=1
162   - end
163   -
164   - end
165   -
166   - end
  152 + choices.each do |r|
  153 + if l.id == r.id
  154 + next
  155 + else
  156 + #p = the_prompts.find{|o| o.left_choice_id == l.id && o.right_choice_id == r.id}
  157 + keystring = "#{l.id},#{r.id}"
  158 + p = the_prompts_hash[keystring]
  159 + if p.nil?
  160 + inserts.push("(NULL, #{q.id}, NULL, #{l.id}, '#{created_timestring}', '#{updated_timestring}', NULL, 0, #{r.id}, NULL, NULL)")
  161 + promptscount+=1
  162 + end
  163 +
  164 + end
  165 +
  166 + end
167 167 end
168 168  
169   - print "Added #{promptscount} to #{q.id}\n"
170   - sql = "INSERT INTO `prompts` (`algorithm_id`, `question_id`, `voter_id`, `left_choice_id`, `created_at`, `updated_at`, `tracking`, `votes_count`, `right_choice_id`, `active`, `randomkey`) VALUES #{inserts.join(', ')}"
171   - unless inserts.empty?
172   - ActiveRecord::Base.connection.execute(sql)
173   - end
  169 + print "Added #{promptscount} to #{q.id}\n"
  170 + sql = "INSERT INTO `prompts` (`algorithm_id`, `question_id`, `voter_id`, `left_choice_id`, `created_at`, `updated_at`, `tracking`, `votes_count`, `right_choice_id`, `active`, `randomkey`) VALUES #{inserts.join(', ')}"
  171 + unless inserts.empty?
  172 + ActiveRecord::Base.connection.execute(sql)
  173 + end
174 174  
175   - Question.update_counters(q.id, :prompts_count => promptscount)
  175 + Question.update_counters(q.id, :prompts_count => promptscount)
176 176  
177 177  
178 178 end
... ... @@ -185,337 +185,339 @@ namespace :test_api do
185 185 desc "Dump votes of a question by left vs right id"
186 186 task(:make_csv => :environment) do
187 187  
188   - q = Question.find(214)
189   -
190   -
191   - the_prompts = q.prompts_hash_by_choice_ids
192   -
193   - #hash_of_choice_ids_from_left_to_right_to_votes
194   - the_hash = {}
195   - q.choices.each do |l|
196   - q.choices.each do |r|
197   - next if l.id == r.id
198   -
199   - if not the_hash.has_key?(l.id)
200   - the_hash[l.id] = {}
201   - the_hash[l.id][l.id] = 0
202   - end
203   -
204   - p = the_prompts["#{l.id}, #{r.id}"]
205   - if p.nil?
206   - the_hash[l.id][r.id] = 0
207   - else
208   - the_hash[l.id][r.id] = p.appearances.size
209   - end
210   - end
211   - end
212   -
213   - the_hash.sort.each do |xval, row|
214   - rowarray = []
215   - row.sort.each do |yval, cell|
216   - rowarray << cell
217   - end
218   - puts rowarray.join(", ")
219   - end
  188 + q = Question.find(214)
  189 +
  190 +
  191 + the_prompts = q.prompts_hash_by_choice_ids
  192 +
  193 + #hash_of_choice_ids_from_left_to_right_to_votes
  194 + the_hash = {}
  195 + q.choices.each do |l|
  196 + q.choices.each do |r|
  197 + next if l.id == r.id
  198 +
  199 + if not the_hash.has_key?(l.id)
  200 + the_hash[l.id] = {}
  201 + the_hash[l.id][l.id] = 0
  202 + end
  203 +
  204 + p = the_prompts["#{l.id}, #{r.id}"]
  205 + if p.nil?
  206 + the_hash[l.id][r.id] = 0
  207 + else
  208 + the_hash[l.id][r.id] = p.appearances.size
  209 + end
  210 + end
  211 + end
  212 +
  213 + the_hash.sort.each do |xval, row|
  214 + rowarray = []
  215 + row.sort.each do |yval, cell|
  216 + rowarray << cell
  217 + end
  218 + puts rowarray.join(", ")
  219 + end
220 220 end
221 221  
222 222  
223 223 desc "Generate density information for each question - should be run nightly"
224 224 task(:generate_density_information => :environment) do
225 225  
226   - # calculating densities is expensive, so only do it for questions with new data
227   - question_ids = Vote.count(:conditions => ['date(created_at) = ?', Date.yesterday], :group => 'question_id').keys()
  226 + # calculating densities is expensive, so only do it for questions with new data
  227 + question_ids = Vote.count(:conditions => ['date(created_at) = ?', Date.yesterday], :group => 'question_id').keys()
228 228  
229   - Question.find(:all, :conditions => {:id => question_ids}).each do |q|
230   - q.save_densities!
231   - end
  229 + Question.find(:all, :conditions => {:id => question_ids}).each do |q|
  230 + q.save_densities!
  231 + end
232 232  
233   - # we can just copy the previous night's data for remaining questions
234   -
235   - Question.find(:all, :conditions => ['id NOT IN (?)', question_ids]).each do |q|
236   - densities = q.densities.find(:all, :conditions => ['date(created_at) = ?', Date.yesterday])
  233 + # we can just copy the previous night's data for remaining questions
  234 +
  235 + Question.find(:all, :conditions => ['id NOT IN (?)', question_ids]).each do |q|
  236 + densities = q.densities.find(:all, :conditions => ['date(created_at) = ?', Date.yesterday])
237 237  
238 238  
239   - densities.each do |d|
240   - new_d = d.clone
241   - new_d.created_at = new_d.updated_at = Time.now
242   - new_d.save!
243   - end
  239 + densities.each do |d|
  240 + new_d = d.clone
  241 + new_d.created_at = new_d.updated_at = Time.now
  242 + new_d.save!
  243 + end
244 244  
245   - if densities.blank?
246   - #fallback in case there wasn't a successful run yesterday
247   - q.save_densities!
248   -
249   - end
  245 + if densities.blank?
  246 + #fallback in case there wasn't a successful run yesterday
  247 + q.save_densities!
  248 +
  249 + end
250 250  
251   - end
  251 + end
252 252 end
253 253  
254   - desc "Description here"
255   - task(:question_vote_consistency => :environment) do
256   - questions = Question.find(:all)
257   - errors = []
258   - successes = []
259   -
260   - questions.each do |question|
  254 + desc "Description here"
  255 + task(:question_vote_consistency => :environment) do
  256 + questions = Question.find(:all)
  257 + errors = []
  258 + successes = []
  259 +
  260 + questions.each do |question|
  261 +
  262 + message, error_occurred = check_basic_balanced_stats(question)
  263 + # hack for now, get around to doing this with block/yield to
  264 + # get rid of duplication
  265 + if error_occurred
  266 + errors << message
  267 + else
  268 + successes << message
  269 + end
261 270  
262   - message, error_occurred = check_basic_balanced_stats(question)
263   - #hack for now, get around to doing this with block /yield to get rid of duplication
264   - if error_occurred
265   - errors << message
266   - else
267   - successes << message
268   - end
269 271  
  272 + message, error_occurred = check_each_choice_appears_within_n_stddevs(question)
  273 + if error_occurred
  274 + errors << message
  275 + else
  276 + successes << message
  277 + end
270 278  
271   - message, error_occurred = check_each_choice_appears_within_n_stddevs(question)
272   - if error_occurred
273   - errors << message
274   - else
275   - successes << message
276   - end
277   -
278   - message, error_occurred = check_each_choice_equally_likely_to_appear_left_or_right(question)
279   - if error_occurred
280   - errors << message
281   - else
282   - successes << message
283   - end
284   -
285   -
286   -
287   - message, error_occurred = check_object_counter_cache_values_match_actual_values(question)
288   - if error_occurred
289   - errors << message
290   - else
291   - successes << message
292   - end
  279 + message, error_occurred = check_each_choice_equally_likely_to_appear_left_or_right(question)
  280 + if error_occurred
  281 + errors << message
  282 + else
  283 + successes << message
  284 + end
293 285  
294 286  
295   - #catchup specific
296   - if question.uses_catchup?
297   - message, error_occurred = check_prompt_cache_hit_rate(question)
298   - if error_occurred
299   - errors << message
300   - else
301   - successes << message
302   - end
303   - end
304   - end
305 287  
306   - message, error_occurred = ensure_all_votes_and_skips_have_unique_appearance
307   -
308   - if error_occurred
309   - errors << message
310   - else
311   - successes << message
312   - end
  288 + message, error_occurred = check_object_counter_cache_values_match_actual_values(question)
  289 + if error_occurred
  290 + errors << message
  291 + else
  292 + successes << message
  293 + end
313 294  
314   - message, error_occurred = response_time_tests
315 295  
316   - if error_occurred
317   - errors << message
318   - else
319   - successes << message
320   - end
  296 + #catchup specific
  297 + if question.uses_catchup?
  298 + message, error_occurred = check_prompt_cache_hit_rate(question)
  299 + if error_occurred
  300 + errors << message
  301 + else
  302 + successes << message
  303 + end
  304 + end
321 305  
322   - email_text = "Conducted the following tests on API data and found the following results\n" +
323   - "For each of the #{questions.length} questions in the database: \n"
324   - errors.each do |e|
325   - email_text += " Test FAILED: " + e + "\n"
326   - end
  306 + end
327 307  
328   - successes.uniq.each do |s|
329   - s.split("\n").each do |m| # some successes have several lines
330   - email_text += " Test Passed: " + m + "\n"
331   - end
332   - end
333   -
334   - puts email_text
  308 + message, error_occurred = ensure_all_votes_and_skips_have_unique_appearance
335 309  
336   - if errors.empty?
337   - CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", email_text)
338   - else
339   - CronMailer.deliver_info_message(CRON_EMAIL.to_a + ERRORS_EMAIL.to_a, "Error! Failure of API Vote Consistency " , email_text)
340   - end
  310 + if error_occurred
  311 + errors << message
  312 + else
  313 + successes << message
  314 + end
  315 +
  316 + message, error_occurred = response_time_tests
  317 +
  318 + if error_occurred
  319 + errors << message
  320 + else
  321 + successes << message
  322 + end
  323 +
  324 + email_text = "Conducted the following tests on API data and found the following results\n" + "For each of the #{questions.length} questions in the database: \n"
  325 + errors.each do |e|
  326 + email_text += " Test FAILED: " + e + "\n"
  327 + end
  328 +
  329 + successes.uniq.each do |s|
  330 + s.split("\n").each do |m| # some successes have several lines
  331 + email_text += " Test Passed: " + m + "\n"
  332 + end
  333 + end
  334 +
  335 + puts email_text
  336 +
  337 + if errors.empty?
  338 + CronMailer.deliver_info_message(CRON_EMAIL, "Test of API Vote Consistency passed", email_text)
  339 + else
  340 + CronMailer.deliver_info_message(CRON_EMAIL.to_a + ERRORS_EMAIL.to_a, "Error! Failure of API Vote Consistency " , email_text)
  341 + end
  342 +
  343 + end
341 344  
342   - end
343 345 def check_basic_balanced_stats(question)
344   - error_message = ""
345   - success_message = "2 x Total Wins = Total Votes\n" +
346   - "Total Votes (wins + losses) is Even\n" +
347   - "Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +
348   - "Total generated prompts on left = Total generated prompts on right"
349   - total_wins =0
350   - total_votes =0
351   - total_generated_prompts_on_left = 0
352   - total_generated_prompts_on_right = 0
353   - total_scores_gte_fifty= 0
354   - total_scores_lte_fifty= 0
355   - error_bool = false
356   -
357   - question.choices.each do |choice|
358   -
359   - if choice.wins
360   - total_wins += choice.wins
361   - total_votes += choice.wins
362   - end
363   -
364   - if choice.losses
  346 + error_message = ""
  347 + success_message = "2 x Total Wins = Total Votes\n" +
  348 + "Total Votes (wins + losses) is Even\n" +
  349 + "Total Votes (wins + losses) = 2 x the number of vote objects that belong to the question\n" +
  350 + "Total generated prompts on left = Total generated prompts on right"
  351 + total_wins =0
  352 + total_votes =0
  353 + total_generated_prompts_on_left = 0
  354 + total_generated_prompts_on_right = 0
  355 + total_scores_gte_fifty= 0
  356 + total_scores_lte_fifty= 0
  357 + error_bool = false
  358 +
  359 + question.choices.each do |choice|
  360 +
  361 + if choice.wins
  362 + total_wins += choice.wins
  363 + total_votes += choice.wins
  364 + end
  365 +
  366 + if choice.losses
365 367 total_votes += choice.losses
366   - end
  368 + end
367 369  
368   - total_generated_prompts_on_left += choice.prompts_on_the_left.size
369   - total_generated_prompts_on_right += choice.prompts_on_the_right.size
  370 + total_generated_prompts_on_left += choice.prompts_on_the_left.size
  371 + total_generated_prompts_on_right += choice.prompts_on_the_right.size
370 372  
371   - cached_score = choice.score.to_f
372   - generated_score = choice.compute_score.to_f
  373 + cached_score = choice.score.to_f
  374 + generated_score = choice.compute_score.to_f
373 375  
374   - delta = 0.001
  376 + delta = 0.001
375 377  
376   - if (cached_score - generated_score).abs >= delta
377   - error_message += "Error! The cached_score is not equal to the calculated score for choice #{choice.id}\n"
  378 + if (cached_score - generated_score).abs >= delta
  379 + error_message += "Error! The cached_score is not equal to the calculated score for choice #{choice.id}\n"
378 380  
379   - print "This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
  381 + print "This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
380 382  
381 383  
382   - end
  384 + end
383 385  
384   - if cached_score == 0.0 || cached_score == 100.0 || cached_score.nil?
385   - error_message += "Error! The cached_score for choice #{choice.id} is exactly 0 or 100, the value: #{cached_score}"
386   - print "Either 0 or 100 This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
387   - end
  386 + if cached_score == 0.0 || cached_score == 100.0 || cached_score.nil?
  387 + error_message += "Error! The cached_score for choice #{choice.id} is exactly 0 or 100, the value: #{cached_score}"
  388 + print "Either 0 or 100 This score is wrong! #{choice.id} , Question ID: #{question.id}, #{cached_score}, #{generated_score}, updated: #{choice.updated_at}\n"
  389 + end
388 390  
389 391  
390   - if cached_score >= 50
391   - total_scores_gte_fifty +=1
392   - end
393   - if cached_score <= 50
394   - total_scores_lte_fifty +=1
395   - end
  392 + if cached_score >= 50
  393 + total_scores_gte_fifty +=1
  394 + end
  395 + if cached_score <= 50
  396 + total_scores_lte_fifty +=1
  397 + end
396 398  
397   - if (choice.wins != choice.votes.count)
398   - error_message += "Error!: Cached choice wins != actual choice wins for choice #{choice.id}\n"
399   - error_bool= true
400   - end
401   -
402   - if (choice.losses != question.votes.count(:conditions => {:loser_choice_id => choice.id}))
403   - error_message += "Error!: Cached choice wins != actual choice wins for choice #{choice.id}\n"
404   - error_bool= true
405   - end
  399 + if (choice.wins != choice.votes.count)
  400 + error_message += "Error!: Cached choice wins != actual choice wins for choice #{choice.id}\n"
  401 + error_bool= true
  402 + end
  403 +
  404 + if (choice.losses != question.votes.count(:conditions => {:loser_choice_id => choice.id}))
  405 + error_message += "Error!: Cached choice wins != actual choice wins for choice #{choice.id}\n"
  406 + error_bool= true
  407 + end
406 408  
407 409 end
408   -
409   -
410   - if (2*total_wins != total_votes)
411   - error_message += "Error 1: 2 x Total Wins != Total votes"
412   - error_bool= true
413   - end
414   -
415   - if(total_votes % 2 != 0)
416   - error_message += "Error 2: Total votes is not Even!"
417   - error_bool= true
418   - end
419   -
420   - if(total_votes != 2* question.votes_count)
421   - error_message += "Error 3: Total votes != 2 x # vote objects"
422   - error_bool = true
423   - end
424   -
425   - if(total_generated_prompts_on_right != total_generated_prompts_on_right)
426   - error_message += "Error 4: Total generated prompts on left != Total generated prompts on right"
427   - error_bool = true
428   - end
429   -
430   - if(total_scores_lte_fifty == question.choices.size || total_scores_gte_fifty == question.choices.size) && (total_scores_lte_fifty != total_scores_gte_fifty)
431   - error_message += "Error: The scores of all choices are either all above 50, or all below 50. This is probably wrong"
432   - error_bool = true
433   - puts "Error score fifty: #{question.id}"
434   - end
435   -
436   - if error_bool
437   - error_message += "Question #{question.id}: 2*wins = #{2*total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
438   - end
  410 +
  411 +
  412 + if (2*total_wins != total_votes)
  413 + error_message += "Error 1: 2 x Total Wins != Total votes"
  414 + error_bool= true
  415 + end
  416 +
  417 + if(total_votes % 2 != 0)
  418 + error_message += "Error 2: Total votes is not Even!"
  419 + error_bool= true
  420 + end
  421 +
  422 + if(total_votes != 2* question.votes_count)
  423 + error_message += "Error 3: Total votes != 2 x # vote objects"
  424 + error_bool = true
  425 + end
  426 +
  427 + if(total_generated_prompts_on_right != total_generated_prompts_on_right)
  428 + error_message += "Error 4: Total generated prompts on left != Total generated prompts on right"
  429 + error_bool = true
  430 + end
  431 +
  432 + if(total_scores_lte_fifty == question.choices.size || total_scores_gte_fifty == question.choices.size) && (total_scores_lte_fifty != total_scores_gte_fifty)
  433 + error_message += "Error: The scores of all choices are either all above 50, or all below 50. This is probably wrong"
  434 + error_bool = true
  435 + puts "Error score fifty: #{question.id}"
  436 + end
  437 +
  438 + if error_bool
  439 + error_message += "Question #{question.id}: 2*wins = #{2*total_wins}, total votes = #{total_votes}, vote_count = #{question.votes_count}\n"
  440 + end
439 441 return error_message.blank? ? [success_message, false] : [error_message, true]
440 442 end
441 443 def check_each_choice_appears_within_n_stddevs(question)
442   - error_message =""
443   - success_message = "Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question " +
444   - "(Note: this applies only to seed choices (not user submitted) and choices currently marked active)"
  444 + error_message =""
  445 + success_message = "Each choice has appeared n times, where n falls within 6 stddevs of the mean number of appearances for a question " +
  446 + "(Note: this applies only to seed choices (not user submitted) and choices currently marked active)"
445 447  
446   - wins_by_choice_id = question.votes.active.count(:group => :choice_id)
447   - losses_by_choice_id= question.votes.active.count(:conditions => "loser_choice_id IS NOT NULL", :group => :loser_choice_id)
  448 + wins_by_choice_id = question.votes.active.count(:group => :choice_id)
  449 + losses_by_choice_id= question.votes.active.count(:conditions => "loser_choice_id IS NOT NULL", :group => :loser_choice_id)
448 450  
449   - #Rails returns an ordered hash, which doesn't allow for blocks to change merging logic.
450   - #A little hack to create a normal hash
451   - wins_hash = {}
452   - wins_hash.merge!(wins_by_choice_id)
453   - losses_hash = {}
454   - losses_hash.merge!(losses_by_choice_id)
  451 + #Rails returns an ordered hash, which doesn't allow for blocks to change merging logic.
  452 + #A little hack to create a normal hash
  453 + wins_hash = {}
  454 + wins_hash.merge!(wins_by_choice_id)
  455 + losses_hash = {}
  456 + losses_hash.merge!(losses_by_choice_id)
455 457  
456 458  
457 459  
458   - appearances_by_choice_id = wins_hash.merge(losses_hash) do |key, oldval, newval| oldval + newval end
  460 + appearances_by_choice_id = wins_hash.merge(losses_hash) do |key, oldval, newval| oldval + newval end
459 461  
460   - sum = total_appearances = appearances_by_choice_id.values.inject(0) {|sum, x| sum +=x}
461   - mean = average_appearances = total_appearances.to_f / appearances_by_choice_id.size.to_f
  462 + sum = total_appearances = appearances_by_choice_id.values.inject(0) {|sum, x| sum +=x}
  463 + mean = average_appearances = total_appearances.to_f / appearances_by_choice_id.size.to_f
462 464  
463   - if sum > 0:
464   - stddev = Math.sqrt( appearances_by_choice_id.values.inject(0) { |sum, e| sum + (e - mean) ** 2 } / appearances_by_choice_id.size.to_f )
  465 + if sum > 0:
  466 + stddev = Math.sqrt( appearances_by_choice_id.values.inject(0) { |sum, e| sum + (e - mean) ** 2 } / appearances_by_choice_id.size.to_f )
465 467  
466 468 appearances_by_choice_id.each do |choice_id, n_i|
467   - if (n_i < (mean - 6*stddev)) || (n_i > mean + 6 *stddev)
468   - error_message += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question"
469   - end
470   - end
471   - end
  469 + if (n_i < (mean - 6*stddev)) || (n_i > mean + 6 *stddev)
  470 + error_message += "Choice #{choice_id} in Question ##{question.id} has an irregular number of appearances: #{n_i}, as compared to the mean: #{mean} and stddev #{stddev} for this question"
  471 + end
  472 + end
  473 + end
472 474  
473 475 return error_message.blank? ? [success_message, false] : [error_message, true]
474 476 end
475 477 def check_each_choice_equally_likely_to_appear_left_or_right(question)
476   - error_message = ""
477   - success_message = "All choices have equal probability of appearing on left or right (within error params)"
478   - question.choices.each do |c|
479   - left_prompts_ids = c.prompts_on_the_left.ids_only
480   - right_prompts_ids = c.prompts_on_the_right.ids_only
481   -
482   - left_appearances = question.appearances.count(:conditions => {:prompt_id => left_prompts_ids})
483   - right_appearances = question.appearances.count(:conditions => {:prompt_id => right_prompts_ids})
484   -
485   - n = left_appearances + right_appearances
486   -
487   - if n == 0
488   - next
489   - end
490   - est_p = right_appearances.to_f / n.to_f
491   - z = (est_p - 0.5).abs / Math.sqrt((0.5 * 0.5) / n.to_f)
492   -
493   - if z > 6
494   - error_message += "Error: Choice ID #{c.id} seems to favor one side: Left Appearances #{left_appearances}, Right Appearances: #{right_appearances}, z = #{z}\n"
495   - end
496   - end
  478 + error_message = ""
  479 + success_message = "All choices have equal probability of appearing on left or right (within error params)"
  480 + question.choices.each do |c|
  481 + left_prompts_ids = c.prompts_on_the_left.ids_only
  482 + right_prompts_ids = c.prompts_on_the_right.ids_only
  483 +
  484 + left_appearances = question.appearances.count(:conditions => {:prompt_id => left_prompts_ids})
  485 + right_appearances = question.appearances.count(:conditions => {:prompt_id => right_prompts_ids})
  486 +
  487 + n = left_appearances + right_appearances
  488 +
  489 + if n == 0
  490 + next
  491 + end
  492 + est_p = right_appearances.to_f / n.to_f
  493 + z = (est_p - 0.5).abs / Math.sqrt((0.5 * 0.5) / n.to_f)
  494 +
  495 + if z > 6
  496 + error_message += "Error: Choice ID #{c.id} seems to favor one side: Left Appearances #{left_appearances}, Right Appearances: #{right_appearances}, z = #{z}\n"
  497 + end
  498 + end
497 499 return error_message.blank? ? [success_message, false] : [error_message, true]
498 500 end
499 501 def check_prompt_cache_hit_rate(question)
500   - error_message = ""
501   - success_message = "At least 90% of prompts on catchup algorithm questions were served from cache\n"
502   -
503   - misses = question.get_prompt_cache_misses(Date.yesterday).to_i
504   - hits = question.get_prompt_cache_hits(Date.yesterday).to_i
505   -
506   - question.expire_prompt_cache_tracking_keys(Date.yesterday)
507   -
508   - yesterday_appearances = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
509   -
510   - if misses + hits != yesterday_appearances
511   - error_message += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_appearances}, Actual: #{misses+hits}\n"
512   - end
513   -
514   - if yesterday_appearances > 5 # this test isn't worthwhile for small numbers of appearances
515   - miss_rate = misses.to_f / yesterday_appearances.to_f
516   - if miss_rate > 0.1
517   - error_message += "Warning! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}, total appearances yesterday: #{yesterday_appearances}\n"
518   - end
  502 + error_message = ""
  503 + success_message = "At least 90% of prompts on catchup algorithm questions were served from cache\n"
  504 +
  505 + misses = question.get_prompt_cache_misses(Date.yesterday).to_i
  506 + hits = question.get_prompt_cache_hits(Date.yesterday).to_i
  507 +
  508 + question.expire_prompt_cache_tracking_keys(Date.yesterday)
  509 +
  510 + yesterday_appearances = question.appearances.count(:conditions => ['date(created_at) = ?', Date.yesterday])
  511 +
  512 + if misses + hits != yesterday_appearances
  513 + error_message += "Error! Question #{question.id} isn't tracking prompt cache hits and misses accurately! Expected #{yesterday_appearances}, Actual: #{misses+hits}\n"
  514 + end
  515 +
  516 + if yesterday_appearances > 5 # this test isn't worthwhile for small numbers of appearances
  517 + miss_rate = misses.to_f / yesterday_appearances.to_f
  518 + if miss_rate > 0.1
  519 + error_message += "Warning! Question #{question.id} has less than 90% of appearances taken from a pre-generated cache! Expected <#{0.1}, Actual: #{miss_rate}, total appearances yesterday: #{yesterday_appearances}\n"
  520 + end
519 521 end
520 522 return error_message.blank? ? [success_message, false] : [error_message, true]
521 523 end
... ... @@ -524,30 +526,30 @@ namespace :test_api do
524 526 error_message = ""
525 527 success_message = "All cached object values match actual values within database"
526 528 # Checks that counter_cache is working as expected
527   - cached_prompts_size = question.prompts.size
528   - actual_prompts_size = question.prompts.count
529   -
530   - if cached_prompts_size != actual_prompts_size
531   - error_message += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n"
532   - end
533   -
534   - cached_votes_size = question.votes.size
535   - actual_votes_size = question.votes.count
536   -
537   - if cached_votes_size != actual_votes_size
538   - error_message += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n"
539   - end
540   -
541   - cached_choices_size = question.choices.size
542   - actual_choices_size = question.choices.count
543   -
544   - if cached_choices_size != actual_choices_size
545   - error_message+= "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n"
546   - end
547   -
548   - #if cached_prompts_size != question.choices.size **2 - question.choices.size
549   - # error_message += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n"
550   - #end
  529 + cached_prompts_size = question.prompts.size
  530 + actual_prompts_size = question.prompts.count
  531 +
  532 + if cached_prompts_size != actual_prompts_size
  533 + error_message += "Error! Question #{question.id} has an inconsistent # of prompts! cached#: #{cached_prompts_size}, actual#: #{actual_prompts_size}\n"
  534 + end
  535 +
  536 + cached_votes_size = question.votes.size
  537 + actual_votes_size = question.votes.count
  538 +
  539 + if cached_votes_size != actual_votes_size
  540 + error_message += "Error! Question #{question.id} has an inconsistent # of votes! cached#: #{cached_votes_size}, actual#: #{actual_votes_size}\n"
  541 + end
  542 +
  543 + cached_choices_size = question.choices.size
  544 + actual_choices_size = question.choices.count
  545 +
  546 + if cached_choices_size != actual_choices_size
  547 + error_message+= "Error! Question #{question.id} has an inconsistent # of choices! cached#: #{cached_choices_size}, actual#: #{actual_choices_size}\n"
  548 + end
  549 +
  550 + #if cached_prompts_size != question.choices.size **2 - question.choices.size
  551 + # error_message += "Error! Question #{question.id} has an incorrect number of prompts! Expected #{question.choices.size **2 - question.choices.size}, Actual: #{cached_prompts_size}\n"
  552 + #end
551 553 return error_message.blank? ? [success_message, false] : [error_message, true]
552 554 end
553 555  
... ... @@ -561,7 +563,7 @@ namespace :test_api do
561 563  
562 564 if (total_answered_appearances != total_votes+ total_skips)
563 565 difference = (total_votes+ total_skips) - total_answered_appearances
564   - error_message += "Error! There are #{difference} votes or skips without associated appearance objects."
  566 + error_message += "Error! There are #{difference} votes or skips without associated appearance objects."
565 567 end
566 568  
567 569 return error_message.blank? ? [success_message, false] : [error_message, true]
... ... @@ -575,32 +577,32 @@ namespace :test_api do
575 577  
576 578 Vote.find_each(:batch_size => 1000, :include => :appearance) do |v|
577 579  
578   - # Subtracting DateTime objects results in the difference in days
579   - server_response_time = v.created_at.to_f - v.appearance.created_at.to_f
580   - if server_response_time < 0
581   - the_error_msg = "Error! Vote #{v.id} was created before the appearance associated with it: Appearance id: #{v.appearance.id}, Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}\n"
  580 + # Subtracting DateTime objects results in the difference in days
  581 + server_response_time = v.created_at.to_f - v.appearance.created_at.to_f
  582 + if server_response_time < 0
  583 + the_error_msg = "Error! Vote #{v.id} was created before the appearance associated with it: Appearance id: #{v.appearance.id}, Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}\n"
582 584  
583   - error_message += the_error_msg
584   - print "Error!" + the_error_msg
585   - end
  585 + error_message += the_error_msg
  586 + print "Error!" + the_error_msg
  587 + end
586 588  
587   - if v.time_viewed && v.time_viewed/1000 > server_response_time
588   - the_error_msg = "Warning! Vote #{v.id} with Appearance #{v.appearance.id}, has a longer client response time than is possible. Server roundtrip time is: #{v.created_at.to_f - v.appearance.created_at.to_f} seconds, but client side response time is: #{v.time_viewed.to_f / 1000.0} seconds\n"
  589 + if v.time_viewed && v.time_viewed/1000 > server_response_time
  590 + the_error_msg = "Warning! Vote #{v.id} with Appearance #{v.appearance.id}, has a longer client response time than is possible. Server roundtrip time is: #{v.created_at.to_f - v.appearance.created_at.to_f} seconds, but client side response time is: #{v.time_viewed.to_f / 1000.0} seconds\n"
589 591  
590   - error_message += the_error_msg
591   - print the_error_msg
  592 + error_message += the_error_msg
  593 + print the_error_msg
592 594  
593 595 elsif v.time_viewed.nil?
594   - if v.created_at > recording_client_time_start_date && v.missing_response_time_exp != 'invalid'
595   - the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, does not have a client response, even though it should! Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
596   - error_message += the_error_msg
597   - print the_error_msg
598   - end
  596 + if v.created_at > recording_client_time_start_date && v.missing_response_time_exp != 'invalid'
  597 + the_error_msg = "Error! Vote #{v.id} with Appearance #{v.appearance.id}, does not have a client response, even though it should! Vote creation time: #{v.created_at.to_s}, Appearance creation time: #{v.appearance.created_at.to_s}, Client side response time: #{v.time_viewed}\n"
  598 + error_message += the_error_msg
  599 + print the_error_msg
  600 + end
599 601  
600   - end
  602 + end
601 603  
602 604 end
603   -
  605 +
604 606 return error_message.blank? ? [success_message, false] : [error_message, true]
605 607 end
606 608 end
... ...