Download as
Browse Code »

Commit ac6d249329acdb2b6dbdbb8ba8fbc26d69cc5f8f

Authored by Dhruv Kapadia
1 parent a43ab92a
Exists in master and in 1 other branch production-vendorized

Speeding up density generation

Showing 2 changed files with 55 additions and 10 deletions   Show diff stats
app/models/question.rb
  Diff comments   View file @ac6d249
... ... @@ -276,19 +276,40 @@ class Question < ActiveRecord::Base
276 276 nonseed_nonseed_sum= 0
277 277 nonseed_nonseed_total= 0
278 278  
  279 + #cache some hashes to prevent tons of sql thrashing
  280 + num_appearances_by_prompt = self.appearances.count(:group => :prompt_id)
  281 +
  282 +
  283 + is_user_created = {}
  284 + self.choices.each do |c|
  285 + is_user_created[c.id] = c.user_created
  286 + end
  287 +
  288 +
279 289 #the_prompts = prompts.find(:all, :include => ['left_choice', 'right_choice'])
280   - prompts.find_each(:include => ['left_choice', 'right_choice']) do |p|
281   - if p.left_choice.user_created == false && p.right_choice.user_created == false
282   - seed_seed_sum += p.appearances.size
  290 + prompts.find_each do |p|
  291 +
  292 + num_appearances = num_appearances_by_prompt[p.id]
  293 +
  294 + if num_appearances.nil?
  295 + num_appearances = 0
  296 + end
  297 +
  298 + left_user_created = is_user_created[p.left_choice_id]
  299 + right_user_created = is_user_created[p.right_choice_id]
  300 +
  301 +
  302 + if left_user_created == false && right_user_created == false
  303 + seed_seed_sum += num_appearances
283 304 seed_seed_total +=1
284   - elsif p.left_choice.user_created == false && p.right_choice.user_created == true
285   - seed_nonseed_sum += p.appearances.size
  305 + elsif left_user_created == false && right_user_created == true
  306 + seed_nonseed_sum += num_appearances
286 307 seed_nonseed_total +=1
287   - elsif p.left_choice.user_created == true && p.right_choice.user_created == false
288   - nonseed_seed_sum += p.appearances.size
  308 + elsif left_user_created == true && right_user_created == false
  309 + nonseed_seed_sum += num_appearances
289 310 nonseed_seed_total +=1
290   - elsif p.left_choice.user_created == true && p.right_choice.user_created == true
291   - nonseed_nonseed_sum += p.appearances.size
  311 + elsif left_user_created == true && right_user_created == true
  312 + nonseed_nonseed_sum += num_appearances
292 313 nonseed_nonseed_total +=1
293 314 end
294 315 end
... ...
lib/tasks/test_api.rake
  Diff comments   View file @ac6d249
... ... @@ -211,9 +211,33 @@ namespace :test_api do
211 211  
212 212 desc "Generate density information for each question - should be run nightly"
213 213 task(:generate_density_information => :environment) do
214   - Question.find(:all).each do |q|
  214 +
  215 + # calculating densities is expensive, so only do it for questions with new data
  216 + question_ids = Vote.count(:conditions => ['date(created_at) = ?', Date.yesterday], :group => 'question_id').keys()
  217 +
  218 + Question.find(:all, :conditions => {:id => question_ids}).each do |q|
215 219 q.save_densities!
216 220 end
  221 +
  222 + # we can just copy the previous night's data for remaining questions
  223 +
  224 + Question.find(:all, :conditions => ['id NOT IN (?)', question_ids]).each do |q|
  225 + densities = q.densities.find(:all, :conditions => ['date(created_at) = ?', Date.yesterday])
  226 +
  227 +
  228 + densities.each do |d|
  229 + new_d = d.clone
  230 + new_d.created_at = new_d.updated_at = Time.now
  231 + new_d.save!
  232 + end
  233 +
  234 + if densities.blank?
  235 + #fallback in case there wasn't a successful run yesterday
  236 + q.save_densities!
  237 +
  238 + end
  239 +
  240 + end
217 241 end
218 242  
219 243 desc "Description here"
... ...