Commit ac6d249329acdb2b6dbdbb8ba8fbc26d69cc5f8f
1 parent
a43ab92a
Exists in
master
and in
1 other branch
Speeding up density generation
Showing
2 changed files
with
55 additions
and
10 deletions
Show diff stats
app/models/question.rb
| @@ -276,19 +276,40 @@ class Question < ActiveRecord::Base | @@ -276,19 +276,40 @@ class Question < ActiveRecord::Base | ||
| 276 | nonseed_nonseed_sum= 0 | 276 | nonseed_nonseed_sum= 0 |
| 277 | nonseed_nonseed_total= 0 | 277 | nonseed_nonseed_total= 0 |
| 278 | 278 | ||
| 279 | + #cache some hashes to prevent tons of sql thrashing | ||
| 280 | + num_appearances_by_prompt = self.appearances.count(:group => :prompt_id) | ||
| 281 | + | ||
| 282 | + | ||
| 283 | + is_user_created = {} | ||
| 284 | + self.choices.each do |c| | ||
| 285 | + is_user_created[c.id] = c.user_created | ||
| 286 | + end | ||
| 287 | + | ||
| 288 | + | ||
| 279 | #the_prompts = prompts.find(:all, :include => ['left_choice', 'right_choice']) | 289 | #the_prompts = prompts.find(:all, :include => ['left_choice', 'right_choice']) |
| 280 | - prompts.find_each(:include => ['left_choice', 'right_choice']) do |p| | ||
| 281 | - if p.left_choice.user_created == false && p.right_choice.user_created == false | ||
| 282 | - seed_seed_sum += p.appearances.size | 290 | + prompts.find_each do |p| |
| 291 | + | ||
| 292 | + num_appearances = num_appearances_by_prompt[p.id] | ||
| 293 | + | ||
| 294 | + if num_appearances.nil? | ||
| 295 | + num_appearances = 0 | ||
| 296 | + end | ||
| 297 | + | ||
| 298 | + left_user_created = is_user_created[p.left_choice_id] | ||
| 299 | + right_user_created = is_user_created[p.right_choice_id] | ||
| 300 | + | ||
| 301 | + | ||
| 302 | + if left_user_created == false && right_user_created == false | ||
| 303 | + seed_seed_sum += num_appearances | ||
| 283 | seed_seed_total +=1 | 304 | seed_seed_total +=1 |
| 284 | - elsif p.left_choice.user_created == false && p.right_choice.user_created == true | ||
| 285 | - seed_nonseed_sum += p.appearances.size | 305 | + elsif left_user_created == false && right_user_created == true |
| 306 | + seed_nonseed_sum += num_appearances | ||
| 286 | seed_nonseed_total +=1 | 307 | seed_nonseed_total +=1 |
| 287 | - elsif p.left_choice.user_created == true && p.right_choice.user_created == false | ||
| 288 | - nonseed_seed_sum += p.appearances.size | 308 | + elsif left_user_created == true && right_user_created == false |
| 309 | + nonseed_seed_sum += num_appearances | ||
| 289 | nonseed_seed_total +=1 | 310 | nonseed_seed_total +=1 |
| 290 | - elsif p.left_choice.user_created == true && p.right_choice.user_created == true | ||
| 291 | - nonseed_nonseed_sum += p.appearances.size | 311 | + elsif left_user_created == true && right_user_created == true |
| 312 | + nonseed_nonseed_sum += num_appearances | ||
| 292 | nonseed_nonseed_total +=1 | 313 | nonseed_nonseed_total +=1 |
| 293 | end | 314 | end |
| 294 | end | 315 | end |
lib/tasks/test_api.rake
| @@ -211,9 +211,33 @@ namespace :test_api do | @@ -211,9 +211,33 @@ namespace :test_api do | ||
| 211 | 211 | ||
| 212 | desc "Generate density information for each question - should be run nightly" | 212 | desc "Generate density information for each question - should be run nightly" |
| 213 | task(:generate_density_information => :environment) do | 213 | task(:generate_density_information => :environment) do |
| 214 | - Question.find(:all).each do |q| | 214 | + |
| 215 | + # calculating densities is expensive, so only do it for questions with new data | ||
| 216 | + question_ids = Vote.count(:conditions => ['date(created_at) = ?', Date.yesterday], :group => 'question_id').keys() | ||
| 217 | + | ||
| 218 | + Question.find(:all, :conditions => {:id => question_ids}).each do |q| | ||
| 215 | q.save_densities! | 219 | q.save_densities! |
| 216 | end | 220 | end |
| 221 | + | ||
| 222 | + # we can just copy the previous night's data for remaining questions | ||
| 223 | + | ||
| 224 | + Question.find(:all, :conditions => ['id NOT IN (?)', question_ids]).each do |q| | ||
| 225 | + densities = q.densities.find(:all, :conditions => ['date(created_at) = ?', Date.yesterday]) | ||
| 226 | + | ||
| 227 | + | ||
| 228 | + densities.each do |d| | ||
| 229 | + new_d = d.clone | ||
| 230 | + new_d.created_at = new_d.updated_at = Time.now | ||
| 231 | + new_d.save! | ||
| 232 | + end | ||
| 233 | + | ||
| 234 | + if densities.blank? | ||
| 235 | + #fallback in case there wasn't a successful run yesterday | ||
| 236 | + q.save_densities! | ||
| 237 | + | ||
| 238 | + end | ||
| 239 | + | ||
| 240 | + end | ||
| 217 | end | 241 | end |
| 218 | 242 | ||
| 219 | desc "Description here" | 243 | desc "Description here" |