Download as
Browse Code »

Commit a7e18ca81bd252838f55911835660dbc504badd3

Authored by Luke Baker
1 parent c6ead088
Exists in master and in 1 other branch production-vendorized

add support for multiple workers 

support the return value from select_all on production / staging
add SQL updates
Showing 1 changed file with 34 additions and 22 deletions   Show diff stats
lib/tasks/prune_db.rake
  Diff comments   View file @a7e18ca
... ... @@ -27,7 +27,9 @@ namespace :prune_db do
27 27 end
28 28  
29 29 desc "Converts all dates from PT to UTC"
30   - task :convert_dates_to_utc => :environment do
  30 + task :convert_dates_to_utc, [:workerid, :workers] => [:environment] do|t,args|
  31 + args.with_defaults(:workerid => 0, :workers => 1)
  32 + raise "workerid can not be greater than workers" if args[:workerid] > args[:workers]
31 33 time_spans = [
32 34 { :gt => "2009-11-01 01:59:59", :lt => "2010-03-14 02:00:00", :h => 8},
33 35 { :gt => "2010-03-14 01:59:59", :lt => "2010-11-07 01:00:00", :h => 7},
... ... @@ -39,25 +41,25 @@ namespace :prune_db do
39 41 { :gt => "2012-03-11 01:59:59", :lt => "2012-11-04 01:00:00", :h => 7}
40 42 ]
41 43 # UTC because Rails will be thinking DB is in UTC when we run this
42   - time_spans.map! do |t|
43   - { :gt => Time.parse("#{t[:gt]} UTC"),
44   - :lt => Time.parse("#{t[:lt]} UTC"),
45   - :h => t[:h] }
46   - end
  44 + #time_spans.map! do |t|
  45 + # { :gt => Time.parse("#{t[:gt]} UTC"),
  46 + # :lt => Time.parse("#{t[:lt]} UTC"),
  47 + # :h => t[:h] }
  48 + #end
47 49 datetime_fields = {
48   - :appearances => ['created_at', 'updated_at'],
49   - :choices => ['created_at', 'updated_at'],
50   - :clicks => ['created_at', 'updated_at'],
  50 + #:appearances => ['created_at', 'updated_at'],
  51 + #:choices => ['created_at', 'updated_at'],
  52 + #:clicks => ['created_at', 'updated_at'],
51 53 :densities => ['created_at', 'updated_at'],
52   - :flags => ['created_at', 'updated_at'],
53   - :prompts => ['created_at', 'updated_at'],
54   - :skips => ['created_at', 'updated_at'],
55   - :votes => ['created_at', 'updated_at'],
56   - :visitors => ['created_at', 'updated_at'],
57   - :users => ['created_at', 'updated_at'],
58   - :questions => ['created_at', 'updated_at'],
59   - :question_versions => ['created_at', 'updated_at'],
60   - :delayed_jobs => ['created_at', 'updated_at', 'run_at', 'locked_at', 'failed_at'],
  54 + #:flags => ['created_at', 'updated_at'],
  55 + #:prompts => ['created_at', 'updated_at'],
  56 + #:skips => ['created_at', 'updated_at'],
  57 + #:votes => ['created_at', 'updated_at'],
  58 + #:visitors => ['created_at', 'updated_at'],
  59 + #:users => ['created_at', 'updated_at'],
  60 + #:questions => ['created_at', 'updated_at'],
  61 + #:question_versions => ['created_at', 'updated_at'],
  62 + #:delayed_jobs => ['created_at', 'updated_at', 'run_at', 'locked_at', 'failed_at'],
61 63 }
62 64  
63 65 STDOUT.sync = true
... ... @@ -66,9 +68,13 @@ namespace :prune_db do
66 68 print "#{table}"
67 69 batch_size = 10000
68 70 i = 0
  71 + where = ''
  72 + if args[:workers] > "1"
  73 + where = "WHERE MOD(id, #{args[:workers]}) = #{args[:workerid]}"
  74 + end
69 75 while true do
70 76 rows = ActiveRecord::Base.connection.select_all(
71   - "SELECT id, #{columns.join(", ")} FROM #{table} ORDER BY id LIMIT #{i*batch_size}, #{batch_size}"
  77 + "SELECT id, #{columns.join(", ")} FROM #{table} #{where} ORDER BY id LIMIT #{i*batch_size}, #{batch_size}"
72 78 )
73 79 print "."
74 80  
... ... @@ -77,7 +83,7 @@ namespace :prune_db do
77 83 # delete any value where the value is blank
78 84 row.delete_if {|key, value| value.blank? }
79 85 row.each do |column, value|
80   - next unless value.class == Time
  86 + next if column == "id"
81 87 time_spans.each do |span|
82 88 if value < span[:lt] && value > span[:gt]
83 89 # if blank then ambiguous and we don't know how to translate
... ... @@ -85,7 +91,7 @@ namespace :prune_db do
85 91 logger.info "AMBIGUOUS: #{table} #{row["id"]} #{column}: #{value}"
86 92 updated_values[column] = nil
87 93 else
88   - updated_values[column] = value + span[:h].hours
  94 + updated_values[column] = Time.parse("#{value} UTC") + span[:h].hours
89 95 end
90 96 break
91 97 end
... ... @@ -99,7 +105,13 @@ namespace :prune_db do
99 105 # remove ambiguous columns (we set them to nil above)
100 106 updated_values.delete_if {|key, value| value.blank? }
101 107 if updated_values.length > 0
102   - logger.info "UPDATE: #{table} #{row.inspect} #{updated_values.inspect}"
  108 + update = "UPDATE #{table} SET #{updated_values.map{|k,v| "#{k} = '#{v.to_formatted_s(:db)}'"}.join(", ")} WHERE id = #{row["id"]}"
  109 + num = ActiveRecord::Base.connection.update_sql(update)
  110 + if num == 1
  111 + logger.info "UPDATE: #{table} #{row.inspect} #{updated_values.inspect}"
  112 + else
  113 + logger.info "UPDATE FAILED: #{table} #{row.inspect} #{updated_values.inspect} #{num.inspect}"
  114 + end
103 115 end
104 116 end
105 117  
... ...