Commit 613a885dc94ae09618550bffba5f94f4cb23b008
1 parent
33212569
Exists in
master
and in
1 other branch
add logging of ambiguous times, times not in spans
Showing
1 changed file
with
30 additions
and
16 deletions
Show diff stats
lib/tasks/prune_db.rake
... | ... | @@ -29,14 +29,14 @@ namespace :prune_db do |
29 | 29 | desc "Converts all dates from PT to UTC" |
30 | 30 | task :convert_dates_to_utc => :environment do |
31 | 31 | time_spans = [ |
32 | - #{ :gt => "2009-11-01 02:00:00", :lt => "2010-03-14 02:00:00", :h => 8}, | |
33 | - #{ :gt => "2010-03-14 02:00:00", :lt => "2010-11-07 01:00:00", :h => 7}, | |
32 | + { :gt => "2009-11-01 02:00:00", :lt => "2010-03-14 02:00:00", :h => 8}, | |
33 | + { :gt => "2010-03-14 02:00:00", :lt => "2010-11-07 01:00:00", :h => 7}, | |
34 | 34 | { :gt => "2010-11-07 01:00:00", :lt => "2010-11-07 02:00:00", :h => nil}, |
35 | - #{ :gt => "2010-11-07 02:00:00", :lt => "2011-03-13 02:00:00", :h => 8}, | |
36 | - #{ :gt => "2011-03-13 02:00:00", :lt => "2011-11-06 01:00:00", :h => 7}, | |
35 | + { :gt => "2010-11-07 02:00:00", :lt => "2011-03-13 02:00:00", :h => 8}, | |
36 | + { :gt => "2011-03-13 02:00:00", :lt => "2011-11-06 01:00:00", :h => 7}, | |
37 | 37 | { :gt => "2011-11-06 01:00:00", :lt => "2011-11-06 02:00:00", :h => nil}, |
38 | - #{ :gt => "2011-11-06 02:00:00", :lt => "2012-03-11 02:00:00", :h => 8}, | |
39 | - #{ :gt => "2012-03-11 02:00:00", :lt => "2012-11-04 01:00:00", :h => 7} | |
38 | + { :gt => "2011-11-06 02:00:00", :lt => "2012-03-11 02:00:00", :h => 8}, | |
39 | + { :gt => "2012-03-11 02:00:00", :lt => "2012-11-04 01:00:00", :h => 7} | |
40 | 40 | ] |
41 | 41 | # UTC because Rails will be thinking DB is in UTC when we run this |
42 | 42 | time_spans.map! do |t| |
... | ... | @@ -61,6 +61,7 @@ namespace :prune_db do |
61 | 61 | } |
62 | 62 | |
63 | 63 | STDOUT.sync = true |
64 | + logger = Rails.logger | |
64 | 65 | datetime_fields.each do |table, columns| |
65 | 66 | print "#{table}" |
66 | 67 | batch_size = 10000 |
... | ... | @@ -72,21 +73,34 @@ namespace :prune_db do |
72 | 73 | print "." |
73 | 74 | |
74 | 75 | rows.each do |row| |
76 | + updated_values = {} | |
77 | + # delete any value where the value is blank | |
78 | + row.delete_if {|key, value| value.blank? } | |
75 | 79 | row.each do |column, value| |
76 | - if value.class == Time | |
77 | - time_spans.each do |span| | |
78 | - # TODO: update row if match and span has :h | |
79 | - # log if :h is nil these will be manually sorted out | |
80 | - # log if no time_span found | |
81 | - if value < span[:lt] && value > span[:gt] | |
82 | - if span[:h].blank? | |
83 | - puts "#{row.inspect}: #{span.inspect}" | |
84 | - end | |
85 | - break | |
80 | + next unless value.class == Time | |
81 | + time_spans.each do |span| | |
82 | + if value < span[:lt] && value > span[:gt] | |
83 | + # if blank then ambiguous and we don't know how to translate | |
84 | + if span[:h].blank? | |
85 | + logger.info "AMBIGUOUS: #{table} #{row["id"]} #{column}: #{value}" | |
86 | + updated_values[column] = nil | |
87 | + else | |
88 | + updated_values[column] = value + span[:h].hours | |
86 | 89 | end |
90 | + break | |
87 | 91 | end |
88 | 92 | end |
89 | 93 | end |
94 | + # Check if some columns did not match any spans | |
95 | + key_diff = row.keys - updated_values.keys - ["id"] | |
96 | + if key_diff.length > 0 | |
97 | + logger.info "MISSING SPAN: #{table} #{row["id"]} #{key_diff.inspect} #{row.inspect}" | |
98 | + end | |
99 | + # remove ambiguous columns (we set them to nil above) | |
100 | + updated_values.delete_if {|key, value| value.blank? } | |
101 | + if updated_values.length > 0 | |
102 | + logger.info "UPDATE: #{table} #{row.inspect} #{updated_values.inspect}" | |
103 | + end | |
90 | 104 | end |
91 | 105 | |
92 | 106 | i+= 1 | ... | ... |