rework CSV export architecture

in order to work with multiple app servers, don't export to file. instead export to redis.

rework CSV export architecture
in order to work with multiple app servers, don't export to file. instead export to redis.
Luke Baker
1 parent a95a3e98
Showing 3 changed files with 40 additions and 80 deletions Show diff stats
app/controllers/questions_controller.rb
app/models/question.rb
spec/models/question_spec.rb
@@ -73,8 +73,8 @@ class QuestionsController &lt; InheritedResources::Base
     # puts "redis key is::::: #{redis_key}"
-    @question.send_later :export_and_delete, type, 
-	                        :response_type => response_type, :redis_key => redis_key, :delete_at => 3.days.from_now
+    @question.send_later :export, type, 
+	                        :response_type => response_type, :redis_key => redis_key
     render :text => "Ok! Please wait for the response (as specified by your response_type)"
@@ -512,36 +512,27 @@ class Question &lt; ActiveRecord::Base
   end
-  def export_and_delete(type, options={})
-    delete_at = options.delete(:delete_at)
-    filename = export(type, options)
-
-    File.send_at(delete_at, :delete, filename)
-    filename
-  end
-
   def export(type, options = {})
     case type
       when 'votes'
-        outfile = "ideamarketplace_#{self.id}_votes.csv"
+        outfile = "ideamarketplace_#{self.id}_votes"
         headers = ['Vote ID', 'Session ID', 'Ideamarketplace ID','Winner ID', 'Winner Text', 'Loser ID', 'Loser Text', 'Prompt ID', 'Left Choice ID', 'Right Choice ID', 'Created at', 'Updated at',  'Response Time (s)', 'Missing Response Time Explanation', 'Session Identifier', 'Valid']
       when 'ideas'
-        outfile = "ideamarketplace_#{self.id}_ideas.csv"
+        outfile = "ideamarketplace_#{self.id}_ideas"
         headers = ['Ideamarketplace ID','Idea ID', 'Idea Text', 'Wins', 'Losses', 'Times involved in Cant Decide', 'Score', 'User Submitted', 'Session ID', 'Created at', 'Last Activity', 'Active', 'Appearances on Left', 'Appearances on Right']
       when 'non_votes'
-        outfile = "ideamarketplace_#{self.id}_non_votes.csv"
+        outfile = "ideamarketplace_#{self.id}_non_votes"
         headers = ['Record Type', 'Record ID', 'Session ID', 'Ideamarketplace ID','Left Choice ID', 'Left Choice Text', 'Right Choice ID', 'Right Choice Text', 'Prompt ID', 'Reason', 'Created at', 'Updated at', 'Response Time (s)', 'Missing Response Time Explanation', 'Session Identifier', 'Valid']
       else 
         raise "Unsupported export type: #{type}"
     end
-    filename = File.join(File.expand_path(Rails.root), "public", "system", "exports", self.id.to_s, Digest::SHA1.hexdigest(outfile + rand(10000000).to_s) + "_" + outfile)
+    name = outfile + "_" + Digest::SHA1.hexdigest(outfile + rand(10000000).to_s) + ".csv"
-    FileUtils::mkdir_p(File.dirname(filename))
-    csv_data = FasterCSV.open(filename, "w") do |csv|
+    csv_data = FasterCSV.generate do |csv|
       csv << headers 
       case type
         when 'votes'
@@ -600,18 +591,23 @@ class Question &lt; ActiveRecord::Base
     end
+
     if options[:response_type] == 'redis'
+      # let's compress this for redis
+      # it should get removed from redis relatively quickly
+      zlib = Zlib::Deflate.new
+      zlibcsv = zlib.deflate(csv_data, Zlib::FINISH)
+      zlib.close
       if options[:redis_key].nil?
         raise "No :redis_key specified"
       end
       #The client should use blpop to listen for a key
       #The client is responsible for deleting the redis key (auto expiration results failure in testing)
-      $redis.lpush(options[:redis_key], filename)
-    #TODO implement response_type == 'email' for use by customers of the API (not local)
+      $redis.lpush(options[:redis_key], zlibcsv)
+    else
+      return csv_data
     end
-
-    filename
   end
   def get_first_unanswered_appearance(visitor, offset=0)
@@ -365,86 +365,60 @@ describe Question do
     it "should export vote data to a csv file" do
-      filename = @aoi_question.export('votes')
+      csv = @aoi_question.export('votes')
-      filename.should_not be nil
-      filename.should match /.*ideamarketplace_#{@aoi_question.id}_votes[.]csv$/
-      File.exists?(filename).should be_true
       # Not specifying exact file syntax, it's likely to change frequently
       #
-      rows = FasterCSV.read(filename)
+      rows = FasterCSV.parse(csv)
       rows.first.should include("Vote ID")
       rows.first.should_not include("Idea ID")
-      File.delete(filename).should be_true
     end
-    it "should notify redis after completing an export, if redis option set" do
+    it "should export zlibed csv to redis after completing an export, if redis option set" do
       redis_key = "test_key123"
       $redis.del(redis_key) # clear if key exists already
-      filename = @aoi_question.export('votes', :response_type => 'redis', :redis_key => redis_key)
-
-      filename.should_not be nil
-      filename.should match /.*ideamarketplace_#{@aoi_question.id}_votes[.]csv$/
-      File.exists?(filename).should be_true
-      $redis.lpop(redis_key).should == filename
+      csv = @aoi_question.export('votes')
+      @aoi_question.export('votes', :response_type => 'redis', :redis_key => redis_key)
+
+      zlibcsv = $redis.lpop(redis_key)
+      zstream = Zlib::Inflate.new
+      buf = zstream.inflate(zlibcsv)
+      zstream.finish
+      zstream.close
+      buf.should == csv
       $redis.del(redis_key) # clean up
-      File.delete(filename).should be_true
     end
     it "should email question owner after completing an export, if email option set" do
       #TODO 
     end
-    it "should export non vote data to a csv file" do 
-      filename = @aoi_question.export('non_votes')
-
-      filename.should_not be nil
-      filename.should match /.*ideamarketplace_#{@aoi_question.id}_non_votes[.]csv$/
-      File.exists?(filename).should be_true
+    it "should export non vote data to a string" do 
+      csv = @aoi_question.export('non_votes')
-      # Not specifying exact file syntax, it's likely to change frequently
-      #
-      rows = FasterCSV.read(filename)
+      rows = FasterCSV.parse(csv)
       rows.first.should include("Record ID")
       rows.first.should include("Record Type")
       rows.first.should_not include("Idea ID")
       # ensure we have more than just the header row
       rows.length.should be > 1
-      File.delete(filename).should_not be_nil 
-
-
     end
-    it "should export idea data to a csv file" do
-      filename = @aoi_question.export('ideas')
+    it "should export idea data to a string" do
+      csv = @aoi_question.export('ideas')
-      filename.should_not be nil
-      filename.should match /.*ideamarketplace_#{@aoi_question.id}_ideas[.]csv$/
-      File.exists?(filename).should be_true
       # Not specifying exact file syntax, it's likely to change frequently
       #
-      rows = FasterCSV.read(filename)
+      rows = FasterCSV.parse(csv)
       rows.first.should include("Idea ID")
       rows.first.should_not include("Skip ID")
-      File.delete(filename).should_not be_nil
-
     end
     it "should raise an error when given an unsupported export type" do
       lambda { @aoi_question.export("blahblahblah") }.should raise_error
     end
-    it "should export data and schedule a job to delete export after X days" do
-      Delayed::Job.delete_all
-      filename = @aoi_question.export_and_delete('votes', :delete_at => 2.days.from_now)
-
-      Delayed::Job.count.should == 1
-      Delayed::Job.delete_all
-      File.delete(filename).should_not be_nil
-
-    end
-
     after(:all) { truncate_all }
   end
@@ -499,15 +473,12 @@ describe Question do
       end
     end
-    it "should export idea data to a csv file with proper escaping" do
-      filename = @aoi_question.export('ideas')
+    it "should export idea data to a string with proper escaping" do
+      csv = @aoi_question.export('ideas')
-      filename.should_not be nil
-      filename.should match /.*ideamarketplace_#{@aoi_question.id}_ideas[.]csv$/
-      File.exists?(filename).should be_true
       # Not specifying exact file syntax, it's likely to change frequently
       #
-      rows = FasterCSV.read(filename)
+      rows = FasterCSV.parse(csv)
       rows.first.should include("Idea ID")
       rows.first.should_not include("Skip ID")
@@ -516,20 +487,14 @@ describe Question do
         # Idea Text
         row[2].should =~ /^foo.bar$/m
       end
-      
-      File.delete(filename).should_not be_nil
-
     end
-    it "should export vote data to a csv file with proper escaping" do
-      filename = @aoi_question.export('votes')
+    it "should export vote data to a string with proper escaping" do
+      csv = @aoi_question.export('votes')
-      filename.should_not be nil
-      filename.should match /.*ideamarketplace_#{@aoi_question.id}_votes[.]csv$/
-      File.exists?(filename).should be_true
       # Not specifying exact file syntax, it's likely to change frequently
       #
-      rows = FasterCSV.read(filename)
+      rows = FasterCSV.parse(csv)
       rows.first.should include("Vote ID")
       rows.first.should_not include("Idea ID")
@@ -540,7 +505,6 @@ describe Question do
         # Loser Text
         row[6].should =~ /^foo.bar$/m
       end
-      File.delete(filename).should be_true
     end
	@@ -73,8 +73,8 @@ class QuestionsController < InheritedResources::Base		@@ -73,8 +73,8 @@ class QuestionsController < InheritedResources::Base
73		73
74	# puts "redis key is::::: #{redis_key}"	74	# puts "redis key is::::: #{redis_key}"
75		75
76	- @question.send_later :export_and_delete, type,
77	- :response_type => response_type, :redis_key => redis_key, :delete_at => 3.days.from_now	76	+ @question.send_later :export, type,
		77	+ :response_type => response_type, :redis_key => redis_key
78		78
79		79
80	render :text => "Ok! Please wait for the response (as specified by your response_type)"	80	render :text => "Ok! Please wait for the response (as specified by your response_type)"