diff --git a/app/controllers/questions_controller.rb b/app/controllers/questions_controller.rb index 77b4eb8..04f8696 100644 --- a/app/controllers/questions_controller.rb +++ b/app/controllers/questions_controller.rb @@ -73,8 +73,8 @@ class QuestionsController < InheritedResources::Base # puts "redis key is::::: #{redis_key}" - @question.send_later :export_and_delete, type, - :response_type => response_type, :redis_key => redis_key, :delete_at => 3.days.from_now + @question.send_later :export, type, + :response_type => response_type, :redis_key => redis_key render :text => "Ok! Please wait for the response (as specified by your response_type)" diff --git a/app/models/question.rb b/app/models/question.rb index d17a6a4..313ff19 100644 --- a/app/models/question.rb +++ b/app/models/question.rb @@ -512,36 +512,27 @@ class Question < ActiveRecord::Base end - def export_and_delete(type, options={}) - delete_at = options.delete(:delete_at) - filename = export(type, options) - - File.send_at(delete_at, :delete, filename) - filename - end - def export(type, options = {}) case type when 'votes' - outfile = "ideamarketplace_#{self.id}_votes.csv" + outfile = "ideamarketplace_#{self.id}_votes" headers = ['Vote ID', 'Session ID', 'Ideamarketplace ID','Winner ID', 'Winner Text', 'Loser ID', 'Loser Text', 'Prompt ID', 'Left Choice ID', 'Right Choice ID', 'Created at', 'Updated at', 'Response Time (s)', 'Missing Response Time Explanation', 'Session Identifier', 'Valid'] when 'ideas' - outfile = "ideamarketplace_#{self.id}_ideas.csv" + outfile = "ideamarketplace_#{self.id}_ideas" headers = ['Ideamarketplace ID','Idea ID', 'Idea Text', 'Wins', 'Losses', 'Times involved in Cant Decide', 'Score', 'User Submitted', 'Session ID', 'Created at', 'Last Activity', 'Active', 'Appearances on Left', 'Appearances on Right'] when 'non_votes' - outfile = "ideamarketplace_#{self.id}_non_votes.csv" + outfile = "ideamarketplace_#{self.id}_non_votes" headers = ['Record Type', 'Record ID', 'Session ID', 'Ideamarketplace ID','Left Choice ID', 'Left Choice Text', 'Right Choice ID', 'Right Choice Text', 'Prompt ID', 'Reason', 'Created at', 'Updated at', 'Response Time (s)', 'Missing Response Time Explanation', 'Session Identifier', 'Valid'] else raise "Unsupported export type: #{type}" end - filename = File.join(File.expand_path(Rails.root), "public", "system", "exports", self.id.to_s, Digest::SHA1.hexdigest(outfile + rand(10000000).to_s) + "_" + outfile) + name = outfile + "_" + Digest::SHA1.hexdigest(outfile + rand(10000000).to_s) + ".csv" - FileUtils::mkdir_p(File.dirname(filename)) - csv_data = FasterCSV.open(filename, "w") do |csv| + csv_data = FasterCSV.generate do |csv| csv << headers case type when 'votes' @@ -600,18 +591,23 @@ class Question < ActiveRecord::Base end + if options[:response_type] == 'redis' + # let's compress this for redis + # it should get removed from redis relatively quickly + zlib = Zlib::Deflate.new + zlibcsv = zlib.deflate(csv_data, Zlib::FINISH) + zlib.close if options[:redis_key].nil? raise "No :redis_key specified" end #The client should use blpop to listen for a key #The client is responsible for deleting the redis key (auto expiration results failure in testing) - $redis.lpush(options[:redis_key], filename) - #TODO implement response_type == 'email' for use by customers of the API (not local) + $redis.lpush(options[:redis_key], zlibcsv) + else + return csv_data end - - filename end def get_first_unanswered_appearance(visitor, offset=0) diff --git a/spec/models/question_spec.rb b/spec/models/question_spec.rb index ce79d0c..fc0c168 100644 --- a/spec/models/question_spec.rb +++ b/spec/models/question_spec.rb @@ -365,86 +365,60 @@ describe Question do it "should export vote data to a csv file" do - filename = @aoi_question.export('votes') + csv = @aoi_question.export('votes') - filename.should_not be nil - filename.should match /.*ideamarketplace_#{@aoi_question.id}_votes[.]csv$/ - File.exists?(filename).should be_true # Not specifying exact file syntax, it's likely to change frequently # - rows = FasterCSV.read(filename) + rows = FasterCSV.parse(csv) rows.first.should include("Vote ID") rows.first.should_not include("Idea ID") - File.delete(filename).should be_true end - it "should notify redis after completing an export, if redis option set" do + it "should export zlibed csv to redis after completing an export, if redis option set" do redis_key = "test_key123" $redis.del(redis_key) # clear if key exists already - filename = @aoi_question.export('votes', :response_type => 'redis', :redis_key => redis_key) - - filename.should_not be nil - filename.should match /.*ideamarketplace_#{@aoi_question.id}_votes[.]csv$/ - File.exists?(filename).should be_true - $redis.lpop(redis_key).should == filename + csv = @aoi_question.export('votes') + @aoi_question.export('votes', :response_type => 'redis', :redis_key => redis_key) + + zlibcsv = $redis.lpop(redis_key) + zstream = Zlib::Inflate.new + buf = zstream.inflate(zlibcsv) + zstream.finish + zstream.close + buf.should == csv $redis.del(redis_key) # clean up - File.delete(filename).should be_true end it "should email question owner after completing an export, if email option set" do #TODO end - it "should export non vote data to a csv file" do - filename = @aoi_question.export('non_votes') - - filename.should_not be nil - filename.should match /.*ideamarketplace_#{@aoi_question.id}_non_votes[.]csv$/ - File.exists?(filename).should be_true + it "should export non vote data to a string" do + csv = @aoi_question.export('non_votes') - # Not specifying exact file syntax, it's likely to change frequently - # - rows = FasterCSV.read(filename) + rows = FasterCSV.parse(csv) rows.first.should include("Record ID") rows.first.should include("Record Type") rows.first.should_not include("Idea ID") # ensure we have more than just the header row rows.length.should be > 1 - File.delete(filename).should_not be_nil - - end - it "should export idea data to a csv file" do - filename = @aoi_question.export('ideas') + it "should export idea data to a string" do + csv = @aoi_question.export('ideas') - filename.should_not be nil - filename.should match /.*ideamarketplace_#{@aoi_question.id}_ideas[.]csv$/ - File.exists?(filename).should be_true # Not specifying exact file syntax, it's likely to change frequently # - rows = FasterCSV.read(filename) + rows = FasterCSV.parse(csv) rows.first.should include("Idea ID") rows.first.should_not include("Skip ID") - File.delete(filename).should_not be_nil - end it "should raise an error when given an unsupported export type" do lambda { @aoi_question.export("blahblahblah") }.should raise_error end - it "should export data and schedule a job to delete export after X days" do - Delayed::Job.delete_all - filename = @aoi_question.export_and_delete('votes', :delete_at => 2.days.from_now) - - Delayed::Job.count.should == 1 - Delayed::Job.delete_all - File.delete(filename).should_not be_nil - - end - after(:all) { truncate_all } end @@ -499,15 +473,12 @@ describe Question do end end - it "should export idea data to a csv file with proper escaping" do - filename = @aoi_question.export('ideas') + it "should export idea data to a string with proper escaping" do + csv = @aoi_question.export('ideas') - filename.should_not be nil - filename.should match /.*ideamarketplace_#{@aoi_question.id}_ideas[.]csv$/ - File.exists?(filename).should be_true # Not specifying exact file syntax, it's likely to change frequently # - rows = FasterCSV.read(filename) + rows = FasterCSV.parse(csv) rows.first.should include("Idea ID") rows.first.should_not include("Skip ID") @@ -516,20 +487,14 @@ describe Question do # Idea Text row[2].should =~ /^foo.bar$/m end - - File.delete(filename).should_not be_nil - end - it "should export vote data to a csv file with proper escaping" do - filename = @aoi_question.export('votes') + it "should export vote data to a string with proper escaping" do + csv = @aoi_question.export('votes') - filename.should_not be nil - filename.should match /.*ideamarketplace_#{@aoi_question.id}_votes[.]csv$/ - File.exists?(filename).should be_true # Not specifying exact file syntax, it's likely to change frequently # - rows = FasterCSV.read(filename) + rows = FasterCSV.parse(csv) rows.first.should include("Vote ID") rows.first.should_not include("Idea ID") @@ -540,7 +505,6 @@ describe Question do # Loser Text row[6].should =~ /^foo.bar$/m end - File.delete(filename).should be_true end -- libgit2 0.21.2