From 0e03ea330c6dfb37a06256e9e1debb093876e57c Mon Sep 17 00:00:00 2001 From: Dhruv Kapadia Date: Thu, 6 May 2010 12:05:37 -0400 Subject: [PATCH] Refactoring data export to run in background, notify redis on completion --- app/controllers/questions_controller.rb | 73 +++++++++++++++++-------------------------------------------------------- app/models/prompt.rb | 1 + app/models/question.rb | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ db/migrate/20100505235409_add_choice_indexes_to_prompts.rb | 11 +++++++++++ db/migrate/20100506003102_add_indexes_to_skips.rb | 11 +++++++++++ spec/models/question_spec.rb | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 221 insertions(+), 58 deletions(-) create mode 100644 db/migrate/20100505235409_add_choice_indexes_to_prompts.rb create mode 100644 db/migrate/20100506003102_add_indexes_to_skips.rb diff --git a/app/controllers/questions_controller.rb b/app/controllers/questions_controller.rb index 689f5f7..8e5e97b 100644 --- a/app/controllers/questions_controller.rb +++ b/app/controllers/questions_controller.rb @@ -155,14 +155,26 @@ class QuestionsController < InheritedResources::Base end def export type = params[:type] + response_type = params[:response_type] - if type == 'votes' - export_votes - elsif type == 'items' - export_items + if response_type == 'redis' + redis_key = params[:redis_key] else - render :text => "Error! Specify a type of export" + render :text => "Error! The only export type supported currently is local through redis!" and return end + + if type.nil? + render :text => "Error! Specify a type of export" and return + end + + @question = current_user.questions.find(params[:id]) + + @question.send_later :export_and_delete, type, + :response_type => response_type, :redis_key => redis_key, :delete_at => 3.days.from_now + + + render :text => "Ok! Please wait for the response (as specified by your response_type)" + # export_type = params[:export_type] # export_format = params[:export_format] #CSV always now, could expand to xml later end @@ -326,57 +338,6 @@ class QuestionsController < InheritedResources::Base end protected - def export_votes - @question = Question.find(params[:id]) - - outfile = "ideamarketplace_#{@question.id}_votes" + Time.now.strftime("%m-%d-%Y") + ".csv" - headers = ['Vote ID', 'Session ID', 'Question ID','Winner ID', 'Winner Text', 'Loser ID', 'Loser Text', - 'Prompt ID', 'Left Choice ID', 'Right Choice ID', 'Created at', 'Updated at'] - @votes = @question.votes - csv_data = FasterCSV.generate do |csv| - csv << headers - @votes.each do |v| - prompt = v.prompt - # these may not exist - loser_data = v.loser_choice.nil? ? "" : "'#{v.loser_choice.data.strip}'" - left_id = v.prompt.nil? ? "" : v.prompt.left_choice_id - right_id = v.prompt.nil? ? "" : v.prompt.right_choice_id - - csv << [ v.id, v.voter_id, v.question_id, v.choice_id, "\'#{v.choice.data.strip}'", v.loser_choice_id, loser_data, - v.prompt_id, left_id, right_id, v.created_at, v.updated_at] - end - end - - send_data(csv_data, - :type => 'text/csv; charset=iso-8859-1; header=present', - :disposition => "attachment; filename=#{outfile}") - end - - def export_items - @question = Question.find(params[:id], :include => [:choices, :prompts]) - - outfile = "ideamarketplace_#{@question.id}_ideas_" + Time.now.strftime("%m-%d-%Y") + ".csv" - headers = ['Ideamarketplace ID','Idea ID', 'Idea Text', 'Wins', 'Losses', 'Score','User Submitted', 'Idea Creator ID', - 'Created at', 'Last Activity', 'Active', 'Local Identifier', - 'Prompts on Left', 'Prompts on Right', 'Prompts Count'] - - csv_data = FasterCSV.generate do |csv| - csv << headers - - #ensure capital format for true and false - @question.choices.each do |c| - user_submitted = (c.item.creator != @question.creator) ? "TRUE" : "FALSE" - - csv << [c.question_id, c.id, "'#{c.data.strip}'", c.wins, c.losses, c.score, user_submitted , c.item.creator_id, - c.created_at, c.updated_at, c.active, c.local_identifier, - c.prompts_on_the_left(true).size, c.prompts_on_the_right(true).size, c.prompts_count] - end - end - - send_data(csv_data, - :type => 'text/csv; charset=iso-8859-1; header=present', - :disposition => "attachment; filename=#{outfile}") - end end class String diff --git a/app/models/prompt.rb b/app/models/prompt.rb index 041a73d..506a1b0 100644 --- a/app/models/prompt.rb +++ b/app/models/prompt.rb @@ -18,6 +18,7 @@ class Prompt < ActiveRecord::Base #named_scope :voted_on_by, proc {|u| { :conditions => { :methodology => methodology } } } named_scope :active, :include => [:left_choice, :right_choice], :conditions => { 'left_choice.active' => true, 'right_choice.active' => true } + named_scope :ids_only, :select => 'id' def self.voted_on_by(u) diff --git a/app/models/question.rb b/app/models/question.rb index 208f8e5..e409130 100644 --- a/app/models/question.rb +++ b/app/models/question.rb @@ -17,6 +17,7 @@ class Question < ActiveRecord::Base end end has_many :votes + has_many :skips has_many :densities has_many :appearances @@ -334,6 +335,98 @@ class Question < ActiveRecord::Base $redis.get(self.pq_key + "_" + date.to_s + "_"+ "hits") end + def export_and_delete(type, options={}) + delete_at = options.delete(:delete_at) + filename = export(type, options) + + File.send_at(delete_at, :delete, filename) + filename + end + + def export(type, options = {}) + + case type + when 'votes' + outfile = "ideamarketplace_#{self.id}_votes.csv" + + headers = ['Vote ID', 'Session ID', 'Question ID','Winner ID', 'Winner Text', 'Loser ID', 'Loser Text', + 'Prompt ID', 'Left Choice ID', 'Right Choice ID', 'Created at', 'Updated at', 'Response Time (ms)'] + + when 'ideas' + outfile = "ideamarketplace_#{self.id}_ideas.csv" + headers = ['Ideamarketplace ID','Idea ID', 'Idea Text', 'Wins', 'Losses', 'Times involved in Cant Decide', 'Score', + 'User Submitted', 'Idea Creator ID', 'Created at', 'Last Activity', 'Active', 'Local Identifier', + 'Appearances on Left', 'Appearances on Right'] + when 'skips' + outfile = "ideamarketplace_#{self.id}_skips.csv" + headers = ['Skip ID', 'Session ID', 'Question ID','Left Choice ID', 'Left Choice Text', + 'Right Choice ID', 'Right Choice Text', 'Prompt ID', 'Appearance ID', 'Reason', + 'Created at', 'Updated at', 'Response Time (ms)'] + else + raise "Unsupported export type: #{type}" + end + + filename = File.join(File.expand_path(Rails.root), "public", "system", "exports", + self.id.to_s, Digest::SHA1.hexdigest(outfile + rand(10000000).to_s) + "_" + outfile) + + FileUtils::mkdir_p(File.dirname(filename)) + csv_data = FasterCSV.open(filename, "w") do |csv| + csv << headers + + case type + when 'votes' + + self.votes.find_each(:include => [:prompt, :choice, :loser_choice]) do |v| + prompt = v.prompt + # these may not exist + loser_data = v.loser_choice.nil? ? "" : "'#{v.loser_choice.data.strip}'" + left_id = v.prompt.nil? ? "" : v.prompt.left_choice_id + right_id = v.prompt.nil? ? "" : v.prompt.right_choice_id + + csv << [ v.id, v.voter_id, v.question_id, v.choice_id, "\'#{v.choice.data.strip}'", v.loser_choice_id, loser_data, + v.prompt_id, left_id, right_id, v.created_at, v.updated_at, v.time_viewed] + end + + when 'ideas' + self.choices.each do |c| + user_submitted = c.user_created ? "TRUE" : "FALSE" + left_prompts_ids = c.prompts_on_the_left.ids_only + right_prompts_ids = c.prompts_on_the_right.ids_only + + left_appearances = self.appearances.count(:conditions => {:prompt_id => left_prompts_ids}) + right_appearances = self.appearances.count(:conditions => {:prompt_id => right_prompts_ids}) + + num_skips = self.skips.count(:conditions => {:prompt_id => left_prompts_ids + right_prompts_ids}) + + csv << [c.question_id, c.id, "'#{c.data.strip}'", c.wins, c.losses, num_skips, c.score, + user_submitted , c.item.creator_id, c.created_at, c.updated_at, c.active, c.local_identifier, + left_appearances, right_appearances] + end + when 'skips' + + self.skips.find_each(:include => :prompt) do |s| + prompt = s.prompt + csv << [ s.id, s.skipper_id, s.question_id, s.prompt.left_choice.id, s.prompt.left_choice.data.strip, + s.prompt.right_choice.id, s.prompt.right_choice.data.strip, s.prompt_id, s.appearance_id, s.skip_reason, + s.created_at, s.updated_at, s.time_viewed] + end + end + + end + + if options[:response_type] == 'redis' + + if options[:redis_key].nil? + raise "No :redis_key specified" + end + + $redis.lpush(options[:redis_key], filename) + #TODO implement response_type == 'email' for use by customers of the API (not local) + end + + filename + end + end diff --git a/db/migrate/20100505235409_add_choice_indexes_to_prompts.rb b/db/migrate/20100505235409_add_choice_indexes_to_prompts.rb new file mode 100644 index 0000000..2687fdb --- /dev/null +++ b/db/migrate/20100505235409_add_choice_indexes_to_prompts.rb @@ -0,0 +1,11 @@ +class AddChoiceIndexesToPrompts < ActiveRecord::Migration + def self.up + add_index :prompts, :left_choice_id + add_index :prompts, :right_choice_id + end + + def self.down + remove_index :prompts, :left_choice_id + remove_index :prompts, :right_choice_id + end +end diff --git a/db/migrate/20100506003102_add_indexes_to_skips.rb b/db/migrate/20100506003102_add_indexes_to_skips.rb new file mode 100644 index 0000000..301dec6 --- /dev/null +++ b/db/migrate/20100506003102_add_indexes_to_skips.rb @@ -0,0 +1,11 @@ +class AddIndexesToSkips < ActiveRecord::Migration + def self.up + add_index :skips, :question_id + add_index :skips, :prompt_id + end + + def self.down + remove_index :skips, :question_id + remove_index :skips, :prompt_id + end +end diff --git a/spec/models/question_spec.rb b/spec/models/question_spec.rb index 2168db5..f461c27 100644 --- a/spec/models/question_spec.rb +++ b/spec/models/question_spec.rb @@ -51,6 +51,7 @@ describe Question do prompt = q.catchup_choose_prompt prompt.active?.should == true end + context "catchup algorithm" do before(:all) do @@ -127,10 +128,95 @@ describe Question do @catchup_q.pop_prompt_queue.should == nil end + end + + context "exporting data" do + before(:each) do + user = Factory.create(:user) + @question = Factory.create(:aoi_question, :site => user, :creator => user.default_visitor) + + end + + + it "should export vote data to a csv file" do + filename = @question.export('votes') + + filename.should_not be nil + filename.should match /.*ideamarketplace_#{@question.id}_votes[.]csv$/ + File.exists?(filename).should be_true + # Not specifying exact file syntax, it's likely to change frequently + # + rows = FasterCSV.read(filename) + rows.first.should include("Vote ID") + rows.first.should_not include("Idea ID") + File.delete(filename).should be_true + + end + + it "should notify redis after completing an export, if redis option set" do + redis_key = "test_key123" + $redis.del(redis_key) # clear if key exists already + filename = @question.export('votes', :response_type => 'redis', :redis_key => redis_key) + + filename.should_not be nil + filename.should match /.*ideamarketplace_#{@question.id}_votes[.]csv$/ + File.exists?(filename).should be_true + $redis.lpop(redis_key).should == filename + + # Not specifying exact file syntax, it's likely to change frequently + $redis.del(redis_key) # clean up + + end + it "should email question owner after completing an export, if email option set" do + #TODO + end + it "should export skip data to a csv file" do + filename = @question.export('skips') + + filename.should_not be nil + filename.should match /.*ideamarketplace_#{@question.id}_skips[.]csv$/ + File.exists?(filename).should be_true + + # Not specifying exact file syntax, it's likely to change frequently + # + rows = FasterCSV.read(filename) + rows.first.should include("Skip ID") + rows.first.should_not include("Idea ID") + File.delete(filename).should_not be_nil + + + end + + it "should export idea data to a csv file" do + filename = @question.export('ideas') + + filename.should_not be nil + filename.should match /.*ideamarketplace_#{@question.id}_ideas[.]csv$/ + File.exists?(filename).should be_true + # Not specifying exact file syntax, it's likely to change frequently + # + rows = FasterCSV.read(filename) + rows.first.should include("Idea ID") + rows.first.should_not include("Skip ID") + File.delete(filename).should_not be_nil + + end + + it "should raise an error when given an unsupported export type" do + lambda { @question.export("blahblahblah") }.should raise_error + end + + it "should export data and schedule a job to delete export after X days" do + Delayed::Job.delete_all + filename = @question.export_and_delete('votes', :delete_at => 2.days.from_now) + + Delayed::Job.count.should == 1 + Delayed::Job.delete_all + File.delete(filename).should_not be_nil + + end end - - #q = @aoi_clone.create_question("foobarbaz", {:name => 'foo'}) end -- libgit2 0.21.2