From cdca25dccb5c9ab31e106283164e06411768327b Mon Sep 17 00:00:00 2001 From: Evandro Jr Date: Mon, 24 Nov 2014 19:57:21 -0300 Subject: [PATCH] Added last harvest for each dspace server --- plugins/virtuoso/controllers/virtuoso_plugin_admin_controller.rb | 7 ++++++- plugins/virtuoso/lib/virtuoso_plugin/dspace_harvest.rb | 60 +++++++++++++++++++++++++++++++++++++++++------------------- plugins/virtuoso/test/functional/virtuoso_plugin_admin_controller_test.rb | 19 +++++++++++-------- plugins/virtuoso/test/unit/dspace_harvest_test.rb | 30 ++++++++++++++++++++++++++---- plugins/virtuoso/views/virtuoso_plugin_admin/_server_list_item.html.erb | 11 +++++++++-- plugins/virtuoso/views/virtuoso_plugin_admin/index.html.erb | 18 ++++++------------ 6 files changed, 99 insertions(+), 46 deletions(-) diff --git a/plugins/virtuoso/controllers/virtuoso_plugin_admin_controller.rb b/plugins/virtuoso/controllers/virtuoso_plugin_admin_controller.rb index ca5460b..358d0a2 100644 --- a/plugins/virtuoso/controllers/virtuoso_plugin_admin_controller.rb +++ b/plugins/virtuoso/controllers/virtuoso_plugin_admin_controller.rb @@ -4,7 +4,12 @@ class VirtuosoPluginAdminController < AdminController settings = params[:settings] settings ||= {} @settings = Noosfero::Plugin::Settings.new(environment, VirtuosoPlugin, settings) - @harvest_running = VirtuosoPlugin::DspaceHarvest.new(environment).find_job.present? + @harvest_running = {} + if @settings.dspace_servers.present? + @settings.dspace_servers.each do |i| + @harvest_running[i['dspace_uri']] = VirtuosoPlugin::DspaceHarvest.new(environment, i).find_job(i['dspace_uri']).present? + end + end if request.post? settings[:dspace_servers].delete_if do | server | server[:dspace_uri].empty? diff --git a/plugins/virtuoso/lib/virtuoso_plugin/dspace_harvest.rb b/plugins/virtuoso/lib/virtuoso_plugin/dspace_harvest.rb index 1619b27..8d43521 100644 --- a/plugins/virtuoso/lib/virtuoso_plugin/dspace_harvest.rb +++ b/plugins/virtuoso/lib/virtuoso_plugin/dspace_harvest.rb @@ -1,12 +1,24 @@ #inspired by https://github.com/code4lib/ruby-oai/blob/master/lib/oai/harvester/harvest.rb class VirtuosoPlugin::DspaceHarvest - def initialize(environment, dspace_uri = nil) + attr_reader :environment, :dspace_settings + + def initialize(environment, dspace_settings = nil) @environment = environment - @dspace_uri = dspace_uri + @dspace_settings = dspace_settings + end + + def dspace_uri + unless dspace_settings.nil? + dspace_settings["dspace_uri"] if dspace_settings.has_key?("dspace_uri") + end end - attr_reader :environment + def last_harvest + unless dspace_settings.nil? + dspace_settings["last_harvest"] if dspace_settings.has_key?("last_harvest") + end + end def plugin @plugin ||= VirtuosoPlugin.new(self) @@ -15,7 +27,7 @@ class VirtuosoPlugin::DspaceHarvest delegate :settings, :to => :plugin def dspace_client - @dspace_client ||= OAI::Client.new("#{@dspace_uri}/oai/request") + @dspace_client ||= OAI::Client.new("#{dspace_uri}/oai/request") end def triplify(record) @@ -26,7 +38,7 @@ class VirtuosoPlugin::DspaceHarvest settings.ontology_mapping.each do |mapping| values = [metadata.extract_field(mapping[:source])].flatten.compact values.each do |value| - query = RDF::Virtuoso::Query.insert_data([RDF::URI.new(subject_identifier), RDF::URI.new(mapping[:target]), value]).graph(RDF::URI.new(@dspace_uri)) + query = RDF::Virtuoso::Query.insert_data([RDF::URI.new(subject_identifier), RDF::URI.new(mapping[:target]), value]).graph(RDF::URI.new(dspace_uri)) plugin.virtuoso_client.insert(query) end end @@ -34,8 +46,8 @@ class VirtuosoPlugin::DspaceHarvest def run harvest_time = Time.now.utc - params = settings.last_harvest ? {:from => settings.last_harvest.utc} : {} - puts "starting harvest #{params} #{@dspace_uri} #{settings.virtuoso_uri}" + params = last_harvest ? {:from => last_harvest.utc} : {} + puts "starting harvest #{params} #{dspace_uri} #{settings.virtuoso_uri}" begin records = dspace_client.list_records(params) records.each do |record| @@ -48,16 +60,26 @@ class VirtuosoPlugin::DspaceHarvest raise ex end end - settings.last_harvest = harvest_time - settings.save! puts "ending harvest #{harvest_time}" end + def save_harvest_time_settings(harvest_time) + dspace_settings = {"dspace_uri" => dspace_uri, "last_harvest" => last_harvest} + settings.dspace_servers.each do |s| + if s["dspace_uri"] == dspace_uri + settings.dspace_servers.delete(dspace_settings) + end + end + @dspace_settings = {"dspace_uri" => dspace_uri, "last_harvest" => harvest_time} + settings.dspace_servers << @dspace_settings + settings.save! + end + def self.harvest_all(environment, from_start) settings = Noosfero::Plugin::Settings.new(environment, VirtuosoPlugin) if settings.dspace_servers.present? - settings.dspace_servers.each do |k, v| - harvest = VirtuosoPlugin::DspaceHarvest.new(environment, k[:dspace_uri]) + settings.dspace_servers.each do |i| + harvest = VirtuosoPlugin::DspaceHarvest.new(environment, i) harvest.start(from_start) end end @@ -66,22 +88,22 @@ class VirtuosoPlugin::DspaceHarvest def start(from_start = false) if find_job.empty? if from_start - settings.last_harvest = nil - settings.save! + save_harvest_time_settings(nil) end - job = VirtuosoPlugin::DspaceHarvest::Job.new(@environment.id, @dspace_uri) + job = VirtuosoPlugin::DspaceHarvest::Job.new(@environment.id, dspace_uri) Delayed::Job.enqueue(job) end end - def find_job - Delayed::Job.where(:handler => "--- !ruby/struct:VirtuosoPlugin::DspaceHarvest::Job\nenvironment_id: #{@environment.id}\ndspace_uri: #{@dspace_uri}\n") + def find_job(_dspace_uri=nil) + _dspace_uri ||= dspace_uri + Delayed::Job.where(:handler => "--- !ruby/struct:VirtuosoPlugin::DspaceHarvest::Job\nenvironment_id: #{@environment.id}\ndspace_uri: #{_dspace_uri}\n") end class Job < Struct.new(:environment_id, :dspace_uri) def perform - environment = Environment.find(environment_id) - harvest = VirtuosoPlugin::DspaceHarvest.new(environment, dspace_uri) + environment = Environment.find(environment_id, dspace_uri) + harvest = VirtuosoPlugin::DspaceHarvest.new(environment, {"dspace_uri" => dspace_uri, "last_harvest" => last_harvest}) harvest.run end end @@ -90,7 +112,7 @@ class VirtuosoPlugin::DspaceHarvest def extract_identifier(record) parsed_identifier = /oai:(.+):(\d+\/\d+)/.match(record.header.identifier) - "#{@dspace_uri}/handle/#{parsed_identifier[2]}" + "#{dspace_uri}/handle/#{parsed_identifier[2]}" end end diff --git a/plugins/virtuoso/test/functional/virtuoso_plugin_admin_controller_test.rb b/plugins/virtuoso/test/functional/virtuoso_plugin_admin_controller_test.rb index fc19536..57063cd 100644 --- a/plugins/virtuoso/test/functional/virtuoso_plugin_admin_controller_test.rb +++ b/plugins/virtuoso/test/functional/virtuoso_plugin_admin_controller_test.rb @@ -17,9 +17,11 @@ class VirtuosoPluginAdminControllerTest < ActionController::TestCase :virtuoso_readonly_username=>"readonly_username", :virtuoso_readonly_password=>"readonly_password", :dspace_servers=>[ - {"dspace_uri"=>"http://dspace1.noosfero.com"}, + {"dspace_uri"=>"http://dspace1.noosfero.com","last_harvest" => 5 }, {"dspace_uri"=>"http://dspace2.noosfero.com"}, - {"dspace_uri"=>"http://dspace3.noosfero.com"} + {"dspace_uri"=>"http://dspace3.noosfero.com", "last_harvest" => 0}, + {"dspace_uri"=>"http://dspace4.noosfero.com", "last_harvest" => nil}, + {"dspace_uri"=>"http://dspace5.noosfero.com", "last_harvest" => 9}, ] } end @@ -35,6 +37,7 @@ class VirtuosoPluginAdminControllerTest < ActionController::TestCase assert_equal 'http://dspace1.noosfero.com', @settings.settings[:dspace_servers][0][:dspace_uri] assert_equal 'http://dspace2.noosfero.com', @settings.settings[:dspace_servers][1][:dspace_uri] assert_equal 'http://dspace3.noosfero.com', @settings.settings[:dspace_servers][2][:dspace_uri] + assert_equal "9", @settings.settings[:dspace_servers][4][:last_harvest] assert_redirected_to :action => 'index' end @@ -45,7 +48,7 @@ class VirtuosoPluginAdminControllerTest < ActionController::TestCase should 'create delayed job to start harvest on force action' do post :index, :settings => mock_settings - harvest = VirtuosoPlugin::DspaceHarvest.new(environment, "http://dspace1.noosfero.com") + harvest = VirtuosoPlugin::DspaceHarvest.new(environment, {"dspace_uri"=>"http://dspace1.noosfero.com", "last_harvest" => 5 }) assert !harvest.find_job.present? get :force_harvest assert harvest.find_job.present? @@ -54,17 +57,17 @@ class VirtuosoPluginAdminControllerTest < ActionController::TestCase should 'force harvest from start' do post :index, :settings => mock_settings get :force_harvest, :from_start => true - harvest = VirtuosoPlugin::DspaceHarvest.new(environment, "http://dspace2.noosfero.com") + harvest = VirtuosoPlugin::DspaceHarvest.new(environment, {"dspace_uri"=>"http://dspace4.noosfero.com", "last_harvest" => nil}) assert harvest.find_job.present? assert_equal nil, harvest.settings.last_harvest end - + should 'not create delayed job to start harvest on force action without settings' do post :index, :settings => mock_settings - harvest = VirtuosoPlugin::DspaceHarvest.new(environment, "http://dspace8.noosfero.com") + harvest = VirtuosoPlugin::DspaceHarvest.new(environment) assert !harvest.find_job.present?, "testing if no job is running" get :force_harvest - assert !harvest.find_job.present?, "testing if no job is running again" + assert !harvest.find_job.present?, "testing if no job is running again" end - + end diff --git a/plugins/virtuoso/test/unit/dspace_harvest_test.rb b/plugins/virtuoso/test/unit/dspace_harvest_test.rb index feb2a0d..445cebb 100644 --- a/plugins/virtuoso/test/unit/dspace_harvest_test.rb +++ b/plugins/virtuoso/test/unit/dspace_harvest_test.rb @@ -15,17 +15,39 @@ class DspaceHarvestTest < ActiveSupport::TestCase :virtuoso_readonly_username=>"readonly_username", :virtuoso_readonly_password=>"readonly_password", :dspace_servers=>[ - {"dspace_uri"=>"http://dspace1.noosfero.com"}, + {"dspace_uri"=>"http://dspace1.noosfero.com","last_harvest" => 5 }, {"dspace_uri"=>"http://dspace2.noosfero.com"}, - {"dspace_uri"=>"http://dspace3.noosfero.com"} + {"dspace_uri"=>"http://dspace3.noosfero.com", "last_harvest" => 0}, + {"dspace_uri"=>"http://dspace4.noosfero.com", "last_harvest" => nil}, + {"dspace_uri"=>"http://dspace5.noosfero.com", "last_harvest" => 9}, ] } end + should 'initialize with dspace_uri' do + harvest = VirtuosoPlugin::DspaceHarvest.new(environment, {"dspace_uri"=>"http://dspace1.noosfero.com"}) + assert harvest.dspace_uri, "http://dspace1.noosfero.com" + end + + should 'initialize with dspace_uri and last_harvest' do + harvest = VirtuosoPlugin::DspaceHarvest.new(environment, {"dspace_uri"=>"http://dspace9.noosfero.com", "last_harvest" => 5}) + assert harvest.dspace_uri, "http://dspace9.noosfero.com" + assert harvest.last_harvest, 5 + end + + should 'save_harvest_time_settings' do + @settings = Noosfero::Plugin::Settings.new(environment, VirtuosoPlugin, mock_settings) + harvest = VirtuosoPlugin::DspaceHarvest.new(environment, {"dspace_uri"=>"http://dspace5.noosfero.com", "last_harvest" => 9}) + assert harvest.last_harvest, 9 + harvest.save_harvest_time_settings(10) + @settings = Noosfero::Plugin::Settings.new(environment.reload, VirtuosoPlugin) + assert harvest.last_harvest, 10 + end + should 'create delayed job when start' do @settings = Noosfero::Plugin::Settings.new(environment, VirtuosoPlugin, mock_settings) @settings.save! - harvest = VirtuosoPlugin::DspaceHarvest.new(environment, "http://dspace1.noosfero.com") + harvest = VirtuosoPlugin::DspaceHarvest.new(environment, {"dspace_uri"=>"http://dspace1.noosfero.com", "last_harvest" => 5}) assert !harvest.find_job.present? harvest.start assert harvest.find_job.present? @@ -34,7 +56,7 @@ class DspaceHarvestTest < ActiveSupport::TestCase should 'not duplicate harvest job' do @settings = Noosfero::Plugin::Settings.new(environment, VirtuosoPlugin, mock_settings) @settings.save! - harvest = VirtuosoPlugin::DspaceHarvest.new(environment, "http://dspace1.noosfero.com") + harvest = VirtuosoPlugin::DspaceHarvest.new(environment, {"dspace_uri"=>"http://dspace1.noosfero.com", "last_harvest" => 5}) assert_difference "harvest.find_job.count", 1 do 5.times { harvest.start } end diff --git a/plugins/virtuoso/views/virtuoso_plugin_admin/_server_list_item.html.erb b/plugins/virtuoso/views/virtuoso_plugin_admin/_server_list_item.html.erb index 34c489d..d6b8a45 100644 --- a/plugins/virtuoso/views/virtuoso_plugin_admin/_server_list_item.html.erb +++ b/plugins/virtuoso/views/virtuoso_plugin_admin/_server_list_item.html.erb @@ -2,6 +2,13 @@ <% value = server_list_item[:dspace_uri] if server_list_item %> <%= text_field_tag 'settings[dspace_servers][][dspace_uri]', value, { :class => 'link-name', :maxlength => 60, :size=> 58 } %> <%= button_without_text(:delete, _('Delete'), "#" , :class=>"delete-server-list-row") %> -

+ <% last_execution = server_list_item[:last_harvest] if server_list_item + if last_execution %> +
+ <%= _('Last execution:') %> + <%= time_ago_as_sentence last_execution %> +
+ <% end %> +

- + diff --git a/plugins/virtuoso/views/virtuoso_plugin_admin/index.html.erb b/plugins/virtuoso/views/virtuoso_plugin_admin/index.html.erb index 00f2a7c..be33f5d 100644 --- a/plugins/virtuoso/views/virtuoso_plugin_admin/index.html.erb +++ b/plugins/virtuoso/views/virtuoso_plugin_admin/index.html.erb @@ -11,35 +11,29 @@ <%= labelled_form_field _('Virtuoso Read-Only Username:'), f.text_field(:virtuoso_readonly_username, :size=> 60) %> <%= labelled_form_field _('Virtuoso Read-Only Password:'), f.password_field(:virtuoso_readonly_password, :size=> 60) %> -
+
- + <%= _('Dspace Servers\' URL:') %>
<%= render :partial => 'server_list_item', :collection=>@settings.dspace_servers %>
-
+
<%= render :partial => 'server_list_item', :locals => {:dspace_uri => "http://"} %> -
+
<%= link_to_function(_('New Dspace Server'), 'add_new_server();', :class => 'button icon-add with-text') %>
-
+ <% button_bar do %> <%= submit_button(:save, _('Save'), :cancel => {:controller => 'plugins', :action => 'index'}) %> <% end %> <% end %>
- <% if @settings.last_harvest %> -
- <%= _('Last execution:') %> - <%= time_ago_as_sentence @settings.last_harvest %> -
-<% end %>
- <% if @harvest_running %> + <% if @harvest_running.has_value?(true) %> <%= _('Running...') %> <% else %> <%= button :next, _('Force harvest'), :action => :force_harvest %> -- libgit2 0.21.2