virtuoso: added a plugin for integration with virtuoso

Victor Costa
1 parent 9c61af94
Showing 9 changed files with 226 additions and 0 deletions Show diff stats
plugins/virtuoso/Gemfile
plugins/virtuoso/controllers/virtuoso_plugin_admin_controller.rb
plugins/virtuoso/lib/virtuoso_plugin.rb
plugins/virtuoso/lib/virtuoso_plugin/dspace_harvest.rb
plugins/virtuoso/lib/virtuoso_plugin/dublin_core_metadata.rb
plugins/virtuoso/test/functional/virtuoso_plugin_admin_controller_test.rb
plugins/virtuoso/test/test_helper.rb
plugins/virtuoso/test/unit/dspace_harvest_test.rb
plugins/virtuoso/views/virtuoso_plugin_admin/index.html.erb
@@ -0,0 +1,3 @@
+gem 'rdf'
+gem 'rdf-virtuoso'
+gem 'oai'
@@ -0,0 +1,23 @@
+class VirtuosoPluginAdminController < AdminController
+
+  def index
+    settings = params[:settings]
+    settings ||= {}
+    @settings = Noosfero::Plugin::Settings.new(environment, VirtuosoPlugin, settings)
+    @harvest_running = VirtuosoPlugin::DspaceHarvest.new(environment).find_job.present?
+
+    if request.post?
+      @settings.save!
+      session[:notice] = 'Settings succefully saved.'
+      redirect_to :action => 'index'
+    end
+  end
+
+  def force_harvest
+    harvest = VirtuosoPlugin::DspaceHarvest.new(environment)
+    harvest.start
+    session[:notice] = _('Harvest started')
+    redirect_to :action => :index
+  end
+
+end
@@ -0,0 +1,11 @@
+class VirtuosoPlugin < Noosfero::Plugin
+
+  def self.plugin_name
+    "Virtuoso integration"
+  end
+
+  def self.plugin_description
+    _('Virtuoso integration')
+  end
+
+end
@@ -0,0 +1,71 @@
+#inspired by https://github.com/code4lib/ruby-oai/blob/master/lib/oai/harvester/harvest.rb
+class VirtuosoPlugin::DspaceHarvest
+
+  DC_CONVERSION = [:title, :creator, :subject, :description, :date, :type, :identifier, :language, :rights, :format]
+
+  def initialize(environment)
+    @environment = environment
+  end
+
+  def settings
+    @settings ||= Noosfero::Plugin::Settings.new(@environment, VirtuosoPlugin)
+  end
+
+  def dspace_client
+    @dspace_client ||= OAI::Client.new("#{settings.dspace_uri}/oai/request")
+  end
+
+  def virtuoso_client
+    @virtuoso_client ||= RDF::Virtuoso::Repository.new("#{settings.virtuoso_uri}/sparql", :update_uri => "#{settings.virtuoso_uri}/sparql-auth", :username => settings.virtuoso_username, :password => settings.virtuoso_password, :auth_method => 'digest', :timeout => 30)
+  end
+
+  def triplify(record)
+    metadata = VirtuosoPlugin::DublinCoreMetadata.new(record.metadata)
+    puts "triplify #{record.header.identifier}"
+
+    DC_CONVERSION.each do |c|
+      values = [metadata.send(c)].flatten.compact
+      values.each do |value|
+        query = RDF::Virtuoso::Query.insert_data([RDF::URI.new(metadata.identifier), RDF::URI.new("http://purl.org/dc/elements/1.1/#{c}"), value]).graph(RDF::URI.new(settings.dspace_uri))
+        virtuoso_client.insert(query)
+      end
+    end
+  end
+
+  def run
+    harvest_time = Time.now.utc
+    params = settings.last_harvest ? {:from => settings.last_harvest.utc} : {}
+    puts "starting harvest #{params}"
+    begin
+      records = dspace_client.list_records(params)
+      records.each do |record|
+        triplify(record)
+      end
+    rescue Exception => ex
+      puts ex.to_s
+    end
+    settings.last_harvest = harvest_time
+    settings.save!
+    puts "ending harvest #{harvest_time}"
+  end
+
+  def start
+    if find_job.empty?
+      job = VirtuosoPlugin::DspaceHarvest::Job.new(@environment.id)
+      Delayed::Job.enqueue(job)
+    end
+  end
+
+  def find_job
+    Delayed::Job.where(:handler => "--- !ruby/struct:VirtuosoPlugin::DspaceHarvest::Job\nenvironment_id: #{@environment.id}\n")
+  end
+
+  class Job < Struct.new(:environment_id)
+    def perform
+      environment = Environment.find(environment_id)
+      harvest = VirtuosoPlugin::DspaceHarvest.new(environment)
+      harvest.run
+    end
+  end
+
+end
@@ -0,0 +1,20 @@
+class VirtuosoPlugin::DublinCoreMetadata
+
+  include OAI::XPath
+
+  attr_accessor :date, :title, :creator, :subject, :description, :date, :type, :identifier, :language, :rights, :format
+
+  def initialize(element)
+    @title = xpath(element, './/dc:title')
+    @creator = xpath(element, './/dc:creator')
+    @subject = xpath_all(element, './/dc:subject').map(&:text)
+    @description = xpath(element, './/dc:description')
+    @date = xpath_all(element, './/dc:date').map(&:text)
+    @type = xpath(element, './/dc:type')
+    @identifier = xpath(element, './/dc:identifier')
+    @language = xpath(element, './/dc:language')
+    @rights = xpath_all(element, './/dc:rights').map(&:text)
+    @format = xpath(element, './/dc:format')
+  end
+
+end
@@ -0,0 +1,38 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class VirtuosoPluginAdminControllerTest < ActionController::TestCase
+
+  def setup
+    @environment = Environment.default
+    @profile = create_user('profile').person
+    login_as(@profile.identifier)
+  end
+
+  attr_reader :environment
+
+  should 'save virtuoso plugin settings' do
+    post :index, :settings => {'virtuoso_uri' => 'http://virtuoso.noosfero.com',
+                               'virtuoso_username' => 'username',
+                               'virtuoso_password' => 'password',
+                               'dspace_uri' => 'http://dspace.noosfero.com'}
+    @settings = Noosfero::Plugin::Settings.new(environment.reload, VirtuosoPlugin)
+    assert_equal 'http://virtuoso.noosfero.com', @settings.settings[:virtuoso_uri]
+    assert_equal 'username', @settings.settings[:virtuoso_username]
+    assert_equal 'password', @settings.settings[:virtuoso_password]
+    assert_equal 'http://dspace.noosfero.com', @settings.settings[:dspace_uri]
+    assert_redirected_to :action => 'index'
+  end
+
+  should 'redirect to index after save' do
+    post :index, :settings => {"virtuoso_uri" => 'http://virtuoso.noosfero.com'}
+    assert_redirected_to :action => 'index'
+  end
+
+  should 'create delayed job to start harvest on force action' do
+    harvest = VirtuosoPlugin::DspaceHarvest.new(environment)
+    assert !harvest.find_job.present?
+    get :force_harvest
+    assert harvest.find_job.present?
+  end
+
+end
@@ -0,0 +1 @@
+require File.dirname(__FILE__) + '/../../../test/test_helper'
@@ -0,0 +1,25 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class DspaceHarvestTest < ActiveSupport::TestCase
+
+  def setup
+    @environment = Environment.default
+  end
+
+  attr_reader :environment
+
+  should 'create delayed job when start' do
+    harvest = VirtuosoPlugin::DspaceHarvest.new(environment)
+    assert !harvest.find_job.present?
+    harvest.start
+    assert harvest.find_job.present?
+  end
+
+  should 'not duplicate harvest job' do
+    harvest = VirtuosoPlugin::DspaceHarvest.new(environment)
+    assert_difference "harvest.find_job.count", 1 do
+      5.times { harvest.start }
+    end
+  end
+
+end
@@ -0,0 +1,34 @@
+<h1><%= _('Virtuoso settings')%></h1>
+
+<%= form_for(:settings) do |f| %>
+
+  <strong>
+    <%= labelled_form_field _('Virtuoso URL:'), f.text_field(:virtuoso_uri) %>
+    <%= labelled_form_field _('Virtuoso Username:'), f.text_field(:virtuoso_username) %>
+    <%= labelled_form_field _('Virtuoso Password:'), f.password_field(:virtuoso_password) %>
+    <%= labelled_form_field _('DSpace URL:'), f.text_field(:dspace_uri) %>
+  </strong>
+
+  <% button_bar do %>
+    <%= submit_button(:save, _('Save'), :cancel => {:controller => 'plugins', :action => 'index'}) %>
+  <% end %>
+
+<% end %>
+
+<hr/>
+<div class="harvest">
+  <% if @settings.last_harvest %>
+  <div class="date">
+    <span class="label"><strong><%= _('Last execution:') %></strong></span>
+    <span class="value"><%= time_ago_as_sentence @settings.last_harvest %></span>
+  </div>
+<% end %>
+  <br/>
+  <div class="actions">
+    <% if @harvest_running %>
+      <%= _('Running...') %>
+    <% else %>
+      <%= button :next, _('Force harvest'), :action => :force_harvest %>
+    <% end %>
+  </div>
+</div>
@@ -0,0 +1,3 @@		@@ -0,0 +1,3 @@
	1	+gem 'rdf'
	2	+gem 'rdf-virtuoso'
	3	+gem 'oai'
@@ -0,0 +1,23 @@		@@ -0,0 +1,23 @@
	1	+class VirtuosoPluginAdminController < AdminController
	2	+
	3	+ def index
	4	+ settings = params[:settings]
	5	+ settings \|\|= {}
	6	+ @settings = Noosfero::Plugin::Settings.new(environment, VirtuosoPlugin, settings)
	7	+ @harvest_running = VirtuosoPlugin::DspaceHarvest.new(environment).find_job.present?
	8	+
	9	+ if request.post?
	10	+ @settings.save!
	11	+ session[:notice] = 'Settings succefully saved.'
	12	+ redirect_to :action => 'index'
	13	+ end
	14	+ end
	15	+
	16	+ def force_harvest
	17	+ harvest = VirtuosoPlugin::DspaceHarvest.new(environment)
	18	+ harvest.start
	19	+ session[:notice] = _('Harvest started')
	20	+ redirect_to :action => :index
	21	+ end
	22	+
	23	+end
@@ -0,0 +1,11 @@		@@ -0,0 +1,11 @@
	1	+class VirtuosoPlugin < Noosfero::Plugin
	2	+
	3	+ def self.plugin_name
	4	+ "Virtuoso integration"
	5	+ end
	6	+
	7	+ def self.plugin_description
	8	+ _('Virtuoso integration')
	9	+ end
	10	+
	11	+end
@@ -0,0 +1,71 @@		@@ -0,0 +1,71 @@
	1	+#inspired by https://github.com/code4lib/ruby-oai/blob/master/lib/oai/harvester/harvest.rb
	2	+class VirtuosoPlugin::DspaceHarvest
	3	+
	4	+ DC_CONVERSION = [:title, :creator, :subject, :description, :date, :type, :identifier, :language, :rights, :format]
	5	+
	6	+ def initialize(environment)
	7	+ @environment = environment
	8	+ end
	9	+
	10	+ def settings
	11	+ @settings \|\|= Noosfero::Plugin::Settings.new(@environment, VirtuosoPlugin)
	12	+ end
	13	+
	14	+ def dspace_client
	15	+ @dspace_client \|\|= OAI::Client.new("#{settings.dspace_uri}/oai/request")
	16	+ end
	17	+
	18	+ def virtuoso_client
	19	+ @virtuoso_client \|\|= RDF::Virtuoso::Repository.new("#{settings.virtuoso_uri}/sparql", :update_uri => "#{settings.virtuoso_uri}/sparql-auth", :username => settings.virtuoso_username, :password => settings.virtuoso_password, :auth_method => 'digest', :timeout => 30)
	20	+ end
	21	+
	22	+ def triplify(record)
	23	+ metadata = VirtuosoPlugin::DublinCoreMetadata.new(record.metadata)
	24	+ puts "triplify #{record.header.identifier}"
	25	+
	26	+ DC_CONVERSION.each do \|c\|
	27	+ values = [metadata.send(c)].flatten.compact
	28	+ values.each do \|value\|
	29	+ query = RDF::Virtuoso::Query.insert_data([RDF::URI.new(metadata.identifier), RDF::URI.new("http://purl.org/dc/elements/1.1/#{c}"), value]).graph(RDF::URI.new(settings.dspace_uri))
	30	+ virtuoso_client.insert(query)
	31	+ end
	32	+ end
	33	+ end
	34	+
	35	+ def run
	36	+ harvest_time = Time.now.utc
	37	+ params = settings.last_harvest ? {:from => settings.last_harvest.utc} : {}
	38	+ puts "starting harvest #{params}"
	39	+ begin
	40	+ records = dspace_client.list_records(params)
	41	+ records.each do \|record\|
	42	+ triplify(record)
	43	+ end
	44	+ rescue Exception => ex
	45	+ puts ex.to_s
	46	+ end
	47	+ settings.last_harvest = harvest_time
	48	+ settings.save!
	49	+ puts "ending harvest #{harvest_time}"
	50	+ end
	51	+
	52	+ def start
	53	+ if find_job.empty?
	54	+ job = VirtuosoPlugin::DspaceHarvest::Job.new(@environment.id)
	55	+ Delayed::Job.enqueue(job)
	56	+ end
	57	+ end
	58	+
	59	+ def find_job
	60	+ Delayed::Job.where(:handler => "--- !ruby/struct:VirtuosoPlugin::DspaceHarvest::Job\nenvironment_id: #{@environment.id}\n")
	61	+ end
	62	+
	63	+ class Job < Struct.new(:environment_id)
	64	+ def perform
	65	+ environment = Environment.find(environment_id)
	66	+ harvest = VirtuosoPlugin::DspaceHarvest.new(environment)
	67	+ harvest.run
	68	+ end
	69	+ end
	70	+
	71	+end
@@ -0,0 +1,20 @@		@@ -0,0 +1,20 @@
	1	+class VirtuosoPlugin::DublinCoreMetadata
	2	+
	3	+ include OAI::XPath
	4	+
	5	+ attr_accessor :date, :title, :creator, :subject, :description, :date, :type, :identifier, :language, :rights, :format
	6	+
	7	+ def initialize(element)
	8	+ @title = xpath(element, './/dc:title')
	9	+ @creator = xpath(element, './/dc:creator')
	10	+ @subject = xpath_all(element, './/dc:subject').map(&:text)
	11	+ @description = xpath(element, './/dc:description')
	12	+ @date = xpath_all(element, './/dc:date').map(&:text)
	13	+ @type = xpath(element, './/dc:type')
	14	+ @identifier = xpath(element, './/dc:identifier')
	15	+ @language = xpath(element, './/dc:language')
	16	+ @rights = xpath_all(element, './/dc:rights').map(&:text)
	17	+ @format = xpath(element, './/dc:format')
	18	+ end
	19	+
	20	+end
@@ -0,0 +1,38 @@		@@ -0,0 +1,38 @@
	1	+require File.dirname(__FILE__) + '/../test_helper'
	2	+
	3	+class VirtuosoPluginAdminControllerTest < ActionController::TestCase
	4	+
	5	+ def setup
	6	+ @environment = Environment.default
	7	+ @profile = create_user('profile').person
	8	+ login_as(@profile.identifier)
	9	+ end
	10	+
	11	+ attr_reader :environment
	12	+
	13	+ should 'save virtuoso plugin settings' do
	14	+ post :index, :settings => {'virtuoso_uri' => 'http://virtuoso.noosfero.com',
	15	+ 'virtuoso_username' => 'username',
	16	+ 'virtuoso_password' => 'password',
	17	+ 'dspace_uri' => 'http://dspace.noosfero.com'}
	18	+ @settings = Noosfero::Plugin::Settings.new(environment.reload, VirtuosoPlugin)
	19	+ assert_equal 'http://virtuoso.noosfero.com', @settings.settings[:virtuoso_uri]
	20	+ assert_equal 'username', @settings.settings[:virtuoso_username]
	21	+ assert_equal 'password', @settings.settings[:virtuoso_password]
	22	+ assert_equal 'http://dspace.noosfero.com', @settings.settings[:dspace_uri]
	23	+ assert_redirected_to :action => 'index'
	24	+ end
	25	+
	26	+ should 'redirect to index after save' do
	27	+ post :index, :settings => {"virtuoso_uri" => 'http://virtuoso.noosfero.com'}
	28	+ assert_redirected_to :action => 'index'
	29	+ end
	30	+
	31	+ should 'create delayed job to start harvest on force action' do
	32	+ harvest = VirtuosoPlugin::DspaceHarvest.new(environment)
	33	+ assert !harvest.find_job.present?
	34	+ get :force_harvest
	35	+ assert harvest.find_job.present?
	36	+ end
	37	+
	38	+end
	@@ -0,0 +1 @@		@@ -0,0 +1 @@
		1	+require File.dirname(__FILE__) + '/../../../test/test_helper'
@@ -0,0 +1,25 @@		@@ -0,0 +1,25 @@
	1	+require File.dirname(__FILE__) + '/../test_helper'
	2	+
	3	+class DspaceHarvestTest < ActiveSupport::TestCase
	4	+
	5	+ def setup
	6	+ @environment = Environment.default
	7	+ end
	8	+
	9	+ attr_reader :environment
	10	+
	11	+ should 'create delayed job when start' do
	12	+ harvest = VirtuosoPlugin::DspaceHarvest.new(environment)
	13	+ assert !harvest.find_job.present?
	14	+ harvest.start
	15	+ assert harvest.find_job.present?
	16	+ end
	17	+
	18	+ should 'not duplicate harvest job' do
	19	+ harvest = VirtuosoPlugin::DspaceHarvest.new(environment)
	20	+ assert_difference "harvest.find_job.count", 1 do
	21	+ 5.times { harvest.start }
	22	+ end
	23	+ end
	24	+
	25	+end
@@ -0,0 +1,34 @@		@@ -0,0 +1,34 @@
	1	+<h1><%= _('Virtuoso settings')%></h1>
	2	+
	3	+<%= form_for(:settings) do \|f\| %>
	4	+
	5	+ <strong>
	6	+ <%= labelled_form_field _('Virtuoso URL:'), f.text_field(:virtuoso_uri) %>
	7	+ <%= labelled_form_field _('Virtuoso Username:'), f.text_field(:virtuoso_username) %>
	8	+ <%= labelled_form_field _('Virtuoso Password:'), f.password_field(:virtuoso_password) %>
	9	+ <%= labelled_form_field _('DSpace URL:'), f.text_field(:dspace_uri) %>
	10	+ </strong>
	11	+
	12	+ <% button_bar do %>
	13	+ <%= submit_button(:save, _('Save'), :cancel => {:controller => 'plugins', :action => 'index'}) %>
	14	+ <% end %>
	15	+
	16	+<% end %>
	17	+
	18	+<hr/>
	19	+<div class="harvest">
	20	+ <% if @settings.last_harvest %>
	21	+ <div class="date">
	22	+ <span class="label"><strong><%= _('Last execution:') %></strong></span>
	23	+ <span class="value"><%= time_ago_as_sentence @settings.last_harvest %></span>
	24	+ </div>
	25	+<% end %>
	26	+ <br/>
	27	+ <div class="actions">
	28	+ <% if @harvest_running %>
	29	+ <%= _('Running...') %>
	30	+ <% else %>
	31	+ <%= button :next, _('Force harvest'), :action => :force_harvest %>
	32	+ <% end %>
	33	+ </div>
	34	+</div>