diff --git a/Gemfile b/Gemfile index f96a866..f0760e0 100644 --- a/Gemfile +++ b/Gemfile @@ -16,6 +16,8 @@ gem 'hpricot' gem 'nokogiri' gem 'rake', :require => false +gem 'whenever', :require => false + # FIXME list here all actual dependencies (i.e. the ones in debian/control), # with their GEM names (not the Debian package names) diff --git a/Gemfile.lock b/Gemfile.lock index 5485d00..64f6ef0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -46,6 +46,7 @@ GEM xpath (~> 2.0) childprocess (0.3.3) ffi (~> 1.0.6) + chronic (0.10.2) cucumber (1.0.6) builder (>= 2.1.2) diff-lcs (>= 1.1.2) @@ -149,6 +150,9 @@ GEM polyglot (>= 0.3.1) tzinfo (0.3.33) websocket (1.0.7) + whenever (0.9.2) + activesupport (>= 2.3.4) + chronic (>= 0.6.3) will_paginate (3.0.3) xpath (2.0.0) nokogiri (~> 1.3) @@ -181,4 +185,5 @@ DEPENDENCIES ruby-feedparser selenium-webdriver thin + whenever will_paginate diff --git a/app/controllers/admin/region_validators_controller.rb b/app/controllers/admin/region_validators_controller.rb index 801e03b..c7e9a19 100644 --- a/app/controllers/admin/region_validators_controller.rb +++ b/app/controllers/admin/region_validators_controller.rb @@ -33,7 +33,7 @@ class RegionValidatorsController < AdminController def load_region_and_search @region = environment.regions.find(params[:id]) if params[:search] - @search = find_by_contents(:organizations, Organization, params[:search])[:results].reject {|item| @region.validator_ids.include?(item.id) } + @search = find_by_contents(:organizations, environment, Organization, params[:search])[:results].reject {|item| @region.validator_ids.include?(item.id) } end end diff --git a/app/controllers/admin/users_controller.rb b/app/controllers/admin/users_controller.rb index 6e8eace..b827994 100644 --- a/app/controllers/admin/users_controller.rb +++ b/app/controllers/admin/users_controller.rb @@ -18,7 +18,7 @@ class UsersController < AdminController end scope = scope.order('name ASC') @q = params[:q] - @collection = find_by_contents(:people, scope, @q, {:per_page => per_page, :page => params[:npage]})[:results] + @collection = find_by_contents(:people, environment, scope, @q, {:per_page => per_page, :page => params[:npage]})[:results] end def set_admin_role diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index b4d1767..1aa3f0e 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -174,17 +174,15 @@ class ApplicationController < ActionController::Base end end - def find_by_contents(asset, scope, query, paginate_options={:page => 1}, options={}) - plugins.dispatch_first(:find_by_contents, asset, scope, query, paginate_options, options) || - fallback_find_by_contents(asset, scope, query, paginate_options, options) - end - - private + include SearchTermHelper - def fallback_find_by_contents(asset, scope, query, paginate_options, options) - scope = scope.like_search(query) unless query.blank? - scope = scope.send(options[:filter]) unless options[:filter].blank? - {:results => scope.paginate(paginate_options)} + def find_by_contents(asset, context, scope, query, paginate_options={:page => 1}, options={}) + search = plugins.dispatch_first(:find_by_contents, asset, scope, query, paginate_options, options) + register_search_term(query, scope.count, search[:results].count, context, asset) + search end + def find_suggestions(query, context, asset, options={}) + plugins.dispatch_first(:find_suggestions, query, context, asset, options) + end end diff --git a/app/controllers/my_profile/cms_controller.rb b/app/controllers/my_profile/cms_controller.rb index 0a00238..3e57354 100644 --- a/app/controllers/my_profile/cms_controller.rb +++ b/app/controllers/my_profile/cms_controller.rb @@ -307,7 +307,7 @@ class CmsController < MyProfileController def search query = params[:q] - results = find_by_contents(:uploaded_files, profile.files.published, query)[:results] + results = find_by_contents(:uploaded_files, profile, profile.files.published, query)[:results] render :text => article_list_to_json(results), :content_type => 'application/json' end diff --git a/app/controllers/public/profile_search_controller.rb b/app/controllers/public/profile_search_controller.rb index 39515e0..32b7fec 100644 --- a/app/controllers/public/profile_search_controller.rb +++ b/app/controllers/public/profile_search_controller.rb @@ -11,7 +11,7 @@ class ProfileSearchController < PublicController if params[:where] == 'environment' redirect_to :controller => 'search', :query => @q else - @results = find_by_contents(:articles, profile.articles.published, @q, {:per_page => 10, :page => params[:page]})[:results] + @results = find_by_contents(:articles, profile, profile.articles.published, @q, {:per_page => 10, :page => params[:page]})[:results] end end end diff --git a/app/controllers/public/search_controller.rb b/app/controllers/public/search_controller.rb index eae18f7..9235704 100644 --- a/app/controllers/public/search_controller.rb +++ b/app/controllers/public/search_controller.rb @@ -4,11 +4,11 @@ class SearchController < PublicController include SearchHelper include ActionView::Helpers::NumberHelper - before_filter :redirect_asset_param, :except => :assets - before_filter :load_category - before_filter :load_search_assets - before_filter :load_query - before_filter :load_filter + before_filter :redirect_asset_param, :except => [:assets, :suggestions] + before_filter :load_category, :except => :suggestions + before_filter :load_search_assets, :except => :suggestions + before_filter :load_query, :except => :suggestions + before_filter :load_order, :except => :suggestions # Backwards compatibility with old URLs def redirect_asset_param @@ -20,7 +20,7 @@ class SearchController < PublicController def index @searches = {} - @order = [] + @assets = [] @names = {} @results_only = true @@ -28,7 +28,7 @@ class SearchController < PublicController load_query @asset = key send(key) - @order << key + @assets << key @names[key] = _(description) end @asset = nil @@ -42,7 +42,7 @@ class SearchController < PublicController # view the summary of one category def category_index @searches = {} - @order = [] + @assets = [] @names = {} limit = MULTIPLE_SEARCH_LIMIT [ @@ -53,7 +53,7 @@ class SearchController < PublicController [ :communities, _('Communities'), :recent_communities ], [ :articles, _('Contents'), :recent_articles ] ].each do |asset, name, filter| - @order << asset + @assets << asset @searches[asset]= {:results => @category.send(filter, limit)} raise "No total_entries for: #{asset}" unless @searches[asset][:results].respond_to?(:total_entries) @names[asset] = name @@ -143,12 +143,16 @@ class SearchController < PublicController render :partial => 'events/events' end + def suggestions + render :text => find_suggestions(params[:term], environment, params[:asset]).to_json + end + ####################################################### protected def load_query @asset = (params[:asset] || params[:action]).to_sym - @order ||= [@asset] + @assets ||= [@asset] @searches ||= {} @query = params[:query] || '' @@ -185,11 +189,11 @@ class SearchController < PublicController @names = @titles if @names.nil? end - def load_filter - @filter = 'more_recent' + def load_order + @order = 'more_recent' if SEARCHES.keys.include?(@asset.to_sym) - available_filters = asset_class(@asset)::SEARCH_FILTERS - @filter = params[:filter] if available_filters.include?(params[:filter]) + available_orders = asset_class(@asset)::SEARCH_FILTERS[:order] + @order = params[:order] if available_orders.include?(params[:order]) end end @@ -213,7 +217,7 @@ class SearchController < PublicController end def full_text_search - @searches[@asset] = find_by_contents(@asset, @scope, @query, paginate_options, {:category => @category, :filter => @filter}) + @searches[@asset] = find_by_contents(@asset, environment, @scope, @query, paginate_options, {:category => @category, :filter => @order}) end private diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 54149ef..7fde986 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -1415,4 +1415,8 @@ module ApplicationHelper content_tag('ul', article.versions.map {|v| link_to("r#{v.version}", @page.url.merge(:version => v.version))}) end + def search_input_with_suggestions(name, asset, default, options = {}) + text_field_tag name, default, options.merge({:id => 'search-input', 'data-asset' => asset}) + end + end diff --git a/app/helpers/layout_helper.rb b/app/helpers/layout_helper.rb index 64d0541..c967ec1 100644 --- a/app/helpers/layout_helper.rb +++ b/app/helpers/layout_helper.rb @@ -27,6 +27,7 @@ module LayoutHelper 'thickbox', 'lightbox', 'colorbox', + 'selectordie', pngfix_stylesheet_path, ] + tokeninput_stylesheets plugins_stylesheets = @plugins.select(&:stylesheet?).map { |plugin| plugin.class.public_path('style.css') } diff --git a/app/helpers/search_helper.rb b/app/helpers/search_helper.rb index 1f1114e..14a6c3a 100644 --- a/app/helpers/search_helper.rb +++ b/app/helpers/search_helper.rb @@ -14,11 +14,23 @@ module SearchHelper :events, _('Events'), ] - FILTER_TRANSLATION = { - 'more_popular' => _('More popular'), - 'more_active' => _('More active'), - 'more_recent' => _('More recent'), - 'more_comments' => _('More comments') + FILTERS_TRANSLATIONS = { + :order => _('Order'), + :display => _('Display') + } + + FILTERS_OPTIONS_TRANSLATION = { + :order => { + 'more_popular' => _('More popular'), + 'more_active' => _('More active'), + 'more_recent' => _('More recent'), + 'more_comments' => _('More comments') + }, + :display => { + 'map' => _('Map'), + 'full' => _('Full'), + 'compact' => _('Compact') + } } # FIXME remove it after search_controler refactored @@ -50,7 +62,7 @@ module SearchHelper end def display?(asset, mode) - defined?(asset_class(asset)::SEARCH_DISPLAYS) && asset_class(asset)::SEARCH_DISPLAYS.include?(mode.to_s) + defined?(asset_class(asset)::SEARCH_FILTERS[:display]) && asset_class(asset)::SEARCH_FILTERS[:display].include?(mode.to_s) end def display_results(searches=nil, asset=nil) @@ -87,50 +99,37 @@ module SearchHelper end end - def display_selector(asset, display, float = 'right') - display = nil if display.blank? - display ||= asset_class(asset).default_search_display - if [display?(asset, :map), display?(asset, :compact), display?(asset, :full)].select {|option| option}.count > 1 - compact_link = display?(asset, :compact) ? (display == 'compact' ? _('Compact') : link_to(_('Compact'), params.merge(:display => 'compact'))) : nil - map_link = display?(asset, :map) ? (display == 'map' ? _('Map') : link_to(_('Map'), params.merge(:display => 'map'))) : nil - full_link = display?(asset, :full) ? (display == 'full' ? _('Full') : link_to(_('Full'), params.merge(:display => 'full'))) : nil - content_tag('div', - content_tag('strong', _('Display')) + ': ' + [compact_link, map_link, full_link].compact.join(' | ').html_safe, - :class => 'search-customize-options' - ) + def select_filter(name, options) + if options.size <= 1 + return + else + options = options.map {|option| [FILTERS_OPTIONS_TRANSLATION[name][option], option]} + options = options_for_select(options, :selected => params[name]) + select_tag(name, options) end end - def filter_selector(asset, filter, float = 'right') + def filters(asset) + return if !asset klass = asset_class(asset) - if klass::SEARCH_FILTERS.count > 1 - options = options_for_select(klass::SEARCH_FILTERS.map {|f| [FILTER_TRANSLATION[f], f]}, filter) - url_params = url_for(params.merge(:filter => 'FILTER')) - onchange = "document.location.href = '#{url_params}'.replace('FILTER', this.value)" - select_field = select_tag(:filter, options, :onchange => onchange) - content_tag('div', - content_tag('strong', _('Filter')) + ': ' + select_field, - :class => "search-customize-options" - ) - end + content_tag('div', klass::SEARCH_FILTERS.map do |name, options| + select_filter(name, options) + end.join("\n"), :id => 'search-filters') + end + + def assets_links(selected) + assets = SEARCHES.keys + content_tag('ul', + assets.map do |asset| + options = {} + options.merge!(:class => 'selected') if selected.to_s == asset.to_s + content_tag('li', asset_link(asset), options) + end.join("\n"), + :id => 'assets-links') end - def filter_title(asset, filter) - { - 'articles_more_recent' => _('More recent contents from network'), - 'articles_more_popular' => _('More viewed contents from network'), - 'articles_more_comments' => _('Most commented contents from network'), - 'people_more_recent' => _('More recent people from network'), - 'people_more_active' => _('More active people from network'), - 'people_more_popular' => _('More popular people from network'), - 'communities_more_recent' => _('More recent communities from network'), - 'communities_more_active' => _('More active communities from network'), - 'communities_more_popular' => _('More popular communities from network'), - 'enterprises_more_recent' => _('More recent enterprises from network'), - 'enterprises_more_active' => _('More active enterprises from network'), - 'enterprises_more_popular' => _('More popular enterprises from network'), - 'products_more_recent' => _('Highlights'), - }[asset.to_s + '_' + filter].to_s + def asset_link(asset) + link_to(SEARCHES[asset], "/search/#{asset}") end end diff --git a/app/helpers/search_term_helper.rb b/app/helpers/search_term_helper.rb new file mode 100644 index 0000000..10bfa50 --- /dev/null +++ b/app/helpers/search_term_helper.rb @@ -0,0 +1,17 @@ +module SearchTermHelper + def register_search_term(term, total, indexed, context, asset='all') + normalized_term = normalize_term(term) + if normalized_term.present? + search_term = SearchTerm.find_or_create(normalized_term, context, asset) + SearchTermOccurrence.create!(:search_term => search_term, :total => total, :indexed => indexed) + end + end + + #FIXME For some reason the job is created but nothing is ran. + #handle_asynchronously :register_search_term + + #TODO Think better on how to normalize them properly + def normalize_term(search_term) + search_term + end +end diff --git a/app/models/article.rb b/app/models/article.rb index de7c1bd..9d5d0ee 100644 --- a/app/models/article.rb +++ b/app/models/article.rb @@ -14,13 +14,10 @@ class Article < ActiveRecord::Base :filename => 1, } - SEARCH_FILTERS = %w[ - more_recent - more_popular - more_comments - ] - - SEARCH_DISPLAYS = %w[full] + SEARCH_FILTERS = { + :order => %w[more_recent more_popular more_comments], + :display => %w[full] + } def self.default_search_display 'full' diff --git a/app/models/enterprise.rb b/app/models/enterprise.rb index 87baedd..27e27fe 100644 --- a/app/models/enterprise.rb +++ b/app/models/enterprise.rb @@ -6,6 +6,11 @@ class Enterprise < Organization SEARCH_DISPLAYS += %w[map full] + SEARCH_FILTERS = { + :order => %w[more_recent more_popular more_active], + :display => %w[compact full map] + } + def self.type_name _('Enterprise') end diff --git a/app/models/environment.rb b/app/models/environment.rb index 840a666..8df8993 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -10,6 +10,7 @@ class Environment < ActiveRecord::Base self.partial_updates = false has_many :tasks, :dependent => :destroy, :as => 'target' + has_many :search_terms, :as => :context IDENTIFY_SCRIPTS = /(php[0-9s]?|[sp]htm[l]?|pl|py|cgi|rb)/ diff --git a/app/models/organization.rb b/app/models/organization.rb index e0a3eb1..d750ece 100644 --- a/app/models/organization.rb +++ b/app/models/organization.rb @@ -3,10 +3,11 @@ class Organization < Profile attr_accessible :moderated_articles, :foundation_year, :contact_person, :acronym, :legal_form, :economic_activity, :management_information, :cnpj, :display_name, :enable_contact_us - SEARCH_FILTERS += %w[ - more_popular - more_active - ] + SEARCH_FILTERS = { + :order => %w[more_recent more_popular more_active], + :display => %w[compact] + } + settings_items :closed, :type => :boolean, :default => false def closed? diff --git a/app/models/person.rb b/app/models/person.rb index d0d0521..cbc3fd7 100644 --- a/app/models/person.rb +++ b/app/models/person.rb @@ -3,10 +3,11 @@ class Person < Profile attr_accessible :organization, :contact_information, :sex, :birth_date, :cell_phone, :comercial_phone, :jabber_id, :personal_website, :nationality, :address_reference, :district, :schooling, :schooling_status, :formation, :custom_formation, :area_of_study, :custom_area_of_study, :professional_activity, :organization_website - SEARCH_FILTERS += %w[ - more_popular - more_active - ] + SEARCH_FILTERS = { + :order => %w[more_recent more_popular more_active], + :display => %w[compact] + } + def self.type_name _('Person') diff --git a/app/models/product.rb b/app/models/product.rb index 7a8070e..2cfebc4 100644 --- a/app/models/product.rb +++ b/app/models/product.rb @@ -5,11 +5,10 @@ class Product < ActiveRecord::Base :description => 1, } - SEARCH_FILTERS = %w[ - more_recent - ] - - SEARCH_DISPLAYS = %w[map full] + SEARCH_FILTERS = { + :order => %w[more_recent], + :display => %w[map full] + } attr_accessible :name, :product_category, :highlighted, :price, :enterprise, :image_builder, :description, :available, :qualifiers diff --git a/app/models/profile.rb b/app/models/profile.rb index 1506d8d..2c31c0e 100644 --- a/app/models/profile.rb +++ b/app/models/profile.rb @@ -17,11 +17,10 @@ class Profile < ActiveRecord::Base :nickname => 2, } - SEARCH_FILTERS = %w[ - more_recent - ] - - SEARCH_DISPLAYS = %w[compact] + SEARCH_FILTERS = { + :order => %w[more_recent], + :display => %w[compact] + } def self.default_search_display 'compact' @@ -138,6 +137,8 @@ class Profile < ActiveRecord::Base has_many :comments_received, :class_name => 'Comment', :through => :articles, :source => :comments + has_many :search_terms, :as => :context + def scraps(scrap=nil) scrap = scrap.is_a?(Scrap) ? scrap.id : scrap scrap.nil? ? Scrap.all_scraps(self) : Scrap.all_scraps(self).find(scrap) diff --git a/app/models/search_term.rb b/app/models/search_term.rb new file mode 100644 index 0000000..f45b22a --- /dev/null +++ b/app/models/search_term.rb @@ -0,0 +1,62 @@ +class SearchTerm < ActiveRecord::Base + validates_presence_of :term, :context + validates_uniqueness_of :term, :scope => [:context_id, :context_type, :asset] + + belongs_to :context, :polymorphic => true + has_many :occurrences, :class_name => 'SearchTermOccurrence' + + attr_accessible :term, :context, :asset + + def self.calculate_scores + os = occurrences_scores + find_each { |search_term| search_term.calculate_score(os) } + end + + def self.find_or_create(term, context, asset='all') + context.search_terms.where(:term => term, :asset => asset).first || context.search_terms.create!(:term => term, :asset=> asset) + end + + # Fast way of getting the occurrences score for each search_term. Ugly but fast! + # + # Each occurrence of a search_term has a score that is smaller the older the + # occurrence happened. We subtract the amount of time between now and the + # moment it happened from the total time any occurrence is valid to happen. E.g.: + # The expiration time is 100 days and an occurrence happened 3 days ago. + # Therefore the score is 97. Them we sum every score to get the total score + # for a search term. + def self.occurrences_scores + ActiveSupport::OrderedHash[*ActiveRecord::Base.connection.execute( + joins(:occurrences). + select("search_terms.id, sum(#{SearchTermOccurrence::EXPIRATION_TIME.to_i} - extract(epoch from (now() - search_term_occurrences.created_at))) as value"). + where("search_term_occurrences.created_at > ?", DateTime.now - SearchTermOccurrence::EXPIRATION_TIME). + group("search_terms.id"). + order('value DESC'). + to_sql + ).map {|result| [result['id'].to_i, result['value'].to_i]}.flatten] + end + + def calculate_occurrence(occurrences_scores) + max_score = occurrences_scores.first[1] + (occurrences_scores[id]/max_score.to_f)*100 + end + + def calculate_relevance(valid_occurrences) + indexed = valid_occurrences.last.indexed.to_f + total = valid_occurrences.last.total.to_f + (1 - indexed/total)*100 + end + + def calculate_score(occurrences_scores) + valid_occurrences = occurrences.valid + if valid_occurrences.present? + # These scores vary from 1~100 + self.occurrence_score = calculate_occurrence(occurrences_scores) + self.relevance_score = calculate_relevance(valid_occurrences) + else + self.occurrence_score = 0 + self.relevance_score = 0 + end + self.score = (occurrence_score * relevance_score)/100.0 + self.save! + end +end diff --git a/app/models/search_term_occurrence.rb b/app/models/search_term_occurrence.rb new file mode 100644 index 0000000..75a8f29 --- /dev/null +++ b/app/models/search_term_occurrence.rb @@ -0,0 +1,9 @@ +class SearchTermOccurrence < ActiveRecord::Base + belongs_to :search_term + validates_presence_of :search_term + attr_accessible :search_term, :created_at, :total, :indexed + + EXPIRATION_TIME = 1.year + + scope :valid, :conditions => ["search_term_occurrences.created_at > ?", DateTime.now - EXPIRATION_TIME] +end diff --git a/app/views/layouts/_javascript.html.erb b/app/views/layouts/_javascript.html.erb index 33c05d4..60ce783 100644 --- a/app/views/layouts/_javascript.html.erb +++ b/app/views/layouts/_javascript.html.erb @@ -2,8 +2,8 @@ 'jquery-2.1.1.min', 'jquery-migrate-1.2.1', 'jquery.noconflict.js', 'jquery.cycle.all.min.js', 'thickbox.js', 'lightbox', 'colorbox', 'jquery-ui-1.10.4/js/jquery-ui-1.10.4.min', 'jquery.scrollTo', 'jquery.form.js', 'jquery-validation/jquery.validate', -'jquery.cookie', 'jquery.ba-bbq.min.js', 'reflection', 'jquery.tokeninput', -'add-and-join', 'report-abuse', 'catalog', 'manage-products', 'autogrow', +'jquery.cookie', 'jquery.ba-bbq.min.js', 'reflection', 'jquery.tokeninput', 'jquery.typewatch', +'add-and-join', 'report-abuse', 'catalog', 'manage-products', 'autogrow', 'select-or-die/_src/selectordie', 'jquery-timepicker-addon/dist/jquery-ui-timepicker-addon', 'application.js', 'rails.js', :cache => 'cache/application' %> <% language = FastGettext.locale %> diff --git a/app/views/layouts/_user.html.erb b/app/views/layouts/_user.html.erb index 5366b6d..beb8064 100644 --- a/app/views/layouts/_user.html.erb +++ b/app/views/layouts/_user.html.erb @@ -10,16 +10,16 @@ <%= _("%s") % thickbox_inline_popup_link('' + _('Login') + '', login_url, 'inlineLoginBox', :id => 'link_login') %> <%= @plugins.dispatch(:alternative_authentication_link).collect { |content| instance_exec(&content) }.join("") %> - + - <% unless @plugins.dispatch(:allow_user_registration).include?(false) %> - <%= _("or %s") % link_to('' + _('Sign up') + '', :controller => 'account', :action => 'signup')%> - <% end %> - - <% end %> -