Download as
Browse Code »

Commit e0416de4174d637cc2e0e330662b59f3eaefefdf

Authored by Braulio Bhavamitra
1 parent 38fbfe9d
Exists in staging and in 18 other branches all_pending_tasks_api, api_roles, comments_permissions, elasticsearch, elasticsearch_categories, elasticsearch_filter, elasticsearch_to_merge, environment-exposes-api, external_followers, federation_oauth_provider, follower_permition, master, master_profile_followers, oauth_external_login, private-scraps-rebase, production, profile_api_improvements, user_mention

analytics: identify bots and filter them out by default

Showing 16 changed files with 240 additions and 66 deletions   Show diff stats
plugins/analytics/Gemfile 0 → 100644
  Diff comments   View file @e0416de
... ... @@ -0,0 +1,2 @@
  1 +gem 'browser', '~> 2.2.0'
  2 +
... ...
plugins/analytics/controllers/myprofile/analytics_plugin/stats_controller.rb
  Diff comments   View file @e0416de
... ... @@ -7,12 +7,39 @@ class AnalyticsPlugin::StatsController < MyProfileController
7 7 def index
8 8 end
9 9  
  10 + def edit
  11 + return render_access_denied unless user.has_permission? 'edit_profile', profile
  12 +
  13 + params[:analytics_settings][:enabled] = params[:analytics_settings][:enabled] == 'true'
  14 + params[:analytics_settings][:anonymous] = params[:analytics_settings][:anonymous] == 'true'
  15 + @settings = profile.analytics_settings params[:analytics_settings] || {}
  16 + @settings.save!
  17 + render nothing: true
  18 + end
  19 +
  20 + def view
  21 + params[:profile_ids] ||= [profile.id]
  22 + ids = params[:profile_ids].map(&:to_i)
  23 + user.adminships # FIXME just to cache #adminship_ids
  24 + ids = ids.select{ |id| id.in? user.adminship_ids } unless @user_is_admin
  25 +
  26 + @profiles = environment.profiles.find ids
  27 + @user = environment.people.find params[:user_id]
  28 + @visits = AnalyticsPlugin::Visit.eager_load(:users_page_views).
  29 + where(profile_id: ids, analytics_plugin_page_views: {user_id: @user.id})
  30 +
  31 + render partial: 'table', locals: {visits: @visits}
  32 +
  33 + end
  34 +
10 35 protected
11 36  
12   - def default_url_options
13   - # avoid rails' use_relative_controller!
14   - {use_route: '/'}
  37 + # inherit routes from core skipping use_relative_controller!
  38 + def url_for options
  39 + options[:controller] = "/#{options[:controller]}" if options.is_a? Hash and options[:controller] and not options[:controller].to_s.starts_with? '/'
  40 + super options
15 41 end
  42 + helper_method :url_for
16 43  
17 44 def skip_page_view
18 45 @analytics_skip_page_view = true
... ...
plugins/analytics/controllers/profile/analytics_plugin/time_on_page_controller.rb
  Diff comments   View file @e0416de
... ... @@ -7,7 +7,10 @@ class AnalyticsPlugin::TimeOnPageController < ProfileController
7 7 Noosfero::Scheduler::Defer.later do
8 8 page_view = profile.page_views.where(request_id: params[:id]).first
9 9 page_view.request = request
10   - page_view.page_load!
  10 + AnalyticsPlugin::PageView.transaction do
  11 + page_view.page_load! Time.at(params[:time].to_i)
  12 + page_view.update_column :title, params[:title] if params[:title].present?
  13 + end
11 14 end
12 15  
13 16 render nothing: true
... ...
plugins/analytics/db/migrate/20151030122634_add_title_and_is_bot_to_analytics_plugin_page_view.rb 0 → 100644
  Diff comments   View file @e0416de
... ... @@ -0,0 +1,40 @@
  1 +class AddTitleAndIsBotToAnalyticsPluginPageView < ActiveRecord::Migration
  2 +
  3 + def up
  4 + add_column :analytics_plugin_page_views, :title, :text
  5 + add_column :analytics_plugin_page_views, :is_bot, :boolean
  6 +
  7 + # missing indexes for performance
  8 + add_index :analytics_plugin_page_views, :type
  9 + add_index :analytics_plugin_page_views, :visit_id
  10 + add_index :analytics_plugin_page_views, :request_started_at
  11 + add_index :analytics_plugin_page_views, :page_loaded_at
  12 + add_index :analytics_plugin_page_views, :is_bot
  13 +
  14 + AnalyticsPlugin::PageView.transaction do
  15 + AnalyticsPlugin::PageView.find_each do |page_view|
  16 + page_view.send :fill_is_bot
  17 + page_view.update_column :is_bot, page_view.is_bot
  18 + end
  19 + end
  20 +
  21 + change_table :analytics_plugin_visits do |t|
  22 + t.timestamps
  23 + end
  24 + AnalyticsPlugin::Visit.transaction do
  25 + AnalyticsPlugin::Visit.find_each do |visit|
  26 + visit.created_at = visit.page_views.first.request_started_at
  27 + visit.updated_at = visit.page_views.last.request_started_at
  28 + visit.save!
  29 + end
  30 + end
  31 +
  32 + # never used
  33 + remove_column :analytics_plugin_page_views, :track_id
  34 + end
  35 +
  36 + def down
  37 + say "this migration can't be reverted"
  38 + end
  39 +
  40 +end
... ...
plugins/analytics/lib/analytics_plugin.rb
  Diff comments   View file @e0416de
... ... @@ -13,4 +13,15 @@ module AnalyticsPlugin
13 13 I18n.t'analytics_plugin.lib.plugin.description'
14 14 end
15 15  
  16 + def self.clear_non_users
  17 + ActiveRecord::Base.transaction do
  18 + AnalyticsPlugin::PageView.bots.delete_all
  19 + AnalyticsPlugin::PageView.not_page_loaded.delete_all
  20 + # delete_all does not work here
  21 + AnalyticsPlugin::Visit.without_page_views.destroy_all
  22 + end
  23 + end
  24 +
16 25 end
  26 +
  27 +Browser::Bot.detect_empty_ua!
... ...
plugins/analytics/lib/analytics_plugin/base.rb
  Diff comments   View file @e0416de
... ... @@ -3,6 +3,7 @@ class AnalyticsPlugin::Base &lt; Noosfero::Plugin
3 3  
4 4 def body_ending
5 5 return unless profile and profile.analytics_enabled?
  6 + return if @analytics_skip_page_view
6 7 lambda do
7 8 render 'analytics_plugin/body_ending'
8 9 end
... ... @@ -12,6 +13,7 @@ class AnalyticsPlugin::Base &lt; Noosfero::Plugin
12 13 ['analytics'].map{ |j| "javascripts/#{j}" }
13 14 end
14 15  
  16 + # FIXME: not reloading on development, need server restart
15 17 def application_controller_filters
16 18 [{
17 19 type: 'around_filter', options: {}, block: -> &block do
... ... @@ -23,15 +25,12 @@ class AnalyticsPlugin::Base &lt; Noosfero::Plugin
23 25 return unless profile and profile.analytics_enabled?
24 26  
25 27 Noosfero::Scheduler::Defer.later 'analytics: register page view' do
26   - page_view = profile.page_views.build request: request, profile_id: profile,
  28 + page_view = profile.page_views.build request: request, profile_id: profile.id,
27 29 request_started_at: request_started_at, request_finished_at: request_finished_at
28   -
29 30 unless profile.analytics_anonymous?
30   - session_id = session.id
31 31 page_view.user = user
32   - page_view.session_id = session_id
  32 + page_view.session_id = session.id
33 33 end
34   -
35 34 page_view.save!
36 35 end
37 36 end,
... ... @@ -39,6 +38,7 @@ class AnalyticsPlugin::Base &lt; Noosfero::Plugin
39 38 end
40 39  
41 40 def control_panel_buttons
  41 + return unless user.is_admin? environment
42 42 {
43 43 title: I18n.t('analytics_plugin.lib.plugin.panel_button'),
44 44 icon: 'analytics-access',
... ...
plugins/analytics/lib/ext/profile.rb
  Diff comments   View file @e0416de
1 1 require_dependency 'profile'
2   -require_dependency 'community'
3 2  
4   -([Profile] + Profile.descendants).each do |subclass|
5   -subclass.class_eval do
  3 +class Profile
6 4  
7   - has_many :visits, foreign_key: :profile_id, class_name: 'AnalyticsPlugin::Visit'
8   - has_many :page_views, foreign_key: :profile_id, class_name: 'AnalyticsPlugin::PageView'
  5 + has_many :users_visits, -> { latest.with_users_page_views }, foreign_key: :profile_id, class_name: 'AnalyticsPlugin::Visit'
9 6  
10   -end
11   -end
  7 + has_many :visits, -> { latest.eager_load :page_views }, foreign_key: :profile_id, class_name: 'AnalyticsPlugin::Visit'
  8 + has_many :page_views, foreign_key: :profile_id, class_name: 'AnalyticsPlugin::PageView'
12 9  
13   -class Profile
  10 + has_many :user_visits, -> { latest.eager_load :page_views }, foreign_key: :user_id, class_name: 'AnalyticsPlugin::PageView'
  11 + has_many :user_page_views, foreign_key: :user_id, class_name: 'AnalyticsPlugin::PageView'
14 12  
15 13 def analytics_settings attrs = {}
16 14 @analytics_settings ||= Noosfero::Plugin::Settings.new self, ::AnalyticsPlugin, attrs
... ...
plugins/analytics/locales/en.yml
  Diff comments   View file @e0416de
... ... @@ -9,10 +9,14 @@ en: &amp;en
9 9  
10 10 views:
11 11 stats:
  12 + enable: "Enable tracking on the profile '%{profile}'"
  13 + anonymous: "Don't associate users' login"
  14 + config_save: "Configuration saved"
12 15 user: 'User'
13 16 initial_time: 'Time'
  17 + ip: 'IP'
14 18 pages: 'Pages'
15 19  
16   -en-US:
  20 +en_US:
17 21 <<: *en
18 22  
... ...
plugins/analytics/locales/pt.yml
  Diff comments   View file @e0416de
... ... @@ -9,9 +9,14 @@ pt: &amp;pt
9 9  
10 10 views:
11 11 stats:
  12 + enable: "Ativar rastreio no perfil '%{profile}'"
  13 + anonymous: "Não associar login de usuários"
  14 + config_save: "Configuração salva"
12 15 user: 'Usuário'
13 16 initial_time: 'Horário'
  17 + ip: 'IP'
14 18 pages: 'Páginas'
15 19  
16   -pt-BR:
  20 +pt_BR:
17 21 <<: *pt
  22 +
... ...
plugins/analytics/models/analytics_plugin/page_view.rb
  Diff comments   View file @e0416de
... ... @@ -10,22 +10,34 @@ class AnalyticsPlugin::PageView &lt; ApplicationRecord
10 10  
11 11 acts_as_having_settings field: :options
12 12  
13   - belongs_to :visit, class_name: 'AnalyticsPlugin::Visit'
14   - belongs_to :referer_page_view, class_name: 'AnalyticsPlugin::PageView'
  13 + belongs_to :profile, validate: true
  14 + belongs_to :visit, class_name: 'AnalyticsPlugin::Visit', touch: true, validate: true
15 15  
16   - belongs_to :user, class_name: 'Person'
17   - belongs_to :session, primary_key: :session_id, foreign_key: :session_id, class_name: 'Session'
18   - belongs_to :profile
  16 + belongs_to :referer_page_view, class_name: 'AnalyticsPlugin::PageView', validate: false
19 17  
20   - validates_presence_of :visit
21   - validates_presence_of :request, on: :create
22   - validates_presence_of :url
  18 + belongs_to :user, class_name: 'Person', validate: false
  19 + belongs_to :session, primary_key: :session_id, foreign_key: :session_id, class_name: 'Session', validate: false
  20 +
  21 + validates :request, presence: true, on: :create
  22 + validates :url, presence: true
23 23  
24 24 before_validation :extract_request_data, on: :create
25 25 before_validation :fill_referer_page_view, on: :create
26 26 before_validation :fill_visit, on: :create
  27 + before_validation :fill_is_bot, on: :create
  28 +
  29 + after_update :destroy_empty_visit
  30 + after_destroy :destroy_empty_visit
  31 +
  32 + scope :in_sequence, -> { order 'analytics_plugin_page_views.request_started_at ASC' }
  33 +
  34 + scope :page_loaded, -> { where 'analytics_plugin_page_views.page_loaded_at IS NOT NULL' }
  35 + scope :not_page_loaded, -> { where 'analytics_plugin_page_views.page_loaded_at IS NULL' }
27 36  
28   - scope :latest, -> { order 'request_started_at DESC' }
  37 + scope :no_bots, -> { where.not is_bot: true }
  38 + scope :bots, -> { where is_bot: true }
  39 +
  40 + scope :loaded_users, -> { in_sequence.page_loaded.no_bots }
29 41  
30 42 def request_duration
31 43 self.request_finished_at - self.request_started_at
... ... @@ -43,8 +55,8 @@ class AnalyticsPlugin::PageView &lt; ApplicationRecord
43 55 Time.now < self.user_last_time_seen + AnalyticsPlugin::TimeOnPageUpdateInterval
44 56 end
45 57  
46   - def page_load!
47   - self.page_loaded_at = Time.now
  58 + def page_load! time
  59 + self.page_loaded_at = time
48 60 self.update_column :page_loaded_at, self.page_loaded_at
49 61 end
50 62  
... ... @@ -56,6 +68,16 @@ class AnalyticsPlugin::PageView &lt; ApplicationRecord
56 68 self.update_column :time_on_page, self.time_on_page
57 69 end
58 70  
  71 + def find_referer_page_view
  72 + return if self.referer_url.blank?
  73 + AnalyticsPlugin::PageView.order('request_started_at DESC').
  74 + where(url: self.referer_url, session_id: self.session_id, user_id: self.user_id, profile_id: self.profile_id).first
  75 + end
  76 +
  77 + def browser
  78 + @browser ||= Browser.new self.user_agent
  79 + end
  80 +
59 81 protected
60 82  
61 83 def extract_request_data
... ... @@ -64,16 +86,29 @@ class AnalyticsPlugin::PageView &lt; ApplicationRecord
64 86 self.user_agent = self.request.headers['User-Agent']
65 87 self.request_id = self.request.env['action_dispatch.request_id']
66 88 self.remote_ip = self.request.remote_ip
  89 + true
67 90 end
68 91  
69 92 def fill_referer_page_view
70   - self.referer_page_view = AnalyticsPlugin::PageView.order('request_started_at DESC').
71   - where(url: self.referer_url, session_id: self.session_id, user_id: self.user_id, profile_id: self.profile_id).first if self.referer_url.present?
  93 + self.referer_page_view = self.find_referer_page_view
  94 + true
72 95 end
73 96  
74 97 def fill_visit
75 98 self.visit = self.referer_page_view.visit if self.referer_page_view and self.referer_page_view.user_on_page?
76 99 self.visit ||= AnalyticsPlugin::Visit.new profile: profile
  100 + true
  101 + end
  102 +
  103 + def fill_is_bot
  104 + self.is_bot = self.browser.bot?
  105 + true
  106 + end
  107 +
  108 + def destroy_empty_visit
  109 + return unless self.visit_id_changed?
  110 + old_visit = AnalyticsPlugin::Visit.find self.visit_id_was
  111 + old_visit.destroy if old_visit.page_views.empty?
77 112 end
78 113  
79 114 end
... ...
plugins/analytics/models/analytics_plugin/visit.rb
  Diff comments   View file @e0416de
... ... @@ -5,10 +5,16 @@ class AnalyticsPlugin::Visit &lt; ApplicationRecord
5 5  
6 6 belongs_to :profile
7 7 has_many :page_views, class_name: 'AnalyticsPlugin::PageView', dependent: :destroy
  8 + has_many :users_page_views, -> { loaded_users }, class_name: 'AnalyticsPlugin::PageView', dependent: :destroy
8 9  
9   - default_scope -> { joins(:page_views).includes :page_views }
  10 + scope :latest, -> { order 'updated_at DESC' }
10 11  
11   - scope :latest, -> { order 'analytics_plugin_page_views.request_started_at DESC' }
  12 + scope :with_users_page_views, -> {
  13 + eager_load(:users_page_views).where.not analytics_plugin_page_views: {visit_id: nil}
  14 + }
  15 + scope :without_page_views, -> {
  16 + eager_load(:page_views).where analytics_plugin_page_views: {visit_id: nil}
  17 + }
12 18  
13 19 def first_page_view
14 20 self.page_views.first
... ...
plugins/analytics/public/javascripts/analytics.js
  Diff comments   View file @e0416de
1 1 analytics = {
  2 +
  3 + t: function (key, options) {
  4 + return I18n.t(key, $.extend(options, {scope: 'analytics_plugin'}))
  5 + },
  6 +
2 7 requestId: '',
3 8  
4 9 timeOnPage: {
... ... @@ -27,7 +32,7 @@ analytics = {
27 32  
28 33 pageLoad: function() {
29 34 $.ajax(analytics.timeOnPage.baseUrl+'/page_load', {
30   - type: 'POST', data: {id: analytics.requestId},
  35 + type: 'POST', data: {id: analytics.requestId, title: document.title, time: Math.floor(Date.now()/1000)},
31 36 success: function(data) {
32 37 },
33 38 });
... ...
plugins/analytics/public/javascripts/views/settings.tag.slim 0 → 100644
  Diff comments   View file @e0416de
... ... @@ -0,0 +1,33 @@
  1 +analytics-settings
  2 + .checkbox
  3 + label name='enabled'
  4 + input type='checkbox' name='enabled' value='1' checked='{settings.enabled}' onchange='{toggleEnabled}'
  5 + |{anl.t('views.stats.enable', {profile: noosfero.profile})}
  6 +
  7 + .checkbox if='{settings.enabled}'
  8 + label name='anonymous'
  9 + input type='checkbox' name='anonymous' value='1' checked='{settings.anonymous}' onchange='{toggleAnonymous}'
  10 + |{anl.t('views.stats.anonymous')}
  11 +
  12 + javascript:
  13 + this.anl = window.analytics
  14 + this.settings = opts.settings
  15 + this.updateUrl = Routes.analytics_plugin_stats_path({profile: noosfero.profile, action: 'edit'})
  16 +
  17 + toggleEnabled (e) {
  18 + this.settings.enabled = !this.settings.enabled
  19 + this.update()
  20 + this.save(e)
  21 + }
  22 + toggleAnonymous (e) {
  23 + this.settings.anonymous = !this.settings.anonymous
  24 + this.save(e)
  25 + }
  26 +
  27 + save (e) {
  28 + var self = this
  29 + $.post(this.updateUrl, {analytics_settings: this.settings}, function() {
  30 + display_notice(self.anl.t('views.stats.config_save'))
  31 + })
  32 + }
  33 +
... ...
plugins/analytics/test/functional/content_viewer_controller_test.rb
  Diff comments   View file @e0416de
... ... @@ -37,7 +37,7 @@ class ContentViewerControllerTest &lt; ActionController::TestCase
37 37 @request.env['HTTP_REFERER'] = first_url
38 38 get :view_page, profile: @community.identifier, page: @community.articles.last.path.split('/')
39 39 assert_equal 2, @community.page_views.count
40   - assert_equal 2, @community.visits.count
  40 + assert_equal 1, @community.visits.count
41 41  
42 42 second_page_view = @community.page_views.order(:id).last
43 43 assert_equal first_page_view, second_page_view.referer_page_view
... ... @@ -48,7 +48,7 @@ class ContentViewerControllerTest &lt; ActionController::TestCase
48 48 future = Time.now + 2*AnalyticsPlugin::TimeOnPageUpdateInterval
49 49 Time.stubs(:now).returns(future)
50 50 get :view_page, profile: @community.identifier, page: @community.articles.last.path.split('/')
51   - assert_equal 3, @community.visits.count
  51 + assert_equal 2, @community.visits.count
52 52 end
53 53  
54 54 end
... ...
plugins/analytics/views/analytics_plugin/stats/_table.html.slim
  Diff comments   View file @e0416de
1 1  
2   -table#analytics-stats.table data-toggle='table' data-striped='true' data-sortable='true' data-icons-prefix='fa'
3   - thead
4   - - unless profile.analytics_anonymous?
  2 +.table-responsive
  3 + table#analytics-stats.table data-toggle='table' data-striped='true' data-sortable='true' data-icons-prefix='fa'
  4 + thead
5 5 th= t'analytics_plugin.views.stats.user'
6   - th= t'analytics_plugin.views.stats.initial_time'
7   - th= t'analytics_plugin.views.stats.pages'
  6 + th= t'analytics_plugin.views.stats.initial_time'
  7 + th= t'analytics_plugin.views.stats.ip'
  8 + th= t'analytics_plugin.views.stats.pages'
8 9  
9   - tbody
10   - - profile.visits.each do |visit|
11   - tr
12   - td= link_to visit.user.name, visit.user.url
13   - td
14   - div data-toggle="tooltip" data-title='#{l visit.initial_time}'
15   - = time_ago_in_words(visit.initial_time)
16   - |&nbsp
17   - = _'ago'
18   - td
19   - - visit.page_views.each do |page_view|
20   - = link_to page_view.url, page_view.url
21   - |&nbsp;
22   - = "(#{distance_of_time_in_words page_view.time_on_page})"
23   - |&nbsp;->&nbsp;
  10 + tbody
  11 + - visits.each do |visit|
  12 + tr data-visit-id='#{visit.id}'
  13 + td= link_to visit.user.name, visit.user.url if visit.user
  14 + td
  15 + div data-toggle="tooltip" data-title='#{l visit.initial_time}'
  16 + = time_ago_in_words visit.initial_time
  17 + |&nbsp
  18 + = _'ago'
  19 + td= visit.users_page_views.first.remote_ip
  20 + td
  21 + ol
  22 + - visit.users_page_views.each do |page_view|
  23 + li
  24 + = link_to (if page_view.title.present? then page_view.title else page_view.url end), page_view.url, target: '_blank'
  25 + |&nbsp;
  26 + = "(#{distance_of_time_in_words page_view.time_on_page})"
24 27  
25 28 javascript:
26 29 $('#analytics-stats').bootstrapTable({
27 30 striped: true,
28   - columns: [
29   - {sortable: true},
30   - {sortable: true},
31   - {sortable: true},
32   - ],
33 31 })
34 32  
35 33 $(document).ready(function() {
... ...
plugins/analytics/views/analytics_plugin/stats/index.html.slim
  Diff comments   View file @e0416de
1   -- content_for :head
2   - = javascript_include_tag 'https://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.8.1/bootstrap-table-all.min.js'
3   - = stylesheet_link_tag 'https://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.8.1/bootstrap-table.css'
  1 += render 'shared/bootstrap_table'
  2 +
  3 += button :back, _('Back to control panel'), controller: 'profile_editor'
  4 +
  5 += js_translations_include plugin: :analytics
  6 += javascript_include_tag 'plugins/analytics/javascripts/views/settings'
  7 +analytics-settings data-opts="#{CGI.escapeHTML({settings: {enabled: profile.analytics_settings.enabled, anonymous: profile.analytics_settings.anonymous}}.to_json)}" data-riot=''
  8 +/ needs html_safe to work
  9 +/= riot_component :analytics_settings, settings: {enabled: profile.analytics_settings.enabled, anonymous: profile.analytics_settings.anonymous}
  10 +
  11 += render 'table', visits: profile.users_visits.limit(50)
4 12  
5   -= render 'table'
... ...