Commit 5f0427fb150acb7f6f722957c54d607fe7ba5d72

Authored by Victor Costa
2 parents c560f738 a1c29241
Exists in staging and in 1 other branch production

Merge branch 'master' into staging

Conflicts:
	lib/feed_handler.rb
	test/unit/feed_handler_test.rb
app/models/environment.rb
@@ -13,7 +13,9 @@ class Environment < ActiveRecord::Base @@ -13,7 +13,9 @@ class Environment < ActiveRecord::Base
13 :reports_lower_bound, :noreply_email, 13 :reports_lower_bound, :noreply_email,
14 :signup_welcome_screen_body, :members_whitelist_enabled, 14 :signup_welcome_screen_body, :members_whitelist_enabled,
15 :members_whitelist, :highlighted_news_amount, 15 :members_whitelist, :highlighted_news_amount,
16 - :portal_news_amount, :date_format, :signup_intro 16 + :portal_news_amount, :date_format, :signup_intro,
  17 + :enable_feed_proxy, :http_feed_proxy, :https_feed_proxy,
  18 + :disable_feed_ssl
17 19
18 has_many :users 20 has_many :users
19 21
app/models/external_feed.rb
@@ -12,6 +12,8 @@ class ExternalFeed < ActiveRecord::Base @@ -12,6 +12,8 @@ class ExternalFeed < ActiveRecord::Base
12 12
13 attr_accessible :address, :enabled, :only_once 13 attr_accessible :address, :enabled, :only_once
14 14
  15 + delegate :environment, :to => :blog, :allow_nil => true
  16 +
15 def add_item(title, link, date, content) 17 def add_item(title, link, date, content)
16 return if content.blank? 18 return if content.blank?
17 doc = Nokogiri::HTML.fragment content 19 doc = Nokogiri::HTML.fragment content
app/views/features/index.html.erb
@@ -49,6 +49,25 @@ Check all the features you want to enable for your environment, uncheck all the @@ -49,6 +49,25 @@ Check all the features you want to enable for your environment, uncheck all the
49 </div> 49 </div>
50 <hr/> 50 <hr/>
51 51
  52 +<h3><%= _('Feed') %></h3>
  53 + <div class="option">
  54 + <%= check_box :environment, :enable_feed_proxy %>
  55 + <label><%= _('Enable feed proxy') %></label>
  56 + </div>
  57 + <div class="input">
  58 + <div class="info"><%= _('HTTP feed proxy address:') %></div>
  59 + <%= text_field :environment, :http_feed_proxy %>
  60 + </div>
  61 + <div class="input">
  62 + <div class="info"><%= _('HTTPS feed proxy address:') %></div>
  63 + <%= text_field :environment, :https_feed_proxy %>
  64 + </div>
  65 + <div class="option">
  66 + <%= check_box :environment, :disable_feed_ssl %>
  67 + <label><%= _('Disable feed SSL') %></label>
  68 + </div>
  69 +<hr/>
  70 +
52 <div> 71 <div>
53 <% button_bar do %> 72 <% button_bar do %>
54 <%= submit_button('save', _('Save changes')) %> 73 <%= submit_button('save', _('Save changes')) %>
db/migrate/20160408011124_add_enable_feed_proxy_to_environments.rb 0 → 100644
@@ -0,0 +1,5 @@ @@ -0,0 +1,5 @@
  1 +class AddEnableFeedProxyToEnvironments < ActiveRecord::Migration
  2 + def change
  3 + add_column :environments, :enable_feed_proxy, :boolean, default: false
  4 + end
  5 +end
db/migrate/20160408011622_add_http_feed_proxy_to_environments.rb 0 → 100644
@@ -0,0 +1,5 @@ @@ -0,0 +1,5 @@
  1 +class AddHttpFeedProxyToEnvironments < ActiveRecord::Migration
  2 + def change
  3 + add_column :environments, :http_feed_proxy, :string
  4 + end
  5 +end
db/migrate/20160408011635_add_https_feed_proxy_to_environments.rb 0 → 100644
@@ -0,0 +1,5 @@ @@ -0,0 +1,5 @@
  1 +class AddHttpsFeedProxyToEnvironments < ActiveRecord::Migration
  2 + def change
  3 + add_column :environments, :https_feed_proxy, :string
  4 + end
  5 +end
db/migrate/20160408011720_add_disable_feed_ssl_to_environments.rb 0 → 100644
@@ -0,0 +1,5 @@ @@ -0,0 +1,5 @@
  1 +class AddDisableFeedSslToEnvironments < ActiveRecord::Migration
  2 + def change
  3 + add_column :environments, :disable_feed_ssl, :boolean, default: false
  4 + end
  5 +end
@@ -11,7 +11,7 @@ @@ -11,7 +11,7 @@
11 # 11 #
12 # It's strongly recommended that you check this file into your version control system. 12 # It's strongly recommended that you check this file into your version control system.
13 13
14 -ActiveRecord::Schema.define(version: 20160324132518) do 14 +ActiveRecord::Schema.define(version: 20160408011720) do
15 15
16 # These are extensions that must be enabled in order to support this database 16 # These are extensions that must be enabled in order to support this database
17 enable_extension "plpgsql" 17 enable_extension "plpgsql"
@@ -391,6 +391,10 @@ ActiveRecord::Schema.define(version: 20160324132518) do @@ -391,6 +391,10 @@ ActiveRecord::Schema.define(version: 20160324132518) do
391 t.string "noreply_email" 391 t.string "noreply_email"
392 t.string "redirection_after_signup", default: "keep_on_same_page" 392 t.string "redirection_after_signup", default: "keep_on_same_page"
393 t.string "date_format", default: "month_name_with_year" 393 t.string "date_format", default: "month_name_with_year"
  394 + t.boolean "enable_feed_proxy", default: false
  395 + t.string "http_feed_proxy"
  396 + t.string "https_feed_proxy"
  397 + t.boolean "disable_feed_ssl", default: false
394 end 398 end
395 399
396 create_table "external_feeds", force: :cascade do |t| 400 create_table "external_feeds", force: :cascade do |t|
features/step_definitions/web_steps.rb
@@ -204,7 +204,7 @@ Then /^the &quot;([^&quot;]*)&quot; field(?: within &quot;([^&quot;]*)&quot;)? should not contain &quot;([^&quot;]*)&quot;$/ @@ -204,7 +204,7 @@ Then /^the &quot;([^&quot;]*)&quot; field(?: within &quot;([^&quot;]*)&quot;)? should not contain &quot;([^&quot;]*)&quot;$/
204 end 204 end
205 end 205 end
206 206
207 -Then /^the "([^"]*)" checkbox(?: within "([^"]*)")? should be checked$/ do |label, selector| 207 +Then /^the "([^"]*)" (?:checkbox|radio button)(?: within "([^"]*)")? should be checked$/ do |label, selector|
208 with_scope(selector) do 208 with_scope(selector) do
209 field_checked = find_field(label)['checked'] 209 field_checked = find_field(label)['checked']
210 if field_checked.respond_to? :should 210 if field_checked.respond_to? :should
@@ -215,7 +215,7 @@ Then /^the &quot;([^&quot;]*)&quot; checkbox(?: within &quot;([^&quot;]*)&quot;)? should be checked$/ do |labe @@ -215,7 +215,7 @@ Then /^the &quot;([^&quot;]*)&quot; checkbox(?: within &quot;([^&quot;]*)&quot;)? should be checked$/ do |labe
215 end 215 end
216 end 216 end
217 217
218 -Then /^the "([^"]*)" checkbox(?: within "([^"]*)")? should not be checked$/ do |label, selector| 218 +Then /^the "([^"]*)" (?:checkbox|radio button)(?: within "([^"]*)")? should not be checked$/ do |label, selector|
219 with_scope(selector) do 219 with_scope(selector) do
220 field_checked = find_field(label)['checked'] 220 field_checked = find_field(label)['checked']
221 if field_checked.respond_to? :should 221 if field_checked.respond_to? :should
lib/feed_handler.rb
@@ -31,7 +31,7 @@ class FeedHandler @@ -31,7 +31,7 @@ class FeedHandler
31 end 31 end
32 end 32 end
33 33
34 - def fetch(address) 34 + def fetch(address, header = {})
35 begin 35 begin
36 content = "" 36 content = ""
37 block = lambda { |s| content = s.read } 37 block = lambda { |s| content = s.read }
@@ -42,13 +42,7 @@ class FeedHandler @@ -42,13 +42,7 @@ class FeedHandler
42 if !valid_url?(address) 42 if !valid_url?(address)
43 raise InvalidUrl.new("\"%s\" is not a valid URL" % address) 43 raise InvalidUrl.new("\"%s\" is not a valid URL" % address)
44 end 44 end
45 - header = {"User-Agent" => "Noosfero/#{Noosfero::VERSION}"}  
46 - if address.starts_with?("https://")  
47 - header.merge!(:proxy => ENV['FEED_HTTPS_PROXY']) if ENV['FEED_HTTPS_PROXY']  
48 - else  
49 - header.merge!(:proxy => ENV['FEED_HTTP_PROXY']) if ENV['FEED_HTTP_PROXY']  
50 - end  
51 - header.merge!(:ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE) if ENV['SSL_VERIFY_NONE'] 45 + header.merge!("User-Agent" => "Noosfero/#{Noosfero::VERSION}")
52 open(address, header, &block) 46 open(address, header, &block)
53 end 47 end
54 return content 48 return content
@@ -57,6 +51,17 @@ class FeedHandler @@ -57,6 +51,17 @@ class FeedHandler
57 end 51 end
58 end 52 end
59 53
  54 + def fetch_through_proxy(address, environment)
  55 + header = {}
  56 + if address.starts_with?("https://")
  57 + header.merge!(:proxy => environment.https_feed_proxy) if environment.https_feed_proxy
  58 + else
  59 + header.merge!(:proxy => environment.http_feed_proxy) if environment.http_feed_proxy
  60 + end
  61 + header.merge!(:ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE) if environment.disable_feed_ssl
  62 + fetch(address, header)
  63 + end
  64 +
60 def process(container) 65 def process(container)
61 begin 66 begin
62 container.class.transaction do 67 container.class.transaction do
@@ -97,7 +102,11 @@ class FeedHandler @@ -97,7 +102,11 @@ class FeedHandler
97 102
98 def actually_process_container(container) 103 def actually_process_container(container)
99 container.clear 104 container.clear
100 - content = fetch(container.address) 105 + if container.environment.enable_feed_proxy
  106 + content = fetch_through_proxy(container.address, container.environment)
  107 + else
  108 + content = fetch(container.address)
  109 + end
101 container.fetched_at = Time.now 110 container.fetched_at = Time.now
102 parsed_feed = parse(content) 111 parsed_feed = parse(content)
103 container.feed_title = parsed_feed.title 112 container.feed_title = parsed_feed.title
plugins/spaminator/Gemfile
1 -gem "progressbar" 1 +gem "ruby-progressbar"
plugins/spaminator/lib/spaminator_plugin/spaminator.rb
@@ -72,14 +72,14 @@ class SpaminatorPlugin::Spaminator @@ -72,14 +72,14 @@ class SpaminatorPlugin::Spaminator
72 self.class.log("Starting to process all comments") 72 self.class.log("Starting to process all comments")
73 comments = comments_to_process 73 comments = comments_to_process
74 total = comments.count 74 total = comments.count
75 - pbar = ProgressBar.new("☢ Comments", total) if Rails.env.development? 75 + pbar = ProgressBar.create(title: "☢ Comments", total: total, format: '%t: |%B| %E') if Rails.env.development?
76 comments.each do |comment| 76 comments.each do |comment|
77 begin 77 begin
78 process_comment(comment) 78 process_comment(comment)
79 rescue 79 rescue
80 register_fail(:comments, comment) 80 register_fail(:comments, comment)
81 end 81 end
82 - pbar.inc if Rails.env.development? 82 + pbar.increment if Rails.env.development?
83 end 83 end
84 @report.processed_comments = total 84 @report.processed_comments = total
85 pbar.finish if Rails.env.development? 85 pbar.finish if Rails.env.development?
@@ -90,11 +90,11 @@ class SpaminatorPlugin::Spaminator @@ -90,11 +90,11 @@ class SpaminatorPlugin::Spaminator
90 self.class.log("Starting to process all people") 90 self.class.log("Starting to process all people")
91 people = people_to_process 91 people = people_to_process
92 total = people.count 92 total = people.count
93 - pbar = ProgressBar.new("☢ People", total) if Rails.env.development? 93 + pbar = ProgressBar.create(title: "☢ People", total: total, format: '%t: |%B| %E') if Rails.env.development?
94 people.find_each do |person| 94 people.find_each do |person|
95 process_person_by_comments(person) 95 process_person_by_comments(person)
96 process_person_by_no_network(person) 96 process_person_by_no_network(person)
97 - pbar.inc if Rails.env.development? 97 + pbar.increment if Rails.env.development?
98 end 98 end
99 @report.processed_people = total 99 @report.processed_people = total
100 pbar.finish if Rails.env.development? 100 pbar.finish if Rails.env.development?
test/unit/environment_test.rb
@@ -1723,4 +1723,42 @@ class EnvironmentTest &lt; ActiveSupport::TestCase @@ -1723,4 +1723,42 @@ class EnvironmentTest &lt; ActiveSupport::TestCase
1723 refute environment.errors[:date_format.to_s].present? 1723 refute environment.errors[:date_format.to_s].present?
1724 end 1724 end
1725 1725
  1726 + should 'respond to enable_feed_proxy' do
  1727 + assert_respond_to Environment.new, :enable_feed_proxy
  1728 + end
  1729 +
  1730 + should 'set enable_feed_proxy on environment' do
  1731 + e = fast_create(Environment, :name => 'Enterprise test')
  1732 + e.enable_feed_proxy = true
  1733 + e.save
  1734 + assert_equal true, e.enable_feed_proxy
  1735 + end
  1736 +
  1737 + should 'not enable feed proxy when enable by default' do
  1738 + assert_equal false, Environment.new.enable_feed_proxy
  1739 + end
  1740 +
  1741 + should 'respond to disable_feed_ssl' do
  1742 + assert_respond_to Environment.new, :disable_feed_ssl
  1743 + end
  1744 +
  1745 + should 'set disable_feed_ssl on environment' do
  1746 + e = fast_create(Environment, :name => 'Enterprise test')
  1747 + e.disable_feed_ssl = true
  1748 + e.save
  1749 + assert_equal true, e.disable_feed_ssl
  1750 + end
  1751 +
  1752 + should 'not disable feed ssl when enable by default' do
  1753 + assert_equal false, Environment.new.disable_feed_ssl
  1754 + end
  1755 +
  1756 + should 'respond to http_feed_proxy' do
  1757 + assert_respond_to Environment.new, :http_feed_proxy
  1758 + end
  1759 +
  1760 + should 'respond to https_feed_proxy' do
  1761 + assert_respond_to Environment.new, :https_feed_proxy
  1762 + end
  1763 +
1726 end 1764 end
test/unit/feed_handler_test.rb
@@ -12,7 +12,7 @@ class FeedHandlerTest &lt; ActiveSupport::TestCase @@ -12,7 +12,7 @@ class FeedHandlerTest &lt; ActiveSupport::TestCase
12 @container ||= create(:feed_reader_block) 12 @container ||= create(:feed_reader_block)
13 end 13 end
14 14
15 - should 'fetch feed content' do 15 + should 'fetch feed content with proxy disabled and SSL enabled' do
16 content = handler.fetch(container.address) 16 content = handler.fetch(container.address)
17 assert_match /<description>Feed content<\/description>/, content 17 assert_match /<description>Feed content<\/description>/, content
18 assert_match /<title>Feed for unit tests<\/title>/, content 18 assert_match /<title>Feed for unit tests<\/title>/, content
@@ -28,6 +28,7 @@ class FeedHandlerTest &lt; ActiveSupport::TestCase @@ -28,6 +28,7 @@ class FeedHandlerTest &lt; ActiveSupport::TestCase
28 end 28 end
29 29
30 should 'process feed and populate container' do 30 should 'process feed and populate container' do
  31 + container.stubs(:environment).returns(Environment.new)
31 handler.process(container) 32 handler.process(container)
32 assert_equal 'Feed for unit tests', container.feed_title 33 assert_equal 'Feed for unit tests', container.feed_title
33 assert_equivalent ["First POST", "Second POST", "Last POST"], container.feed_items.map {|item| item[:title]} 34 assert_equivalent ["First POST", "Second POST", "Last POST"], container.feed_items.map {|item| item[:title]}
@@ -62,6 +63,7 @@ class FeedHandlerTest &lt; ActiveSupport::TestCase @@ -62,6 +63,7 @@ class FeedHandlerTest &lt; ActiveSupport::TestCase
62 end 63 end
63 64
64 should 'save only latest N posts from feed' do 65 should 'save only latest N posts from feed' do
  66 + container.stubs(:environment).returns(Environment.new)
65 container.limit = 1 67 container.limit = 1
66 handler.process(container) 68 handler.process(container)
67 assert_equal 1, container.feed_items.size 69 assert_equal 1, container.feed_items.size
@@ -85,6 +87,7 @@ class FeedHandlerTest &lt; ActiveSupport::TestCase @@ -85,6 +87,7 @@ class FeedHandlerTest &lt; ActiveSupport::TestCase
85 87
86 should 'identifies itself as noosfero user agent' do 88 should 'identifies itself as noosfero user agent' do
87 handler.expects(:open).with('http://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}"}, anything).returns('bli content') 89 handler.expects(:open).with('http://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}"}, anything).returns('bli content')
  90 +
88 assert_equal 'bli content', handler.fetch('http://site.org/feed.xml') 91 assert_equal 'bli content', handler.fetch('http://site.org/feed.xml')
89 end 92 end
90 93
@@ -150,36 +153,73 @@ class FeedHandlerTest &lt; ActiveSupport::TestCase @@ -150,36 +153,73 @@ class FeedHandlerTest &lt; ActiveSupport::TestCase
150 end 153 end
151 end 154 end
152 155
153 - should 'set proxy when FEED_HTTP_PROXY is setted from env' do  
154 - ENV.stubs('[]').with('FEED_HTTP_PROXY').returns('http://127.0.0.1:3128') 156 + should 'set proxy when http_feed_proxy is setted from env' do
  157 + env = Environment.new
  158 + env.expects(:disable_feed_ssl).returns(false)
  159 + env.expects(:http_feed_proxy).twice.returns('http://127.0.0.1:3128')
  160 +
155 handler.expects(:open).with('http://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}", :proxy => 'http://127.0.0.1:3128'}, anything).returns('bli content') 161 handler.expects(:open).with('http://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}", :proxy => 'http://127.0.0.1:3128'}, anything).returns('bli content')
156 - assert_equal 'bli content', handler.fetch('http://site.org/feed.xml') 162 +
  163 + assert_equal 'bli content', handler.fetch_through_proxy('http://site.org/feed.xml', env)
157 end 164 end
158 165
159 - should 'set proxy when FEED_HTTPS_PROXY is setted from env' do  
160 - ENV.stubs('[]').with('FEED_HTTPS_PROXY').returns('http://127.0.0.1:3128') 166 + should 'set proxy when https_feed_proxy is setted from env' do
  167 + env = Environment.new
  168 + env.expects(:disable_feed_ssl).returns(false)
  169 + env.expects(:https_feed_proxy).twice.returns('http://127.0.0.1:3128')
  170 +
161 handler.expects(:open).with('https://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}", :proxy => 'http://127.0.0.1:3128'}, anything).returns('bli content') 171 handler.expects(:open).with('https://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}", :proxy => 'http://127.0.0.1:3128'}, anything).returns('bli content')
162 - assert_equal 'bli content', handler.fetch('https://site.org/feed.xml') 172 +
  173 + assert_equal 'bli content', handler.fetch_through_proxy('https://site.org/feed.xml', env)
163 end 174 end
164 175
165 should 'use https proxy for https address when both env variables were defined' do 176 should 'use https proxy for https address when both env variables were defined' do
166 - ENV.stubs('[]').with('FEED_HTTPS_PROXY').returns('http://127.0.0.2:3128')  
167 - ENV.stubs('[]').with('FEED_HTTP_PROXY').returns('http://127.0.0.1:3128') 177 + env = Environment.new
  178 + env.expects(:disable_feed_ssl).returns(false)
  179 + env.expects(:https_feed_proxy).twice.returns('http://127.0.0.2:3128')
  180 +
168 handler.expects(:open).with('https://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}", :proxy => 'http://127.0.0.2:3128'}, anything).returns('bli content') 181 handler.expects(:open).with('https://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}", :proxy => 'http://127.0.0.2:3128'}, anything).returns('bli content')
169 - assert_equal 'bli content', handler.fetch('https://site.org/feed.xml') 182 +
  183 + assert_equal 'bli content', handler.fetch_through_proxy('https://site.org/feed.xml', env)
170 end 184 end
171 185
172 should 'use http proxy for http address when both env variables were defined' do 186 should 'use http proxy for http address when both env variables were defined' do
173 - ENV.stubs('[]').with('FEED_HTTPS_PROXY').returns('http://127.0.0.2:3128')  
174 - ENV.stubs('[]').with('FEED_HTTP_PROXY').returns('http://127.0.0.1:3128') 187 + env = Environment.new
  188 + env.expects(:disable_feed_ssl).returns(false)
  189 + env.expects(:http_feed_proxy).twice.returns('http://127.0.0.1:3128')
  190 +
175 handler.expects(:open).with('http://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}", :proxy => 'http://127.0.0.1:3128'}, anything).returns('bli content') 191 handler.expects(:open).with('http://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}", :proxy => 'http://127.0.0.1:3128'}, anything).returns('bli content')
176 - assert_equal 'bli content', handler.fetch('http://site.org/feed.xml') 192 +
  193 + assert_equal 'bli content', handler.fetch_through_proxy('http://site.org/feed.xml', env)
177 end 194 end
178 195
179 should 'not verify ssl when define env parameter SSL_VERIFY_NONE' do 196 should 'not verify ssl when define env parameter SSL_VERIFY_NONE' do
180 - ENV.stubs('[]').with('SSL_VERIFY_NONE').returns(true) 197 + env = Environment.new
  198 + env.expects(:disable_feed_ssl).returns(true)
  199 +
181 handler.expects(:open).with('http://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}", :ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE}, anything) 200 handler.expects(:open).with('http://site.org/feed.xml', {"User-Agent" => "Noosfero/#{Noosfero::VERSION}", :ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE}, anything)
182 - handler.fetch('http://site.org/feed.xml') 201 +
  202 + handler.fetch_through_proxy('http://site.org/feed.xml', env)
  203 + end
  204 +
  205 + [:external_feed, :feed_reader_block].each do |container_class|
  206 + should "not use proxy settings for #{container_class} from one environment on another" do
  207 + one = create(:environment)
  208 + one.expects(:enable_feed_proxy).returns(true)
  209 + container_one = create(container_class)
  210 + container_one.stubs(:environment).returns(one)
  211 +
  212 + handler.expects(:fetch_through_proxy).with(container_one.address, one)
  213 + handler.process(container_one)
  214 +
  215 + another = create(:environment)
  216 + another.expects(:enable_feed_proxy).returns(false)
  217 + container_another = create(container_class)
  218 + container_another.stubs(:environment).returns(another)
  219 +
  220 + handler.expects(:fetch_through_proxy).never.with(container_another.address, another)
  221 + handler.process(container_another)
  222 + end
183 end 223 end
184 224
185 end 225 end