Download as
Browse Code »

Commit 7279e8c1774f3116dd74ca462447189730f9eff8

Authored by Saito
1 parent 5719f02f
Exists in master and in 4 other branches 6-8-stable, 7-0-stable, 7-0-stable-spb, spb-stable

Revert "merge charlock_holmes to master" 

This reverts commit a25a85b9396cbc07bf2bfcd12d3c0a6685df66d6.
Showing 3 changed files with 11 additions and 9 deletions   Show diff stats
@@ -22,9 +22,9 @@ gem "acts_as_list" @@ -22,9 +22,9 @@ gem "acts_as_list"
22 gem "rdiscount" 22 gem "rdiscount"
23 gem "acts-as-taggable-on", "~> 2.1.0" 23 gem "acts-as-taggable-on", "~> 2.1.0"
24 gem "drapper" 24 gem "drapper"
  25 +gem "rchardet19", "~> 1.3.5"
25 gem "resque" 26 gem "resque"
26 gem "httparty" 27 gem "httparty"
27 -gem "charlock_holmes"  
28 28
29 group :assets do 29 group :assets do
30 gem "sass-rails", "~> 3.1.0" 30 gem "sass-rails", "~> 3.1.0"
@@ -77,7 +77,6 @@ GEM @@ -77,7 +77,6 @@ GEM
77 xpath (~> 0.1.4) 77 xpath (~> 0.1.4)
78 carrierwave (0.5.8) 78 carrierwave (0.5.8)
79 activesupport (~> 3.0) 79 activesupport (~> 3.0)
80 - charlock_holmes (0.6.8)  
81 childprocess (0.2.2) 80 childprocess (0.2.2)
82 ffi (~> 1.0.6) 81 ffi (~> 1.0.6)
83 coffee-rails (3.1.1) 82 coffee-rails (3.1.1)
@@ -173,6 +172,7 @@ GEM @@ -173,6 +172,7 @@ GEM
173 rdoc (~> 3.4) 172 rdoc (~> 3.4)
174 thor (~> 0.14.6) 173 thor (~> 0.14.6)
175 rake (0.9.2.2) 174 rake (0.9.2.2)
  175 + rchardet19 (1.3.5)
176 rdiscount (1.6.8) 176 rdiscount (1.6.8)
177 rdoc (3.11) 177 rdoc (3.11)
178 json (~> 1.4) 178 json (~> 1.4)
@@ -285,7 +285,6 @@ DEPENDENCIES @@ -285,7 +285,6 @@ DEPENDENCIES
285 awesome_print 285 awesome_print
286 capybara 286 capybara
287 carrierwave 287 carrierwave
288 - charlock_holmes  
289 coffee-rails (~> 3.1.0) 288 coffee-rails (~> 3.1.0)
290 database_cleaner 289 database_cleaner
291 devise (= 1.5.0) 290 devise (= 1.5.0)
@@ -303,6 +302,7 @@ DEPENDENCIES @@ -303,6 +302,7 @@ DEPENDENCIES
303 pygments.rb (= 0.2.3) 302 pygments.rb (= 0.2.3)
304 rails (= 3.1.1) 303 rails (= 3.1.1)
305 rails-footnotes (~> 3.7.5) 304 rails-footnotes (~> 3.7.5)
  305 + rchardet19 (~> 1.3.5)
306 rdiscount 306 rdiscount
307 resque 307 resque
308 rspec-rails 308 rspec-rails
@@ -17,13 +17,15 @@ module Utils @@ -17,13 +17,15 @@ module Utils
17 end 17 end
18 18
19 module CharEncode 19 module CharEncode
20 - def encode(content)  
21 - content ||= ''  
22 - detection = CharlockHolmes::EncodingDetector.detect(content)  
23 - if hash = detection  
24 - content = CharlockHolmes::Converter.convert(content, hash[:encoding], 'UTF-8') if hash[:encoding] 20 + def encode(string)
  21 + return '' unless string
  22 + cd = CharDet.detect(string)
  23 + if cd.confidence > 0.6
  24 + string.force_encoding(cd.encoding)
25 end 25 end
26 - content 26 + string.encode("utf-8", :undef => :replace, :replace => "?", :invalid => :replace)
  27 + rescue
  28 + "Invalid Encoding"
27 end 29 end
28 end 30 end
29 31