Merge pull request #868 from SaitoWu/bugfix/encoding

Bugfix/encoding

Merge pull request #868 from SaitoWu/bugfix/encoding
Bugfix/encoding
Dmitriy Zaporozhets
2 parents af226ae9 e851cb07
Showing 6 changed files with 38 additions and 29 deletions Show diff stats
app/helpers/commits_helper.rb
app/views/refs/_tree.html.haml
app/views/refs/_tree_file.html.haml
config/initializers/gitlabhq/20_grit_ext.rb
lib/gitlab/encode.rb
lib/graph_commit.rb
@@ -58,14 +58,14 @@ module CommitsHelper
       next if line.match(/^\-\-\- a/)
       next if line.match(/^\+\+\+ b/)
  
-      full_line = html_escape(line.gsub(/\n/, '')).force_encoding("UTF-8")
+      full_line = html_escape(line.gsub(/\n/, ''))
  
       if line.match(/^@@ -/)
         type = "match"
  
         line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0
         line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0
-                
+
         next if line_old == 1 && line_new == 1
         yield(full_line, type, nil, nil, nil)
         next
@@ -42,9 +42,9 @@
         .readme
           - if content.name =~ /\.(md|markdown)$/i
             = preserve do
-              = markdown(content.data.detect_encoding!)
+              = markdown(content.data)
           - else
-            = simple_format(content.data.detect_encoding!)
+            = simple_format(content.data)
  
 - if params[:path]
   - history_path = tree_file_project_ref_path(@project, @ref, params[:path])
@@ -13,7 +13,7 @@
       #tree-readme-holder
         .readme
           = preserve do
-            = markdown(file.data.detect_encoding!)
+            = markdown(file.data)
     - else
       .view_file_content
         - unless file.empty?
 require 'grit'
 require 'pygments'
  
+Grit::Git.git_timeout = GIT_OPTS["git_timeout"]
+Grit::Git.git_max_size = GIT_OPTS["git_max_size"]
+
 Grit::Blob.class_eval do
   include Linguist::BlobHelper
-end
  
-#monkey patch raw_object from string
-Grit::GitRuby::Internal::RawObject.class_eval do
-  def content
-    @content
+  def data
+    @data ||= @repo.git.cat_file({:p => true}, id)
+    Gitlab::Encode.utf8 @data
   end
 end
  
-
 Grit::Diff.class_eval do
   def old_path
-    Gitlab::Encode.utf8 a_path
+    Gitlab::Encode.utf8 @a_path
   end
  
   def new_path
-    Gitlab::Encode.utf8 b_path
+    Gitlab::Encode.utf8 @b_path
   end
-end
  
-Grit::Git.git_timeout = GIT_OPTS["git_timeout"]
-Grit::Git.git_max_size = GIT_OPTS["git_max_size"]
+  def diff
+    Gitlab::Encode.utf8 @diff
+  end
+end
 # Patch Strings to enable detect_encoding! on views
 require 'charlock_holmes/string'
 module Gitlab
-  module Encode 
+  module Encode
     extend self
  
     def utf8 message
+      # return nil if message is nil
       return nil unless message
  
-      detect = CharlockHolmes::EncodingDetector.detect(message) rescue {}
+      message.force_encoding("utf-8")
+      # return message if message type is binary
+      detect = CharlockHolmes::EncodingDetector.detect(message)
+      return message if detect[:type] == :binary
  
-      # It's better to default to UTF-8 as sometimes it's wrongly detected as another charset
-      if detect[:encoding] && detect[:confidence] == 100
-        CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
-      else
-        message
-      end.force_encoding("utf-8")
+      # if message is utf-8 encoding, just return it
+      return message if message.valid_encoding?
  
-    # Prevent app from crash cause of 
-    # encoding errors
+      # if message is not utf-8 encoding, convert it
+      if detect[:encoding]
+        message.force_encoding(detect[:encoding])
+        message.encode!("utf-8", detect[:encoding], :undef => :replace, :replace => "", :invalid => :replace)
+      end
+
+      # ensure message encoding is utf8
+      message.valid_encoding? ? message : raise
+
+    # Prevent app from crash cause of encoding errors
     rescue
-      "--broken encoding: #{encoding}"
+      "--broken encoding: #{detect[:encoding]}"
     end
  
     def detect_encoding message
@@ -96,13 +96,13 @@ class GraphCommit
     h[:parents] = self.parents.collect do |p|
       [p.id,0,0]
     end
-    h[:author]  = author.name.force_encoding("UTF-8")
+    h[:author]  = Gitlab::Encode.utf8(author.name)
     h[:time]    = time
     h[:space]   = space
     h[:refs]    = refs.collect{|r|r.name}.join(" ") unless refs.nil?
     h[:id]      = sha
     h[:date]    = date
-    h[:message] = message.force_encoding("UTF-8")
+    h[:message] = Gitlab::Encode.utf8(message)
     h[:login]   = author.email
     h
   end
...	...	@@ -58,14 +58,14 @@ module CommitsHelper
58	58	next if line.match(/^\-\-\- a/)
59	59	next if line.match(/^\+\+\+ b/)
60	60
61		- full_line = html_escape(line.gsub(/\n/, '')).force_encoding("UTF-8")
	61	+ full_line = html_escape(line.gsub(/\n/, ''))
62	62
63	63	if line.match(/^@@ -/)
64	64	type = "match"
65	65
66	66	line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0
67	67	line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0
68		-
	68	+
69	69	next if line_old == 1 && line_new == 1
70	70	yield(full_line, type, nil, nil, nil)
71	71	next
...	...
...	...	@@ -42,9 +42,9 @@
42	42	.readme
43	43	- if content.name =~ /\.(md\|markdown)$/i
44	44	= preserve do
45		- = markdown(content.data.detect_encoding!)
	45	+ = markdown(content.data)
46	46	- else
47		- = simple_format(content.data.detect_encoding!)
	47	+ = simple_format(content.data)
48	48
49	49	- if params[:path]
50	50	- history_path = tree_file_project_ref_path(@project, @ref, params[:path])
...	...
...	...	@@ -13,7 +13,7 @@
13	13	#tree-readme-holder
14	14	.readme
15	15	= preserve do
16		- = markdown(file.data.detect_encoding!)
	16	+ = markdown(file.data)
17	17	- else
18	18	.view_file_content
19	19	- unless file.empty?
...	...
1	1	require 'grit'
2	2	require 'pygments'
3	3
	4	+Grit::Git.git_timeout = GIT_OPTS["git_timeout"]
	5	+Grit::Git.git_max_size = GIT_OPTS["git_max_size"]
	6	+
4	7	Grit::Blob.class_eval do
5	8	include Linguist::BlobHelper
6		-end
7	9
8		-#monkey patch raw_object from string
9		-Grit::GitRuby::Internal::RawObject.class_eval do
10		- def content
11		- @content
	10	+ def data
	11	+ @data \|\|= @repo.git.cat_file({:p => true}, id)
	12	+ Gitlab::Encode.utf8 @data
12	13	end
13	14	end
14	15
15		-
16	16	Grit::Diff.class_eval do
17	17	def old_path
18		- Gitlab::Encode.utf8 a_path
	18	+ Gitlab::Encode.utf8 @a_path
19	19	end
20	20
21	21	def new_path
22		- Gitlab::Encode.utf8 b_path
	22	+ Gitlab::Encode.utf8 @b_path
23	23	end
24		-end
25	24
26		-Grit::Git.git_timeout = GIT_OPTS["git_timeout"]
27		-Grit::Git.git_max_size = GIT_OPTS["git_max_size"]
	25	+ def diff
	26	+ Gitlab::Encode.utf8 @diff
	27	+ end
	28	+end
...	...
1	1	# Patch Strings to enable detect_encoding! on views
2	2	require 'charlock_holmes/string'
3	3	module Gitlab
4		- module Encode
	4	+ module Encode
5	5	extend self
6	6
7	7	def utf8 message
	8	+ # return nil if message is nil
8	9	return nil unless message
9	10
10		- detect = CharlockHolmes::EncodingDetector.detect(message) rescue {}
	11	+ message.force_encoding("utf-8")
	12	+ # return message if message type is binary
	13	+ detect = CharlockHolmes::EncodingDetector.detect(message)
	14	+ return message if detect[:type] == :binary
11	15
12		- # It's better to default to UTF-8 as sometimes it's wrongly detected as another charset
13		- if detect[:encoding] && detect[:confidence] == 100
14		- CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
15		- else
16		- message
17		- end.force_encoding("utf-8")
	16	+ # if message is utf-8 encoding, just return it
	17	+ return message if message.valid_encoding?
18	18
19		- # Prevent app from crash cause of
20		- # encoding errors
	19	+ # if message is not utf-8 encoding, convert it
	20	+ if detect[:encoding]
	21	+ message.force_encoding(detect[:encoding])
	22	+ message.encode!("utf-8", detect[:encoding], :undef => :replace, :replace => "", :invalid => :replace)
	23	+ end
	24	+
	25	+ # ensure message encoding is utf8
	26	+ message.valid_encoding? ? message : raise
	27	+
	28	+ # Prevent app from crash cause of encoding errors
21	29	rescue
22		- "--broken encoding: #{encoding}"
	30	+ "--broken encoding: #{detect[:encoding]}"
23	31	end
24	32
25	33	def detect_encoding message
...	...
...	...	@@ -96,13 +96,13 @@ class GraphCommit
96	96	h[:parents] = self.parents.collect do \|p\|
97	97	[p.id,0,0]
98	98	end
99		- h[:author] = author.name.force_encoding("UTF-8")
	99	+ h[:author] = Gitlab::Encode.utf8(author.name)
100	100	h[:time] = time
101	101	h[:space] = space
102	102	h[:refs] = refs.collect{\|r\|r.name}.join(" ") unless refs.nil?
103	103	h[:id] = sha
104	104	h[:date] = date
105		- h[:message] = message.force_encoding("UTF-8")
	105	+ h[:message] = Gitlab::Encode.utf8(message)
106	106	h[:login] = author.email
107	107	h
108	108	end
...	...