fix the issue on github #157.

directly force_encoding is wrong, must detect the file string's encoding. then force_encoding the string to it's encoding. last convert it to utf-8.

fix the issue on github #157.
directly force_encoding is wrong, must detect the file string's encoding. then force_encoding the string to it's encoding. last convert it to utf-8.
Saito
1 parent 75fa0632
Showing 5 changed files with 23 additions and 8 deletions Show diff stats
app/helpers/commits_helper.rb
app/models/commit.rb
app/views/commits/_text_file.html.haml
lib/graph_commit.rb
lib/utils.rb
 module CommitsHelper
+  include Utils::CharEncode
   def diff_line(line, line_new = 0, line_old = 0)
     full_line = html_escape(line.gsub(/\n/, ''))
     color = if line[0] == "+"
 class Commit
+  include Utils::CharEncode
+
   attr_accessor :commit
   attr_accessor :head
  
@@ -20,7 +22,7 @@ class Commit
   end
  
   def safe_message
-    message.force_encoding(Encoding::UTF_8)
+    encode(message)
   end
  
   def created_at
@@ -28,10 +30,10 @@ class Commit
   end
  
   def author_email
-    author.email.force_encoding(Encoding::UTF_8)
+    encode(author.email)
   end
  
   def author_name
-    author.name.force_encoding(Encoding::UTF_8)
+    encode(author.name)
   end
 end
@@ -2,7 +2,7 @@
 - line_new = 0
 - lines_arr = diff.diff.lines.to_a
 - lines_arr.each do |line|
-  - line.force_encoding(Encoding::UTF_8)
+  - encode(line)
   - next if line.match(/^--- \/dev\/null/)
   - next if line.match(/^--- a/)
   - next if line.match(/^\+\+\+ b/)
 require "grit"
  
 class GraphCommit
+  include Utils::CharEncode
   attr_accessor :time, :space
   attr_accessor :refs
  
@@ -65,7 +66,7 @@ class GraphCommit
   # @param [GraphCommit] the commit object.
   # @param [Hash<String,GraphCommit>] map of commits
   #
-  # @return [Fixnum] max space used.  
+  # @return [Fixnum] max space used.
   def self.mark_chain(mark, commit, map)
     commit.space = mark  if commit.space == 0
     m1 = mark - 1
@@ -96,13 +97,13 @@ class GraphCommit
     h[:parents] = self.parents.collect do |p|
       [p.id,0,0]
     end
-    h[:author]  = author.name.force_encoding("UTF-8")
+    h[:author]  = encode(author.name)
     h[:time]    = time
     h[:space]   = space
     h[:refs]    = refs.collect{|r|r.name}.join(" ") unless refs.nil?
     h[:id]      = sha
     h[:date]    = date
-    h[:message] = message.force_encoding("UTF-8")
+    h[:message] = encode(message)
     h[:login]   = author.email
     h
   end
@@ -16,9 +16,20 @@ module Utils
     end
   end
  
+  module CharEncode
+    def encode(string)
+      cd = CharDet.detect(string)
+      if cd.confidence > 0.6
+        string.force_encoding(cd.encoding)
+      end
+      string.encode("utf-8", :undef => :replace, :replace => "?", :invalid => :replace)
+    end
+  end
+
   module Colorize
+    include CharEncode
     def colorize
-      system_colorize(data, name)
+      system_colorize(encode(data), name)
     end
  
     def system_colorize(data, file_name)
1	1	module CommitsHelper
	2	+ include Utils::CharEncode
2	3	def diff_line(line, line_new = 0, line_old = 0)
3	4	full_line = html_escape(line.gsub(/\n/, ''))
4	5	color = if line[0] == "+"
...	...
1	1	class Commit
	2	+ include Utils::CharEncode
	3	+
2	4	attr_accessor :commit
3	5	attr_accessor :head
4	6
...	...	@@ -20,7 +22,7 @@ class Commit
20	22	end
21	23
22	24	def safe_message
23		- message.force_encoding(Encoding::UTF_8)
	25	+ encode(message)
24	26	end
25	27
26	28	def created_at
...	...	@@ -28,10 +30,10 @@ class Commit
28	30	end
29	31
30	32	def author_email
31		- author.email.force_encoding(Encoding::UTF_8)
	33	+ encode(author.email)
32	34	end
33	35
34	36	def author_name
35		- author.name.force_encoding(Encoding::UTF_8)
	37	+ encode(author.name)
36	38	end
37	39	end
...	...
...	...	@@ -2,7 +2,7 @@
2	2	- line_new = 0
3	3	- lines_arr = diff.diff.lines.to_a
4	4	- lines_arr.each do \|line\|
5		- - line.force_encoding(Encoding::UTF_8)
	5	+ - encode(line)
6	6	- next if line.match(/^--- \/dev\/null/)
7	7	- next if line.match(/^--- a/)
8	8	- next if line.match(/^\+\+\+ b/)
...	...
1	1	require "grit"
2	2
3	3	class GraphCommit
	4	+ include Utils::CharEncode
4	5	attr_accessor :time, :space
5	6	attr_accessor :refs
6	7
...	...	@@ -65,7 +66,7 @@ class GraphCommit
65	66	# @param [GraphCommit] the commit object.
66	67	# @param [Hash<String,GraphCommit>] map of commits
67	68	#
68		- # @return [Fixnum] max space used.
	69	+ # @return [Fixnum] max space used.
69	70	def self.mark_chain(mark, commit, map)
70	71	commit.space = mark if commit.space == 0
71	72	m1 = mark - 1
...	...	@@ -96,13 +97,13 @@ class GraphCommit
96	97	h[:parents] = self.parents.collect do \|p\|
97	98	[p.id,0,0]
98	99	end
99		- h[:author] = author.name.force_encoding("UTF-8")
	100	+ h[:author] = encode(author.name)
100	101	h[:time] = time
101	102	h[:space] = space
102	103	h[:refs] = refs.collect{\|r\|r.name}.join(" ") unless refs.nil?
103	104	h[:id] = sha
104	105	h[:date] = date
105		- h[:message] = message.force_encoding("UTF-8")
	106	+ h[:message] = encode(message)
106	107	h[:login] = author.email
107	108	h
108	109	end
...	...
...	...	@@ -16,9 +16,20 @@ module Utils
16	16	end
17	17	end
18	18
	19	+ module CharEncode
	20	+ def encode(string)
	21	+ cd = CharDet.detect(string)
	22	+ if cd.confidence > 0.6
	23	+ string.force_encoding(cd.encoding)
	24	+ end
	25	+ string.encode("utf-8", :undef => :replace, :replace => "?", :invalid => :replace)
	26	+ end
	27	+ end
	28	+
19	29	module Colorize
	30	+ include CharEncode
20	31	def colorize
21		- system_colorize(data, name)
	32	+ system_colorize(encode(data), name)
22	33	end
23	34
24	35	def system_colorize(data, file_name)
...	...