Remove caractere especial 'marcador' ao classificar sentença.

Erickson Silva
1 parent a5237893
Showing 1 changed file with 2 additions and 1 deletions Show diff stats
src/ClassificaSentencas.py
@@ -50,6 +50,7 @@ class ClassificaSentencas(object):
 			decodificada = regex2.sub('',regex.sub('"',s.replace("–", "-").replace("—", "-"))).decode("utf-8")
 		except:
 			decodificada = s.decode("utf-8")
+
 		return Toqueniza.TOK_PORT.tokenize(decodificada)
  
 	def obter_classificacao_morfologica(self):
@@ -63,7 +64,7 @@ class ClassificaSentencas(object):
 		while (anotada[0][1] is None):
 			time.sleep(random.choice(sleep_times))
 			anotada = AnotaCorpus.anota_sentencas([s],etiquetador,"hunpos")[0]
-		regex = re.compile('[%s]' % re.escape('!"#&()*+,-./:;<=>?@[\]^_`{|}~'))
+		regex = re.compile('[%s]' % re.escape(u'\u2022''!"#&()*+,-./:;<=>?@[\]^_`{|}~'))
 		tag_punctuation = [".",",","QT","("]
 		anotada_corrigida = []
 		for x in anotada:
...	...	@@ -50,6 +50,7 @@ class ClassificaSentencas(object):
50	50	decodificada = regex2.sub('',regex.sub('"',s.replace("–", "-").replace("—", "-"))).decode("utf-8")
51	51	except:
52	52	decodificada = s.decode("utf-8")
	53	+
53	54	return Toqueniza.TOK_PORT.tokenize(decodificada)
54	55
55	56	def obter_classificacao_morfologica(self):
...	...	@@ -63,7 +64,7 @@ class ClassificaSentencas(object):
63	64	while (anotada[0][1] is None):
64	65	time.sleep(random.choice(sleep_times))
65	66	anotada = AnotaCorpus.anota_sentencas([s],etiquetador,"hunpos")[0]
66		- regex = re.compile('[%s]' % re.escape('!"#&()*+,-./:;<=>?@[\]^_`{\|}~'))
	67	+ regex = re.compile('[%s]' % re.escape(u'\u2022''!"#&()*+,-./:;<=>?@[\]^_`{\|}~'))
67	68	tag_punctuation = [".",",","QT","("]
68	69	anotada_corrigida = []
69	70	for x in anotada:
...	...