Download as
Browse Code »

Commit 0b57611984f4692ec270c49faaeca0744cec59eb

Authored by Wesnydy Ribeiro
1 parent 1ccd01c4
Exists in master and in 3 other branches OpenSigns, coppe, exchange

Bundles List service implementation

Showing 5 changed files with 211 additions and 4 deletions   Show diff stats
core/Trie.py 0 → 100755
  Diff comments   View file @0b57611
@@ -0,0 +1,63 @@ @@ -0,0 +1,63 @@
  1 +#!/usr/bin/python
  2 +# -*- coding: utf-8 -*-
  3 +
  4 +class Node:
  5 + def __init__(self, end = False):
  6 + self.end = end
  7 + self.children = {}
  8 +
  9 +class Trie:
  10 + def __init__(self, characters):
  11 + self.characters = characters
  12 + self._parse_characters()
  13 + self.root = Node()
  14 +
  15 + def _parse_characters(self):
  16 + self.characters_keys = {}
  17 + for i in xrange(len(self.characters)):
  18 + self.characters_keys[self.characters[i]] = i
  19 +
  20 + def _key_of(self, item, i):
  21 + return self.characters_keys[item[i].encode('utf-8')]
  22 +
  23 + def _add(self, item):
  24 + node = self.root
  25 + item = list(item.decode('utf-8'))
  26 + for i in xrange(len(item)):
  27 + key = self._key_of(item, i)
  28 + if not node.children.has_key(key):
  29 + node.children[key] = Node()
  30 + node = node.children[key]
  31 + node.end = True
  32 +
  33 + def _to_json(self, node):
  34 + keys = [];
  35 + children = {}
  36 + for key, value in node.children.iteritems():
  37 + keys.append(key)
  38 + children[key] = self._to_json(value)
  39 + keys.sort()
  40 + return { 'end': node.end, 'keys': keys, 'children': children }
  41 +
  42 + def add(self, data):
  43 + if type(data) is list:
  44 + for d in data:
  45 + if not d in ignore:
  46 + self._add(d)
  47 + else:
  48 + self._add(data)
  49 +
  50 + def to_json(self):
  51 + return self._to_json(self.root)
  52 +
  53 +chars = ["'", '$', ',', '_', '%', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
  54 + 'A', 'Á', 'Â', 'Ã', 'À', 'B', 'C', 'Ç', 'D', 'E', 'É', 'Ê', 'F', 'G', 'H',
  55 + 'I', 'Í', 'J', 'K', 'L', 'M', 'N', 'O', 'Ó', 'Ô', 'Õ', 'P', 'Q', 'R', 'S',
  56 + 'T', 'U', 'Ú', 'V', 'W', 'X', 'Y', 'Z']
  57 +
  58 +ignore = ["[INTERROGAÇÃO]", "[EXCLAMAÇÃO]", "[PONTO]"]
  59 +
  60 +def gen(data):
  61 + trie = Trie(chars)
  62 + trie.add(data)
  63 + return { 'characters': chars, 'keys': trie.characters_keys, 'trie': trie.to_json() }
core/indexer.py 0 → 100755
  Diff comments   View file @0b57611
@@ -0,0 +1,128 @@ @@ -0,0 +1,128 @@
  1 +#!/usr/bin/env python
  2 +# -*- coding: utf-8 -*-
  3 +
  4 +"""
  5 +Author: Erickson Silva
  6 +E-Mail: erickson.silva@lavid.ufpb.br
  7 +
  8 +Author: Wesnydy Lima Ribeiro
  9 +E-Mail: wesnydy@lavid.ufpb.br
  10 +"""
  11 +
  12 +import os
  13 +import Trie
  14 +import json
  15 +import pika
  16 +import PikaManager
  17 +
  18 +from time import sleep
  19 +
  20 +TRIE=None
  21 +
  22 +BUNDLES_PATH=None
  23 +BUNDLES_LIST={}
  24 +
  25 +# Manager of queues connections.
  26 +manager = PikaManager.PikaManager("localhost")
  27 +
  28 +def generate_trie():
  29 + global TRIE
  30 + signs = list(BUNDLES_LIST["DEFAULT"])
  31 + TRIE = json.dumps(Trie.gen(signs))
  32 +
  33 +def list_files(path):
  34 + files = []
  35 + for fname in os.listdir(path):
  36 + path_mount = os.path.join(path, fname)
  37 + if not os.path.isdir(path_mount):
  38 + files.append(fname)
  39 + return files
  40 +
  41 +def check_platform_files():
  42 + android = set(list_files(BUNDLES_PATH["ANDROID"]))
  43 + ios = set(list_files(BUNDLES_PATH["IOS"]))
  44 + webgl = set(list_files(BUNDLES_PATH["WEBGL"]))
  45 + standalone = set(list_files(BUNDLES_PATH["STANDALONE"]))
  46 + if android == ios and ios == webgl and webgl == standalone:
  47 + return standalone
  48 + raise RuntimeError("Inconsistent signs. Check files.")
  49 +
  50 +def list_bundles():
  51 + global BUNDLES_LIST
  52 + states = ["AC", "AL", "AP", "AM", "BA", "CE", "DF", "ES", "GO", "MA", "MT",
  53 + "MS", "MG", "PA", "PB", "PR", "PE", "PI", "RJ", "RN", "RS", "RO",
  54 + "RR", "SC", "SP", "SE", "TO"]
  55 + BUNDLES_LIST["DEFAULT"] = check_platform_files()
  56 + for platform, path in BUNDLES_PATH.iteritems():
  57 + BUNDLES_LIST[platform] = {}
  58 + for state in states:
  59 + try:
  60 + BUNDLES_LIST[platform].update({state:set(os.listdir(os.path.join(path, state)))})
  61 + except OSError:
  62 + BUNDLES_LIST[platform].update({state:set([])})
  63 +
  64 +def load_bundles_paths():
  65 + global BUNDLES_PATH
  66 + try:
  67 + SIGNS_PATH=os.environ['SIGNS_VLIBRAS']
  68 + except KeyError:
  69 + raise EnvironmentError("Environment variable 'SIGNS_VLIBRAS' not found.")
  70 + IOS_SIGNS_PATH=os.path.join(SIGNS_PATH, "IOS")
  71 + ANDROID_SIGNS_PATH=os.path.join(SIGNS_PATH, "ANDROID")
  72 + STANDALONE_SIGNS_PATH=os.path.join(SIGNS_PATH, "STANDALONE")
  73 + WEBGL_SIGNS_PATH=os.path.join(SIGNS_PATH, "WEBGL")
  74 + BUNDLES_PATH={"IOS":IOS_SIGNS_PATH, "ANDROID":ANDROID_SIGNS_PATH,
  75 + "STANDALONE":STANDALONE_SIGNS_PATH, "WEBGL":WEBGL_SIGNS_PATH}
  76 + list_bundles()
  77 + generate_trie()
  78 +
  79 +def keep_alive(conn_send, conn_receive):
  80 + """
  81 + Keep the connection alive.
  82 +
  83 + Parameters
  84 + ----------
  85 + conn_send : object
  86 + Connection of writer.
  87 + conn_receive : object
  88 + Connection of receiver.
  89 + """
  90 + while True:
  91 + sleep(30)
  92 + try:
  93 + conn_send.process_data_events()
  94 + conn_receive.process_data_events()
  95 + except:
  96 + continue
  97 +
  98 +# start_new_thread(keep_alive, (manager.get_conn_send(), manager.get_conn_receive()))
  99 +
  100 +def run(ch, method, properties, body):
  101 + """
  102 + Execute the worker.
  103 +
  104 + Parameters
  105 + ----------
  106 + ch : object
  107 + Channel of communication.
  108 + method : function
  109 + Callback method.
  110 + properties : object
  111 + Message containing a set of 14 properties.
  112 + body : string
  113 + Json string containing the necessary arguments for workers.
  114 + """
  115 + print ("Sending list...")
  116 + manager.send_to_queue("lists", TRIE, properties)
  117 + print ("Ok")
  118 +
  119 +print ("Indexing bundles...")
  120 +load_bundles_paths()
  121 +
  122 +print ("Indexer listening...")
  123 +while True:
  124 + try:
  125 + manager.receive_from_queue("signals", run)
  126 + except KeyboardInterrupt:
  127 + manager.close_connections()
  128 + os._exit(0)
translate-api/config/settings.js
  Diff comments   View file @0b57611
@@ -13,7 +13,7 @@ var path = require('path'); @@ -13,7 +13,7 @@ var path = require('path');
13 var config = {}; 13 var config = {};
14 14
15 config.contentsPath = process.env.VLIBRAS_VIDEO_LIBRAS; 15 config.contentsPath = process.env.VLIBRAS_VIDEO_LIBRAS;
16 -config.bundlesPath = process.env.BUNDLES_PATH; 16 +config.bundlesPath = process.env.SIGNS_VLIBRAS;
17 17
18 config.platformsList = [ 'ANDROID', 'IOS', 'WEBGL', 'STANDALONE' ]; 18 config.platformsList = [ 'ANDROID', 'IOS', 'WEBGL', 'STANDALONE' ];
19 19
translate-api/controllers/bundle.js
  Diff comments   View file @0b57611
@@ -9,9 +9,25 @@ @@ -9,9 +9,25 @@
9 * Required libs. 9 * Required libs.
10 */ 10 */
11 var path = require('path') 11 var path = require('path')
  12 + , amqp = require('../helpers/amqpManager')
  13 + , shortid = require('shortid')
12 , settings = require('../config/settings') 14 , settings = require('../config/settings')
13 , error = require('../helpers/error'); 15 , error = require('../helpers/error');
14 16
  17 +exports.index=function(req, res, next) {
  18 + var id = shortid.generate();
  19 + var body = 'signal'; // Just to send something. It has no functionality.
  20 + amqp.sendToQueue(body, id,'signals', false, res, function(err) {
  21 + if (err)
  22 + return error.internalError('An internal communication error has occurred.', next);
  23 + amqp.receiveFromQueue(id, 'lists', false, res, function(err, message) {
  24 + if (err)
  25 + return error.internalError('An internal communication error has occurred.', next);
  26 + res.status(200).json(JSON.parse(message));
  27 + });
  28 + });
  29 +};
  30 +
15 exports.show=function(req, res, next) { 31 exports.show=function(req, res, next) {
16 /** 32 /**
17 * Check the required params. 33 * Check the required params.
translate-api/routes/bundle.js
  Diff comments   View file @0b57611
@@ -5,13 +5,13 @@ @@ -5,13 +5,13 @@
5 5
6 var express = require('express') 6 var express = require('express')
7 , router = express.Router() 7 , router = express.Router()
8 - , bundlesController = require('../controllers/bundle'); 8 + , bundleController = require('../controllers/bundle');
9 9
10 /** 10 /**
11 * Routes to get bundles of a specific platform. 11 * Routes to get bundles of a specific platform.
12 */ 12 */
13 router 13 router
14 - // .get('/signs', bundlesController.index)  
15 - .get('/:platform/:state?/:sign', bundlesController.show) 14 + .get('/signs', bundleController.index)
  15 + .get('/:platform/:state?/:sign', bundleController.show)
16 16
17 module.exports = router; 17 module.exports = router;