Commit 0b57611984f4692ec270c49faaeca0744cec59eb
1 parent
1ccd01c4
Exists in
master
and in
3 other branches
Bundles List service implementation
Showing
5 changed files
with
211 additions
and
4 deletions
Show diff stats
... | ... | @@ -0,0 +1,63 @@ |
1 | +#!/usr/bin/python | |
2 | +# -*- coding: utf-8 -*- | |
3 | + | |
4 | +class Node: | |
5 | + def __init__(self, end = False): | |
6 | + self.end = end | |
7 | + self.children = {} | |
8 | + | |
9 | +class Trie: | |
10 | + def __init__(self, characters): | |
11 | + self.characters = characters | |
12 | + self._parse_characters() | |
13 | + self.root = Node() | |
14 | + | |
15 | + def _parse_characters(self): | |
16 | + self.characters_keys = {} | |
17 | + for i in xrange(len(self.characters)): | |
18 | + self.characters_keys[self.characters[i]] = i | |
19 | + | |
20 | + def _key_of(self, item, i): | |
21 | + return self.characters_keys[item[i].encode('utf-8')] | |
22 | + | |
23 | + def _add(self, item): | |
24 | + node = self.root | |
25 | + item = list(item.decode('utf-8')) | |
26 | + for i in xrange(len(item)): | |
27 | + key = self._key_of(item, i) | |
28 | + if not node.children.has_key(key): | |
29 | + node.children[key] = Node() | |
30 | + node = node.children[key] | |
31 | + node.end = True | |
32 | + | |
33 | + def _to_json(self, node): | |
34 | + keys = []; | |
35 | + children = {} | |
36 | + for key, value in node.children.iteritems(): | |
37 | + keys.append(key) | |
38 | + children[key] = self._to_json(value) | |
39 | + keys.sort() | |
40 | + return { 'end': node.end, 'keys': keys, 'children': children } | |
41 | + | |
42 | + def add(self, data): | |
43 | + if type(data) is list: | |
44 | + for d in data: | |
45 | + if not d in ignore: | |
46 | + self._add(d) | |
47 | + else: | |
48 | + self._add(data) | |
49 | + | |
50 | + def to_json(self): | |
51 | + return self._to_json(self.root) | |
52 | + | |
53 | +chars = ["'", '$', ',', '_', '%', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', | |
54 | + 'A', 'Á', 'Â', 'Ã', 'À', 'B', 'C', 'Ç', 'D', 'E', 'É', 'Ê', 'F', 'G', 'H', | |
55 | + 'I', 'Í', 'J', 'K', 'L', 'M', 'N', 'O', 'Ó', 'Ô', 'Õ', 'P', 'Q', 'R', 'S', | |
56 | + 'T', 'U', 'Ú', 'V', 'W', 'X', 'Y', 'Z'] | |
57 | + | |
58 | +ignore = ["[INTERROGAÇÃO]", "[EXCLAMAÇÃO]", "[PONTO]"] | |
59 | + | |
60 | +def gen(data): | |
61 | + trie = Trie(chars) | |
62 | + trie.add(data) | |
63 | + return { 'characters': chars, 'keys': trie.characters_keys, 'trie': trie.to_json() } | ... | ... |
... | ... | @@ -0,0 +1,128 @@ |
1 | +#!/usr/bin/env python | |
2 | +# -*- coding: utf-8 -*- | |
3 | + | |
4 | +""" | |
5 | +Author: Erickson Silva | |
6 | +E-Mail: erickson.silva@lavid.ufpb.br | |
7 | + | |
8 | +Author: Wesnydy Lima Ribeiro | |
9 | +E-Mail: wesnydy@lavid.ufpb.br | |
10 | +""" | |
11 | + | |
12 | +import os | |
13 | +import Trie | |
14 | +import json | |
15 | +import pika | |
16 | +import PikaManager | |
17 | + | |
18 | +from time import sleep | |
19 | + | |
20 | +TRIE=None | |
21 | + | |
22 | +BUNDLES_PATH=None | |
23 | +BUNDLES_LIST={} | |
24 | + | |
25 | +# Manager of queues connections. | |
26 | +manager = PikaManager.PikaManager("localhost") | |
27 | + | |
28 | +def generate_trie(): | |
29 | + global TRIE | |
30 | + signs = list(BUNDLES_LIST["DEFAULT"]) | |
31 | + TRIE = json.dumps(Trie.gen(signs)) | |
32 | + | |
33 | +def list_files(path): | |
34 | + files = [] | |
35 | + for fname in os.listdir(path): | |
36 | + path_mount = os.path.join(path, fname) | |
37 | + if not os.path.isdir(path_mount): | |
38 | + files.append(fname) | |
39 | + return files | |
40 | + | |
41 | +def check_platform_files(): | |
42 | + android = set(list_files(BUNDLES_PATH["ANDROID"])) | |
43 | + ios = set(list_files(BUNDLES_PATH["IOS"])) | |
44 | + webgl = set(list_files(BUNDLES_PATH["WEBGL"])) | |
45 | + standalone = set(list_files(BUNDLES_PATH["STANDALONE"])) | |
46 | + if android == ios and ios == webgl and webgl == standalone: | |
47 | + return standalone | |
48 | + raise RuntimeError("Inconsistent signs. Check files.") | |
49 | + | |
50 | +def list_bundles(): | |
51 | + global BUNDLES_LIST | |
52 | + states = ["AC", "AL", "AP", "AM", "BA", "CE", "DF", "ES", "GO", "MA", "MT", | |
53 | + "MS", "MG", "PA", "PB", "PR", "PE", "PI", "RJ", "RN", "RS", "RO", | |
54 | + "RR", "SC", "SP", "SE", "TO"] | |
55 | + BUNDLES_LIST["DEFAULT"] = check_platform_files() | |
56 | + for platform, path in BUNDLES_PATH.iteritems(): | |
57 | + BUNDLES_LIST[platform] = {} | |
58 | + for state in states: | |
59 | + try: | |
60 | + BUNDLES_LIST[platform].update({state:set(os.listdir(os.path.join(path, state)))}) | |
61 | + except OSError: | |
62 | + BUNDLES_LIST[platform].update({state:set([])}) | |
63 | + | |
64 | +def load_bundles_paths(): | |
65 | + global BUNDLES_PATH | |
66 | + try: | |
67 | + SIGNS_PATH=os.environ['SIGNS_VLIBRAS'] | |
68 | + except KeyError: | |
69 | + raise EnvironmentError("Environment variable 'SIGNS_VLIBRAS' not found.") | |
70 | + IOS_SIGNS_PATH=os.path.join(SIGNS_PATH, "IOS") | |
71 | + ANDROID_SIGNS_PATH=os.path.join(SIGNS_PATH, "ANDROID") | |
72 | + STANDALONE_SIGNS_PATH=os.path.join(SIGNS_PATH, "STANDALONE") | |
73 | + WEBGL_SIGNS_PATH=os.path.join(SIGNS_PATH, "WEBGL") | |
74 | + BUNDLES_PATH={"IOS":IOS_SIGNS_PATH, "ANDROID":ANDROID_SIGNS_PATH, | |
75 | + "STANDALONE":STANDALONE_SIGNS_PATH, "WEBGL":WEBGL_SIGNS_PATH} | |
76 | + list_bundles() | |
77 | + generate_trie() | |
78 | + | |
79 | +def keep_alive(conn_send, conn_receive): | |
80 | + """ | |
81 | + Keep the connection alive. | |
82 | + | |
83 | + Parameters | |
84 | + ---------- | |
85 | + conn_send : object | |
86 | + Connection of writer. | |
87 | + conn_receive : object | |
88 | + Connection of receiver. | |
89 | + """ | |
90 | + while True: | |
91 | + sleep(30) | |
92 | + try: | |
93 | + conn_send.process_data_events() | |
94 | + conn_receive.process_data_events() | |
95 | + except: | |
96 | + continue | |
97 | + | |
98 | +# start_new_thread(keep_alive, (manager.get_conn_send(), manager.get_conn_receive())) | |
99 | + | |
100 | +def run(ch, method, properties, body): | |
101 | + """ | |
102 | + Execute the worker. | |
103 | + | |
104 | + Parameters | |
105 | + ---------- | |
106 | + ch : object | |
107 | + Channel of communication. | |
108 | + method : function | |
109 | + Callback method. | |
110 | + properties : object | |
111 | + Message containing a set of 14 properties. | |
112 | + body : string | |
113 | + Json string containing the necessary arguments for workers. | |
114 | + """ | |
115 | + print ("Sending list...") | |
116 | + manager.send_to_queue("lists", TRIE, properties) | |
117 | + print ("Ok") | |
118 | + | |
119 | +print ("Indexing bundles...") | |
120 | +load_bundles_paths() | |
121 | + | |
122 | +print ("Indexer listening...") | |
123 | +while True: | |
124 | + try: | |
125 | + manager.receive_from_queue("signals", run) | |
126 | + except KeyboardInterrupt: | |
127 | + manager.close_connections() | |
128 | + os._exit(0) | ... | ... |
translate-api/config/settings.js
... | ... | @@ -13,7 +13,7 @@ var path = require('path'); |
13 | 13 | var config = {}; |
14 | 14 | |
15 | 15 | config.contentsPath = process.env.VLIBRAS_VIDEO_LIBRAS; |
16 | -config.bundlesPath = process.env.BUNDLES_PATH; | |
16 | +config.bundlesPath = process.env.SIGNS_VLIBRAS; | |
17 | 17 | |
18 | 18 | config.platformsList = [ 'ANDROID', 'IOS', 'WEBGL', 'STANDALONE' ]; |
19 | 19 | ... | ... |
translate-api/controllers/bundle.js
... | ... | @@ -9,9 +9,25 @@ |
9 | 9 | * Required libs. |
10 | 10 | */ |
11 | 11 | var path = require('path') |
12 | + , amqp = require('../helpers/amqpManager') | |
13 | + , shortid = require('shortid') | |
12 | 14 | , settings = require('../config/settings') |
13 | 15 | , error = require('../helpers/error'); |
14 | 16 | |
17 | +exports.index=function(req, res, next) { | |
18 | + var id = shortid.generate(); | |
19 | + var body = 'signal'; // Just to send something. It has no functionality. | |
20 | + amqp.sendToQueue(body, id,'signals', false, res, function(err) { | |
21 | + if (err) | |
22 | + return error.internalError('An internal communication error has occurred.', next); | |
23 | + amqp.receiveFromQueue(id, 'lists', false, res, function(err, message) { | |
24 | + if (err) | |
25 | + return error.internalError('An internal communication error has occurred.', next); | |
26 | + res.status(200).json(JSON.parse(message)); | |
27 | + }); | |
28 | + }); | |
29 | +}; | |
30 | + | |
15 | 31 | exports.show=function(req, res, next) { |
16 | 32 | /** |
17 | 33 | * Check the required params. | ... | ... |
translate-api/routes/bundle.js
... | ... | @@ -5,13 +5,13 @@ |
5 | 5 | |
6 | 6 | var express = require('express') |
7 | 7 | , router = express.Router() |
8 | - , bundlesController = require('../controllers/bundle'); | |
8 | + , bundleController = require('../controllers/bundle'); | |
9 | 9 | |
10 | 10 | /** |
11 | 11 | * Routes to get bundles of a specific platform. |
12 | 12 | */ |
13 | 13 | router |
14 | - // .get('/signs', bundlesController.index) | |
15 | - .get('/:platform/:state?/:sign', bundlesController.show) | |
14 | + .get('/signs', bundleController.index) | |
15 | + .get('/:platform/:state?/:sign', bundleController.show) | |
16 | 16 | |
17 | 17 | module.exports = router; | ... | ... |