diff --git a/core/Trie.py b/core/Trie.py new file mode 100755 index 0000000..f4eb6f0 --- /dev/null +++ b/core/Trie.py @@ -0,0 +1,63 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +class Node: + def __init__(self, end = False): + self.end = end + self.children = {} + +class Trie: + def __init__(self, characters): + self.characters = characters + self._parse_characters() + self.root = Node() + + def _parse_characters(self): + self.characters_keys = {} + for i in xrange(len(self.characters)): + self.characters_keys[self.characters[i]] = i + + def _key_of(self, item, i): + return self.characters_keys[item[i].encode('utf-8')] + + def _add(self, item): + node = self.root + item = list(item.decode('utf-8')) + for i in xrange(len(item)): + key = self._key_of(item, i) + if not node.children.has_key(key): + node.children[key] = Node() + node = node.children[key] + node.end = True + + def _to_json(self, node): + keys = []; + children = {} + for key, value in node.children.iteritems(): + keys.append(key) + children[key] = self._to_json(value) + keys.sort() + return { 'end': node.end, 'keys': keys, 'children': children } + + def add(self, data): + if type(data) is list: + for d in data: + if not d in ignore: + self._add(d) + else: + self._add(data) + + def to_json(self): + return self._to_json(self.root) + +chars = ["'", '$', ',', '_', '%', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'A', 'Á', 'Â', 'Ã', 'À', 'B', 'C', 'Ç', 'D', 'E', 'É', 'Ê', 'F', 'G', 'H', + 'I', 'Í', 'J', 'K', 'L', 'M', 'N', 'O', 'Ó', 'Ô', 'Õ', 'P', 'Q', 'R', 'S', + 'T', 'U', 'Ú', 'V', 'W', 'X', 'Y', 'Z'] + +ignore = ["[INTERROGAÇÃO]", "[EXCLAMAÇÃO]", "[PONTO]"] + +def gen(data): + trie = Trie(chars) + trie.add(data) + return { 'characters': chars, 'keys': trie.characters_keys, 'trie': trie.to_json() } diff --git a/core/indexer.py b/core/indexer.py new file mode 100755 index 0000000..cb1a19b --- /dev/null +++ b/core/indexer.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Author: Erickson Silva +E-Mail: erickson.silva@lavid.ufpb.br + +Author: Wesnydy Lima Ribeiro +E-Mail: wesnydy@lavid.ufpb.br +""" + +import os +import Trie +import json +import pika +import PikaManager + +from time import sleep + +TRIE=None + +BUNDLES_PATH=None +BUNDLES_LIST={} + +# Manager of queues connections. +manager = PikaManager.PikaManager("localhost") + +def generate_trie(): + global TRIE + signs = list(BUNDLES_LIST["DEFAULT"]) + TRIE = json.dumps(Trie.gen(signs)) + +def list_files(path): + files = [] + for fname in os.listdir(path): + path_mount = os.path.join(path, fname) + if not os.path.isdir(path_mount): + files.append(fname) + return files + +def check_platform_files(): + android = set(list_files(BUNDLES_PATH["ANDROID"])) + ios = set(list_files(BUNDLES_PATH["IOS"])) + webgl = set(list_files(BUNDLES_PATH["WEBGL"])) + standalone = set(list_files(BUNDLES_PATH["STANDALONE"])) + if android == ios and ios == webgl and webgl == standalone: + return standalone + raise RuntimeError("Inconsistent signs. Check files.") + +def list_bundles(): + global BUNDLES_LIST + states = ["AC", "AL", "AP", "AM", "BA", "CE", "DF", "ES", "GO", "MA", "MT", + "MS", "MG", "PA", "PB", "PR", "PE", "PI", "RJ", "RN", "RS", "RO", + "RR", "SC", "SP", "SE", "TO"] + BUNDLES_LIST["DEFAULT"] = check_platform_files() + for platform, path in BUNDLES_PATH.iteritems(): + BUNDLES_LIST[platform] = {} + for state in states: + try: + BUNDLES_LIST[platform].update({state:set(os.listdir(os.path.join(path, state)))}) + except OSError: + BUNDLES_LIST[platform].update({state:set([])}) + +def load_bundles_paths(): + global BUNDLES_PATH + try: + SIGNS_PATH=os.environ['SIGNS_VLIBRAS'] + except KeyError: + raise EnvironmentError("Environment variable 'SIGNS_VLIBRAS' not found.") + IOS_SIGNS_PATH=os.path.join(SIGNS_PATH, "IOS") + ANDROID_SIGNS_PATH=os.path.join(SIGNS_PATH, "ANDROID") + STANDALONE_SIGNS_PATH=os.path.join(SIGNS_PATH, "STANDALONE") + WEBGL_SIGNS_PATH=os.path.join(SIGNS_PATH, "WEBGL") + BUNDLES_PATH={"IOS":IOS_SIGNS_PATH, "ANDROID":ANDROID_SIGNS_PATH, + "STANDALONE":STANDALONE_SIGNS_PATH, "WEBGL":WEBGL_SIGNS_PATH} + list_bundles() + generate_trie() + +def keep_alive(conn_send, conn_receive): + """ + Keep the connection alive. + + Parameters + ---------- + conn_send : object + Connection of writer. + conn_receive : object + Connection of receiver. + """ + while True: + sleep(30) + try: + conn_send.process_data_events() + conn_receive.process_data_events() + except: + continue + +# start_new_thread(keep_alive, (manager.get_conn_send(), manager.get_conn_receive())) + +def run(ch, method, properties, body): + """ + Execute the worker. + + Parameters + ---------- + ch : object + Channel of communication. + method : function + Callback method. + properties : object + Message containing a set of 14 properties. + body : string + Json string containing the necessary arguments for workers. + """ + print ("Sending list...") + manager.send_to_queue("lists", TRIE, properties) + print ("Ok") + +print ("Indexing bundles...") +load_bundles_paths() + +print ("Indexer listening...") +while True: + try: + manager.receive_from_queue("signals", run) + except KeyboardInterrupt: + manager.close_connections() + os._exit(0) diff --git a/translate-api/config/settings.js b/translate-api/config/settings.js index c64d7de..be980a2 100644 --- a/translate-api/config/settings.js +++ b/translate-api/config/settings.js @@ -13,7 +13,7 @@ var path = require('path'); var config = {}; config.contentsPath = process.env.VLIBRAS_VIDEO_LIBRAS; -config.bundlesPath = process.env.BUNDLES_PATH; +config.bundlesPath = process.env.SIGNS_VLIBRAS; config.platformsList = [ 'ANDROID', 'IOS', 'WEBGL', 'STANDALONE' ]; diff --git a/translate-api/controllers/bundle.js b/translate-api/controllers/bundle.js index 9e74a38..a0e9d4a 100644 --- a/translate-api/controllers/bundle.js +++ b/translate-api/controllers/bundle.js @@ -9,9 +9,25 @@ * Required libs. */ var path = require('path') + , amqp = require('../helpers/amqpManager') + , shortid = require('shortid') , settings = require('../config/settings') , error = require('../helpers/error'); +exports.index=function(req, res, next) { + var id = shortid.generate(); + var body = 'signal'; // Just to send something. It has no functionality. + amqp.sendToQueue(body, id,'signals', false, res, function(err) { + if (err) + return error.internalError('An internal communication error has occurred.', next); + amqp.receiveFromQueue(id, 'lists', false, res, function(err, message) { + if (err) + return error.internalError('An internal communication error has occurred.', next); + res.status(200).json(JSON.parse(message)); + }); + }); +}; + exports.show=function(req, res, next) { /** * Check the required params. diff --git a/translate-api/routes/bundle.js b/translate-api/routes/bundle.js index 3d367a7..3606100 100644 --- a/translate-api/routes/bundle.js +++ b/translate-api/routes/bundle.js @@ -5,13 +5,13 @@ var express = require('express') , router = express.Router() - , bundlesController = require('../controllers/bundle'); + , bundleController = require('../controllers/bundle'); /** * Routes to get bundles of a specific platform. */ router - // .get('/signs', bundlesController.index) - .get('/:platform/:state?/:sign', bundlesController.show) + .get('/signs', bundleController.index) + .get('/:platform/:state?/:sign', bundleController.show) module.exports = router; -- libgit2 0.21.2