indexer.py
3.48 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Author: Erickson Silva
E-Mail: erickson.silva@lavid.ufpb.br
Author: Wesnydy Lima Ribeiro
E-Mail: wesnydy@lavid.ufpb.br
"""
import os
import Trie
import json
import pika
import PikaManager
from time import sleep
TRIE=None
BUNDLES_PATH=None
BUNDLES_LIST={}
# Manager of queues connections.
manager = PikaManager.PikaManager("rabbit")
def generate_trie():
global TRIE
signs = list(BUNDLES_LIST["DEFAULT"])
TRIE = json.dumps(Trie.gen(signs))
def list_files(path):
files = []
for fname in os.listdir(path):
path_mount = os.path.join(path, fname)
if not os.path.isdir(path_mount):
files.append(fname)
return files
def check_platform_files():
android = set(list_files(BUNDLES_PATH["ANDROID"]))
ios = set(list_files(BUNDLES_PATH["IOS"]))
webgl = set(list_files(BUNDLES_PATH["WEBGL"]))
standalone = set(list_files(BUNDLES_PATH["STANDALONE"]))
if android == ios and ios == webgl and webgl == standalone:
return standalone
raise RuntimeError("Inconsistent signs. Check files.")
def list_bundles():
global BUNDLES_LIST
states = ["AC", "AL", "AP", "AM", "BA", "CE", "DF", "ES", "GO", "MA", "MT",
"MS", "MG", "PA", "PB", "PR", "PE", "PI", "RJ", "RN", "RS", "RO",
"RR", "SC", "SP", "SE", "TO"]
BUNDLES_LIST["DEFAULT"] = check_platform_files()
for platform, path in BUNDLES_PATH.iteritems():
BUNDLES_LIST[platform] = {}
for state in states:
try:
BUNDLES_LIST[platform].update({state:set(os.listdir(os.path.join(path, state)))})
except OSError:
BUNDLES_LIST[platform].update({state:set([])})
def load_bundles_paths():
global BUNDLES_PATH
try:
SIGNS_PATH=os.environ['SIGNS_VLIBRAS']
except KeyError:
raise EnvironmentError("Environment variable 'SIGNS_VLIBRAS' not found.")
IOS_SIGNS_PATH=os.path.join(SIGNS_PATH, "IOS")
ANDROID_SIGNS_PATH=os.path.join(SIGNS_PATH, "ANDROID")
STANDALONE_SIGNS_PATH=os.path.join(SIGNS_PATH, "STANDALONE")
WEBGL_SIGNS_PATH=os.path.join(SIGNS_PATH, "WEBGL")
BUNDLES_PATH={"IOS":IOS_SIGNS_PATH, "ANDROID":ANDROID_SIGNS_PATH,
"STANDALONE":STANDALONE_SIGNS_PATH, "WEBGL":WEBGL_SIGNS_PATH}
list_bundles()
generate_trie()
def keep_alive(conn_send, conn_receive):
"""
Keep the connection alive.
Parameters
----------
conn_send : object
Connection of writer.
conn_receive : object
Connection of receiver.
"""
while True:
sleep(30)
try:
conn_send.process_data_events()
conn_receive.process_data_events()
except:
continue
# start_new_thread(keep_alive, (manager.get_conn_send(), manager.get_conn_receive()))
def run(ch, method, properties, body):
"""
Execute the worker.
Parameters
----------
ch : object
Channel of communication.
method : function
Callback method.
properties : object
Message containing a set of 14 properties.
body : string
Json string containing the necessary arguments for workers.
"""
print ("Sending list...")
manager.send_to_queue("lists", TRIE, properties)
print ("Ok")
print ("Indexing bundles...")
load_bundles_paths()
print ("Indexer listening...")
while True:
try:
manager.receive_from_queue("signals", run)
except KeyboardInterrupt:
manager.close_connections()
os._exit(0)