Download as
Browse Code »

Commit 5ddca534b66a44709261acc6832f4c5894d8f248

Authored by root
1 parent d7419300
Exists in release

Atualização das informações de log. 

Atualização dos diretórios.
Atualização das variaveis de ambiente.
Adição do docker-compose.
Showing 7 changed files with 214 additions and 119 deletions   Show diff stats
Dockerfile
  Diff comments   View file @5ddca53
... ... @@ -76,6 +76,7 @@ COPY ./core/renderer.py /root/renderer.py
76 76 COPY ./core/translator.py /root/translator.py
77 77 COPY ./core/PikaManager.py /root/PikaManager.py
78 78 COPY ./core/processManager.py /root/processManager.py
  79 +COPY ./log/ /root/log/
79 80 COPY ./unityVideo/ /root/unityVideo/
80 81 COPY ./vlibras-translate/ /root/vlibras-translate/
81 82 COPY ./vlibras-libs/ /root/vlibras-libs/
... ... @@ -83,8 +84,9 @@ COPY ./vlibras-api/ /root/vlibras-api/
83 84  
84 85 #Workers environment variables
85 86 ENV VLIBRAS_VIDEO_CREATOR="/root/unityVideo/videoCreator.x86_64"
86   -ENV VLIBRAS_VIDEO_LIBRAS="/root/storage/libras"
87   -ENV VLIBRAS_VIDEO_MIXED="/root/storage/videos"
  87 +ENV VLIBRAS_VIDEO_LIBRAS="/storage/libras"
  88 +ENV VLIBRAS_VIDEO_MIXED="/storage/videos"
  89 +ENV VLIBRAS_VIDEO_SCREENS="/storage/frames"
88 90  
89 91 #Translator environment variables
90 92 ENV HUNPOS_TAGGER="/root/vlibras-libs/aelius/bin/hunpos-tag"
... ... @@ -93,15 +95,14 @@ ENV TRANSLATE_DATA="/root/vlibras-translate/data"
93 95 ENV NLTK_DATA="/root/vlibras-libs/aelius/nltk_data"
94 96 ENV PYTHONPATH=":/root/vlibras-libs/aelius:/root/vlibras-translate/src:/root/vlibras-libs/aelius:/root/vlibras-translate/src"
95 97  
  98 +
96 99 #ENV NODE_PATH=/root/vlibras-api/node_modules/;%NODE_PATH%
97 100  
98 101 #Portas de comunicacao do container
99   -EXPOSE 27017
100   -EXPOSE 5672
101   -EXPOSE 80:80
102   -#EXPOSE 15672
103 102 WORKDIR /root/vlibras-api/
104   -VOLUME ["/root/volume/", "/storage/"]
  103 +
  104 +VOLUME ["/storage/"]
  105 +
105 106 RUN npm cache clean
106 107 RUN rm -rf node_modules
107 108 RUN npm i
... ... @@ -112,4 +113,4 @@ RUN node --version
112 113 RUN /usr/bin/mongod &
113 114  
114 115 #Comando de entrada quando inicializado
115   -ENTRYPOINT (/usr/bin/mongod & redis-server & rabbitmq-server start & sleep 3m) ; (node server.js & sleep 10) ; python /root/extractor.py & python /root/translator.py & python /root/renderer.py & python /root/mixer.py
  116 +ENTRYPOINT (/usr/bin/mongod & redis-server & rabbitmq-server start & sleep 3m) ; (node /root/vlibras-api/server.js & sleep 10) ; python /root/processManager.py
... ...
core/PikaManager.py
  Diff comments   View file @5ddca53
1   -
2 1 """
3 2 Author: Caio Marcelo Campoy Guedes
4 3 E-Mail: caiomcg@gmail.com
5 4  
6   -Author: Caio Marcelo Campoy Guedes
7   -E-Mail: caiomcg@gmail.com
  5 +Author: Erickson Silva
  6 +E-Mail: erickson.silva@lavid.ufpb.br
8 7  
9   -Author: Caio Marcelo Campoy Guedes
10   -E-Mail: caiomcg@gmail.com
  8 +Author: Jorismar Barbosa
  9 +E-Mail: jorismar.barbosa@lavid.ufpb.br
  10 +
  11 +Author: Wesnydy Lima Ribeiro
  12 +E-Mail: wesnydy@lavid.ufpb.br
11 13 """
12 14  
13 15 import pika
... ... @@ -15,90 +17,161 @@ import json
15 17  
16 18 class PikaManager:
17 19  
18   - def __init__(self, ip):
19   - """
20   - Initialize the class without
21   -
22   - Parameters
23   - ----------
24   - ip : string
25   - The server IP.
26   - """
27   - self.server_ip = ip
28   - self.MAX_ERR_ATTEMPT = 3
29   -
30   - def __init__(self, ip, username, password):
31   - self.server_ip = ip
32   - self.MAX_ERR_ATTEMPT = 3
33   - self.add_credentials(username, password)
34   - self.add_blockConnection()
35   -
36   - def add_credentials(self, username, password):
37   - self.credentials = pika.PlainCredentials(username, password)
38   -
39   - def add_blockConnection(self):
40   - self.conn_send = pika.BlockingConnection(pika.ConnectionParameters(host = self.server_ip, credentials = self.credentials, heartbeat_interval = 0))
41   - self.conn_receive = pika.BlockingConnection(pika.ConnectionParameters(host = self.server_ip, credentials = self.credentials, heartbeat_interval = 0))
42   -
43   - def _reload_connection(self, connection):
44   - try:
45   - connection.close()
46   - except:
47   - pass #HANDLE
48   - connection = pika.BlockingConnection(pika.ConnectionParameters(host = self.server_ip, heartbeat_interval = 0)) # ?
49   -
50   - def _setup_channel(self, connection):
51   - """
52   - Atempt to create a connection.
53   -
54   - Parameters
55   - ----------
56   - connection : Object
57   - Connection to setup.
58   - Returns
59   - -------
60   - Object
61   - None if cailed to connect. The object if successfuly connected.
62   - """
63   - attempts = 0
64   -
65   - while attempts < self.MAX_ERR_ATTEMPT:
66   - try:
67   - channel = connection.channel()
68   - return channel
69   - except:
70   - self._reload_connection(connection)
71   - attempts += 1
72   - print("Send Error: Attempt(" + str(attempts) + ")")
73   -
74   - if attempts == self.MAX_ERR_ATTEMPT:
75   - return None
76   -
77   - def send_to_queue(self, queue_name, body, props):
78   - channel = self._setup_channel(self.conn_send)
79   - if channel != None:
80   - channel.queue_declare(queue = queue_name)
81   - channel.basic_publish(exchange = '', routing_key = queue_name, properties = pika.BasicProperties(correlation_id = props.correlation_id), body = json.dumps(body))
82   - channel.close()
83   -
84   - def receive_from_queue(self, queue_name, callback):
85   - channel = self._setup_channel(self.conn_receive)
86   - if channel != None:
87   - channel.queue_declare(queue = queue_name)
88   - channel.basic_qos(prefetch_count = 1)
89   - channel.basic_consume(callback, queue = queue_name, no_ack = True)
90   - channel.start_consuming()
91   - channel.close()
92   -
93   - def get_conn_send(self):
94   - return self.conn_send
95   -
96   - def get_conn_receive(self):
97   - return self.conn_receive
98   -
99   - def close_connections(self):
100   - try:
101   - self.conn_receive.close()
102   - self.conn_send.close()
103   - except:
104   - pass
  20 + def __init__(self, ip, username=None, password=None):
  21 + """
  22 + Initialize the class with credentials.
  23 +
  24 + Parameters
  25 + ----------
  26 + ip : string
  27 + The server IP.
  28 + username : string
  29 + The user login.
  30 + password : string
  31 + The user password.
  32 + """
  33 + self.server_ip = ip
  34 + self.MAX_ERR_ATTEMPT = 3
  35 +
  36 + if not all((username, password)):
  37 + self.add_localBlockConnection()
  38 + else:
  39 + self.add_credentials(username, password)
  40 + self.add_blockConnection()
  41 +
  42 + def add_credentials(self, username, password):
  43 + """
  44 + Add user credentials.
  45 +
  46 + Parameters
  47 + ----------
  48 + username : string
  49 + The user login.
  50 + password : string
  51 + The user password.
  52 + """
  53 + self.credentials = pika.PlainCredentials(username, password)
  54 +
  55 + def add_localBlockConnection(self):
  56 + """
  57 + Create the blocking connection object.
  58 + """
  59 + self.conn_send = pika.BlockingConnection(pika.ConnectionParameters(host = self.server_ip, heartbeat_interval = 0))
  60 + self.conn_receive = pika.BlockingConnection(pika.ConnectionParameters(host = self.server_ip, heartbeat_interval = 0))
  61 +
  62 + def add_blockConnection(self):
  63 + """
  64 + Create the blocking connection object. Credentials are used.
  65 + """
  66 + self.conn_send = pika.BlockingConnection(pika.ConnectionParameters(host = self.server_ip, credentials = self.credentials, heartbeat_interval = 0))
  67 + self.conn_receive = pika.BlockingConnection(pika.ConnectionParameters(host = self.server_ip, credentials = self.credentials, heartbeat_interval = 0))
  68 +
  69 + def _reload_connection(self, connection):
  70 + """
  71 + Reload a specific connection.
  72 +
  73 + Parameters
  74 + ----------
  75 + connection : Object
  76 + The connection to be reloaded.
  77 + """
  78 + try:
  79 + connection.close()
  80 + except:
  81 + pass
  82 + connection = pika.BlockingConnection(pika.ConnectionParameters(host = self.server_ip, heartbeat_interval = 0)) # ?
  83 +
  84 + def _setup_channel(self, connection):
  85 + """
  86 + Atempt to create a connection.
  87 +
  88 + Parameters
  89 + ----------
  90 + connection : Object
  91 + Connection to setup.
  92 + Returns
  93 + -------
  94 + Object
  95 + None if cailed to connect. The object if successfuly connected.
  96 + """
  97 + attempts = 0
  98 +
  99 + while attempts < self.MAX_ERR_ATTEMPT:
  100 + try:
  101 + channel = connection.channel()
  102 + return channel
  103 + except:
  104 + self._reload_connection(connection)
  105 + attempts += 1
  106 + print("Send Error: Attempt(" + str(attempts) + ")")
  107 +
  108 + if attempts == self.MAX_ERR_ATTEMPT:
  109 + return None
  110 +
  111 + def send_to_queue(self, queue_name, body, props):
  112 + """
  113 + Send a message to the queue.
  114 +
  115 + Parameters
  116 + ----------
  117 + queue_name : string
  118 + Queue that receives the message.
  119 + body : string
  120 + The message to be sent.
  121 + props : Object
  122 + Object containing a set of 14 properties.
  123 + """
  124 + channel = self._setup_channel(self.conn_send)
  125 + if channel != None:
  126 + channel.queue_declare(queue = queue_name)
  127 + channel.basic_publish(exchange = '', routing_key = queue_name, properties = pika.BasicProperties(correlation_id = props.correlation_id), body = json.dumps(body))
  128 + channel.close()
  129 +
  130 + def receive_from_queue(self, queue_name, callback):
  131 + """
  132 + Receive a message from the queue.
  133 +
  134 + Parameters
  135 + ----------
  136 + queue_name : string
  137 + Queue where the message will be received.
  138 + callback : function
  139 + Function that process the message.
  140 + """
  141 + channel = self._setup_channel(self.conn_receive)
  142 + if channel != None:
  143 + channel.queue_declare(queue = queue_name)
  144 + channel.basic_qos(prefetch_count = 1)
  145 + channel.basic_consume(callback, queue = queue_name, no_ack = True)
  146 + channel.start_consuming()
  147 + channel.close()
  148 +
  149 + def get_conn_send(self):
  150 + """
  151 + Get the send connection.
  152 +
  153 + Returns
  154 + -------
  155 + The send connection.
  156 + """
  157 + return self.conn_send
  158 +
  159 + def get_conn_receive(self):
  160 + """
  161 + Get the receive connection.
  162 +
  163 + Returns
  164 + -------
  165 + The receive connection.
  166 + """
  167 + return self.conn_receive
  168 +
  169 + def close_connections(self):
  170 + """
  171 + Close all connections.
  172 + """
  173 + try:
  174 + self.conn_receive.close()
  175 + self.conn_send.close()
  176 + except:
  177 + pass
... ...
core/extractor.py
  Diff comments   View file @5ddca53
... ... @@ -30,11 +30,11 @@ from urllib import urlretrieve
30 30 logger = logging.getLogger("extractor")
31 31 logger.setLevel(logging.DEBUG)
32 32  
33   -fh = logging.FileHandler("../log/extractor.log")
  33 +fh = logging.FileHandler("/root/log/extractor.log")
34 34 fh.setLevel(logging.DEBUG)
35 35  
36 36 ch = logging.StreamHandler()
37   -ch.setLevel(logging.ERROR)
  37 +ch.setLevel(logging.INFO)
38 38  
39 39 formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
40 40 fh.setFormatter(formatter)
... ... @@ -44,7 +44,8 @@ logger.addHandler(fh)
44 44 logger.addHandler(ch)
45 45  
46 46 # Manager of queues connections.
47   -manager = PikaManager.PikaManager("150.165.205.10", "test", "test")
  47 +#manager = PikaManager.PikaManager("150.165.205.10", "test", "test")
  48 +manager = PikaManager.PikaManager("localhost")
48 49  
49 50 def run(ch, method, properties, body):
50 51 """
... ...
core/mixer.py
  Diff comments   View file @5ddca53
... ... @@ -26,15 +26,19 @@ from thread import start_new_thread
26 26 from time import sleep
27 27 from urllib import urlretrieve
28 28  
  29 +def make_dir_if_exists(path):
  30 + if not os.path.exists(path):
  31 + os.makedirs(path)
  32 +
29 33 # Logging configuration.
30 34 logger = logging.getLogger('mixer')
31 35 logger.setLevel(logging.DEBUG)
32 36  
33   -fh = logging.FileHandler('../log/mixer.log')
  37 +fh = logging.FileHandler('/root/log/mixer.log')
34 38 fh.setLevel(logging.DEBUG)
35 39  
36 40 ch = logging.StreamHandler()
37   -ch.setLevel(logging.ERROR)
  41 +ch.setLevel(logging.INFO)
38 42  
39 43 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
40 44 fh.setFormatter(formatter)
... ... @@ -44,9 +48,13 @@ logger.addHandler(fh)
44 48 logger.addHandler(ch)
45 49  
46 50 # Manager of queues connections.
47   -manager = PikaManager.PikaManager("150.165.205.10", "test", "test")
  51 +#manager = PikaManager.PikaManager("150.165.205.10", "test", "test")
  52 +
  53 +manager = PikaManager.PikaManager("localhost")
48 54  
49 55 PATH_MIXED_VIDEO = os.getenv("VLIBRAS_VIDEO_MIXED")
  56 +#Create Path if Needed
  57 +make_dir_if_exists(PATH_MIXED_VIDEO)
50 58  
51 59 def main_video_height(main_video):
52 60 """
... ...
core/processManager.py
  Diff comments   View file @5ddca53
... ... @@ -15,7 +15,7 @@ def signalHandler(signal, frame):
15 15  
16 16 def spawnRenderer():
17 17 while KEEP_RUNNING:
18   - proc = subprocess.Popen(["./renderer.py"], shell=True, stdout=subprocess.PIPE)
  18 + proc = subprocess.Popen(["/root/renderer.py"], shell=True, stdout=subprocess.PIPE)
19 19 print("Process RENDERER PID: " + str(proc.pid))
20 20 try:
21 21 stdoutdata, stderrdata = proc.communicate()
... ... @@ -25,7 +25,7 @@ def spawnRenderer():
25 25  
26 26 def spawnMixer():
27 27 while KEEP_RUNNING:
28   - proc = subprocess.Popen(["./mixer.py"], shell=True, stdout=subprocess.PIPE)
  28 + proc = subprocess.Popen(["/root/mixer.py"], shell=True, stdout=subprocess.PIPE)
29 29 print("Process MIXER PID: " + str(proc.pid))
30 30 try:
31 31 stdoutdata, stderrdata = proc.communicate()
... ... @@ -35,7 +35,7 @@ def spawnMixer():
35 35  
36 36 def spawnTranslator():
37 37 while KEEP_RUNNING:
38   - proc = subprocess.Popen(["./translator.py"], shell=True, stdout=subprocess.PIPE)
  38 + proc = subprocess.Popen(["/root/translator.py"], shell=True, stdout=subprocess.PIPE)
39 39 print("Process TRANSLATOR PID: " + str(proc.pid))
40 40 try:
41 41 stdoutdata, stderrdata = proc.communicate()
... ... @@ -45,7 +45,7 @@ def spawnTranslator():
45 45  
46 46 def spawnExtractor():
47 47 while KEEP_RUNNING:
48   - proc = subprocess.Popen(["./extractor.py"], shell=True, stdout=subprocess.PIPE)
  48 + proc = subprocess.Popen(["/root/extractor.py"], shell=True, stdout=subprocess.PIPE)
49 49 print("Process EXTRACTOR PID: " + str(proc.pid))
50 50 try:
51 51 stdoutdata, stderrdata = proc.communicate()
... ... @@ -67,4 +67,4 @@ if __name__ == &quot;__main__&quot;:
67 67  
68 68 t4 = Thread(target=spawnMixer, args=())
69 69 t4.start()
70   - #thread.start_new_thread(spawnWorker, (worker,))
71 70 \ No newline at end of file
  71 + #thread.start_new_thread(spawnWorker, (worker,))
... ...
core/renderer.py
  Diff comments   View file @5ddca53
... ... @@ -18,15 +18,20 @@ from time import sleep
18 18 #Temporary
19 19 from shutil import rmtree
20 20  
  21 +#Temporary
  22 +def make_dir_if_exists(path):
  23 + if not os.path.exists(path):
  24 + os.makedirs(path)
  25 +
21 26 # Logging configuration.
22 27 logger = logging.getLogger('renderer')
23 28 logger.setLevel(logging.DEBUG)
24 29  
25   -fh = logging.FileHandler('../log/renderer.log')
  30 +fh = logging.FileHandler('/root/log/renderer.log')
26 31 fh.setLevel(logging.DEBUG)
27 32  
28 33 ch = logging.StreamHandler()
29   -ch.setLevel(logging.ERROR)
  34 +ch.setLevel(logging.INFO)
30 35  
31 36 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
32 37 fh.setFormatter(formatter)
... ... @@ -36,16 +41,21 @@ logger.addHandler(fh)
36 41 logger.addHandler(ch)
37 42  
38 43 # Manager of queues connections.
39   -manager = PikaManager.PikaManager("150.165.205.10", "test", "test")
  44 +#manager = PikaManager.PikaManager("150.165.205.10", "test", "test")
  45 +
  46 +manager = PikaManager.PikaManager("localhost")
40 47  
41 48 TCP_IP = '0.0.0.0'
42 49 TCP_PORT = 5555
43 50  
44 51 PATH_LIBRAS = os.getenv("VLIBRAS_VIDEO_LIBRAS")
45 52 VIDEO_CREATOR = os.getenv("VLIBRAS_VIDEO_CREATOR")
  53 +PATH_SCREENS = os.getenv("VLIBRAS_VIDEO_SCREENS")
46 54  
47   -#Temporary
48   -PATH_SCREENS = "/storage/frames/"
  55 +#Create Paths if needed
  56 +make_dir_if_exists(PATH_LIBRAS)
  57 +make_dir_if_exists(VIDEO_CREATOR)
  58 +make_dir_if_exists(PATH_SCREENS)
49 59  
50 60 # Status of renderer to process new requests. Answer one request at a time.
51 61 worker_available = True
... ...
core/translator.py
  Diff comments   View file @5ddca53
... ... @@ -29,11 +29,11 @@ from time import sleep
29 29 logger = logging.getLogger("translator")
30 30 logger.setLevel(logging.DEBUG)
31 31  
32   -fh = logging.FileHandler("../log/translator.log")
  32 +fh = logging.FileHandler("/root/log/translator.log")
33 33 fh.setLevel(logging.DEBUG)
34 34  
35 35 ch = logging.StreamHandler()
36   -ch.setLevel(logging.ERROR)
  36 +ch.setLevel(logging.INFO)
37 37  
38 38 formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
39 39 fh.setFormatter(formatter)
... ... @@ -43,7 +43,9 @@ logger.addHandler(fh)
43 43 logger.addHandler(ch)
44 44  
45 45 # Manager of queues connections.
46   -manager = PikaManager.PikaManager("150.165.205.10", "test", "test")
  46 +#manager = PikaManager.PikaManager("150.165.205.10", "test", "test")
  47 +
  48 +manager = PikaManager.PikaManager("localhost")
47 49  
48 50 def run(ch, method, properties, body):
49 51 """
... ...