Download as
Browse Code »

Commit 1ef47e53ae63d3428846bf68383f102a857c8c8b

Authored by André Araújo
1 parent 5ff89612
Exists in master

Integração da criação de tarefas com DB API

Showing 5 changed files with 158 additions and 239 deletions   Show diff stats
1 -TASKMGR_DIR = $(CURDIR)  
2 -TASKMGR_INIT_SCRIPT = wikilibras-taskmgr  
3 -TASKMGR_INIT = ${TASKMGR_DIR}/${TASKMGR_INIT_SCRIPT}  
4 -TASKMGR_ETC_INIT = /etc/init.d/${TASKMGR_INIT_SCRIPT}  
5 -TASKMGR_LOG = ${TASKMGR_DIR}/events.log  
6 -TASKMGR_REQUIREMENTS = ${TASKMGR_DIR}/requirements.txt  
7 -TASKMGR_SETTINGS = ${TASKMGR_DIR}/settings_local.json  
8 -TASKMGR_SETTINGS_T = ${TASKMGR_SETTINGS}.template  
9 -TASKMGR_DATABASE = ${TASKMGR_DIR}/database.json  
10 -  
11 -PATH_CORRETOR = $(subst $(shell basename "$(CURDIR)"),corretor_sinais,$(CURDIR))  
12 -PATH_VALIDADOR = $(subst $(shell basename "$(CURDIR)"),validador_sinais,$(CURDIR))  
13 -PATH_WIKILIBRAS = $(subst $(shell basename "$(CURDIR)"),wikilibrasV2,$(CURDIR))  
14 -  
15 -SED_REPLACE_ALL_PATH = sed -i "s\#$(1)\#$(2)\#g" "$(3)"  
16 -  
17 -install: uninstall build config startup-enable  
18 -  
19 -build: 1 +TASKMGR_DIR := $(CURDIR)
  2 +TASKMGR_INIT_SCRIPT := wikilibras-taskmgr
  3 +TASKMGR_INIT := ${TASKMGR_DIR}/${TASKMGR_INIT_SCRIPT}
  4 +TASKMGR_ETC_INIT := /etc/init.d/${TASKMGR_INIT_SCRIPT}
  5 +TASKMGR_LOG := ${TASKMGR_DIR}/events.log
  6 +TASKMGR_REQUIREMENTS := ${TASKMGR_DIR}/requirements.txt
  7 +TASKMGR_SETTINGS := ${TASKMGR_DIR}/settings_local.json
  8 +TASKMGR_SETTINGS_T := ${TASKMGR_SETTINGS}.template
  9 +TASKMGR_DATABASE := ${TASKMGR_DIR}/database.json
  10 +VLIBRAS_WIKILIBRAS_ENV := /etc/profile.d/lavid/vlibras/wikilibras_env.sh
  11 +SED_REPLACE_ALL = sed -i "s/$(1)/$(2)/g" "$(3)"
  12 +SED_REPLACE_ALL_PATH = sed -i "s\#$(1)\#$(2)\#g" "$(3)"
  13 +
  14 +-include ${VLIBRAS_WIKILIBRAS_ENV}
  15 +
  16 +install: uninstall env config autostart-enable
  17 +
  18 +env:
20 @ ( \ 19 @ ( \
21 - cd "${TASKMGR_DIR}"; \  
22 virtualenv ./env; \ 20 virtualenv ./env; \
23 . ./env/bin/activate; \ 21 . ./env/bin/activate; \
24 pip install -U pip; \ 22 pip install -U pip; \
@@ -27,19 +25,18 @@ build: @@ -27,19 +25,18 @@ build:
27 ) 25 )
28 26
29 clean: 27 clean:
30 - @ sudo find ${TASKMGR_DIR} -regextype posix-awk -regex "(.*.log|.*.pyc)" -type f -delete 28 + @ sudo find . -regextype posix-awk -regex "(.*.log|.*.pyc)" -type f -delete
31 29
32 config: 30 config:
33 @ cp "${TASKMGR_SETTINGS_T}" "${TASKMGR_SETTINGS}" 31 @ cp "${TASKMGR_SETTINGS_T}" "${TASKMGR_SETTINGS}"
34 - @ $(call SED_REPLACE_ALL_PATH,<path-to-corretor>,${PATH_CORRETOR},${TASKMGR_SETTINGS})  
35 - @ $(call SED_REPLACE_ALL_PATH,<path-to-validador>,${PATH_VALIDADOR},${TASKMGR_SETTINGS})  
36 - @ $(call SED_REPLACE_ALL_PATH,<path-to-wikilibras>,${PATH_WIKILIBRAS},${TASKMGR_SETTINGS}) 32 + @ $(call SED_REPLACE_ALL,my-api-key,${PYBOSSA_API_KEY},${TASKMGR_SETTINGS})
  33 + @ $(call SED_REPLACE_ALL,localhost,${LOCALHOST},${TASKMGR_SETTINGS})
37 34
38 -startup-disable: 35 +autostart-disable:
39 @ sudo update-rc.d -f "${TASKMGR_INIT_SCRIPT}" remove 36 @ sudo update-rc.d -f "${TASKMGR_INIT_SCRIPT}" remove
40 @ sudo rm -f "${TASKMGR_ETC_INIT}" 37 @ sudo rm -f "${TASKMGR_ETC_INIT}"
41 38
42 -startup-enable: 39 +autostart-enable:
43 @ sed "s#<path-to-project>#${TASKMGR_DIR}#" "${TASKMGR_INIT}" | sudo tee "${TASKMGR_ETC_INIT}" > /dev/null 40 @ sed "s#<path-to-project>#${TASKMGR_DIR}#" "${TASKMGR_INIT}" | sudo tee "${TASKMGR_ETC_INIT}" > /dev/null
44 @ sudo chmod 755 "${TASKMGR_ETC_INIT}" 41 @ sudo chmod 755 "${TASKMGR_ETC_INIT}"
45 @ sudo chown "root:root" "${TASKMGR_ETC_INIT}" 42 @ sudo chown "root:root" "${TASKMGR_ETC_INIT}"
@@ -47,18 +44,11 @@ startup-enable: @@ -47,18 +44,11 @@ startup-enable:
47 44
48 run: 45 run:
49 @ ( \ 46 @ ( \
50 - cd "${TASKMGR_DIR}"; \  
51 . ./env/bin/activate; \ 47 . ./env/bin/activate; \
52 python main.py; \ 48 python main.py; \
53 ) 49 )
54 50
55 -uninstall: clean startup-disable 51 +uninstall: clean autostart-disable
56 @ rm -rf ./env/ 52 @ rm -rf ./env/
57 53
58 -reset: clean  
59 - @ if [ -e "${TASKMGR_DATABASE}" ]; \  
60 - then \  
61 - cat "${TASKMGR_DATABASE}"; \  
62 - echo ""; \  
63 - sudo rm -f "${TASKMGR_DATABASE}"; \  
64 - fi 54 +.PHONY: clean config run uninstall
1 -## Copie o arquivo modelo de configuração 1 +### Configuração de Instalação
  2 +
  3 +#### Copie o arquivo modelo de configuração
2 4
3 ```sh 5 ```sh
4 $ cp settings_local.json.template settings_local.json 6 $ cp settings_local.json.template settings_local.json
5 ``` 7 ```
6 8
7 -## Edite o arquivo de configuração 9 +#### Edite o arquivo de configuração
8 10
9 ```sh 11 ```sh
10 $ gedit settings_local.json 12 $ gedit settings_local.json
11 ``` 13 ```
12 14
13 -- Substitua o valor da chave "db_host": "localhost" pelo endereço IP da API do banco de dados de sinais.  
14 - Exemplo: "db_host": "150.250.350.45".  
15 -  
16 -- Substitua "<path-to-corretor>" pelo caminho do diretório completo do projeto "corretor_sinais".  
17 -  
18 -- Substitua "<path-to-validador>" pelo caminho do diretório completo do projeto "validador_sinais".  
19 -  
20 -- Substitua "<path-to-wikilibras>" pelo caminho do diretório completo do projeto "wikilibrasV2". 15 +#### Substitua os valores: localhost, my-api-key
  16 +```json
  17 +{
  18 + "db_host": "localhost",
  19 + "pb_api_key": "my-api-key",
  20 + "pb_endpoint": "http://localhost/pybossa/"
  21 +}
  22 +```
21 23
22 -- Substitua my-api-key por sua API key do Pybossa. 24 +#### Exemplo:
  25 +```json
  26 +{
  27 + "db_host": "192.168.1.100",
  28 + "pb_api_key": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
  29 + "pb_endpoint": "http://192.168.1.100/pybossa/"
  30 +}
  31 +```
23 32
24 -## Instale dependências 33 +### Instalação de dependências
25 34
26 ```sh 35 ```sh
27 $ make install 36 $ make install
28 ``` 37 ```
29 38
30 -## Execute teste (Opcional) 39 +#### Execute teste (Opcional)
31 40
32 ```sh 41 ```sh
33 $ make run 42 $ make run
34 ``` 43 ```
35 44
36 -## Habilitar a inicialização automática do serviço 45 +#### Habilitar a inicialização automática do serviço
37 46
38 ```sh 47 ```sh
39 -$ make enable-startup 48 +$ make autostart-enable
40 ``` 49 ```
41 50
42 -## Desabilitar a inicialização automática do serviço 51 +#### Desabilitar a inicialização automática do serviço
43 52
44 ```sh 53 ```sh
45 -$ make disable-startup 54 +$ make autostart-disable
46 ``` 55 ```
1 # -*- coding: UTF-8 -*- 1 # -*- coding: UTF-8 -*-
2 2
  3 +import json
3 import os 4 import os
4 import pyutil 5 import pyutil
5 import sys 6 import sys
6 import task_manager 7 import task_manager
7 import time 8 import time
8 9
  10 +def load_json(filename):
  11 + r = {}
  12 + try:
  13 + fp = open(filename, "r")
  14 + if (fp):
  15 + r = json.load(fp)
  16 + fp.close()
  17 + except:
  18 + pyutil.print_stack_trace()
  19 + return r
  20 +
  21 +def save_json(data, filename, idt = 4):
  22 + try:
  23 + fp = open(filename, "w")
  24 + if (fp):
  25 + json.dump(
  26 + data,
  27 + fp,
  28 + ensure_ascii = False,
  29 + indent = idt,
  30 + sort_keys = True
  31 + )
  32 + fp.close()
  33 + except:
  34 + pyutil.print_stack_trace()
  35 + return False
  36 + return True
  37 +
9 def main(): 38 def main():
10 seconds = 10 39 seconds = 10
11 json_data = None 40 json_data = None
12 - task = None 41 + task_creator = None
13 pyutil.log("wikilibras task manager started") 42 pyutil.log("wikilibras task manager started")
14 - while (task == None): 43 + while (task_creator == None):
15 try: 44 try:
16 - json_data = task_manager.load_json(os.path.join(os.path.dirname(os.path.abspath('__file__')), "settings_local.json"))  
17 - task = task_manager.task_config(json_data) 45 + json_data = load_json(os.path.join(os.path.dirname(os.path.abspath("__file__")), "settings_local.json"))
  46 + os.path.dirname(os.path.abspath("__file__"))
  47 + task_creator = task_manager.task_creator(json_data)
18 except: 48 except:
19 - task = None  
20 - print("Waiting for Network to Get Projects ID's") 49 + task_creator = None
  50 + print("Waiting for Network to Get Project ID")
  51 + pyutil.print_stack_trace()
21 time.sleep(10) 52 time.sleep(10)
22 -  
23 - if (json_data != None and task != None): 53 + if (json_data != None and task_creator != None):
  54 + # print(task_creator.to_string())
24 pyutil.log("wikilibras task manager checking for tasks") 55 pyutil.log("wikilibras task manager checking for tasks")
25 while (True): 56 while (True):
26 - task.run(task.data["corretor"])  
27 - task.run(task.data["validador"])  
28 - task.run(task.data["wikilibras"])  
29 - print("Waiting %d seconds to check new tasks" % (seconds)) 57 + task_creator.run()
  58 + # print("Waiting %d seconds to check new tasks" % (seconds))
30 time.sleep(seconds) 59 time.sleep(seconds)
31 60
32 -if __name__ == '__main__': 61 +if __name__ == "__main__":
33 main() 62 main()
settings_local.json.template
  Diff comments   View file @1ef47e5
@@ -5,15 +5,12 @@ @@ -5,15 +5,12 @@
5 "pb_api_key": "my-api-key", 5 "pb_api_key": "my-api-key",
6 "pb_endpoint": "http://localhost/pybossa/", 6 "pb_endpoint": "http://localhost/pybossa/",
7 "corretor": { 7 "corretor": {
8 - "short_name": "corretor_sinais",  
9 - "video_path": "<path-to-corretor>/view/videos" 8 + "short_name": "corretor_sinais"
10 }, 9 },
11 "validador": { 10 "validador": {
12 - "short_name": "validador_sinais",  
13 - "video_path": "<path-to-validador>/view/videos" 11 + "short_name": "validador_sinais"
14 }, 12 },
15 "wikilibras": { 13 "wikilibras": {
16 - "short_name": "wikilibras",  
17 - "video_path": "<path-to-wikilibras>/view/videos" 14 + "short_name": "wikilibras"
18 } 15 }
19 } 16 }
task_manager.py
  Diff comments   View file @1ef47e5
@@ -8,29 +8,7 @@ import requests @@ -8,29 +8,7 @@ import requests
8 import shutil 8 import shutil
9 import logging 9 import logging
10 10
11 -def load_json(filename):  
12 - r = {}  
13 - try:  
14 - fp = open(filename, 'r')  
15 - if (fp):  
16 - r = json.load(fp)  
17 - fp.close()  
18 - except:  
19 - pass  
20 - return r  
21 -  
22 -def save_json(data, filename, idt = 4):  
23 - try:  
24 - fp = open(filename, 'w')  
25 - if (fp):  
26 - json.dump(data, fp, ensure_ascii = False, indent = idt, sort_keys = True)  
27 - fp.close()  
28 - except:  
29 - pyutil.print_stack_trace()  
30 - return False  
31 - return True  
32 -  
33 -class task_config(): 11 +class task_creator():
34 def __init__(self, json_data): 12 def __init__(self, json_data):
35 self.data = {} 13 self.data = {}
36 self.data["db_host"] = json_data.get("db_host", "localhost") 14 self.data["db_host"] = json_data.get("db_host", "localhost")
@@ -68,158 +46,74 @@ class task_config(): @@ -68,158 +46,74 @@ class task_config():
68 except: 46 except:
69 return {} 47 return {}
70 48
71 - def get_data_by_version(self, version):  
72 - r = self.__get__("sinais?version=%d" % (version))  
73 - if (r != {}):  
74 - # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4))  
75 - return r.json()  
76 - else:  
77 - return []  
78 -  
79 - def get_data_by_version_selo(self, version, selo):  
80 - r = self.__get__("sinais?version=%d&selo=%s" % (version, selo))  
81 - if (r != {}):  
82 - # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4))  
83 - return r.json()  
84 - else:  
85 - return []  
86 -  
87 - def get_data_by_selo(self, selo):  
88 - r = self.__get__("sinais?&selo=%s" % (selo)) 49 + def get_newtasks(self, selo = None):
  50 + url = "newtasks?task_manager_alive=true"
  51 + if (isinstance(selo, int)):
  52 + url += "?selo=%d" % (selo)
  53 + r = self.__get__(url)
89 if (r != {}): 54 if (r != {}):
90 # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4)) 55 # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4))
91 return r.json() 56 return r.json()
92 else: 57 else:
93 return [] 58 return []
94 59
95 - def get_listall(self):  
96 - r = self.__get__("listall")  
97 - if (r != {}):  
98 - # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4))  
99 - return r.json()  
100 - else:  
101 - return []  
102 -  
103 - def get_version(self):  
104 - r = self.__get__("version")  
105 - if (r != {}):  
106 - version = r.json()["version"]  
107 - if (isinstance(version, int)):  
108 - return version  
109 - return 0  
110 -  
111 - def get_file(self, url, filename): 60 + def run(self):
112 try: 61 try:
113 - r = requests.get(url, stream = True)  
114 - if (r.status_code == 200):  
115 - with open(filename, 'wb') as f:  
116 - for chunk in r.iter_content(chunk_size = 1024):  
117 - if chunk:  
118 - f.write(chunk)  
119 - return True  
120 - except:  
121 - pass  
122 - return False  
123 -  
124 - def run(self, project):  
125 - try:  
126 - video_path = project.get("video_path", "view/videos")  
127 - proj_name = project.get("short_name", "")  
128 - proj_id = project.get("project_id", "")  
129 - database = os.path.join(os.path.dirname(os.path.abspath('__file__')), "database.json")  
130 - database_bak = database + ".bak"  
131 - control = load_json(database)  
132 - changed = False  
133 - # server_version = self.get_version()  
134 - # for i in range(1, server_version + 1):  
135 - dbquery = None  
136 -  
137 - # selos  
138 - # 1 - wikilibras  
139 - # 2 - especialista  
140 - # 3 - invalido_wikilibras  
141 - # 4 - invalido_especialista  
142 - # 5 - animadores  
143 - # 6 - invalido_animadores  
144 - # 7 - null  
145 -  
146 - if (proj_name == "corretor_sinais"):  
147 - dbquery = self.get_data_by_selo("invalido_especialista")  
148 -  
149 - if (proj_name == "validador_sinais"):  
150 - dbquery = self.get_data_by_selo("wikilibras") + self.get_data_by_selo("animadores")  
151 -  
152 - if (proj_name == "wikilibras"):  
153 - dbquery = self.get_data_by_selo("null")  
154 -  
155 - for j in dbquery:  
156 - add_task = False  
157 - current_sinal_name = str(j["nome"]).upper()  
158 - current_version = int(j["version"])  
159 -  
160 - if (current_sinal_name in control):  
161 - if (control[current_sinal_name] < current_version):  
162 - control[current_sinal_name] = current_version  
163 - changed = True  
164 - add_task = True 62 + dbquery = self.get_newtasks()
  63 + tasks = []
  64 + if (dbquery == []):
  65 + print("Wikilibras DB API not running at 'http://%s:%s'" % (self.data["db_host"], self.data["db_port"]))
  66 + return
  67 + if (dbquery["status"]):
  68 + tasks = dbquery["data"]
  69 + else:
  70 + pyutil.log("DecodeError: " + str(dbquery))
  71 + return
  72 + for i in tasks:
  73 + id_selo = i["idSelo"]
  74 + nome_sinal = i["nome"]
  75 + id_sinal = i["idSinal"]
  76 + usuario = i["usuario"]
  77 + estado = i["estado"]
  78 + classe = i["classe"]
  79 + blend_file = i["blender"]
  80 + video_ref = i["file"]
  81 + video_ava = i["avatar"]
  82 + proj_name = None
  83 + if (id_selo == 7):
  84 + proj_name = "wikilibras";
  85 + elif (id_selo == 4):
  86 + proj_name = "corretor";
  87 + elif ((id_selo == 1) or (id_selo == 5)):
  88 + proj_name = "validador";
165 else: 89 else:
166 - control[current_sinal_name] = current_version  
167 - changed = True  
168 - add_task = True  
169 -  
170 - if(add_task):  
171 - avatar_url = "http://%s:%s/blender/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"nome"] + ".blend"))  
172 - video_ref_url = "http://%s:%s/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"file"]))  
173 - video_ava_url = "http://%s:%s/avatar/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"nome"] + ".webm"))  
174 -  
175 - avatar_out = os.path.join(video_path, str(j[u"nome"] + "_AVATAR.blend"))  
176 - video_ref_out = os.path.join(video_path, str(j[u"nome"] + "_REF.webm"))  
177 - video_ava_out = os.path.join(video_path, str(j[u"nome"] + "_AVATAR.webm"))  
178 -  
179 - # cria diretorio de destino caso nao exista  
180 - if not os.path.exists(video_path):  
181 - os.makedirs(video_path)  
182 -  
183 - # remove arquivos existentes relativos a pasta "avatar_out"  
184 - if pyutil.file_exists(avatar_out):  
185 - os.unlink(avatar_out)  
186 -  
187 - # remove arquivos existentes relativos a pasta "video_ref_out"  
188 - if pyutil.file_exists(video_ref_out):  
189 - os.unlink(video_ref_out)  
190 -  
191 - # remove arquivos existentes relativos a pasta "video_ava_out"  
192 - if pyutil.file_exists(video_ava_out):  
193 - os.unlink(video_ava_out)  
194 -  
195 - # faz download do arquivo blend  
196 - self.get_file(avatar_url, avatar_out)  
197 -  
198 - # faz download do video renderizado  
199 - self.get_file(video_ava_url, video_ava_out)  
200 -  
201 - # faz download do video de referencia  
202 - if (self.get_file(video_ref_url, video_ref_out)):  
203 - pyutil.log("%s: creating task: %s version: %s stamp: %s" % (proj_name, j[u"nome"], j[u"version"], j[u"nomeSelo"]))  
204 - task = dict(sign_name = j[u"nome"], submission_date = pyutil.get_date_now())  
205 - response_task = pbclient.create_task(proj_id, task)  
206 - pyutil.log(str(response_task))  
207 - else:  
208 - # nao foi possivel fazer o download do video de referencia (tarefa nao adicionada)  
209 - pyutil.log("%s: file not found: %s" % (proj_name, video_ref_url))  
210 -  
211 - if (changed):  
212 - # verifica se arquivo database.json existe para realizar backup  
213 - if ((os.path.isfile(database) == 1) and (os.path.exists(database) == 1)):  
214 -  
215 - # verifica se arquivo database.json.bak existe para remover  
216 - if ((os.path.isfile(database_bak) == 1) and (os.path.exists(database_bak) == 1)):  
217 - os.remove(database_bak)  
218 -  
219 - # faz backup antes de salvar nova base de dados  
220 - shutil.copy(database, database_bak)  
221 -  
222 - # salva nova base de dados  
223 - save_json(control, database, 2) 90 + continue
  91 + proj_id = self.data[proj_name]["project_id"]
  92 + public_users = "/public/users"
  93 + if (usuario != None):
  94 + public_users += "/" + usuario
  95 + if (estado != None):
  96 + public_users += "/" + estado
  97 + if (classe != None):
  98 + public_users += "/" + classe
  99 + task = dict(
  100 + sign_name = nome_sinal,
  101 + submission_date = pyutil.get_date_now()
  102 + )
  103 + if (video_ref != None):
  104 + task["video_ref"] = ("%s/%s" % (public_users, video_ref))
  105 + if (video_ava != None):
  106 + task["video_ava"] = ("%s/%s" % (public_users, video_ava))
  107 + if (blend_file != None):
  108 + task["blend"] = ("%s/%s" % (public_users, blend_file))
  109 + # print(repr(task).decode("unicode-escape"))
  110 + # print(json.dumps(task, ensure_ascii = True, indent = 4, sort_keys = False))
  111 + response_task = pbclient.create_task(proj_id, task)
  112 + pyutil.log("[%s] created task[%d] '%s'" % (proj_name, int(response_task.id), nome_sinal))
  113 + # TODO
  114 + # verificar se o sinal foi atualizado corretamente,
  115 + # caso contrario persistir lista com (id_sinal, task_id)
  116 + # para atualizar posteriormente
  117 + self.__get__("updatetask?idsinal=%d&idtask=%d" % (int(id_sinal), int(response_task.id)))
224 except: 118 except:
225 pyutil.print_stack_trace() 119 pyutil.print_stack_trace()