Download as
Browse Code »

Commit 1ef47e53ae63d3428846bf68383f102a857c8c8b

Authored by André Araújo
1 parent 5ff89612
Exists in master

Integração da criação de tarefas com DB API

Showing 5 changed files with 158 additions and 239 deletions   Show diff stats
Makefile
  Diff comments   View file @1ef47e5
1   -TASKMGR_DIR = $(CURDIR)
2   -TASKMGR_INIT_SCRIPT = wikilibras-taskmgr
3   -TASKMGR_INIT = ${TASKMGR_DIR}/${TASKMGR_INIT_SCRIPT}
4   -TASKMGR_ETC_INIT = /etc/init.d/${TASKMGR_INIT_SCRIPT}
5   -TASKMGR_LOG = ${TASKMGR_DIR}/events.log
6   -TASKMGR_REQUIREMENTS = ${TASKMGR_DIR}/requirements.txt
7   -TASKMGR_SETTINGS = ${TASKMGR_DIR}/settings_local.json
8   -TASKMGR_SETTINGS_T = ${TASKMGR_SETTINGS}.template
9   -TASKMGR_DATABASE = ${TASKMGR_DIR}/database.json
10   -
11   -PATH_CORRETOR = $(subst $(shell basename "$(CURDIR)"),corretor_sinais,$(CURDIR))
12   -PATH_VALIDADOR = $(subst $(shell basename "$(CURDIR)"),validador_sinais,$(CURDIR))
13   -PATH_WIKILIBRAS = $(subst $(shell basename "$(CURDIR)"),wikilibrasV2,$(CURDIR))
14   -
15   -SED_REPLACE_ALL_PATH = sed -i "s\#$(1)\#$(2)\#g" "$(3)"
16   -
17   -install: uninstall build config startup-enable
18   -
19   -build:
  1 +TASKMGR_DIR := $(CURDIR)
  2 +TASKMGR_INIT_SCRIPT := wikilibras-taskmgr
  3 +TASKMGR_INIT := ${TASKMGR_DIR}/${TASKMGR_INIT_SCRIPT}
  4 +TASKMGR_ETC_INIT := /etc/init.d/${TASKMGR_INIT_SCRIPT}
  5 +TASKMGR_LOG := ${TASKMGR_DIR}/events.log
  6 +TASKMGR_REQUIREMENTS := ${TASKMGR_DIR}/requirements.txt
  7 +TASKMGR_SETTINGS := ${TASKMGR_DIR}/settings_local.json
  8 +TASKMGR_SETTINGS_T := ${TASKMGR_SETTINGS}.template
  9 +TASKMGR_DATABASE := ${TASKMGR_DIR}/database.json
  10 +VLIBRAS_WIKILIBRAS_ENV := /etc/profile.d/lavid/vlibras/wikilibras_env.sh
  11 +SED_REPLACE_ALL = sed -i "s/$(1)/$(2)/g" "$(3)"
  12 +SED_REPLACE_ALL_PATH = sed -i "s\#$(1)\#$(2)\#g" "$(3)"
  13 +
  14 +-include ${VLIBRAS_WIKILIBRAS_ENV}
  15 +
  16 +install: uninstall env config autostart-enable
  17 +
  18 +env:
20 19 @ ( \
21   - cd "${TASKMGR_DIR}"; \
22 20 virtualenv ./env; \
23 21 . ./env/bin/activate; \
24 22 pip install -U pip; \
... ... @@ -27,19 +25,18 @@ build:
27 25 )
28 26  
29 27 clean:
30   - @ sudo find ${TASKMGR_DIR} -regextype posix-awk -regex "(.*.log|.*.pyc)" -type f -delete
  28 + @ sudo find . -regextype posix-awk -regex "(.*.log|.*.pyc)" -type f -delete
31 29  
32 30 config:
33 31 @ cp "${TASKMGR_SETTINGS_T}" "${TASKMGR_SETTINGS}"
34   - @ $(call SED_REPLACE_ALL_PATH,<path-to-corretor>,${PATH_CORRETOR},${TASKMGR_SETTINGS})
35   - @ $(call SED_REPLACE_ALL_PATH,<path-to-validador>,${PATH_VALIDADOR},${TASKMGR_SETTINGS})
36   - @ $(call SED_REPLACE_ALL_PATH,<path-to-wikilibras>,${PATH_WIKILIBRAS},${TASKMGR_SETTINGS})
  32 + @ $(call SED_REPLACE_ALL,my-api-key,${PYBOSSA_API_KEY},${TASKMGR_SETTINGS})
  33 + @ $(call SED_REPLACE_ALL,localhost,${LOCALHOST},${TASKMGR_SETTINGS})
37 34  
38   -startup-disable:
  35 +autostart-disable:
39 36 @ sudo update-rc.d -f "${TASKMGR_INIT_SCRIPT}" remove
40 37 @ sudo rm -f "${TASKMGR_ETC_INIT}"
41 38  
42   -startup-enable:
  39 +autostart-enable:
43 40 @ sed "s#<path-to-project>#${TASKMGR_DIR}#" "${TASKMGR_INIT}" | sudo tee "${TASKMGR_ETC_INIT}" > /dev/null
44 41 @ sudo chmod 755 "${TASKMGR_ETC_INIT}"
45 42 @ sudo chown "root:root" "${TASKMGR_ETC_INIT}"
... ... @@ -47,18 +44,11 @@ startup-enable:
47 44  
48 45 run:
49 46 @ ( \
50   - cd "${TASKMGR_DIR}"; \
51 47 . ./env/bin/activate; \
52 48 python main.py; \
53 49 )
54 50  
55   -uninstall: clean startup-disable
  51 +uninstall: clean autostart-disable
56 52 @ rm -rf ./env/
57 53  
58   -reset: clean
59   - @ if [ -e "${TASKMGR_DATABASE}" ]; \
60   - then \
61   - cat "${TASKMGR_DATABASE}"; \
62   - echo ""; \
63   - sudo rm -f "${TASKMGR_DATABASE}"; \
64   - fi
  54 +.PHONY: clean config run uninstall
... ...
README.md
  Diff comments   View file @1ef47e5
1   -## Copie o arquivo modelo de configuração
  1 +### Configuração de Instalação
  2 +
  3 +#### Copie o arquivo modelo de configuração
2 4  
3 5 ```sh
4 6 $ cp settings_local.json.template settings_local.json
5 7 ```
6 8  
7   -## Edite o arquivo de configuração
  9 +#### Edite o arquivo de configuração
8 10  
9 11 ```sh
10 12 $ gedit settings_local.json
11 13 ```
12 14  
13   -- Substitua o valor da chave "db_host": "localhost" pelo endereço IP da API do banco de dados de sinais.
14   - Exemplo: "db_host": "150.250.350.45".
15   -
16   -- Substitua "<path-to-corretor>" pelo caminho do diretório completo do projeto "corretor_sinais".
17   -
18   -- Substitua "<path-to-validador>" pelo caminho do diretório completo do projeto "validador_sinais".
19   -
20   -- Substitua "<path-to-wikilibras>" pelo caminho do diretório completo do projeto "wikilibrasV2".
  15 +#### Substitua os valores: localhost, my-api-key
  16 +```json
  17 +{
  18 + "db_host": "localhost",
  19 + "pb_api_key": "my-api-key",
  20 + "pb_endpoint": "http://localhost/pybossa/"
  21 +}
  22 +```
21 23  
22   -- Substitua my-api-key por sua API key do Pybossa.
  24 +#### Exemplo:
  25 +```json
  26 +{
  27 + "db_host": "192.168.1.100",
  28 + "pb_api_key": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX",
  29 + "pb_endpoint": "http://192.168.1.100/pybossa/"
  30 +}
  31 +```
23 32  
24   -## Instale dependências
  33 +### Instalação de dependências
25 34  
26 35 ```sh
27 36 $ make install
28 37 ```
29 38  
30   -## Execute teste (Opcional)
  39 +#### Execute teste (Opcional)
31 40  
32 41 ```sh
33 42 $ make run
34 43 ```
35 44  
36   -## Habilitar a inicialização automática do serviço
  45 +#### Habilitar a inicialização automática do serviço
37 46  
38 47 ```sh
39   -$ make enable-startup
  48 +$ make autostart-enable
40 49 ```
41 50  
42   -## Desabilitar a inicialização automática do serviço
  51 +#### Desabilitar a inicialização automática do serviço
43 52  
44 53 ```sh
45   -$ make disable-startup
  54 +$ make autostart-disable
46 55 ```
... ...
main.py
  Diff comments   View file @1ef47e5
1 1 # -*- coding: UTF-8 -*-
2 2  
  3 +import json
3 4 import os
4 5 import pyutil
5 6 import sys
6 7 import task_manager
7 8 import time
8 9  
  10 +def load_json(filename):
  11 + r = {}
  12 + try:
  13 + fp = open(filename, "r")
  14 + if (fp):
  15 + r = json.load(fp)
  16 + fp.close()
  17 + except:
  18 + pyutil.print_stack_trace()
  19 + return r
  20 +
  21 +def save_json(data, filename, idt = 4):
  22 + try:
  23 + fp = open(filename, "w")
  24 + if (fp):
  25 + json.dump(
  26 + data,
  27 + fp,
  28 + ensure_ascii = False,
  29 + indent = idt,
  30 + sort_keys = True
  31 + )
  32 + fp.close()
  33 + except:
  34 + pyutil.print_stack_trace()
  35 + return False
  36 + return True
  37 +
9 38 def main():
10 39 seconds = 10
11 40 json_data = None
12   - task = None
  41 + task_creator = None
13 42 pyutil.log("wikilibras task manager started")
14   - while (task == None):
  43 + while (task_creator == None):
15 44 try:
16   - json_data = task_manager.load_json(os.path.join(os.path.dirname(os.path.abspath('__file__')), "settings_local.json"))
17   - task = task_manager.task_config(json_data)
  45 + json_data = load_json(os.path.join(os.path.dirname(os.path.abspath("__file__")), "settings_local.json"))
  46 + os.path.dirname(os.path.abspath("__file__"))
  47 + task_creator = task_manager.task_creator(json_data)
18 48 except:
19   - task = None
20   - print("Waiting for Network to Get Projects ID's")
  49 + task_creator = None
  50 + print("Waiting for Network to Get Project ID")
  51 + pyutil.print_stack_trace()
21 52 time.sleep(10)
22   -
23   - if (json_data != None and task != None):
  53 + if (json_data != None and task_creator != None):
  54 + # print(task_creator.to_string())
24 55 pyutil.log("wikilibras task manager checking for tasks")
25 56 while (True):
26   - task.run(task.data["corretor"])
27   - task.run(task.data["validador"])
28   - task.run(task.data["wikilibras"])
29   - print("Waiting %d seconds to check new tasks" % (seconds))
  57 + task_creator.run()
  58 + # print("Waiting %d seconds to check new tasks" % (seconds))
30 59 time.sleep(seconds)
31 60  
32   -if __name__ == '__main__':
  61 +if __name__ == "__main__":
33 62 main()
... ...
settings_local.json.template
  Diff comments   View file @1ef47e5
... ... @@ -5,15 +5,12 @@
5 5 "pb_api_key": "my-api-key",
6 6 "pb_endpoint": "http://localhost/pybossa/",
7 7 "corretor": {
8   - "short_name": "corretor_sinais",
9   - "video_path": "<path-to-corretor>/view/videos"
  8 + "short_name": "corretor_sinais"
10 9 },
11 10 "validador": {
12   - "short_name": "validador_sinais",
13   - "video_path": "<path-to-validador>/view/videos"
  11 + "short_name": "validador_sinais"
14 12 },
15 13 "wikilibras": {
16   - "short_name": "wikilibras",
17   - "video_path": "<path-to-wikilibras>/view/videos"
  14 + "short_name": "wikilibras"
18 15 }
19 16 }
... ...
task_manager.py
  Diff comments   View file @1ef47e5
... ... @@ -8,29 +8,7 @@ import requests
8 8 import shutil
9 9 import logging
10 10  
11   -def load_json(filename):
12   - r = {}
13   - try:
14   - fp = open(filename, 'r')
15   - if (fp):
16   - r = json.load(fp)
17   - fp.close()
18   - except:
19   - pass
20   - return r
21   -
22   -def save_json(data, filename, idt = 4):
23   - try:
24   - fp = open(filename, 'w')
25   - if (fp):
26   - json.dump(data, fp, ensure_ascii = False, indent = idt, sort_keys = True)
27   - fp.close()
28   - except:
29   - pyutil.print_stack_trace()
30   - return False
31   - return True
32   -
33   -class task_config():
  11 +class task_creator():
34 12 def __init__(self, json_data):
35 13 self.data = {}
36 14 self.data["db_host"] = json_data.get("db_host", "localhost")
... ... @@ -68,158 +46,74 @@ class task_config():
68 46 except:
69 47 return {}
70 48  
71   - def get_data_by_version(self, version):
72   - r = self.__get__("sinais?version=%d" % (version))
73   - if (r != {}):
74   - # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4))
75   - return r.json()
76   - else:
77   - return []
78   -
79   - def get_data_by_version_selo(self, version, selo):
80   - r = self.__get__("sinais?version=%d&selo=%s" % (version, selo))
81   - if (r != {}):
82   - # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4))
83   - return r.json()
84   - else:
85   - return []
86   -
87   - def get_data_by_selo(self, selo):
88   - r = self.__get__("sinais?&selo=%s" % (selo))
  49 + def get_newtasks(self, selo = None):
  50 + url = "newtasks?task_manager_alive=true"
  51 + if (isinstance(selo, int)):
  52 + url += "?selo=%d" % (selo)
  53 + r = self.__get__(url)
89 54 if (r != {}):
90 55 # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4))
91 56 return r.json()
92 57 else:
93 58 return []
94 59  
95   - def get_listall(self):
96   - r = self.__get__("listall")
97   - if (r != {}):
98   - # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4))
99   - return r.json()
100   - else:
101   - return []
102   -
103   - def get_version(self):
104   - r = self.__get__("version")
105   - if (r != {}):
106   - version = r.json()["version"]
107   - if (isinstance(version, int)):
108   - return version
109   - return 0
110   -
111   - def get_file(self, url, filename):
  60 + def run(self):
112 61 try:
113   - r = requests.get(url, stream = True)
114   - if (r.status_code == 200):
115   - with open(filename, 'wb') as f:
116   - for chunk in r.iter_content(chunk_size = 1024):
117   - if chunk:
118   - f.write(chunk)
119   - return True
120   - except:
121   - pass
122   - return False
123   -
124   - def run(self, project):
125   - try:
126   - video_path = project.get("video_path", "view/videos")
127   - proj_name = project.get("short_name", "")
128   - proj_id = project.get("project_id", "")
129   - database = os.path.join(os.path.dirname(os.path.abspath('__file__')), "database.json")
130   - database_bak = database + ".bak"
131   - control = load_json(database)
132   - changed = False
133   - # server_version = self.get_version()
134   - # for i in range(1, server_version + 1):
135   - dbquery = None
136   -
137   - # selos
138   - # 1 - wikilibras
139   - # 2 - especialista
140   - # 3 - invalido_wikilibras
141   - # 4 - invalido_especialista
142   - # 5 - animadores
143   - # 6 - invalido_animadores
144   - # 7 - null
145   -
146   - if (proj_name == "corretor_sinais"):
147   - dbquery = self.get_data_by_selo("invalido_especialista")
148   -
149   - if (proj_name == "validador_sinais"):
150   - dbquery = self.get_data_by_selo("wikilibras") + self.get_data_by_selo("animadores")
151   -
152   - if (proj_name == "wikilibras"):
153   - dbquery = self.get_data_by_selo("null")
154   -
155   - for j in dbquery:
156   - add_task = False
157   - current_sinal_name = str(j["nome"]).upper()
158   - current_version = int(j["version"])
159   -
160   - if (current_sinal_name in control):
161   - if (control[current_sinal_name] < current_version):
162   - control[current_sinal_name] = current_version
163   - changed = True
164   - add_task = True
  62 + dbquery = self.get_newtasks()
  63 + tasks = []
  64 + if (dbquery == []):
  65 + print("Wikilibras DB API not running at 'http://%s:%s'" % (self.data["db_host"], self.data["db_port"]))
  66 + return
  67 + if (dbquery["status"]):
  68 + tasks = dbquery["data"]
  69 + else:
  70 + pyutil.log("DecodeError: " + str(dbquery))
  71 + return
  72 + for i in tasks:
  73 + id_selo = i["idSelo"]
  74 + nome_sinal = i["nome"]
  75 + id_sinal = i["idSinal"]
  76 + usuario = i["usuario"]
  77 + estado = i["estado"]
  78 + classe = i["classe"]
  79 + blend_file = i["blender"]
  80 + video_ref = i["file"]
  81 + video_ava = i["avatar"]
  82 + proj_name = None
  83 + if (id_selo == 7):
  84 + proj_name = "wikilibras";
  85 + elif (id_selo == 4):
  86 + proj_name = "corretor";
  87 + elif ((id_selo == 1) or (id_selo == 5)):
  88 + proj_name = "validador";
165 89 else:
166   - control[current_sinal_name] = current_version
167   - changed = True
168   - add_task = True
169   -
170   - if(add_task):
171   - avatar_url = "http://%s:%s/blender/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"nome"] + ".blend"))
172   - video_ref_url = "http://%s:%s/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"file"]))
173   - video_ava_url = "http://%s:%s/avatar/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"nome"] + ".webm"))
174   -
175   - avatar_out = os.path.join(video_path, str(j[u"nome"] + "_AVATAR.blend"))
176   - video_ref_out = os.path.join(video_path, str(j[u"nome"] + "_REF.webm"))
177   - video_ava_out = os.path.join(video_path, str(j[u"nome"] + "_AVATAR.webm"))
178   -
179   - # cria diretorio de destino caso nao exista
180   - if not os.path.exists(video_path):
181   - os.makedirs(video_path)
182   -
183   - # remove arquivos existentes relativos a pasta "avatar_out"
184   - if pyutil.file_exists(avatar_out):
185   - os.unlink(avatar_out)
186   -
187   - # remove arquivos existentes relativos a pasta "video_ref_out"
188   - if pyutil.file_exists(video_ref_out):
189   - os.unlink(video_ref_out)
190   -
191   - # remove arquivos existentes relativos a pasta "video_ava_out"
192   - if pyutil.file_exists(video_ava_out):
193   - os.unlink(video_ava_out)
194   -
195   - # faz download do arquivo blend
196   - self.get_file(avatar_url, avatar_out)
197   -
198   - # faz download do video renderizado
199   - self.get_file(video_ava_url, video_ava_out)
200   -
201   - # faz download do video de referencia
202   - if (self.get_file(video_ref_url, video_ref_out)):
203   - pyutil.log("%s: creating task: %s version: %s stamp: %s" % (proj_name, j[u"nome"], j[u"version"], j[u"nomeSelo"]))
204   - task = dict(sign_name = j[u"nome"], submission_date = pyutil.get_date_now())
205   - response_task = pbclient.create_task(proj_id, task)
206   - pyutil.log(str(response_task))
207   - else:
208   - # nao foi possivel fazer o download do video de referencia (tarefa nao adicionada)
209   - pyutil.log("%s: file not found: %s" % (proj_name, video_ref_url))
210   -
211   - if (changed):
212   - # verifica se arquivo database.json existe para realizar backup
213   - if ((os.path.isfile(database) == 1) and (os.path.exists(database) == 1)):
214   -
215   - # verifica se arquivo database.json.bak existe para remover
216   - if ((os.path.isfile(database_bak) == 1) and (os.path.exists(database_bak) == 1)):
217   - os.remove(database_bak)
218   -
219   - # faz backup antes de salvar nova base de dados
220   - shutil.copy(database, database_bak)
221   -
222   - # salva nova base de dados
223   - save_json(control, database, 2)
  90 + continue
  91 + proj_id = self.data[proj_name]["project_id"]
  92 + public_users = "/public/users"
  93 + if (usuario != None):
  94 + public_users += "/" + usuario
  95 + if (estado != None):
  96 + public_users += "/" + estado
  97 + if (classe != None):
  98 + public_users += "/" + classe
  99 + task = dict(
  100 + sign_name = nome_sinal,
  101 + submission_date = pyutil.get_date_now()
  102 + )
  103 + if (video_ref != None):
  104 + task["video_ref"] = ("%s/%s" % (public_users, video_ref))
  105 + if (video_ava != None):
  106 + task["video_ava"] = ("%s/%s" % (public_users, video_ava))
  107 + if (blend_file != None):
  108 + task["blend"] = ("%s/%s" % (public_users, blend_file))
  109 + # print(repr(task).decode("unicode-escape"))
  110 + # print(json.dumps(task, ensure_ascii = True, indent = 4, sort_keys = False))
  111 + response_task = pbclient.create_task(proj_id, task)
  112 + pyutil.log("[%s] created task[%d] '%s'" % (proj_name, int(response_task.id), nome_sinal))
  113 + # TODO
  114 + # verificar se o sinal foi atualizado corretamente,
  115 + # caso contrario persistir lista com (id_sinal, task_id)
  116 + # para atualizar posteriormente
  117 + self.__get__("updatetask?idsinal=%d&idtask=%d" % (int(id_sinal), int(response_task.id)))
224 118 except:
225 119 pyutil.print_stack_trace()
... ...