diff --git a/Makefile b/Makefile index 1ac76b8..5048e64 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,22 @@ -TASKMGR_DIR = $(CURDIR) -TASKMGR_INIT_SCRIPT = wikilibras-taskmgr -TASKMGR_INIT = ${TASKMGR_DIR}/${TASKMGR_INIT_SCRIPT} -TASKMGR_ETC_INIT = /etc/init.d/${TASKMGR_INIT_SCRIPT} -TASKMGR_LOG = ${TASKMGR_DIR}/events.log -TASKMGR_REQUIREMENTS = ${TASKMGR_DIR}/requirements.txt -TASKMGR_SETTINGS = ${TASKMGR_DIR}/settings_local.json -TASKMGR_SETTINGS_T = ${TASKMGR_SETTINGS}.template -TASKMGR_DATABASE = ${TASKMGR_DIR}/database.json - -PATH_CORRETOR = $(subst $(shell basename "$(CURDIR)"),corretor_sinais,$(CURDIR)) -PATH_VALIDADOR = $(subst $(shell basename "$(CURDIR)"),validador_sinais,$(CURDIR)) -PATH_WIKILIBRAS = $(subst $(shell basename "$(CURDIR)"),wikilibrasV2,$(CURDIR)) - -SED_REPLACE_ALL_PATH = sed -i "s\#$(1)\#$(2)\#g" "$(3)" - -install: uninstall build config startup-enable - -build: +TASKMGR_DIR := $(CURDIR) +TASKMGR_INIT_SCRIPT := wikilibras-taskmgr +TASKMGR_INIT := ${TASKMGR_DIR}/${TASKMGR_INIT_SCRIPT} +TASKMGR_ETC_INIT := /etc/init.d/${TASKMGR_INIT_SCRIPT} +TASKMGR_LOG := ${TASKMGR_DIR}/events.log +TASKMGR_REQUIREMENTS := ${TASKMGR_DIR}/requirements.txt +TASKMGR_SETTINGS := ${TASKMGR_DIR}/settings_local.json +TASKMGR_SETTINGS_T := ${TASKMGR_SETTINGS}.template +TASKMGR_DATABASE := ${TASKMGR_DIR}/database.json +VLIBRAS_WIKILIBRAS_ENV := /etc/profile.d/lavid/vlibras/wikilibras_env.sh +SED_REPLACE_ALL = sed -i "s/$(1)/$(2)/g" "$(3)" +SED_REPLACE_ALL_PATH = sed -i "s\#$(1)\#$(2)\#g" "$(3)" + +-include ${VLIBRAS_WIKILIBRAS_ENV} + +install: uninstall env config autostart-enable + +env: @ ( \ - cd "${TASKMGR_DIR}"; \ virtualenv ./env; \ . ./env/bin/activate; \ pip install -U pip; \ @@ -27,19 +25,18 @@ build: ) clean: - @ sudo find ${TASKMGR_DIR} -regextype posix-awk -regex "(.*.log|.*.pyc)" -type f -delete + @ sudo find . -regextype posix-awk -regex "(.*.log|.*.pyc)" -type f -delete config: @ cp "${TASKMGR_SETTINGS_T}" "${TASKMGR_SETTINGS}" - @ $(call SED_REPLACE_ALL_PATH,,${PATH_CORRETOR},${TASKMGR_SETTINGS}) - @ $(call SED_REPLACE_ALL_PATH,,${PATH_VALIDADOR},${TASKMGR_SETTINGS}) - @ $(call SED_REPLACE_ALL_PATH,,${PATH_WIKILIBRAS},${TASKMGR_SETTINGS}) + @ $(call SED_REPLACE_ALL,my-api-key,${PYBOSSA_API_KEY},${TASKMGR_SETTINGS}) + @ $(call SED_REPLACE_ALL,localhost,${LOCALHOST},${TASKMGR_SETTINGS}) -startup-disable: +autostart-disable: @ sudo update-rc.d -f "${TASKMGR_INIT_SCRIPT}" remove @ sudo rm -f "${TASKMGR_ETC_INIT}" -startup-enable: +autostart-enable: @ sed "s##${TASKMGR_DIR}#" "${TASKMGR_INIT}" | sudo tee "${TASKMGR_ETC_INIT}" > /dev/null @ sudo chmod 755 "${TASKMGR_ETC_INIT}" @ sudo chown "root:root" "${TASKMGR_ETC_INIT}" @@ -47,18 +44,11 @@ startup-enable: run: @ ( \ - cd "${TASKMGR_DIR}"; \ . ./env/bin/activate; \ python main.py; \ ) -uninstall: clean startup-disable +uninstall: clean autostart-disable @ rm -rf ./env/ -reset: clean - @ if [ -e "${TASKMGR_DATABASE}" ]; \ - then \ - cat "${TASKMGR_DATABASE}"; \ - echo ""; \ - sudo rm -f "${TASKMGR_DATABASE}"; \ - fi +.PHONY: clean config run uninstall diff --git a/README.md b/README.md index d6a4fb1..27c213e 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,55 @@ -## Copie o arquivo modelo de configuração +### Configuração de Instalação + +#### Copie o arquivo modelo de configuração ```sh $ cp settings_local.json.template settings_local.json ``` -## Edite o arquivo de configuração +#### Edite o arquivo de configuração ```sh $ gedit settings_local.json ``` -- Substitua o valor da chave "db_host": "localhost" pelo endereço IP da API do banco de dados de sinais. - Exemplo: "db_host": "150.250.350.45". - -- Substitua "" pelo caminho do diretório completo do projeto "corretor_sinais". - -- Substitua "" pelo caminho do diretório completo do projeto "validador_sinais". - -- Substitua "" pelo caminho do diretório completo do projeto "wikilibrasV2". +#### Substitua os valores: localhost, my-api-key +```json +{ + "db_host": "localhost", + "pb_api_key": "my-api-key", + "pb_endpoint": "http://localhost/pybossa/" +} +``` -- Substitua my-api-key por sua API key do Pybossa. +#### Exemplo: +```json +{ + "db_host": "192.168.1.100", + "pb_api_key": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", + "pb_endpoint": "http://192.168.1.100/pybossa/" +} +``` -## Instale dependências +### Instalação de dependências ```sh $ make install ``` -## Execute teste (Opcional) +#### Execute teste (Opcional) ```sh $ make run ``` -## Habilitar a inicialização automática do serviço +#### Habilitar a inicialização automática do serviço ```sh -$ make enable-startup +$ make autostart-enable ``` -## Desabilitar a inicialização automática do serviço +#### Desabilitar a inicialização automática do serviço ```sh -$ make disable-startup +$ make autostart-disable ``` diff --git a/main.py b/main.py index 7aabec9..d8f93c9 100644 --- a/main.py +++ b/main.py @@ -1,33 +1,62 @@ # -*- coding: UTF-8 -*- +import json import os import pyutil import sys import task_manager import time +def load_json(filename): + r = {} + try: + fp = open(filename, "r") + if (fp): + r = json.load(fp) + fp.close() + except: + pyutil.print_stack_trace() + return r + +def save_json(data, filename, idt = 4): + try: + fp = open(filename, "w") + if (fp): + json.dump( + data, + fp, + ensure_ascii = False, + indent = idt, + sort_keys = True + ) + fp.close() + except: + pyutil.print_stack_trace() + return False + return True + def main(): seconds = 10 json_data = None - task = None + task_creator = None pyutil.log("wikilibras task manager started") - while (task == None): + while (task_creator == None): try: - json_data = task_manager.load_json(os.path.join(os.path.dirname(os.path.abspath('__file__')), "settings_local.json")) - task = task_manager.task_config(json_data) + json_data = load_json(os.path.join(os.path.dirname(os.path.abspath("__file__")), "settings_local.json")) + os.path.dirname(os.path.abspath("__file__")) + task_creator = task_manager.task_creator(json_data) except: - task = None - print("Waiting for Network to Get Projects ID's") + task_creator = None + print("Waiting for Network to Get Project ID") + pyutil.print_stack_trace() time.sleep(10) - - if (json_data != None and task != None): + if (json_data != None and task_creator != None): + # print(task_creator.to_string()) pyutil.log("wikilibras task manager checking for tasks") while (True): - task.run(task.data["corretor"]) - task.run(task.data["validador"]) - task.run(task.data["wikilibras"]) - print("Waiting %d seconds to check new tasks" % (seconds)) + task_creator.run() + # print("Waiting %d seconds to check new tasks" % (seconds)) time.sleep(seconds) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/settings_local.json.template b/settings_local.json.template index a69997b..f820416 100644 --- a/settings_local.json.template +++ b/settings_local.json.template @@ -5,15 +5,12 @@ "pb_api_key": "my-api-key", "pb_endpoint": "http://localhost/pybossa/", "corretor": { - "short_name": "corretor_sinais", - "video_path": "/view/videos" + "short_name": "corretor_sinais" }, "validador": { - "short_name": "validador_sinais", - "video_path": "/view/videos" + "short_name": "validador_sinais" }, "wikilibras": { - "short_name": "wikilibras", - "video_path": "/view/videos" + "short_name": "wikilibras" } } diff --git a/task_manager.py b/task_manager.py index 5e6ca3a..733db46 100644 --- a/task_manager.py +++ b/task_manager.py @@ -8,29 +8,7 @@ import requests import shutil import logging -def load_json(filename): - r = {} - try: - fp = open(filename, 'r') - if (fp): - r = json.load(fp) - fp.close() - except: - pass - return r - -def save_json(data, filename, idt = 4): - try: - fp = open(filename, 'w') - if (fp): - json.dump(data, fp, ensure_ascii = False, indent = idt, sort_keys = True) - fp.close() - except: - pyutil.print_stack_trace() - return False - return True - -class task_config(): +class task_creator(): def __init__(self, json_data): self.data = {} self.data["db_host"] = json_data.get("db_host", "localhost") @@ -68,158 +46,74 @@ class task_config(): except: return {} - def get_data_by_version(self, version): - r = self.__get__("sinais?version=%d" % (version)) - if (r != {}): - # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4)) - return r.json() - else: - return [] - - def get_data_by_version_selo(self, version, selo): - r = self.__get__("sinais?version=%d&selo=%s" % (version, selo)) - if (r != {}): - # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4)) - return r.json() - else: - return [] - - def get_data_by_selo(self, selo): - r = self.__get__("sinais?&selo=%s" % (selo)) + def get_newtasks(self, selo = None): + url = "newtasks?task_manager_alive=true" + if (isinstance(selo, int)): + url += "?selo=%d" % (selo) + r = self.__get__(url) if (r != {}): # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4)) return r.json() else: return [] - def get_listall(self): - r = self.__get__("listall") - if (r != {}): - # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4)) - return r.json() - else: - return [] - - def get_version(self): - r = self.__get__("version") - if (r != {}): - version = r.json()["version"] - if (isinstance(version, int)): - return version - return 0 - - def get_file(self, url, filename): + def run(self): try: - r = requests.get(url, stream = True) - if (r.status_code == 200): - with open(filename, 'wb') as f: - for chunk in r.iter_content(chunk_size = 1024): - if chunk: - f.write(chunk) - return True - except: - pass - return False - - def run(self, project): - try: - video_path = project.get("video_path", "view/videos") - proj_name = project.get("short_name", "") - proj_id = project.get("project_id", "") - database = os.path.join(os.path.dirname(os.path.abspath('__file__')), "database.json") - database_bak = database + ".bak" - control = load_json(database) - changed = False - # server_version = self.get_version() - # for i in range(1, server_version + 1): - dbquery = None - - # selos - # 1 - wikilibras - # 2 - especialista - # 3 - invalido_wikilibras - # 4 - invalido_especialista - # 5 - animadores - # 6 - invalido_animadores - # 7 - null - - if (proj_name == "corretor_sinais"): - dbquery = self.get_data_by_selo("invalido_especialista") - - if (proj_name == "validador_sinais"): - dbquery = self.get_data_by_selo("wikilibras") + self.get_data_by_selo("animadores") - - if (proj_name == "wikilibras"): - dbquery = self.get_data_by_selo("null") - - for j in dbquery: - add_task = False - current_sinal_name = str(j["nome"]).upper() - current_version = int(j["version"]) - - if (current_sinal_name in control): - if (control[current_sinal_name] < current_version): - control[current_sinal_name] = current_version - changed = True - add_task = True + dbquery = self.get_newtasks() + tasks = [] + if (dbquery == []): + print("Wikilibras DB API not running at 'http://%s:%s'" % (self.data["db_host"], self.data["db_port"])) + return + if (dbquery["status"]): + tasks = dbquery["data"] + else: + pyutil.log("DecodeError: " + str(dbquery)) + return + for i in tasks: + id_selo = i["idSelo"] + nome_sinal = i["nome"] + id_sinal = i["idSinal"] + usuario = i["usuario"] + estado = i["estado"] + classe = i["classe"] + blend_file = i["blender"] + video_ref = i["file"] + video_ava = i["avatar"] + proj_name = None + if (id_selo == 7): + proj_name = "wikilibras"; + elif (id_selo == 4): + proj_name = "corretor"; + elif ((id_selo == 1) or (id_selo == 5)): + proj_name = "validador"; else: - control[current_sinal_name] = current_version - changed = True - add_task = True - - if(add_task): - avatar_url = "http://%s:%s/blender/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"nome"] + ".blend")) - video_ref_url = "http://%s:%s/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"file"])) - video_ava_url = "http://%s:%s/avatar/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"nome"] + ".webm")) - - avatar_out = os.path.join(video_path, str(j[u"nome"] + "_AVATAR.blend")) - video_ref_out = os.path.join(video_path, str(j[u"nome"] + "_REF.webm")) - video_ava_out = os.path.join(video_path, str(j[u"nome"] + "_AVATAR.webm")) - - # cria diretorio de destino caso nao exista - if not os.path.exists(video_path): - os.makedirs(video_path) - - # remove arquivos existentes relativos a pasta "avatar_out" - if pyutil.file_exists(avatar_out): - os.unlink(avatar_out) - - # remove arquivos existentes relativos a pasta "video_ref_out" - if pyutil.file_exists(video_ref_out): - os.unlink(video_ref_out) - - # remove arquivos existentes relativos a pasta "video_ava_out" - if pyutil.file_exists(video_ava_out): - os.unlink(video_ava_out) - - # faz download do arquivo blend - self.get_file(avatar_url, avatar_out) - - # faz download do video renderizado - self.get_file(video_ava_url, video_ava_out) - - # faz download do video de referencia - if (self.get_file(video_ref_url, video_ref_out)): - pyutil.log("%s: creating task: %s version: %s stamp: %s" % (proj_name, j[u"nome"], j[u"version"], j[u"nomeSelo"])) - task = dict(sign_name = j[u"nome"], submission_date = pyutil.get_date_now()) - response_task = pbclient.create_task(proj_id, task) - pyutil.log(str(response_task)) - else: - # nao foi possivel fazer o download do video de referencia (tarefa nao adicionada) - pyutil.log("%s: file not found: %s" % (proj_name, video_ref_url)) - - if (changed): - # verifica se arquivo database.json existe para realizar backup - if ((os.path.isfile(database) == 1) and (os.path.exists(database) == 1)): - - # verifica se arquivo database.json.bak existe para remover - if ((os.path.isfile(database_bak) == 1) and (os.path.exists(database_bak) == 1)): - os.remove(database_bak) - - # faz backup antes de salvar nova base de dados - shutil.copy(database, database_bak) - - # salva nova base de dados - save_json(control, database, 2) + continue + proj_id = self.data[proj_name]["project_id"] + public_users = "/public/users" + if (usuario != None): + public_users += "/" + usuario + if (estado != None): + public_users += "/" + estado + if (classe != None): + public_users += "/" + classe + task = dict( + sign_name = nome_sinal, + submission_date = pyutil.get_date_now() + ) + if (video_ref != None): + task["video_ref"] = ("%s/%s" % (public_users, video_ref)) + if (video_ava != None): + task["video_ava"] = ("%s/%s" % (public_users, video_ava)) + if (blend_file != None): + task["blend"] = ("%s/%s" % (public_users, blend_file)) + # print(repr(task).decode("unicode-escape")) + # print(json.dumps(task, ensure_ascii = True, indent = 4, sort_keys = False)) + response_task = pbclient.create_task(proj_id, task) + pyutil.log("[%s] created task[%d] '%s'" % (proj_name, int(response_task.id), nome_sinal)) + # TODO + # verificar se o sinal foi atualizado corretamente, + # caso contrario persistir lista com (id_sinal, task_id) + # para atualizar posteriormente + self.__get__("updatetask?idsinal=%d&idtask=%d" % (int(id_sinal), int(response_task.id))) except: pyutil.print_stack_trace() -- libgit2 0.21.2