diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bb2e5cb --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +TASKMGR_DIR = $(CURDIR) +TASKMGR_INIT_SCRIPT = wikilibras-taskmgr.init +TASKMGR_INIT = ${TASKMGR_DIR}/${TASKMGR_INIT_SCRIPT} +TASKMGR_INIT_ETC = /etc/init.d/${TASKMGR_INIT_SCRIPT} +TASKMGR_LOG = ${TASKMGR_DIR}/events.log +TASKMGR_REQUIREMENTS = ${TASKMGR_DIR}/requirements.txt +TASKMGR_ENV = ${TASKMGR_DIR}/env +TASKMGR_ENV_BIN = ${TASKMGR_ENV}/bin +TASKMGR_ENV_ACTIVATE = ${TASKMGR_ENV_BIN}/activate +TASKMGR_ENV_PYTHON = ${TASKMGR_ENV_BIN}/python +TASKMGR_ENV_PIP = ${TASKMGR_ENV_BIN}/pip +TASKMGR_SETTINGS = ${TASKMGR_DIR}/settings_local.json +TASKMGR_SETTINGS_T = ${TASKMGR_SETTINGS}.template +TASKMGR_DATABASE = ${TASKMGR_DIR}/database.json +TASKMGR_DATABASE_B = ${TASKMGR_PROJECT}.bak + +install: build enable-startup + +build: uninstall + @rm -f ${TASKMGR_LOG} + @( \ + cd ${TASKMGR_DIR}; \ + virtualenv ${TASKMGR_ENV}; \ + . ${TASKMGR_ENV_ACTIVATE}; \ + ${TASKMGR_ENV_PIP} install -U pip; \ + ${TASKMGR_ENV_PIP} install -r ${TASKMGR_REQUIREMENTS}; \ + ) + +clean: + @find ${TASKMGR_DIR} -regextype posix-awk -regex "(.*.log|.*.pyc)" -type f -delete + +disable-startup: + @sudo update-rc.d -f ${TASKMGR_INIT_SCRIPT} remove + @sudo rm -f ${TASKMGR_INIT_ETC} + +enable-startup: + @sed "s##${TASKMGR_DIR}#" ${TASKMGR_INIT} | sudo tee ${TASKMGR_INIT_ETC} + @sudo chmod 755 ${TASKMGR_INIT_ETC} + @sudo chown "root:root" ${TASKMGR_INIT_ETC} + @sudo update-rc.d -f ${TASKMGR_INIT_SCRIPT} defaults + +run: + @( \ + cd ${TASKMGR_DIR}; \ + . ${TASKMGR_ENV_ACTIVATE}; \ + ${TASKMGR_ENV_PYTHON} ${TASKMGR_DIR}/main.py; \ + ) + +uninstall: clean disable-startup + @rm -rf ${TASKMGR_ENV} + +reset: + @if [ -e "${TASKMGR_DATABASE}" ]; then cat ${TASKMGR_DATABASE}; echo ""; sudo rm -f ${TASKMGR_DATABASE} ${TASKMGR_LOG}; fi diff --git a/README.md b/README.md new file mode 100644 index 0000000..d6a4fb1 --- /dev/null +++ b/README.md @@ -0,0 +1,46 @@ +## Copie o arquivo modelo de configuração + +```sh +$ cp settings_local.json.template settings_local.json +``` + +## Edite o arquivo de configuração + +```sh +$ gedit settings_local.json +``` + +- Substitua o valor da chave "db_host": "localhost" pelo endereço IP da API do banco de dados de sinais. + Exemplo: "db_host": "150.250.350.45". + +- Substitua "" pelo caminho do diretório completo do projeto "corretor_sinais". + +- Substitua "" pelo caminho do diretório completo do projeto "validador_sinais". + +- Substitua "" pelo caminho do diretório completo do projeto "wikilibrasV2". + +- Substitua my-api-key por sua API key do Pybossa. + +## Instale dependências + +```sh +$ make install +``` + +## Execute teste (Opcional) + +```sh +$ make run +``` + +## Habilitar a inicialização automática do serviço + +```sh +$ make enable-startup +``` + +## Desabilitar a inicialização automática do serviço + +```sh +$ make disable-startup +``` diff --git a/inicial.txt b/inicial.txt deleted file mode 100644 index 5556abd..0000000 --- a/inicial.txt +++ /dev/null @@ -1 +0,0 @@ -Repositorio taskmgr Criado diff --git a/main.py b/main.py new file mode 100644 index 0000000..5bbf2c3 --- /dev/null +++ b/main.py @@ -0,0 +1,32 @@ +# -*- coding: UTF-8 -*- + +import os +import pyutil +import sys +import task_manager +import time + +def main(): + seconds = 10 + json_data = None + task = None + while (task == None): + try: + json_data = task_manager.load_json(os.path.join(os.path.dirname(os.path.abspath('__file__')), "settings_local.json")) + task = task_manager.task_config(json_data) + except: + task = None + print("Waiting for Network to Get Projects ID's") + time.sleep(10) + + if (json_data != None and task != None): + pyutil.log("wikilibras task manager started") + while (True): + task.run(task.data["corretor"]) + task.run(task.data["validador"]) + task.run(task.data["wikilibras"]) + print("Waiting %d seconds to check new tasks" % (seconds)) + time.sleep(seconds) + +if __name__ == '__main__': + main() diff --git a/pyutil.py b/pyutil.py new file mode 100644 index 0000000..f406e69 --- /dev/null +++ b/pyutil.py @@ -0,0 +1,74 @@ +# -*- coding: UTF-8 -*- + +import datetime +import logging +import os +import shutil +import sys + +# @def funcao para obter data e hora atual do sistema +# @param string formato de data e hora +# @return string retorna data e hora do sistema no momento da chamada +def getTimeStamp(date_fmt="%Y-%m-%d %H:%M:%S.%f"): + if ("%f" in date_fmt): + # [:-3] remove 3 casas decimais dos milisegundos (ms) + return datetime.datetime.now().strftime(date_fmt)[:-3] + else: + return datetime.datetime.now().strftime(date_fmt) + +# @def funcao para gravar log dos eventos em arquivo +# @param string mensagem a ser salva +# @param int indice do tipo de log 0: apenas print, 1: debug, 2: info, 3: warn, 4: error, 5: critical +# @param string caminho completo do arquivo de logs +# @param string formato de tempo utilizado +# @return none +def log(msg="", log_level=2, log_file = os.path.join(os.path.dirname(os.path.abspath('__file__')), "events.log")): + dict_level = { + 0: ["Print", None, None], + 1: ["DEBUG", logging.DEBUG, logging.debug], + 2: ["INFO", logging.INFO, logging.info], + 3: ["WARNING", logging.WARN, logging.warn], + 4: ["ERROR", logging.ERROR, logging.error], + 5: ["CRITICAL", logging.CRITICAL, logging.critical] + } + # log_format = "[%(asctime)s.%(msecs).03d] %(levelname)s: : %(message)s" + log_format = "[%(asctime)s.%(msecs).03d] %(levelname)s: %(message)s" + date_fmt = "%Y-%m-%d %H:%M:%S" + logging.basicConfig(filename=log_file, datefmt=date_fmt, format=log_format, level=dict_level[log_level][1]) + logging.Formatter(fmt="%(asctime)s", datefmt=date_fmt) + log_level %= len(dict_level) + write_mode = dict_level[log_level][2] + print("[%s] %s: %s" % (getTimeStamp(), dict_level[log_level][0], msg)) + if (write_mode != None): + write_mode(msg) + return + +def file_exists(file_path): + if ((os.path.isfile(file_path) == 1) and (os.path.exists(file_path) == 1)): + return True + else: + return False + +# @def funcao para exibir excecao +# @param string deve ser passado: "__file__" para identificar em qual modulo ocorreu a excecao +# @return int retorna 1 +def print_stack_trace(): + error = "\n File name: %s\n Function name: %s\n Line code: %s\n Type exception: %s\n Message: %s" % ( + os.path.basename(sys.exc_info()[2].tb_frame.f_code.co_filename), + sys.exc_info()[2].tb_frame.f_code.co_name, + sys.exc_info()[2].tb_lineno, + sys.exc_info()[0].__name__, + sys.exc_info()[1] + ) + log(error, 4) + return 1 + +def get_date_now(): + return datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + +def is_int(string): + try: + int(string) + return True + except ValueError: + return False diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..38c0b29 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pybossa-client +requests >= 1.2 \ No newline at end of file diff --git a/settings_local.json.template b/settings_local.json.template new file mode 100644 index 0000000..a69997b --- /dev/null +++ b/settings_local.json.template @@ -0,0 +1,19 @@ +{ + "db_host": "localhost", + "db_timeout": 15, + "db_port": 200, + "pb_api_key": "my-api-key", + "pb_endpoint": "http://localhost/pybossa/", + "corretor": { + "short_name": "corretor_sinais", + "video_path": "/view/videos" + }, + "validador": { + "short_name": "validador_sinais", + "video_path": "/view/videos" + }, + "wikilibras": { + "short_name": "wikilibras", + "video_path": "/view/videos" + } +} diff --git a/task_manager.py b/task_manager.py new file mode 100644 index 0000000..0d3c523 --- /dev/null +++ b/task_manager.py @@ -0,0 +1,224 @@ +# -*- coding: UTF-8 -*- + +import json +import os +import pbclient +import pyutil +import requests +import shutil +import logging + +def load_json(filename): + r = {} + try: + fp = open(filename, 'r') + if (fp): + r = json.load(fp) + fp.close() + except: + pass + return r + +def save_json(data, filename, idt = 4): + try: + fp = open(filename, 'w') + if (fp): + json.dump(data, fp, ensure_ascii = False, indent = idt, sort_keys = True) + fp.close() + except: + pyutil.print_stack_trace() + return False + return True + +class task_config(): + def __init__(self, json_data): + self.data = {} + self.data["db_host"] = json_data.get("db_host", "localhost") + self.data["db_timeout"] = json_data.get("db_timeout", 30) + self.data["db_port"] = json_data.get("db_port", 200) + self.data["pb_api_key"] = json_data.get("pb_api_key", "") + self.data["pb_endpoint"] = json_data.get("pb_endpoint", "http://localhost/pybossa/") + self.data["corretor"] = dict(json_data.get("corretor", {})) + self.data["validador"] = dict(json_data.get("validador", {})) + self.data["wikilibras"] = dict(json_data.get("wikilibras", {})) + self.set_pb_config(self.data["pb_api_key"], self.data["pb_endpoint"]) + self.data["corretor"]["project_id"] = self.get_pb_project_id(self.data["corretor"]["short_name"]) + self.data["validador"]["project_id"] = self.get_pb_project_id(self.data["validador"]["short_name"]) + self.data["wikilibras"]["project_id"] = self.get_pb_project_id(self.data["wikilibras"]["short_name"]) + logging.getLogger("requests").setLevel(logging.ERROR) + + def set_pb_config(self, pb_api_key, pb_endpoint): + pbclient.set('api_key', self.data["pb_api_key"]) + pbclient.set('endpoint', self.data["pb_endpoint"]) + + def get_pb_project_id(self, pb_short_name): + projects = pbclient.find_project(short_name = pb_short_name) + if (len(projects) > 0): + return projects[0].id + else: + return None + + def to_string(self): + return json.dumps(self.data, sort_keys = True, ensure_ascii = False, indent = 4) + + def __get__(self, endpoint): + try: + r = requests.get(('http://%s:%s/%s' % (self.data["db_host"], self.data["db_port"], endpoint)), timeout = self.data["db_timeout"]) + return r + except: + return {} + + def get_data_by_version(self, version): + r = self.__get__("sinais?version=%d" % (version)) + if (r != {}): + # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4)) + return r.json() + else: + return [] + + def get_data_by_version_selo(self, version, selo): + r = self.__get__("sinais?version=%d&selo=%s" % (version, selo)) + if (r != {}): + # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4)) + return r.json() + else: + return [] + + def get_data_by_selo(self, selo): + r = self.__get__("sinais?&selo=%s" % (selo)) + if (r != {}): + # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4)) + return r.json() + else: + return [] + + def get_listall(self): + r = self.__get__("listall") + if (r != {}): + # pyutil.log(json.dumps(r.json(), ensure_ascii = False, indent = 4)) + return r.json() + else: + return [] + + def get_version(self): + r = self.__get__("version") + if (r != {}): + version = r.json()["version"] + if (isinstance(version, int)): + return version + return 0 + + def get_file(self, url, filename): + try: + r = requests.get(url, stream = True) + if (r.status_code == 200): + with open(filename, 'wb') as f: + for chunk in r.iter_content(chunk_size = 1024): + if chunk: + f.write(chunk) + return True + except: + pass + return False + + def run(self, project): + try: + video_path = project.get("video_path", "view/videos") + proj_name = project.get("short_name", "") + proj_id = project.get("project_id", "") + database = os.path.join(os.path.dirname(os.path.abspath('__file__')), "database.json") + database_bak = database + ".bak" + control = load_json(database) + changed = False + # server_version = self.get_version() + # for i in range(1, server_version + 1): + dbquery = None + + # selos + # 1 - wikilibras + # 2 - especialista + # 3 - invalido_wikilibras + # 4 - invalido_especialista + # 5 - animadores + # 6 - invalido_animadores + # 7 - null + + if (proj_name == "corretor_sinais"): + dbquery = self.get_data_by_selo("invalido_especialista") + + if (proj_name == "validador_sinais"): + dbquery = self.get_data_by_selo("wikilibras") + self.get_data_by_selo("animadores") + + if (proj_name == "wikilibras"): + dbquery = self.get_data_by_selo("null") + + for j in dbquery: + add_task = False + current_sinal_name = str(j["nome"]).upper() + current_version = int(j["version"]) + + if (current_sinal_name in control): + if (control[current_sinal_name] < current_version): + control[current_sinal_name] = current_version + changed = True + add_task = True + else: + control[current_sinal_name] = current_version + changed = True + add_task = True + + if(add_task): + avatar_url = "http://%s:%s/blender/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"nome"] + ".blend")) + video_ref_url = "http://%s:%s/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"file"])) + video_ava_url = "http://%s:%s/avatar/%s" % (str(self.data["db_host"]), str(self.data["db_port"]), str(j[u"nome"] + ".webm")) + + avatar_out = os.path.join(video_path, str(j[u"nome"] + "_AVATAR.blend")) + video_ref_out = os.path.join(video_path, str(j[u"nome"] + "_REF.webm")) + video_ava_out = os.path.join(video_path, str(j[u"nome"] + "_AVATAR.webm")) + + # cria diretorio de destino caso nao exista + if not os.path.exists(video_path): + os.makedirs(video_path) + + # remove arquivos existentes relativos a pasta "avatar_out" + if pyutil.file_exists(avatar_out): + os.unlink(avatar_out) + + # remove arquivos existentes relativos a pasta "video_ref_out" + if pyutil.file_exists(video_ref_out): + os.unlink(video_ref_out) + + # remove arquivos existentes relativos a pasta "video_ava_out" + if pyutil.file_exists(video_ava_out): + os.unlink(video_ava_out) + + # faz download do arquivo blend + self.get_file(avatar_url, avatar_out) + + # faz download do video renderizado + self.get_file(video_ava_url, video_ava_out) + + # faz download do video de referencia + if (self.get_file(video_ref_url, video_ref_out)): + pyutil.log("%s: creating task: %s version: %s stamp: %s" % (proj_name, j[u"nome"], j[u"version"], j[u"nomeSelo"])) + task = dict(sign_name = j[u"nome"], submission_date = pyutil.get_date_now()) + # pbclient.create_task(proj_id, task) + else: + # nao foi possivel fazer o download do video de referencia (tarefa nao adicionada) + pyutil.log("%s: file not found: %s" % (proj_name, video_ref_url)) + + if (changed): + # verifica se arquivo database.json existe para realizar backup + if ((os.path.isfile(database) == 1) and (os.path.exists(database) == 1)): + + # verifica se arquivo database.json.bak existe para remover + if ((os.path.isfile(database_bak) == 1) and (os.path.exists(database_bak) == 1)): + os.remove(database_bak) + + # faz backup antes de salvar nova base de dados + shutil.copy(database, database_bak) + + # salva nova base de dados + save_json(control, database, 2) + except: + pyutil.print_stack_trace() diff --git a/wikilibras-taskmgr.init b/wikilibras-taskmgr.init new file mode 100644 index 0000000..2100111 --- /dev/null +++ b/wikilibras-taskmgr.init @@ -0,0 +1,34 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: wikilibras-taskmgr.init +# Required-Start: $all +# Required-Stop: $all +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: This service update tasks for Wikilibras +# Description: Enable service provided by Wikilibras API at boot +### END INIT INFO + +set -e + +export PATH=$PATH:/usr/local/bin + +CWD="" + +case "$1" in + start) + cd "$CWD" + . env/bin/activate + python main.py & + ;; + stop) + cd "$CWD" + make stop + ;; + *) + echo "Usage: /etc/init.d/wikilibras-taskmgr.init {start|stop}" + exit 1 + ;; +esac + +exit 0 -- libgit2 0.21.2