From 4395076d7fa9d2aac8e71e13f679e50d2f8cb067 Mon Sep 17 00:00:00 2001 From: Eduardo Santos Date: Tue, 21 Oct 2014 14:03:51 -0200 Subject: [PATCH] Várias correções nas coletas das impressoras, incluindo problema que só exportava o último contador. --- README.md | 25 ++++++++++++++++++++++++- cocar/commands/scan_commands.py | 228 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- cocar/model/printer.py | 34 ++++++++++++++++++++++++++-------- cocar/session.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- cocar/xml_utils.py | 24 ++++++++++++++++++++---- 5 files changed, 373 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index d38f01d..a0978fd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -cocar-agente +Instalação ============ Módulo agente coletor para o software Cocar @@ -11,3 +11,26 @@ Para funcionar é necessário primeiro instalar o pacote da distribuição e só virtualenv --system-site-packages -p /usr/bin/python2.7 cocar-agente + +Operação +================ + +Descrição dos principais comandos de operação + +* Varredura contínua de rede + +
+/srv/cocar-agente/bin/paster scan continous_scan
+
+ +* Leitura e export do contador das impressoras + +
+/srv/cocar-agente/bin/paster scan printer_scan -t 10000000
+
+ +* Coleta de MAC address que não foi inicialmente identificado + +
+/srv/cocar-agente/bin/paster scan scan_mac_all -a eth0 -t 10
+
\ No newline at end of file diff --git a/cocar/commands/scan_commands.py b/cocar/commands/scan_commands.py index 0350f61..60570fa 100644 --- a/cocar/commands/scan_commands.py +++ b/cocar/commands/scan_commands.py @@ -16,10 +16,11 @@ from ..model.printer import Printer, PrinterCounter from ..model.host import Host from ..model.computer import Computer from ..csv_utils import NetworkCSV -from ..session import NmapSession, SnmpSession +from ..session import NmapSession, SnmpSession, ArpSession from multiprocessing import Process, Queue from ..xml_utils import NmapXML from sqlalchemy.exc import IntegrityError +from sqlalchemy import and_ log = logging.getLogger() @@ -75,6 +76,12 @@ class ScanCommands(command.Command): help='Arquivo individual de rede para ser carregado' ) + parser.add_option('-a', '--iface', + action='store', + dest='iface', + help='Interface de rede para utilizar no Arping' + ) + def __init__(self, name): """ Constructor method @@ -134,6 +141,15 @@ class ScanCommands(command.Command): if cmd == 'import_printers': self.import_printers() return + if cmd == 'get_mac': + self.get_mac() + return + if cmd == 'scan_mac': + self.scan_mac() + return + if cmd == 'scan_mac_all': + self.scan_mac() + return else: log.error('Command "%s" not recognized' % (cmd,)) @@ -286,6 +302,7 @@ class ScanCommands(command.Command): while True: log.info("Iniciando scan de redes...") self.scan_networks() + log.info("Scan de redes finalizado. Iniciando procedimento de " "identificação de ativos de rede, computadores e impressoras") self.load_network_files() @@ -305,17 +322,31 @@ class ScanCommands(command.Command): log.info("EXPORT DE IMPRESSORAS FINALIZADO!!! Reiniciando as coletas") #time.sleep(600) + def scan_mac_all(self): + """ + Fica varrendo a rede tentando arrumar os MAC's + """ + print("*** Aperente CTRL+C para encerrar a execução ***") + + while True: + self.scan_mac() + log.info("SCAN DE MAC FINALIZADO!!!") + def export_printers(self): """ Exporta todos os contadores para o Cocar """ session = self.cocar.Session - results = session.query(Printer).all() + results = session.query(Printer).join( + PrinterCounter.__table__, + PrinterCounter.network_ip == Printer.network_ip + ).all() for printer in results: log.info("Exportando impressora %s", printer.network_ip) printer.export_printer(server_url=self.cocar.config.get('cocar', 'server_url'), session=session) session.close() + log.info("EXPORT DAS IMPRESSORAS FINALIZADO!!! %s IMPRESSORAS EXPORTADAS!!!", len(results)) def get_printer_attribute(self): """ @@ -457,6 +488,30 @@ class ScanCommands(command.Command): session = self.cocar.Session for hostname in nmap_xml.hosts.keys(): host = nmap_xml.identify_host(hostname, timeout=self.options.timeout) + # Antes de tudo verifica se ele já está na tabela de contadores + counter = session.query( + PrinterCounter.__table__ + ).outerjoin( + Printer.__table__, + PrinterCounter.network_ip == Printer.network_ip + ).filter( + and_( + PrinterCounter.network_ip == hostname, + Printer.network_ip.is_(None) + ) + ).first() + + if counter is not None: + # Agora insere a impressora + log.info("Inserindo impressora com o IP %s", hostname) + session.execute( + Printer.__table__.insert().values( + network_ip=host.network_ip + ) + ) + session.flush() + continue + if isinstance(host, Printer): # Vê se a impressora já está na base results = session.query(Printer).filter(Printer.network_ip == hostname).first() @@ -476,8 +531,21 @@ class ScanCommands(command.Command): network_ip=hostname ) ) - session.flush() log.info("Impressora %s adicionada novamente com sucesso", hostname) + + # Agora atualiza informações do host + if host.mac_address is not None: + session.execute( + Host.__table__.update().values( + mac_address=host.mac_address, + name=host.name, + ports=host.ports + ).where( + Host.network_ip == hostname + ) + ) + session.flush() + log.info("Informações do host %s atualizadas com sucesso", hostname) else: log.error("ERRO!!! Host não encontrado com o IP!!! %s", hostname) else: @@ -492,8 +560,34 @@ class ScanCommands(command.Command): session.flush() except IntegrityError, e: log.error("Erro adicionando computador com o IP %s. IP Repetido\n%s", hostname, e.message) + # Agora atualiza informações do host + if host.mac_address is not None: + session.execute( + Host.__table__.update().values( + mac_address=host.mac_address, + name=host.name, + ports=host.ports + ).where( + Host.network_ip == hostname + ) + ) + session.flush() + log.info("Informações do host %s atualizadas com sucesso", hostname) else: log.info("Computador com o IP %s já cadastrado", hostname) + # Agora atualiza informações do host + if host.mac_address is not None: + session.execute( + Host.__table__.update().values( + mac_address=host.mac_address, + name=host.name, + ports=host.ports + ).where( + Host.network_ip == hostname + ) + ) + session.flush() + log.info("Informações do host %s atualizadas com sucesso", hostname) else: # Insere host genérico results = session.query(Host).filter(Host.network_ip == hostname).first() @@ -504,12 +598,42 @@ class ScanCommands(command.Command): session.flush() except IntegrityError, e: log.error("Erro adicionando host genérico com o IP %s. IP Repetido\n%s", hostname, e.message) + + # Agora atualiza informações do host + if host.mac_address is not None: + session.execute( + Host.__table__.update().values( + mac_address=host.mac_address, + name=host.name, + ports=host.ports + ).where( + Host.network_ip == hostname + ) + ) + session.flush() + log.info("Informações do host %s atualizadas com sucesso", hostname) else: log.info("Host genérico com o IP %s já cadastrado", hostname) + # Agora atualiza informações do host + if host.mac_address is not None: + session.execute( + Host.__table__.update().values( + mac_address=host.mac_address, + name=host.name, + ports=host.ports + ).where( + Host.network_ip == hostname + ) + ) + session.flush() + log.info("Informações do host %s atualizadas com sucesso", hostname) + #session.flush() session.close() + log.info("CARGA DO ARQUIVO DE REDE %s FINALIZADA!!!", network_file) + def import_printers(self): """ Importa impressoras já cadastradas e não presentes na base local @@ -535,6 +659,86 @@ class ScanCommands(command.Command): session.close() + def get_mac(self): + """ + Atualiza MAC Address para o host selecionado + :return: + """ + if type(self.options.hosts) != list: + self.options.hosts = [self.options.hosts] + + session = self.cocar.Session + for host in self.options.hosts: + arp = ArpSession( + host=host, + iface=self.options.iface, + timeout=self.options.timeout + ) + + result = arp.scan() + + if result is not None: + log.debug("Atualizando MAC = %s para host = %s", result, host) + session.execute( + Host.__table__.update().values( + mac_address=result + ).where( + Host.network_ip == host + ) + ) + session.flush() + + session.close() + + def scan_mac(self): + """ + Scan all hosts to update macs + """ + processes = int(self.cocar.config.get('cocar', 'processes')) + # Create queues + task_queue = Queue() + done_queue = Queue() + + session = self.cocar.Session + results = session.query(Host).all() + for host in results: + arp = ArpSession( + host=host.network_ip, + iface=self.options.iface, + timeout=self.options.timeout + ) + task_queue.put(arp) + + #Start worker processes + for i in range(processes): + Process(target=worker_mac, args=(task_queue, done_queue)).start() + + # Get and print results + print 'Unordered results:' + for i in range(len(results)): + host_list = done_queue.get() + log.debug(host_list) + if host_list[1] is None: + log.error("Nao foi possivel encontrar o mac do host %s", host_list[0]) + continue + try: + log.debug("Atualizando MAC = %s para host = %s", host_list[1], host_list[0]) + session.execute( + Host.__table__.update().values( + mac_address=host_list[1] + ).where( + Host.network_ip == host_list[0] + ) + ) + session.flush() + except AttributeError, e: + log.error("Erro na atualização do MAC para host %s\n%s", host_list[0], e.message) + continue + + # Tell child processes to stop + for i in range(processes): + task_queue.put('STOP') + def make_query(host): """This does the actual snmp query @@ -558,6 +762,17 @@ def make_query_printer(host): return host.printer_dict() +def make_query_mac(host): + """This does the actual snmp query + + This is a bit fancy as it accepts both instances + of SnmpSession and host/ip addresses. This + allows a user to customize mass queries with + subsets of different hostnames and community strings + """ + return host.scan_list() + + # Function run by worker processes def worker(inp, output): for func in iter(inp.get, 'STOP'): @@ -569,4 +784,11 @@ def worker(inp, output): def worker_printer(inp, output): for func in iter(inp.get, 'STOP'): result = make_query_printer(func) + output.put(result) + + +# Function run by worker processes +def worker_mac(inp, output): + for func in iter(inp.get, 'STOP'): + result = make_query_mac(func) output.put(result) \ No newline at end of file diff --git a/cocar/model/printer.py b/cocar/model/printer.py index b6a4805..4e6284a 100644 --- a/cocar/model/printer.py +++ b/cocar/model/printer.py @@ -46,13 +46,30 @@ class Printer(Host): """ Exporta todos os contadores para a impressora """ - counter_list = session.query( - PrinterCounter - ).filter( - PrinterCounter.network_ip == self.network_ip - ).all() - - for counter in counter_list: + #query = session.query( + # PrinterCounter + #).filter( + # PrinterCounter.__table__.c.network_ip == self.network_ip + #) + + stm = """SELECT printer_counter.network_ip as ip_address, + host.mac_address, + host.inclusion_date, + host.scantime, + printer.model, + printer.serial, + printer.description, + printer_counter.counter, + printer_counter.counter_time +FROM host +JOIN printer ON host.network_ip = printer.network_ip +JOIN printer_counter ON printer.network_ip = printer_counter.network_ip +WHERE printer_counter.network_ip = '%s'""" % self.network_ip + + counter_list = session.execute(stm, mapper=PrinterCounter).fetchall() + + for elm in counter_list: + counter = PrinterCounter(**elm) print(counter) result = counter.export_counter(server_url, session) if result: @@ -61,6 +78,7 @@ class Printer(Host): log.error("Erro na remocao do contador %s para a impressora %s", counter.counter, self.network_ip) return False + log.info("EXPORT DA IMPRESSORA %s FINALIZADO!!! %s CONTADORES EXPORTADOS!!!", self.network_ip, len(counter_list)) return True @@ -157,7 +175,7 @@ class PrinterCounter(Printer): return False if response.status_code == 200: - log.info("Contador para a impressora %s com contador %s" + log.info("Contador para a impressora %s com contador %s " "exportado com sucesso", self.network_ip, self.counter) # Remove o contador session.execute( diff --git a/cocar/session.py b/cocar/session.py index 242437e..52b9658 100644 --- a/cocar/session.py +++ b/cocar/session.py @@ -7,6 +7,7 @@ import netsnmp import subprocess import logging from . import Cocar +import re log = logging.getLogger() @@ -247,4 +248,80 @@ class NmapSession(object): log.error("Install nmap: sudo apt-get install nmap") return False - return True \ No newline at end of file + return True + + +class ArpSession(object): + """ + Classe para buscar informações de MAC do ativo + """ + def __init__(self, + host, + iface='eth0', + timeout='10'): + """ + :param host: Endereço IP do host a ser escaneado + :param mac: MAC address do host + :param timeout: Timeout esperando pelo reply da interface + """ + self.host = host + self.iface = iface + self.timeout = timeout + + def scan(self): + """ + + :return: Somente MAc + """ + log.debug("Iniciando scan para o host %s", self.host) + try: + scanv = subprocess.Popen(["sudo", + "arping", + "-I", + self.iface, + "-c", + '1', + "-w", + self.timeout, + self.host], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate()[0] + + match = re.search("(\[)(.*)(\])", scanv) + + if match: + return match.group(2) + + return match + except OSError: + log.error("Install arping: sudo apt-get install arping") + return None + + def scan_list(self): + """ + + :return: List com host e MAC + """ + log.debug("Iniciando scan para o host %s", self.host) + try: + scanv = subprocess.Popen(["sudo", + "arping", + "-I", + self.iface, + "-c", + '1', + "-w", + self.timeout, + self.host], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate()[0] + + match = re.search("(\[)(.*)(\])", scanv) + + if match: + return [self.host, match.group(2)] + + return [self.host, match] + except OSError: + log.error("Install arping: sudo apt-get install arping") + return None \ No newline at end of file diff --git a/cocar/xml_utils.py b/cocar/xml_utils.py index 2bea17c..b7a8528 100644 --- a/cocar/xml_utils.py +++ b/cocar/xml_utils.py @@ -32,10 +32,7 @@ class NmapXML(object): if addr.get('addrtype') == 'ipv4': host = addr.get('addr') elif addr.get('addrtype') == 'mac': - mac = { - 'address': addr.get('addr'), - 'vendor': addr.get('vendor') - } + mac = addr.get('addr') # A chave do dicionário é o IP self.hosts[host] = dict() @@ -129,6 +126,25 @@ class NmapXML(object): open_ports=host['ports'], ) return printer + elif host.get('os'): + # Nesse caso já sei que é computador. Precisa identificar o OS + for os in host['os'].keys(): + if int(host['os'][os]['accuracy']) > accuracy: + accuracy = int(host['os'][os]['accuracy']) + os_final = os + + scantime = int(host.get('endtime')) - int(host.get('starttime')) + computer = model.computer.Computer( + ip_address=hostname, + mac_address=host.get('mac'), + hostname=host.get('hostname'), + inclusion_date=host.get('endtime'), + scantime=scantime, + open_ports=host.get('ports'), + so=host['os'][os_final] + ) + + return computer else: # Desiste e retorna host genérico host = model.host.Host( -- libgit2 0.21.2