Download as
Browse Code »

Commit 4395076d7fa9d2aac8e71e13f679e50d2f8cb067

Authored by Eduardo Santos
Committed by Eduardo Santos
1 parent be5b7045
Exists in master

Várias correções nas coletas das impressoras, incluindo problema que só exportava o último contador.

Showing 5 changed files with 373 additions and 17 deletions   Show diff stats
README.md
  Diff comments   View file @4395076
1   -cocar-agente
  1 +Instalação
2 2 ============
3 3  
4 4 Módulo agente coletor para o software Cocar
... ... @@ -11,3 +11,26 @@ Para funcionar é necessário primeiro instalar o pacote da distribuição e só
11 11 virtualenv --system-site-packages -p /usr/bin/python2.7 cocar-agente
12 12 </pre>
13 13  
  14 +
  15 +Operação
  16 +================
  17 +
  18 +Descrição dos principais comandos de operação
  19 +
  20 +* Varredura contínua de rede
  21 +
  22 +<pre>
  23 +/srv/cocar-agente/bin/paster scan continous_scan
  24 +</pre>
  25 +
  26 +* Leitura e export do contador das impressoras
  27 +
  28 +<pre>
  29 +/srv/cocar-agente/bin/paster scan printer_scan -t 10000000
  30 +</pre>
  31 +
  32 +* Coleta de MAC address que não foi inicialmente identificado
  33 +
  34 +<pre>
  35 +/srv/cocar-agente/bin/paster scan scan_mac_all -a eth0 -t 10
  36 +</pre>
14 37 \ No newline at end of file
... ...
cocar/commands/scan_commands.py
  Diff comments   View file @4395076
... ... @@ -16,10 +16,11 @@ from ..model.printer import Printer, PrinterCounter
16 16 from ..model.host import Host
17 17 from ..model.computer import Computer
18 18 from ..csv_utils import NetworkCSV
19   -from ..session import NmapSession, SnmpSession
  19 +from ..session import NmapSession, SnmpSession, ArpSession
20 20 from multiprocessing import Process, Queue
21 21 from ..xml_utils import NmapXML
22 22 from sqlalchemy.exc import IntegrityError
  23 +from sqlalchemy import and_
23 24  
24 25 log = logging.getLogger()
25 26  
... ... @@ -75,6 +76,12 @@ class ScanCommands(command.Command):
75 76 help='Arquivo individual de rede para ser carregado'
76 77 )
77 78  
  79 + parser.add_option('-a', '--iface',
  80 + action='store',
  81 + dest='iface',
  82 + help='Interface de rede para utilizar no Arping'
  83 + )
  84 +
78 85 def __init__(self, name):
79 86 """
80 87 Constructor method
... ... @@ -134,6 +141,15 @@ class ScanCommands(command.Command):
134 141 if cmd == 'import_printers':
135 142 self.import_printers()
136 143 return
  144 + if cmd == 'get_mac':
  145 + self.get_mac()
  146 + return
  147 + if cmd == 'scan_mac':
  148 + self.scan_mac()
  149 + return
  150 + if cmd == 'scan_mac_all':
  151 + self.scan_mac()
  152 + return
137 153 else:
138 154 log.error('Command "%s" not recognized' % (cmd,))
139 155  
... ... @@ -286,6 +302,7 @@ class ScanCommands(command.Command):
286 302 while True:
287 303 log.info("Iniciando scan de redes...")
288 304 self.scan_networks()
  305 +
289 306 log.info("Scan de redes finalizado. Iniciando procedimento de "
290 307 "identificação de ativos de rede, computadores e impressoras")
291 308 self.load_network_files()
... ... @@ -305,17 +322,31 @@ class ScanCommands(command.Command):
305 322 log.info("EXPORT DE IMPRESSORAS FINALIZADO!!! Reiniciando as coletas")
306 323 #time.sleep(600)
307 324  
  325 + def scan_mac_all(self):
  326 + """
  327 + Fica varrendo a rede tentando arrumar os MAC's
  328 + """
  329 + print("*** Aperente CTRL+C para encerrar a execução ***")
  330 +
  331 + while True:
  332 + self.scan_mac()
  333 + log.info("SCAN DE MAC FINALIZADO!!!")
  334 +
308 335 def export_printers(self):
309 336 """
310 337 Exporta todos os contadores para o Cocar
311 338 """
312 339 session = self.cocar.Session
313   - results = session.query(Printer).all()
  340 + results = session.query(Printer).join(
  341 + PrinterCounter.__table__,
  342 + PrinterCounter.network_ip == Printer.network_ip
  343 + ).all()
314 344 for printer in results:
315 345 log.info("Exportando impressora %s", printer.network_ip)
316 346 printer.export_printer(server_url=self.cocar.config.get('cocar', 'server_url'), session=session)
317 347  
318 348 session.close()
  349 + log.info("EXPORT DAS IMPRESSORAS FINALIZADO!!! %s IMPRESSORAS EXPORTADAS!!!", len(results))
319 350  
320 351 def get_printer_attribute(self):
321 352 """
... ... @@ -457,6 +488,30 @@ class ScanCommands(command.Command):
457 488 session = self.cocar.Session
458 489 for hostname in nmap_xml.hosts.keys():
459 490 host = nmap_xml.identify_host(hostname, timeout=self.options.timeout)
  491 + # Antes de tudo verifica se ele já está na tabela de contadores
  492 + counter = session.query(
  493 + PrinterCounter.__table__
  494 + ).outerjoin(
  495 + Printer.__table__,
  496 + PrinterCounter.network_ip == Printer.network_ip
  497 + ).filter(
  498 + and_(
  499 + PrinterCounter.network_ip == hostname,
  500 + Printer.network_ip.is_(None)
  501 + )
  502 + ).first()
  503 +
  504 + if counter is not None:
  505 + # Agora insere a impressora
  506 + log.info("Inserindo impressora com o IP %s", hostname)
  507 + session.execute(
  508 + Printer.__table__.insert().values(
  509 + network_ip=host.network_ip
  510 + )
  511 + )
  512 + session.flush()
  513 + continue
  514 +
460 515 if isinstance(host, Printer):
461 516 # Vê se a impressora já está na base
462 517 results = session.query(Printer).filter(Printer.network_ip == hostname).first()
... ... @@ -476,8 +531,21 @@ class ScanCommands(command.Command):
476 531 network_ip=hostname
477 532 )
478 533 )
479   - session.flush()
480 534 log.info("Impressora %s adicionada novamente com sucesso", hostname)
  535 +
  536 + # Agora atualiza informações do host
  537 + if host.mac_address is not None:
  538 + session.execute(
  539 + Host.__table__.update().values(
  540 + mac_address=host.mac_address,
  541 + name=host.name,
  542 + ports=host.ports
  543 + ).where(
  544 + Host.network_ip == hostname
  545 + )
  546 + )
  547 + session.flush()
  548 + log.info("Informações do host %s atualizadas com sucesso", hostname)
481 549 else:
482 550 log.error("ERRO!!! Host não encontrado com o IP!!! %s", hostname)
483 551 else:
... ... @@ -492,8 +560,34 @@ class ScanCommands(command.Command):
492 560 session.flush()
493 561 except IntegrityError, e:
494 562 log.error("Erro adicionando computador com o IP %s. IP Repetido\n%s", hostname, e.message)
  563 + # Agora atualiza informações do host
  564 + if host.mac_address is not None:
  565 + session.execute(
  566 + Host.__table__.update().values(
  567 + mac_address=host.mac_address,
  568 + name=host.name,
  569 + ports=host.ports
  570 + ).where(
  571 + Host.network_ip == hostname
  572 + )
  573 + )
  574 + session.flush()
  575 + log.info("Informações do host %s atualizadas com sucesso", hostname)
495 576 else:
496 577 log.info("Computador com o IP %s já cadastrado", hostname)
  578 + # Agora atualiza informações do host
  579 + if host.mac_address is not None:
  580 + session.execute(
  581 + Host.__table__.update().values(
  582 + mac_address=host.mac_address,
  583 + name=host.name,
  584 + ports=host.ports
  585 + ).where(
  586 + Host.network_ip == hostname
  587 + )
  588 + )
  589 + session.flush()
  590 + log.info("Informações do host %s atualizadas com sucesso", hostname)
497 591 else:
498 592 # Insere host genérico
499 593 results = session.query(Host).filter(Host.network_ip == hostname).first()
... ... @@ -504,12 +598,42 @@ class ScanCommands(command.Command):
504 598 session.flush()
505 599 except IntegrityError, e:
506 600 log.error("Erro adicionando host genérico com o IP %s. IP Repetido\n%s", hostname, e.message)
  601 +
  602 + # Agora atualiza informações do host
  603 + if host.mac_address is not None:
  604 + session.execute(
  605 + Host.__table__.update().values(
  606 + mac_address=host.mac_address,
  607 + name=host.name,
  608 + ports=host.ports
  609 + ).where(
  610 + Host.network_ip == hostname
  611 + )
  612 + )
  613 + session.flush()
  614 + log.info("Informações do host %s atualizadas com sucesso", hostname)
507 615 else:
508 616 log.info("Host genérico com o IP %s já cadastrado", hostname)
509 617  
  618 + # Agora atualiza informações do host
  619 + if host.mac_address is not None:
  620 + session.execute(
  621 + Host.__table__.update().values(
  622 + mac_address=host.mac_address,
  623 + name=host.name,
  624 + ports=host.ports
  625 + ).where(
  626 + Host.network_ip == hostname
  627 + )
  628 + )
  629 + session.flush()
  630 + log.info("Informações do host %s atualizadas com sucesso", hostname)
  631 +
510 632 #session.flush()
511 633 session.close()
512 634  
  635 + log.info("CARGA DO ARQUIVO DE REDE %s FINALIZADA!!!", network_file)
  636 +
513 637 def import_printers(self):
514 638 """
515 639 Importa impressoras já cadastradas e não presentes na base local
... ... @@ -535,6 +659,86 @@ class ScanCommands(command.Command):
535 659  
536 660 session.close()
537 661  
  662 + def get_mac(self):
  663 + """
  664 + Atualiza MAC Address para o host selecionado
  665 + :return:
  666 + """
  667 + if type(self.options.hosts) != list:
  668 + self.options.hosts = [self.options.hosts]
  669 +
  670 + session = self.cocar.Session
  671 + for host in self.options.hosts:
  672 + arp = ArpSession(
  673 + host=host,
  674 + iface=self.options.iface,
  675 + timeout=self.options.timeout
  676 + )
  677 +
  678 + result = arp.scan()
  679 +
  680 + if result is not None:
  681 + log.debug("Atualizando MAC = %s para host = %s", result, host)
  682 + session.execute(
  683 + Host.__table__.update().values(
  684 + mac_address=result
  685 + ).where(
  686 + Host.network_ip == host
  687 + )
  688 + )
  689 + session.flush()
  690 +
  691 + session.close()
  692 +
  693 + def scan_mac(self):
  694 + """
  695 + Scan all hosts to update macs
  696 + """
  697 + processes = int(self.cocar.config.get('cocar', 'processes'))
  698 + # Create queues
  699 + task_queue = Queue()
  700 + done_queue = Queue()
  701 +
  702 + session = self.cocar.Session
  703 + results = session.query(Host).all()
  704 + for host in results:
  705 + arp = ArpSession(
  706 + host=host.network_ip,
  707 + iface=self.options.iface,
  708 + timeout=self.options.timeout
  709 + )
  710 + task_queue.put(arp)
  711 +
  712 + #Start worker processes
  713 + for i in range(processes):
  714 + Process(target=worker_mac, args=(task_queue, done_queue)).start()
  715 +
  716 + # Get and print results
  717 + print 'Unordered results:'
  718 + for i in range(len(results)):
  719 + host_list = done_queue.get()
  720 + log.debug(host_list)
  721 + if host_list[1] is None:
  722 + log.error("Nao foi possivel encontrar o mac do host %s", host_list[0])
  723 + continue
  724 + try:
  725 + log.debug("Atualizando MAC = %s para host = %s", host_list[1], host_list[0])
  726 + session.execute(
  727 + Host.__table__.update().values(
  728 + mac_address=host_list[1]
  729 + ).where(
  730 + Host.network_ip == host_list[0]
  731 + )
  732 + )
  733 + session.flush()
  734 + except AttributeError, e:
  735 + log.error("Erro na atualização do MAC para host %s\n%s", host_list[0], e.message)
  736 + continue
  737 +
  738 + # Tell child processes to stop
  739 + for i in range(processes):
  740 + task_queue.put('STOP')
  741 +
538 742  
539 743 def make_query(host):
540 744 """This does the actual snmp query
... ... @@ -558,6 +762,17 @@ def make_query_printer(host):
558 762 return host.printer_dict()
559 763  
560 764  
  765 +def make_query_mac(host):
  766 + """This does the actual snmp query
  767 +
  768 + This is a bit fancy as it accepts both instances
  769 + of SnmpSession and host/ip addresses. This
  770 + allows a user to customize mass queries with
  771 + subsets of different hostnames and community strings
  772 + """
  773 + return host.scan_list()
  774 +
  775 +
561 776 # Function run by worker processes
562 777 def worker(inp, output):
563 778 for func in iter(inp.get, 'STOP'):
... ... @@ -569,4 +784,11 @@ def worker(inp, output):
569 784 def worker_printer(inp, output):
570 785 for func in iter(inp.get, 'STOP'):
571 786 result = make_query_printer(func)
  787 + output.put(result)
  788 +
  789 +
  790 +# Function run by worker processes
  791 +def worker_mac(inp, output):
  792 + for func in iter(inp.get, 'STOP'):
  793 + result = make_query_mac(func)
572 794 output.put(result)
573 795 \ No newline at end of file
... ...
cocar/model/printer.py
  Diff comments   View file @4395076
... ... @@ -46,13 +46,30 @@ class Printer(Host):
46 46 """
47 47 Exporta todos os contadores para a impressora
48 48 """
49   - counter_list = session.query(
50   - PrinterCounter
51   - ).filter(
52   - PrinterCounter.network_ip == self.network_ip
53   - ).all()
54   -
55   - for counter in counter_list:
  49 + #query = session.query(
  50 + # PrinterCounter
  51 + #).filter(
  52 + # PrinterCounter.__table__.c.network_ip == self.network_ip
  53 + #)
  54 +
  55 + stm = """SELECT printer_counter.network_ip as ip_address,
  56 + host.mac_address,
  57 + host.inclusion_date,
  58 + host.scantime,
  59 + printer.model,
  60 + printer.serial,
  61 + printer.description,
  62 + printer_counter.counter,
  63 + printer_counter.counter_time
  64 +FROM host
  65 +JOIN printer ON host.network_ip = printer.network_ip
  66 +JOIN printer_counter ON printer.network_ip = printer_counter.network_ip
  67 +WHERE printer_counter.network_ip = '%s'""" % self.network_ip
  68 +
  69 + counter_list = session.execute(stm, mapper=PrinterCounter).fetchall()
  70 +
  71 + for elm in counter_list:
  72 + counter = PrinterCounter(**elm)
56 73 print(counter)
57 74 result = counter.export_counter(server_url, session)
58 75 if result:
... ... @@ -61,6 +78,7 @@ class Printer(Host):
61 78 log.error("Erro na remocao do contador %s para a impressora %s", counter.counter, self.network_ip)
62 79 return False
63 80  
  81 + log.info("EXPORT DA IMPRESSORA %s FINALIZADO!!! %s CONTADORES EXPORTADOS!!!", self.network_ip, len(counter_list))
64 82 return True
65 83  
66 84  
... ... @@ -157,7 +175,7 @@ class PrinterCounter(Printer):
157 175 return False
158 176  
159 177 if response.status_code == 200:
160   - log.info("Contador para a impressora %s com contador %s"
  178 + log.info("Contador para a impressora %s com contador %s "
161 179 "exportado com sucesso", self.network_ip, self.counter)
162 180 # Remove o contador
163 181 session.execute(
... ...
cocar/session.py
  Diff comments   View file @4395076
... ... @@ -7,6 +7,7 @@ import netsnmp
7 7 import subprocess
8 8 import logging
9 9 from . import Cocar
  10 +import re
10 11  
11 12 log = logging.getLogger()
12 13  
... ... @@ -247,4 +248,80 @@ class NmapSession(object):
247 248 log.error("Install nmap: sudo apt-get install nmap")
248 249 return False
249 250  
250   - return True
251 251 \ No newline at end of file
  252 + return True
  253 +
  254 +
  255 +class ArpSession(object):
  256 + """
  257 + Classe para buscar informações de MAC do ativo
  258 + """
  259 + def __init__(self,
  260 + host,
  261 + iface='eth0',
  262 + timeout='10'):
  263 + """
  264 + :param host: Endereço IP do host a ser escaneado
  265 + :param mac: MAC address do host
  266 + :param timeout: Timeout esperando pelo reply da interface
  267 + """
  268 + self.host = host
  269 + self.iface = iface
  270 + self.timeout = timeout
  271 +
  272 + def scan(self):
  273 + """
  274 +
  275 + :return: Somente MAc
  276 + """
  277 + log.debug("Iniciando scan para o host %s", self.host)
  278 + try:
  279 + scanv = subprocess.Popen(["sudo",
  280 + "arping",
  281 + "-I",
  282 + self.iface,
  283 + "-c",
  284 + '1',
  285 + "-w",
  286 + self.timeout,
  287 + self.host],
  288 + stdout=subprocess.PIPE,
  289 + stderr=subprocess.PIPE).communicate()[0]
  290 +
  291 + match = re.search("(\[)(.*)(\])", scanv)
  292 +
  293 + if match:
  294 + return match.group(2)
  295 +
  296 + return match
  297 + except OSError:
  298 + log.error("Install arping: sudo apt-get install arping")
  299 + return None
  300 +
  301 + def scan_list(self):
  302 + """
  303 +
  304 + :return: List com host e MAC
  305 + """
  306 + log.debug("Iniciando scan para o host %s", self.host)
  307 + try:
  308 + scanv = subprocess.Popen(["sudo",
  309 + "arping",
  310 + "-I",
  311 + self.iface,
  312 + "-c",
  313 + '1',
  314 + "-w",
  315 + self.timeout,
  316 + self.host],
  317 + stdout=subprocess.PIPE,
  318 + stderr=subprocess.PIPE).communicate()[0]
  319 +
  320 + match = re.search("(\[)(.*)(\])", scanv)
  321 +
  322 + if match:
  323 + return [self.host, match.group(2)]
  324 +
  325 + return [self.host, match]
  326 + except OSError:
  327 + log.error("Install arping: sudo apt-get install arping")
  328 + return None
252 329 \ No newline at end of file
... ...
cocar/xml_utils.py
  Diff comments   View file @4395076
... ... @@ -32,10 +32,7 @@ class NmapXML(object):
32 32 if addr.get('addrtype') == 'ipv4':
33 33 host = addr.get('addr')
34 34 elif addr.get('addrtype') == 'mac':
35   - mac = {
36   - 'address': addr.get('addr'),
37   - 'vendor': addr.get('vendor')
38   - }
  35 + mac = addr.get('addr')
39 36  
40 37 # A chave do dicionário é o IP
41 38 self.hosts[host] = dict()
... ... @@ -129,6 +126,25 @@ class NmapXML(object):
129 126 open_ports=host['ports'],
130 127 )
131 128 return printer
  129 + elif host.get('os'):
  130 + # Nesse caso já sei que é computador. Precisa identificar o OS
  131 + for os in host['os'].keys():
  132 + if int(host['os'][os]['accuracy']) > accuracy:
  133 + accuracy = int(host['os'][os]['accuracy'])
  134 + os_final = os
  135 +
  136 + scantime = int(host.get('endtime')) - int(host.get('starttime'))
  137 + computer = model.computer.Computer(
  138 + ip_address=hostname,
  139 + mac_address=host.get('mac'),
  140 + hostname=host.get('hostname'),
  141 + inclusion_date=host.get('endtime'),
  142 + scantime=scantime,
  143 + open_ports=host.get('ports'),
  144 + so=host['os'][os_final]
  145 + )
  146 +
  147 + return computer
132 148 else:
133 149 # Desiste e retorna host genérico
134 150 host = model.host.Host(
... ...