From ec1332476d5c5fb2ea22e10f198474005c80a043 Mon Sep 17 00:00:00 2001 From: Antonio Terceiro Date: Mon, 10 Aug 2015 16:20:25 -0300 Subject: [PATCH] Monitoring support with munin --- Rakefile | 6 ++++++ config/prod/config.yaml | 1 + cookbooks/firewall/templates/default/iptables.erb | 8 ++++++++ cookbooks/munin/files/nginx.conf | 12 ++++++++++++ cookbooks/munin/files/packetloss | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ cookbooks/munin/recipes/default.rb | 12 ++++++++++++ cookbooks/munin/recipes/node.rb | 32 ++++++++++++++++++++++++++++++++ cookbooks/munin/templates/hosts.conf.erb | 4 ++++ roles/monitoring_server.rb | 3 +++ roles/server.rb | 2 +- 10 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 cookbooks/munin/files/nginx.conf create mode 100644 cookbooks/munin/files/packetloss create mode 100644 cookbooks/munin/recipes/default.rb create mode 100644 cookbooks/munin/recipes/node.rb create mode 100644 cookbooks/munin/templates/hosts.conf.erb create mode 100644 roles/monitoring_server.rb diff --git a/Rakefile b/Rakefile index d857e80..61aa00d 100644 --- a/Rakefile +++ b/Rakefile @@ -58,6 +58,12 @@ $nodes.each do |node| node.data['firewall'] = firewall end +# In the absence of a dedicated munin master, reverseproxy will do that. +if !config['munin_master'] + config['munin_master'] = ips['reverseproxy'] + $nodes.find { |node| node.hostname == 'reverseproxy' }.data['run_list'] << 'role[monitoring_server]' +end + task :console do require 'pry' binding.pry diff --git a/config/prod/config.yaml b/config/prod/config.yaml index 248493d..574909f 100644 --- a/config/prod/config.yaml +++ b/config/prod/config.yaml @@ -17,3 +17,4 @@ external_outgoing_mail_relay: 189.9.150.53 external_outgoing_mail_domain: serpro.gov.br raven_dsn: https://4418146896924efe9b73d557f803f047:8a59f39b5f584ff589ecf3dd47faaead@sentry.tracy.com.br/13 google_analytics_id: 'UA-64206731-1' +munin_master: 10.21.0.10 diff --git a/cookbooks/firewall/templates/default/iptables.erb b/cookbooks/firewall/templates/default/iptables.erb index 8cffd6e..ca2a20a 100644 --- a/cookbooks/firewall/templates/default/iptables.erb +++ b/cookbooks/firewall/templates/default/iptables.erb @@ -16,11 +16,19 @@ -A INPUT -p icmp --icmp-type 11 -j ACCEPT -A INPUT -p icmp --icmp-type 12 -j ACCEPT +# allow ping between the peers +<% node['peers'].each do |hostname,ip| %> +-A INPUT -s <%= ip %> -p icmp --icmp-type 8 -j ACCEPT +<% end %> + -A INPUT -i lo -j ACCEPT # Everybody need to accept SSH from integration -A INPUT -s <%= node['peers']['integration'] %> -p tcp -m state --state NEW --dport 22 -j ACCEPT +# Everybody needs to accept munin connections from munin master +-A INPUT -s <%= node['config']['munin_master'] %> -p tcp -m state --state NEW --dport 4949 -j ACCEPT + <%= node['firewall'] %> <%= render 'iptables-filter.erb' %> diff --git a/cookbooks/munin/files/nginx.conf b/cookbooks/munin/files/nginx.conf new file mode 100644 index 0000000..afb15ed --- /dev/null +++ b/cookbooks/munin/files/nginx.conf @@ -0,0 +1,12 @@ +location /munin/static/ { + alias /var/www/html/munin/static/; + expires modified +1w; +} +location /munin/ { + # auth_basic "Restricted"; + # # Create the htpasswd file with the htpasswd tool. + # auth_basic_user_file /etc/nginx/munin_htpasswd; + + alias /var/www/html/munin/; + expires modified +310s; +} diff --git a/cookbooks/munin/files/packetloss b/cookbooks/munin/files/packetloss new file mode 100644 index 0000000..f2c19d4 --- /dev/null +++ b/cookbooks/munin/files/packetloss @@ -0,0 +1,60 @@ +#!/bin/sh +# +# Copyright (c) 2009 Sven-Hendrik Haase +# Copyright (C) 2004 Jimmy Olsen +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 dated June, +# 1991. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# +# Plugin to monitor packet loss +# +# Please note that sometimes it can take quite long for the plugin to return +# a value on a network with lots of packet loss. +# You therefore need to account for it by appending the following to your +# plugin-conf.d/munin-node. Remember to restart munin-node afterwards. +# Append the next 3 lines to plugin-conf.d/munin-node: +# [packetloss_*] +# timeout 60 +# user root +# +# Parameters: +# +# ping_args - Arguments to ping (default "-c 2") +# ping_args2 - Arguments after the host name (required for Solaris) +# ping - Ping program to use +# host - Host to ping +# +# Arguments for Solaris: +# ping_args -s +# ping_args2 56 2 +# +#%# family=manual + +file_host=`basename $0 | sed 's/^packetloss_//g'` +host=${host:-${file_host:-www.google.com}} + +if [ "$1" = "config" ]; then + echo "graph_title Packet loss to $host (in %)" + echo 'graph_args --upper-limit 100 -l 0' + echo 'graph_vlabel %' + echo 'graph_category network' + echo 'graph_info This graph shows packet loss statistics.' + echo "packetloss.label $host" + echo "packetloss.info Packet loss statistics for $host." + echo 'packetloss.draw LINE2' + exit 0 +fi + +${ping:-ping} ${ping_args:-'-c 10'} ${host} ${ping_args2} | perl -n -e 'print "packetloss.value $1\n" if /(\d+)% packet loss/;' diff --git a/cookbooks/munin/recipes/default.rb b/cookbooks/munin/recipes/default.rb new file mode 100644 index 0000000..d792283 --- /dev/null +++ b/cookbooks/munin/recipes/default.rb @@ -0,0 +1,12 @@ +package 'munin' + +template '/etc/munin/conf.d/hosts.conf' + +package 'nginx' +service 'nginx' do + supports :reload => true +end +cookbook_file '/etc/nginx/default.d/munin.conf' do + source 'nginx.conf' + notifies :reload, 'service[nginx]' +end diff --git a/cookbooks/munin/recipes/node.rb b/cookbooks/munin/recipes/node.rb new file mode 100644 index 0000000..bd1b02a --- /dev/null +++ b/cookbooks/munin/recipes/node.rb @@ -0,0 +1,32 @@ +package 'munin-node' + +service 'munin-node' do + action [:enable, :start] +end + +directory '/usr/local/share/munin/plugins' do + recursive true +end +cookbook_file '/usr/local/share/munin/plugins/packetloss' do + mode 0755 +end + +node['peers'].each do |hostname,ip| + link '/etc/munin/plugins/packetloss_' + hostname do + to '/usr/local/share/munin/plugins/packetloss' + end +end + +bash "allow connections from munin master" do + ip = node['config']['munin_master'] + code "echo 'cidr_allow #{ip}/32' >> /etc/munin/munin-node.conf" + not_if "grep 'cidr_allow #{ip}/32' /etc/munin/munin-node.conf" + notifies :restart, 'service[munin-node]' +end + +bash "set munin-node hostname" do + hostname = node['fqdn'] + code "sed -i -e '/^host_name\s*localhost/d; $a host_name #{hostname}' /etc/munin/munin-node.conf" + not_if "grep 'host_name #{hostname}' /etc/munin/munin-node.conf" + notifies :restart, 'service[munin-node]' +end diff --git a/cookbooks/munin/templates/hosts.conf.erb b/cookbooks/munin/templates/hosts.conf.erb new file mode 100644 index 0000000..8b6c0d7 --- /dev/null +++ b/cookbooks/munin/templates/hosts.conf.erb @@ -0,0 +1,4 @@ +<% node['peers'].each do |hostname,ip| %> +[<%= hostname %>] + address <%= ip %> +<% end %> diff --git a/roles/monitoring_server.rb b/roles/monitoring_server.rb new file mode 100644 index 0000000..5fc9b8f --- /dev/null +++ b/roles/monitoring_server.rb @@ -0,0 +1,3 @@ +name 'monitoring_server' +description 'Monitoring server' +run_list 'recipe[munin]' diff --git a/roles/server.rb b/roles/server.rb index e6aa725..3abf411 100644 --- a/roles/server.rb +++ b/roles/server.rb @@ -1,3 +1,3 @@ name 'server' description 'Common configuration for all servers' -run_list 'recipe[basics]', 'recipe[firewall]', 'recipe[email::client]' +run_list 'recipe[basics]', 'recipe[firewall]', 'recipe[email::client]', 'recipe[munin::node]' -- libgit2 0.21.2