Monitoring support with munin

- all servers have munin-node installed - unless an external munin master is specified, munin master will be installed to reverseproxy - munin master for production will be managed separately

Monitoring support with munin
- all servers have munin-node installed - unless an external munin master is specified, munin master will be installed to reverseproxy - munin master for production will be managed separately
Antonio Terceiro
1 parent f38cf00c
Showing 10 changed files with 139 additions and 1 deletions Show diff stats
Rakefile
config/prod/config.yaml
cookbooks/firewall/templates/default/iptables.erb
cookbooks/munin/files/nginx.conf
cookbooks/munin/files/packetloss
cookbooks/munin/recipes/default.rb
cookbooks/munin/recipes/node.rb
cookbooks/munin/templates/hosts.conf.erb
roles/monitoring_server.rb
roles/server.rb
@@ -58,6 +58,12 @@ $nodes.each do |node|
   node.data['firewall'] = firewall
 end
+# In the absence of a dedicated munin master, reverseproxy will do that.
+if !config['munin_master']
+  config['munin_master'] = ips['reverseproxy']
+  $nodes.find { |node| node.hostname == 'reverseproxy' }.data['run_list'] << 'role[monitoring_server]'
+end
+
 task :console do
   require 'pry'
   binding.pry
@@ -17,3 +17,4 @@ external_outgoing_mail_relay: 189.9.150.53
 external_outgoing_mail_domain: serpro.gov.br
 raven_dsn: https://4418146896924efe9b73d557f803f047:8a59f39b5f584ff589ecf3dd47faaead@sentry.tracy.com.br/13
 google_analytics_id: 'UA-64206731-1'
+munin_master: 10.21.0.10
@@ -16,11 +16,19 @@
 -A INPUT -p icmp --icmp-type 11 -j ACCEPT
 -A INPUT -p icmp --icmp-type 12 -j ACCEPT
+# allow ping between the peers
+<% node['peers'].each do |hostname,ip| %>
+-A INPUT -s <%= ip %> -p icmp --icmp-type 8 -j ACCEPT
+<% end %>
+
 -A INPUT -i lo -j ACCEPT
 # Everybody need to accept SSH from integration
 -A INPUT -s <%= node['peers']['integration'] %> -p tcp -m state --state NEW --dport 22 -j ACCEPT
+# Everybody needs to accept munin connections from munin master
+-A INPUT -s <%= node['config']['munin_master'] %> -p tcp -m state --state NEW --dport 4949 -j ACCEPT
+
 <%= node['firewall'] %>
 <%= render 'iptables-filter.erb' %>
@@ -0,0 +1,12 @@
+location /munin/static/ {
+  alias /var/www/html/munin/static/;
+  expires modified +1w;
+}
+location /munin/ {
+  # auth_basic            "Restricted";
+  # # Create the htpasswd file with the htpasswd tool.
+  # auth_basic_user_file  /etc/nginx/munin_htpasswd;
+
+  alias /var/www/html/munin/;
+  expires modified +310s;
+}
@@ -0,0 +1,60 @@
+#!/bin/sh
+#
+# Copyright (c) 2009 Sven-Hendrik Haase
+# Copyright (C) 2004 Jimmy Olsen
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2 dated June,
+# 1991.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#
+# Plugin to monitor packet loss
+#
+# Please note that sometimes it can take quite long for the plugin to return
+# a value on a network with lots of packet loss.
+# You therefore need to account for it by appending the following to your
+# plugin-conf.d/munin-node. Remember to restart munin-node afterwards.
+# Append the next 3 lines to plugin-conf.d/munin-node:
+# [packetloss_*]
+# timeout 60
+# user root
+#
+# Parameters:
+#
+# 	ping_args      - Arguments to ping (default "-c 2")
+# 	ping_args2     - Arguments after the host name (required for Solaris)
+# 	ping           - Ping program to use
+# 	host           - Host to ping
+#
+# Arguments for Solaris:
+#      ping_args      -s
+#      ping_args2     56 2
+#
+#%# family=manual
+
+file_host=`basename $0 | sed 's/^packetloss_//g'`
+host=${host:-${file_host:-www.google.com}}
+
+if [ "$1" = "config" ]; then
+	echo "graph_title Packet loss to $host (in %)"
+	echo 'graph_args --upper-limit 100 -l 0'
+	echo 'graph_vlabel %'
+	echo 'graph_category network'
+	echo 'graph_info This graph shows packet loss statistics.'
+	echo "packetloss.label $host"
+	echo "packetloss.info Packet loss statistics for $host."
+	echo 'packetloss.draw LINE2'
+	exit 0
+fi
+
+${ping:-ping} ${ping_args:-'-c 10'} ${host} ${ping_args2} | perl -n -e 'print "packetloss.value $1\n" if /(\d+)% packet loss/;'
@@ -0,0 +1,12 @@
+package 'munin'
+
+template '/etc/munin/conf.d/hosts.conf'
+
+package 'nginx'
+service 'nginx' do
+  supports :reload => true
+end
+cookbook_file '/etc/nginx/default.d/munin.conf' do
+  source 'nginx.conf'
+  notifies :reload, 'service[nginx]'
+end
@@ -0,0 +1,32 @@
+package 'munin-node'
+
+service 'munin-node' do
+  action [:enable, :start]
+end
+
+directory '/usr/local/share/munin/plugins' do
+  recursive true
+end
+cookbook_file '/usr/local/share/munin/plugins/packetloss' do
+  mode 0755
+end
+
+node['peers'].each do |hostname,ip|
+  link '/etc/munin/plugins/packetloss_' + hostname do
+    to '/usr/local/share/munin/plugins/packetloss'
+  end
+end
+
+bash "allow connections from munin master" do
+  ip = node['config']['munin_master']
+  code "echo 'cidr_allow #{ip}/32' >> /etc/munin/munin-node.conf"
+  not_if "grep 'cidr_allow #{ip}/32' /etc/munin/munin-node.conf"
+  notifies :restart, 'service[munin-node]'
+end
+
+bash "set munin-node hostname" do
+  hostname = node['fqdn']
+  code "sed -i -e '/^host_name\s*localhost/d; $a host_name #{hostname}' /etc/munin/munin-node.conf"
+  not_if "grep 'host_name #{hostname}' /etc/munin/munin-node.conf"
+  notifies :restart, 'service[munin-node]'
+end
@@ -0,0 +1,4 @@
+<% node['peers'].each do |hostname,ip| %>
+[<%= hostname %>]
+  address <%= ip %>
+<% end %>
@@ -0,0 +1,3 @@
+name 'monitoring_server'
+description 'Monitoring server'
+run_list 'recipe[munin]'
 name 'server'
 description 'Common configuration for all servers'
-run_list 'recipe[basics]', 'recipe[firewall]', 'recipe[email::client]'
+run_list 'recipe[basics]', 'recipe[firewall]', 'recipe[email::client]', 'recipe[munin::node]'
	@@ -58,6 +58,12 @@ $nodes.each do \|node\|		@@ -58,6 +58,12 @@ $nodes.each do \|node\|
58	node.data['firewall'] = firewall	58	node.data['firewall'] = firewall
59	end	59	end
60		60
		61	+# In the absence of a dedicated munin master, reverseproxy will do that.
		62	+if !config['munin_master']
		63	+ config['munin_master'] = ips['reverseproxy']
		64	+ $nodes.find { \|node\| node.hostname == 'reverseproxy' }.data['run_list'] << 'role[monitoring_server]'
		65	+end
		66	+
61	task :console do	67	task :console do
62	require 'pry'	68	require 'pry'
63	binding.pry	69	binding.pry
	@@ -17,3 +17,4 @@ external_outgoing_mail_relay: 189.9.150.53		@@ -17,3 +17,4 @@ external_outgoing_mail_relay: 189.9.150.53
17	external_outgoing_mail_domain: serpro.gov.br	17	external_outgoing_mail_domain: serpro.gov.br
18	raven_dsn: https://4418146896924efe9b73d557f803f047:8a59f39b5f584ff589ecf3dd47faaead@sentry.tracy.com.br/13	18	raven_dsn: https://4418146896924efe9b73d557f803f047:8a59f39b5f584ff589ecf3dd47faaead@sentry.tracy.com.br/13
19	google_analytics_id: 'UA-64206731-1'	19	google_analytics_id: 'UA-64206731-1'
		20	+munin_master: 10.21.0.10
@@ -0,0 +1,12 @@		@@ -0,0 +1,12 @@
	1	+location /munin/static/ {
	2	+ alias /var/www/html/munin/static/;
	3	+ expires modified +1w;
	4	+}
	5	+location /munin/ {
	6	+ # auth_basic "Restricted";
	7	+ # # Create the htpasswd file with the htpasswd tool.
	8	+ # auth_basic_user_file /etc/nginx/munin_htpasswd;
	9	+
	10	+ alias /var/www/html/munin/;
	11	+ expires modified +310s;
	12	+}
@@ -0,0 +1,60 @@		@@ -0,0 +1,60 @@
	1	+#!/bin/sh
	2	+#
	3	+# Copyright (c) 2009 Sven-Hendrik Haase
	4	+# Copyright (C) 2004 Jimmy Olsen
	5	+#
	6	+# This program is free software; you can redistribute it and/or
	7	+# modify it under the terms of the GNU General Public License
	8	+# as published by the Free Software Foundation; version 2 dated June,
	9	+# 1991.
	10	+#
	11	+# This program is distributed in the hope that it will be useful,
	12	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	14	+# GNU General Public License for more details.
	15	+#
	16	+# You should have received a copy of the GNU General Public License
	17	+# along with this program; if not, write to the Free Software
	18	+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	19	+#
	20	+#
	21	+# Plugin to monitor packet loss
	22	+#
	23	+# Please note that sometimes it can take quite long for the plugin to return
	24	+# a value on a network with lots of packet loss.
	25	+# You therefore need to account for it by appending the following to your
	26	+# plugin-conf.d/munin-node. Remember to restart munin-node afterwards.
	27	+# Append the next 3 lines to plugin-conf.d/munin-node:
	28	+# [packetloss_*]
	29	+# timeout 60
	30	+# user root
	31	+#
	32	+# Parameters:
	33	+#
	34	+# ping_args - Arguments to ping (default "-c 2")
	35	+# ping_args2 - Arguments after the host name (required for Solaris)
	36	+# ping - Ping program to use
	37	+# host - Host to ping
	38	+#
	39	+# Arguments for Solaris:
	40	+# ping_args -s
	41	+# ping_args2 56 2
	42	+#
	43	+#%# family=manual
	44	+
	45	+file_host=`basename $0 \| sed 's/^packetloss_//g'`
	46	+host=${host:-${file_host:-www.google.com}}
	47	+
	48	+if [ "$1" = "config" ]; then
	49	+ echo "graph_title Packet loss to $host (in %)"
	50	+ echo 'graph_args --upper-limit 100 -l 0'
	51	+ echo 'graph_vlabel %'
	52	+ echo 'graph_category network'
	53	+ echo 'graph_info This graph shows packet loss statistics.'
	54	+ echo "packetloss.label $host"
	55	+ echo "packetloss.info Packet loss statistics for $host."
	56	+ echo 'packetloss.draw LINE2'
	57	+ exit 0
	58	+fi
	59	+
	60	+${ping:-ping} ${ping_args:-'-c 10'} ${host} ${ping_args2} \| perl -n -e 'print "packetloss.value $1\n" if /(\d+)% packet loss/;'
@@ -0,0 +1,12 @@		@@ -0,0 +1,12 @@
	1	+package 'munin'
	2	+
	3	+template '/etc/munin/conf.d/hosts.conf'
	4	+
	5	+package 'nginx'
	6	+service 'nginx' do
	7	+ supports :reload => true
	8	+end
	9	+cookbook_file '/etc/nginx/default.d/munin.conf' do
	10	+ source 'nginx.conf'
	11	+ notifies :reload, 'service[nginx]'
	12	+end
@@ -0,0 +1,32 @@		@@ -0,0 +1,32 @@
	1	+package 'munin-node'
	2	+
	3	+service 'munin-node' do
	4	+ action [:enable, :start]
	5	+end
	6	+
	7	+directory '/usr/local/share/munin/plugins' do
	8	+ recursive true
	9	+end
	10	+cookbook_file '/usr/local/share/munin/plugins/packetloss' do
	11	+ mode 0755
	12	+end
	13	+
	14	+node['peers'].each do \|hostname,ip\|
	15	+ link '/etc/munin/plugins/packetloss_' + hostname do
	16	+ to '/usr/local/share/munin/plugins/packetloss'
	17	+ end
	18	+end
	19	+
	20	+bash "allow connections from munin master" do
	21	+ ip = node['config']['munin_master']
	22	+ code "echo 'cidr_allow #{ip}/32' >> /etc/munin/munin-node.conf"
	23	+ not_if "grep 'cidr_allow #{ip}/32' /etc/munin/munin-node.conf"
	24	+ notifies :restart, 'service[munin-node]'
	25	+end
	26	+
	27	+bash "set munin-node hostname" do
	28	+ hostname = node['fqdn']
	29	+ code "sed -i -e '/^host_name\s*localhost/d; $a host_name #{hostname}' /etc/munin/munin-node.conf"
	30	+ not_if "grep 'host_name #{hostname}' /etc/munin/munin-node.conf"
	31	+ notifies :restart, 'service[munin-node]'
	32	+end
@@ -0,0 +1,4 @@		@@ -0,0 +1,4 @@
	1	+<% node['peers'].each do \|hostname,ip\| %>
	2	+[<%= hostname %>]
	3	+ address <%= ip %>
	4	+<% end %>
@@ -0,0 +1,3 @@		@@ -0,0 +1,3 @@
	1	+name 'monitoring_server'
	2	+description 'Monitoring server'
	3	+run_list 'recipe[munin]'
1	name 'server'	1	name 'server'
2	description 'Common configuration for all servers'	2	description 'Common configuration for all servers'
3	-run_list 'recipe[basics]', 'recipe[firewall]', 'recipe[email::client]'	3	+run_list 'recipe[basics]', 'recipe[firewall]', 'recipe[email::client]', 'recipe[munin::node]'