Commit 11aba6020c96b782e802e9bc86692ba09b60ab5d
Exists in
master
and in
4 other branches
Merge branch 'remote_syslog' into 'master'
Remote syslog
Showing
13 changed files
with
175 additions
and
0 deletions
Show diff stats
CHANGELOG
@@ -6,6 +6,7 @@ omnibus-gitlab repository. | @@ -6,6 +6,7 @@ omnibus-gitlab repository. | ||
6 | 7.1.0 | 6 | 7.1.0 |
7 | - Build: explicitly use .forward for sending notifications | 7 | - Build: explicitly use .forward for sending notifications |
8 | - Fix MySQL build for Ubuntu 14.04 | 8 | - Fix MySQL build for Ubuntu 14.04 |
9 | +- Built-in UDP log shipping (Enterprise Edition only) | ||
9 | 10 | ||
10 | 7.0.0-ee.omnibus.1 | 11 | 7.0.0-ee.omnibus.1 |
11 | - Fix MySQL build for Ubuntu 14.04 | 12 | - Fix MySQL build for Ubuntu 14.04 |
README.md
@@ -435,6 +435,29 @@ about the files it generates. | @@ -435,6 +435,29 @@ about the files it generates. | ||
435 | 435 | ||
436 | You can modify svlogd settings via `/etc/gitlab/gitlab.rb` with the following settings: | 436 | You can modify svlogd settings via `/etc/gitlab/gitlab.rb` with the following settings: |
437 | 437 | ||
438 | +### UDP log shipping (GitLab Enterprise Edition only) | ||
439 | + | ||
440 | +You can configure omnibus-gitlab to send syslog-ish log messages via UDP. | ||
441 | + | ||
442 | +```ruby | ||
443 | +logging['udp_log_shipping_host'] = '1.2.3.4' # Your syslog server | ||
444 | +logging['udp_log_shipping_port'] = 1514 # Optional, defaults to 514 (syslog) | ||
445 | +``` | ||
446 | + | ||
447 | +Example log messages: | ||
448 | + | ||
449 | +``` | ||
450 | +<13>Jun 26 06:33:46 ubuntu1204-test production.log: Started GET "/root/my-project/import" for 127.0.0.1 at 2014-06-26 06:33:46 -0700 | ||
451 | +<13>Jun 26 06:33:46 ubuntu1204-test production.log: Processing by ProjectsController#import as HTML | ||
452 | +<13>Jun 26 06:33:46 ubuntu1204-test production.log: Parameters: {"id"=>"root/my-project"} | ||
453 | +<13>Jun 26 06:33:46 ubuntu1204-test production.log: Completed 200 OK in 122ms (Views: 71.9ms | ActiveRecord: 12.2ms) | ||
454 | +<13>Jun 26 06:33:46 ubuntu1204-test gitlab_access.log: 172.16.228.1 - - [26/Jun/2014:06:33:46 -0700] "GET /root/my-project/import HTTP/1.1" 200 5775 "https://172.16.228.169/root/my-project/import" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36" | ||
455 | +2014-06-26_13:33:46.49866 ubuntu1204-test sidekiq: 2014-06-26T13:33:46Z 18107 TID-7nbj0 Sidekiq::Extensions::DelayedMailer JID-bbfb118dd1db20f6c39f5b50 INFO: start | ||
456 | + | ||
457 | +2014-06-26_13:33:46.52608 ubuntu1204-test sidekiq: 2014-06-26T13:33:46Z 18107 TID-7muoc RepositoryImportWorker JID-57ee926c3655fcfa062338ae INFO: start | ||
458 | + | ||
459 | +``` | ||
460 | + | ||
438 | ```ruby | 461 | ```ruby |
439 | # Below are the default values | 462 | # Below are the default values |
440 | logging['svlogd_size'] = 200 * 1024 * 1024 # rotate after 200 MB of log data | 463 | logging['svlogd_size'] = 200 * 1024 * 1024 # rotate after 200 MB of log data |
config/projects/gitlab.rb
@@ -35,6 +35,9 @@ dependency "git" | @@ -35,6 +35,9 @@ dependency "git" | ||
35 | dependency "redis" | 35 | dependency "redis" |
36 | dependency "nginx" | 36 | dependency "nginx" |
37 | dependency "chef-gem" | 37 | dependency "chef-gem" |
38 | +if system("#{Omnibus.project_root}/support/is_gitlab_ee.sh") || system("#{Omnibus.project_root}/support/is_gitlab_com.sh") | ||
39 | + dependency "remote-syslog" | ||
40 | +end | ||
38 | dependency "runit" | 41 | dependency "runit" |
39 | dependency "gitlab-rails" | 42 | dependency "gitlab-rails" |
40 | dependency "gitlab-shell" | 43 | dependency "gitlab-shell" |
@@ -0,0 +1,27 @@ | @@ -0,0 +1,27 @@ | ||
1 | +# | ||
2 | +## Copyright:: Copyright (c) 2014 GitLab B.V. | ||
3 | +## License:: Apache License, Version 2.0 | ||
4 | +## | ||
5 | +## Licensed under the Apache License, Version 2.0 (the "License"); | ||
6 | +## you may not use this file except in compliance with the License. | ||
7 | +## You may obtain a copy of the License at | ||
8 | +## | ||
9 | +## http://www.apache.org/licenses/LICENSE-2.0 | ||
10 | +## | ||
11 | +## Unless required by applicable law or agreed to in writing, software | ||
12 | +## distributed under the License is distributed on an "AS IS" BASIS, | ||
13 | +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
14 | +## See the License for the specific language governing permissions and | ||
15 | +## limitations under the License. | ||
16 | +## | ||
17 | +# | ||
18 | + | ||
19 | +name "remote-syslog" | ||
20 | +default_version "1.6.14" | ||
21 | + | ||
22 | +dependency "ruby" | ||
23 | +dependency "rubygems" | ||
24 | + | ||
25 | +build do | ||
26 | + gem "install remote_syslog -n #{install_dir}/embedded/bin --no-rdoc --no-ri -v #{version}" | ||
27 | +end |
files/gitlab-cookbooks/gitlab/attributes/default.rb
@@ -258,3 +258,16 @@ default['gitlab']['logging']['svlogd_timeout'] = 24 * 60 * 60 # rotate after 24 | @@ -258,3 +258,16 @@ default['gitlab']['logging']['svlogd_timeout'] = 24 * 60 * 60 # rotate after 24 | ||
258 | default['gitlab']['logging']['svlogd_filter'] = "gzip" # compress logs with gzip | 258 | default['gitlab']['logging']['svlogd_filter'] = "gzip" # compress logs with gzip |
259 | default['gitlab']['logging']['svlogd_udp'] = nil # transmit log messages via UDP | 259 | default['gitlab']['logging']['svlogd_udp'] = nil # transmit log messages via UDP |
260 | default['gitlab']['logging']['svlogd_prefix'] = nil # custom prefix for log messages | 260 | default['gitlab']['logging']['svlogd_prefix'] = nil # custom prefix for log messages |
261 | +default['gitlab']['logging']['udp_log_shipping_host'] = nil # remote host to ship log messages to via UDP | ||
262 | +default['gitlab']['logging']['udp_log_shipping_port'] = 514 # remote host to ship log messages to via UDP | ||
263 | + | ||
264 | +### | ||
265 | +# Remote syslog | ||
266 | +### | ||
267 | +default['gitlab']['remote-syslog']['enable'] = false | ||
268 | +default['gitlab']['remote-syslog']['ha'] = false | ||
269 | +default['gitlab']['remote-syslog']['dir'] = "/var/opt/gitlab/remote-syslog" | ||
270 | +default['gitlab']['remote-syslog']['log_directory'] = "/var/log/gitlab/remote-syslog" | ||
271 | +default['gitlab']['remote-syslog']['destination_host'] = "localhost" | ||
272 | +default['gitlab']['remote-syslog']['destination_port'] = 514 | ||
273 | +default['gitlab']['remote-syslog']['services'] = %w{redis nginx unicorn gitlab-rails postgresql sidekiq} |
files/gitlab-cookbooks/gitlab/libraries/gitlab.rb
@@ -44,6 +44,7 @@ module Gitlab | @@ -44,6 +44,7 @@ module Gitlab | ||
44 | sidekiq Mash.new | 44 | sidekiq Mash.new |
45 | nginx Mash.new | 45 | nginx Mash.new |
46 | logging Mash.new | 46 | logging Mash.new |
47 | + remote_syslog Mash.new | ||
47 | node nil | 48 | node nil |
48 | external_url nil | 49 | external_url nil |
49 | git_data_dir nil | 50 | git_data_dir nil |
@@ -120,6 +121,24 @@ module Gitlab | @@ -120,6 +121,24 @@ module Gitlab | ||
120 | Gitlab['gitlab_rails']['satellites_path'] ||= File.join(git_data_dir, "gitlab-satellites") | 121 | Gitlab['gitlab_rails']['satellites_path'] ||= File.join(git_data_dir, "gitlab-satellites") |
121 | end | 122 | end |
122 | 123 | ||
124 | + def parse_udp_log_shipping | ||
125 | + return unless logging['udp_log_shipping_host'] | ||
126 | + | ||
127 | + Gitlab['remote_syslog']['enable'] ||= true | ||
128 | + Gitlab['remote_syslog']['destination_host'] ||= logging['udp_log_shipping_host'] | ||
129 | + | ||
130 | + if logging['udp_log_shipping_port'] | ||
131 | + Gitlab['remote_syslog']['destination_port'] ||= logging['udp_log_shipping_port'] | ||
132 | + Gitlab['logging']['svlogd_udp'] ||= "#{logging['udp_log_shipping_host']}:#{logging['udp_log_shipping_port']}" | ||
133 | + else | ||
134 | + Gitlab['logging']['svlogd_udp'] ||= logging['udp_log_shipping_host'] | ||
135 | + end | ||
136 | + | ||
137 | + %w{redis nginx sidekiq unicorn postgresql remote-syslog}.each do |runit_sv| | ||
138 | + Gitlab[runit_sv.gsub('-', '_')]['svlogd_prefix'] ||= "#{node['hostname']} #{runit_sv}: " | ||
139 | + end | ||
140 | + end | ||
141 | + | ||
123 | def generate_hash | 142 | def generate_hash |
124 | results = { "gitlab" => {} } | 143 | results = { "gitlab" => {} } |
125 | [ | 144 | [ |
@@ -132,6 +151,7 @@ module Gitlab | @@ -132,6 +151,7 @@ module Gitlab | ||
132 | "sidekiq", | 151 | "sidekiq", |
133 | "nginx", | 152 | "nginx", |
134 | "logging", | 153 | "logging", |
154 | + "remote_syslog", | ||
135 | "postgresql" | 155 | "postgresql" |
136 | ].each do |key| | 156 | ].each do |key| |
137 | rkey = key.gsub('_', '-') | 157 | rkey = key.gsub('_', '-') |
@@ -145,6 +165,7 @@ module Gitlab | @@ -145,6 +165,7 @@ module Gitlab | ||
145 | generate_secrets(node_name) | 165 | generate_secrets(node_name) |
146 | parse_external_url | 166 | parse_external_url |
147 | parse_git_data_dir | 167 | parse_git_data_dir |
168 | + parse_udp_log_shipping | ||
148 | # The last step is to convert underscores to hyphens in top-level keys | 169 | # The last step is to convert underscores to hyphens in top-level keys |
149 | generate_hash | 170 | generate_hash |
150 | end | 171 | end |
files/gitlab-cookbooks/gitlab/recipes/default.rb
@@ -70,6 +70,7 @@ include_recipe "runit" | @@ -70,6 +70,7 @@ include_recipe "runit" | ||
70 | "unicorn", | 70 | "unicorn", |
71 | "sidekiq", | 71 | "sidekiq", |
72 | "nginx", | 72 | "nginx", |
73 | + "remote-syslog", | ||
73 | "bootstrap", | 74 | "bootstrap", |
74 | ].each do |service| | 75 | ].each do |service| |
75 | if node["gitlab"][service]["enable"] | 76 | if node["gitlab"][service]["enable"] |
@@ -0,0 +1,48 @@ | @@ -0,0 +1,48 @@ | ||
1 | +# | ||
2 | +# Copyright:: Copyright (c) 2014 GitLab B.V. | ||
3 | +# License:: Apache License, Version 2.0 | ||
4 | +# | ||
5 | +# Licensed under the Apache License, Version 2.0 (the "License"); | ||
6 | +# you may not use this file except in compliance with the License. | ||
7 | +# You may obtain a copy of the License at | ||
8 | +# | ||
9 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
10 | +# | ||
11 | +# Unless required by applicable law or agreed to in writing, software | ||
12 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
14 | +# See the License for the specific language governing permissions and | ||
15 | +# limitations under the License. | ||
16 | +# | ||
17 | + | ||
18 | +remote_syslog_dir = node['gitlab']['remote-syslog']['dir'] | ||
19 | +remote_syslog_log_dir = node['gitlab']['remote-syslog']['log_directory'] | ||
20 | + | ||
21 | +[ | ||
22 | + remote_syslog_dir, | ||
23 | + remote_syslog_log_dir | ||
24 | +].each do |dir| | ||
25 | + directory dir do | ||
26 | + mode "0700" | ||
27 | + end | ||
28 | +end | ||
29 | + | ||
30 | +template File.join(remote_syslog_dir, "remote_syslog.yml") do | ||
31 | + mode "0644" | ||
32 | + variables(node['gitlab']['remote-syslog'].to_hash) | ||
33 | + notifies :restart, 'service[remote-syslog]' if OmnibusHelper.should_notify?("remote-syslog") | ||
34 | +end | ||
35 | + | ||
36 | +runit_service "remote-syslog" do | ||
37 | + down node['gitlab']['remote-syslog']['ha'] | ||
38 | + options({ | ||
39 | + :log_directory => remote_syslog_log_dir | ||
40 | + }.merge(params)) | ||
41 | + log_options node['gitlab']['logging'].to_hash.merge(node['gitlab']['remote-syslog'].to_hash) | ||
42 | +end | ||
43 | + | ||
44 | +if node['gitlab']['bootstrap']['enable'] | ||
45 | + execute "/opt/gitlab/bin/gitlab-ctl start remote-syslog" do | ||
46 | + retries 20 | ||
47 | + end | ||
48 | +end |
files/gitlab-cookbooks/gitlab/recipes/remote-syslog_disable.rb
0 → 100644
@@ -0,0 +1,20 @@ | @@ -0,0 +1,20 @@ | ||
1 | +# | ||
2 | +# Copyright:: Copyright (c) 2014 GitLab.com | ||
3 | +# License:: Apache License, Version 2.0 | ||
4 | +# | ||
5 | +# Licensed under the Apache License, Version 2.0 (the "License"); | ||
6 | +# you may not use this file except in compliance with the License. | ||
7 | +# You may obtain a copy of the License at | ||
8 | +# | ||
9 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
10 | +# | ||
11 | +# Unless required by applicable law or agreed to in writing, software | ||
12 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
14 | +# See the License for the specific language governing permissions and | ||
15 | +# limitations under the License. | ||
16 | +# | ||
17 | + | ||
18 | +runit_service "remote-syslog" do | ||
19 | + action :disable | ||
20 | +end |
files/gitlab-cookbooks/gitlab/templates/default/remote_syslog.yml.erb
0 → 100644
files/gitlab-cookbooks/gitlab/templates/default/sv-remote-syslog-log-config.erb
0 → 100644
files/gitlab-cookbooks/gitlab/templates/default/sv-remote-syslog-log-run.erb
0 → 100644
files/gitlab-cookbooks/gitlab/templates/default/sv-remote-syslog-run.erb
0 → 100644