Commit 11aba6020c96b782e802e9bc86692ba09b60ab5d
Exists in
master
and in
4 other branches
Merge branch 'remote_syslog' into 'master'
Remote syslog
Showing
13 changed files
with
175 additions
and
0 deletions
Show diff stats
CHANGELOG
| @@ -6,6 +6,7 @@ omnibus-gitlab repository. | @@ -6,6 +6,7 @@ omnibus-gitlab repository. | ||
| 6 | 7.1.0 | 6 | 7.1.0 |
| 7 | - Build: explicitly use .forward for sending notifications | 7 | - Build: explicitly use .forward for sending notifications |
| 8 | - Fix MySQL build for Ubuntu 14.04 | 8 | - Fix MySQL build for Ubuntu 14.04 |
| 9 | +- Built-in UDP log shipping (Enterprise Edition only) | ||
| 9 | 10 | ||
| 10 | 7.0.0-ee.omnibus.1 | 11 | 7.0.0-ee.omnibus.1 |
| 11 | - Fix MySQL build for Ubuntu 14.04 | 12 | - Fix MySQL build for Ubuntu 14.04 |
README.md
| @@ -435,6 +435,29 @@ about the files it generates. | @@ -435,6 +435,29 @@ about the files it generates. | ||
| 435 | 435 | ||
| 436 | You can modify svlogd settings via `/etc/gitlab/gitlab.rb` with the following settings: | 436 | You can modify svlogd settings via `/etc/gitlab/gitlab.rb` with the following settings: |
| 437 | 437 | ||
| 438 | +### UDP log shipping (GitLab Enterprise Edition only) | ||
| 439 | + | ||
| 440 | +You can configure omnibus-gitlab to send syslog-ish log messages via UDP. | ||
| 441 | + | ||
| 442 | +```ruby | ||
| 443 | +logging['udp_log_shipping_host'] = '1.2.3.4' # Your syslog server | ||
| 444 | +logging['udp_log_shipping_port'] = 1514 # Optional, defaults to 514 (syslog) | ||
| 445 | +``` | ||
| 446 | + | ||
| 447 | +Example log messages: | ||
| 448 | + | ||
| 449 | +``` | ||
| 450 | +<13>Jun 26 06:33:46 ubuntu1204-test production.log: Started GET "/root/my-project/import" for 127.0.0.1 at 2014-06-26 06:33:46 -0700 | ||
| 451 | +<13>Jun 26 06:33:46 ubuntu1204-test production.log: Processing by ProjectsController#import as HTML | ||
| 452 | +<13>Jun 26 06:33:46 ubuntu1204-test production.log: Parameters: {"id"=>"root/my-project"} | ||
| 453 | +<13>Jun 26 06:33:46 ubuntu1204-test production.log: Completed 200 OK in 122ms (Views: 71.9ms | ActiveRecord: 12.2ms) | ||
| 454 | +<13>Jun 26 06:33:46 ubuntu1204-test gitlab_access.log: 172.16.228.1 - - [26/Jun/2014:06:33:46 -0700] "GET /root/my-project/import HTTP/1.1" 200 5775 "https://172.16.228.169/root/my-project/import" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36" | ||
| 455 | +2014-06-26_13:33:46.49866 ubuntu1204-test sidekiq: 2014-06-26T13:33:46Z 18107 TID-7nbj0 Sidekiq::Extensions::DelayedMailer JID-bbfb118dd1db20f6c39f5b50 INFO: start | ||
| 456 | + | ||
| 457 | +2014-06-26_13:33:46.52608 ubuntu1204-test sidekiq: 2014-06-26T13:33:46Z 18107 TID-7muoc RepositoryImportWorker JID-57ee926c3655fcfa062338ae INFO: start | ||
| 458 | + | ||
| 459 | +``` | ||
| 460 | + | ||
| 438 | ```ruby | 461 | ```ruby |
| 439 | # Below are the default values | 462 | # Below are the default values |
| 440 | logging['svlogd_size'] = 200 * 1024 * 1024 # rotate after 200 MB of log data | 463 | logging['svlogd_size'] = 200 * 1024 * 1024 # rotate after 200 MB of log data |
config/projects/gitlab.rb
| @@ -35,6 +35,9 @@ dependency "git" | @@ -35,6 +35,9 @@ dependency "git" | ||
| 35 | dependency "redis" | 35 | dependency "redis" |
| 36 | dependency "nginx" | 36 | dependency "nginx" |
| 37 | dependency "chef-gem" | 37 | dependency "chef-gem" |
| 38 | +if system("#{Omnibus.project_root}/support/is_gitlab_ee.sh") || system("#{Omnibus.project_root}/support/is_gitlab_com.sh") | ||
| 39 | + dependency "remote-syslog" | ||
| 40 | +end | ||
| 38 | dependency "runit" | 41 | dependency "runit" |
| 39 | dependency "gitlab-rails" | 42 | dependency "gitlab-rails" |
| 40 | dependency "gitlab-shell" | 43 | dependency "gitlab-shell" |
| @@ -0,0 +1,27 @@ | @@ -0,0 +1,27 @@ | ||
| 1 | +# | ||
| 2 | +## Copyright:: Copyright (c) 2014 GitLab B.V. | ||
| 3 | +## License:: Apache License, Version 2.0 | ||
| 4 | +## | ||
| 5 | +## Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 6 | +## you may not use this file except in compliance with the License. | ||
| 7 | +## You may obtain a copy of the License at | ||
| 8 | +## | ||
| 9 | +## http://www.apache.org/licenses/LICENSE-2.0 | ||
| 10 | +## | ||
| 11 | +## Unless required by applicable law or agreed to in writing, software | ||
| 12 | +## distributed under the License is distributed on an "AS IS" BASIS, | ||
| 13 | +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 14 | +## See the License for the specific language governing permissions and | ||
| 15 | +## limitations under the License. | ||
| 16 | +## | ||
| 17 | +# | ||
| 18 | + | ||
| 19 | +name "remote-syslog" | ||
| 20 | +default_version "1.6.14" | ||
| 21 | + | ||
| 22 | +dependency "ruby" | ||
| 23 | +dependency "rubygems" | ||
| 24 | + | ||
| 25 | +build do | ||
| 26 | + gem "install remote_syslog -n #{install_dir}/embedded/bin --no-rdoc --no-ri -v #{version}" | ||
| 27 | +end |
files/gitlab-cookbooks/gitlab/attributes/default.rb
| @@ -258,3 +258,16 @@ default['gitlab']['logging']['svlogd_timeout'] = 24 * 60 * 60 # rotate after 24 | @@ -258,3 +258,16 @@ default['gitlab']['logging']['svlogd_timeout'] = 24 * 60 * 60 # rotate after 24 | ||
| 258 | default['gitlab']['logging']['svlogd_filter'] = "gzip" # compress logs with gzip | 258 | default['gitlab']['logging']['svlogd_filter'] = "gzip" # compress logs with gzip |
| 259 | default['gitlab']['logging']['svlogd_udp'] = nil # transmit log messages via UDP | 259 | default['gitlab']['logging']['svlogd_udp'] = nil # transmit log messages via UDP |
| 260 | default['gitlab']['logging']['svlogd_prefix'] = nil # custom prefix for log messages | 260 | default['gitlab']['logging']['svlogd_prefix'] = nil # custom prefix for log messages |
| 261 | +default['gitlab']['logging']['udp_log_shipping_host'] = nil # remote host to ship log messages to via UDP | ||
| 262 | +default['gitlab']['logging']['udp_log_shipping_port'] = 514 # remote host to ship log messages to via UDP | ||
| 263 | + | ||
| 264 | +### | ||
| 265 | +# Remote syslog | ||
| 266 | +### | ||
| 267 | +default['gitlab']['remote-syslog']['enable'] = false | ||
| 268 | +default['gitlab']['remote-syslog']['ha'] = false | ||
| 269 | +default['gitlab']['remote-syslog']['dir'] = "/var/opt/gitlab/remote-syslog" | ||
| 270 | +default['gitlab']['remote-syslog']['log_directory'] = "/var/log/gitlab/remote-syslog" | ||
| 271 | +default['gitlab']['remote-syslog']['destination_host'] = "localhost" | ||
| 272 | +default['gitlab']['remote-syslog']['destination_port'] = 514 | ||
| 273 | +default['gitlab']['remote-syslog']['services'] = %w{redis nginx unicorn gitlab-rails postgresql sidekiq} |
files/gitlab-cookbooks/gitlab/libraries/gitlab.rb
| @@ -44,6 +44,7 @@ module Gitlab | @@ -44,6 +44,7 @@ module Gitlab | ||
| 44 | sidekiq Mash.new | 44 | sidekiq Mash.new |
| 45 | nginx Mash.new | 45 | nginx Mash.new |
| 46 | logging Mash.new | 46 | logging Mash.new |
| 47 | + remote_syslog Mash.new | ||
| 47 | node nil | 48 | node nil |
| 48 | external_url nil | 49 | external_url nil |
| 49 | git_data_dir nil | 50 | git_data_dir nil |
| @@ -120,6 +121,24 @@ module Gitlab | @@ -120,6 +121,24 @@ module Gitlab | ||
| 120 | Gitlab['gitlab_rails']['satellites_path'] ||= File.join(git_data_dir, "gitlab-satellites") | 121 | Gitlab['gitlab_rails']['satellites_path'] ||= File.join(git_data_dir, "gitlab-satellites") |
| 121 | end | 122 | end |
| 122 | 123 | ||
| 124 | + def parse_udp_log_shipping | ||
| 125 | + return unless logging['udp_log_shipping_host'] | ||
| 126 | + | ||
| 127 | + Gitlab['remote_syslog']['enable'] ||= true | ||
| 128 | + Gitlab['remote_syslog']['destination_host'] ||= logging['udp_log_shipping_host'] | ||
| 129 | + | ||
| 130 | + if logging['udp_log_shipping_port'] | ||
| 131 | + Gitlab['remote_syslog']['destination_port'] ||= logging['udp_log_shipping_port'] | ||
| 132 | + Gitlab['logging']['svlogd_udp'] ||= "#{logging['udp_log_shipping_host']}:#{logging['udp_log_shipping_port']}" | ||
| 133 | + else | ||
| 134 | + Gitlab['logging']['svlogd_udp'] ||= logging['udp_log_shipping_host'] | ||
| 135 | + end | ||
| 136 | + | ||
| 137 | + %w{redis nginx sidekiq unicorn postgresql remote-syslog}.each do |runit_sv| | ||
| 138 | + Gitlab[runit_sv.gsub('-', '_')]['svlogd_prefix'] ||= "#{node['hostname']} #{runit_sv}: " | ||
| 139 | + end | ||
| 140 | + end | ||
| 141 | + | ||
| 123 | def generate_hash | 142 | def generate_hash |
| 124 | results = { "gitlab" => {} } | 143 | results = { "gitlab" => {} } |
| 125 | [ | 144 | [ |
| @@ -132,6 +151,7 @@ module Gitlab | @@ -132,6 +151,7 @@ module Gitlab | ||
| 132 | "sidekiq", | 151 | "sidekiq", |
| 133 | "nginx", | 152 | "nginx", |
| 134 | "logging", | 153 | "logging", |
| 154 | + "remote_syslog", | ||
| 135 | "postgresql" | 155 | "postgresql" |
| 136 | ].each do |key| | 156 | ].each do |key| |
| 137 | rkey = key.gsub('_', '-') | 157 | rkey = key.gsub('_', '-') |
| @@ -145,6 +165,7 @@ module Gitlab | @@ -145,6 +165,7 @@ module Gitlab | ||
| 145 | generate_secrets(node_name) | 165 | generate_secrets(node_name) |
| 146 | parse_external_url | 166 | parse_external_url |
| 147 | parse_git_data_dir | 167 | parse_git_data_dir |
| 168 | + parse_udp_log_shipping | ||
| 148 | # The last step is to convert underscores to hyphens in top-level keys | 169 | # The last step is to convert underscores to hyphens in top-level keys |
| 149 | generate_hash | 170 | generate_hash |
| 150 | end | 171 | end |
files/gitlab-cookbooks/gitlab/recipes/default.rb
| @@ -70,6 +70,7 @@ include_recipe "runit" | @@ -70,6 +70,7 @@ include_recipe "runit" | ||
| 70 | "unicorn", | 70 | "unicorn", |
| 71 | "sidekiq", | 71 | "sidekiq", |
| 72 | "nginx", | 72 | "nginx", |
| 73 | + "remote-syslog", | ||
| 73 | "bootstrap", | 74 | "bootstrap", |
| 74 | ].each do |service| | 75 | ].each do |service| |
| 75 | if node["gitlab"][service]["enable"] | 76 | if node["gitlab"][service]["enable"] |
| @@ -0,0 +1,48 @@ | @@ -0,0 +1,48 @@ | ||
| 1 | +# | ||
| 2 | +# Copyright:: Copyright (c) 2014 GitLab B.V. | ||
| 3 | +# License:: Apache License, Version 2.0 | ||
| 4 | +# | ||
| 5 | +# Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 6 | +# you may not use this file except in compliance with the License. | ||
| 7 | +# You may obtain a copy of the License at | ||
| 8 | +# | ||
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 10 | +# | ||
| 11 | +# Unless required by applicable law or agreed to in writing, software | ||
| 12 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 14 | +# See the License for the specific language governing permissions and | ||
| 15 | +# limitations under the License. | ||
| 16 | +# | ||
| 17 | + | ||
| 18 | +remote_syslog_dir = node['gitlab']['remote-syslog']['dir'] | ||
| 19 | +remote_syslog_log_dir = node['gitlab']['remote-syslog']['log_directory'] | ||
| 20 | + | ||
| 21 | +[ | ||
| 22 | + remote_syslog_dir, | ||
| 23 | + remote_syslog_log_dir | ||
| 24 | +].each do |dir| | ||
| 25 | + directory dir do | ||
| 26 | + mode "0700" | ||
| 27 | + end | ||
| 28 | +end | ||
| 29 | + | ||
| 30 | +template File.join(remote_syslog_dir, "remote_syslog.yml") do | ||
| 31 | + mode "0644" | ||
| 32 | + variables(node['gitlab']['remote-syslog'].to_hash) | ||
| 33 | + notifies :restart, 'service[remote-syslog]' if OmnibusHelper.should_notify?("remote-syslog") | ||
| 34 | +end | ||
| 35 | + | ||
| 36 | +runit_service "remote-syslog" do | ||
| 37 | + down node['gitlab']['remote-syslog']['ha'] | ||
| 38 | + options({ | ||
| 39 | + :log_directory => remote_syslog_log_dir | ||
| 40 | + }.merge(params)) | ||
| 41 | + log_options node['gitlab']['logging'].to_hash.merge(node['gitlab']['remote-syslog'].to_hash) | ||
| 42 | +end | ||
| 43 | + | ||
| 44 | +if node['gitlab']['bootstrap']['enable'] | ||
| 45 | + execute "/opt/gitlab/bin/gitlab-ctl start remote-syslog" do | ||
| 46 | + retries 20 | ||
| 47 | + end | ||
| 48 | +end |
files/gitlab-cookbooks/gitlab/recipes/remote-syslog_disable.rb
0 → 100644
| @@ -0,0 +1,20 @@ | @@ -0,0 +1,20 @@ | ||
| 1 | +# | ||
| 2 | +# Copyright:: Copyright (c) 2014 GitLab.com | ||
| 3 | +# License:: Apache License, Version 2.0 | ||
| 4 | +# | ||
| 5 | +# Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 6 | +# you may not use this file except in compliance with the License. | ||
| 7 | +# You may obtain a copy of the License at | ||
| 8 | +# | ||
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 10 | +# | ||
| 11 | +# Unless required by applicable law or agreed to in writing, software | ||
| 12 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 14 | +# See the License for the specific language governing permissions and | ||
| 15 | +# limitations under the License. | ||
| 16 | +# | ||
| 17 | + | ||
| 18 | +runit_service "remote-syslog" do | ||
| 19 | + action :disable | ||
| 20 | +end |
files/gitlab-cookbooks/gitlab/templates/default/remote_syslog.yml.erb
0 → 100644
files/gitlab-cookbooks/gitlab/templates/default/sv-remote-syslog-log-config.erb
0 → 100644
files/gitlab-cookbooks/gitlab/templates/default/sv-remote-syslog-log-run.erb
0 → 100644
files/gitlab-cookbooks/gitlab/templates/default/sv-remote-syslog-run.erb
0 → 100644