diff --git a/CHANGELOG b/CHANGELOG index 1bf2ce7..ab803e3 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,7 @@ - Support changing the name of the "git" user / group (Michael Fenn) - Configure omniauth in gitlab.yml - Expose more fields under 'extra' in gitlab.yml +- Zero-downtime Unicorn restarts 6.9.0 - Make SSH port in clone URLs configurable (Julien Pivotto) diff --git a/README.md b/README.md index 992f8eb..5d988f8 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,13 @@ sudo gitlab-ctl restart It is also possible to start, stop or restart individual components. ```shell -sudo gitlab-ctl restart unicorn +sudo gitlab-ctl restart sidekiq +``` + +Unicorn supports zero-downtime reloads. These can be triggered as follows: + +```shell +sudo gitlab-ctl hup unicorn ``` ## Configuration diff --git a/files/gitlab-cookbooks/gitlab/attributes/default.rb b/files/gitlab-cookbooks/gitlab/attributes/default.rb index 22f1bdb..2580d72 100644 --- a/files/gitlab-cookbooks/gitlab/attributes/default.rb +++ b/files/gitlab-cookbooks/gitlab/attributes/default.rb @@ -146,6 +146,7 @@ default['gitlab']['unicorn']['worker_processes'] = 2 default['gitlab']['unicorn']['listen'] = '127.0.0.1' default['gitlab']['unicorn']['port'] = 8080 default['gitlab']['unicorn']['socket'] = '/var/opt/gitlab/gitlab-rails/tmp/sockets/gitlab.socket' +default['gitlab']['unicorn']['pidfile'] = '/opt/gitlab/var/unicorn/unicorn.pid' default['gitlab']['unicorn']['tcp_nopush'] = true default['gitlab']['unicorn']['backlog_socket'] = 64 default['gitlab']['unicorn']['worker_timeout'] = 30 diff --git a/files/gitlab-cookbooks/gitlab/recipes/unicorn.rb b/files/gitlab-cookbooks/gitlab/recipes/unicorn.rb index 5f5b49f..0984b4f 100644 --- a/files/gitlab-cookbooks/gitlab/recipes/unicorn.rb +++ b/files/gitlab-cookbooks/gitlab/recipes/unicorn.rb @@ -21,12 +21,14 @@ gitlab_rails_etc_dir = File.join(gitlab_rails_dir, "etc") gitlab_rails_working_dir = File.join(gitlab_rails_dir, "working") unicorn_listen_socket = node['gitlab']['unicorn']['socket'] +unicorn_pidfile = node['gitlab']['unicorn']['pidfile'] unicorn_log_dir = node['gitlab']['unicorn']['log_directory'] unicorn_socket_dir = File.dirname(unicorn_listen_socket) [ unicorn_log_dir, - unicorn_socket_dir + unicorn_socket_dir, + File.dirname(unicorn_pidfile) ].each do |dir_name| directory dir_name do owner node['gitlab']['user']['username'] @@ -51,6 +53,19 @@ unicorn_config File.join(gitlab_rails_etc_dir, "unicorn.rb") do working_directory gitlab_rails_working_dir worker_processes node['gitlab']['unicorn']['worker_processes'] preload_app true + stderr_path File.join(unicorn_log_dir, "unicorn_stderr.log") + stdout_path File.join(unicorn_log_dir, "unicorn_stdout.log") + pid unicorn_pidfile + before_fork <<-'EOS' + old_pid = "#{server.config[:pid]}.oldbin" + if old_pid != server.pid + begin + sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU + Process.kill(sig, File.read(old_pid).to_i) + rescue Errno::ENOENT, Errno::ESRCH + end + end + EOS owner "root" group "root" mode "0644" @@ -59,6 +74,7 @@ end runit_service "unicorn" do down node['gitlab']['unicorn']['ha'] + restart_command 2 # Restart Unicorn using SIGUSR2 options({ :log_directory => unicorn_log_dir }.merge(params)) diff --git a/files/gitlab-cookbooks/gitlab/templates/default/sv-unicorn-run.erb b/files/gitlab-cookbooks/gitlab/templates/default/sv-unicorn-run.erb index 90bf977..32967d6 100644 --- a/files/gitlab-cookbooks/gitlab/templates/default/sv-unicorn-run.erb +++ b/files/gitlab-cookbooks/gitlab/templates/default/sv-unicorn-run.erb @@ -1,6 +1,99 @@ -#!/bin/sh - -cd /opt/gitlab/embedded/service/gitlab-rails +#!/bin/bash +# Let runit capture all script error messages exec 2>&1 -exec chpst -P -U <%= node['gitlab']['user']['username'] %> -u <%= node['gitlab']['user']['username'] %> /usr/bin/env HOME="<%= node['gitlab']['user']['home'] %>" /opt/gitlab/embedded/bin/bundle exec unicorn -E <%= node['gitlab']['gitlab-rails']['environment'] %> -c <%= File.join(node['gitlab']['gitlab-rails']['dir'], "etc", "unicorn.rb") %> /opt/gitlab/embedded/service/gitlab-rails/config.ru + +readonly current_pidfile=<%= node['gitlab']['unicorn']['pidfile'] %> +readonly oldbin_pidfile=${current_pidfile}.oldbin +readonly unicorn_wait_start=1 # time in seconds +readonly unicorn_poll_alive=1 # time in seconds + +function main +{ + cd /opt/gitlab/embedded/service/gitlab-rails + find_us_a_unicorn + trap_signals + wait_for_unicorn_to_exit +} + +function find_us_a_unicorn +{ + adopt ${current_pidfile} + if [[ ${unicorn_pid} ]]; then + echo "adopted existing unicorn master ${unicorn_pid}" + return + fi + + adopt ${oldbin_pidfile} + if [[ ${unicorn_pid} ]]; then + echo "adopted existing oldbin unicorn master ${unicorn_pid}" + return + fi + + echo "starting new unicorn master" + start_unicorn_master + sleep ${unicorn_wait_start} + + adopt ${current_pidfile} + if [[ ${unicorn_pid} ]]; then + echo "adopted new unicorn master ${unicorn_pid}" + return + fi + + echo "failed to start a new unicorn master" + exit +} + +function adopt +{ + local pid=$(cat $1 2>/dev/null) + if alive ${pid} && is_unicorn ${pid}; then + readonly unicorn_pid=${pid} + fi +} + +function alive +{ + kill -0 $1 > /dev/null 2>&1 +} + +function is_unicorn +{ + ps -p $1 -o args | grep -q unicorn +} + +function start_unicorn_master +{ + chpst -P -U <%= node['gitlab']['user']['username'] %> -u <%= node['gitlab']['user']['username'] %> /usr/bin/env HOME="<%= node['gitlab']['user']['home'] %>" /opt/gitlab/embedded/bin/bundle exec unicorn -D -E <%= node['gitlab']['gitlab-rails']['environment'] %> -c <%= File.join(node['gitlab']['gitlab-rails']['dir'], "etc", "unicorn.rb") %> /opt/gitlab/embedded/service/gitlab-rails/config.ru +} + +function trap_signals +{ + # Forward all common runit signals except: + # - HUP which we handle below; + # - KILL which cannot be caught. + for sig in STOP CONT ALRM INT QUIT USR1 USR2 TERM; do + trap "forward_signal ${sig}" ${sig} + done + + # Omnibus-ctl does not have a subcommand that sends USR2 but it can send HUP. + # To allow for reloading unicorn from the command line, translate HUP to + # USR2. + trap "echo 'wrapper received HUP'; forward_signal USR2" HUP +} + +function forward_signal +{ + echo "forwarding $1 to unicorn master ${unicorn_pid}" + kill -$1 ${unicorn_pid} +} + +function wait_for_unicorn_to_exit +{ + while sleep ${unicorn_poll_alive}; do + alive ${unicorn_pid} || break + done +} + +main +echo "wrapper for unicorn master ${unicorn_pid} exiting" -- libgit2 0.21.2