Zero-downtime Unicorn restarts during reconfigure

Jacob Vosmaer
1 parent 1202a727
Showing 5 changed files with 123 additions and 6 deletions Show diff stats
CHANGELOG
README.md
files/gitlab-cookbooks/gitlab/attributes/default.rb
files/gitlab-cookbooks/gitlab/recipes/unicorn.rb
files/gitlab-cookbooks/gitlab/templates/default/sv-unicorn-run.erb
@@ -5,6 +5,7 @@
 - Support changing the name of the "git" user / group (Michael Fenn)
 - Configure omniauth in gitlab.yml
 - Expose more fields under 'extra' in gitlab.yml
+- Zero-downtime Unicorn restarts
 6.9.0
 - Make SSH port in clone URLs configurable (Julien Pivotto)
@@ -131,7 +131,13 @@ sudo gitlab-ctl restart
 It is also possible to start, stop or restart individual components.
 ```shell
-sudo gitlab-ctl restart unicorn
+sudo gitlab-ctl restart sidekiq
+```
+
+Unicorn supports zero-downtime reloads. These can be triggered as follows:
+
+```shell
+sudo gitlab-ctl hup unicorn
 ```
 ## Configuration
@@ -146,6 +146,7 @@ default[&#39;gitlab&#39;][&#39;unicorn&#39;][&#39;worker_processes&#39;] = 2
 default['gitlab']['unicorn']['listen'] = '127.0.0.1'
 default['gitlab']['unicorn']['port'] = 8080
 default['gitlab']['unicorn']['socket'] = '/var/opt/gitlab/gitlab-rails/tmp/sockets/gitlab.socket'
+default['gitlab']['unicorn']['pidfile'] = '/opt/gitlab/var/unicorn/unicorn.pid'
 default['gitlab']['unicorn']['tcp_nopush'] = true
 default['gitlab']['unicorn']['backlog_socket'] = 64
 default['gitlab']['unicorn']['worker_timeout'] = 30
@@ -21,12 +21,14 @@ gitlab_rails_etc_dir = File.join(gitlab_rails_dir, &quot;etc&quot;)
 gitlab_rails_working_dir = File.join(gitlab_rails_dir, "working")
 unicorn_listen_socket = node['gitlab']['unicorn']['socket']
+unicorn_pidfile = node['gitlab']['unicorn']['pidfile']
 unicorn_log_dir = node['gitlab']['unicorn']['log_directory']
 unicorn_socket_dir = File.dirname(unicorn_listen_socket)
 [
   unicorn_log_dir,
-  unicorn_socket_dir
+  unicorn_socket_dir,
+  File.dirname(unicorn_pidfile)
 ].each do |dir_name|
   directory dir_name do
     owner node['gitlab']['user']['username']
@@ -51,6 +53,19 @@ unicorn_config File.join(gitlab_rails_etc_dir, &quot;unicorn.rb&quot;) do
   working_directory gitlab_rails_working_dir
   worker_processes node['gitlab']['unicorn']['worker_processes']
   preload_app true
+  stderr_path File.join(unicorn_log_dir, "unicorn_stderr.log")
+  stdout_path File.join(unicorn_log_dir, "unicorn_stdout.log")
+  pid unicorn_pidfile
+  before_fork <<-'EOS'
+    old_pid = "#{server.config[:pid]}.oldbin"
+    if old_pid != server.pid
+      begin
+        sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
+        Process.kill(sig, File.read(old_pid).to_i)
+      rescue Errno::ENOENT, Errno::ESRCH
+      end
+    end
+  EOS
   owner "root"
   group "root"
   mode "0644"
@@ -59,6 +74,7 @@ end
 runit_service "unicorn" do
   down node['gitlab']['unicorn']['ha']
+  restart_command 2 # Restart Unicorn using SIGUSR2
   options({
     :log_directory => unicorn_log_dir
   }.merge(params))
-#!/bin/sh
-
-cd /opt/gitlab/embedded/service/gitlab-rails
+#!/bin/bash
+# Let runit capture all script error messages
 exec 2>&1
-exec chpst -P -U <%= node['gitlab']['user']['username'] %> -u <%= node['gitlab']['user']['username'] %> /usr/bin/env HOME="<%= node['gitlab']['user']['home'] %>" /opt/gitlab/embedded/bin/bundle exec unicorn -E <%= node['gitlab']['gitlab-rails']['environment'] %> -c <%= File.join(node['gitlab']['gitlab-rails']['dir'], "etc", "unicorn.rb") %> /opt/gitlab/embedded/service/gitlab-rails/config.ru
+
+readonly current_pidfile=<%= node['gitlab']['unicorn']['pidfile'] %>
+readonly oldbin_pidfile=${current_pidfile}.oldbin
+readonly unicorn_wait_start=1 # time in seconds
+readonly unicorn_poll_alive=1 # time in seconds
+
+function main
+{
+  cd /opt/gitlab/embedded/service/gitlab-rails
+  find_us_a_unicorn
+  trap_signals
+  wait_for_unicorn_to_exit
+}
+
+function find_us_a_unicorn
+{
+  adopt ${current_pidfile}
+  if [[ ${unicorn_pid} ]]; then
+    echo "adopted existing unicorn master ${unicorn_pid}"
+    return
+  fi
+
+  adopt ${oldbin_pidfile}
+  if [[ ${unicorn_pid} ]]; then
+    echo "adopted existing oldbin unicorn master ${unicorn_pid}"
+    return
+  fi
+
+  echo "starting new unicorn master"
+  start_unicorn_master
+  sleep ${unicorn_wait_start}
+
+  adopt ${current_pidfile}
+  if [[ ${unicorn_pid} ]]; then
+    echo "adopted new unicorn master ${unicorn_pid}"
+    return
+  fi
+
+  echo "failed to start a new unicorn master"
+  exit
+}
+
+function adopt
+{
+  local pid=$(cat $1 2>/dev/null)
+  if alive ${pid} && is_unicorn ${pid}; then
+    readonly unicorn_pid=${pid}
+  fi
+}
+
+function alive
+{
+  kill -0 $1 > /dev/null 2>&1
+}
+
+function is_unicorn
+{
+  ps -p $1 -o args | grep -q unicorn
+}
+
+function start_unicorn_master
+{
+  chpst -P -U <%= node['gitlab']['user']['username'] %> -u <%= node['gitlab']['user']['username'] %> /usr/bin/env HOME="<%= node['gitlab']['user']['home'] %>" /opt/gitlab/embedded/bin/bundle exec unicorn -D -E <%= node['gitlab']['gitlab-rails']['environment'] %> -c <%= File.join(node['gitlab']['gitlab-rails']['dir'], "etc", "unicorn.rb") %> /opt/gitlab/embedded/service/gitlab-rails/config.ru
+}
+
+function trap_signals
+{
+  # Forward all common runit signals except:
+  # - HUP which we handle below;
+  # - KILL which cannot be caught.
+  for sig in STOP CONT ALRM INT QUIT USR1 USR2 TERM; do
+    trap "forward_signal ${sig}" ${sig}
+  done
+
+  # Omnibus-ctl does not have a subcommand that sends USR2 but it can send HUP.
+  # To allow for reloading unicorn from the command line, translate HUP to
+  # USR2.
+  trap "echo 'wrapper received HUP'; forward_signal USR2" HUP
+}
+
+function forward_signal
+{
+  echo "forwarding $1 to unicorn master ${unicorn_pid}"
+  kill -$1 ${unicorn_pid}
+}
+
+function wait_for_unicorn_to_exit
+{
+  while sleep ${unicorn_poll_alive}; do
+    alive ${unicorn_pid} || break
+  done
+}
+
+main
+echo "wrapper for unicorn master ${unicorn_pid} exiting"
	@@ -5,6 +5,7 @@		@@ -5,6 +5,7 @@
5	- Support changing the name of the "git" user / group (Michael Fenn)	5	- Support changing the name of the "git" user / group (Michael Fenn)
6	- Configure omniauth in gitlab.yml	6	- Configure omniauth in gitlab.yml
7	- Expose more fields under 'extra' in gitlab.yml	7	- Expose more fields under 'extra' in gitlab.yml
		8	+- Zero-downtime Unicorn restarts
8		9
9	6.9.0	10	6.9.0
10	- Make SSH port in clone URLs configurable (Julien Pivotto)	11	- Make SSH port in clone URLs configurable (Julien Pivotto)
	@@ -21,12 +21,14 @@ gitlab_rails_etc_dir = File.join(gitlab_rails_dir, "etc")		@@ -21,12 +21,14 @@ gitlab_rails_etc_dir = File.join(gitlab_rails_dir, "etc")
21	gitlab_rails_working_dir = File.join(gitlab_rails_dir, "working")	21	gitlab_rails_working_dir = File.join(gitlab_rails_dir, "working")
22		22
23	unicorn_listen_socket = node['gitlab']['unicorn']['socket']	23	unicorn_listen_socket = node['gitlab']['unicorn']['socket']
		24	+unicorn_pidfile = node['gitlab']['unicorn']['pidfile']
24	unicorn_log_dir = node['gitlab']['unicorn']['log_directory']	25	unicorn_log_dir = node['gitlab']['unicorn']['log_directory']
25	unicorn_socket_dir = File.dirname(unicorn_listen_socket)	26	unicorn_socket_dir = File.dirname(unicorn_listen_socket)
26		27
27	[	28	[
28	unicorn_log_dir,	29	unicorn_log_dir,
29	- unicorn_socket_dir	30	+ unicorn_socket_dir,
		31	+ File.dirname(unicorn_pidfile)
30	].each do \|dir_name\|	32	].each do \|dir_name\|
31	directory dir_name do	33	directory dir_name do
32	owner node['gitlab']['user']['username']	34	owner node['gitlab']['user']['username']
	@@ -51,6 +53,19 @@ unicorn_config File.join(gitlab_rails_etc_dir, "unicorn.rb") do		@@ -51,6 +53,19 @@ unicorn_config File.join(gitlab_rails_etc_dir, "unicorn.rb") do
51	working_directory gitlab_rails_working_dir	53	working_directory gitlab_rails_working_dir
52	worker_processes node['gitlab']['unicorn']['worker_processes']	54	worker_processes node['gitlab']['unicorn']['worker_processes']
53	preload_app true	55	preload_app true
		56	+ stderr_path File.join(unicorn_log_dir, "unicorn_stderr.log")
		57	+ stdout_path File.join(unicorn_log_dir, "unicorn_stdout.log")
		58	+ pid unicorn_pidfile
		59	+ before_fork <<-'EOS'
		60	+ old_pid = "#{server.config[:pid]}.oldbin"
		61	+ if old_pid != server.pid
		62	+ begin
		63	+ sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
		64	+ Process.kill(sig, File.read(old_pid).to_i)
		65	+ rescue Errno::ENOENT, Errno::ESRCH
		66	+ end
		67	+ end
		68	+ EOS
54	owner "root"	69	owner "root"
55	group "root"	70	group "root"
56	mode "0644"	71	mode "0644"
	@@ -59,6 +74,7 @@ end		@@ -59,6 +74,7 @@ end
59		74
60	runit_service "unicorn" do	75	runit_service "unicorn" do
61	down node['gitlab']['unicorn']['ha']	76	down node['gitlab']['unicorn']['ha']
		77	+ restart_command 2 # Restart Unicorn using SIGUSR2
62	options({	78	options({
63	:log_directory => unicorn_log_dir	79	:log_directory => unicorn_log_dir
64	}.merge(params))	80	}.merge(params))
1	-#!/bin/sh
2	-
3	-cd /opt/gitlab/embedded/service/gitlab-rails	1	+#!/bin/bash
4		2
		3	+# Let runit capture all script error messages
5	exec 2>&1	4	exec 2>&1
6	-exec chpst -P -U <%= node['gitlab']['user']['username'] %> -u <%= node['gitlab']['user']['username'] %> /usr/bin/env HOME="<%= node['gitlab']['user']['home'] %>" /opt/gitlab/embedded/bin/bundle exec unicorn -E <%= node['gitlab']['gitlab-rails']['environment'] %> -c <%= File.join(node['gitlab']['gitlab-rails']['dir'], "etc", "unicorn.rb") %> /opt/gitlab/embedded/service/gitlab-rails/config.ru	5	+
		6	+readonly current_pidfile=<%= node['gitlab']['unicorn']['pidfile'] %>
		7	+readonly oldbin_pidfile=${current_pidfile}.oldbin
		8	+readonly unicorn_wait_start=1 # time in seconds
		9	+readonly unicorn_poll_alive=1 # time in seconds
		10	+
		11	+function main
		12	+{
		13	+ cd /opt/gitlab/embedded/service/gitlab-rails
		14	+ find_us_a_unicorn
		15	+ trap_signals
		16	+ wait_for_unicorn_to_exit
		17	+}
		18	+
		19	+function find_us_a_unicorn
		20	+{
		21	+ adopt ${current_pidfile}
		22	+ if [[ ${unicorn_pid} ]]; then
		23	+ echo "adopted existing unicorn master ${unicorn_pid}"
		24	+ return
		25	+ fi
		26	+
		27	+ adopt ${oldbin_pidfile}
		28	+ if [[ ${unicorn_pid} ]]; then
		29	+ echo "adopted existing oldbin unicorn master ${unicorn_pid}"
		30	+ return
		31	+ fi
		32	+
		33	+ echo "starting new unicorn master"
		34	+ start_unicorn_master
		35	+ sleep ${unicorn_wait_start}
		36	+
		37	+ adopt ${current_pidfile}
		38	+ if [[ ${unicorn_pid} ]]; then
		39	+ echo "adopted new unicorn master ${unicorn_pid}"
		40	+ return
		41	+ fi
		42	+
		43	+ echo "failed to start a new unicorn master"
		44	+ exit
		45	+}
		46	+
		47	+function adopt
		48	+{
		49	+ local pid=$(cat $1 2>/dev/null)
		50	+ if alive ${pid} && is_unicorn ${pid}; then
		51	+ readonly unicorn_pid=${pid}
		52	+ fi
		53	+}
		54	+
		55	+function alive
		56	+{
		57	+ kill -0 $1 > /dev/null 2>&1
		58	+}
		59	+
		60	+function is_unicorn
		61	+{
		62	+ ps -p $1 -o args \| grep -q unicorn
		63	+}
		64	+
		65	+function start_unicorn_master
		66	+{
		67	+ chpst -P -U <%= node['gitlab']['user']['username'] %> -u <%= node['gitlab']['user']['username'] %> /usr/bin/env HOME="<%= node['gitlab']['user']['home'] %>" /opt/gitlab/embedded/bin/bundle exec unicorn -D -E <%= node['gitlab']['gitlab-rails']['environment'] %> -c <%= File.join(node['gitlab']['gitlab-rails']['dir'], "etc", "unicorn.rb") %> /opt/gitlab/embedded/service/gitlab-rails/config.ru
		68	+}
		69	+
		70	+function trap_signals
		71	+{
		72	+ # Forward all common runit signals except:
		73	+ # - HUP which we handle below;
		74	+ # - KILL which cannot be caught.
		75	+ for sig in STOP CONT ALRM INT QUIT USR1 USR2 TERM; do
		76	+ trap "forward_signal ${sig}" ${sig}
		77	+ done
		78	+
		79	+ # Omnibus-ctl does not have a subcommand that sends USR2 but it can send HUP.
		80	+ # To allow for reloading unicorn from the command line, translate HUP to
		81	+ # USR2.
		82	+ trap "echo 'wrapper received HUP'; forward_signal USR2" HUP
		83	+}
		84	+
		85	+function forward_signal
		86	+{
		87	+ echo "forwarding $1 to unicorn master ${unicorn_pid}"
		88	+ kill -$1 ${unicorn_pid}
		89	+}
		90	+
		91	+function wait_for_unicorn_to_exit
		92	+{
		93	+ while sleep ${unicorn_poll_alive}; do
		94	+ alive ${unicorn_pid} \|\| break
		95	+ done
		96	+}
		97	+
		98	+main
		99	+echo "wrapper for unicorn master ${unicorn_pid} exiting"