Zero-downtime Unicorn restarts during reconfigure

Jacob Vosmaer
1 parent 1202a727
Showing 5 changed files with 123 additions and 6 deletions Show diff stats
CHANGELOG
README.md
files/gitlab-cookbooks/gitlab/attributes/default.rb
files/gitlab-cookbooks/gitlab/recipes/unicorn.rb
files/gitlab-cookbooks/gitlab/templates/default/sv-unicorn-run.erb
@@ -5,6 +5,7 @@
 - Support changing the name of the "git" user / group (Michael Fenn)
 - Configure omniauth in gitlab.yml
 - Expose more fields under 'extra' in gitlab.yml
+- Zero-downtime Unicorn restarts
  
 6.9.0
 - Make SSH port in clone URLs configurable (Julien Pivotto)
@@ -131,7 +131,13 @@ sudo gitlab-ctl restart
 It is also possible to start, stop or restart individual components.
  
 ```shell
-sudo gitlab-ctl restart unicorn
+sudo gitlab-ctl restart sidekiq
+```
+
+Unicorn supports zero-downtime reloads. These can be triggered as follows:
+
+```shell
+sudo gitlab-ctl hup unicorn
 ```
  
 ## Configuration
@@ -146,6 +146,7 @@ default[&#39;gitlab&#39;][&#39;unicorn&#39;][&#39;worker_processes&#39;] = 2
 default['gitlab']['unicorn']['listen'] = '127.0.0.1'
 default['gitlab']['unicorn']['port'] = 8080
 default['gitlab']['unicorn']['socket'] = '/var/opt/gitlab/gitlab-rails/tmp/sockets/gitlab.socket'
+default['gitlab']['unicorn']['pidfile'] = '/opt/gitlab/var/unicorn/unicorn.pid'
 default['gitlab']['unicorn']['tcp_nopush'] = true
 default['gitlab']['unicorn']['backlog_socket'] = 64
 default['gitlab']['unicorn']['worker_timeout'] = 30
@@ -21,12 +21,14 @@ gitlab_rails_etc_dir = File.join(gitlab_rails_dir, &quot;etc&quot;)
 gitlab_rails_working_dir = File.join(gitlab_rails_dir, "working")
  
 unicorn_listen_socket = node['gitlab']['unicorn']['socket']
+unicorn_pidfile = node['gitlab']['unicorn']['pidfile']
 unicorn_log_dir = node['gitlab']['unicorn']['log_directory']
 unicorn_socket_dir = File.dirname(unicorn_listen_socket)
  
 [
   unicorn_log_dir,
-  unicorn_socket_dir
+  unicorn_socket_dir,
+  File.dirname(unicorn_pidfile)
 ].each do |dir_name|
   directory dir_name do
     owner node['gitlab']['user']['username']
@@ -51,6 +53,19 @@ unicorn_config File.join(gitlab_rails_etc_dir, &quot;unicorn.rb&quot;) do
   working_directory gitlab_rails_working_dir
   worker_processes node['gitlab']['unicorn']['worker_processes']
   preload_app true
+  stderr_path File.join(unicorn_log_dir, "unicorn_stderr.log")
+  stdout_path File.join(unicorn_log_dir, "unicorn_stdout.log")
+  pid unicorn_pidfile
+  before_fork <<-'EOS'
+    old_pid = "#{server.config[:pid]}.oldbin"
+    if old_pid != server.pid
+      begin
+        sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
+        Process.kill(sig, File.read(old_pid).to_i)
+      rescue Errno::ENOENT, Errno::ESRCH
+      end
+    end
+  EOS
   owner "root"
   group "root"
   mode "0644"
@@ -59,6 +74,7 @@ end
  
 runit_service "unicorn" do
   down node['gitlab']['unicorn']['ha']
+  restart_command 2 # Restart Unicorn using SIGUSR2
   options({
     :log_directory => unicorn_log_dir
   }.merge(params))
-#!/bin/sh
-
-cd /opt/gitlab/embedded/service/gitlab-rails
+#!/bin/bash
  
+# Let runit capture all script error messages
 exec 2>&1
-exec chpst -P -U <%= node['gitlab']['user']['username'] %> -u <%= node['gitlab']['user']['username'] %> /usr/bin/env HOME="<%= node['gitlab']['user']['home'] %>" /opt/gitlab/embedded/bin/bundle exec unicorn -E <%= node['gitlab']['gitlab-rails']['environment'] %> -c <%= File.join(node['gitlab']['gitlab-rails']['dir'], "etc", "unicorn.rb") %> /opt/gitlab/embedded/service/gitlab-rails/config.ru
+
+readonly current_pidfile=<%= node['gitlab']['unicorn']['pidfile'] %>
+readonly oldbin_pidfile=${current_pidfile}.oldbin
+readonly unicorn_wait_start=1 # time in seconds
+readonly unicorn_poll_alive=1 # time in seconds
+
+function main
+{
+  cd /opt/gitlab/embedded/service/gitlab-rails
+  find_us_a_unicorn
+  trap_signals
+  wait_for_unicorn_to_exit
+}
+
+function find_us_a_unicorn
+{
+  adopt ${current_pidfile}
+  if [[ ${unicorn_pid} ]]; then
+    echo "adopted existing unicorn master ${unicorn_pid}"
+    return
+  fi
+
+  adopt ${oldbin_pidfile}
+  if [[ ${unicorn_pid} ]]; then
+    echo "adopted existing oldbin unicorn master ${unicorn_pid}"
+    return
+  fi
+
+  echo "starting new unicorn master"
+  start_unicorn_master
+  sleep ${unicorn_wait_start}
+
+  adopt ${current_pidfile}
+  if [[ ${unicorn_pid} ]]; then
+    echo "adopted new unicorn master ${unicorn_pid}"
+    return
+  fi
+
+  echo "failed to start a new unicorn master"
+  exit
+}
+
+function adopt
+{
+  local pid=$(cat $1 2>/dev/null)
+  if alive ${pid} && is_unicorn ${pid}; then
+    readonly unicorn_pid=${pid}
+  fi
+}
+
+function alive
+{
+  kill -0 $1 > /dev/null 2>&1
+}
+
+function is_unicorn
+{
+  ps -p $1 -o args | grep -q unicorn
+}
+
+function start_unicorn_master
+{
+  chpst -P -U <%= node['gitlab']['user']['username'] %> -u <%= node['gitlab']['user']['username'] %> /usr/bin/env HOME="<%= node['gitlab']['user']['home'] %>" /opt/gitlab/embedded/bin/bundle exec unicorn -D -E <%= node['gitlab']['gitlab-rails']['environment'] %> -c <%= File.join(node['gitlab']['gitlab-rails']['dir'], "etc", "unicorn.rb") %> /opt/gitlab/embedded/service/gitlab-rails/config.ru
+}
+
+function trap_signals
+{
+  # Forward all common runit signals except:
+  # - HUP which we handle below;
+  # - KILL which cannot be caught.
+  for sig in STOP CONT ALRM INT QUIT USR1 USR2 TERM; do
+    trap "forward_signal ${sig}" ${sig}
+  done
+
+  # Omnibus-ctl does not have a subcommand that sends USR2 but it can send HUP.
+  # To allow for reloading unicorn from the command line, translate HUP to
+  # USR2.
+  trap "echo 'wrapper received HUP'; forward_signal USR2" HUP
+}
+
+function forward_signal
+{
+  echo "forwarding $1 to unicorn master ${unicorn_pid}"
+  kill -$1 ${unicorn_pid}
+}
+
+function wait_for_unicorn_to_exit
+{
+  while sleep ${unicorn_poll_alive}; do
+    alive ${unicorn_pid} || break
+  done
+}
+
+main
+echo "wrapper for unicorn master ${unicorn_pid} exiting"
...	...	@@ -5,6 +5,7 @@
5	5	- Support changing the name of the "git" user / group (Michael Fenn)
6	6	- Configure omniauth in gitlab.yml
7	7	- Expose more fields under 'extra' in gitlab.yml
	8	+- Zero-downtime Unicorn restarts
8	9
9	10	6.9.0
10	11	- Make SSH port in clone URLs configurable (Julien Pivotto)
...	...
...	...	@@ -131,7 +131,13 @@ sudo gitlab-ctl restart
131	131	It is also possible to start, stop or restart individual components.
132	132
133	133	```shell
134		-sudo gitlab-ctl restart unicorn
	134	+sudo gitlab-ctl restart sidekiq
	135	+```
	136	+
	137	+Unicorn supports zero-downtime reloads. These can be triggered as follows:
	138	+
	139	+```shell
	140	+sudo gitlab-ctl hup unicorn
135	141	```
136	142
137	143	## Configuration
...	...
...	...	@@ -146,6 +146,7 @@ default['gitlab']['unicorn']['worker_processes'] = 2
146	146	default['gitlab']['unicorn']['listen'] = '127.0.0.1'
147	147	default['gitlab']['unicorn']['port'] = 8080
148	148	default['gitlab']['unicorn']['socket'] = '/var/opt/gitlab/gitlab-rails/tmp/sockets/gitlab.socket'
	149	+default['gitlab']['unicorn']['pidfile'] = '/opt/gitlab/var/unicorn/unicorn.pid'
149	150	default['gitlab']['unicorn']['tcp_nopush'] = true
150	151	default['gitlab']['unicorn']['backlog_socket'] = 64
151	152	default['gitlab']['unicorn']['worker_timeout'] = 30
...	...
...	...	@@ -21,12 +21,14 @@ gitlab_rails_etc_dir = File.join(gitlab_rails_dir, "etc")
21	21	gitlab_rails_working_dir = File.join(gitlab_rails_dir, "working")
22	22
23	23	unicorn_listen_socket = node['gitlab']['unicorn']['socket']
	24	+unicorn_pidfile = node['gitlab']['unicorn']['pidfile']
24	25	unicorn_log_dir = node['gitlab']['unicorn']['log_directory']
25	26	unicorn_socket_dir = File.dirname(unicorn_listen_socket)
26	27
27	28	[
28	29	unicorn_log_dir,
29		- unicorn_socket_dir
	30	+ unicorn_socket_dir,
	31	+ File.dirname(unicorn_pidfile)
30	32	].each do \|dir_name\|
31	33	directory dir_name do
32	34	owner node['gitlab']['user']['username']
...	...	@@ -51,6 +53,19 @@ unicorn_config File.join(gitlab_rails_etc_dir, "unicorn.rb") do
51	53	working_directory gitlab_rails_working_dir
52	54	worker_processes node['gitlab']['unicorn']['worker_processes']
53	55	preload_app true
	56	+ stderr_path File.join(unicorn_log_dir, "unicorn_stderr.log")
	57	+ stdout_path File.join(unicorn_log_dir, "unicorn_stdout.log")
	58	+ pid unicorn_pidfile
	59	+ before_fork <<-'EOS'
	60	+ old_pid = "#{server.config[:pid]}.oldbin"
	61	+ if old_pid != server.pid
	62	+ begin
	63	+ sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
	64	+ Process.kill(sig, File.read(old_pid).to_i)
	65	+ rescue Errno::ENOENT, Errno::ESRCH
	66	+ end
	67	+ end
	68	+ EOS
54	69	owner "root"
55	70	group "root"
56	71	mode "0644"
...	...	@@ -59,6 +74,7 @@ end
59	74
60	75	runit_service "unicorn" do
61	76	down node['gitlab']['unicorn']['ha']
	77	+ restart_command 2 # Restart Unicorn using SIGUSR2
62	78	options({
63	79	:log_directory => unicorn_log_dir
64	80	}.merge(params))
...	...
1		-#!/bin/sh
2		-
3		-cd /opt/gitlab/embedded/service/gitlab-rails
	1	+#!/bin/bash
4	2
	3	+# Let runit capture all script error messages
5	4	exec 2>&1
6		-exec chpst -P -U <%= node['gitlab']['user']['username'] %> -u <%= node['gitlab']['user']['username'] %> /usr/bin/env HOME="<%= node['gitlab']['user']['home'] %>" /opt/gitlab/embedded/bin/bundle exec unicorn -E <%= node['gitlab']['gitlab-rails']['environment'] %> -c <%= File.join(node['gitlab']['gitlab-rails']['dir'], "etc", "unicorn.rb") %> /opt/gitlab/embedded/service/gitlab-rails/config.ru
	5	+
	6	+readonly current_pidfile=<%= node['gitlab']['unicorn']['pidfile'] %>
	7	+readonly oldbin_pidfile=${current_pidfile}.oldbin
	8	+readonly unicorn_wait_start=1 # time in seconds
	9	+readonly unicorn_poll_alive=1 # time in seconds
	10	+
	11	+function main
	12	+{
	13	+ cd /opt/gitlab/embedded/service/gitlab-rails
	14	+ find_us_a_unicorn
	15	+ trap_signals
	16	+ wait_for_unicorn_to_exit
	17	+}
	18	+
	19	+function find_us_a_unicorn
	20	+{
	21	+ adopt ${current_pidfile}
	22	+ if [[ ${unicorn_pid} ]]; then
	23	+ echo "adopted existing unicorn master ${unicorn_pid}"
	24	+ return
	25	+ fi
	26	+
	27	+ adopt ${oldbin_pidfile}
	28	+ if [[ ${unicorn_pid} ]]; then
	29	+ echo "adopted existing oldbin unicorn master ${unicorn_pid}"
	30	+ return
	31	+ fi
	32	+
	33	+ echo "starting new unicorn master"
	34	+ start_unicorn_master
	35	+ sleep ${unicorn_wait_start}
	36	+
	37	+ adopt ${current_pidfile}
	38	+ if [[ ${unicorn_pid} ]]; then
	39	+ echo "adopted new unicorn master ${unicorn_pid}"
	40	+ return
	41	+ fi
	42	+
	43	+ echo "failed to start a new unicorn master"
	44	+ exit
	45	+}
	46	+
	47	+function adopt
	48	+{
	49	+ local pid=$(cat $1 2>/dev/null)
	50	+ if alive ${pid} && is_unicorn ${pid}; then
	51	+ readonly unicorn_pid=${pid}
	52	+ fi
	53	+}
	54	+
	55	+function alive
	56	+{
	57	+ kill -0 $1 > /dev/null 2>&1
	58	+}
	59	+
	60	+function is_unicorn
	61	+{
	62	+ ps -p $1 -o args \| grep -q unicorn
	63	+}
	64	+
	65	+function start_unicorn_master
	66	+{
	67	+ chpst -P -U <%= node['gitlab']['user']['username'] %> -u <%= node['gitlab']['user']['username'] %> /usr/bin/env HOME="<%= node['gitlab']['user']['home'] %>" /opt/gitlab/embedded/bin/bundle exec unicorn -D -E <%= node['gitlab']['gitlab-rails']['environment'] %> -c <%= File.join(node['gitlab']['gitlab-rails']['dir'], "etc", "unicorn.rb") %> /opt/gitlab/embedded/service/gitlab-rails/config.ru
	68	+}
	69	+
	70	+function trap_signals
	71	+{
	72	+ # Forward all common runit signals except:
	73	+ # - HUP which we handle below;
	74	+ # - KILL which cannot be caught.
	75	+ for sig in STOP CONT ALRM INT QUIT USR1 USR2 TERM; do
	76	+ trap "forward_signal ${sig}" ${sig}
	77	+ done
	78	+
	79	+ # Omnibus-ctl does not have a subcommand that sends USR2 but it can send HUP.
	80	+ # To allow for reloading unicorn from the command line, translate HUP to
	81	+ # USR2.
	82	+ trap "echo 'wrapper received HUP'; forward_signal USR2" HUP
	83	+}
	84	+
	85	+function forward_signal
	86	+{
	87	+ echo "forwarding $1 to unicorn master ${unicorn_pid}"
	88	+ kill -$1 ${unicorn_pid}
	89	+}
	90	+
	91	+function wait_for_unicorn_to_exit
	92	+{
	93	+ while sleep ${unicorn_poll_alive}; do
	94	+ alive ${unicorn_pid} \|\| break
	95	+ done
	96	+}
	97	+
	98	+main
	99	+echo "wrapper for unicorn master ${unicorn_pid} exiting"
...	...