Updated the init script, now waits for pids

Rovanion Luckey · GitLab
1 parent c465636a
Showing 2 changed files with 67 additions and 39 deletions Show diff stats
CHANGELOG
lib/support/init.d/gitlab
+v 6.3.0
+  - Init script now waits for pids to appear after (re)starting before reporting status (Rovanion Luckey)
 v 6.2.0
   - Public project pages are now visible to everyone (files, issues, wik, etc.)
     THIS MEANS YOUR ISSUES AND WIKI FOR PUBLIC PROJECTS ARE PUBLICLY VISIBLE AFTER THE UPGRADE
@@ -20,8 +20,8 @@ RAILS_ENV=&quot;production&quot;
 # Script variable names should be lower-case not to conflict with internal
 # /bin/sh variables such as PATH, EDITOR or SHELL.
-app_root="/home/git/gitlab"
 app_user="git"
+app_root="/home/$app_user/gitlab"
 pid_path="$app_root/tmp/pids"
 socket_path="$app_root/tmp/sockets"
 web_server_pid_path="$pid_path/unicorn.pid"
@@ -44,6 +44,7 @@ fi
 ### Init Script functions
+## Gets the pids from the files
 check_pids(){
   if ! mkdir -p "$pid_path"; then
     echo "Could not create the path $pid_path needed to store the pids."
@@ -62,12 +63,29 @@ check_pids(){
   fi
 }
+## Called when we have started the two processes and are waiting for their pid files.
+wait_for_pids(){
+  # We are sleeping a bit here mostly because sidekiq is slow at writing it's pid
+  i=0;
+  while [ ! -f $web_server_pid_path -o ! -f $sidekiq_pid_path ]; do
+    sleep 0.1;
+    i=$((i+1))
+    if [ $((i%10)) = 0 ]; then
+      echo -n "."
+    elif [ $((i)) = 301 ]; then
+      echo "Waited 30s for the processes to write their pids, something probably went wrong."
+      exit 1;
+    fi
+  done
+  echo
+}
+
 # We use the pids in so many parts of the script it makes sense to always check them.
 # Only after start() is run should the pids change. Sidekiq sets it's own pid.
 check_pids
-# Checks whether the different parts of the service are already running or not.
+## Checks whether the different parts of the service are already running or not.
 check_status(){
   check_pids
   # If the web server is running kill -0 $wpid returns true, or rather 0.
@@ -86,7 +104,7 @@ check_status(){
   fi
 }
-# Check for stale pids and remove them if necessary
+## Check for stale pids and remove them if necessary.
 check_stale_pids(){
   check_status
   # If there is a pid it is something else than 0, the service is running if
@@ -94,7 +112,7 @@ check_stale_pids(){
   if [ "$wpid" != "0" -a "$web_status" != "0" ]; then
     echo "Removing stale Unicorn web server pid. This is most likely caused by the web server crashing the last time it ran."
     if ! rm "$web_server_pid_path"; then
-      echo "Unable to remove stale pid, exiting"
+      echo "Unable to remove stale pid, exiting."
       exit 1
     fi
   fi
@@ -107,7 +125,7 @@ check_stale_pids(){
   fi
 }
-# If no parts of the service is running, bail out.
+## If no parts of the service is running, bail out.
 exit_if_not_running(){
   check_stale_pids
   if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then
@@ -116,86 +134,92 @@ exit_if_not_running(){
   fi
 }
-# Starts Unicorn and Sidekiq.
+## Starts Unicorn and Sidekiq if they're not running.
 start() {
   check_stale_pids
+  if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then
+    echo -n "Starting both the GitLab Unicorn and Sidekiq"
+  elif [ "$web_status" != "0" ]; then
+    echo -n "Starting GitLab Sidekiq"
+  elif [ "$sidekiq_status" != "0" ]; then
+    echo -n "Starting GitLab Unicorn"
+  fi
+
   # Then check if the service is running. If it is: don't start again.
   if [ "$web_status" = "0" ]; then
     echo "The Unicorn web server already running with pid $wpid, not restarting."
   else
-    echo "Starting the GitLab Unicorn web server..."
     # Remove old socket if it exists
     rm -f "$socket_path"/gitlab.socket 2>/dev/null
-    # Start the webserver
-    RAILS_ENV=$RAILS_ENV script/web start
+    # Start the web server
+    RAILS_ENV=$RAILS_ENV script/web start &
   fi
   # If sidekiq is already running, don't start it again.
   if [ "$sidekiq_status" = "0" ]; then
     echo "The Sidekiq job dispatcher is already running with pid $spid, not restarting"
   else
-    echo "Starting the GitLab Sidekiq event dispatcher..."
-    RAILS_ENV=$RAILS_ENV script/background_jobs start
-    # We are sleeping a bit here because sidekiq is slow at writing it's pid
-    sleep 2
+    RAILS_ENV=$RAILS_ENV script/background_jobs start &
   fi
+  # Wait for the pids to be planted
+  wait_for_pids
   # Finally check the status to tell wether or not GitLab is running
-  status
+  print_status
 }
-# Asks the Unicorn and the Sidekiq if they would be so kind as to stop, if not kills them.
+## Asks the Unicorn and the Sidekiq if they would be so kind as to stop, if not kills them.
 stop() {
   exit_if_not_running
+
+  if [ "$web_status" = "0" -a "$sidekiq_status" = "0" ]; then
+    echo -n "Shutting down both Unicorn and Sidekiq"
+  elif [ "$web_status" = "0" ]; then
+    echo -n "Shutting down Sidekiq"
+  elif [ "$sidekiq_status" = "0" ]; then
+    echo -n "Shutting down Unicorn"
+  fi
+
   # If the Unicorn web server is running, tell it to stop;
   if [ "$web_status" = "0" ]; then
-    RAILS_ENV=$RAILS_ENV script/web stop
-    echo "Stopping the GitLab Unicorn web server..."
-    stopping=true
-  else
-    echo "The Unicorn web was not running, doing nothing."
+     RAILS_ENV=$RAILS_ENV script/web stop
   fi
   # And do the same thing for the Sidekiq.
   if [ "$sidekiq_status" = "0" ]; then
-    printf "Stopping Sidekiq job dispatcher."
     RAILS_ENV=$RAILS_ENV script/background_jobs stop
-    stopping=true
-  else
-    echo "The Sidekiq was not running, must have run out of breath."
   fi
-
   # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script.
-  while [ "$stopping" = "true" ]; do
+  while [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; do
     sleep 1
     check_status
-    if [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; then
-      printf "."
-    else
+    printf "."
+    if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then
       printf "\n"
       break
     fi
   done
+
   sleep 1
   # Cleaning up unused pids
   rm "$web_server_pid_path" 2>/dev/null
   # rm "$sidekiq_pid_path" # Sidekiq seems to be cleaning up it's own pid.
-  status
+  print_status
 }
-# Returns the status of GitLab and it's components
-status() {
+## Prints the status of GitLab and it's components.
+print_status() {
   check_status
   if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then
     echo "GitLab is not running."
     return
   fi
   if [ "$web_status" = "0" ]; then
-      echo "The GitLab Unicorn webserver with pid $wpid is running."
+      echo "The GitLab Unicorn web server with pid $wpid is running."
   else
-      printf "The GitLab Unicorn webserver is server is \033[31mnot running\033[0m.\n"33[31mnot runningserver is \033[31mnot running\033[0m.\n"33[0m.\n"
+      printf "The GitLab Unicorn web server is server is \033[31mnot running\033[0m.\n"33[31mnot runningserver is \033[31mnot running\033[0m.\n"33[0m.\n"
   fi
   if [ "$sidekiq_status" = "0" ]; then
       echo "The GitLab Sidekiq job dispatcher with pid $spid is running."
@@ -207,6 +231,7 @@ status() {
   fi
 }
+## Tells unicorn to reload it's config and Sidekiq to restart
 reload(){
   exit_if_not_running
   if [ "$wpid" = "0" ];then
@@ -218,11 +243,12 @@ reload(){
   echo "Done."
   echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..."
   RAILS_ENV=$RAILS_ENV script/background_jobs restart
-  # Waiting 2 seconds for sidekiq to write it.
-  sleep 2
-  status
+
+  wait_for_pids
+  print_status
 }
+## Restarts Sidekiq and Unicorn.
 restart(){
   check_status
   if [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; then
@@ -232,7 +258,7 @@ restart(){
 }
-## Finally the input handling.
+### Finally the input handling.
 case "$1" in
   start)
@@ -248,7 +274,7 @@ case &quot;$1&quot; in
 	reload
         ;;
   status)
-        status
+        print_status
         ;;
   *)
         echo "Usage: service gitlab {start|stop|restart|reload|status}"
		1	+v 6.3.0
		2	+ - Init script now waits for pids to appear after (re)starting before reporting status (Rovanion Luckey)
1	v 6.2.0	3	v 6.2.0
2	- Public project pages are now visible to everyone (files, issues, wik, etc.)	4	- Public project pages are now visible to everyone (files, issues, wik, etc.)
3	THIS MEANS YOUR ISSUES AND WIKI FOR PUBLIC PROJECTS ARE PUBLICLY VISIBLE AFTER THE UPGRADE	5	THIS MEANS YOUR ISSUES AND WIKI FOR PUBLIC PROJECTS ARE PUBLICLY VISIBLE AFTER THE UPGRADE
	@@ -20,8 +20,8 @@ RAILS_ENV="production"		@@ -20,8 +20,8 @@ RAILS_ENV="production"
20		20
21	# Script variable names should be lower-case not to conflict with internal	21	# Script variable names should be lower-case not to conflict with internal
22	# /bin/sh variables such as PATH, EDITOR or SHELL.	22	# /bin/sh variables such as PATH, EDITOR or SHELL.
23	-app_root="/home/git/gitlab"
24	app_user="git"	23	app_user="git"
		24	+app_root="/home/$app_user/gitlab"
25	pid_path="$app_root/tmp/pids"	25	pid_path="$app_root/tmp/pids"
26	socket_path="$app_root/tmp/sockets"	26	socket_path="$app_root/tmp/sockets"
27	web_server_pid_path="$pid_path/unicorn.pid"	27	web_server_pid_path="$pid_path/unicorn.pid"
	@@ -44,6 +44,7 @@ fi		@@ -44,6 +44,7 @@ fi
44		44
45	### Init Script functions	45	### Init Script functions
46		46
		47	+## Gets the pids from the files
47	check_pids(){	48	check_pids(){
48	if ! mkdir -p "$pid_path"; then	49	if ! mkdir -p "$pid_path"; then
49	echo "Could not create the path $pid_path needed to store the pids."	50	echo "Could not create the path $pid_path needed to store the pids."
	@@ -62,12 +63,29 @@ check_pids(){		@@ -62,12 +63,29 @@ check_pids(){
62	fi	63	fi
63	}	64	}
64		65
		66	+## Called when we have started the two processes and are waiting for their pid files.
		67	+wait_for_pids(){
		68	+ # We are sleeping a bit here mostly because sidekiq is slow at writing it's pid
		69	+ i=0;
		70	+ while [ ! -f $web_server_pid_path -o ! -f $sidekiq_pid_path ]; do
		71	+ sleep 0.1;
		72	+ i=$((i+1))
		73	+ if [ $((i%10)) = 0 ]; then
		74	+ echo -n "."
		75	+ elif [ $((i)) = 301 ]; then
		76	+ echo "Waited 30s for the processes to write their pids, something probably went wrong."
		77	+ exit 1;
		78	+ fi
		79	+ done
		80	+ echo
		81	+}
		82	+
65	# We use the pids in so many parts of the script it makes sense to always check them.	83	# We use the pids in so many parts of the script it makes sense to always check them.
66	# Only after start() is run should the pids change. Sidekiq sets it's own pid.	84	# Only after start() is run should the pids change. Sidekiq sets it's own pid.
67	check_pids	85	check_pids
68		86
69		87
70	-# Checks whether the different parts of the service are already running or not.	88	+## Checks whether the different parts of the service are already running or not.
71	check_status(){	89	check_status(){
72	check_pids	90	check_pids
73	# If the web server is running kill -0 $wpid returns true, or rather 0.	91	# If the web server is running kill -0 $wpid returns true, or rather 0.
	@@ -86,7 +104,7 @@ check_status(){		@@ -86,7 +104,7 @@ check_status(){
86	fi	104	fi
87	}	105	}
88		106
89	-# Check for stale pids and remove them if necessary	107	+## Check for stale pids and remove them if necessary.
90	check_stale_pids(){	108	check_stale_pids(){
91	check_status	109	check_status
92	# If there is a pid it is something else than 0, the service is running if	110	# If there is a pid it is something else than 0, the service is running if
	@@ -94,7 +112,7 @@ check_stale_pids(){		@@ -94,7 +112,7 @@ check_stale_pids(){
94	if [ "$wpid" != "0" -a "$web_status" != "0" ]; then	112	if [ "$wpid" != "0" -a "$web_status" != "0" ]; then
95	echo "Removing stale Unicorn web server pid. This is most likely caused by the web server crashing the last time it ran."	113	echo "Removing stale Unicorn web server pid. This is most likely caused by the web server crashing the last time it ran."
96	if ! rm "$web_server_pid_path"; then	114	if ! rm "$web_server_pid_path"; then
97	- echo "Unable to remove stale pid, exiting"	115	+ echo "Unable to remove stale pid, exiting."
98	exit 1	116	exit 1
99	fi	117	fi
100	fi	118	fi
	@@ -107,7 +125,7 @@ check_stale_pids(){		@@ -107,7 +125,7 @@ check_stale_pids(){
107	fi	125	fi
108	}	126	}
109		127
110	-# If no parts of the service is running, bail out.	128	+## If no parts of the service is running, bail out.
111	exit_if_not_running(){	129	exit_if_not_running(){
112	check_stale_pids	130	check_stale_pids
113	if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then	131	if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then
	@@ -116,86 +134,92 @@ exit_if_not_running(){		@@ -116,86 +134,92 @@ exit_if_not_running(){
116	fi	134	fi
117	}	135	}
118		136
119	-# Starts Unicorn and Sidekiq.	137	+## Starts Unicorn and Sidekiq if they're not running.
120	start() {	138	start() {
121	check_stale_pids	139	check_stale_pids
122		140
		141	+ if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then
		142	+ echo -n "Starting both the GitLab Unicorn and Sidekiq"
		143	+ elif [ "$web_status" != "0" ]; then
		144	+ echo -n "Starting GitLab Sidekiq"
		145	+ elif [ "$sidekiq_status" != "0" ]; then
		146	+ echo -n "Starting GitLab Unicorn"
		147	+ fi
		148	+
123	# Then check if the service is running. If it is: don't start again.	149	# Then check if the service is running. If it is: don't start again.
124	if [ "$web_status" = "0" ]; then	150	if [ "$web_status" = "0" ]; then
125	echo "The Unicorn web server already running with pid $wpid, not restarting."	151	echo "The Unicorn web server already running with pid $wpid, not restarting."
126	else	152	else
127	- echo "Starting the GitLab Unicorn web server..."
128	# Remove old socket if it exists	153	# Remove old socket if it exists
129	rm -f "$socket_path"/gitlab.socket 2>/dev/null	154	rm -f "$socket_path"/gitlab.socket 2>/dev/null
130	- # Start the webserver
131	- RAILS_ENV=$RAILS_ENV script/web start	155	+ # Start the web server
		156	+ RAILS_ENV=$RAILS_ENV script/web start &
132	fi	157	fi
133		158
134	# If sidekiq is already running, don't start it again.	159	# If sidekiq is already running, don't start it again.
135	if [ "$sidekiq_status" = "0" ]; then	160	if [ "$sidekiq_status" = "0" ]; then
136	echo "The Sidekiq job dispatcher is already running with pid $spid, not restarting"	161	echo "The Sidekiq job dispatcher is already running with pid $spid, not restarting"
137	else	162	else
138	- echo "Starting the GitLab Sidekiq event dispatcher..."
139	- RAILS_ENV=$RAILS_ENV script/background_jobs start
140	- # We are sleeping a bit here because sidekiq is slow at writing it's pid
141	- sleep 2	163	+ RAILS_ENV=$RAILS_ENV script/background_jobs start &
142	fi	164	fi
143		165
		166	+ # Wait for the pids to be planted
		167	+ wait_for_pids
144	# Finally check the status to tell wether or not GitLab is running	168	# Finally check the status to tell wether or not GitLab is running
145	- status	169	+ print_status
146	}	170	}
147		171
148	-# Asks the Unicorn and the Sidekiq if they would be so kind as to stop, if not kills them.	172	+## Asks the Unicorn and the Sidekiq if they would be so kind as to stop, if not kills them.
149	stop() {	173	stop() {
150	exit_if_not_running	174	exit_if_not_running
		175	+
		176	+ if [ "$web_status" = "0" -a "$sidekiq_status" = "0" ]; then
		177	+ echo -n "Shutting down both Unicorn and Sidekiq"
		178	+ elif [ "$web_status" = "0" ]; then
		179	+ echo -n "Shutting down Sidekiq"
		180	+ elif [ "$sidekiq_status" = "0" ]; then
		181	+ echo -n "Shutting down Unicorn"
		182	+ fi
		183	+
151	# If the Unicorn web server is running, tell it to stop;	184	# If the Unicorn web server is running, tell it to stop;
152	if [ "$web_status" = "0" ]; then	185	if [ "$web_status" = "0" ]; then
153	- RAILS_ENV=$RAILS_ENV script/web stop
154	- echo "Stopping the GitLab Unicorn web server..."
155	- stopping=true
156	- else
157	- echo "The Unicorn web was not running, doing nothing."	186	+ RAILS_ENV=$RAILS_ENV script/web stop
158	fi	187	fi
159	# And do the same thing for the Sidekiq.	188	# And do the same thing for the Sidekiq.
160	if [ "$sidekiq_status" = "0" ]; then	189	if [ "$sidekiq_status" = "0" ]; then
161	- printf "Stopping Sidekiq job dispatcher."
162	RAILS_ENV=$RAILS_ENV script/background_jobs stop	190	RAILS_ENV=$RAILS_ENV script/background_jobs stop
163	- stopping=true
164	- else
165	- echo "The Sidekiq was not running, must have run out of breath."
166	fi	191	fi
167		192
168	-
169	# If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script.	193	# If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script.
170	- while [ "$stopping" = "true" ]; do	194	+ while [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; do
171	sleep 1	195	sleep 1
172	check_status	196	check_status
173	- if [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; then
174	- printf "."
175	- else	197	+ printf "."
		198	+ if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then
176	printf "\n"	199	printf "\n"
177	break	200	break
178	fi	201	fi
179	done	202	done
		203	+
180	sleep 1	204	sleep 1
181	# Cleaning up unused pids	205	# Cleaning up unused pids
182	rm "$web_server_pid_path" 2>/dev/null	206	rm "$web_server_pid_path" 2>/dev/null
183	# rm "$sidekiq_pid_path" # Sidekiq seems to be cleaning up it's own pid.	207	# rm "$sidekiq_pid_path" # Sidekiq seems to be cleaning up it's own pid.
184		208
185	- status	209	+ print_status
186	}	210	}
187		211
188	-# Returns the status of GitLab and it's components
189	-status() {	212	+## Prints the status of GitLab and it's components.
		213	+print_status() {
190	check_status	214	check_status
191	if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then	215	if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then
192	echo "GitLab is not running."	216	echo "GitLab is not running."
193	return	217	return
194	fi	218	fi
195	if [ "$web_status" = "0" ]; then	219	if [ "$web_status" = "0" ]; then
196	- echo "The GitLab Unicorn webserver with pid $wpid is running."	220	+ echo "The GitLab Unicorn web server with pid $wpid is running."
197	else	221	else
198	- printf "The GitLab Unicorn webserver is server is \033[31mnot running\033[0m.\n"33[31mnot runningserver is \033[31mnot running\033[0m.\n"33[0m.\n"	222	+ printf "The GitLab Unicorn web server is server is \033[31mnot running\033[0m.\n"33[31mnot runningserver is \033[31mnot running\033[0m.\n"33[0m.\n"
199	fi	223	fi
200	if [ "$sidekiq_status" = "0" ]; then	224	if [ "$sidekiq_status" = "0" ]; then
201	echo "The GitLab Sidekiq job dispatcher with pid $spid is running."	225	echo "The GitLab Sidekiq job dispatcher with pid $spid is running."
	@@ -207,6 +231,7 @@ status() {		@@ -207,6 +231,7 @@ status() {
207	fi	231	fi
208	}	232	}
209		233
		234	+## Tells unicorn to reload it's config and Sidekiq to restart
210	reload(){	235	reload(){
211	exit_if_not_running	236	exit_if_not_running
212	if [ "$wpid" = "0" ];then	237	if [ "$wpid" = "0" ];then
	@@ -218,11 +243,12 @@ reload(){		@@ -218,11 +243,12 @@ reload(){
218	echo "Done."	243	echo "Done."
219	echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..."	244	echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..."
220	RAILS_ENV=$RAILS_ENV script/background_jobs restart	245	RAILS_ENV=$RAILS_ENV script/background_jobs restart
221	- # Waiting 2 seconds for sidekiq to write it.
222	- sleep 2
223	- status	246	+
		247	+ wait_for_pids
		248	+ print_status
224	}	249	}
225		250
		251	+## Restarts Sidekiq and Unicorn.
226	restart(){	252	restart(){
227	check_status	253	check_status
228	if [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; then	254	if [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; then
	@@ -232,7 +258,7 @@ restart(){		@@ -232,7 +258,7 @@ restart(){
232	}	258	}
233		259
234		260
235	-## Finally the input handling.	261	+### Finally the input handling.
236		262
237	case "$1" in	263	case "$1" in
238	start)	264	start)
	@@ -248,7 +274,7 @@ case "$1" in		@@ -248,7 +274,7 @@ case "$1" in
248	reload	274	reload
249	;;	275	;;
250	status)	276	status)
251	- status	277	+ print_status
252	;;	278	;;
253	*)	279	*)
254	echo "Usage: service gitlab {start\|stop\|restart\|reload\|status}"	280	echo "Usage: service gitlab {start\|stop\|restart\|reload\|status}"