sidekiq

Dmitriy Zaporozhets
1 parent 7bfd38ea
Showing 14 changed files with 75 additions and 94 deletions Show diff stats
Gemfile
Gemfile.lock
Procfile
app/controllers/admin/dashboard_controller.rb
app/mailers/notify.rb
app/models/system_hook.rb
app/views/admin/dashboard/index.html.haml
app/views/admin/resque/show.html.haml
app/views/layouts/admin.html.haml
app/workers/post_receive.rb
app/workers/system_hook_worker.rb
config/initializers/4_resque.rb
config/routes.rb
lib/tasks/resque.rake
@@ -81,8 +81,10 @@ gem &quot;acts-as-taggable-on&quot;, &quot;2.3.3&quot;
 gem "draper", "~> 0.18.0"
  
 # Background jobs
-gem "resque", git: "https://github.com/gitlabhq/resque.git", ref: "9ef4700306dd946a3ac000612428967ce0c32213"
-gem 'resque_mailer'
+gem 'slim'
+gem 'sinatra', :require => nil
+gem 'sidekiq', '2.6.4'
+gem 'sidekiq_mailer'
  
 # HTTP requests
 gem "httparty"
@@ -157,7 +159,6 @@ group :test do
   gem "simplecov", require: false
   gem "shoulda-matchers", "1.3.0"
   gem 'email_spec'
-  gem 'resque_spec'
   gem "webmock"
   gem 'test_after_commit'
 end
@@ -55,17 +55,6 @@ GIT
     raphael-rails (2.1.0)
  
 GIT
-  remote: https://github.com/gitlabhq/resque.git
-  revision: 9ef4700306dd946a3ac000612428967ce0c32213
-  ref: 9ef4700306dd946a3ac000612428967ce0c32213
-  specs:
-    resque (2.0.0.pre.1)
-      json
-      redis-namespace (~> 1.0)
-      sinatra (>= 0.9.2)
-      vegas (~> 0.1.2)
-
-GIT
   remote: https://github.com/jonleighton/poltergeist.git
   revision: 5c2e092001074a8cf09f332d3714e9ba150bc8ca
   ref: 5c2e092001074a8cf09f332d3714e9ba150bc8ca
@@ -132,6 +121,9 @@ GEM
     carrierwave (0.7.1)
       activemodel (>= 3.2.0)
       activesupport (>= 3.2.0)
+    celluloid (0.12.4)
+      facter (>= 1.6.12)
+      timers (>= 1.0.0)
     charlock_holmes (0.6.9)
     childprocess (0.3.6)
       ffi (~> 1.0, >= 1.0.6)
@@ -150,6 +142,7 @@ GEM
     coffee-script-source (1.4.0)
     colored (1.2)
     colorize (0.5.8)
+    connection_pool (1.0.0)
     crack (0.3.1)
     daemons (1.1.9)
     devise (2.1.2)
@@ -169,6 +162,7 @@ GEM
     eventmachine (1.0.0)
     execjs (1.4.0)
       multi_json (~> 1.0)
+    facter (1.6.17)
     factory_girl (4.1.0)
       activesupport (>= 3.0.0)
     factory_girl_rails (4.1.0)
@@ -370,11 +364,6 @@ GEM
     redis (3.0.2)
     redis-namespace (1.2.1)
       redis (~> 3.0.0)
-    resque_mailer (2.1.0)
-      actionmailer (~> 3.0)
-    resque_spec (0.12.5)
-      resque (>= 1.19.0)
-      rspec (>= 2.5.0)
     rspec (2.12.0)
       rspec-core (~> 2.12.0)
       rspec-expectations (~> 2.12.0)
@@ -412,6 +401,16 @@ GEM
     sexp_processor (4.1.3)
     shoulda-matchers (1.3.0)
       activesupport (>= 3.0.0)
+    sidekiq (2.6.4)
+      celluloid (~> 0.12.0)
+      connection_pool (~> 1.0)
+      multi_json (~> 1)
+      redis (~> 3)
+      redis-namespace
+    sidekiq_mailer (0.0.4)
+      actionmailer (~> 3.0)
+      activesupport (~> 3.0)
+      sidekiq (~> 2.3)
     simplecov (0.7.1)
       multi_json (~> 1.0)
       simplecov-html (~> 0.7.1)
@@ -421,6 +420,9 @@ GEM
       rack-protection (~> 1.2)
       tilt (~> 1.3, >= 1.3.3)
     six (0.2.0)
+    slim (1.3.6)
+      temple (~> 0.5.5)
+      tilt (~> 1.3.3)
     slop (3.3.3)
     spinach (0.5.2)
       colorize
@@ -435,6 +437,7 @@ GEM
       rack (~> 1.0)
       tilt (~> 1.1, != 1.3.0)
     stamp (0.3.0)
+    temple (0.5.5)
     test_after_commit (0.0.1)
     therubyracer (0.10.2)
       libv8 (~> 3.3.10)
@@ -444,6 +447,7 @@ GEM
       rack (>= 1.0.0)
     thor (0.16.0)
     tilt (1.3.3)
+    timers (1.0.2)
     treetop (1.4.12)
       polyglot
       polyglot (>= 0.3.1)
@@ -455,8 +459,6 @@ GEM
       kgio (~> 2.6)
       rack
       raindrops (~> 0.7)
-    vegas (0.1.11)
-      rack (>= 1.0.0)
     virtus (0.5.2)
       backports (~> 2.6.1)
     warden (1.2.1)
@@ -534,17 +536,18 @@ DEPENDENCIES
   rb-fsevent
   rb-inotify
   redcarpet (~> 2.2.2)
-  resque!
-  resque_mailer
-  resque_spec
   rspec-rails
   sass-rails (~> 3.2.5)
   sdoc
   seed-fu
   settingslogic
   shoulda-matchers (= 1.3.0)
+  sidekiq (= 2.6.4)
+  sidekiq_mailer
   simplecov
+  sinatra
   six
+  slim
   spinach-rails
   stamp
   test_after_commit
 web: bundle exec rails s -p $PORT
-worker: bundle exec rake environment resque:work QUEUE=* VVERBOSE=1
+worker: bundle exec sidekiq -q post_receive,mailer,system_hook,common
@@ -3,10 +3,6 @@ class Admin::DashboardController &lt; AdminController
     @projects = Project.order("created_at DESC").limit(10)
     @users = User.order("created_at DESC").limit(10)
  
-    @resque_accessible = true
-    @workers = Resque.workers
-    @pending_jobs = Resque.size(:post_receive)
-
   rescue Redis::InheritedError
     @resque_accessible = false
   end
 class Notify < ActionMailer::Base
-  include Resque::Mailer
+  include Sidekiq::Mailer
   add_template_helper ApplicationHelper
   add_template_helper GitlabMarkdownHelper
  
@@ -19,6 +19,6 @@ class SystemHook &lt; WebHook
   end
  
   def async_execute(data)
-    Resque.enqueue(SystemHookWorker, id, data)
+    Sidekiq::Client.enqueue(SystemHookWorker, id, data)
   end
 end
 .admin_dash.row
-  .span3
+  .span4
     .ui-box
       %h5.title Projects
       .data.padded
@@ -7,7 +7,7 @@
           %h1= Project.count
         %hr
         = link_to 'New Project', new_project_path, class: "btn small"
-  .span3
+  .span4
     .ui-box
       %h5.title Groups
       .data.padded
@@ -15,7 +15,7 @@
           %h1= Group.count
         %hr
         = link_to 'New Group', new_admin_group_path, class: "btn small"
-  .span3
+  .span4
     .ui-box
       %h5.title Users
       .data.padded
@@ -23,25 +23,6 @@
           %h1= User.count
         %hr
         = link_to 'New User', new_admin_user_path, class: "btn small"
-  .span3
-    .ui-box
-      %h5.title
-        Resque Workers
-      .data.padded
-        - if @resque_accessible
-          = link_to admin_resque_path do
-            %h1{class: @workers.present? ? "cgreen" : "cred"}
-              = @workers.count
-          %hr
-          %p
-            %strong{class: @pending_jobs > 0 ? "cred" : "cgreen"}
-              #{@pending_jobs} post receive jobs waiting
-        - else
-          = link_to admin_resque_path do
-            %h1.cdark ?
-          %hr
-          %p
-            %strong Resque status unknown
  
 .row
   .span6
-%h3.page_title Resque
+%h3.page_title Background Jobs
 %br
 .ui-box
-  %iframe{src: resque_path, width: '100%', height: 600, style: "border: none"}
+  %iframe{src: sidekiq_path, width: '100%', height: 900, style: "border: none"}
@@ -19,6 +19,6 @@
         = nav_link(controller: :hooks) do
           = link_to "Hooks", admin_hooks_path
         = nav_link(controller: :resque) do
-          = link_to "Resque", admin_resque_path
+          = link_to "Background Jobs", admin_resque_path
  
       .content= yield
 class PostReceive
-  @queue = :post_receive
+  include Sidekiq::Worker
  
-  def self.perform(repo_path, oldrev, newrev, ref, identifier)
+  sidekiq_options queue: :post_receive
+
+  def perform(repo_path, oldrev, newrev, ref, identifier)
     repo_path.gsub!(Gitlab.config.gitolite.repos_path.to_s, "")
     repo_path.gsub!(/.git$/, "")
     repo_path.gsub!(/^\//, "")
 class SystemHookWorker
-  @queue = :system_hook
+  include Sidekiq::Worker
  
-  def self.perform(hook_id, data)
+  sidekiq_options queue: :system_hook
+
+  def perform(hook_id, data)
     SystemHook.find(hook_id).execute data
   end
 end
 # Custom Redis configuration
 config_file = Rails.root.join('config', 'resque.yml')
  
-if File.exists?(config_file)
-  resque_config = YAML.load_file(config_file)
-  Resque.redis = resque_config[Rails.env]
-end
-Resque.redis.namespace = 'resque:gitlab'
-Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection }
-
-# Authentication
-require 'resque/server'
-class ResqueAuthentication
-  def initialize(app)
-    @app = app
-  end
+resque_url = if File.exists?(config_file)
+               YAML.load_file(config_file)[Rails.env]
+             else
+               "localhost:6379"
+             end
  
-  def call(env)
-    account = env['warden'].authenticate!(:database_authenticatable, :rememberable, scope: :user)
-    raise "Access denied" if !account.admin?
-    @app.call(env)
-  end
+Sidekiq.configure_server do |config|
+  config.redis = {
+    url: "redis://#{resque_url}",
+    namespace: 'resque:gitlab'
+  }
 end
  
-Resque::Server.use ResqueAuthentication
-
-# Mailer
-Resque::Mailer.excluded_environments = []
-
+Sidekiq.configure_client do |config|
+  config.redis = {
+    url: "redis://#{resque_url}",
+    namespace: 'resque:gitlab'
+  }
+end
+require 'sidekiq/web'
+
 Gitlab::Application.routes.draw do
   #
   # Search
@@ -8,9 +10,10 @@ Gitlab::Application.routes.draw do
   require 'api'
   mount Gitlab::API => '/api'
  
-  # Optionally, enable Resque here
-  require 'resque/server'
-  mount Resque::Server => '/info/resque', as: 'resque'
+  constraint = lambda { |request| request.env["warden"].authenticate? and request.env['warden'].user.admin? }
+  constraints constraint do
+    mount Sidekiq::Web, at: "/admin/workers", as: :sidekiq
+  end
  
   # Enable Grack support
   mount Grack::Bundle.new({
@@ -2,20 +2,20 @@ require &#39;resque/tasks&#39;
  
 namespace :resque do
   task setup: :environment do
-    Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection }
+    #Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection }
   end
  
   desc "Resque | kill all workers (using -QUIT), god will take care of them"
   task :stop_workers => :environment do
-    pids = Array.new
+    #pids = Array.new
  
-    Resque.workers.each do |worker|
-      pids << worker.to_s.split(/:/).second
-    end
+    #Resque.workers.each do |worker|
+      #pids << worker.to_s.split(/:/).second
+    #end
  
-    if pids.size > 0
-      system("kill -QUIT #{pids.join(' ')}")
-    end
+    #if pids.size > 0
+      #system("kill -QUIT #{pids.join(' ')}")
+    #end
   end
 end
...	...	@@ -81,8 +81,10 @@ gem "acts-as-taggable-on", "2.3.3"
81	81	gem "draper", "~> 0.18.0"
82	82
83	83	# Background jobs
84		-gem "resque", git: "https://github.com/gitlabhq/resque.git", ref: "9ef4700306dd946a3ac000612428967ce0c32213"
85		-gem 'resque_mailer'
	84	+gem 'slim'
	85	+gem 'sinatra', :require => nil
	86	+gem 'sidekiq', '2.6.4'
	87	+gem 'sidekiq_mailer'
86	88
87	89	# HTTP requests
88	90	gem "httparty"
...	...	@@ -157,7 +159,6 @@ group :test do
157	159	gem "simplecov", require: false
158	160	gem "shoulda-matchers", "1.3.0"
159	161	gem 'email_spec'
160		- gem 'resque_spec'
161	162	gem "webmock"
162	163	gem 'test_after_commit'
163	164	end
...	...
...	...	@@ -55,17 +55,6 @@ GIT
55	55	raphael-rails (2.1.0)
56	56
57	57	GIT
58		- remote: https://github.com/gitlabhq/resque.git
59		- revision: 9ef4700306dd946a3ac000612428967ce0c32213
60		- ref: 9ef4700306dd946a3ac000612428967ce0c32213
61		- specs:
62		- resque (2.0.0.pre.1)
63		- json
64		- redis-namespace (~> 1.0)
65		- sinatra (>= 0.9.2)
66		- vegas (~> 0.1.2)
67		-
68		-GIT
69	58	remote: https://github.com/jonleighton/poltergeist.git
70	59	revision: 5c2e092001074a8cf09f332d3714e9ba150bc8ca
71	60	ref: 5c2e092001074a8cf09f332d3714e9ba150bc8ca
...	...	@@ -132,6 +121,9 @@ GEM
132	121	carrierwave (0.7.1)
133	122	activemodel (>= 3.2.0)
134	123	activesupport (>= 3.2.0)
	124	+ celluloid (0.12.4)
	125	+ facter (>= 1.6.12)
	126	+ timers (>= 1.0.0)
135	127	charlock_holmes (0.6.9)
136	128	childprocess (0.3.6)
137	129	ffi (~> 1.0, >= 1.0.6)
...	...	@@ -150,6 +142,7 @@ GEM
150	142	coffee-script-source (1.4.0)
151	143	colored (1.2)
152	144	colorize (0.5.8)
	145	+ connection_pool (1.0.0)
153	146	crack (0.3.1)
154	147	daemons (1.1.9)
155	148	devise (2.1.2)
...	...	@@ -169,6 +162,7 @@ GEM
169	162	eventmachine (1.0.0)
170	163	execjs (1.4.0)
171	164	multi_json (~> 1.0)
	165	+ facter (1.6.17)
172	166	factory_girl (4.1.0)
173	167	activesupport (>= 3.0.0)
174	168	factory_girl_rails (4.1.0)
...	...	@@ -370,11 +364,6 @@ GEM
370	364	redis (3.0.2)
371	365	redis-namespace (1.2.1)
372	366	redis (~> 3.0.0)
373		- resque_mailer (2.1.0)
374		- actionmailer (~> 3.0)
375		- resque_spec (0.12.5)
376		- resque (>= 1.19.0)
377		- rspec (>= 2.5.0)
378	367	rspec (2.12.0)
379	368	rspec-core (~> 2.12.0)
380	369	rspec-expectations (~> 2.12.0)
...	...	@@ -412,6 +401,16 @@ GEM
412	401	sexp_processor (4.1.3)
413	402	shoulda-matchers (1.3.0)
414	403	activesupport (>= 3.0.0)
	404	+ sidekiq (2.6.4)
	405	+ celluloid (~> 0.12.0)
	406	+ connection_pool (~> 1.0)
	407	+ multi_json (~> 1)
	408	+ redis (~> 3)
	409	+ redis-namespace
	410	+ sidekiq_mailer (0.0.4)
	411	+ actionmailer (~> 3.0)
	412	+ activesupport (~> 3.0)
	413	+ sidekiq (~> 2.3)
415	414	simplecov (0.7.1)
416	415	multi_json (~> 1.0)
417	416	simplecov-html (~> 0.7.1)
...	...	@@ -421,6 +420,9 @@ GEM
421	420	rack-protection (~> 1.2)
422	421	tilt (~> 1.3, >= 1.3.3)
423	422	six (0.2.0)
	423	+ slim (1.3.6)
	424	+ temple (~> 0.5.5)
	425	+ tilt (~> 1.3.3)
424	426	slop (3.3.3)
425	427	spinach (0.5.2)
426	428	colorize
...	...	@@ -435,6 +437,7 @@ GEM
435	437	rack (~> 1.0)
436	438	tilt (~> 1.1, != 1.3.0)
437	439	stamp (0.3.0)
	440	+ temple (0.5.5)
438	441	test_after_commit (0.0.1)
439	442	therubyracer (0.10.2)
440	443	libv8 (~> 3.3.10)
...	...	@@ -444,6 +447,7 @@ GEM
444	447	rack (>= 1.0.0)
445	448	thor (0.16.0)
446	449	tilt (1.3.3)
	450	+ timers (1.0.2)
447	451	treetop (1.4.12)
448	452	polyglot
449	453	polyglot (>= 0.3.1)
...	...	@@ -455,8 +459,6 @@ GEM
455	459	kgio (~> 2.6)
456	460	rack
457	461	raindrops (~> 0.7)
458		- vegas (0.1.11)
459		- rack (>= 1.0.0)
460	462	virtus (0.5.2)
461	463	backports (~> 2.6.1)
462	464	warden (1.2.1)
...	...	@@ -534,17 +536,18 @@ DEPENDENCIES
534	536	rb-fsevent
535	537	rb-inotify
536	538	redcarpet (~> 2.2.2)
537		- resque!
538		- resque_mailer
539		- resque_spec
540	539	rspec-rails
541	540	sass-rails (~> 3.2.5)
542	541	sdoc
543	542	seed-fu
544	543	settingslogic
545	544	shoulda-matchers (= 1.3.0)
	545	+ sidekiq (= 2.6.4)
	546	+ sidekiq_mailer
546	547	simplecov
	548	+ sinatra
547	549	six
	550	+ slim
548	551	spinach-rails
549	552	stamp
550	553	test_after_commit
...	...
1	1	web: bundle exec rails s -p $PORT
2		-worker: bundle exec rake environment resque:work QUEUE=* VVERBOSE=1
	2	+worker: bundle exec sidekiq -q post_receive,mailer,system_hook,common
...	...
...	...	@@ -3,10 +3,6 @@ class Admin::DashboardController < AdminController
3	3	@projects = Project.order("created_at DESC").limit(10)
4	4	@users = User.order("created_at DESC").limit(10)
5	5
6		- @resque_accessible = true
7		- @workers = Resque.workers
8		- @pending_jobs = Resque.size(:post_receive)
9		-
10	6	rescue Redis::InheritedError
11	7	@resque_accessible = false
12	8	end
...	...
1	1	class Notify < ActionMailer::Base
2		- include Resque::Mailer
	2	+ include Sidekiq::Mailer
3	3	add_template_helper ApplicationHelper
4	4	add_template_helper GitlabMarkdownHelper
5	5
...	...
...	...	@@ -19,6 +19,6 @@ class SystemHook < WebHook
19	19	end
20	20
21	21	def async_execute(data)
22		- Resque.enqueue(SystemHookWorker, id, data)
	22	+ Sidekiq::Client.enqueue(SystemHookWorker, id, data)
23	23	end
24	24	end
...	...
1	1	.admin_dash.row
2		- .span3
	2	+ .span4
3	3	.ui-box
4	4	%h5.title Projects
5	5	.data.padded
...	...	@@ -7,7 +7,7 @@
7	7	%h1= Project.count
8	8	%hr
9	9	= link_to 'New Project', new_project_path, class: "btn small"
10		- .span3
	10	+ .span4
11	11	.ui-box
12	12	%h5.title Groups
13	13	.data.padded
...	...	@@ -15,7 +15,7 @@
15	15	%h1= Group.count
16	16	%hr
17	17	= link_to 'New Group', new_admin_group_path, class: "btn small"
18		- .span3
	18	+ .span4
19	19	.ui-box
20	20	%h5.title Users
21	21	.data.padded
...	...	@@ -23,25 +23,6 @@
23	23	%h1= User.count
24	24	%hr
25	25	= link_to 'New User', new_admin_user_path, class: "btn small"
26		- .span3
27		- .ui-box
28		- %h5.title
29		- Resque Workers
30		- .data.padded
31		- - if @resque_accessible
32		- = link_to admin_resque_path do
33		- %h1{class: @workers.present? ? "cgreen" : "cred"}
34		- = @workers.count
35		- %hr
36		- %p
37		- %strong{class: @pending_jobs > 0 ? "cred" : "cgreen"}
38		- #{@pending_jobs} post receive jobs waiting
39		- - else
40		- = link_to admin_resque_path do
41		- %h1.cdark ?
42		- %hr
43		- %p
44		- %strong Resque status unknown
45	26
46	27	.row
47	28	.span6
...	...
1		-%h3.page_title Resque
	1	+%h3.page_title Background Jobs
2	2	%br
3	3	.ui-box
4		- %iframe{src: resque_path, width: '100%', height: 600, style: "border: none"}
	4	+ %iframe{src: sidekiq_path, width: '100%', height: 900, style: "border: none"}
...	...
...	...	@@ -19,6 +19,6 @@
19	19	= nav_link(controller: :hooks) do
20	20	= link_to "Hooks", admin_hooks_path
21	21	= nav_link(controller: :resque) do
22		- = link_to "Resque", admin_resque_path
	22	+ = link_to "Background Jobs", admin_resque_path
23	23
24	24	.content= yield
...	...
1	1	class PostReceive
2		- @queue = :post_receive
	2	+ include Sidekiq::Worker
3	3
4		- def self.perform(repo_path, oldrev, newrev, ref, identifier)
	4	+ sidekiq_options queue: :post_receive
	5	+
	6	+ def perform(repo_path, oldrev, newrev, ref, identifier)
5	7	repo_path.gsub!(Gitlab.config.gitolite.repos_path.to_s, "")
6	8	repo_path.gsub!(/.git$/, "")
7	9	repo_path.gsub!(/^\//, "")
...	...