Download as
Browse Code »

Commit c7bb3a1f726be189ccce51bdd631b26eb4f64db1

Authored by Dmitriy Zaporozhets
1 parent 7bfd38ea
Exists in master and in 4 other branches 6-8-stable, 7-0-stable, 7-0-stable-spb, spb-stable

sidekiq

Showing 14 changed files with 75 additions and 94 deletions   Show diff stats
@@ -81,8 +81,10 @@ gem "acts-as-taggable-on", "2.3.3" @@ -81,8 +81,10 @@ gem "acts-as-taggable-on", "2.3.3"
81 gem "draper", "~> 0.18.0" 81 gem "draper", "~> 0.18.0"
82 82
83 # Background jobs 83 # Background jobs
84 -gem "resque", git: "https://github.com/gitlabhq/resque.git", ref: "9ef4700306dd946a3ac000612428967ce0c32213"  
85 -gem 'resque_mailer' 84 +gem 'slim'
  85 +gem 'sinatra', :require => nil
  86 +gem 'sidekiq', '2.6.4'
  87 +gem 'sidekiq_mailer'
86 88
87 # HTTP requests 89 # HTTP requests
88 gem "httparty" 90 gem "httparty"
@@ -157,7 +159,6 @@ group :test do @@ -157,7 +159,6 @@ group :test do
157 gem "simplecov", require: false 159 gem "simplecov", require: false
158 gem "shoulda-matchers", "1.3.0" 160 gem "shoulda-matchers", "1.3.0"
159 gem 'email_spec' 161 gem 'email_spec'
160 - gem 'resque_spec'  
161 gem "webmock" 162 gem "webmock"
162 gem 'test_after_commit' 163 gem 'test_after_commit'
163 end 164 end
@@ -55,17 +55,6 @@ GIT @@ -55,17 +55,6 @@ GIT
55 raphael-rails (2.1.0) 55 raphael-rails (2.1.0)
56 56
57 GIT 57 GIT
58 - remote: https://github.com/gitlabhq/resque.git  
59 - revision: 9ef4700306dd946a3ac000612428967ce0c32213  
60 - ref: 9ef4700306dd946a3ac000612428967ce0c32213  
61 - specs:  
62 - resque (2.0.0.pre.1)  
63 - json  
64 - redis-namespace (~> 1.0)  
65 - sinatra (>= 0.9.2)  
66 - vegas (~> 0.1.2)  
67 -  
68 -GIT  
69 remote: https://github.com/jonleighton/poltergeist.git 58 remote: https://github.com/jonleighton/poltergeist.git
70 revision: 5c2e092001074a8cf09f332d3714e9ba150bc8ca 59 revision: 5c2e092001074a8cf09f332d3714e9ba150bc8ca
71 ref: 5c2e092001074a8cf09f332d3714e9ba150bc8ca 60 ref: 5c2e092001074a8cf09f332d3714e9ba150bc8ca
@@ -132,6 +121,9 @@ GEM @@ -132,6 +121,9 @@ GEM
132 carrierwave (0.7.1) 121 carrierwave (0.7.1)
133 activemodel (>= 3.2.0) 122 activemodel (>= 3.2.0)
134 activesupport (>= 3.2.0) 123 activesupport (>= 3.2.0)
  124 + celluloid (0.12.4)
  125 + facter (>= 1.6.12)
  126 + timers (>= 1.0.0)
135 charlock_holmes (0.6.9) 127 charlock_holmes (0.6.9)
136 childprocess (0.3.6) 128 childprocess (0.3.6)
137 ffi (~> 1.0, >= 1.0.6) 129 ffi (~> 1.0, >= 1.0.6)
@@ -150,6 +142,7 @@ GEM @@ -150,6 +142,7 @@ GEM
150 coffee-script-source (1.4.0) 142 coffee-script-source (1.4.0)
151 colored (1.2) 143 colored (1.2)
152 colorize (0.5.8) 144 colorize (0.5.8)
  145 + connection_pool (1.0.0)
153 crack (0.3.1) 146 crack (0.3.1)
154 daemons (1.1.9) 147 daemons (1.1.9)
155 devise (2.1.2) 148 devise (2.1.2)
@@ -169,6 +162,7 @@ GEM @@ -169,6 +162,7 @@ GEM
169 eventmachine (1.0.0) 162 eventmachine (1.0.0)
170 execjs (1.4.0) 163 execjs (1.4.0)
171 multi_json (~> 1.0) 164 multi_json (~> 1.0)
  165 + facter (1.6.17)
172 factory_girl (4.1.0) 166 factory_girl (4.1.0)
173 activesupport (>= 3.0.0) 167 activesupport (>= 3.0.0)
174 factory_girl_rails (4.1.0) 168 factory_girl_rails (4.1.0)
@@ -370,11 +364,6 @@ GEM @@ -370,11 +364,6 @@ GEM
370 redis (3.0.2) 364 redis (3.0.2)
371 redis-namespace (1.2.1) 365 redis-namespace (1.2.1)
372 redis (~> 3.0.0) 366 redis (~> 3.0.0)
373 - resque_mailer (2.1.0)  
374 - actionmailer (~> 3.0)  
375 - resque_spec (0.12.5)  
376 - resque (>= 1.19.0)  
377 - rspec (>= 2.5.0)  
378 rspec (2.12.0) 367 rspec (2.12.0)
379 rspec-core (~> 2.12.0) 368 rspec-core (~> 2.12.0)
380 rspec-expectations (~> 2.12.0) 369 rspec-expectations (~> 2.12.0)
@@ -412,6 +401,16 @@ GEM @@ -412,6 +401,16 @@ GEM
412 sexp_processor (4.1.3) 401 sexp_processor (4.1.3)
413 shoulda-matchers (1.3.0) 402 shoulda-matchers (1.3.0)
414 activesupport (>= 3.0.0) 403 activesupport (>= 3.0.0)
  404 + sidekiq (2.6.4)
  405 + celluloid (~> 0.12.0)
  406 + connection_pool (~> 1.0)
  407 + multi_json (~> 1)
  408 + redis (~> 3)
  409 + redis-namespace
  410 + sidekiq_mailer (0.0.4)
  411 + actionmailer (~> 3.0)
  412 + activesupport (~> 3.0)
  413 + sidekiq (~> 2.3)
415 simplecov (0.7.1) 414 simplecov (0.7.1)
416 multi_json (~> 1.0) 415 multi_json (~> 1.0)
417 simplecov-html (~> 0.7.1) 416 simplecov-html (~> 0.7.1)
@@ -421,6 +420,9 @@ GEM @@ -421,6 +420,9 @@ GEM
421 rack-protection (~> 1.2) 420 rack-protection (~> 1.2)
422 tilt (~> 1.3, >= 1.3.3) 421 tilt (~> 1.3, >= 1.3.3)
423 six (0.2.0) 422 six (0.2.0)
  423 + slim (1.3.6)
  424 + temple (~> 0.5.5)
  425 + tilt (~> 1.3.3)
424 slop (3.3.3) 426 slop (3.3.3)
425 spinach (0.5.2) 427 spinach (0.5.2)
426 colorize 428 colorize
@@ -435,6 +437,7 @@ GEM @@ -435,6 +437,7 @@ GEM
435 rack (~> 1.0) 437 rack (~> 1.0)
436 tilt (~> 1.1, != 1.3.0) 438 tilt (~> 1.1, != 1.3.0)
437 stamp (0.3.0) 439 stamp (0.3.0)
  440 + temple (0.5.5)
438 test_after_commit (0.0.1) 441 test_after_commit (0.0.1)
439 therubyracer (0.10.2) 442 therubyracer (0.10.2)
440 libv8 (~> 3.3.10) 443 libv8 (~> 3.3.10)
@@ -444,6 +447,7 @@ GEM @@ -444,6 +447,7 @@ GEM
444 rack (>= 1.0.0) 447 rack (>= 1.0.0)
445 thor (0.16.0) 448 thor (0.16.0)
446 tilt (1.3.3) 449 tilt (1.3.3)
  450 + timers (1.0.2)
447 treetop (1.4.12) 451 treetop (1.4.12)
448 polyglot 452 polyglot
449 polyglot (>= 0.3.1) 453 polyglot (>= 0.3.1)
@@ -455,8 +459,6 @@ GEM @@ -455,8 +459,6 @@ GEM
455 kgio (~> 2.6) 459 kgio (~> 2.6)
456 rack 460 rack
457 raindrops (~> 0.7) 461 raindrops (~> 0.7)
458 - vegas (0.1.11)  
459 - rack (>= 1.0.0)  
460 virtus (0.5.2) 462 virtus (0.5.2)
461 backports (~> 2.6.1) 463 backports (~> 2.6.1)
462 warden (1.2.1) 464 warden (1.2.1)
@@ -534,17 +536,18 @@ DEPENDENCIES @@ -534,17 +536,18 @@ DEPENDENCIES
534 rb-fsevent 536 rb-fsevent
535 rb-inotify 537 rb-inotify
536 redcarpet (~> 2.2.2) 538 redcarpet (~> 2.2.2)
537 - resque!  
538 - resque_mailer  
539 - resque_spec  
540 rspec-rails 539 rspec-rails
541 sass-rails (~> 3.2.5) 540 sass-rails (~> 3.2.5)
542 sdoc 541 sdoc
543 seed-fu 542 seed-fu
544 settingslogic 543 settingslogic
545 shoulda-matchers (= 1.3.0) 544 shoulda-matchers (= 1.3.0)
  545 + sidekiq (= 2.6.4)
  546 + sidekiq_mailer
546 simplecov 547 simplecov
  548 + sinatra
547 six 549 six
  550 + slim
548 spinach-rails 551 spinach-rails
549 stamp 552 stamp
550 test_after_commit 553 test_after_commit
1 web: bundle exec rails s -p $PORT 1 web: bundle exec rails s -p $PORT
2 -worker: bundle exec rake environment resque:work QUEUE=* VVERBOSE=1 2 +worker: bundle exec sidekiq -q post_receive,mailer,system_hook,common
app/controllers/admin/dashboard_controller.rb
  Diff comments   View file @c7bb3a1
@@ -3,10 +3,6 @@ class Admin::DashboardController < AdminController @@ -3,10 +3,6 @@ class Admin::DashboardController < AdminController
3 @projects = Project.order("created_at DESC").limit(10) 3 @projects = Project.order("created_at DESC").limit(10)
4 @users = User.order("created_at DESC").limit(10) 4 @users = User.order("created_at DESC").limit(10)
5 5
6 - @resque_accessible = true  
7 - @workers = Resque.workers  
8 - @pending_jobs = Resque.size(:post_receive)  
9 -  
10 rescue Redis::InheritedError 6 rescue Redis::InheritedError
11 @resque_accessible = false 7 @resque_accessible = false
12 end 8 end
app/mailers/notify.rb
  Diff comments   View file @c7bb3a1
1 class Notify < ActionMailer::Base 1 class Notify < ActionMailer::Base
2 - include Resque::Mailer 2 + include Sidekiq::Mailer
3 add_template_helper ApplicationHelper 3 add_template_helper ApplicationHelper
4 add_template_helper GitlabMarkdownHelper 4 add_template_helper GitlabMarkdownHelper
5 5
app/models/system_hook.rb
  Diff comments   View file @c7bb3a1
@@ -19,6 +19,6 @@ class SystemHook &lt; WebHook @@ -19,6 +19,6 @@ class SystemHook &lt; WebHook
19 end 19 end
20 20
21 def async_execute(data) 21 def async_execute(data)
22 - Resque.enqueue(SystemHookWorker, id, data) 22 + Sidekiq::Client.enqueue(SystemHookWorker, id, data)
23 end 23 end
24 end 24 end
app/views/admin/dashboard/index.html.haml
  Diff comments   View file @c7bb3a1
1 .admin_dash.row 1 .admin_dash.row
2 - .span3 2 + .span4
3 .ui-box 3 .ui-box
4 %h5.title Projects 4 %h5.title Projects
5 .data.padded 5 .data.padded
@@ -7,7 +7,7 @@ @@ -7,7 +7,7 @@
7 %h1= Project.count 7 %h1= Project.count
8 %hr 8 %hr
9 = link_to 'New Project', new_project_path, class: "btn small" 9 = link_to 'New Project', new_project_path, class: "btn small"
10 - .span3 10 + .span4
11 .ui-box 11 .ui-box
12 %h5.title Groups 12 %h5.title Groups
13 .data.padded 13 .data.padded
@@ -15,7 +15,7 @@ @@ -15,7 +15,7 @@
15 %h1= Group.count 15 %h1= Group.count
16 %hr 16 %hr
17 = link_to 'New Group', new_admin_group_path, class: "btn small" 17 = link_to 'New Group', new_admin_group_path, class: "btn small"
18 - .span3 18 + .span4
19 .ui-box 19 .ui-box
20 %h5.title Users 20 %h5.title Users
21 .data.padded 21 .data.padded
@@ -23,25 +23,6 @@ @@ -23,25 +23,6 @@
23 %h1= User.count 23 %h1= User.count
24 %hr 24 %hr
25 = link_to 'New User', new_admin_user_path, class: "btn small" 25 = link_to 'New User', new_admin_user_path, class: "btn small"
26 - .span3  
27 - .ui-box  
28 - %h5.title  
29 - Resque Workers  
30 - .data.padded  
31 - - if @resque_accessible  
32 - = link_to admin_resque_path do  
33 - %h1{class: @workers.present? ? "cgreen" : "cred"}  
34 - = @workers.count  
35 - %hr  
36 - %p  
37 - %strong{class: @pending_jobs > 0 ? "cred" : "cgreen"}  
38 - #{@pending_jobs} post receive jobs waiting  
39 - - else  
40 - = link_to admin_resque_path do  
41 - %h1.cdark ?  
42 - %hr  
43 - %p  
44 - %strong Resque status unknown  
45 26
46 .row 27 .row
47 .span6 28 .span6
app/views/admin/resque/show.html.haml
  Diff comments   View file @c7bb3a1
1 -%h3.page_title Resque 1 +%h3.page_title Background Jobs
2 %br 2 %br
3 .ui-box 3 .ui-box
4 - %iframe{src: resque_path, width: '100%', height: 600, style: "border: none"} 4 + %iframe{src: sidekiq_path, width: '100%', height: 900, style: "border: none"}
app/views/layouts/admin.html.haml
  Diff comments   View file @c7bb3a1
@@ -19,6 +19,6 @@ @@ -19,6 +19,6 @@
19 = nav_link(controller: :hooks) do 19 = nav_link(controller: :hooks) do
20 = link_to "Hooks", admin_hooks_path 20 = link_to "Hooks", admin_hooks_path
21 = nav_link(controller: :resque) do 21 = nav_link(controller: :resque) do
22 - = link_to "Resque", admin_resque_path 22 + = link_to "Background Jobs", admin_resque_path
23 23
24 .content= yield 24 .content= yield
app/workers/post_receive.rb
  Diff comments   View file @c7bb3a1
1 class PostReceive 1 class PostReceive
2 - @queue = :post_receive 2 + include Sidekiq::Worker
3 3
4 - def self.perform(repo_path, oldrev, newrev, ref, identifier) 4 + sidekiq_options queue: :post_receive
  5 +
  6 + def perform(repo_path, oldrev, newrev, ref, identifier)
5 repo_path.gsub!(Gitlab.config.gitolite.repos_path.to_s, "") 7 repo_path.gsub!(Gitlab.config.gitolite.repos_path.to_s, "")
6 repo_path.gsub!(/.git$/, "") 8 repo_path.gsub!(/.git$/, "")
7 repo_path.gsub!(/^\//, "") 9 repo_path.gsub!(/^\//, "")
app/workers/system_hook_worker.rb
  Diff comments   View file @c7bb3a1
1 class SystemHookWorker 1 class SystemHookWorker
2 - @queue = :system_hook 2 + include Sidekiq::Worker
3 3
4 - def self.perform(hook_id, data) 4 + sidekiq_options queue: :system_hook
  5 +
  6 + def perform(hook_id, data)
5 SystemHook.find(hook_id).execute data 7 SystemHook.find(hook_id).execute data
6 end 8 end
7 end 9 end
config/initializers/4_resque.rb
  Diff comments   View file @c7bb3a1
1 # Custom Redis configuration 1 # Custom Redis configuration
2 config_file = Rails.root.join('config', 'resque.yml') 2 config_file = Rails.root.join('config', 'resque.yml')
3 3
4 -if File.exists?(config_file)  
5 - resque_config = YAML.load_file(config_file)  
6 - Resque.redis = resque_config[Rails.env]  
7 -end  
8 -Resque.redis.namespace = 'resque:gitlab'  
9 -Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection }  
10 -  
11 -# Authentication  
12 -require 'resque/server'  
13 -class ResqueAuthentication  
14 - def initialize(app)  
15 - @app = app  
16 - end 4 +resque_url = if File.exists?(config_file)
  5 + YAML.load_file(config_file)[Rails.env]
  6 + else
  7 + "localhost:6379"
  8 + end
17 9
18 - def call(env)  
19 - account = env['warden'].authenticate!(:database_authenticatable, :rememberable, scope: :user)  
20 - raise "Access denied" if !account.admin?  
21 - @app.call(env)  
22 - end 10 +Sidekiq.configure_server do |config|
  11 + config.redis = {
  12 + url: "redis://#{resque_url}",
  13 + namespace: 'resque:gitlab'
  14 + }
23 end 15 end
24 16
25 -Resque::Server.use ResqueAuthentication  
26 -  
27 -# Mailer  
28 -Resque::Mailer.excluded_environments = []  
29 - 17 +Sidekiq.configure_client do |config|
  18 + config.redis = {
  19 + url: "redis://#{resque_url}",
  20 + namespace: 'resque:gitlab'
  21 + }
  22 +end
config/routes.rb
  Diff comments   View file @c7bb3a1
  1 +require 'sidekiq/web'
  2 +
1 Gitlab::Application.routes.draw do 3 Gitlab::Application.routes.draw do
2 # 4 #
3 # Search 5 # Search
@@ -8,9 +10,10 @@ Gitlab::Application.routes.draw do @@ -8,9 +10,10 @@ Gitlab::Application.routes.draw do
8 require 'api' 10 require 'api'
9 mount Gitlab::API => '/api' 11 mount Gitlab::API => '/api'
10 12
11 - # Optionally, enable Resque here  
12 - require 'resque/server'  
13 - mount Resque::Server => '/info/resque', as: 'resque' 13 + constraint = lambda { |request| request.env["warden"].authenticate? and request.env['warden'].user.admin? }
  14 + constraints constraint do
  15 + mount Sidekiq::Web, at: "/admin/workers", as: :sidekiq
  16 + end
14 17
15 # Enable Grack support 18 # Enable Grack support
16 mount Grack::Bundle.new({ 19 mount Grack::Bundle.new({
lib/tasks/resque.rake
  Diff comments   View file @c7bb3a1
@@ -2,20 +2,20 @@ require &#39;resque/tasks&#39; @@ -2,20 +2,20 @@ require &#39;resque/tasks&#39;
2 2
3 namespace :resque do 3 namespace :resque do
4 task setup: :environment do 4 task setup: :environment do
5 - Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection } 5 + #Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection }
6 end 6 end
7 7
8 desc "Resque | kill all workers (using -QUIT), god will take care of them" 8 desc "Resque | kill all workers (using -QUIT), god will take care of them"
9 task :stop_workers => :environment do 9 task :stop_workers => :environment do
10 - pids = Array.new 10 + #pids = Array.new
11 11
12 - Resque.workers.each do |worker|  
13 - pids << worker.to_s.split(/:/).second  
14 - end 12 + #Resque.workers.each do |worker|
  13 + #pids << worker.to_s.split(/:/).second
  14 + #end
15 15
16 - if pids.size > 0  
17 - system("kill -QUIT #{pids.join(' ')}")  
18 - end 16 + #if pids.size > 0
  17 + #system("kill -QUIT #{pids.join(' ')}")
  18 + #end
19 end 19 end
20 end 20 end
21 21