Download as
Browse Code »

Commit df6db81e2ab36410c377f8a9a712756653de0e2a

Authored by Dmitriy Zaporozhets
2 parents b4f16faa f7ade3b6
Exists in master and in 4 other branches 6-8-stable, 7-0-stable, 7-0-stable-spb, spb-stable

Merge branch 'features/async_gitolite' of dev.gitlabhq.com:gitlab/gitlabhq

Showing 16 changed files with 127 additions and 64 deletions   Show diff stats
1 web: bundle exec unicorn_rails -p $PORT 1 web: bundle exec unicorn_rails -p $PORT
2 -worker: bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default 2 +worker: bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default,gitolite
app/contexts/projects/create_context.rb
  Diff comments   View file @df6db81
@@ -32,16 +32,10 @@ module Projects @@ -32,16 +32,10 @@ module Projects
32 @project.namespace_id = current_user.namespace_id 32 @project.namespace_id = current_user.namespace_id
33 end 33 end
34 34
35 - Project.transaction do  
36 - @project.creator = current_user  
37 - @project.save! 35 + @project.creator = current_user
38 36
39 - # Add user as project master  
40 - @project.users_projects.create!(project_access: UsersProject::MASTER, user: current_user)  
41 -  
42 - # when project saved no team member exist so  
43 - # project repository should be updated after first user add  
44 - @project.update_repository 37 + if @project.save
  38 + @project.users_projects.create(project_access: UsersProject::MASTER, user: current_user)
45 end 39 end
46 40
47 @project 41 @project
app/models/project.rb
  Diff comments   View file @df6db81
@@ -299,6 +299,9 @@ class Project < ActiveRecord::Base @@ -299,6 +299,9 @@ class Project < ActiveRecord::Base
299 def trigger_post_receive(oldrev, newrev, ref, user) 299 def trigger_post_receive(oldrev, newrev, ref, user)
300 data = post_receive_data(oldrev, newrev, ref, user) 300 data = post_receive_data(oldrev, newrev, ref, user)
301 301
  302 + # Create satellite
  303 + self.satellite.create unless self.satellite.exists?
  304 +
302 # Create push event 305 # Create push event
303 self.observe_push(data) 306 self.observe_push(data)
304 307
@@ -313,9 +316,6 @@ class Project < ActiveRecord::Base @@ -313,9 +316,6 @@ class Project < ActiveRecord::Base
313 self.execute_services(data.dup) 316 self.execute_services(data.dup)
314 end 317 end
315 318
316 - # Create satellite  
317 - self.satellite.create unless self.satellite.exists?  
318 -  
319 # Discover the default branch, but only if it hasn't already been set to 319 # Discover the default branch, but only if it hasn't already been set to
320 # something else 320 # something else
321 if repository && default_branch.nil? 321 if repository && default_branch.nil?
@@ -460,11 +460,17 @@ class Project < ActiveRecord::Base @@ -460,11 +460,17 @@ class Project < ActiveRecord::Base
460 end 460 end
461 461
462 def update_repository 462 def update_repository
463 - gitolite.update_repository(self) 463 + GitoliteWorker.perform_async(
  464 + :update_repository,
  465 + self.id
  466 + )
464 end 467 end
465 468
466 def destroy_repository 469 def destroy_repository
467 - gitolite.remove_repository(self) 470 + GitoliteWorker.perform_async(
  471 + :remove_repository,
  472 + self.path_with_namespace
  473 + )
468 end 474 end
469 475
470 def repo_exists? 476 def repo_exists?
app/models/protected_branch.rb
  Diff comments   View file @df6db81
@@ -22,7 +22,7 @@ class ProtectedBranch < ActiveRecord::Base @@ -22,7 +22,7 @@ class ProtectedBranch < ActiveRecord::Base
22 after_destroy :update_repository 22 after_destroy :update_repository
23 23
24 def update_repository 24 def update_repository
25 - gitolite.update_repository(project) 25 + project.update_repository
26 end 26 end
27 27
28 def commit 28 def commit
app/models/users_project.rb
  Diff comments   View file @df6db81
@@ -82,9 +82,13 @@ class UsersProject < ActiveRecord::Base @@ -82,9 +82,13 @@ class UsersProject < ActiveRecord::Base
82 users_project.save 82 users_project.save
83 end 83 end
84 end 84 end
85 - Gitlab::Gitolite.new.update_repositories(Project.where(id: project_ids))  
86 end 85 end
87 86
  87 + GitoliteWorker.perform_async(
  88 + :update_repositories,
  89 + project_ids
  90 + )
  91 +
88 true 92 true
89 rescue 93 rescue
90 false 94 false
@@ -97,9 +101,13 @@ class UsersProject < ActiveRecord::Base @@ -97,9 +101,13 @@ class UsersProject < ActiveRecord::Base
97 users_project.skip_git = true 101 users_project.skip_git = true
98 users_project.destroy 102 users_project.destroy
99 end 103 end
100 - Gitlab::Gitolite.new.update_repositories(Project.where(id: project_ids))  
101 end 104 end
102 105
  106 + GitoliteWorker.perform_async(
  107 + :update_repositories,
  108 + project_ids
  109 + )
  110 +
103 true 111 true
104 rescue 112 rescue
105 false 113 false
@@ -129,7 +137,7 @@ class UsersProject < ActiveRecord::Base @@ -129,7 +137,7 @@ class UsersProject < ActiveRecord::Base
129 end 137 end
130 138
131 def update_repository 139 def update_repository
132 - gitolite.update_repository(project) 140 + project.update_repository
133 end 141 end
134 142
135 def project_access_human 143 def project_access_human
app/observers/project_observer.rb
  Diff comments   View file @df6db81
@@ -10,6 +10,7 @@ class ProjectObserver < ActiveRecord::Observer @@ -10,6 +10,7 @@ class ProjectObserver < ActiveRecord::Observer
10 def after_destroy(project) 10 def after_destroy(project)
11 log_info("Project \"#{project.name}\" was removed") 11 log_info("Project \"#{project.name}\" was removed")
12 12
  13 + project.satellite.destroy
13 project.destroy_repository 14 project.destroy_repository
14 end 15 end
15 16
app/workers/gitolite_worker.rb 0 → 100644
  Diff comments   View file @df6db81
@@ -0,0 +1,10 @@ @@ -0,0 +1,10 @@
  1 +class GitoliteWorker
  2 + include Sidekiq::Worker
  3 + include Gitolited
  4 +
  5 + sidekiq_options queue: :gitolite
  6 +
  7 + def perform(action, arg)
  8 + gitolite.send(action, arg)
  9 + end
  10 +end
app/workers/post_receive.rb
  Diff comments   View file @df6db81
@@ -13,13 +13,14 @@ class PostReceive @@ -13,13 +13,14 @@ class PostReceive
13 13
14 # Ignore push from non-gitlab users 14 # Ignore push from non-gitlab users
15 user = if identifier.eql? Gitlab.config.gitolite.admin_key 15 user = if identifier.eql? Gitlab.config.gitolite.admin_key
16 - email = project.repository.commit(newrev).author.email rescue nil  
17 - User.find_by_email(email) if email  
18 - elsif /^[A-Z0-9._%a-z\-]+@(?:[A-Z0-9a-z\-]+\.)+[A-Za-z]{2,4}$/.match(identifier)  
19 - User.find_by_email(identifier)  
20 - else  
21 - Key.find_by_identifier(identifier).try(:user)  
22 - end 16 + email = project.repository.commit(newrev).author.email rescue nil
  17 + User.find_by_email(email) if email
  18 + elsif /^[A-Z0-9._%a-z\-]+@(?:[A-Z0-9a-z\-]+\.)+[A-Za-z]{2,4}$/.match(identifier)
  19 + User.find_by_email(identifier)
  20 + else
  21 + Key.find_by_identifier(identifier).try(:user)
  22 + end
  23 +
23 return false unless user 24 return false unless user
24 25
25 project.trigger_post_receive(oldrev, newrev, ref, user) 26 project.trigger_post_receive(oldrev, newrev, ref, user)
lib/gitlab/backend/gitolite.rb
  Diff comments   View file @df6db81
@@ -22,7 +22,13 @@ module Gitlab @@ -22,7 +22,13 @@ module Gitlab
22 end 22 end
23 end 23 end
24 24
25 - def update_repository project 25 + # Update project config in gitolite by project id
  26 + #
  27 + # Ex.
  28 + # update_repository(23)
  29 + #
  30 + def update_repository(project_id)
  31 + project = Project.find(project_id)
26 config.update_project!(project) 32 config.update_project!(project)
27 end 33 end
28 34
@@ -33,8 +39,28 @@ module Gitlab @@ -33,8 +39,28 @@ module Gitlab
33 end 39 end
34 end 40 end
35 41
36 - def remove_repository project  
37 - config.destroy_project!(project) 42 + # Remove repository from gitolite
  43 + #
  44 + # name - project path with namespace
  45 + #
  46 + # Ex.
  47 + # remove_repository("gitlab/gitlab-ci")
  48 + #
  49 + def remove_repository(name)
  50 + config.destroy_project!(name)
  51 + end
  52 +
  53 + # Update projects configs in gitolite by project ids
  54 + #
  55 + # Ex.
  56 + # update_repositories([1, 4, 6])
  57 + #
  58 + def update_repositories(project_ids)
  59 + projects = Project.where(id: project_ids)
  60 +
  61 + config.apply do |config|
  62 + config.update_projects(projects)
  63 + end
38 end 64 end
39 65
40 def url_to_repo path 66 def url_to_repo path
@@ -45,12 +71,6 @@ module Gitlab @@ -45,12 +71,6 @@ module Gitlab
45 config.admin_all_repo! 71 config.admin_all_repo!
46 end 72 end
47 73
48 - def update_repositories projects  
49 - config.apply do |config|  
50 - config.update_projects(projects)  
51 - end  
52 - end  
53 -  
54 alias_method :create_repository, :update_repository 74 alias_method :create_repository, :update_repository
55 end 75 end
56 end 76 end
lib/gitlab/backend/gitolite_config.rb
  Diff comments   View file @df6db81
@@ -4,6 +4,8 @@ require 'fileutils' @@ -4,6 +4,8 @@ require 'fileutils'
4 4
5 module Gitlab 5 module Gitlab
6 class GitoliteConfig 6 class GitoliteConfig
  7 + include Gitlab::Popen
  8 +
7 class PullError < StandardError; end 9 class PullError < StandardError; end
8 class PushError < StandardError; end 10 class PushError < StandardError; end
9 class BrokenGitolite < StandardError; end 11 class BrokenGitolite < StandardError; end
@@ -87,12 +89,14 @@ module Gitlab @@ -87,12 +89,14 @@ module Gitlab
87 Gitlab::GitLogger.error(message) 89 Gitlab::GitLogger.error(message)
88 end 90 end
89 91
90 - def destroy_project(project)  
91 - # do rm-rf only if repository exists  
92 - if project.repository  
93 - FileUtils.rm_rf(project.repository.path_to_repo)  
94 - end  
95 - conf.rm_repo(project.path_with_namespace) 92 + def path_to_repo(name)
  93 + File.join(Gitlab.config.gitolite.repos_path, "#{name}.git")
  94 + end
  95 +
  96 + def destroy_project(name)
  97 + full_path = path_to_repo(name)
  98 + FileUtils.rm_rf(full_path) if File.exists?(full_path)
  99 + conf.rm_repo(name)
96 end 100 end
97 101
98 def clean_repo repo_name 102 def clean_repo repo_name
@@ -210,14 +214,14 @@ module Gitlab @@ -210,14 +214,14 @@ module Gitlab
210 end 214 end
211 215
212 def push 216 def push
213 - output, status = popen('git add -A') 217 + output, status = popen('git add -A', tmp_conf_path)
214 raise "Git add failed." unless status.zero? 218 raise "Git add failed." unless status.zero?
215 219
216 # git commit returns 0 on success, and 1 if there is nothing to commit 220 # git commit returns 0 on success, and 1 if there is nothing to commit
217 - output, status = popen('git commit -m "GitLab"') 221 + output, status = popen('git commit -m "GitLab"', tmp_conf_path)
218 raise "Git add failed." unless [0,1].include?(status) 222 raise "Git add failed." unless [0,1].include?(status)
219 223
220 - output, status = popen('git push') 224 + output, status = popen('git push', tmp_conf_path)
221 225
222 if output =~ /remote\: FATAL/ 226 if output =~ /remote\: FATAL/
223 raise BrokenGitolite, output 227 raise BrokenGitolite, output
@@ -230,20 +234,8 @@ module Gitlab @@ -230,20 +234,8 @@ module Gitlab
230 end 234 end
231 end 235 end
232 236
233 - def popen(cmd, path = nil)  
234 - path ||= File.join(config_tmp_dir,'gitolite')  
235 - vars = { "PWD" => path }  
236 - options = { :chdir => path }  
237 -  
238 - @cmd_output = ""  
239 - @cmd_status = 0  
240 - Open3.popen3(vars, cmd, options) do |stdin, stdout, stderr, wait_thr|  
241 - @cmd_status = wait_thr.value.exitstatus  
242 - @cmd_output << stdout.read  
243 - @cmd_output << stderr.read  
244 - end  
245 -  
246 - return @cmd_output, @cmd_status 237 + def tmp_conf_path
  238 + File.join(config_tmp_dir,'gitolite')
247 end 239 end
248 end 240 end
249 end 241 end
lib/gitlab/popen.rb 0 → 100644
  Diff comments   View file @df6db81
@@ -0,0 +1,18 @@ @@ -0,0 +1,18 @@
  1 +module Gitlab
  2 + module Popen
  3 + def popen(cmd, path)
  4 + vars = { "PWD" => path }
  5 + options = { :chdir => path }
  6 +
  7 + @cmd_output = ""
  8 + @cmd_status = 0
  9 + Open3.popen3(vars, cmd, options) do |stdin, stdout, stderr, wait_thr|
  10 + @cmd_status = wait_thr.value.exitstatus
  11 + @cmd_output << stdout.read
  12 + @cmd_output << stderr.read
  13 + end
  14 +
  15 + return @cmd_output, @cmd_status
  16 + end
  17 + end
  18 +end
lib/gitlab/satellite/satellite.rb
  Diff comments   View file @df6db81
@@ -3,6 +3,8 @@ module Gitlab @@ -3,6 +3,8 @@ module Gitlab
3 3
4 module Satellite 4 module Satellite
5 class Satellite 5 class Satellite
  6 + include Gitlab::Popen
  7 +
6 PARKING_BRANCH = "__parking_branch" 8 PARKING_BRANCH = "__parking_branch"
7 9
8 attr_accessor :project 10 attr_accessor :project
@@ -24,8 +26,10 @@ module Gitlab @@ -24,8 +26,10 @@ module Gitlab
24 end 26 end
25 27
26 def create 28 def create
27 - create_cmd = "git clone #{project.url_to_repo} #{path}"  
28 - if system(create_cmd) 29 + output, status = popen("git clone #{project.url_to_repo} #{path}",
  30 + Gitlab.config.satellites.path)
  31 +
  32 + if status.zero?
29 true 33 true
30 else 34 else
31 Gitlab::GitLogger.error("Failed to create satellite for #{project.name_with_namespace}") 35 Gitlab::GitLogger.error("Failed to create satellite for #{project.name_with_namespace}")
@@ -66,6 +70,10 @@ module Gitlab @@ -66,6 +70,10 @@ module Gitlab
66 @repo ||= Grit::Repo.new(path) 70 @repo ||= Grit::Repo.new(path)
67 end 71 end
68 72
  73 + def destroy
  74 + FileUtils.rm_rf(path)
  75 + end
  76 +
69 private 77 private
70 78
71 # Clear the working directory 79 # Clear the working directory
lib/tasks/sidekiq.rake
  Diff comments   View file @df6db81
@@ -6,7 +6,7 @@ namespace :sidekiq do @@ -6,7 +6,7 @@ namespace :sidekiq do
6 6
7 desc "GITLAB | Start sidekiq" 7 desc "GITLAB | Start sidekiq"
8 task :start do 8 task :start do
9 - run "nohup bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default -e #{Rails.env} -P #{pidfile} >> #{Rails.root.join("log", "sidekiq.log")} 2>&1 &" 9 + run "nohup bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,gitolite,common,default -e #{Rails.env} -P #{pidfile} >> #{Rails.root.join("log", "sidekiq.log")} 2>&1 &"
10 end 10 end
11 11
12 def pidfile 12 def pidfile
spec/factories.rb
  Diff comments   View file @df6db81
@@ -12,7 +12,7 @@ FactoryGirl.define do @@ -12,7 +12,7 @@ FactoryGirl.define do
12 factory :user, aliases: [:author, :assignee, :owner, :creator] do 12 factory :user, aliases: [:author, :assignee, :owner, :creator] do
13 email { Faker::Internet.email } 13 email { Faker::Internet.email }
14 name 14 name
15 - username { Faker::Internet.user_name } 15 + sequence(:username) { |n| "#{Faker::Internet.user_name}#{n}" }
16 password "123456" 16 password "123456"
17 password_confirmation { password } 17 password_confirmation { password }
18 18
spec/lib/gitolite_spec.rb
  Diff comments   View file @df6db81
1 require 'spec_helper' 1 require 'spec_helper'
2 2
3 describe Gitlab::Gitolite do 3 describe Gitlab::Gitolite do
4 - let(:project) { double('Project', path: 'diaspora') } 4 + let(:project) { double('Project', id: 7, path: 'diaspora') }
5 let(:gitolite_config) { double('Gitlab::GitoliteConfig') } 5 let(:gitolite_config) { double('Gitlab::GitoliteConfig') }
6 let(:gitolite) { Gitlab::Gitolite.new } 6 let(:gitolite) { Gitlab::Gitolite.new }
7 7
8 before do 8 before do
9 gitolite.stub(config: gitolite_config) 9 gitolite.stub(config: gitolite_config)
  10 + Project.stub(find: project)
10 end 11 end
11 12
12 it { should respond_to :set_key } 13 it { should respond_to :set_key }
@@ -20,6 +21,6 @@ describe Gitlab::Gitolite do @@ -20,6 +21,6 @@ describe Gitlab::Gitolite do
20 21
21 it "should call config update" do 22 it "should call config update" do
22 gitolite_config.should_receive(:update_project!) 23 gitolite_config.should_receive(:update_project!)
23 - gitolite.update_repository project 24 + gitolite.update_repository(project.id)
24 end 25 end
25 end 26 end
spec/support/stubbed_repository.rb
  Diff comments   View file @df6db81
@@ -21,6 +21,10 @@ class Project @@ -21,6 +21,10 @@ class Project
21 true 21 true
22 end 22 end
23 23
  24 + def destroy
  25 + true
  26 + end
  27 +
24 def create 28 def create
25 true 29 true
26 end 30 end