Download as
Browse Code »

Commit df6db81e2ab36410c377f8a9a712756653de0e2a

Authored by Dmitriy Zaporozhets
2 parents b4f16faa f7ade3b6
Exists in master and in 4 other branches 6-8-stable, 7-0-stable, 7-0-stable-spb, spb-stable

Merge branch 'features/async_gitolite' of dev.gitlabhq.com:gitlab/gitlabhq

Showing 16 changed files with 127 additions and 64 deletions   Show diff stats
Procfile
  Diff comments   View file @df6db81
1 1 web: bundle exec unicorn_rails -p $PORT
2   -worker: bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default
  2 +worker: bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default,gitolite
... ...
app/contexts/projects/create_context.rb
  Diff comments   View file @df6db81
... ... @@ -32,16 +32,10 @@ module Projects
32 32 @project.namespace_id = current_user.namespace_id
33 33 end
34 34  
35   - Project.transaction do
36   - @project.creator = current_user
37   - @project.save!
  35 + @project.creator = current_user
38 36  
39   - # Add user as project master
40   - @project.users_projects.create!(project_access: UsersProject::MASTER, user: current_user)
41   -
42   - # when project saved no team member exist so
43   - # project repository should be updated after first user add
44   - @project.update_repository
  37 + if @project.save
  38 + @project.users_projects.create(project_access: UsersProject::MASTER, user: current_user)
45 39 end
46 40  
47 41 @project
... ...
app/models/project.rb
  Diff comments   View file @df6db81
... ... @@ -299,6 +299,9 @@ class Project < ActiveRecord::Base
299 299 def trigger_post_receive(oldrev, newrev, ref, user)
300 300 data = post_receive_data(oldrev, newrev, ref, user)
301 301  
  302 + # Create satellite
  303 + self.satellite.create unless self.satellite.exists?
  304 +
302 305 # Create push event
303 306 self.observe_push(data)
304 307  
... ... @@ -313,9 +316,6 @@ class Project < ActiveRecord::Base
313 316 self.execute_services(data.dup)
314 317 end
315 318  
316   - # Create satellite
317   - self.satellite.create unless self.satellite.exists?
318   -
319 319 # Discover the default branch, but only if it hasn't already been set to
320 320 # something else
321 321 if repository && default_branch.nil?
... ... @@ -460,11 +460,17 @@ class Project < ActiveRecord::Base
460 460 end
461 461  
462 462 def update_repository
463   - gitolite.update_repository(self)
  463 + GitoliteWorker.perform_async(
  464 + :update_repository,
  465 + self.id
  466 + )
464 467 end
465 468  
466 469 def destroy_repository
467   - gitolite.remove_repository(self)
  470 + GitoliteWorker.perform_async(
  471 + :remove_repository,
  472 + self.path_with_namespace
  473 + )
468 474 end
469 475  
470 476 def repo_exists?
... ...
app/models/protected_branch.rb
  Diff comments   View file @df6db81
... ... @@ -22,7 +22,7 @@ class ProtectedBranch < ActiveRecord::Base
22 22 after_destroy :update_repository
23 23  
24 24 def update_repository
25   - gitolite.update_repository(project)
  25 + project.update_repository
26 26 end
27 27  
28 28 def commit
... ...
app/models/users_project.rb
  Diff comments   View file @df6db81
... ... @@ -82,9 +82,13 @@ class UsersProject < ActiveRecord::Base
82 82 users_project.save
83 83 end
84 84 end
85   - Gitlab::Gitolite.new.update_repositories(Project.where(id: project_ids))
86 85 end
87 86  
  87 + GitoliteWorker.perform_async(
  88 + :update_repositories,
  89 + project_ids
  90 + )
  91 +
88 92 true
89 93 rescue
90 94 false
... ... @@ -97,9 +101,13 @@ class UsersProject < ActiveRecord::Base
97 101 users_project.skip_git = true
98 102 users_project.destroy
99 103 end
100   - Gitlab::Gitolite.new.update_repositories(Project.where(id: project_ids))
101 104 end
102 105  
  106 + GitoliteWorker.perform_async(
  107 + :update_repositories,
  108 + project_ids
  109 + )
  110 +
103 111 true
104 112 rescue
105 113 false
... ... @@ -129,7 +137,7 @@ class UsersProject < ActiveRecord::Base
129 137 end
130 138  
131 139 def update_repository
132   - gitolite.update_repository(project)
  140 + project.update_repository
133 141 end
134 142  
135 143 def project_access_human
... ...
app/observers/project_observer.rb
  Diff comments   View file @df6db81
... ... @@ -10,6 +10,7 @@ class ProjectObserver < ActiveRecord::Observer
10 10 def after_destroy(project)
11 11 log_info("Project \"#{project.name}\" was removed")
12 12  
  13 + project.satellite.destroy
13 14 project.destroy_repository
14 15 end
15 16  
... ...
app/workers/gitolite_worker.rb 0 → 100644
  Diff comments   View file @df6db81
... ... @@ -0,0 +1,10 @@
  1 +class GitoliteWorker
  2 + include Sidekiq::Worker
  3 + include Gitolited
  4 +
  5 + sidekiq_options queue: :gitolite
  6 +
  7 + def perform(action, arg)
  8 + gitolite.send(action, arg)
  9 + end
  10 +end
... ...
app/workers/post_receive.rb
  Diff comments   View file @df6db81
... ... @@ -13,13 +13,14 @@ class PostReceive
13 13  
14 14 # Ignore push from non-gitlab users
15 15 user = if identifier.eql? Gitlab.config.gitolite.admin_key
16   - email = project.repository.commit(newrev).author.email rescue nil
17   - User.find_by_email(email) if email
18   - elsif /^[A-Z0-9._%a-z\-]+@(?:[A-Z0-9a-z\-]+\.)+[A-Za-z]{2,4}$/.match(identifier)
19   - User.find_by_email(identifier)
20   - else
21   - Key.find_by_identifier(identifier).try(:user)
22   - end
  16 + email = project.repository.commit(newrev).author.email rescue nil
  17 + User.find_by_email(email) if email
  18 + elsif /^[A-Z0-9._%a-z\-]+@(?:[A-Z0-9a-z\-]+\.)+[A-Za-z]{2,4}$/.match(identifier)
  19 + User.find_by_email(identifier)
  20 + else
  21 + Key.find_by_identifier(identifier).try(:user)
  22 + end
  23 +
23 24 return false unless user
24 25  
25 26 project.trigger_post_receive(oldrev, newrev, ref, user)
... ...
lib/gitlab/backend/gitolite.rb
  Diff comments   View file @df6db81
... ... @@ -22,7 +22,13 @@ module Gitlab
22 22 end
23 23 end
24 24  
25   - def update_repository project
  25 + # Update project config in gitolite by project id
  26 + #
  27 + # Ex.
  28 + # update_repository(23)
  29 + #
  30 + def update_repository(project_id)
  31 + project = Project.find(project_id)
26 32 config.update_project!(project)
27 33 end
28 34  
... ... @@ -33,8 +39,28 @@ module Gitlab
33 39 end
34 40 end
35 41  
36   - def remove_repository project
37   - config.destroy_project!(project)
  42 + # Remove repository from gitolite
  43 + #
  44 + # name - project path with namespace
  45 + #
  46 + # Ex.
  47 + # remove_repository("gitlab/gitlab-ci")
  48 + #
  49 + def remove_repository(name)
  50 + config.destroy_project!(name)
  51 + end
  52 +
  53 + # Update projects configs in gitolite by project ids
  54 + #
  55 + # Ex.
  56 + # update_repositories([1, 4, 6])
  57 + #
  58 + def update_repositories(project_ids)
  59 + projects = Project.where(id: project_ids)
  60 +
  61 + config.apply do |config|
  62 + config.update_projects(projects)
  63 + end
38 64 end
39 65  
40 66 def url_to_repo path
... ... @@ -45,12 +71,6 @@ module Gitlab
45 71 config.admin_all_repo!
46 72 end
47 73  
48   - def update_repositories projects
49   - config.apply do |config|
50   - config.update_projects(projects)
51   - end
52   - end
53   -
54 74 alias_method :create_repository, :update_repository
55 75 end
56 76 end
... ...
lib/gitlab/backend/gitolite_config.rb
  Diff comments   View file @df6db81
... ... @@ -4,6 +4,8 @@ require 'fileutils'
4 4  
5 5 module Gitlab
6 6 class GitoliteConfig
  7 + include Gitlab::Popen
  8 +
7 9 class PullError < StandardError; end
8 10 class PushError < StandardError; end
9 11 class BrokenGitolite < StandardError; end
... ... @@ -87,12 +89,14 @@ module Gitlab
87 89 Gitlab::GitLogger.error(message)
88 90 end
89 91  
90   - def destroy_project(project)
91   - # do rm-rf only if repository exists
92   - if project.repository
93   - FileUtils.rm_rf(project.repository.path_to_repo)
94   - end
95   - conf.rm_repo(project.path_with_namespace)
  92 + def path_to_repo(name)
  93 + File.join(Gitlab.config.gitolite.repos_path, "#{name}.git")
  94 + end
  95 +
  96 + def destroy_project(name)
  97 + full_path = path_to_repo(name)
  98 + FileUtils.rm_rf(full_path) if File.exists?(full_path)
  99 + conf.rm_repo(name)
96 100 end
97 101  
98 102 def clean_repo repo_name
... ... @@ -210,14 +214,14 @@ module Gitlab
210 214 end
211 215  
212 216 def push
213   - output, status = popen('git add -A')
  217 + output, status = popen('git add -A', tmp_conf_path)
214 218 raise "Git add failed." unless status.zero?
215 219  
216 220 # git commit returns 0 on success, and 1 if there is nothing to commit
217   - output, status = popen('git commit -m "GitLab"')
  221 + output, status = popen('git commit -m "GitLab"', tmp_conf_path)
218 222 raise "Git add failed." unless [0,1].include?(status)
219 223  
220   - output, status = popen('git push')
  224 + output, status = popen('git push', tmp_conf_path)
221 225  
222 226 if output =~ /remote\: FATAL/
223 227 raise BrokenGitolite, output
... ... @@ -230,20 +234,8 @@ module Gitlab
230 234 end
231 235 end
232 236  
233   - def popen(cmd, path = nil)
234   - path ||= File.join(config_tmp_dir,'gitolite')
235   - vars = { "PWD" => path }
236   - options = { :chdir => path }
237   -
238   - @cmd_output = ""
239   - @cmd_status = 0
240   - Open3.popen3(vars, cmd, options) do |stdin, stdout, stderr, wait_thr|
241   - @cmd_status = wait_thr.value.exitstatus
242   - @cmd_output << stdout.read
243   - @cmd_output << stderr.read
244   - end
245   -
246   - return @cmd_output, @cmd_status
  237 + def tmp_conf_path
  238 + File.join(config_tmp_dir,'gitolite')
247 239 end
248 240 end
249 241 end
... ...
lib/gitlab/popen.rb 0 → 100644
  Diff comments   View file @df6db81
... ... @@ -0,0 +1,18 @@
  1 +module Gitlab
  2 + module Popen
  3 + def popen(cmd, path)
  4 + vars = { "PWD" => path }
  5 + options = { :chdir => path }
  6 +
  7 + @cmd_output = ""
  8 + @cmd_status = 0
  9 + Open3.popen3(vars, cmd, options) do |stdin, stdout, stderr, wait_thr|
  10 + @cmd_status = wait_thr.value.exitstatus
  11 + @cmd_output << stdout.read
  12 + @cmd_output << stderr.read
  13 + end
  14 +
  15 + return @cmd_output, @cmd_status
  16 + end
  17 + end
  18 +end
... ...
lib/gitlab/satellite/satellite.rb
  Diff comments   View file @df6db81
... ... @@ -3,6 +3,8 @@ module Gitlab
3 3  
4 4 module Satellite
5 5 class Satellite
  6 + include Gitlab::Popen
  7 +
6 8 PARKING_BRANCH = "__parking_branch"
7 9  
8 10 attr_accessor :project
... ... @@ -24,8 +26,10 @@ module Gitlab
24 26 end
25 27  
26 28 def create
27   - create_cmd = "git clone #{project.url_to_repo} #{path}"
28   - if system(create_cmd)
  29 + output, status = popen("git clone #{project.url_to_repo} #{path}",
  30 + Gitlab.config.satellites.path)
  31 +
  32 + if status.zero?
29 33 true
30 34 else
31 35 Gitlab::GitLogger.error("Failed to create satellite for #{project.name_with_namespace}")
... ... @@ -66,6 +70,10 @@ module Gitlab
66 70 @repo ||= Grit::Repo.new(path)
67 71 end
68 72  
  73 + def destroy
  74 + FileUtils.rm_rf(path)
  75 + end
  76 +
69 77 private
70 78  
71 79 # Clear the working directory
... ...
lib/tasks/sidekiq.rake
  Diff comments   View file @df6db81
... ... @@ -6,7 +6,7 @@ namespace :sidekiq do
6 6  
7 7 desc "GITLAB | Start sidekiq"
8 8 task :start do
9   - run "nohup bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default -e #{Rails.env} -P #{pidfile} >> #{Rails.root.join("log", "sidekiq.log")} 2>&1 &"
  9 + run "nohup bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,gitolite,common,default -e #{Rails.env} -P #{pidfile} >> #{Rails.root.join("log", "sidekiq.log")} 2>&1 &"
10 10 end
11 11  
12 12 def pidfile
... ...
spec/factories.rb
  Diff comments   View file @df6db81
... ... @@ -12,7 +12,7 @@ FactoryGirl.define do
12 12 factory :user, aliases: [:author, :assignee, :owner, :creator] do
13 13 email { Faker::Internet.email }
14 14 name
15   - username { Faker::Internet.user_name }
  15 + sequence(:username) { |n| "#{Faker::Internet.user_name}#{n}" }
16 16 password "123456"
17 17 password_confirmation { password }
18 18  
... ...
spec/lib/gitolite_spec.rb
  Diff comments   View file @df6db81
1 1 require 'spec_helper'
2 2  
3 3 describe Gitlab::Gitolite do
4   - let(:project) { double('Project', path: 'diaspora') }
  4 + let(:project) { double('Project', id: 7, path: 'diaspora') }
5 5 let(:gitolite_config) { double('Gitlab::GitoliteConfig') }
6 6 let(:gitolite) { Gitlab::Gitolite.new }
7 7  
8 8 before do
9 9 gitolite.stub(config: gitolite_config)
  10 + Project.stub(find: project)
10 11 end
11 12  
12 13 it { should respond_to :set_key }
... ... @@ -20,6 +21,6 @@ describe Gitlab::Gitolite do
20 21  
21 22 it "should call config update" do
22 23 gitolite_config.should_receive(:update_project!)
23   - gitolite.update_repository project
  24 + gitolite.update_repository(project.id)
24 25 end
25 26 end
... ...
spec/support/stubbed_repository.rb
  Diff comments   View file @df6db81
... ... @@ -21,6 +21,10 @@ class Project
21 21 true
22 22 end
23 23  
  24 + def destroy
  25 + true
  26 + end
  27 +
24 28 def create
25 29 true
26 30 end
... ...