Repository import during project creation often return timeout for medium and large repos.

So lets do it async. First create project, then import repo and create satellite with Sidekiq

Repository import during project creation often return timeout for medium and large repos.
So lets do it async. First create project, then import repo and create satellite with Sidekiq
Dmitriy Zaporozhets
1 parent e2f946fd
Showing 7 changed files with 80 additions and 55 deletions Show diff stats
app/contexts/projects/create_context.rb
app/models/project.rb
app/observers/project_observer.rb
app/views/projects/empty.html.haml
app/workers/repository_import_worker.rb
db/migrate/20130812143708_add_import_url_to_project.rb
db/schema.rb
@@ -45,20 +45,6 @@ module Projects
  
       @project.creator = current_user
  
-      # Import project from cloneable resource
-      if @project.valid? && @project.import_url.present?
-        shell = Gitlab::Shell.new
-
-        if shell.import_repository(@project.path_with_namespace, @project.import_url)
-          # We should create satellite for imported repo
-          @project.satellite.create unless @project.satellite.exists?
-          @project.imported = true
-          true
-        else
-          @project.errors.add(:import_url, 'cannot clone repo')
-        end
-      end
-
       if @project.save
         unless @project.group
           @project.users_projects.create(project_access: UsersProject::MASTER, user: current_user)
@@ -37,8 +37,6 @@ class Project &lt; ActiveRecord::Base
  
   acts_as_taggable_on :labels, :issues_default_labels
  
-  attr_accessor :import_url
-
   # Relations
   belongs_to :creator,      foreign_key: "creator_id", class_name: "User"
   belongs_to :group,        foreign_key: "namespace_id", conditions: "type = 'Group'"
@@ -157,6 +155,10 @@ class Project &lt; ActiveRecord::Base
     import_url.present?
   end
  
+  def imported?
+    imported
+  end
+
   def check_limit
     unless creator.can_create_project?
       errors[:limit_reached] << ("Your own projects limit is #{creator.projects_limit}! Please contact administrator to increase it")
@@ -411,10 +413,6 @@ class Project &lt; ActiveRecord::Base
     !(forked_project_link.nil? || forked_project_link.forked_from_project.nil?)
   end
  
-  def imported?
-    imported
-  end
-
   def personal?
     !group
   end
 class ProjectObserver < BaseObserver
   def after_create(project)
-    return true if project.forked? || project.imported?
-
-    GitlabShellWorker.perform_async(
-      :add_repository,
-      project.path_with_namespace
-    )
-
-    log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
+    return true if project.forked?
+
+    if project.import?
+      RepositoryImportWorker.perform_in(5.seconds, project.id)
+    else
+      GitlabShellWorker.perform_async(
+        :add_repository,
+        project.path_with_namespace
+      )
+
+      log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
+    end
   end
  
   def after_update(project)
 = render 'clone_panel'
  
-%div.git-empty
-  %fieldset
-    %legend Git global setup:
-    %pre.dark
-      :preserve
-        git config --global user.name "#{current_user.name}"
-        git config --global user.email "#{current_user.email}"
+- if @project.import? && !@project.imported
+  .save-project-loader
+    %center
+      = image_tag "ajax_loader.gif"
+      %h3 Importing repository.
+      %p.monospace git clone --bare #{@project.import_url}
+      %p Please wait until we import repository for you. Refresh at will.
  
-  %fieldset
-    %legend Create Repository
-    %pre.dark
-      :preserve
-        mkdir #{@project.path}
-        cd #{@project.path}
-        git init
-        touch README
-        git add README
-        git commit -m 'first commit'
-        git remote add origin #{@project.url_to_repo}
-        git push -u origin master
+- else
+  %div.git-empty
+    %fieldset
+      %legend Git global setup:
+      %pre.dark
+        :preserve
+          git config --global user.name "#{current_user.name}"
+          git config --global user.email "#{current_user.email}"
  
-  %fieldset
-    %legend Existing Git Repo?
-    %pre.dark
-      :preserve
-        cd existing_git_repo
-        git remote add origin #{@project.url_to_repo}
-        git push -u origin master
+    %fieldset
+      %legend Create Repository
+      %pre.dark
+        :preserve
+          mkdir #{@project.path}
+          cd #{@project.path}
+          git init
+          touch README
+          git add README
+          git commit -m 'first commit'
+          git remote add origin #{@project.url_to_repo}
+          git push -u origin master
+
+    %fieldset
+      %legend Existing Git Repo?
+      %pre.dark
+        :preserve
+          cd existing_git_repo
+          git remote add origin #{@project.url_to_repo}
+          git push -u origin master
  
   - if can? current_user, :remove_project, @project
     .prepend-top-20
@@ -0,0 +1,22 @@
+class RepositoryImportWorker
+  include Sidekiq::Worker
+  include Gitlab::ShellAdapter
+
+  sidekiq_options queue: :gitlab_shell
+
+  def perform(project_id)
+    project = Project.find(project_id)
+    result = gitlab_shell.send(:import_repository,
+                               project.path_with_namespace,
+                               project.import_url)
+
+    if result
+      project.imported = true
+      project.save
+      project.satellite.create unless project.satellite.exists?
+      project.discover_default_branch
+    else
+      project.imported = false
+    end
+  end
+end
@@ -0,0 +1,5 @@
+class AddImportUrlToProject < ActiveRecord::Migration
+  def change
+    add_column :projects, :import_url, :string
+  end
+end
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
  
-ActiveRecord::Schema.define(:version => 20130804151314) do
+ActiveRecord::Schema.define(:version => 20130812143708) do
  
   create_table "deploy_keys_projects", :force => true do |t|
     t.integer  "deploy_key_id", :null => false
@@ -178,6 +178,7 @@ ActiveRecord::Schema.define(:version =&gt; 20130804151314) do
     t.boolean  "snippets_enabled",       :default => true,     :null => false
     t.datetime "last_activity_at"
     t.boolean  "imported",               :default => false,    :null => false
+    t.string   "import_url"
   end
  
   add_index "projects", ["creator_id"], :name => "index_projects_on_owner_id"
...	...	@@ -45,20 +45,6 @@ module Projects
45	45
46	46	@project.creator = current_user
47	47
48		- # Import project from cloneable resource
49		- if @project.valid? && @project.import_url.present?
50		- shell = Gitlab::Shell.new
51		-
52		- if shell.import_repository(@project.path_with_namespace, @project.import_url)
53		- # We should create satellite for imported repo
54		- @project.satellite.create unless @project.satellite.exists?
55		- @project.imported = true
56		- true
57		- else
58		- @project.errors.add(:import_url, 'cannot clone repo')
59		- end
60		- end
61		-
62	48	if @project.save
63	49	unless @project.group
64	50	@project.users_projects.create(project_access: UsersProject::MASTER, user: current_user)
...	...
...	...	@@ -37,8 +37,6 @@ class Project < ActiveRecord::Base
37	37
38	38	acts_as_taggable_on :labels, :issues_default_labels
39	39
40		- attr_accessor :import_url
41		-
42	40	# Relations
43	41	belongs_to :creator, foreign_key: "creator_id", class_name: "User"
44	42	belongs_to :group, foreign_key: "namespace_id", conditions: "type = 'Group'"
...	...	@@ -157,6 +155,10 @@ class Project < ActiveRecord::Base
157	155	import_url.present?
158	156	end
159	157
	158	+ def imported?
	159	+ imported
	160	+ end
	161	+
160	162	def check_limit
161	163	unless creator.can_create_project?
162	164	errors[:limit_reached] << ("Your own projects limit is #{creator.projects_limit}! Please contact administrator to increase it")
...	...	@@ -411,10 +413,6 @@ class Project < ActiveRecord::Base
411	413	!(forked_project_link.nil? \|\| forked_project_link.forked_from_project.nil?)
412	414	end
413	415
414		- def imported?
415		- imported
416		- end
417		-
418	416	def personal?
419	417	!group
420	418	end
...	...
1	1	class ProjectObserver < BaseObserver
2	2	def after_create(project)
3		- return true if project.forked? \|\| project.imported?
4		-
5		- GitlabShellWorker.perform_async(
6		- :add_repository,
7		- project.path_with_namespace
8		- )
9		-
10		- log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
	3	+ return true if project.forked?
	4	+
	5	+ if project.import?
	6	+ RepositoryImportWorker.perform_in(5.seconds, project.id)
	7	+ else
	8	+ GitlabShellWorker.perform_async(
	9	+ :add_repository,
	10	+ project.path_with_namespace
	11	+ )
	12	+
	13	+ log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
	14	+ end
11	15	end
12	16
13	17	def after_update(project)
...	...
1	1	= render 'clone_panel'
2	2
3		-%div.git-empty
4		- %fieldset
5		- %legend Git global setup:
6		- %pre.dark
7		- :preserve
8		- git config --global user.name "#{current_user.name}"
9		- git config --global user.email "#{current_user.email}"
	3	+- if @project.import? && !@project.imported
	4	+ .save-project-loader
	5	+ %center
	6	+ = image_tag "ajax_loader.gif"
	7	+ %h3 Importing repository.
	8	+ %p.monospace git clone --bare #{@project.import_url}
	9	+ %p Please wait until we import repository for you. Refresh at will.
10	10
11		- %fieldset
12		- %legend Create Repository
13		- %pre.dark
14		- :preserve
15		- mkdir #{@project.path}
16		- cd #{@project.path}
17		- git init
18		- touch README
19		- git add README
20		- git commit -m 'first commit'
21		- git remote add origin #{@project.url_to_repo}
22		- git push -u origin master
	11	+- else
	12	+ %div.git-empty
	13	+ %fieldset
	14	+ %legend Git global setup:
	15	+ %pre.dark
	16	+ :preserve
	17	+ git config --global user.name "#{current_user.name}"
	18	+ git config --global user.email "#{current_user.email}"
23	19
24		- %fieldset
25		- %legend Existing Git Repo?
26		- %pre.dark
27		- :preserve
28		- cd existing_git_repo
29		- git remote add origin #{@project.url_to_repo}
30		- git push -u origin master
	20	+ %fieldset
	21	+ %legend Create Repository
	22	+ %pre.dark
	23	+ :preserve
	24	+ mkdir #{@project.path}
	25	+ cd #{@project.path}
	26	+ git init
	27	+ touch README
	28	+ git add README
	29	+ git commit -m 'first commit'
	30	+ git remote add origin #{@project.url_to_repo}
	31	+ git push -u origin master
	32	+
	33	+ %fieldset
	34	+ %legend Existing Git Repo?
	35	+ %pre.dark
	36	+ :preserve
	37	+ cd existing_git_repo
	38	+ git remote add origin #{@project.url_to_repo}
	39	+ git push -u origin master
31	40
32	41	- if can? current_user, :remove_project, @project
33	42	.prepend-top-20
...	...
...	...	@@ -0,0 +1,22 @@
	1	+class RepositoryImportWorker
	2	+ include Sidekiq::Worker
	3	+ include Gitlab::ShellAdapter
	4	+
	5	+ sidekiq_options queue: :gitlab_shell
	6	+
	7	+ def perform(project_id)
	8	+ project = Project.find(project_id)
	9	+ result = gitlab_shell.send(:import_repository,
	10	+ project.path_with_namespace,
	11	+ project.import_url)
	12	+
	13	+ if result
	14	+ project.imported = true
	15	+ project.save
	16	+ project.satellite.create unless project.satellite.exists?
	17	+ project.discover_default_branch
	18	+ else
	19	+ project.imported = false
	20	+ end
	21	+ end
	22	+end
...	...
...	...	@@ -0,0 +1,5 @@
	1	+class AddImportUrlToProject < ActiveRecord::Migration
	2	+ def change
	3	+ add_column :projects, :import_url, :string
	4	+ end
	5	+end
...	...
...	...	@@ -11,7 +11,7 @@
11	11	#
12	12	# It's strongly recommended to check this file into your version control system.
13	13
14		-ActiveRecord::Schema.define(:version => 20130804151314) do
	14	+ActiveRecord::Schema.define(:version => 20130812143708) do
15	15
16	16	create_table "deploy_keys_projects", :force => true do \|t\|
17	17	t.integer "deploy_key_id", :null => false
...	...	@@ -178,6 +178,7 @@ ActiveRecord::Schema.define(:version => 20130804151314) do
178	178	t.boolean "snippets_enabled", :default => true, :null => false
179	179	t.datetime "last_activity_at"
180	180	t.boolean "imported", :default => false, :null => false
	181	+ t.string "import_url"
181	182	end
182	183
183	184	add_index "projects", ["creator_id"], :name => "index_projects_on_owner_id"
...	...