Repository import during project creation often return timeout for medium and large repos.

So lets do it async. First create project, then import repo and create satellite with Sidekiq

Repository import during project creation often return timeout for medium and large repos.
So lets do it async. First create project, then import repo and create satellite with Sidekiq
Dmitriy Zaporozhets
1 parent e2f946fd
Showing 7 changed files with 80 additions and 55 deletions Show diff stats
app/contexts/projects/create_context.rb
app/models/project.rb
app/observers/project_observer.rb
app/views/projects/empty.html.haml
app/workers/repository_import_worker.rb
db/migrate/20130812143708_add_import_url_to_project.rb
db/schema.rb
@@ -45,20 +45,6 @@ module Projects
       @project.creator = current_user
-      # Import project from cloneable resource
-      if @project.valid? && @project.import_url.present?
-        shell = Gitlab::Shell.new
-
-        if shell.import_repository(@project.path_with_namespace, @project.import_url)
-          # We should create satellite for imported repo
-          @project.satellite.create unless @project.satellite.exists?
-          @project.imported = true
-          true
-        else
-          @project.errors.add(:import_url, 'cannot clone repo')
-        end
-      end
-
       if @project.save
         unless @project.group
           @project.users_projects.create(project_access: UsersProject::MASTER, user: current_user)
@@ -37,8 +37,6 @@ class Project &lt; ActiveRecord::Base
   acts_as_taggable_on :labels, :issues_default_labels
-  attr_accessor :import_url
-
   # Relations
   belongs_to :creator,      foreign_key: "creator_id", class_name: "User"
   belongs_to :group,        foreign_key: "namespace_id", conditions: "type = 'Group'"
@@ -157,6 +155,10 @@ class Project &lt; ActiveRecord::Base
     import_url.present?
   end
+  def imported?
+    imported
+  end
+
   def check_limit
     unless creator.can_create_project?
       errors[:limit_reached] << ("Your own projects limit is #{creator.projects_limit}! Please contact administrator to increase it")
@@ -411,10 +413,6 @@ class Project &lt; ActiveRecord::Base
     !(forked_project_link.nil? || forked_project_link.forked_from_project.nil?)
   end
-  def imported?
-    imported
-  end
-
   def personal?
     !group
   end
 class ProjectObserver < BaseObserver
   def after_create(project)
-    return true if project.forked? || project.imported?
-
-    GitlabShellWorker.perform_async(
-      :add_repository,
-      project.path_with_namespace
-    )
-
-    log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
+    return true if project.forked?
+
+    if project.import?
+      RepositoryImportWorker.perform_in(5.seconds, project.id)
+    else
+      GitlabShellWorker.perform_async(
+        :add_repository,
+        project.path_with_namespace
+      )
+
+      log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
+    end
   end
   def after_update(project)
 = render 'clone_panel'
-%div.git-empty
-  %fieldset
-    %legend Git global setup:
-    %pre.dark
-      :preserve
-        git config --global user.name "#{current_user.name}"
-        git config --global user.email "#{current_user.email}"
+- if @project.import? && !@project.imported
+  .save-project-loader
+    %center
+      = image_tag "ajax_loader.gif"
+      %h3 Importing repository.
+      %p.monospace git clone --bare #{@project.import_url}
+      %p Please wait until we import repository for you. Refresh at will.
-  %fieldset
-    %legend Create Repository
-    %pre.dark
-      :preserve
-        mkdir #{@project.path}
-        cd #{@project.path}
-        git init
-        touch README
-        git add README
-        git commit -m 'first commit'
-        git remote add origin #{@project.url_to_repo}
-        git push -u origin master
+- else
+  %div.git-empty
+    %fieldset
+      %legend Git global setup:
+      %pre.dark
+        :preserve
+          git config --global user.name "#{current_user.name}"
+          git config --global user.email "#{current_user.email}"
-  %fieldset
-    %legend Existing Git Repo?
-    %pre.dark
-      :preserve
-        cd existing_git_repo
-        git remote add origin #{@project.url_to_repo}
-        git push -u origin master
+    %fieldset
+      %legend Create Repository
+      %pre.dark
+        :preserve
+          mkdir #{@project.path}
+          cd #{@project.path}
+          git init
+          touch README
+          git add README
+          git commit -m 'first commit'
+          git remote add origin #{@project.url_to_repo}
+          git push -u origin master
+
+    %fieldset
+      %legend Existing Git Repo?
+      %pre.dark
+        :preserve
+          cd existing_git_repo
+          git remote add origin #{@project.url_to_repo}
+          git push -u origin master
   - if can? current_user, :remove_project, @project
     .prepend-top-20
@@ -0,0 +1,22 @@
+class RepositoryImportWorker
+  include Sidekiq::Worker
+  include Gitlab::ShellAdapter
+
+  sidekiq_options queue: :gitlab_shell
+
+  def perform(project_id)
+    project = Project.find(project_id)
+    result = gitlab_shell.send(:import_repository,
+                               project.path_with_namespace,
+                               project.import_url)
+
+    if result
+      project.imported = true
+      project.save
+      project.satellite.create unless project.satellite.exists?
+      project.discover_default_branch
+    else
+      project.imported = false
+    end
+  end
+end
@@ -0,0 +1,5 @@
+class AddImportUrlToProject < ActiveRecord::Migration
+  def change
+    add_column :projects, :import_url, :string
+  end
+end
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
-ActiveRecord::Schema.define(:version => 20130804151314) do
+ActiveRecord::Schema.define(:version => 20130812143708) do
   create_table "deploy_keys_projects", :force => true do |t|
     t.integer  "deploy_key_id", :null => false
@@ -178,6 +178,7 @@ ActiveRecord::Schema.define(:version =&gt; 20130804151314) do
     t.boolean  "snippets_enabled",       :default => true,     :null => false
     t.datetime "last_activity_at"
     t.boolean  "imported",               :default => false,    :null => false
+    t.string   "import_url"
   end
   add_index "projects", ["creator_id"], :name => "index_projects_on_owner_id"
	@@ -45,20 +45,6 @@ module Projects		@@ -45,20 +45,6 @@ module Projects
45		45
46	@project.creator = current_user	46	@project.creator = current_user
47		47
48	- # Import project from cloneable resource
49	- if @project.valid? && @project.import_url.present?
50	- shell = Gitlab::Shell.new
51	-
52	- if shell.import_repository(@project.path_with_namespace, @project.import_url)
53	- # We should create satellite for imported repo
54	- @project.satellite.create unless @project.satellite.exists?
55	- @project.imported = true
56	- true
57	- else
58	- @project.errors.add(:import_url, 'cannot clone repo')
59	- end
60	- end
61	-
62	if @project.save	48	if @project.save
63	unless @project.group	49	unless @project.group
64	@project.users_projects.create(project_access: UsersProject::MASTER, user: current_user)	50	@project.users_projects.create(project_access: UsersProject::MASTER, user: current_user)
@@ -0,0 +1,22 @@		@@ -0,0 +1,22 @@
	1	+class RepositoryImportWorker
	2	+ include Sidekiq::Worker
	3	+ include Gitlab::ShellAdapter
	4	+
	5	+ sidekiq_options queue: :gitlab_shell
	6	+
	7	+ def perform(project_id)
	8	+ project = Project.find(project_id)
	9	+ result = gitlab_shell.send(:import_repository,
	10	+ project.path_with_namespace,
	11	+ project.import_url)
	12	+
	13	+ if result
	14	+ project.imported = true
	15	+ project.save
	16	+ project.satellite.create unless project.satellite.exists?
	17	+ project.discover_default_branch
	18	+ else
	19	+ project.imported = false
	20	+ end
	21	+ end
	22	+end
@@ -0,0 +1,5 @@		@@ -0,0 +1,5 @@
	1	+class AddImportUrlToProject < ActiveRecord::Migration
	2	+ def change
	3	+ add_column :projects, :import_url, :string
	4	+ end
	5	+end