Merge branch 'import-timeout' of https://dev.gitlab.org/dzaporozhets/gitlabhq in…

…to dzaporozhets/gitlabhq-import-timeout Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> Conflicts: CHANGELOG db/schema.rb

Merge branch 'import-timeout' of https://dev.gitlab.org/dzaporozhets/gitlabhq in…
…to dzaporozhets/gitlabhq-import-timeout Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> Conflicts: CHANGELOG db/schema.rb
Dmitriy Zaporozhets
2 parents 58e4e6b0 9064fba0
Showing 14 changed files with 189 additions and 98 deletions Show diff stats
CHANGELOG
app/controllers/projects_controller.rb
app/models/project.rb
app/observers/project_observer.rb
app/services/projects/create_service.rb
app/views/projects/create.js.haml
app/views/projects/empty.html.haml
app/views/projects/import.html.haml
app/views/projects/new.html.haml
app/workers/repository_import_worker.rb
config/routes.rb
db/migrate/20140312145357_add_import_status_to_project.rb
db/migrate/20140313092127_migrate_already_imported_projects.rb
db/schema.rb
@@ -17,6 +17,7 @@ v 6.7.0
   - Add GFM autocompletion for MergeRequests (Robert Speicher)
   - Add webhook when a new tag is pushed (Jeroen van Baarsen)
   - Add button for toggling inline comments in diff view
+  - Add retry feature for repository import
 v 6.6.2
   - Fix 500 error on branch/tag create or remove via UI
@@ -5,7 +5,7 @@ class ProjectsController &lt; ApplicationController
   # Authorize
   before_filter :authorize_read_project!, except: [:index, :new, :create]
-  before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive]
+  before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive, :retry_import]
   before_filter :require_non_empty_project, only: [:blob, :tree, :graph]
   layout 'navless', only: [:new, :create, :fork]
@@ -21,16 +21,9 @@ class ProjectsController &lt; ApplicationController
   def create
     @project = ::Projects::CreateService.new(current_user, params[:project]).execute
+    flash[:notice] = 'Project was successfully created.' if @project.saved?
     respond_to do |format|
-      flash[:notice] = 'Project was successfully created.' if @project.saved?
-      format.html do
-        if @project.saved?
-          redirect_to @project
-        else
-          render "new"
-        end
-      end
       format.js
     end
   end
@@ -55,6 +48,11 @@ class ProjectsController &lt; ApplicationController
   end
   def show
+    if @project.import_in_progress?
+      redirect_to import_project_path(@project)
+      return
+    end
+
     return authenticate_user! unless @project.public? || current_user
     limit = (params[:limit] || 20).to_i
@@ -67,9 +65,7 @@ class ProjectsController &lt; ApplicationController
         if @project.empty_repo?
           render "projects/empty", layout: user_layout
         else
-          if current_user
-            @last_push = current_user.recent_push(@project.id)
-          end
+          @last_push = current_user.recent_push(@project.id) if current_user
           render :show, layout: user_layout
         end
       end
@@ -77,6 +73,28 @@ class ProjectsController &lt; ApplicationController
     end
   end
+  def import
+    if project.import_finished?
+      redirect_to @project
+      return
+    end
+  end
+
+  def retry_import
+    unless @project.import_failed?
+      redirect_to import_project_path(@project)
+    end
+
+    @project.import_url = params[:project][:import_url]
+
+    if @project.save
+      @project.reload
+      @project.import_retry
+    end
+
+    redirect_to import_project_path(@project)
+  end
+
   def destroy
     return access_denied! unless can?(current_user, :remove_project, project)
@@ -28,7 +28,6 @@ class Project &lt; ActiveRecord::Base
   include Gitlab::VisibilityLevel
   extend Enumerize
-  default_value_for :imported, false
   default_value_for :archived, false
   ActsAsTaggableOn.strict_case_match = true
@@ -59,13 +58,10 @@ class Project &lt; ActiveRecord::Base
   has_one :gemnasium_service, dependent: :destroy
   has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id"
   has_one :forked_from_project, through: :forked_project_link
-
   # Merge Requests for target project should be removed with it
   has_many :merge_requests,     dependent: :destroy, foreign_key: "target_project_id"
-
   # Merge requests from source project should be kept when source project was removed
   has_many :fork_merge_requests, foreign_key: "source_project_id", class_name: MergeRequest
-
   has_many :issues, -> { order "state DESC, created_at DESC" }, dependent: :destroy
   has_many :services,           dependent: :destroy
   has_many :events,             dependent: :destroy
@@ -74,10 +70,8 @@ class Project &lt; ActiveRecord::Base
   has_many :snippets,           dependent: :destroy, class_name: "ProjectSnippet"
   has_many :hooks,              dependent: :destroy, class_name: "ProjectHook"
   has_many :protected_branches, dependent: :destroy
-
   has_many :users_projects, dependent: :destroy
   has_many :users, through: :users_projects
-
   has_many :deploy_keys_projects, dependent: :destroy
   has_many :deploy_keys, through: :deploy_keys_projects
@@ -97,15 +91,12 @@ class Project &lt; ActiveRecord::Base
   validates :issues_enabled, :wall_enabled, :merge_requests_enabled,
             :wiki_enabled, inclusion: { in: [true, false] }
   validates :issues_tracker_id, length: { maximum: 255 }, allow_blank: true
-
   validates :namespace, presence: true
   validates_uniqueness_of :name, scope: :namespace_id
   validates_uniqueness_of :path, scope: :namespace_id
-
   validates :import_url,
     format: { with: URI::regexp(%w(git http https)), message: "should be a valid url" },
     if: :import?
-
   validate :check_limit, on: :create
   # Scopes
@@ -118,14 +109,36 @@ class Project &lt; ActiveRecord::Base
   scope :sorted_by_activity, -> { reorder("projects.last_activity_at DESC") }
   scope :personal, ->(user) { where(namespace_id: user.namespace_id) }
   scope :joined, ->(user) { where("namespace_id != ?", user.namespace_id) }
-
   scope :public_only, -> { where(visibility_level: Project::PUBLIC) }
   scope :public_and_internal_only, -> { where(visibility_level: Project.public_and_internal_levels) }
-
   scope :non_archived, -> { where(archived: false) }
   enumerize :issues_tracker, in: (Gitlab.config.issues_tracker.keys).append(:gitlab), default: :gitlab
+  state_machine :import_status, initial: :none do
+    event :import_start do
+      transition :none => :started
+    end
+
+    event :import_finish do
+      transition :started => :finished
+    end
+
+    event :import_fail do
+      transition :started => :failed
+    end
+
+    event :import_retry do
+      transition :failed => :started
+    end
+
+    state :started
+    state :finished
+    state :failed
+
+    after_transition any => :started, :do => :add_import_job
+  end
+
   class << self
     def public_and_internal_levels
       [Project::PUBLIC, Project::INTERNAL]
@@ -202,12 +215,28 @@ class Project &lt; ActiveRecord::Base
     id && persisted?
   end
+  def add_import_job
+    RepositoryImportWorker.perform_in(2.seconds, id)
+  end
+
   def import?
     import_url.present?
   end
   def imported?
-    imported
+    import_finished?
+  end
+
+  def import_in_progress?
+    import? && import_status == 'started'
+  end
+
+  def import_failed?
+    import_status == 'failed'
+  end
+
+  def import_finished?
+    import_status == 'finished'
   end
   def check_limit
 class ProjectObserver < BaseObserver
   def after_create(project)
-    project.update_column(:last_activity_at, project.created_at)
-
-    return true if project.forked?
-
-    if project.import?
-      RepositoryImportWorker.perform_in(5.seconds, project.id)
-    else
-      GitlabShellWorker.perform_async(
-        :add_repository,
-        project.path_with_namespace
-      )
-
-      log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
-    end
-
-    if project.wiki_enabled?
-      begin
-        # force the creation of a wiki,
-        GollumWiki.new(project, project.owner).wiki
-      rescue GollumWiki::CouldNotCreateWikiError => ex
-        # Prevent project observer crash
-        # if failed to create wiki
-        nil
-      end
-    end
+    log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
   end
   def after_update(project)
@@ -58,6 +58,29 @@ module Projects
             user: current_user
           )
         end
+
+        @project.update_column(:last_activity_at, @project.created_at)
+
+        if @project.import?
+          @project.import_start
+        else
+          GitlabShellWorker.perform_async(
+            :add_repository,
+            @project.path_with_namespace
+          )
+
+        end
+
+        if @project.wiki_enabled?
+          begin
+            # force the creation of a wiki,
+            GollumWiki.new(@project, @project.owner).wiki
+          rescue GollumWiki::CouldNotCreateWikiError => ex
+            # Prevent project observer crash
+            # if failed to create wiki
+            nil
+          end
+        end
       end
       @project
 - if @project.saved?
-  :plain
-    location.href = "#{project_path(@project)}";
+  - if @project.import?
+    :plain
+      location.href = "#{import_project_path(@project)}";
+  - else
+    :plain
+      location.href = "#{project_path(@project)}";
 - else
   :plain
     $(".project-edit-errors").html("#{escape_javascript(render('errors'))}");
 = render "home_panel"
-- if @project.import? && !@project.imported
-  .save-project-loader
-    %center
-      %h2
-        %i.icon-spinner.icon-spin
-        Importing repository.
-      %p.monospace git clone --bare #{@project.import_url}
-      %p Please wait while we import the repository for you. Refresh at will.
-      :javascript
-        new ProjectImport();
+%div.git-empty
+  %fieldset
+    %legend Git global setup:
+    %pre.dark
+      :preserve
+        git config --global user.name "#{git_user_name}"
+        git config --global user.email "#{git_user_email}"
-- else
-  %div.git-empty
-    %fieldset
-      %legend Git global setup:
-      %pre.dark
-        :preserve
-          git config --global user.name "#{git_user_name}"
-          git config --global user.email "#{git_user_email}"
+  %fieldset
+    %legend Create Repository
+    %pre.dark
+      :preserve
+        mkdir #{@project.path}
+        cd #{@project.path}
+        git init
+        touch README
+        git add README
+        git commit -m 'first commit'
+        git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
+        git push -u origin master
-    %fieldset
-      %legend Create Repository
-      %pre.dark
-        :preserve
-          mkdir #{@project.path}
-          cd #{@project.path}
-          git init
-          touch README
-          git add README
-          git commit -m 'first commit'
-          git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
-          git push -u origin master
+  %fieldset
+    %legend Existing Git Repo?
+    %pre.dark
+      :preserve
+        cd existing_git_repo
+        git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
+        git push -u origin master
-    %fieldset
-      %legend Existing Git Repo?
-      %pre.dark
-        :preserve
-          cd existing_git_repo
-          git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
-          git push -u origin master
-
-  - if can? current_user, :remove_project, @project
-    .prepend-top-20
-      = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"
+- if can? current_user, :remove_project, @project
+  .prepend-top-20
+    = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"
@@ -0,0 +1,30 @@
+- if @project.import_in_progress?
+  .save-project-loader
+    %center
+      %h2
+        %i.icon-spinner.icon-spin
+        Import in progress.
+      %p.monospace git clone --bare #{@project.import_url}
+      %p Please wait while we import the repository for you. Refresh at will.
+      :javascript
+        new ProjectImport();
+
+- elsif @project.import_failed?
+  .save-project-loader
+    %center
+      %h2
+        Import failed. Retry?
+  %hr
+  - if can?(current_user, :admin_project, @project)
+    = form_for @project, url: retry_import_project_path(@project), method: :put, html: { class: 'form-horizontal' } do |f|
+      .form-group.import-url-data
+        = f.label :import_url, class: 'control-label' do
+          %span Import existing repo
+        .col-sm-10
+          = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
+          .bs-callout.bs-callout-info
+            This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
+            %br
+            The import will time out after 2 minutes. For big repositories, use a clone/push combination.
+      .form-actions
+        = f.submit 'Retry import', class: "btn btn-create", tabindex: 4
@@ -23,6 +23,7 @@
           .col-sm-2
           .col-sm-10
             = link_to "#", class: 'js-toggle-button' do
+              %i.icon-edit
               %span Customize repository name?
         .js-toggle-content.hide
           .form-group
@@ -46,8 +47,10 @@
               %span Import existing repo
             .col-sm-10
               = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
-              .light
-                URL must be cloneable
+              .bs-callout.bs-callout-info
+                This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
+                %br
+                The import will time out after 2 minutes. For big repositories, use a clone/push combination.
       %hr
       .form-group
@@ -11,11 +11,11 @@ class RepositoryImportWorker
                                project.import_url)
     if result
-      project.imported = true
+      project.import_finish
       project.save
       project.satellite.create unless project.satellite.exists?
     else
-      project.imported = false
+      project.import_fail
     end
   end
 end
@@ -179,6 +179,8 @@ Gitlab::Application.routes.draw do
       post :archive
       post :unarchive
       get :autocomplete_sources
+      get :import
+      put :retry_import
     end
     scope module: :projects do
@@ -0,0 +1,5 @@
+class AddImportStatusToProject < ActiveRecord::Migration
+  def change
+    add_column :projects, :import_status, :string
+  end
+end
@@ -0,0 +1,12 @@
+class MigrateAlreadyImportedProjects < ActiveRecord::Migration
+  def up
+    Project.where(imported: true).update_all(import_status: "finished")
+    Project.where(imported: false).update_all(import_status: "none")
+    remove_column :projects, :imported
+  end
+
+  def down
+    add_column :projects, :imported, :boolean, default: false
+    Project.where(import_status: 'finished').update_all(imported: true)
+  end
+end
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20140305193308) do
+ActiveRecord::Schema.define(version: 20140313092127) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -214,10 +214,10 @@ ActiveRecord::Schema.define(version: 20140305193308) do
     t.string   "issues_tracker_id"
     t.boolean  "snippets_enabled",       default: true,     null: false
     t.datetime "last_activity_at"
-    t.boolean  "imported",               default: false,    null: false
     t.string   "import_url"
     t.integer  "visibility_level",       default: 0,        null: false
     t.boolean  "archived",               default: false,    null: false
+    t.string   "import_status"
   end
   add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree
	@@ -17,6 +17,7 @@ v 6.7.0		@@ -17,6 +17,7 @@ v 6.7.0
17	- Add GFM autocompletion for MergeRequests (Robert Speicher)	17	- Add GFM autocompletion for MergeRequests (Robert Speicher)
18	- Add webhook when a new tag is pushed (Jeroen van Baarsen)	18	- Add webhook when a new tag is pushed (Jeroen van Baarsen)
19	- Add button for toggling inline comments in diff view	19	- Add button for toggling inline comments in diff view
		20	+ - Add retry feature for repository import
20		21
21	v 6.6.2	22	v 6.6.2
22	- Fix 500 error on branch/tag create or remove via UI	23	- Fix 500 error on branch/tag create or remove via UI
1	class ProjectObserver < BaseObserver	1	class ProjectObserver < BaseObserver
2	def after_create(project)	2	def after_create(project)
3	- project.update_column(:last_activity_at, project.created_at)
4	-
5	- return true if project.forked?
6	-
7	- if project.import?
8	- RepositoryImportWorker.perform_in(5.seconds, project.id)
9	- else
10	- GitlabShellWorker.perform_async(
11	- :add_repository,
12	- project.path_with_namespace
13	- )
14	-
15	- log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
16	- end
17	-
18	- if project.wiki_enabled?
19	- begin
20	- # force the creation of a wiki,
21	- GollumWiki.new(project, project.owner).wiki
22	- rescue GollumWiki::CouldNotCreateWikiError => ex
23	- # Prevent project observer crash
24	- # if failed to create wiki
25	- nil
26	- end
27	- end	3	+ log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
28	end	4	end
29		5
30	def after_update(project)	6	def after_update(project)
	@@ -58,6 +58,29 @@ module Projects		@@ -58,6 +58,29 @@ module Projects
58	user: current_user	58	user: current_user
59	)	59	)
60	end	60	end
		61	+
		62	+ @project.update_column(:last_activity_at, @project.created_at)
		63	+
		64	+ if @project.import?
		65	+ @project.import_start
		66	+ else
		67	+ GitlabShellWorker.perform_async(
		68	+ :add_repository,
		69	+ @project.path_with_namespace
		70	+ )
		71	+
		72	+ end
		73	+
		74	+ if @project.wiki_enabled?
		75	+ begin
		76	+ # force the creation of a wiki,
		77	+ GollumWiki.new(@project, @project.owner).wiki
		78	+ rescue GollumWiki::CouldNotCreateWikiError => ex
		79	+ # Prevent project observer crash
		80	+ # if failed to create wiki
		81	+ nil
		82	+ end
		83	+ end
61	end	84	end
62		85
63	@project	86	@project
1	= render "home_panel"	1	= render "home_panel"
2		2
3	-- if @project.import? && !@project.imported
4	- .save-project-loader
5	- %center
6	- %h2
7	- %i.icon-spinner.icon-spin
8	- Importing repository.
9	- %p.monospace git clone --bare #{@project.import_url}
10	- %p Please wait while we import the repository for you. Refresh at will.
11	- :javascript
12	- new ProjectImport();	3	+%div.git-empty
		4	+ %fieldset
		5	+ %legend Git global setup:
		6	+ %pre.dark
		7	+ :preserve
		8	+ git config --global user.name "#{git_user_name}"
		9	+ git config --global user.email "#{git_user_email}"
13		10
14	-- else
15	- %div.git-empty
16	- %fieldset
17	- %legend Git global setup:
18	- %pre.dark
19	- :preserve
20	- git config --global user.name "#{git_user_name}"
21	- git config --global user.email "#{git_user_email}"	11	+ %fieldset
		12	+ %legend Create Repository
		13	+ %pre.dark
		14	+ :preserve
		15	+ mkdir #{@project.path}
		16	+ cd #{@project.path}
		17	+ git init
		18	+ touch README
		19	+ git add README
		20	+ git commit -m 'first commit'
		21	+ git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
		22	+ git push -u origin master
22		23
23	- %fieldset
24	- %legend Create Repository
25	- %pre.dark
26	- :preserve
27	- mkdir #{@project.path}
28	- cd #{@project.path}
29	- git init
30	- touch README
31	- git add README
32	- git commit -m 'first commit'
33	- git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
34	- git push -u origin master	24	+ %fieldset
		25	+ %legend Existing Git Repo?
		26	+ %pre.dark
		27	+ :preserve
		28	+ cd existing_git_repo
		29	+ git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
		30	+ git push -u origin master
35		31
36	- %fieldset
37	- %legend Existing Git Repo?
38	- %pre.dark
39	- :preserve
40	- cd existing_git_repo
41	- git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
42	- git push -u origin master
43	-
44	- - if can? current_user, :remove_project, @project
45	- .prepend-top-20
46	- = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"	32	+- if can? current_user, :remove_project, @project
		33	+ .prepend-top-20
		34	+ = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"
@@ -0,0 +1,30 @@		@@ -0,0 +1,30 @@
	1	+- if @project.import_in_progress?
	2	+ .save-project-loader
	3	+ %center
	4	+ %h2
	5	+ %i.icon-spinner.icon-spin
	6	+ Import in progress.
	7	+ %p.monospace git clone --bare #{@project.import_url}
	8	+ %p Please wait while we import the repository for you. Refresh at will.
	9	+ :javascript
	10	+ new ProjectImport();
	11	+
	12	+- elsif @project.import_failed?
	13	+ .save-project-loader
	14	+ %center
	15	+ %h2
	16	+ Import failed. Retry?
	17	+ %hr
	18	+ - if can?(current_user, :admin_project, @project)
	19	+ = form_for @project, url: retry_import_project_path(@project), method: :put, html: { class: 'form-horizontal' } do \|f\|
	20	+ .form-group.import-url-data
	21	+ = f.label :import_url, class: 'control-label' do
	22	+ %span Import existing repo
	23	+ .col-sm-10
	24	+ = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
	25	+ .bs-callout.bs-callout-info
	26	+ This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
	27	+ %br
	28	+ The import will time out after 2 minutes. For big repositories, use a clone/push combination.
	29	+ .form-actions
	30	+ = f.submit 'Retry import', class: "btn btn-create", tabindex: 4
	@@ -11,11 +11,11 @@ class RepositoryImportWorker		@@ -11,11 +11,11 @@ class RepositoryImportWorker
11	project.import_url)	11	project.import_url)
12		12
13	if result	13	if result
14	- project.imported = true	14	+ project.import_finish
15	project.save	15	project.save
16	project.satellite.create unless project.satellite.exists?	16	project.satellite.create unless project.satellite.exists?
17	else	17	else
18	- project.imported = false	18	+ project.import_fail
19	end	19	end
20	end	20	end
21	end	21	end
@@ -0,0 +1,5 @@		@@ -0,0 +1,5 @@
	1	+class AddImportStatusToProject < ActiveRecord::Migration
	2	+ def change
	3	+ add_column :projects, :import_status, :string
	4	+ end
	5	+end
@@ -0,0 +1,12 @@		@@ -0,0 +1,12 @@
	1	+class MigrateAlreadyImportedProjects < ActiveRecord::Migration
	2	+ def up
	3	+ Project.where(imported: true).update_all(import_status: "finished")
	4	+ Project.where(imported: false).update_all(import_status: "none")
	5	+ remove_column :projects, :imported
	6	+ end
	7	+
	8	+ def down
	9	+ add_column :projects, :imported, :boolean, default: false
	10	+ Project.where(import_status: 'finished').update_all(imported: true)
	11	+ end
	12	+end