Commit 0d7986a8c7806a5d7630283f26667f954a5e61f7

Authored by Dmitriy Zaporozhets
2 parents 58e4e6b0 9064fba0

Merge branch 'import-timeout' of https://dev.gitlab.org/dzaporozhets/gitlabhq in…

…to dzaporozhets/gitlabhq-import-timeout

Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

Conflicts:
	CHANGELOG
	db/schema.rb
CHANGELOG
... ... @@ -17,6 +17,7 @@ v 6.7.0
17 17 - Add GFM autocompletion for MergeRequests (Robert Speicher)
18 18 - Add webhook when a new tag is pushed (Jeroen van Baarsen)
19 19 - Add button for toggling inline comments in diff view
  20 + - Add retry feature for repository import
20 21  
21 22 v 6.6.2
22 23 - Fix 500 error on branch/tag create or remove via UI
... ...
app/controllers/projects_controller.rb
... ... @@ -5,7 +5,7 @@ class ProjectsController &lt; ApplicationController
5 5  
6 6 # Authorize
7 7 before_filter :authorize_read_project!, except: [:index, :new, :create]
8   - before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive]
  8 + before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive, :retry_import]
9 9 before_filter :require_non_empty_project, only: [:blob, :tree, :graph]
10 10  
11 11 layout 'navless', only: [:new, :create, :fork]
... ... @@ -21,16 +21,9 @@ class ProjectsController &lt; ApplicationController
21 21  
22 22 def create
23 23 @project = ::Projects::CreateService.new(current_user, params[:project]).execute
  24 + flash[:notice] = 'Project was successfully created.' if @project.saved?
24 25  
25 26 respond_to do |format|
26   - flash[:notice] = 'Project was successfully created.' if @project.saved?
27   - format.html do
28   - if @project.saved?
29   - redirect_to @project
30   - else
31   - render "new"
32   - end
33   - end
34 27 format.js
35 28 end
36 29 end
... ... @@ -55,6 +48,11 @@ class ProjectsController &lt; ApplicationController
55 48 end
56 49  
57 50 def show
  51 + if @project.import_in_progress?
  52 + redirect_to import_project_path(@project)
  53 + return
  54 + end
  55 +
58 56 return authenticate_user! unless @project.public? || current_user
59 57  
60 58 limit = (params[:limit] || 20).to_i
... ... @@ -67,9 +65,7 @@ class ProjectsController &lt; ApplicationController
67 65 if @project.empty_repo?
68 66 render "projects/empty", layout: user_layout
69 67 else
70   - if current_user
71   - @last_push = current_user.recent_push(@project.id)
72   - end
  68 + @last_push = current_user.recent_push(@project.id) if current_user
73 69 render :show, layout: user_layout
74 70 end
75 71 end
... ... @@ -77,6 +73,28 @@ class ProjectsController &lt; ApplicationController
77 73 end
78 74 end
79 75  
  76 + def import
  77 + if project.import_finished?
  78 + redirect_to @project
  79 + return
  80 + end
  81 + end
  82 +
  83 + def retry_import
  84 + unless @project.import_failed?
  85 + redirect_to import_project_path(@project)
  86 + end
  87 +
  88 + @project.import_url = params[:project][:import_url]
  89 +
  90 + if @project.save
  91 + @project.reload
  92 + @project.import_retry
  93 + end
  94 +
  95 + redirect_to import_project_path(@project)
  96 + end
  97 +
80 98 def destroy
81 99 return access_denied! unless can?(current_user, :remove_project, project)
82 100  
... ...
app/models/project.rb
... ... @@ -28,7 +28,6 @@ class Project &lt; ActiveRecord::Base
28 28 include Gitlab::VisibilityLevel
29 29 extend Enumerize
30 30  
31   - default_value_for :imported, false
32 31 default_value_for :archived, false
33 32  
34 33 ActsAsTaggableOn.strict_case_match = true
... ... @@ -59,13 +58,10 @@ class Project &lt; ActiveRecord::Base
59 58 has_one :gemnasium_service, dependent: :destroy
60 59 has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id"
61 60 has_one :forked_from_project, through: :forked_project_link
62   -
63 61 # Merge Requests for target project should be removed with it
64 62 has_many :merge_requests, dependent: :destroy, foreign_key: "target_project_id"
65   -
66 63 # Merge requests from source project should be kept when source project was removed
67 64 has_many :fork_merge_requests, foreign_key: "source_project_id", class_name: MergeRequest
68   -
69 65 has_many :issues, -> { order "state DESC, created_at DESC" }, dependent: :destroy
70 66 has_many :services, dependent: :destroy
71 67 has_many :events, dependent: :destroy
... ... @@ -74,10 +70,8 @@ class Project &lt; ActiveRecord::Base
74 70 has_many :snippets, dependent: :destroy, class_name: "ProjectSnippet"
75 71 has_many :hooks, dependent: :destroy, class_name: "ProjectHook"
76 72 has_many :protected_branches, dependent: :destroy
77   -
78 73 has_many :users_projects, dependent: :destroy
79 74 has_many :users, through: :users_projects
80   -
81 75 has_many :deploy_keys_projects, dependent: :destroy
82 76 has_many :deploy_keys, through: :deploy_keys_projects
83 77  
... ... @@ -97,15 +91,12 @@ class Project &lt; ActiveRecord::Base
97 91 validates :issues_enabled, :wall_enabled, :merge_requests_enabled,
98 92 :wiki_enabled, inclusion: { in: [true, false] }
99 93 validates :issues_tracker_id, length: { maximum: 255 }, allow_blank: true
100   -
101 94 validates :namespace, presence: true
102 95 validates_uniqueness_of :name, scope: :namespace_id
103 96 validates_uniqueness_of :path, scope: :namespace_id
104   -
105 97 validates :import_url,
106 98 format: { with: URI::regexp(%w(git http https)), message: "should be a valid url" },
107 99 if: :import?
108   -
109 100 validate :check_limit, on: :create
110 101  
111 102 # Scopes
... ... @@ -118,14 +109,36 @@ class Project &lt; ActiveRecord::Base
118 109 scope :sorted_by_activity, -> { reorder("projects.last_activity_at DESC") }
119 110 scope :personal, ->(user) { where(namespace_id: user.namespace_id) }
120 111 scope :joined, ->(user) { where("namespace_id != ?", user.namespace_id) }
121   -
122 112 scope :public_only, -> { where(visibility_level: Project::PUBLIC) }
123 113 scope :public_and_internal_only, -> { where(visibility_level: Project.public_and_internal_levels) }
124   -
125 114 scope :non_archived, -> { where(archived: false) }
126 115  
127 116 enumerize :issues_tracker, in: (Gitlab.config.issues_tracker.keys).append(:gitlab), default: :gitlab
128 117  
  118 + state_machine :import_status, initial: :none do
  119 + event :import_start do
  120 + transition :none => :started
  121 + end
  122 +
  123 + event :import_finish do
  124 + transition :started => :finished
  125 + end
  126 +
  127 + event :import_fail do
  128 + transition :started => :failed
  129 + end
  130 +
  131 + event :import_retry do
  132 + transition :failed => :started
  133 + end
  134 +
  135 + state :started
  136 + state :finished
  137 + state :failed
  138 +
  139 + after_transition any => :started, :do => :add_import_job
  140 + end
  141 +
129 142 class << self
130 143 def public_and_internal_levels
131 144 [Project::PUBLIC, Project::INTERNAL]
... ... @@ -202,12 +215,28 @@ class Project &lt; ActiveRecord::Base
202 215 id && persisted?
203 216 end
204 217  
  218 + def add_import_job
  219 + RepositoryImportWorker.perform_in(2.seconds, id)
  220 + end
  221 +
205 222 def import?
206 223 import_url.present?
207 224 end
208 225  
209 226 def imported?
210   - imported
  227 + import_finished?
  228 + end
  229 +
  230 + def import_in_progress?
  231 + import? && import_status == 'started'
  232 + end
  233 +
  234 + def import_failed?
  235 + import_status == 'failed'
  236 + end
  237 +
  238 + def import_finished?
  239 + import_status == 'finished'
211 240 end
212 241  
213 242 def check_limit
... ...
app/observers/project_observer.rb
1 1 class ProjectObserver < BaseObserver
2 2 def after_create(project)
3   - project.update_column(:last_activity_at, project.created_at)
4   -
5   - return true if project.forked?
6   -
7   - if project.import?
8   - RepositoryImportWorker.perform_in(5.seconds, project.id)
9   - else
10   - GitlabShellWorker.perform_async(
11   - :add_repository,
12   - project.path_with_namespace
13   - )
14   -
15   - log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
16   - end
17   -
18   - if project.wiki_enabled?
19   - begin
20   - # force the creation of a wiki,
21   - GollumWiki.new(project, project.owner).wiki
22   - rescue GollumWiki::CouldNotCreateWikiError => ex
23   - # Prevent project observer crash
24   - # if failed to create wiki
25   - nil
26   - end
27   - end
  3 + log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
28 4 end
29 5  
30 6 def after_update(project)
... ...
app/services/projects/create_service.rb
... ... @@ -58,6 +58,29 @@ module Projects
58 58 user: current_user
59 59 )
60 60 end
  61 +
  62 + @project.update_column(:last_activity_at, @project.created_at)
  63 +
  64 + if @project.import?
  65 + @project.import_start
  66 + else
  67 + GitlabShellWorker.perform_async(
  68 + :add_repository,
  69 + @project.path_with_namespace
  70 + )
  71 +
  72 + end
  73 +
  74 + if @project.wiki_enabled?
  75 + begin
  76 + # force the creation of a wiki,
  77 + GollumWiki.new(@project, @project.owner).wiki
  78 + rescue GollumWiki::CouldNotCreateWikiError => ex
  79 + # Prevent project observer crash
  80 + # if failed to create wiki
  81 + nil
  82 + end
  83 + end
61 84 end
62 85  
63 86 @project
... ...
app/views/projects/create.js.haml
1 1 - if @project.saved?
2   - :plain
3   - location.href = "#{project_path(@project)}";
  2 + - if @project.import?
  3 + :plain
  4 + location.href = "#{import_project_path(@project)}";
  5 + - else
  6 + :plain
  7 + location.href = "#{project_path(@project)}";
4 8 - else
5 9 :plain
6 10 $(".project-edit-errors").html("#{escape_javascript(render('errors'))}");
... ...
app/views/projects/empty.html.haml
1 1 = render "home_panel"
2 2  
3   -- if @project.import? && !@project.imported
4   - .save-project-loader
5   - %center
6   - %h2
7   - %i.icon-spinner.icon-spin
8   - Importing repository.
9   - %p.monospace git clone --bare #{@project.import_url}
10   - %p Please wait while we import the repository for you. Refresh at will.
11   - :javascript
12   - new ProjectImport();
  3 +%div.git-empty
  4 + %fieldset
  5 + %legend Git global setup:
  6 + %pre.dark
  7 + :preserve
  8 + git config --global user.name "#{git_user_name}"
  9 + git config --global user.email "#{git_user_email}"
13 10  
14   -- else
15   - %div.git-empty
16   - %fieldset
17   - %legend Git global setup:
18   - %pre.dark
19   - :preserve
20   - git config --global user.name "#{git_user_name}"
21   - git config --global user.email "#{git_user_email}"
  11 + %fieldset
  12 + %legend Create Repository
  13 + %pre.dark
  14 + :preserve
  15 + mkdir #{@project.path}
  16 + cd #{@project.path}
  17 + git init
  18 + touch README
  19 + git add README
  20 + git commit -m 'first commit'
  21 + git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
  22 + git push -u origin master
22 23  
23   - %fieldset
24   - %legend Create Repository
25   - %pre.dark
26   - :preserve
27   - mkdir #{@project.path}
28   - cd #{@project.path}
29   - git init
30   - touch README
31   - git add README
32   - git commit -m 'first commit'
33   - git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
34   - git push -u origin master
  24 + %fieldset
  25 + %legend Existing Git Repo?
  26 + %pre.dark
  27 + :preserve
  28 + cd existing_git_repo
  29 + git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
  30 + git push -u origin master
35 31  
36   - %fieldset
37   - %legend Existing Git Repo?
38   - %pre.dark
39   - :preserve
40   - cd existing_git_repo
41   - git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
42   - git push -u origin master
43   -
44   - - if can? current_user, :remove_project, @project
45   - .prepend-top-20
46   - = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"
  32 +- if can? current_user, :remove_project, @project
  33 + .prepend-top-20
  34 + = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"
... ...
app/views/projects/import.html.haml 0 → 100644
... ... @@ -0,0 +1,30 @@
  1 +- if @project.import_in_progress?
  2 + .save-project-loader
  3 + %center
  4 + %h2
  5 + %i.icon-spinner.icon-spin
  6 + Import in progress.
  7 + %p.monospace git clone --bare #{@project.import_url}
  8 + %p Please wait while we import the repository for you. Refresh at will.
  9 + :javascript
  10 + new ProjectImport();
  11 +
  12 +- elsif @project.import_failed?
  13 + .save-project-loader
  14 + %center
  15 + %h2
  16 + Import failed. Retry?
  17 + %hr
  18 + - if can?(current_user, :admin_project, @project)
  19 + = form_for @project, url: retry_import_project_path(@project), method: :put, html: { class: 'form-horizontal' } do |f|
  20 + .form-group.import-url-data
  21 + = f.label :import_url, class: 'control-label' do
  22 + %span Import existing repo
  23 + .col-sm-10
  24 + = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
  25 + .bs-callout.bs-callout-info
  26 + This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
  27 + %br
  28 + The import will time out after 2 minutes. For big repositories, use a clone/push combination.
  29 + .form-actions
  30 + = f.submit 'Retry import', class: "btn btn-create", tabindex: 4
... ...
app/views/projects/new.html.haml
... ... @@ -23,6 +23,7 @@
23 23 .col-sm-2
24 24 .col-sm-10
25 25 = link_to "#", class: 'js-toggle-button' do
  26 + %i.icon-edit
26 27 %span Customize repository name?
27 28 .js-toggle-content.hide
28 29 .form-group
... ... @@ -46,8 +47,10 @@
46 47 %span Import existing repo
47 48 .col-sm-10
48 49 = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
49   - .light
50   - URL must be cloneable
  50 + .bs-callout.bs-callout-info
  51 + This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
  52 + %br
  53 + The import will time out after 2 minutes. For big repositories, use a clone/push combination.
51 54 %hr
52 55  
53 56 .form-group
... ...
app/workers/repository_import_worker.rb
... ... @@ -11,11 +11,11 @@ class RepositoryImportWorker
11 11 project.import_url)
12 12  
13 13 if result
14   - project.imported = true
  14 + project.import_finish
15 15 project.save
16 16 project.satellite.create unless project.satellite.exists?
17 17 else
18   - project.imported = false
  18 + project.import_fail
19 19 end
20 20 end
21 21 end
... ...
config/routes.rb
... ... @@ -179,6 +179,8 @@ Gitlab::Application.routes.draw do
179 179 post :archive
180 180 post :unarchive
181 181 get :autocomplete_sources
  182 + get :import
  183 + put :retry_import
182 184 end
183 185  
184 186 scope module: :projects do
... ...
db/migrate/20140312145357_add_import_status_to_project.rb 0 → 100644
... ... @@ -0,0 +1,5 @@
  1 +class AddImportStatusToProject < ActiveRecord::Migration
  2 + def change
  3 + add_column :projects, :import_status, :string
  4 + end
  5 +end
... ...
db/migrate/20140313092127_migrate_already_imported_projects.rb 0 → 100644
... ... @@ -0,0 +1,12 @@
  1 +class MigrateAlreadyImportedProjects < ActiveRecord::Migration
  2 + def up
  3 + Project.where(imported: true).update_all(import_status: "finished")
  4 + Project.where(imported: false).update_all(import_status: "none")
  5 + remove_column :projects, :imported
  6 + end
  7 +
  8 + def down
  9 + add_column :projects, :imported, :boolean, default: false
  10 + Project.where(import_status: 'finished').update_all(imported: true)
  11 + end
  12 +end
... ...
db/schema.rb
... ... @@ -11,7 +11,7 @@
11 11 #
12 12 # It's strongly recommended that you check this file into your version control system.
13 13  
14   -ActiveRecord::Schema.define(version: 20140305193308) do
  14 +ActiveRecord::Schema.define(version: 20140313092127) do
15 15  
16 16 # These are extensions that must be enabled in order to support this database
17 17 enable_extension "plpgsql"
... ... @@ -214,10 +214,10 @@ ActiveRecord::Schema.define(version: 20140305193308) do
214 214 t.string "issues_tracker_id"
215 215 t.boolean "snippets_enabled", default: true, null: false
216 216 t.datetime "last_activity_at"
217   - t.boolean "imported", default: false, null: false
218 217 t.string "import_url"
219 218 t.integer "visibility_level", default: 0, null: false
220 219 t.boolean "archived", default: false, null: false
  220 + t.string "import_status"
221 221 end
222 222  
223 223 add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree
... ...