Commit 0d7986a8c7806a5d7630283f26667f954a5e61f7
Exists in
spb-stable
and in
3 other branches
Merge branch 'import-timeout' of https://dev.gitlab.org/dzaporozhets/gitlabhq in…
…to dzaporozhets/gitlabhq-import-timeout Signed-off-by: Dmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com> Conflicts: CHANGELOG db/schema.rb
Showing
14 changed files
with
189 additions
and
98 deletions
Show diff stats
CHANGELOG
... | ... | @@ -17,6 +17,7 @@ v 6.7.0 |
17 | 17 | - Add GFM autocompletion for MergeRequests (Robert Speicher) |
18 | 18 | - Add webhook when a new tag is pushed (Jeroen van Baarsen) |
19 | 19 | - Add button for toggling inline comments in diff view |
20 | + - Add retry feature for repository import | |
20 | 21 | |
21 | 22 | v 6.6.2 |
22 | 23 | - Fix 500 error on branch/tag create or remove via UI | ... | ... |
app/controllers/projects_controller.rb
... | ... | @@ -5,7 +5,7 @@ class ProjectsController < ApplicationController |
5 | 5 | |
6 | 6 | # Authorize |
7 | 7 | before_filter :authorize_read_project!, except: [:index, :new, :create] |
8 | - before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive] | |
8 | + before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive, :retry_import] | |
9 | 9 | before_filter :require_non_empty_project, only: [:blob, :tree, :graph] |
10 | 10 | |
11 | 11 | layout 'navless', only: [:new, :create, :fork] |
... | ... | @@ -21,16 +21,9 @@ class ProjectsController < ApplicationController |
21 | 21 | |
22 | 22 | def create |
23 | 23 | @project = ::Projects::CreateService.new(current_user, params[:project]).execute |
24 | + flash[:notice] = 'Project was successfully created.' if @project.saved? | |
24 | 25 | |
25 | 26 | respond_to do |format| |
26 | - flash[:notice] = 'Project was successfully created.' if @project.saved? | |
27 | - format.html do | |
28 | - if @project.saved? | |
29 | - redirect_to @project | |
30 | - else | |
31 | - render "new" | |
32 | - end | |
33 | - end | |
34 | 27 | format.js |
35 | 28 | end |
36 | 29 | end |
... | ... | @@ -55,6 +48,11 @@ class ProjectsController < ApplicationController |
55 | 48 | end |
56 | 49 | |
57 | 50 | def show |
51 | + if @project.import_in_progress? | |
52 | + redirect_to import_project_path(@project) | |
53 | + return | |
54 | + end | |
55 | + | |
58 | 56 | return authenticate_user! unless @project.public? || current_user |
59 | 57 | |
60 | 58 | limit = (params[:limit] || 20).to_i |
... | ... | @@ -67,9 +65,7 @@ class ProjectsController < ApplicationController |
67 | 65 | if @project.empty_repo? |
68 | 66 | render "projects/empty", layout: user_layout |
69 | 67 | else |
70 | - if current_user | |
71 | - @last_push = current_user.recent_push(@project.id) | |
72 | - end | |
68 | + @last_push = current_user.recent_push(@project.id) if current_user | |
73 | 69 | render :show, layout: user_layout |
74 | 70 | end |
75 | 71 | end |
... | ... | @@ -77,6 +73,28 @@ class ProjectsController < ApplicationController |
77 | 73 | end |
78 | 74 | end |
79 | 75 | |
76 | + def import | |
77 | + if project.import_finished? | |
78 | + redirect_to @project | |
79 | + return | |
80 | + end | |
81 | + end | |
82 | + | |
83 | + def retry_import | |
84 | + unless @project.import_failed? | |
85 | + redirect_to import_project_path(@project) | |
86 | + end | |
87 | + | |
88 | + @project.import_url = params[:project][:import_url] | |
89 | + | |
90 | + if @project.save | |
91 | + @project.reload | |
92 | + @project.import_retry | |
93 | + end | |
94 | + | |
95 | + redirect_to import_project_path(@project) | |
96 | + end | |
97 | + | |
80 | 98 | def destroy |
81 | 99 | return access_denied! unless can?(current_user, :remove_project, project) |
82 | 100 | ... | ... |
app/models/project.rb
... | ... | @@ -28,7 +28,6 @@ class Project < ActiveRecord::Base |
28 | 28 | include Gitlab::VisibilityLevel |
29 | 29 | extend Enumerize |
30 | 30 | |
31 | - default_value_for :imported, false | |
32 | 31 | default_value_for :archived, false |
33 | 32 | |
34 | 33 | ActsAsTaggableOn.strict_case_match = true |
... | ... | @@ -59,13 +58,10 @@ class Project < ActiveRecord::Base |
59 | 58 | has_one :gemnasium_service, dependent: :destroy |
60 | 59 | has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id" |
61 | 60 | has_one :forked_from_project, through: :forked_project_link |
62 | - | |
63 | 61 | # Merge Requests for target project should be removed with it |
64 | 62 | has_many :merge_requests, dependent: :destroy, foreign_key: "target_project_id" |
65 | - | |
66 | 63 | # Merge requests from source project should be kept when source project was removed |
67 | 64 | has_many :fork_merge_requests, foreign_key: "source_project_id", class_name: MergeRequest |
68 | - | |
69 | 65 | has_many :issues, -> { order "state DESC, created_at DESC" }, dependent: :destroy |
70 | 66 | has_many :services, dependent: :destroy |
71 | 67 | has_many :events, dependent: :destroy |
... | ... | @@ -74,10 +70,8 @@ class Project < ActiveRecord::Base |
74 | 70 | has_many :snippets, dependent: :destroy, class_name: "ProjectSnippet" |
75 | 71 | has_many :hooks, dependent: :destroy, class_name: "ProjectHook" |
76 | 72 | has_many :protected_branches, dependent: :destroy |
77 | - | |
78 | 73 | has_many :users_projects, dependent: :destroy |
79 | 74 | has_many :users, through: :users_projects |
80 | - | |
81 | 75 | has_many :deploy_keys_projects, dependent: :destroy |
82 | 76 | has_many :deploy_keys, through: :deploy_keys_projects |
83 | 77 | |
... | ... | @@ -97,15 +91,12 @@ class Project < ActiveRecord::Base |
97 | 91 | validates :issues_enabled, :wall_enabled, :merge_requests_enabled, |
98 | 92 | :wiki_enabled, inclusion: { in: [true, false] } |
99 | 93 | validates :issues_tracker_id, length: { maximum: 255 }, allow_blank: true |
100 | - | |
101 | 94 | validates :namespace, presence: true |
102 | 95 | validates_uniqueness_of :name, scope: :namespace_id |
103 | 96 | validates_uniqueness_of :path, scope: :namespace_id |
104 | - | |
105 | 97 | validates :import_url, |
106 | 98 | format: { with: URI::regexp(%w(git http https)), message: "should be a valid url" }, |
107 | 99 | if: :import? |
108 | - | |
109 | 100 | validate :check_limit, on: :create |
110 | 101 | |
111 | 102 | # Scopes |
... | ... | @@ -118,14 +109,36 @@ class Project < ActiveRecord::Base |
118 | 109 | scope :sorted_by_activity, -> { reorder("projects.last_activity_at DESC") } |
119 | 110 | scope :personal, ->(user) { where(namespace_id: user.namespace_id) } |
120 | 111 | scope :joined, ->(user) { where("namespace_id != ?", user.namespace_id) } |
121 | - | |
122 | 112 | scope :public_only, -> { where(visibility_level: Project::PUBLIC) } |
123 | 113 | scope :public_and_internal_only, -> { where(visibility_level: Project.public_and_internal_levels) } |
124 | - | |
125 | 114 | scope :non_archived, -> { where(archived: false) } |
126 | 115 | |
127 | 116 | enumerize :issues_tracker, in: (Gitlab.config.issues_tracker.keys).append(:gitlab), default: :gitlab |
128 | 117 | |
118 | + state_machine :import_status, initial: :none do | |
119 | + event :import_start do | |
120 | + transition :none => :started | |
121 | + end | |
122 | + | |
123 | + event :import_finish do | |
124 | + transition :started => :finished | |
125 | + end | |
126 | + | |
127 | + event :import_fail do | |
128 | + transition :started => :failed | |
129 | + end | |
130 | + | |
131 | + event :import_retry do | |
132 | + transition :failed => :started | |
133 | + end | |
134 | + | |
135 | + state :started | |
136 | + state :finished | |
137 | + state :failed | |
138 | + | |
139 | + after_transition any => :started, :do => :add_import_job | |
140 | + end | |
141 | + | |
129 | 142 | class << self |
130 | 143 | def public_and_internal_levels |
131 | 144 | [Project::PUBLIC, Project::INTERNAL] |
... | ... | @@ -202,12 +215,28 @@ class Project < ActiveRecord::Base |
202 | 215 | id && persisted? |
203 | 216 | end |
204 | 217 | |
218 | + def add_import_job | |
219 | + RepositoryImportWorker.perform_in(2.seconds, id) | |
220 | + end | |
221 | + | |
205 | 222 | def import? |
206 | 223 | import_url.present? |
207 | 224 | end |
208 | 225 | |
209 | 226 | def imported? |
210 | - imported | |
227 | + import_finished? | |
228 | + end | |
229 | + | |
230 | + def import_in_progress? | |
231 | + import? && import_status == 'started' | |
232 | + end | |
233 | + | |
234 | + def import_failed? | |
235 | + import_status == 'failed' | |
236 | + end | |
237 | + | |
238 | + def import_finished? | |
239 | + import_status == 'finished' | |
211 | 240 | end |
212 | 241 | |
213 | 242 | def check_limit | ... | ... |
app/observers/project_observer.rb
1 | 1 | class ProjectObserver < BaseObserver |
2 | 2 | def after_create(project) |
3 | - project.update_column(:last_activity_at, project.created_at) | |
4 | - | |
5 | - return true if project.forked? | |
6 | - | |
7 | - if project.import? | |
8 | - RepositoryImportWorker.perform_in(5.seconds, project.id) | |
9 | - else | |
10 | - GitlabShellWorker.perform_async( | |
11 | - :add_repository, | |
12 | - project.path_with_namespace | |
13 | - ) | |
14 | - | |
15 | - log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"") | |
16 | - end | |
17 | - | |
18 | - if project.wiki_enabled? | |
19 | - begin | |
20 | - # force the creation of a wiki, | |
21 | - GollumWiki.new(project, project.owner).wiki | |
22 | - rescue GollumWiki::CouldNotCreateWikiError => ex | |
23 | - # Prevent project observer crash | |
24 | - # if failed to create wiki | |
25 | - nil | |
26 | - end | |
27 | - end | |
3 | + log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"") | |
28 | 4 | end |
29 | 5 | |
30 | 6 | def after_update(project) | ... | ... |
app/services/projects/create_service.rb
... | ... | @@ -58,6 +58,29 @@ module Projects |
58 | 58 | user: current_user |
59 | 59 | ) |
60 | 60 | end |
61 | + | |
62 | + @project.update_column(:last_activity_at, @project.created_at) | |
63 | + | |
64 | + if @project.import? | |
65 | + @project.import_start | |
66 | + else | |
67 | + GitlabShellWorker.perform_async( | |
68 | + :add_repository, | |
69 | + @project.path_with_namespace | |
70 | + ) | |
71 | + | |
72 | + end | |
73 | + | |
74 | + if @project.wiki_enabled? | |
75 | + begin | |
76 | + # force the creation of a wiki, | |
77 | + GollumWiki.new(@project, @project.owner).wiki | |
78 | + rescue GollumWiki::CouldNotCreateWikiError => ex | |
79 | + # Prevent project observer crash | |
80 | + # if failed to create wiki | |
81 | + nil | |
82 | + end | |
83 | + end | |
61 | 84 | end |
62 | 85 | |
63 | 86 | @project | ... | ... |
app/views/projects/create.js.haml
1 | 1 | - if @project.saved? |
2 | - :plain | |
3 | - location.href = "#{project_path(@project)}"; | |
2 | + - if @project.import? | |
3 | + :plain | |
4 | + location.href = "#{import_project_path(@project)}"; | |
5 | + - else | |
6 | + :plain | |
7 | + location.href = "#{project_path(@project)}"; | |
4 | 8 | - else |
5 | 9 | :plain |
6 | 10 | $(".project-edit-errors").html("#{escape_javascript(render('errors'))}"); | ... | ... |
app/views/projects/empty.html.haml
1 | 1 | = render "home_panel" |
2 | 2 | |
3 | -- if @project.import? && !@project.imported | |
4 | - .save-project-loader | |
5 | - %center | |
6 | - %h2 | |
7 | - %i.icon-spinner.icon-spin | |
8 | - Importing repository. | |
9 | - %p.monospace git clone --bare #{@project.import_url} | |
10 | - %p Please wait while we import the repository for you. Refresh at will. | |
11 | - :javascript | |
12 | - new ProjectImport(); | |
3 | +%div.git-empty | |
4 | + %fieldset | |
5 | + %legend Git global setup: | |
6 | + %pre.dark | |
7 | + :preserve | |
8 | + git config --global user.name "#{git_user_name}" | |
9 | + git config --global user.email "#{git_user_email}" | |
13 | 10 | |
14 | -- else | |
15 | - %div.git-empty | |
16 | - %fieldset | |
17 | - %legend Git global setup: | |
18 | - %pre.dark | |
19 | - :preserve | |
20 | - git config --global user.name "#{git_user_name}" | |
21 | - git config --global user.email "#{git_user_email}" | |
11 | + %fieldset | |
12 | + %legend Create Repository | |
13 | + %pre.dark | |
14 | + :preserve | |
15 | + mkdir #{@project.path} | |
16 | + cd #{@project.path} | |
17 | + git init | |
18 | + touch README | |
19 | + git add README | |
20 | + git commit -m 'first commit' | |
21 | + git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')} | |
22 | + git push -u origin master | |
22 | 23 | |
23 | - %fieldset | |
24 | - %legend Create Repository | |
25 | - %pre.dark | |
26 | - :preserve | |
27 | - mkdir #{@project.path} | |
28 | - cd #{@project.path} | |
29 | - git init | |
30 | - touch README | |
31 | - git add README | |
32 | - git commit -m 'first commit' | |
33 | - git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')} | |
34 | - git push -u origin master | |
24 | + %fieldset | |
25 | + %legend Existing Git Repo? | |
26 | + %pre.dark | |
27 | + :preserve | |
28 | + cd existing_git_repo | |
29 | + git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')} | |
30 | + git push -u origin master | |
35 | 31 | |
36 | - %fieldset | |
37 | - %legend Existing Git Repo? | |
38 | - %pre.dark | |
39 | - :preserve | |
40 | - cd existing_git_repo | |
41 | - git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')} | |
42 | - git push -u origin master | |
43 | - | |
44 | - - if can? current_user, :remove_project, @project | |
45 | - .prepend-top-20 | |
46 | - = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right" | |
32 | +- if can? current_user, :remove_project, @project | |
33 | + .prepend-top-20 | |
34 | + = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right" | ... | ... |
... | ... | @@ -0,0 +1,30 @@ |
1 | +- if @project.import_in_progress? | |
2 | + .save-project-loader | |
3 | + %center | |
4 | + %h2 | |
5 | + %i.icon-spinner.icon-spin | |
6 | + Import in progress. | |
7 | + %p.monospace git clone --bare #{@project.import_url} | |
8 | + %p Please wait while we import the repository for you. Refresh at will. | |
9 | + :javascript | |
10 | + new ProjectImport(); | |
11 | + | |
12 | +- elsif @project.import_failed? | |
13 | + .save-project-loader | |
14 | + %center | |
15 | + %h2 | |
16 | + Import failed. Retry? | |
17 | + %hr | |
18 | + - if can?(current_user, :admin_project, @project) | |
19 | + = form_for @project, url: retry_import_project_path(@project), method: :put, html: { class: 'form-horizontal' } do |f| | |
20 | + .form-group.import-url-data | |
21 | + = f.label :import_url, class: 'control-label' do | |
22 | + %span Import existing repo | |
23 | + .col-sm-10 | |
24 | + = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git' | |
25 | + .bs-callout.bs-callout-info | |
26 | + This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git. | |
27 | + %br | |
28 | + The import will time out after 2 minutes. For big repositories, use a clone/push combination. | |
29 | + .form-actions | |
30 | + = f.submit 'Retry import', class: "btn btn-create", tabindex: 4 | ... | ... |
app/views/projects/new.html.haml
... | ... | @@ -23,6 +23,7 @@ |
23 | 23 | .col-sm-2 |
24 | 24 | .col-sm-10 |
25 | 25 | = link_to "#", class: 'js-toggle-button' do |
26 | + %i.icon-edit | |
26 | 27 | %span Customize repository name? |
27 | 28 | .js-toggle-content.hide |
28 | 29 | .form-group |
... | ... | @@ -46,8 +47,10 @@ |
46 | 47 | %span Import existing repo |
47 | 48 | .col-sm-10 |
48 | 49 | = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git' |
49 | - .light | |
50 | - URL must be cloneable | |
50 | + .bs-callout.bs-callout-info | |
51 | + This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git. | |
52 | + %br | |
53 | + The import will time out after 2 minutes. For big repositories, use a clone/push combination. | |
51 | 54 | %hr |
52 | 55 | |
53 | 56 | .form-group | ... | ... |
app/workers/repository_import_worker.rb
... | ... | @@ -11,11 +11,11 @@ class RepositoryImportWorker |
11 | 11 | project.import_url) |
12 | 12 | |
13 | 13 | if result |
14 | - project.imported = true | |
14 | + project.import_finish | |
15 | 15 | project.save |
16 | 16 | project.satellite.create unless project.satellite.exists? |
17 | 17 | else |
18 | - project.imported = false | |
18 | + project.import_fail | |
19 | 19 | end |
20 | 20 | end |
21 | 21 | end | ... | ... |
config/routes.rb
db/migrate/20140312145357_add_import_status_to_project.rb
0 → 100644
db/migrate/20140313092127_migrate_already_imported_projects.rb
0 → 100644
... | ... | @@ -0,0 +1,12 @@ |
1 | +class MigrateAlreadyImportedProjects < ActiveRecord::Migration | |
2 | + def up | |
3 | + Project.where(imported: true).update_all(import_status: "finished") | |
4 | + Project.where(imported: false).update_all(import_status: "none") | |
5 | + remove_column :projects, :imported | |
6 | + end | |
7 | + | |
8 | + def down | |
9 | + add_column :projects, :imported, :boolean, default: false | |
10 | + Project.where(import_status: 'finished').update_all(imported: true) | |
11 | + end | |
12 | +end | ... | ... |
db/schema.rb
... | ... | @@ -11,7 +11,7 @@ |
11 | 11 | # |
12 | 12 | # It's strongly recommended that you check this file into your version control system. |
13 | 13 | |
14 | -ActiveRecord::Schema.define(version: 20140305193308) do | |
14 | +ActiveRecord::Schema.define(version: 20140313092127) do | |
15 | 15 | |
16 | 16 | # These are extensions that must be enabled in order to support this database |
17 | 17 | enable_extension "plpgsql" |
... | ... | @@ -214,10 +214,10 @@ ActiveRecord::Schema.define(version: 20140305193308) do |
214 | 214 | t.string "issues_tracker_id" |
215 | 215 | t.boolean "snippets_enabled", default: true, null: false |
216 | 216 | t.datetime "last_activity_at" |
217 | - t.boolean "imported", default: false, null: false | |
218 | 217 | t.string "import_url" |
219 | 218 | t.integer "visibility_level", default: 0, null: false |
220 | 219 | t.boolean "archived", default: false, null: false |
220 | + t.string "import_status" | |
221 | 221 | end |
222 | 222 | |
223 | 223 | add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree | ... | ... |