Download as
Browse Code »

Commit c33d5e16fe5f5dde4f270adaf7fb6fe5b9552018

Authored by Dmitriy Zaporozhets
1 parent 362d82d1
Exists in master and in 4 other branches 6-8-stable, 7-0-stable, 7-0-stable-spb, spb-stable

refactor backup/restore

Showing 5 changed files with 140 additions and 103 deletions   Show diff stats
doc/install/databases.md
  Diff comments   View file @c33d5e1
@@ -21,7 +21,7 @@ GitLab supports the following databases: @@ -21,7 +21,7 @@ GitLab supports the following databases:
21 mysql> CREATE DATABASE IF NOT EXISTS `gitlabhq_production` DEFAULT CHARACTER SET `utf8` COLLATE `utf8_unicode_ci`; 21 mysql> CREATE DATABASE IF NOT EXISTS `gitlabhq_production` DEFAULT CHARACTER SET `utf8` COLLATE `utf8_unicode_ci`;
22 22
23 # Grant the GitLab user necessary permissopns on the table. 23 # Grant the GitLab user necessary permissopns on the table.
24 - mysql> GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER ON `gitlabhq_production`.* TO 'gitlab'@'localhost'; 24 + mysql> GRANT SELECT, LOCK TABLES, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER ON `gitlabhq_production`.* TO 'gitlab'@'localhost';
25 25
26 # Quit the database session 26 # Quit the database session
27 mysql> \q 27 mysql> \q
lib/backup.rb
View file @362d82d
@@ -1,56 +0,0 @@ @@ -1,56 +0,0 @@
1 -require 'yaml'  
2 -  
3 -class Backup  
4 - attr_reader :config, :db_dir  
5 -  
6 - def initialize  
7 - @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]  
8 - @db_dir = File.join(Gitlab.config.backup.path, 'db')  
9 - FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir)  
10 - end  
11 -  
12 - def backup_db  
13 - case config["adapter"]  
14 - when /^mysql/ then  
15 - system("mysqldump #{mysql_args} #{config['database']} > #{db_file_name}")  
16 - when "postgresql" then  
17 - pg_env  
18 - system("pg_dump #{config['database']} > #{db_file_name}")  
19 - end  
20 - end  
21 -  
22 - def restore_db  
23 - case config["adapter"]  
24 - when /^mysql/ then  
25 - system("mysql #{mysql_args} #{config['database']} < #{db_file_name}")  
26 - when "postgresql" then  
27 - pg_env  
28 - system("pg_restore #{config['database']} #{db_file_name}")  
29 - end  
30 - end  
31 -  
32 - protected  
33 -  
34 - def db_file_name  
35 - File.join(db_dir, 'database.sql')  
36 - end  
37 -  
38 - def mysql_args  
39 - args = {  
40 - 'host' => '--host',  
41 - 'port' => '--port',  
42 - 'socket' => '--socket',  
43 - 'username' => '--user',  
44 - 'encoding' => '--default-character-set',  
45 - 'password' => '--password'  
46 - }  
47 - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact.join(' ')  
48 - end  
49 -  
50 - def pg_env  
51 - ENV['PGUSER'] = config["username"] if config["username"]  
52 - ENV['PGHOST'] = config["host"] if config["host"]  
53 - ENV['PGPORT'] = config["port"].to_s if config["port"]  
54 - ENV['PGPASSWORD'] = config["password"].to_s if config["password"]  
55 - end  
56 -end  
lib/backup/database.rb 0 → 100644
  Diff comments   View file @c33d5e1
@@ -0,0 +1,58 @@ @@ -0,0 +1,58 @@
  1 +require 'yaml'
  2 +
  3 +module Backup
  4 + class Database
  5 + attr_reader :config, :db_dir
  6 +
  7 + def initialize
  8 + @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
  9 + @db_dir = File.join(Gitlab.config.backup.path, 'db')
  10 + FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir)
  11 + end
  12 +
  13 + def dump
  14 + case config["adapter"]
  15 + when /^mysql/ then
  16 + system("mysqldump #{mysql_args} #{config['database']} > #{db_file_name}")
  17 + when "postgresql" then
  18 + pg_env
  19 + system("pg_dump #{config['database']} > #{db_file_name}")
  20 + end
  21 + end
  22 +
  23 + def restore
  24 + case config["adapter"]
  25 + when /^mysql/ then
  26 + system("mysql #{mysql_args} #{config['database']} < #{db_file_name}")
  27 + when "postgresql" then
  28 + pg_env
  29 + system("pg_restore #{config['database']} #{db_file_name}")
  30 + end
  31 + end
  32 +
  33 + protected
  34 +
  35 + def db_file_name
  36 + File.join(db_dir, 'database.sql')
  37 + end
  38 +
  39 + def mysql_args
  40 + args = {
  41 + 'host' => '--host',
  42 + 'port' => '--port',
  43 + 'socket' => '--socket',
  44 + 'username' => '--user',
  45 + 'encoding' => '--default-character-set',
  46 + 'password' => '--password'
  47 + }
  48 + args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact.join(' ')
  49 + end
  50 +
  51 + def pg_env
  52 + ENV['PGUSER'] = config["username"] if config["username"]
  53 + ENV['PGHOST'] = config["host"] if config["host"]
  54 + ENV['PGPORT'] = config["port"].to_s if config["port"]
  55 + ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
  56 + end
  57 + end
  58 +end
lib/backup/repository.rb 0 → 100644
  Diff comments   View file @c33d5e1
@@ -0,0 +1,74 @@ @@ -0,0 +1,74 @@
  1 +require 'yaml'
  2 +
  3 +module Backup
  4 + class Repository
  5 + attr_reader :repos_path
  6 +
  7 + def dump
  8 + prepare
  9 +
  10 + Project.find_each(batch_size: 1000) do |project|
  11 + print " * #{project.path_with_namespace} ... "
  12 +
  13 + if project.empty_repo?
  14 + puts "[SKIPPED]".cyan
  15 + next
  16 + end
  17 +
  18 + # Create namespace dir if missing
  19 + FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace
  20 +
  21 + if system("cd #{path_to_repo(project)} > /dev/null 2>&1 && git bundle create #{path_to_bundle(project)} --all > /dev/null 2>&1")
  22 + puts "[DONE]".green
  23 + else
  24 + puts "[FAILED]".red
  25 + end
  26 + end
  27 + end
  28 +
  29 + def restore
  30 + if File.exists?(repos_path)
  31 + # Move repos dir to 'repositories.old' dir
  32 + bk_repos_path = File.join(repos_path, '..', 'repositories.old')
  33 + FileUtils.mv(repos_path, bk_repos_path)
  34 + end
  35 +
  36 + FileUtils.mkdir_p(repos_path)
  37 +
  38 + Project.find_each(batch_size: 1000) do |project|
  39 + print "#{project.path_with_namespace} ... "
  40 +
  41 + project.namespace.ensure_dir_exist if project.namespace
  42 +
  43 + if system("git clone --bare #{path_to_bundle(project)} #{path_to_repo(project)} > /dev/null 2>&1")
  44 + puts "[DONE]".green
  45 + else
  46 + puts "[FAILED]".red
  47 + end
  48 + end
  49 + end
  50 +
  51 + protected
  52 +
  53 + def path_to_repo(project)
  54 + File.join(repos_path, project.path_with_namespace + '.git')
  55 + end
  56 +
  57 + def path_to_bundle(project)
  58 + File.join(backup_repos_path, project.path_with_namespace + ".bundle")
  59 + end
  60 +
  61 + def repos_path
  62 + Gitlab.config.gitlab_shell.repos_path
  63 + end
  64 +
  65 + def backup_repos_path
  66 + File.join(Gitlab.config.backup.path, "repositories")
  67 + end
  68 +
  69 + def prepare
  70 + FileUtils.rm_rf(backup_repos_path)
  71 + FileUtils.mkdir_p(backup_repos_path)
  72 + end
  73 + end
  74 +end
lib/tasks/gitlab/backup.rake
  Diff comments   View file @c33d5e1
@@ -111,67 +111,28 @@ namespace :gitlab do @@ -111,67 +111,28 @@ namespace :gitlab do
111 111
112 namespace :repo do 112 namespace :repo do
113 task :create => :environment do 113 task :create => :environment do
114 - backup_path_repo = File.join(Gitlab.config.backup.path, "repositories")  
115 - FileUtils.mkdir_p(backup_path_repo) until Dir.exists?(backup_path_repo)  
116 puts "Dumping repositories ...".blue 114 puts "Dumping repositories ...".blue
117 -  
118 - Project.find_each(:batch_size => 1000) do |project|  
119 - print " * #{project.path_with_namespace} ... "  
120 -  
121 - if project.empty_repo?  
122 - puts "[SKIPPED]".cyan  
123 - next  
124 - end  
125 -  
126 - # Create namespace dir if missing  
127 - FileUtils.mkdir_p(File.join(backup_path_repo, project.namespace.path)) if project.namespace  
128 -  
129 - # Build a destination path for backup  
130 - path_to_bundle = File.join(backup_path_repo, project.path_with_namespace + ".bundle")  
131 -  
132 - if Kernel.system("cd #{project.repository.path_to_repo} > /dev/null 2>&1 && git bundle create #{path_to_bundle} --all > /dev/null 2>&1")  
133 - puts "[DONE]".green  
134 - else  
135 - puts "[FAILED]".red  
136 - end  
137 - end 115 + Backup::Repository.new.dump
  116 + puts "done".green
138 end 117 end
139 118
140 task :restore => :environment do 119 task :restore => :environment do
141 - backup_path_repo = File.join(Gitlab.config.backup.path, "repositories")  
142 - repos_path = Gitlab.config.gitlab_shell.repos_path  
143 -  
144 - puts "Restoring repositories ... "  
145 -  
146 - Project.find_each(:batch_size => 1000) do |project|  
147 - print "#{project.path_with_namespace} ... "  
148 -  
149 - if project.namespace  
150 - project.namespace.ensure_dir_exist  
151 - end  
152 -  
153 - # Build a backup path  
154 - path_to_bundle = File.join(backup_path_repo, project.path_with_namespace + ".bundle")  
155 -  
156 - if Kernel.system("git clone --bare #{path_to_bundle} #{project.repository.path_to_repo} > /dev/null 2>&1")  
157 - puts "[DONE]".green  
158 - else  
159 - puts "[FAILED]".red  
160 - end  
161 - end 120 + puts "Restoring repositories ...".blue
  121 + Backup::Repository.new.restore
  122 + puts "done".green
162 end 123 end
163 end 124 end
164 125
165 namespace :db do 126 namespace :db do
166 task :create => :environment do 127 task :create => :environment do
167 puts "Dumping database ... ".blue 128 puts "Dumping database ... ".blue
168 - Backup.new.backup_db 129 + Backup::Database.new.dump
169 puts "done".green 130 puts "done".green
170 end 131 end
171 132
172 task :restore => :environment do 133 task :restore => :environment do
173 puts "Restoring database ... ".blue 134 puts "Restoring database ... ".blue
174 - Backup.new.restore_db 135 + Backup::Database.new.restore
175 puts "done".green 136 puts "done".green
176 end 137 end
177 end 138 end