Download as
Browse Code »

Commit c33d5e16fe5f5dde4f270adaf7fb6fe5b9552018

Authored by Dmitriy Zaporozhets
1 parent 362d82d1
Exists in master and in 4 other branches 6-8-stable, 7-0-stable, 7-0-stable-spb, spb-stable

refactor backup/restore

Showing 5 changed files with 140 additions and 103 deletions   Show diff stats
doc/install/databases.md
  Diff comments   View file @c33d5e1
... ... @@ -21,7 +21,7 @@ GitLab supports the following databases:
21 21 mysql> CREATE DATABASE IF NOT EXISTS `gitlabhq_production` DEFAULT CHARACTER SET `utf8` COLLATE `utf8_unicode_ci`;
22 22  
23 23 # Grant the GitLab user necessary permissopns on the table.
24   - mysql> GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER ON `gitlabhq_production`.* TO 'gitlab'@'localhost';
  24 + mysql> GRANT SELECT, LOCK TABLES, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER ON `gitlabhq_production`.* TO 'gitlab'@'localhost';
25 25  
26 26 # Quit the database session
27 27 mysql> \q
... ...
lib/backup.rb
View file @362d82d
... ... @@ -1,56 +0,0 @@
1   -require 'yaml'
2   -
3   -class Backup
4   - attr_reader :config, :db_dir
5   -
6   - def initialize
7   - @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
8   - @db_dir = File.join(Gitlab.config.backup.path, 'db')
9   - FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir)
10   - end
11   -
12   - def backup_db
13   - case config["adapter"]
14   - when /^mysql/ then
15   - system("mysqldump #{mysql_args} #{config['database']} > #{db_file_name}")
16   - when "postgresql" then
17   - pg_env
18   - system("pg_dump #{config['database']} > #{db_file_name}")
19   - end
20   - end
21   -
22   - def restore_db
23   - case config["adapter"]
24   - when /^mysql/ then
25   - system("mysql #{mysql_args} #{config['database']} < #{db_file_name}")
26   - when "postgresql" then
27   - pg_env
28   - system("pg_restore #{config['database']} #{db_file_name}")
29   - end
30   - end
31   -
32   - protected
33   -
34   - def db_file_name
35   - File.join(db_dir, 'database.sql')
36   - end
37   -
38   - def mysql_args
39   - args = {
40   - 'host' => '--host',
41   - 'port' => '--port',
42   - 'socket' => '--socket',
43   - 'username' => '--user',
44   - 'encoding' => '--default-character-set',
45   - 'password' => '--password'
46   - }
47   - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact.join(' ')
48   - end
49   -
50   - def pg_env
51   - ENV['PGUSER'] = config["username"] if config["username"]
52   - ENV['PGHOST'] = config["host"] if config["host"]
53   - ENV['PGPORT'] = config["port"].to_s if config["port"]
54   - ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
55   - end
56   -end
lib/backup/database.rb 0 → 100644
  Diff comments   View file @c33d5e1
... ... @@ -0,0 +1,58 @@
  1 +require 'yaml'
  2 +
  3 +module Backup
  4 + class Database
  5 + attr_reader :config, :db_dir
  6 +
  7 + def initialize
  8 + @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
  9 + @db_dir = File.join(Gitlab.config.backup.path, 'db')
  10 + FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir)
  11 + end
  12 +
  13 + def dump
  14 + case config["adapter"]
  15 + when /^mysql/ then
  16 + system("mysqldump #{mysql_args} #{config['database']} > #{db_file_name}")
  17 + when "postgresql" then
  18 + pg_env
  19 + system("pg_dump #{config['database']} > #{db_file_name}")
  20 + end
  21 + end
  22 +
  23 + def restore
  24 + case config["adapter"]
  25 + when /^mysql/ then
  26 + system("mysql #{mysql_args} #{config['database']} < #{db_file_name}")
  27 + when "postgresql" then
  28 + pg_env
  29 + system("pg_restore #{config['database']} #{db_file_name}")
  30 + end
  31 + end
  32 +
  33 + protected
  34 +
  35 + def db_file_name
  36 + File.join(db_dir, 'database.sql')
  37 + end
  38 +
  39 + def mysql_args
  40 + args = {
  41 + 'host' => '--host',
  42 + 'port' => '--port',
  43 + 'socket' => '--socket',
  44 + 'username' => '--user',
  45 + 'encoding' => '--default-character-set',
  46 + 'password' => '--password'
  47 + }
  48 + args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact.join(' ')
  49 + end
  50 +
  51 + def pg_env
  52 + ENV['PGUSER'] = config["username"] if config["username"]
  53 + ENV['PGHOST'] = config["host"] if config["host"]
  54 + ENV['PGPORT'] = config["port"].to_s if config["port"]
  55 + ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
  56 + end
  57 + end
  58 +end
... ...
lib/backup/repository.rb 0 → 100644
  Diff comments   View file @c33d5e1
... ... @@ -0,0 +1,74 @@
  1 +require 'yaml'
  2 +
  3 +module Backup
  4 + class Repository
  5 + attr_reader :repos_path
  6 +
  7 + def dump
  8 + prepare
  9 +
  10 + Project.find_each(batch_size: 1000) do |project|
  11 + print " * #{project.path_with_namespace} ... "
  12 +
  13 + if project.empty_repo?
  14 + puts "[SKIPPED]".cyan
  15 + next
  16 + end
  17 +
  18 + # Create namespace dir if missing
  19 + FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace
  20 +
  21 + if system("cd #{path_to_repo(project)} > /dev/null 2>&1 && git bundle create #{path_to_bundle(project)} --all > /dev/null 2>&1")
  22 + puts "[DONE]".green
  23 + else
  24 + puts "[FAILED]".red
  25 + end
  26 + end
  27 + end
  28 +
  29 + def restore
  30 + if File.exists?(repos_path)
  31 + # Move repos dir to 'repositories.old' dir
  32 + bk_repos_path = File.join(repos_path, '..', 'repositories.old')
  33 + FileUtils.mv(repos_path, bk_repos_path)
  34 + end
  35 +
  36 + FileUtils.mkdir_p(repos_path)
  37 +
  38 + Project.find_each(batch_size: 1000) do |project|
  39 + print "#{project.path_with_namespace} ... "
  40 +
  41 + project.namespace.ensure_dir_exist if project.namespace
  42 +
  43 + if system("git clone --bare #{path_to_bundle(project)} #{path_to_repo(project)} > /dev/null 2>&1")
  44 + puts "[DONE]".green
  45 + else
  46 + puts "[FAILED]".red
  47 + end
  48 + end
  49 + end
  50 +
  51 + protected
  52 +
  53 + def path_to_repo(project)
  54 + File.join(repos_path, project.path_with_namespace + '.git')
  55 + end
  56 +
  57 + def path_to_bundle(project)
  58 + File.join(backup_repos_path, project.path_with_namespace + ".bundle")
  59 + end
  60 +
  61 + def repos_path
  62 + Gitlab.config.gitlab_shell.repos_path
  63 + end
  64 +
  65 + def backup_repos_path
  66 + File.join(Gitlab.config.backup.path, "repositories")
  67 + end
  68 +
  69 + def prepare
  70 + FileUtils.rm_rf(backup_repos_path)
  71 + FileUtils.mkdir_p(backup_repos_path)
  72 + end
  73 + end
  74 +end
... ...
lib/tasks/gitlab/backup.rake
  Diff comments   View file @c33d5e1
... ... @@ -111,67 +111,28 @@ namespace :gitlab do
111 111  
112 112 namespace :repo do
113 113 task :create => :environment do
114   - backup_path_repo = File.join(Gitlab.config.backup.path, "repositories")
115   - FileUtils.mkdir_p(backup_path_repo) until Dir.exists?(backup_path_repo)
116 114 puts "Dumping repositories ...".blue
117   -
118   - Project.find_each(:batch_size => 1000) do |project|
119   - print " * #{project.path_with_namespace} ... "
120   -
121   - if project.empty_repo?
122   - puts "[SKIPPED]".cyan
123   - next
124   - end
125   -
126   - # Create namespace dir if missing
127   - FileUtils.mkdir_p(File.join(backup_path_repo, project.namespace.path)) if project.namespace
128   -
129   - # Build a destination path for backup
130   - path_to_bundle = File.join(backup_path_repo, project.path_with_namespace + ".bundle")
131   -
132   - if Kernel.system("cd #{project.repository.path_to_repo} > /dev/null 2>&1 && git bundle create #{path_to_bundle} --all > /dev/null 2>&1")
133   - puts "[DONE]".green
134   - else
135   - puts "[FAILED]".red
136   - end
137   - end
  115 + Backup::Repository.new.dump
  116 + puts "done".green
138 117 end
139 118  
140 119 task :restore => :environment do
141   - backup_path_repo = File.join(Gitlab.config.backup.path, "repositories")
142   - repos_path = Gitlab.config.gitlab_shell.repos_path
143   -
144   - puts "Restoring repositories ... "
145   -
146   - Project.find_each(:batch_size => 1000) do |project|
147   - print "#{project.path_with_namespace} ... "
148   -
149   - if project.namespace
150   - project.namespace.ensure_dir_exist
151   - end
152   -
153   - # Build a backup path
154   - path_to_bundle = File.join(backup_path_repo, project.path_with_namespace + ".bundle")
155   -
156   - if Kernel.system("git clone --bare #{path_to_bundle} #{project.repository.path_to_repo} > /dev/null 2>&1")
157   - puts "[DONE]".green
158   - else
159   - puts "[FAILED]".red
160   - end
161   - end
  120 + puts "Restoring repositories ...".blue
  121 + Backup::Repository.new.restore
  122 + puts "done".green
162 123 end
163 124 end
164 125  
165 126 namespace :db do
166 127 task :create => :environment do
167 128 puts "Dumping database ... ".blue
168   - Backup.new.backup_db
  129 + Backup::Database.new.dump
169 130 puts "done".green
170 131 end
171 132  
172 133 task :restore => :environment do
173 134 puts "Restoring database ... ".blue
174   - Backup.new.restore_db
  135 + Backup::Database.new.restore
175 136 puts "done".green
176 137 end
177 138 end
... ...