diff --git a/Rakefile b/Rakefile index 2063f98..d857e80 100644 --- a/Rakefile +++ b/Rakefile @@ -81,35 +81,35 @@ end task :backup => ssh_config_file do # setup - sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'rm -rf /tmp/backups' - sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'rm -rf /tmp/backups' sh 'mkdir', '-p', 'backups' # integration - sh 'scp', '-F', ssh_config_file, 'utils/migration/backup_integration.sh', 'integration:/tmp' - sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', '/tmp/backup_integration.sh' - sh 'scp', '-F', ssh_config_file, 'integration:/tmp/backups/*', 'backups/' + sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'chmod a+xr /.snapshots' + sh 'scp', '-F', ssh_config_file, 'integration:/.snapshots/hourly.0/spb/*', 'backups/' # social - sh 'scp', '-F', ssh_config_file, 'utils/migration/backup_social.sh', 'social:/tmp' - sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', '/tmp/backup_social.sh' - sh 'scp', '-F', ssh_config_file, 'social:/tmp/backups/*', 'backups/' + sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'chmod a+xr /.snapshots' + sh 'scp', '-F', ssh_config_file, 'social:/.snapshots/hourly.0/spb/*', 'backups/' end -task :restore => ssh_config_file do +task :restore => [ssh_config_file, config_file] do # setup sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'rm -rf /tmp/backups' + sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'systemctl stop colab' sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'rm -rf /tmp/backups' sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'systemctl stop noosfero' + sh 'ssh', '-F', ssh_config_file, 'database', 'sudo', 'sudo -u postgres dropdb colab 2> /dev/null' + sh 'ssh', '-F', ssh_config_file, 'database', 'sudo', 'sudo -u postgres createdb colab --owner colab 2> /dev/null' sh 'ssh', '-F', ssh_config_file, 'database', 'sudo', 'sudo -u postgres dropdb noosfero 2> /dev/null' sh 'ssh', '-F', ssh_config_file, 'database', 'sudo', 'sudo -u postgres createdb noosfero --owner noosfero 2> /dev/null' #integration sh 'scp', '-r', '-F', ssh_config_file, 'backups', 'integration:/tmp' sh 'scp', '-F', ssh_config_file, 'utils/migration/restore_integration.sh', 'integration:/tmp' - sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', '/tmp/restore_integration.sh' + sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', "env SPB_URL=#{config['lists_hostname']} /tmp/restore_integration.sh" #social sh 'scp', '-r', '-F', ssh_config_file, 'backups', 'social:/tmp' sh 'scp', '-F', ssh_config_file, 'utils/migration/restore_social.sh', 'social:/tmp' sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', '/tmp/restore_social.sh' sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'systemctl start noosfero' + sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'systemctl start colab' end task :bootstrap_common => 'config/local/ssh_config' diff --git a/cookbooks/backup/files/host-integration/backup_spb.sh b/cookbooks/backup/files/host-integration/backup_spb.sh new file mode 100755 index 0000000..9490506 --- /dev/null +++ b/cookbooks/backup/files/host-integration/backup_spb.sh @@ -0,0 +1,18 @@ +#!/bin/bash +#colab-admin dumpdata > /tmp/backups/colab_dump.json +pg_dump -h database -U colab colab > colab.dump +# GitLab Backup +cd /usr/lib/gitlab +sudo -u git bundle exec rake gitlab:backup:create RAILS_ENV=production +cd - +# TODO fix regular expression +mv /var/lib/gitlab/backups/*_gitlab_backup.tar . + +tar -czf gitlab_shell_ssh.tar.gz /var/lib/gitlab-shell/.ssh/ + +# Mailman Backup +cd /var/lib/mailman +tar -cpzf mailman_backup.tar.gz lists/ data/ archives/ +cd - +mv /var/lib/mailman/mailman_backup.tar.gz . + diff --git a/cookbooks/backup/files/host-social/backup_spb.sh b/cookbooks/backup/files/host-social/backup_spb.sh new file mode 100755 index 0000000..08244bf --- /dev/null +++ b/cookbooks/backup/files/host-social/backup_spb.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd /usr/lib/noosfero +RAILS_ENV=production sudo -u noosfero bundle exec rake backup +cd - +# TODO fix regular expression +mv /usr/lib/noosfero/tmp/backup/*.tar.gz noosfero_backup.tar.gz diff --git a/cookbooks/backup/files/rsnapshot-spb b/cookbooks/backup/files/rsnapshot-spb new file mode 100644 index 0000000..a3a0769 --- /dev/null +++ b/cookbooks/backup/files/rsnapshot-spb @@ -0,0 +1,4 @@ +0 */6 * * * root rsnapshot hourly +59 23 * * * root rsnapshot daily +40 23 * * 7 root rsnapshot weekly +20 23 1 * * root rsnapshot monthly diff --git a/cookbooks/backup/files/rsnapshot.conf b/cookbooks/backup/files/rsnapshot.conf new file mode 100644 index 0000000..2bce291 --- /dev/null +++ b/cookbooks/backup/files/rsnapshot.conf @@ -0,0 +1,220 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULES: # +# # +# This file requires tabs between elements # +# # +# Directories require a trailing slash: # +# right: /home/ # +# wrong: /home # +# # +################################################# + +####################### +# CONFIG FILE VERSION # +####################### + +config_version 1.2 + +########################### +# SNAPSHOT ROOT DIRECTORY # +########################### + +# All snapshots will be stored under this root directory. +# +snapshot_root /.snapshots/ + +# If no_create_root is enabled, rsnapshot will not automatically create the +# snapshot_root directory. This is particularly useful if you are backing +# up to removable media, such as a FireWire or USB drive. +# +#no_create_root 1 + +################################# +# EXTERNAL PROGRAM DEPENDENCIES # +################################# + +# LINUX USERS: Be sure to uncomment "cmd_cp". This gives you extra features. +# EVERYONE ELSE: Leave "cmd_cp" commented out for compatibility. +# +# See the README file or the man page for more details. +# +cmd_cp /usr/bin/cp + +# uncomment this to use the rm program instead of the built-in perl routine. +# +cmd_rm /usr/bin/rm + +# rsync must be enabled for anything to work. This is the only command that +# must be enabled. +# +cmd_rsync /usr/bin/rsync + +# Uncomment this to enable remote ssh backups over rsync. +# +#cmd_ssh /usr/bin/ssh + +# Comment this out to disable syslog support. +# +cmd_logger /usr/bin/logger + +# Uncomment this to specify the path to "du" for disk usage checks. +# If you have an older version of "du", you may also want to check the +# "du_args" parameter below. +# +cmd_du /usr/bin/du + +# Uncomment this to specify the path to rsnapshot-diff. +# +#cmd_rsnapshot_diff /usr/local/bin/rsnapshot-diff + +# Specify the path to a script (and any optional arguments) to run right +# before rsnapshot syncs files +# +#cmd_preexec /path/to/preexec/script + +# Specify the path to a script (and any optional arguments) to run right +# after rsnapshot syncs files +# +#cmd_postexec /path/to/postexec/script + +######################################### +# BACKUP INTERVALS # +# Must be unique and in ascending order # +# i.e. hourly, daily, weekly, etc. # +######################################### + +interval hourly 4 +interval daily 6 +interval weekly 2 +interval monthly 2 + +############################################ +# GLOBAL OPTIONS # +# All are optional, with sensible defaults # +############################################ + +# Verbose level, 1 through 5. +# 1 Quiet Print fatal errors only +# 2 Default Print errors and warnings only +# 3 Verbose Show equivalent shell commands being executed +# 4 Extra Verbose Show extra verbose information +# 5 Debug mode Everything +# +verbose 2 + +# Same as "verbose" above, but controls the amount of data sent to the +# logfile, if one is being used. The default is 3. +# +loglevel 3 + +# If you enable this, data will be written to the file you specify. The +# amount of data written is controlled by the "loglevel" parameter. +# +logfile /var/log/rsnapshot + +# If enabled, rsnapshot will write a lockfile to prevent two instances +# from running simultaneously (and messing up the snapshot_root). +# If you enable this, make sure the lockfile directory is not world +# writable. Otherwise anyone can prevent the program from running. +# +lockfile /var/run/rsnapshot.pid + +# Default rsync args. All rsync commands have at least these options set. +# +#rsync_short_args -a +#rsync_long_args --delete --numeric-ids --relative --delete-excluded + +# ssh has no args passed by default, but you can specify some here. +# +#ssh_args -p 22 + +# Default arguments for the "du" program (for disk space reporting). +# The GNU version of "du" is preferred. See the man page for more details. +# If your version of "du" doesn't support the -h flag, try -k flag instead. +# +#du_args -csh + +# If this is enabled, rsync won't span filesystem partitions within a +# backup point. This essentially passes the -x option to rsync. +# The default is 0 (off). +# +#one_fs 0 + +# The include and exclude parameters, if enabled, simply get passed directly +# to rsync. If you have multiple include/exclude patterns, put each one on a +# separate line. Please look up the --include and --exclude options in the +# rsync man page for more details on how to specify file name patterns. +# +#include ??? +#include ??? +#exclude ??? +#exclude ??? + +# The include_file and exclude_file parameters, if enabled, simply get +# passed directly to rsync. Please look up the --include-from and +# --exclude-from options in the rsync man page for more details. +# +#include_file /path/to/include/file +#exclude_file /path/to/exclude/file + +# If your version of rsync supports --link-dest, consider enable this. +# This is the best way to support special files (FIFOs, etc) cross-platform. +# The default is 0 (off). +# +#link_dest 0 + +# When sync_first is enabled, it changes the default behaviour of rsnapshot. +# Normally, when rsnapshot is called with its lowest interval +# (i.e.: "rsnapshot hourly"), it will sync files AND rotate the lowest +# intervals. With sync_first enabled, "rsnapshot sync" handles the file sync, +# and all interval calls simply rotate files. See the man page for more +# details. The default is 0 (off). +# +#sync_first 0 + +# If enabled, rsnapshot will move the oldest directory for each interval +# to [interval_name].delete, then it will remove the lockfile and delete +# that directory just before it exits. The default is 0 (off). +# +#use_lazy_deletes 0 + +# Number of rsync re-tries. If you experience any network problems or +# network card issues that tend to cause ssh to crap-out with +# "Corrupted MAC on input" errors, for example, set this to a non-zero +# value to have the rsync operation re-tried +# +#rsync_numtries 0 + +############################### +### BACKUP POINTS / SCRIPTS ### +############################### + +# LOCALHOST +#backup /home/ localhost/ +#backup /etc/ localhost/ +#backup /usr/local/ localhost/ +#backup /var/log/rsnapshot localhost/ +#backup /etc/passwd localhost/ +#backup /home/foo/My Documents/ localhost/ +#backup /foo/bar/ localhost/ one_fs=1, rsync_short_args=-urltvpog +#backup_script /usr/local/bin/backup_pgsql.sh localhost/postgres/ + +# EXAMPLE.COM +#backup_script /bin/date "+ backup of example.com started at %c" unused1 +#backup root@example.com:/home/ example.com/ +rsync_long_args=--bwlimit=16,exclude=core +#backup root@example.com:/etc/ example.com/ exclude=mtab,exclude=core +#backup_script ssh root@example.com "mysqldump -A > /var/db/dump/mysql.sql" unused2 +#backup root@example.com:/var/db/dump/ example.com/ +#backup_script /bin/date "+ backup of example.com ended at %c" unused9 +#backup_script /bin/date "+ backup of example.com ended at %c" unused9 + +backup_script /usr/local/bin/backup_spb.sh spb/ + +# CVS.SOURCEFORGE.NET +#backup_script /usr/local/bin/backup_rsnapshot_cvsroot.sh rsnapshot.cvs.sourceforge.net/ + +# RSYNC.SAMBA.ORG +#backup rsync://rsync.samba.org/rsyncftp/ rsync.samba.org/rsyncftp/ diff --git a/cookbooks/backup/recipes/default.rb b/cookbooks/backup/recipes/default.rb new file mode 100644 index 0000000..63d0dc1 --- /dev/null +++ b/cookbooks/backup/recipes/default.rb @@ -0,0 +1,34 @@ +package 'cronie' +package 'rsnapshot' + +cookbook_file '/etc/rsnapshot.conf' do + owner 'root' + group 'root' + mode 0644 +end + +#case node.name +#when "social" +# cookbook_file '/usr/local/bin/backup_noosfero.sh' do +# owner 'root' +# group 'root' +# mode 0755 +# end +#end + +cookbook_file '/usr/local/bin/backup_spb.sh' do + owner 'root' + group 'root' + mode 0755 +end + +cookbook_file '/etc/cron.d/rsnapshot-spb' do + owner 'root' + group 'root' + mode 0644 +end + +service 'crond' do + action [:enable, :restart] +end + diff --git a/roles/integration_server.rb b/roles/integration_server.rb index f263121..4782e33 100644 --- a/roles/integration_server.rb +++ b/roles/integration_server.rb @@ -11,4 +11,5 @@ run_list *[ 'recipe[colab]', 'recipe[colab::nginx]', 'recipe[gitlab]', + 'recipe[backup]', ] diff --git a/roles/social_server.rb b/roles/social_server.rb index ed9e3a0..231f532 100644 --- a/roles/social_server.rb +++ b/roles/social_server.rb @@ -2,5 +2,6 @@ name 'database_server' description 'Social networking server' run_list *[ 'recipe[basics::nginx]', - 'recipe[noosfero]' + 'recipe[noosfero]', + 'recipe[backup]' ] diff --git a/utils/migration/backup_integration.sh b/utils/migration/backup_integration.sh deleted file mode 100755 index c339b59..0000000 --- a/utils/migration/backup_integration.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -echo 'I: starting integration backup...' - -echo 'I: Creating /tmp/backups/ on integration' -mkdir -p /tmp/backups/ - -# Colab Backup -echo 'I: dumping colab data' -colab-admin dumpdata > /tmp/backups/colab_dump.json - -# GitLab Backup -echo 'cleaning gitlab backups directory' -sudo rm -rf /var/lib/gitlab/backups/* -echo 'I: creating gitlab backup' -cd /usr/lib/gitlab -sudo -u git bundle exec rake gitlab:backup:create RAILS_ENV=production > /dev/null -# TODO fix regular expression -mv /var/lib/gitlab/backups/*_gitlab_backup.tar /tmp/backups - -echo 'I: creating gitlab shell ssh backup' -tar -czf /tmp/backups/gitlab_shell_ssh.tar.gz /var/lib/gitlab-shell/.ssh/ 2> /dev/null - -# Mailman Backup -cd /var/lib/mailman -echo 'I: creating mailman backups' -tar -czf /tmp/backups/mailman_backup.tar.gz lists/ data/ archives/ - -echo 'I: integration backup done.' - diff --git a/utils/migration/restore_integration.sh b/utils/migration/restore_integration.sh index 67daab0..a982919 100755 --- a/utils/migration/restore_integration.sh +++ b/utils/migration/restore_integration.sh @@ -3,7 +3,8 @@ echo 'Starting restore on integration...' # Colab Restore echo 'restoring colab...' -colab-admin loaddata /tmp/backups/colab_dump.json > /dev/null +psql -U colab -h database colab < /tmp/backups/colab.dump 1> /dev/null 2> /dev/null +colab-admin migrate > /dev/null echo 'done.' # Gitlab Restore @@ -14,12 +15,16 @@ echo 'restoring gitlab...' mv /tmp/backups/*_gitlab_backup.tar /var/lib/gitlab/backups/ cd /usr/lib/gitlab sudo -u git bundle exec rake gitlab:backup:restore RAILS_ENV=production force=yes 1> /dev/null 2>/dev/null +sudo rm -rf /var/lib/gitlab/backups/* echo 'done.' # Mailman Restore echo 'restoring mailman...' -mv /tmp/backups/mailman_backup.tar.gz /var/lib/mailman/ +sudo mv /tmp/backups/mailman_backup.tar.gz /var/lib/mailman/ cd /var/lib/mailman -tar -vxzf mailman_backup.tar.gz 1> /dev/null 2> /dev/null +sudo tar -xzf mailman_backup.tar.gz +sudo rm mailman_backup.tar.gz +cd /usr/lib/mailman/bin +for list in `sudo ls /var/lib/mailman/lists`; do sudo ./withlist -l -r fix_url $list -u $SPB_URL 1> /dev/null 2> /dev/null; done echo 'done.' -- libgit2 0.21.2