Download as
Browse Code »

Commit ce25b7d242b6c28537e12e45bec8ba38c3626588

Authored by Athos
1 parent 1dab81ce
Exists in master and in 89 other branches 3.x, add_sisp_to_chef, add_super_archives_plugin, api_for_colab, automates_core_packing, backup_not_prod, cdtc_configuration, changes_in_buttons_on_content_panel, colab_automated_login, colab_spb_plugin_recipe, colab_widgets_settings, design_validation, dev-lappis, dev_env_minimal, disable_email_dev, docs, fix_breadcrumbs_position, fix_categories_software_link, fix_edit_institution, fix_edit_software_with_another_license, fix_get_license_info, fix_gitlab_assets_permission, fix_list_style_inside_article, fix_list_style_on_folder_elements, fix_members_pagination, fix_merge_request_url, fix_models_translations, fix_no_license, fix_software_api, fix_software_block_migration, fix_software_communities_translations, fix_software_communities_unit_test, fix_style_create_institution_admin_panel, fix_superarchives_imports, fix_sym_links_noosfero, focus_search_field_theme, gov-user-refactoring, gov-user-refactoring-rails4, header_fix, institution_modal_on_rating, kalibro-conf-refactoring, kalibro-processor-package, lxc_settings, margin_fix, mezuro_cookbook, performance, prezento, r3, refactor_download_block, refactor_software_communities, refactor_software_for_sisp, register_page, release-process, release-process-v2, remove-unused-images, remove_backup_emails, remove_broken_theme, remove_secondary_email_from_user, remove_sisp_buttons, removing_super_archives_email, review_message, scope2method, signals_user_noosfero, sisp_catalog_header, sisp_colab_config, sisp_dev, sisp_dev_master, sisp_simple_version, software_as_organization, software_catalog_style_fix, software_communities_html_refactor, software_infos_api, spb_minimal_env, spb_to_rails4, spec_refactor, stable-4.1, stable-4.2, stable-4.x, stable-devel, support_docs, syslog, temp_soft_comm_refactoring, theme_header, theme_javascript_refactory, thread_dropdown, thread_page, update_search_by_categories, update_software_api, update_softwares_boxes

Fix backup and restore tasks 

- The new backup task uses the rsnapshot package
- Cron job to run the backups
- Each server holds its own backups
- Backup task just downloads data to the host
Showing 10 changed files with 303 additions and 44 deletions   Show diff stats
@@ -81,35 +81,35 @@ end @@ -81,35 +81,35 @@ end
81 81
82 task :backup => ssh_config_file do 82 task :backup => ssh_config_file do
83 # setup 83 # setup
84 - sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'rm -rf /tmp/backups'  
85 - sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'rm -rf /tmp/backups'  
86 sh 'mkdir', '-p', 'backups' 84 sh 'mkdir', '-p', 'backups'
87 # integration 85 # integration
88 - sh 'scp', '-F', ssh_config_file, 'utils/migration/backup_integration.sh', 'integration:/tmp'  
89 - sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', '/tmp/backup_integration.sh'  
90 - sh 'scp', '-F', ssh_config_file, 'integration:/tmp/backups/*', 'backups/' 86 + sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'chmod a+xr /.snapshots'
  87 + sh 'scp', '-F', ssh_config_file, 'integration:/.snapshots/hourly.0/spb/*', 'backups/'
91 # social 88 # social
92 - sh 'scp', '-F', ssh_config_file, 'utils/migration/backup_social.sh', 'social:/tmp'  
93 - sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', '/tmp/backup_social.sh'  
94 - sh 'scp', '-F', ssh_config_file, 'social:/tmp/backups/*', 'backups/' 89 + sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'chmod a+xr /.snapshots'
  90 + sh 'scp', '-F', ssh_config_file, 'social:/.snapshots/hourly.0/spb/*', 'backups/'
95 end 91 end
96 92
97 -task :restore => ssh_config_file do 93 +task :restore => [ssh_config_file, config_file] do
98 # setup 94 # setup
99 sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'rm -rf /tmp/backups' 95 sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'rm -rf /tmp/backups'
  96 + sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'systemctl stop colab'
100 sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'rm -rf /tmp/backups' 97 sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'rm -rf /tmp/backups'
101 sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'systemctl stop noosfero' 98 sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'systemctl stop noosfero'
  99 + sh 'ssh', '-F', ssh_config_file, 'database', 'sudo', 'sudo -u postgres dropdb colab 2> /dev/null'
  100 + sh 'ssh', '-F', ssh_config_file, 'database', 'sudo', 'sudo -u postgres createdb colab --owner colab 2> /dev/null'
102 sh 'ssh', '-F', ssh_config_file, 'database', 'sudo', 'sudo -u postgres dropdb noosfero 2> /dev/null' 101 sh 'ssh', '-F', ssh_config_file, 'database', 'sudo', 'sudo -u postgres dropdb noosfero 2> /dev/null'
103 sh 'ssh', '-F', ssh_config_file, 'database', 'sudo', 'sudo -u postgres createdb noosfero --owner noosfero 2> /dev/null' 102 sh 'ssh', '-F', ssh_config_file, 'database', 'sudo', 'sudo -u postgres createdb noosfero --owner noosfero 2> /dev/null'
104 #integration 103 #integration
105 sh 'scp', '-r', '-F', ssh_config_file, 'backups', 'integration:/tmp' 104 sh 'scp', '-r', '-F', ssh_config_file, 'backups', 'integration:/tmp'
106 sh 'scp', '-F', ssh_config_file, 'utils/migration/restore_integration.sh', 'integration:/tmp' 105 sh 'scp', '-F', ssh_config_file, 'utils/migration/restore_integration.sh', 'integration:/tmp'
107 - sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', '/tmp/restore_integration.sh' 106 + sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', "env SPB_URL=#{config['lists_hostname']} /tmp/restore_integration.sh"
108 #social 107 #social
109 sh 'scp', '-r', '-F', ssh_config_file, 'backups', 'social:/tmp' 108 sh 'scp', '-r', '-F', ssh_config_file, 'backups', 'social:/tmp'
110 sh 'scp', '-F', ssh_config_file, 'utils/migration/restore_social.sh', 'social:/tmp' 109 sh 'scp', '-F', ssh_config_file, 'utils/migration/restore_social.sh', 'social:/tmp'
111 sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', '/tmp/restore_social.sh' 110 sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', '/tmp/restore_social.sh'
112 sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'systemctl start noosfero' 111 sh 'ssh', '-F', ssh_config_file, 'social', 'sudo', 'systemctl start noosfero'
  112 + sh 'ssh', '-F', ssh_config_file, 'integration', 'sudo', 'systemctl start colab'
113 end 113 end
114 114
115 task :bootstrap_common => 'config/local/ssh_config' 115 task :bootstrap_common => 'config/local/ssh_config'
cookbooks/backup/files/host-integration/backup_spb.sh 0 → 100755
  Diff comments   View file @ce25b7d
@@ -0,0 +1,18 @@ @@ -0,0 +1,18 @@
  1 +#!/bin/bash
  2 +#colab-admin dumpdata > /tmp/backups/colab_dump.json
  3 +pg_dump -h database -U colab colab > colab.dump
  4 +# GitLab Backup
  5 +cd /usr/lib/gitlab
  6 +sudo -u git bundle exec rake gitlab:backup:create RAILS_ENV=production
  7 +cd -
  8 +# TODO fix regular expression
  9 +mv /var/lib/gitlab/backups/*_gitlab_backup.tar .
  10 +
  11 +tar -czf gitlab_shell_ssh.tar.gz /var/lib/gitlab-shell/.ssh/
  12 +
  13 +# Mailman Backup
  14 +cd /var/lib/mailman
  15 +tar -cpzf mailman_backup.tar.gz lists/ data/ archives/
  16 +cd -
  17 +mv /var/lib/mailman/mailman_backup.tar.gz .
  18 +
cookbooks/backup/files/host-social/backup_spb.sh 0 → 100755
  Diff comments   View file @ce25b7d
@@ -0,0 +1,6 @@ @@ -0,0 +1,6 @@
  1 +#!/bin/bash
  2 +cd /usr/lib/noosfero
  3 +RAILS_ENV=production sudo -u noosfero bundle exec rake backup
  4 +cd -
  5 +# TODO fix regular expression
  6 +mv /usr/lib/noosfero/tmp/backup/*.tar.gz noosfero_backup.tar.gz
cookbooks/backup/files/rsnapshot-spb 0 → 100644
  Diff comments   View file @ce25b7d
@@ -0,0 +1,4 @@ @@ -0,0 +1,4 @@
  1 +0 */6 * * * root rsnapshot hourly
  2 +59 23 * * * root rsnapshot daily
  3 +40 23 * * 7 root rsnapshot weekly
  4 +20 23 1 * * root rsnapshot monthly
cookbooks/backup/files/rsnapshot.conf 0 → 100644
  Diff comments   View file @ce25b7d
@@ -0,0 +1,220 @@ @@ -0,0 +1,220 @@
  1 +#################################################
  2 +# rsnapshot.conf - rsnapshot configuration file #
  3 +#################################################
  4 +# #
  5 +# PLEASE BE AWARE OF THE FOLLOWING RULES: #
  6 +# #
  7 +# This file requires tabs between elements #
  8 +# #
  9 +# Directories require a trailing slash: #
  10 +# right: /home/ #
  11 +# wrong: /home #
  12 +# #
  13 +#################################################
  14 +
  15 +#######################
  16 +# CONFIG FILE VERSION #
  17 +#######################
  18 +
  19 +config_version 1.2
  20 +
  21 +###########################
  22 +# SNAPSHOT ROOT DIRECTORY #
  23 +###########################
  24 +
  25 +# All snapshots will be stored under this root directory.
  26 +#
  27 +snapshot_root /.snapshots/
  28 +
  29 +# If no_create_root is enabled, rsnapshot will not automatically create the
  30 +# snapshot_root directory. This is particularly useful if you are backing
  31 +# up to removable media, such as a FireWire or USB drive.
  32 +#
  33 +#no_create_root 1
  34 +
  35 +#################################
  36 +# EXTERNAL PROGRAM DEPENDENCIES #
  37 +#################################
  38 +
  39 +# LINUX USERS: Be sure to uncomment "cmd_cp". This gives you extra features.
  40 +# EVERYONE ELSE: Leave "cmd_cp" commented out for compatibility.
  41 +#
  42 +# See the README file or the man page for more details.
  43 +#
  44 +cmd_cp /usr/bin/cp
  45 +
  46 +# uncomment this to use the rm program instead of the built-in perl routine.
  47 +#
  48 +cmd_rm /usr/bin/rm
  49 +
  50 +# rsync must be enabled for anything to work. This is the only command that
  51 +# must be enabled.
  52 +#
  53 +cmd_rsync /usr/bin/rsync
  54 +
  55 +# Uncomment this to enable remote ssh backups over rsync.
  56 +#
  57 +#cmd_ssh /usr/bin/ssh
  58 +
  59 +# Comment this out to disable syslog support.
  60 +#
  61 +cmd_logger /usr/bin/logger
  62 +
  63 +# Uncomment this to specify the path to "du" for disk usage checks.
  64 +# If you have an older version of "du", you may also want to check the
  65 +# "du_args" parameter below.
  66 +#
  67 +cmd_du /usr/bin/du
  68 +
  69 +# Uncomment this to specify the path to rsnapshot-diff.
  70 +#
  71 +#cmd_rsnapshot_diff /usr/local/bin/rsnapshot-diff
  72 +
  73 +# Specify the path to a script (and any optional arguments) to run right
  74 +# before rsnapshot syncs files
  75 +#
  76 +#cmd_preexec /path/to/preexec/script
  77 +
  78 +# Specify the path to a script (and any optional arguments) to run right
  79 +# after rsnapshot syncs files
  80 +#
  81 +#cmd_postexec /path/to/postexec/script
  82 +
  83 +#########################################
  84 +# BACKUP INTERVALS #
  85 +# Must be unique and in ascending order #
  86 +# i.e. hourly, daily, weekly, etc. #
  87 +#########################################
  88 +
  89 +interval hourly 4
  90 +interval daily 6
  91 +interval weekly 2
  92 +interval monthly 2
  93 +
  94 +############################################
  95 +# GLOBAL OPTIONS #
  96 +# All are optional, with sensible defaults #
  97 +############################################
  98 +
  99 +# Verbose level, 1 through 5.
  100 +# 1 Quiet Print fatal errors only
  101 +# 2 Default Print errors and warnings only
  102 +# 3 Verbose Show equivalent shell commands being executed
  103 +# 4 Extra Verbose Show extra verbose information
  104 +# 5 Debug mode Everything
  105 +#
  106 +verbose 2
  107 +
  108 +# Same as "verbose" above, but controls the amount of data sent to the
  109 +# logfile, if one is being used. The default is 3.
  110 +#
  111 +loglevel 3
  112 +
  113 +# If you enable this, data will be written to the file you specify. The
  114 +# amount of data written is controlled by the "loglevel" parameter.
  115 +#
  116 +logfile /var/log/rsnapshot
  117 +
  118 +# If enabled, rsnapshot will write a lockfile to prevent two instances
  119 +# from running simultaneously (and messing up the snapshot_root).
  120 +# If you enable this, make sure the lockfile directory is not world
  121 +# writable. Otherwise anyone can prevent the program from running.
  122 +#
  123 +lockfile /var/run/rsnapshot.pid
  124 +
  125 +# Default rsync args. All rsync commands have at least these options set.
  126 +#
  127 +#rsync_short_args -a
  128 +#rsync_long_args --delete --numeric-ids --relative --delete-excluded
  129 +
  130 +# ssh has no args passed by default, but you can specify some here.
  131 +#
  132 +#ssh_args -p 22
  133 +
  134 +# Default arguments for the "du" program (for disk space reporting).
  135 +# The GNU version of "du" is preferred. See the man page for more details.
  136 +# If your version of "du" doesn't support the -h flag, try -k flag instead.
  137 +#
  138 +#du_args -csh
  139 +
  140 +# If this is enabled, rsync won't span filesystem partitions within a
  141 +# backup point. This essentially passes the -x option to rsync.
  142 +# The default is 0 (off).
  143 +#
  144 +#one_fs 0
  145 +
  146 +# The include and exclude parameters, if enabled, simply get passed directly
  147 +# to rsync. If you have multiple include/exclude patterns, put each one on a
  148 +# separate line. Please look up the --include and --exclude options in the
  149 +# rsync man page for more details on how to specify file name patterns.
  150 +#
  151 +#include ???
  152 +#include ???
  153 +#exclude ???
  154 +#exclude ???
  155 +
  156 +# The include_file and exclude_file parameters, if enabled, simply get
  157 +# passed directly to rsync. Please look up the --include-from and
  158 +# --exclude-from options in the rsync man page for more details.
  159 +#
  160 +#include_file /path/to/include/file
  161 +#exclude_file /path/to/exclude/file
  162 +
  163 +# If your version of rsync supports --link-dest, consider enable this.
  164 +# This is the best way to support special files (FIFOs, etc) cross-platform.
  165 +# The default is 0 (off).
  166 +#
  167 +#link_dest 0
  168 +
  169 +# When sync_first is enabled, it changes the default behaviour of rsnapshot.
  170 +# Normally, when rsnapshot is called with its lowest interval
  171 +# (i.e.: "rsnapshot hourly"), it will sync files AND rotate the lowest
  172 +# intervals. With sync_first enabled, "rsnapshot sync" handles the file sync,
  173 +# and all interval calls simply rotate files. See the man page for more
  174 +# details. The default is 0 (off).
  175 +#
  176 +#sync_first 0
  177 +
  178 +# If enabled, rsnapshot will move the oldest directory for each interval
  179 +# to [interval_name].delete, then it will remove the lockfile and delete
  180 +# that directory just before it exits. The default is 0 (off).
  181 +#
  182 +#use_lazy_deletes 0
  183 +
  184 +# Number of rsync re-tries. If you experience any network problems or
  185 +# network card issues that tend to cause ssh to crap-out with
  186 +# "Corrupted MAC on input" errors, for example, set this to a non-zero
  187 +# value to have the rsync operation re-tried
  188 +#
  189 +#rsync_numtries 0
  190 +
  191 +###############################
  192 +### BACKUP POINTS / SCRIPTS ###
  193 +###############################
  194 +
  195 +# LOCALHOST
  196 +#backup /home/ localhost/
  197 +#backup /etc/ localhost/
  198 +#backup /usr/local/ localhost/
  199 +#backup /var/log/rsnapshot localhost/
  200 +#backup /etc/passwd localhost/
  201 +#backup /home/foo/My Documents/ localhost/
  202 +#backup /foo/bar/ localhost/ one_fs=1, rsync_short_args=-urltvpog
  203 +#backup_script /usr/local/bin/backup_pgsql.sh localhost/postgres/
  204 +
  205 +# EXAMPLE.COM
  206 +#backup_script /bin/date "+ backup of example.com started at %c" unused1
  207 +#backup root@example.com:/home/ example.com/ +rsync_long_args=--bwlimit=16,exclude=core
  208 +#backup root@example.com:/etc/ example.com/ exclude=mtab,exclude=core
  209 +#backup_script ssh root@example.com "mysqldump -A > /var/db/dump/mysql.sql" unused2
  210 +#backup root@example.com:/var/db/dump/ example.com/
  211 +#backup_script /bin/date "+ backup of example.com ended at %c" unused9
  212 +#backup_script /bin/date "+ backup of example.com ended at %c" unused9
  213 +
  214 +backup_script /usr/local/bin/backup_spb.sh spb/
  215 +
  216 +# CVS.SOURCEFORGE.NET
  217 +#backup_script /usr/local/bin/backup_rsnapshot_cvsroot.sh rsnapshot.cvs.sourceforge.net/
  218 +
  219 +# RSYNC.SAMBA.ORG
  220 +#backup rsync://rsync.samba.org/rsyncftp/ rsync.samba.org/rsyncftp/
cookbooks/backup/recipes/default.rb 0 → 100644
  Diff comments   View file @ce25b7d
@@ -0,0 +1,34 @@ @@ -0,0 +1,34 @@
  1 +package 'cronie'
  2 +package 'rsnapshot'
  3 +
  4 +cookbook_file '/etc/rsnapshot.conf' do
  5 + owner 'root'
  6 + group 'root'
  7 + mode 0644
  8 +end
  9 +
  10 +#case node.name
  11 +#when "social"
  12 +# cookbook_file '/usr/local/bin/backup_noosfero.sh' do
  13 +# owner 'root'
  14 +# group 'root'
  15 +# mode 0755
  16 +# end
  17 +#end
  18 +
  19 +cookbook_file '/usr/local/bin/backup_spb.sh' do
  20 + owner 'root'
  21 + group 'root'
  22 + mode 0755
  23 +end
  24 +
  25 +cookbook_file '/etc/cron.d/rsnapshot-spb' do
  26 + owner 'root'
  27 + group 'root'
  28 + mode 0644
  29 +end
  30 +
  31 +service 'crond' do
  32 + action [:enable, :restart]
  33 +end
  34 +
roles/integration_server.rb
  Diff comments   View file @ce25b7d
@@ -11,4 +11,5 @@ run_list *[ @@ -11,4 +11,5 @@ run_list *[
11 'recipe[colab]', 11 'recipe[colab]',
12 'recipe[colab::nginx]', 12 'recipe[colab::nginx]',
13 'recipe[gitlab]', 13 'recipe[gitlab]',
  14 + 'recipe[backup]',
14 ] 15 ]
roles/social_server.rb
  Diff comments   View file @ce25b7d
@@ -2,5 +2,6 @@ name 'database_server' @@ -2,5 +2,6 @@ name 'database_server'
2 description 'Social networking server' 2 description 'Social networking server'
3 run_list *[ 3 run_list *[
4 'recipe[basics::nginx]', 4 'recipe[basics::nginx]',
5 - 'recipe[noosfero]' 5 + 'recipe[noosfero]',
  6 + 'recipe[backup]'
6 ] 7 ]
utils/migration/backup_integration.sh
View file @1dab81c
@@ -1,30 +0,0 @@ @@ -1,30 +0,0 @@
1 -#!/bin/bash  
2 -  
3 -echo 'I: starting integration backup...'  
4 -  
5 -echo 'I: Creating /tmp/backups/ on integration'  
6 -mkdir -p /tmp/backups/  
7 -  
8 -# Colab Backup  
9 -echo 'I: dumping colab data'  
10 -colab-admin dumpdata > /tmp/backups/colab_dump.json  
11 -  
12 -# GitLab Backup  
13 -echo 'cleaning gitlab backups directory'  
14 -sudo rm -rf /var/lib/gitlab/backups/*  
15 -echo 'I: creating gitlab backup'  
16 -cd /usr/lib/gitlab  
17 -sudo -u git bundle exec rake gitlab:backup:create RAILS_ENV=production > /dev/null  
18 -# TODO fix regular expression  
19 -mv /var/lib/gitlab/backups/*_gitlab_backup.tar /tmp/backups  
20 -  
21 -echo 'I: creating gitlab shell ssh backup'  
22 -tar -czf /tmp/backups/gitlab_shell_ssh.tar.gz /var/lib/gitlab-shell/.ssh/ 2> /dev/null  
23 -  
24 -# Mailman Backup  
25 -cd /var/lib/mailman  
26 -echo 'I: creating mailman backups'  
27 -tar -czf /tmp/backups/mailman_backup.tar.gz lists/ data/ archives/  
28 -  
29 -echo 'I: integration backup done.'  
30 -  
utils/migration/restore_integration.sh
  Diff comments   View file @ce25b7d
@@ -3,7 +3,8 @@ @@ -3,7 +3,8 @@
3 echo 'Starting restore on integration...' 3 echo 'Starting restore on integration...'
4 # Colab Restore 4 # Colab Restore
5 echo 'restoring colab...' 5 echo 'restoring colab...'
6 -colab-admin loaddata /tmp/backups/colab_dump.json > /dev/null 6 +psql -U colab -h database colab < /tmp/backups/colab.dump 1> /dev/null 2> /dev/null
  7 +colab-admin migrate > /dev/null
7 echo 'done.' 8 echo 'done.'
8 9
9 # Gitlab Restore 10 # Gitlab Restore
@@ -14,12 +15,16 @@ echo &#39;restoring gitlab...&#39; @@ -14,12 +15,16 @@ echo &#39;restoring gitlab...&#39;
14 mv /tmp/backups/*_gitlab_backup.tar /var/lib/gitlab/backups/ 15 mv /tmp/backups/*_gitlab_backup.tar /var/lib/gitlab/backups/
15 cd /usr/lib/gitlab 16 cd /usr/lib/gitlab
16 sudo -u git bundle exec rake gitlab:backup:restore RAILS_ENV=production force=yes 1> /dev/null 2>/dev/null 17 sudo -u git bundle exec rake gitlab:backup:restore RAILS_ENV=production force=yes 1> /dev/null 2>/dev/null
  18 +sudo rm -rf /var/lib/gitlab/backups/*
17 echo 'done.' 19 echo 'done.'
18 20
19 # Mailman Restore 21 # Mailman Restore
20 echo 'restoring mailman...' 22 echo 'restoring mailman...'
21 -mv /tmp/backups/mailman_backup.tar.gz /var/lib/mailman/ 23 +sudo mv /tmp/backups/mailman_backup.tar.gz /var/lib/mailman/
22 cd /var/lib/mailman 24 cd /var/lib/mailman
23 -tar -vxzf mailman_backup.tar.gz 1> /dev/null 2> /dev/null 25 +sudo tar -xzf mailman_backup.tar.gz
  26 +sudo rm mailman_backup.tar.gz
  27 +cd /usr/lib/mailman/bin
  28 +for list in `sudo ls /var/lib/mailman/lists`; do sudo ./withlist -l -r fix_url $list -u $SPB_URL 1> /dev/null 2> /dev/null; done
24 29
25 echo 'done.' 30 echo 'done.'