Download as
Browse Code »

Commit 1c8339abbf9ed02041baa81fdda64d058e30240a

Authored by Sergio Oliveira
2 parents 733d4cda 7c6bb027
Exists in master and in 39 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, check-port, colab-signals, colab-vcard, colab_search, colab_tag, colab_tag_merge, community_association, detach_super_archives, fix-dashboard, fix-message-preview, fix_app_label, fix_cache_count_spb, fix_header, fixed_gitlab_import, go_to_profile_panel, header_footer, layout-fix, mobile_user_scalable, move_out_plugins, paginate-threads, profile_integration, refactor-data-import, remove-gitlab-plugin, removing_namespace, search_block, search_filters, spb-release/3.0, split_screen, stable, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd, workin_whoosh_temp

Merge branch 'update_puppet' of github.com:colab-community/colab into update_puppet

Showing 2 changed files with 136 additions and 98 deletions   Show diff stats
fabfile.py
  Diff comments   View file @1c8339a
... ... @@ -9,23 +9,24 @@ from fabric.api import env, run, sudo, local
9 9 from fabric.contrib.files import exists
10 10 from fabric.context_managers import prefix, cd, settings, shell_env
11 11  
12   -LINUX_DISTRO = ''
13 12 DEBIAN_FAMILY = ['debian', 'ubuntu']
14 13 REDHAT_FAMILY = ['centos', 'fedora']
15 14  
16   -DISTRO_CMD = {'debian': ('apt-get', {
17   - 'install': '-y',
18   - 'update': '-y',
19   - }),
20   - 'redhat': ('yum', {
21   - 'install': '-y',
22   - 'update': '-y',
23   - })
24   - }
  15 +DISTRO_CMD = {
  16 + 'debian': ('apt-get', {
  17 + 'install': '-y',
  18 + 'update': '-y',
  19 + }),
  20 + 'redhat': ('yum', {
  21 + 'install': '-y',
  22 + 'update': '-y',
  23 + })
  24 +}
25 25  
26 26 APP_USER = APP_NAME = VENV_NAME = 'colab'
27 27 REPO_URL = 'git@github.com:colab-community/colab.git'
28 28  
  29 +
29 30 environments = {
30 31 'dev': {
31 32 'hosts': ['127.0.0.1'],
... ... @@ -47,19 +48,23 @@ WORKON_ENV = '{} && workon {}'.format(SOURCE_VENV, VENV_NAME)
47 48 MANAGE_PATH = os.path.join(REPO_PATH, 'src')
48 49 SETTINGS_PATH = os.path.join(MANAGE_PATH, APP_NAME)
49 50  
  51 +
50 52 def get_distro_family():
51   - linux_name = run('python -c "import platform; print platform.dist()[0]"').lower()
  53 + cmd = 'python -c "import platform; print platform.dist()[0]"'
  54 + linux_name = run(cmd).lower()
52 55 if linux_name in DEBIAN_FAMILY:
53   - return 'debian'
  56 + return 'debian'
54 57 elif linux_name in REDHAT_FAMILY:
55   - return 'redhat'
56   - else :
57   - error(colors.red('Distribuiton `{}` not supported.'.format(linux_name)))
58   - exit(1)
  58 + return 'redhat'
  59 + else:
  60 + error(colors.red('Distribuiton `{}` not supported'.format(linux_name)))
  61 + exit(1)
  62 +
59 63  
60   -def cmd(family, command, args = ''):
  64 +def cmd(family, command, args=''):
61 65 pkgmanager, commands = DISTRO_CMD[family]
62   - return '{} {} {} {}'.format(pkgmanager, command, commands[command], args)
  66 + return ' '.join([pkgmanager, command, commands[command], args])
  67 +
63 68  
64 69 @task
65 70 def environment(name=DEFAULT_ENVIRONMENT):
... ... @@ -80,9 +85,11 @@ def environment(name=DEFAULT_ENVIRONMENT):
80 85 env.update(environments[name])
81 86 env.environment = name
82 87  
83   -def package_install(pkg):
  88 +
  89 +def package_install(pkg):
84 90 family = get_distro_family()
85   - sudo(cmd(family, 'install', pkg ))
  91 + sudo(cmd(family, 'install', pkg))
  92 +
86 93  
87 94 def install_requirements():
88 95 with cd(REPO_PATH), prefix(WORKON_ENV):
... ... @@ -96,12 +103,14 @@ def install_requirements():
96 103 else:
97 104 run('pip install -r requirements.txt')
98 105  
  106 +
99 107 def mkvirtualenv():
100 108 if not exists('~/.virtualenvs/' + VENV_NAME):
101 109 with prefix(SOURCE_VENV):
102 110 run('mkvirtualenv ' + VENV_NAME)
103 111 return True
104 112  
  113 +
105 114 def manage(command):
106 115 django_settings = env.get('django_settings')
107 116 env_vars = {}
... ... @@ -112,17 +121,21 @@ def manage(command):
112 121 with cd(MANAGE_PATH), prefix(WORKON_ENV):
113 122 run('python manage.py {}'.format(command))
114 123  
  124 +
115 125 def syncdb():
116 126 manage('syncdb')
117 127  
  128 +
118 129 def migrate():
119 130 manage('migrate')
120 131  
  132 +
121 133 def collectstatic():
122 134 sudo('mkdir -p /usr/share/nginx/{}'.format(APP_NAME))
123 135 sudo('chown {} /usr/share/nginx/{}'.format(env.user, APP_NAME))
124 136 manage('collectstatic --noinput')
125 137  
  138 +
126 139 def create_local_settings():
127 140 with cd(SETTINGS_PATH), settings(user=env.superuser):
128 141 env_local_settings = 'local_settings-{}.py'.format(env.environment)
... ... @@ -131,6 +144,7 @@ def create_local_settings():
131 144 run('ln -s {} {}'.format(env_local_settings, 'local_settings.py'))
132 145 run('chown {} local_settings.py'.format(env.user))
133 146  
  147 +
134 148 def update_code():
135 149 if env.is_vagrant:
136 150 if not exists(REPO_PATH):
... ... @@ -143,26 +157,29 @@ def update_code():
143 157 with cd(REPO_PATH):
144 158 run('git pull')
145 159  
  160 +
146 161 @task
147 162 def bootstrap():
148 163 """Bootstrap machine to run fabric tasks"""
  164 +
149 165 with settings(user=env.superuser):
150   - family = get_distro_family()
151   - sudo(cmd(family, 'update'))
152   -
153   - if not exists('/usr/bin/git'):
  166 + family = get_distro_family()
  167 + sudo(cmd(family, 'update'))
  168 +
  169 + if not exists('/usr/bin/git'):
154 170 package_install('git-core')
155 171  
156 172 if env.is_vagrant:
157   - groups = ['sudo' ,'vagrant']
  173 + groups = ['sudo', 'vagrant']
158 174 local('chmod -fR g+w {}'.format(PROJECT_PATH))
159 175 else:
160 176 groups = ['sudo']
161 177  
162   - for group in groups:
163   - sudo('groupadd -f {}'.format(group))
  178 + for group in groups:
  179 + sudo('groupadd -f {}'.format(group))
164 180  
165   - sudo('useradd {} -G {} -m -s /bin/bash'.format(APP_USER, ','.join(groups)))
  181 + command = 'useradd {} -G {} -m -s /bin/bash'
  182 + sudo(command.format(APP_USER, ','.join(groups)))
166 183  
167 184 ssh_dir = '/home/{0}/.ssh/'.format(APP_USER)
168 185 if not exists(ssh_dir):
... ... @@ -182,11 +199,13 @@ def bootstrap():
182 199 sudo('chmod 440 {}'.format(tmp_file))
183 200 sudo('mv {} {}'.format(tmp_file, sudoers_file))
184 201  
  202 +
185 203 @task
186 204 def provision():
187 205 """Run puppet"""
188 206  
189 207 update_code()
  208 +
190 209 puppet_path = os.path.join(REPO_PATH, 'puppet/')
191 210 modules_path = os.path.join(puppet_path, 'modules')
192 211 puppet_modules = '{}:/etc/puppet/modules'.format(modules_path)
... ... @@ -205,6 +224,7 @@ def provision():
205 224  
206 225 sudo('puppet apply --modulepath={} {}'.format(puppet_modules, cmd))
207 226  
  227 +
208 228 @task
209 229 def ssh_keygen():
210 230 """Create SSH credentials"""
... ... @@ -218,6 +238,7 @@ def ssh_keygen():
218 238 print('')
219 239 print('Add the key above to your github repository deploy keys')
220 240  
  241 +
221 242 @task
222 243 def deploy(noprovision=False):
223 244 """Deploy and run the new code (master branch)"""
... ... @@ -239,5 +260,6 @@ def deploy(noprovision=False):
239 260  
240 261 sudo('supervisorctl start all')
241 262  
  263 +
242 264 # Main
243 265 environment()
... ...
puppet/bootstrap.py
  Diff comments   View file @1c8339a
1   -#!/usr/bin/env python2.7
  1 +#!/usr/bin/env python
2 2  
3 3 import os
4   -import apt
5   -import apt_pkg
  4 +import re
6 5 import locale
7 6 import platform
8 7 import subprocess
9 8 import urllib
10 9  
11   -from apt import debfile
12 10 from distutils.version import StrictVersion
13 11 from pkg_resources import parse_requirements
14 12 from shutil import copyfile
15   -from subprocess import check_output
16 13  
17   -
18   -PUPPET_TARGET_VERSION="3.4.3-1"
  14 +PUPPET_TARGET_VERSION = "3.6.2"
19 15 PUPPET_DIR = os.path.join(os.path.dirname(__file__))
20 16 MODULES_FILE_PATH = os.path.join(PUPPET_DIR, 'modules.txt')
21 17  
22 18  
23   -def get_package(name):
24   - cache = apt.cache.Cache()
25   - if name in cache:
26   - return cache[name], cache
27   -
28   - return None, None
29   -
30   -
31   -def pkg_available(name):
32   - pkg = get_package(name)[0]
33   - if pkg and pkg.versions.get(PUPPET_TARGET_VERSION):
34   - return True
35   -
36   - return False
37   -
38   -
39   -def config_puppetlabs_repo():
40   - dist = platform.dist()[-1]
41   -
42   - url = 'http://apt.puppetlabs.com/puppetlabs-release-{}.deb'.format(dist)
43   - filename = '/tmp/puppet_apt.deb'
44   - try:
45   - urllib.urlretrieve(url, filename)
46   - except IOError:
47   - print "Could not install puppet"
48   - raise
49   -
50   - deb_pkg = debfile.DebPackage(filename)
51   - if deb_pkg.check():
52   - deb_pkg.install()
53   - cache = apt.cache.Cache()
54   - cache.update()
55   - cache.open()
56   -
57   -
58   -def install_puppet(upgrade=False):
59   - pkg, cache = get_package('puppet')
60   -
61   - pkg.candidate = pkg.versions.get(PUPPET_TARGET_VERSION)
62   - if upgrade:
63   - pkg.mark_upgrade()
  19 +def get_release_name():
  20 + distro = platform.dist()[0].lower()
  21 + if distro == 'centos':
  22 + with open('/etc/centos-release') as release_file:
  23 + regex = r'CentOS release (\d)'
  24 + release = re.search(regex, release_file.read()).group(1)
  25 + return 'centos', release
  26 + elif distro == 'ubuntu':
  27 + with open('/etc/lsb-release') as release_file:
  28 + regex = r'DISTRIB_CODENAME=([a-z]+)'
  29 + release = re.search(regex, release_file.read()).group(1)
  30 + return 'ubuntu', release
64 31 else:
65   - pkg.mark_install()
66   -
67   - cache.commit()
  32 + return '', ''
68 33  
69 34  
70 35 def get_modules_installed():
71   - modules_list = check_output(['puppet', 'module', 'list']).split('\n')
  36 + modules_list = os.popen('puppet module list').read().split('\n')
72 37 modules_list = [line for line in modules_list if 'no modules' not in line]
73 38 modules_list = [item.split()[1:] for item in modules_list]
74 39 modules_list = [item for item in modules_list if item]
... ... @@ -83,9 +48,9 @@ def get_modules_installed():
83 48  
84 49 def run(cmd, module, version=''):
85 50 if version:
86   - version = ' --version {}'.format(version)
  51 + version = ' --version %s' % (version)
87 52  
88   - cmd = 'puppet module {} {}{}'.format(cmd, module, version)
  53 + cmd = 'puppet module %s %s%s' % (cmd, module, version)
89 54 process = subprocess.Popen(cmd, shell=True, executable='/bin/bash')
90 55 process.wait()
91 56  
... ... @@ -97,16 +62,19 @@ def install_puppet_modules():
97 62 modules_requirements = modules_file.read().replace('/', '-')
98 63  
99 64 for module in parse_requirements(modules_requirements):
100   -
101 65 current_cmd, compare, version, version_comparison = '', '', '', None
102 66 if module.project_name in modules_installed:
103   -
104 67 if module.specs:
105 68 compare, version = module.specs[0]
106   - version_comparison = apt_pkg.version_compare(
107   - modules_installed[module.project_name],
108   - version
109   - )
  69 +
  70 + tmp_version = modules_installed[module.project_name]
  71 + installed_version = StrictVersion(tmp_version)
  72 + required_version = StrictVersion(version)
  73 +
  74 + if installed_version >= required_version:
  75 + version_comparison = 0
  76 + else:
  77 + version_comparison = -1
110 78 else:
111 79 continue
112 80  
... ... @@ -125,17 +93,65 @@ def install_puppet_modules():
125 93 if not version_comparison or version_comparison < 0:
126 94 run(current_cmd, module.project_name)
127 95  
  96 +
  97 +def iscentos(distro):
  98 + return distro == 'centos'
  99 +
  100 +
  101 +def isubuntu(distro):
  102 + return distro == 'ubuntu'
  103 +
  104 +
  105 +def download(url, filename):
  106 + try:
  107 + urllib.urlretrieve(url, filename)
  108 + except IOError:
  109 + print "Could not install puppet"
  110 + raise
  111 +
  112 +
128 113 def main():
129   - # If the package not found or if the version is outdated, install puppet
130   - if not pkg_available('puppet'):
131   - config_puppetlabs_repo()
132   -
133   - pkg = get_package('puppet')[0]
134   - if not pkg.is_installed:
135   - install_puppet()
136   - elif apt_pkg.version_compare(pkg.installed.version,
137   - PUPPET_TARGET_VERSION) < 0:
138   - install_puppet(upgrade=True)
  114 + distro, release = get_release_name()
  115 + print('Distro %s, release %s' % (distro, release))
  116 +
  117 + if iscentos(distro):
  118 + cmd = 'rpm'
  119 + flags = '-ivh'
  120 + url = 'http://yum.puppetlabs.com/'
  121 + pkg = 'puppetlabs-release-el-%s.noarch.rpm' % (release)
  122 + update = ['yum', 'update', '-y']
  123 + install = ['yum', 'install', 'puppet', '-y']
  124 + elif isubuntu(distro):
  125 + cmd = 'dpkg'
  126 + flags = '-i'
  127 + url = 'https://apt.puppetlabs.com/'
  128 + pkg = 'puppetlabs-release-%s.deb' % (release)
  129 + update = ['apt-get', 'update', '-y']
  130 + install = ['apt-get', 'install', 'puppet', '-y']
  131 +
  132 + # Needed dependency for import pkg_resources
  133 + pkg_res = ['apt-get', 'install', 'python-pkg-resources', '-y']
  134 + subprocess.call(pkg_res)
  135 + else:
  136 + print('This distribuition is currently not supported!')
  137 + print('exiting...')
  138 + exit(1)
  139 +
  140 + tmp_file = '/tmp/%s' % (pkg)
  141 + download(url + pkg, tmp_file)
  142 + args = [cmd, flags, tmp_file]
  143 +
  144 + # Add repository
  145 + result = subprocess.call(args)
  146 + if result != 0:
  147 + print('Repository %s already set' % pkg)
  148 +
  149 + # Install Puppet
  150 + subprocess.call(update)
  151 + result = subprocess.call(install)
  152 + if result != 0:
  153 + print('Failed installing puppet')
  154 + exit(result)
139 155  
140 156 if os.path.isfile('/vagrant/puppet/hiera.yaml'):
141 157 copyfile('/vagrant/puppet/hiera.yaml', '/etc/puppet/hiera.yaml')
... ...