Commit f1979c278c5e22227b9daa49249395ea1299362e
1 parent
8089173b
Exists in
master
and in
39 other branches
Removed fabfile
Showing
1 changed file
with
0 additions
and
338 deletions
Show diff stats
fabfile.py
@@ -1,338 +0,0 @@ | @@ -1,338 +0,0 @@ | ||
1 | -# coding: utf-8 | ||
2 | - | ||
3 | -import os | ||
4 | - | ||
5 | -from fabric import colors | ||
6 | -from fabric.utils import error | ||
7 | -from fabric.decorators import task | ||
8 | -from fabric.api import env, run, sudo, local | ||
9 | -from fabric.contrib.files import exists | ||
10 | -from fabric.context_managers import prefix, cd, settings, shell_env | ||
11 | - | ||
12 | -DEBIAN_FAMILY = ['debian', 'ubuntu'] | ||
13 | -REDHAT_FAMILY = ['centos', 'fedora'] | ||
14 | - | ||
15 | -DISTRO_CMD = { | ||
16 | - 'debian': ('apt-get', { | ||
17 | - 'install': '-y', | ||
18 | - 'update': '-y', | ||
19 | - }), | ||
20 | - 'redhat': ('yum', { | ||
21 | - 'install': '-y', | ||
22 | - 'update': '-y', | ||
23 | - }) | ||
24 | -} | ||
25 | - | ||
26 | -APP_USER = APP_NAME = VENV_NAME = 'colab' | ||
27 | -REPO_URL = 'https://github.com/colab-community/colab.git' | ||
28 | -REPO_BRANCH = 'master' | ||
29 | - | ||
30 | - | ||
31 | -environments = { | ||
32 | - 'dev': { | ||
33 | - 'hosts': ['127.0.0.1'], | ||
34 | - 'key_filename': '~/.vagrant.d/insecure_private_key', | ||
35 | - 'port': 2222, | ||
36 | - 'is_vagrant': True, | ||
37 | - 'superuser': 'vagrant', | ||
38 | - }, | ||
39 | -} | ||
40 | -DEFAULT_ENVIRONMENT = 'dev' | ||
41 | - | ||
42 | -env.user = APP_USER | ||
43 | -env.use_shell = False | ||
44 | - | ||
45 | -PROJECT_PATH = os.path.join(os.path.dirname(__file__)) | ||
46 | -REPO_PATH = '/home/{}/{}'.format(APP_USER, APP_NAME) | ||
47 | -SOURCE_VENV = 'source /usr/local/bin/virtualenvwrapper.sh' | ||
48 | - | ||
49 | -WORKON_ENV = '{} && workon {}'.format(SOURCE_VENV, VENV_NAME) | ||
50 | -MANAGE_PATH = os.path.join(REPO_PATH, 'src') | ||
51 | -SETTINGS_PATH = os.path.join(MANAGE_PATH, APP_NAME) | ||
52 | - | ||
53 | - | ||
54 | -def get_distro_family(): | ||
55 | - cmd = 'python -c "import platform; print platform.dist()[0]"' | ||
56 | - linux_name = run(cmd).lower() | ||
57 | - if linux_name in DEBIAN_FAMILY: | ||
58 | - return 'debian' | ||
59 | - elif linux_name in REDHAT_FAMILY: | ||
60 | - return 'redhat' | ||
61 | - else: | ||
62 | - error(colors.red('Distribuiton `{}` not supported'.format(linux_name))) | ||
63 | - exit(1) | ||
64 | - | ||
65 | - | ||
66 | -def cmd(family, command, args=''): | ||
67 | - pkgmanager, commands = DISTRO_CMD[family] | ||
68 | - return ' '.join([pkgmanager, command, commands[command], args]) | ||
69 | - | ||
70 | - | ||
71 | -@task | ||
72 | -def environment(name=DEFAULT_ENVIRONMENT): | ||
73 | - """Set the environment where the tasks will be executed""" | ||
74 | - global REPO_URL | ||
75 | - global REPO_BRANCH | ||
76 | - | ||
77 | - try: | ||
78 | - import project_cfg | ||
79 | - except ImportError: | ||
80 | - pass | ||
81 | - else: | ||
82 | - REPO_URL = project_cfg.repository_url | ||
83 | - REPO_BRANCH = project_cfg.branch | ||
84 | - environments.update(project_cfg.environments) | ||
85 | - | ||
86 | - if name not in environments: | ||
87 | - error(colors.red('Environment `{}` does not exist.'.format(name))) | ||
88 | - | ||
89 | - env.update(environments[name]) | ||
90 | - env.environment = name | ||
91 | - | ||
92 | - | ||
93 | -def package_install(pkg): | ||
94 | - family = get_distro_family() | ||
95 | - sudo(cmd(family, 'install', pkg)) | ||
96 | - | ||
97 | - | ||
98 | -def install_requirements(): | ||
99 | - with cd(REPO_PATH), prefix(WORKON_ENV): | ||
100 | - run('pip install -U distribute') | ||
101 | - if not env.is_vagrant: | ||
102 | - run('pip install -r requirements.txt') | ||
103 | - return | ||
104 | - | ||
105 | - if exists('requirements-{}.txt'.format(env.environment)): | ||
106 | - run('pip install -r requirements-{}.txt'.format(env.environment)) | ||
107 | - else: | ||
108 | - run('pip install -r requirements.txt') | ||
109 | - | ||
110 | - | ||
111 | -def mkvirtualenv(): | ||
112 | - if not exists('~/.virtualenvs/' + VENV_NAME): | ||
113 | - | ||
114 | - with prefix(SOURCE_VENV): | ||
115 | - run('mkvirtualenv ' + VENV_NAME) | ||
116 | - return True | ||
117 | - | ||
118 | - | ||
119 | -def manage(command): | ||
120 | - django_settings = env.get('django_settings') | ||
121 | - env_vars = {} | ||
122 | - if django_settings: | ||
123 | - env_vars.update({'DJANGO_SETTINGS_MODULE': django_settings}) | ||
124 | - | ||
125 | - with shell_env(**env_vars): | ||
126 | - with cd(MANAGE_PATH), prefix(WORKON_ENV): | ||
127 | - run('python manage.py {}'.format(command)) | ||
128 | - | ||
129 | - | ||
130 | -def migrate(): | ||
131 | - manage('migrate') | ||
132 | - manage('loaddata super_archives/fixture/initial_data.json') | ||
133 | - | ||
134 | - | ||
135 | -def collectstatic(): | ||
136 | - sudo('mkdir -p /usr/share/nginx/{}'.format(APP_NAME)) | ||
137 | - sudo('chown {} /usr/share/nginx/{}'.format(env.user, APP_NAME)) | ||
138 | - manage('collectstatic --noinput') | ||
139 | - | ||
140 | - | ||
141 | -def create_local_settings(): | ||
142 | - with cd(SETTINGS_PATH): | ||
143 | - env_local_settings = 'local_settings-{}.py'.format(env.environment) | ||
144 | - | ||
145 | - if not exists('local_settings.py') and exists(env_local_settings): | ||
146 | - run('ln -s {} {}'.format(env_local_settings, 'local_settings.py')) | ||
147 | - run('chown {} local_settings.py'.format(env.user)) | ||
148 | - | ||
149 | - | ||
150 | -def update_code(): | ||
151 | - if env.is_vagrant: | ||
152 | - if not exists(REPO_PATH): | ||
153 | - run('ln -s /vagrant/ {}'.format(REPO_PATH)) | ||
154 | - return | ||
155 | - | ||
156 | - if not exists(REPO_PATH): | ||
157 | - run('git clone {} {} -b {}'.format(REPO_URL, REPO_PATH, REPO_BRANCH)) | ||
158 | - else: | ||
159 | - with cd(REPO_PATH): | ||
160 | - run('git checkout {}'.format(REPO_BRANCH)) | ||
161 | - run('git pull') | ||
162 | - | ||
163 | - | ||
164 | -@task | ||
165 | -def bootstrap(): | ||
166 | - """Bootstrap machine to run fabric tasks""" | ||
167 | - | ||
168 | - with settings(user=env.superuser): | ||
169 | - family = get_distro_family() | ||
170 | - if(family == 'debian'): | ||
171 | - sudo(cmd(family, 'update')) | ||
172 | - else: | ||
173 | - package_install('wget') | ||
174 | - | ||
175 | - if not exists('/usr/bin/git'): | ||
176 | - package_install('git-core') | ||
177 | - | ||
178 | - if env.is_vagrant: | ||
179 | - groups = ['sudo', 'vagrant'] | ||
180 | - local('chmod -fR g+w {}'.format(PROJECT_PATH)) | ||
181 | - else: | ||
182 | - groups = ['sudo'] | ||
183 | - | ||
184 | - for group in groups: | ||
185 | - sudo('groupadd -f {}'.format(group)) | ||
186 | - | ||
187 | - command = 'useradd {} -G {} -m -s /bin/bash' | ||
188 | - sudo(command.format(APP_USER, ','.join(groups))) | ||
189 | - | ||
190 | - ssh_dir = '/home/{0}/.ssh/'.format(APP_USER) | ||
191 | - if not exists(ssh_dir): | ||
192 | - sudo('mkdir -p {0}'.format(ssh_dir)) | ||
193 | - sudo('chmod 700 {0}'.format(ssh_dir)) | ||
194 | - sudo('cp ~{}/.ssh/authorized_keys /home/{}/.ssh/'.format( | ||
195 | - env.superuser, | ||
196 | - APP_USER | ||
197 | - )) | ||
198 | - sudo('chown -fR {0}:{0} {1}'.format(APP_USER, ssh_dir)) | ||
199 | - | ||
200 | - sudoers_file = os.path.join('/etc/sudoers.d/', APP_USER) | ||
201 | - tmp_file = os.path.join('/tmp', APP_USER) | ||
202 | - if not exists(sudoers_file): | ||
203 | - sudo('echo "{} ALL=NOPASSWD: ALL" > {}'.format(APP_USER, tmp_file)) | ||
204 | - sudo('chown root:root {}'.format(tmp_file)) | ||
205 | - sudo('chmod 440 {}'.format(tmp_file)) | ||
206 | - sudo('mv {} {}'.format(tmp_file, sudoers_file)) | ||
207 | - | ||
208 | - | ||
209 | -@task | ||
210 | -def provision(): | ||
211 | - """Run puppet""" | ||
212 | - | ||
213 | - update_code() | ||
214 | - | ||
215 | - puppet_path = os.path.join(REPO_PATH, 'puppet/') | ||
216 | - modules_path = os.path.join(puppet_path, 'modules') | ||
217 | - puppet_modules = '{}:/etc/puppet/modules'.format(modules_path) | ||
218 | - | ||
219 | - with cd(puppet_path): | ||
220 | - run('sudo python bootstrap.py') | ||
221 | - | ||
222 | - if env.is_vagrant: | ||
223 | - cmd = os.path.join(puppet_path, 'manifests', 'site.pp') | ||
224 | - else: | ||
225 | - cmd = '-e "include {}"'.format(APP_NAME) | ||
226 | - | ||
227 | - if not exists('/usr/bin/puppet'): | ||
228 | - print(colors.red('Please install `puppet` before continue.')) | ||
229 | - return | ||
230 | - | ||
231 | - sudo('puppet apply --modulepath={} {}'.format(puppet_modules, cmd)) | ||
232 | - | ||
233 | - | ||
234 | -@task | ||
235 | -def ssh_keygen(): | ||
236 | - """Create SSH credentials""" | ||
237 | - | ||
238 | - if not exists('~/.ssh/id_rsa'): | ||
239 | - run("ssh-keygen -f ~/.ssh/id_rsa -N '' -b 1024 -q") | ||
240 | - key = run('cat ~/.ssh/id_rsa.pub') | ||
241 | - | ||
242 | - print('Public key:') | ||
243 | - print(colors.yellow(key)) | ||
244 | - print('') | ||
245 | - print('Add the key above to your github repository deploy keys') | ||
246 | - | ||
247 | - | ||
248 | -@task | ||
249 | -def deploy(noprovision=False): | ||
250 | - """Deploy and run the new code (master branch)""" | ||
251 | - | ||
252 | - if noprovision is False: | ||
253 | - provision() | ||
254 | - else: | ||
255 | - update_code() | ||
256 | - | ||
257 | - fix_path() | ||
258 | - | ||
259 | - install_solr() | ||
260 | - | ||
261 | - mkvirtualenv() | ||
262 | - | ||
263 | - sudo('supervisorctl stop all') | ||
264 | - | ||
265 | - install_requirements() | ||
266 | - create_local_settings() | ||
267 | - collectstatic() | ||
268 | - migrate() | ||
269 | - | ||
270 | - build_schema() | ||
271 | - | ||
272 | - sudo('supervisorctl start all') | ||
273 | - | ||
274 | - | ||
275 | -def fix_path(): | ||
276 | - global SOURCE_VENV | ||
277 | - global WORKON_ENV | ||
278 | - if exists('/usr/bin/virtualenvwrapper.sh'): | ||
279 | - SOURCE_VENV = 'source /usr/bin/virtualenvwrapper.sh' | ||
280 | - WORKON_ENV = '{} && workon {}'.format(SOURCE_VENV, VENV_NAME) | ||
281 | - | ||
282 | - | ||
283 | -@task | ||
284 | -def install_solr(): | ||
285 | - """Install Solr""" | ||
286 | - | ||
287 | - SOLR_PKG = 'https://archive.apache.org/dist/lucene/solr/4.6.1/solr-4.6.1.tgz' | ||
288 | - | ||
289 | - if not exists('~/solr-4.6.1'): | ||
290 | - run('wget {} -O /tmp/solr-4.6.1.tgz'.format(SOLR_PKG)) | ||
291 | - run('tar xzf /tmp/solr-4.6.1.tgz -C /tmp/') | ||
292 | - run('cp -rf /tmp/solr-4.6.1 ~/solr-4.6.1') | ||
293 | - run('mv ~/solr-4.6.1/example ~/solr-4.6.1/colab') | ||
294 | - run('chmod +x ~/solr-4.6.1/colab/start.jar') | ||
295 | - run('rm /tmp/solr-4.6.1.tgz') | ||
296 | - | ||
297 | - with cd('~/solr-4.6.1/colab/solr/collection1/conf/'): | ||
298 | - if not exists('stopwords_en.txt'): | ||
299 | - run('cp stopwords.txt stopwords_en.txt') | ||
300 | - | ||
301 | - | ||
302 | -@task | ||
303 | -def solr(port=8983): | ||
304 | - """Start Solr""" | ||
305 | - with cd('~/solr-4.6.1/colab'), settings(user='colab'): | ||
306 | - run('java -jar start.jar -Djetty.port={}'.format(port)) | ||
307 | - | ||
308 | - | ||
309 | -@task | ||
310 | -def rebuild_index(age=None, batch=None): | ||
311 | - """Rebuild the solr index""" | ||
312 | - age_arg = '' | ||
313 | - if age: | ||
314 | - age_arg = '--age={}'.format(age) | ||
315 | - | ||
316 | - batch_arg = '' | ||
317 | - if batch: | ||
318 | - batch_arg = '--batch-size={}'.format(batch) | ||
319 | - | ||
320 | - manage('rebuild_index {} {}'.format(age_arg, batch_arg)) | ||
321 | - | ||
322 | - | ||
323 | -@task | ||
324 | -def update_index(): | ||
325 | - """Update solr index""" | ||
326 | - manage('update_index') | ||
327 | - | ||
328 | - | ||
329 | -@task | ||
330 | -def build_schema(): | ||
331 | - """Build solr schema""" | ||
332 | - solr_schema_file = '~/solr-4.6.1/colab/solr/collection1/conf/schema.xml' | ||
333 | - manage('build_solr_schema -f {}'.format(solr_schema_file)) | ||
334 | - run(r'sed -i "s/<fields>/<fields>\n<field name=\"_version_\" type=\"long\" indexed=\"true\" stored =\"true\"\/>/" {}'.format(solr_schema_file)) | ||
335 | - | ||
336 | - | ||
337 | -# Main | ||
338 | -environment() |