Commit
16da24cfb318688463f0113e50ff42228be62881
Authored by
Gust
2014-12-11 14:57:44 -0200
Exists in
master
and in
39 other branches
1.12.x , add_requests_as_dep , celery_service , change-passwd-signal , check-port , colab-signals , colab-vcard , colab_search , colab_tag , colab_tag_merge , community_association , detach_super_archives , fix-dashboard , fix-message-preview , fix_app_label , fix_cache_count_spb , fix_header , fixed_gitlab_import , go_to_profile_panel , header_footer , layout-fix , mobile_user_scalable , move_out_plugins , paginate-threads , profile_integration , refactor-data-import , remove-gitlab-plugin , removing_namespace , search_block , search_filters , spb-release/3.0 , split_screen , stable , startplugin , timestamp_plugins , translation_review , user_regex , validate-passwd , workin_whoosh_temp
Add interface for data importation for proxies
-Proxies utilizes the data_api.py fetchData() to expecify how to import
their data into the colab database
-Gitlab need to add the variable auth_token to colab.yml to have the
permission of acces to the gitlabAPI data
Signed-off-by: Gustavo Jaruga <darksshades@gmail.com>
Signed-off-by: Charles Oliveira <18oliveira.charles@gmail.com>
Signed-off-by: Carlos Oliveira <carlospecter@gmail.com>
Showing
13 changed files
with
167 additions
and
1 deletions
Show diff stats
@@ -70,6 +70,7 @@ ROBOTS_NOINDEX: false
@@ -70,6 +70,7 @@ ROBOTS_NOINDEX: false
70
# PROXIED_APPS:
70
# PROXIED_APPS:
71
# gitlab:
71
# gitlab:
72
# upstream: 'http://localhost:8090/gitlab/'
72
# upstream: 'http://localhost:8090/gitlab/'
73
+# auth_token: ''
73
# trac:
74
# trac:
74
# upstream: 'http://localhost:5000/trac/'
75
# upstream: 'http://localhost:5000/trac/'
75
76
@@ -0,0 +1,51 @@
@@ -0,0 +1,51 @@
1
+from colab.proxy.gitlab.models import *
2
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
3
+from django.db.models.fields import DateTimeField
4
+from dateutil.parser import parse
5
+import urllib2
6
+import json
7
+from django.conf import settings
8
+
9
+class GitlabDataAPI(ProxyDataAPI):
10
+
11
+
12
+ def fetchProjects(self):
13
+ page = 1
14
+ projects = []
15
+
16
+ proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
17
+ admin_token = proxy_config.get('auth_token')
18
+
19
+ # Iterates throughout all projects pages
20
+ while(True):
21
+ data = urllib2.urlopen(proxy_config.get('upstream')+'api/v3/projects/all?private_token={}&per_page=100&page={}'.format(admin_token, page))
22
+ json_data = json.load(data)
23
+
24
+ if len(json_data) == 0:
25
+ break
26
+
27
+ page = page + 1
28
+
29
+ for element in json_data:
30
+ project = GitlabProject()
31
+
32
+ for field in GitlabProject._meta.fields:
33
+ value = element[field.name]
34
+ value = parse(element[field.name]) if isinstance(field, DateTimeField) else value
35
+ setattr(project, field.name, value)
36
+
37
+ projects.append(project)
38
+
39
+ return projects
40
+
41
+
42
+ def fetchData(self):
43
+ data = self.fetchProjects()
44
+
45
+ for datum in data:
46
+ datum.save()
47
+
48
+ @property
49
+ def app_label(self):
50
+ return 'gitlab'
51
+
@@ -0,0 +1,28 @@
@@ -0,0 +1,28 @@
1
+# -*- coding: utf-8 -*-
2
+from __future__ import unicode_literals
3
+
4
+from django.db import models, migrations
5
+
6
+
7
+class Migration(migrations.Migration):
8
+
9
+ dependencies = [
10
+ ]
11
+
12
+ operations = [
13
+ migrations.CreateModel(
14
+ name='GitlabProject',
15
+ fields=[
16
+ ('id', models.IntegerField(serialize=False, primary_key=True)),
17
+ ('description', models.TextField()),
18
+ ('public', models.BooleanField(default=True)),
19
+ ('name', models.TextField()),
20
+ ('name_with_namespace', models.TextField()),
21
+ ('created_at', models.DateTimeField(blank=True)),
22
+ ('last_activity_at', models.DateTimeField(blank=True)),
23
+ ],
24
+ options={
25
+ },
26
+ bases=(models.Model,),
27
+ ),
28
+ ]
1
from django.db import models
1
from django.db import models
2
+from django.conf import settings
3
+from colab.accounts.models import User
2
4
3
-# Create your models here.
5
+class GitlabProject(models.Model):
6
+
7
+ id = models.IntegerField(primary_key=True)
8
+ description = models.TextField()
9
+ public = models.BooleanField(default=True)
10
+ name = models.TextField()
11
+ name_with_namespace = models.TextField()
12
+ created_at = models.DateTimeField(blank=True)
13
+ last_activity_at = models.DateTimeField(blank=True)
@@ -0,0 +1,12 @@
@@ -0,0 +1,12 @@
1
+from colab.proxy.gitlab.models import *
2
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
3
+from django.db.models.fields import DateTimeField
4
+from dateutil.parser import parse
5
+import urllib2
6
+import json
7
+
8
+class JenkinsDataAPI(ProxyDataAPI):
9
+
10
+ def fetchData(self):
11
+ pass
12
+
@@ -0,0 +1,12 @@
@@ -0,0 +1,12 @@
1
+from colab.proxy.gitlab.models import *
2
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
3
+from django.db.models.fields import DateTimeField
4
+from dateutil.parser import parse
5
+import urllib2
6
+import json
7
+
8
+class NoosferoDataAPI(ProxyDataAPI):
9
+
10
+ def fetchData(self):
11
+ pass
12
+
@@ -0,0 +1 @@
@@ -0,0 +1 @@
1
+__init__.py
0
\ No newline at end of file
2
\ No newline at end of file
@@ -0,0 +1,21 @@
@@ -0,0 +1,21 @@
1
+#!/usr/bin/env python
2
+
3
+import colab
4
+from django.core.management.base import BaseCommand
5
+from colab.super_archives.models import Message
6
+from django.conf import settings
7
+modules = [ i for i in settings.INSTALLED_APPS if i.startswith("colab.proxy.") ]
8
+for module in modules:
9
+ module += ".data_api"
10
+ __import__(module, locals(), globals())
11
+
12
+class Command(BaseCommand):
13
+ help = "Import proxy data into colab database"
14
+
15
+ def handle(self, *args, **kwargs):
16
+ print "Executing extraction command..."
17
+
18
+ for module in modules:
19
+ extractionClassname = module + ".data_api." + module.split('.')[-1].title() + "DataAPI"
20
+ api = eval(extractionClassname)()
21
+ api.fetchData()
@@ -0,0 +1,6 @@
@@ -0,0 +1,6 @@
1
+
2
+class ProxyDataAPI():
3
+
4
+
5
+ def fetchData(self):
6
+ raise NotImplementedError('fetchData not yet implemented')
0
\ No newline at end of file
7
\ No newline at end of file
@@ -0,0 +1,12 @@
@@ -0,0 +1,12 @@
1
+from colab.proxy.gitlab.models import *
2
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
3
+from django.db.models.fields import DateTimeField
4
+from dateutil.parser import parse
5
+import urllib2
6
+import json
7
+
8
+class RedmineDataAPI(ProxyDataAPI):
9
+
10
+ def fetchData(self):
11
+ pass
12
+
@@ -0,0 +1,12 @@
@@ -0,0 +1,12 @@
1
+from colab.proxy.gitlab.models import *
2
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
3
+from django.db.models.fields import DateTimeField
4
+from dateutil.parser import parse
5
+import urllib2
6
+import json
7
+
8
+class TracDataAPI(ProxyDataAPI):
9
+
10
+ def fetchData(self):
11
+ pass
12
+
