Add interface for data importation for proxies

-Proxies utilizes the data_api.py fetchData() to expecify how to import their data into the colab database -Gitlab need to add the variable auth_token to colab.yml to have the permission of acces to the gitlabAPI data Signed-off-by: Gustavo Jaruga <darksshades@gmail.com> Signed-off-by: Charles Oliveira <18oliveira.charles@gmail.com> Signed-off-by: Carlos Oliveira <carlospecter@gmail.com>

Add interface for data importation for proxies
-Proxies utilizes the data_api.py fetchData() to expecify how to import their data into the colab database -Gitlab need to add the variable auth_token to colab.yml to have the permission of acces to the gitlabAPI data Signed-off-by: Gustavo Jaruga <darksshades@gmail.com> Signed-off-by: Charles Oliveira <18oliveira.charles@gmail.com> Signed-off-by: Carlos Oliveira <carlospecter@gmail.com>
Gust
1 parent 13c0a171
Showing 13 changed files with 167 additions and 1 deletions Show diff stats
colab/management/initconfig.py
colab/proxy/gitlab/data_api.py
colab/proxy/gitlab/migrations/0001_initial.py
colab/proxy/gitlab/migrations/__init__.py
colab/proxy/gitlab/models.py
colab/proxy/jenkins/data_api.py
colab/proxy/noosfero/data_api.py
colab/proxy/proxybase/__init__.py
colab/proxy/proxybase/management/commands/__init__.py
colab/proxy/proxybase/management/commands/import_proxy_data.py
colab/proxy/proxybase/proxy_data_api.py
colab/proxy/redmine/data_api.py
colab/proxy/trac/data_api.py
@@ -70,6 +70,7 @@ ROBOTS_NOINDEX: false
 # PROXIED_APPS:
 #   gitlab:
 #     upstream: 'http://localhost:8090/gitlab/'
+#     auth_token: ''
 #   trac:
 #     upstream: 'http://localhost:5000/trac/'
  
@@ -0,0 +1,51 @@
+from colab.proxy.gitlab.models import *
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
+from django.db.models.fields import DateTimeField
+from dateutil.parser import parse
+import urllib2
+import json
+from django.conf import settings
+
+class GitlabDataAPI(ProxyDataAPI):
+
+
+  def fetchProjects(self):
+    page = 1
+    projects = []
+
+    proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
+    admin_token = proxy_config.get('auth_token')
+
+    # Iterates throughout all projects pages
+    while(True):
+      data = urllib2.urlopen(proxy_config.get('upstream')+'api/v3/projects/all?private_token={}&per_page=100&page={}'.format(admin_token, page))
+      json_data = json.load(data)
+
+      if len(json_data) == 0:
+        break
+
+      page = page + 1
+
+      for element in json_data:
+        project = GitlabProject()
+
+        for field in GitlabProject._meta.fields:
+          value = element[field.name]
+          value = parse(element[field.name]) if isinstance(field, DateTimeField) else value
+          setattr(project, field.name, value)
+
+        projects.append(project)
+
+    return projects
+
+
+  def fetchData(self):
+    data = self.fetchProjects()
+
+    for datum in data:
+      datum.save()
+
+  @property
+  def app_label(self):
+    return 'gitlab'
+
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GitlabProject',
+            fields=[
+                ('id', models.IntegerField(serialize=False, primary_key=True)),
+                ('description', models.TextField()),
+                ('public', models.BooleanField(default=True)),
+                ('name', models.TextField()),
+                ('name_with_namespace', models.TextField()),
+                ('created_at', models.DateTimeField(blank=True)),
+                ('last_activity_at', models.DateTimeField(blank=True)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+    ]
 from django.db import models
+from django.conf import settings
+from colab.accounts.models import User
  
-# Create your models here.
+class GitlabProject(models.Model):
+
+    id = models.IntegerField(primary_key=True)
+    description = models.TextField()
+    public = models.BooleanField(default=True)
+    name = models.TextField()
+    name_with_namespace = models.TextField()
+    created_at = models.DateTimeField(blank=True)
+    last_activity_at = models.DateTimeField(blank=True)
@@ -0,0 +1,12 @@
+from colab.proxy.gitlab.models import *
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
+from django.db.models.fields import DateTimeField
+from dateutil.parser import parse
+import urllib2
+import json
+
+class JenkinsDataAPI(ProxyDataAPI):
+
+  def fetchData(self):
+    pass
+
@@ -0,0 +1,12 @@
+from colab.proxy.gitlab.models import *
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
+from django.db.models.fields import DateTimeField
+from dateutil.parser import parse
+import urllib2
+import json
+
+class NoosferoDataAPI(ProxyDataAPI):
+
+  def fetchData(self):
+    pass
+
@@ -0,0 +1 @@
+__init__.py
 \ No newline at end of file
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+import colab
+from django.core.management.base import BaseCommand
+from colab.super_archives.models import Message
+from django.conf import settings
+modules = [ i for i in settings.INSTALLED_APPS if i.startswith("colab.proxy.") ]
+for module in modules:
+  module += ".data_api"
+  __import__(module, locals(), globals())
+
+class Command(BaseCommand):
+    help = "Import proxy data into colab database"
+
+    def handle(self, *args, **kwargs):
+        print "Executing extraction command..."
+
+        for module in modules:
+          extractionClassname = module + ".data_api." + module.split('.')[-1].title() + "DataAPI"
+          api = eval(extractionClassname)()
+          api.fetchData()
@@ -0,0 +1,6 @@
+
+class ProxyDataAPI():
+
+
+  def fetchData(self):
+    raise NotImplementedError('fetchData not yet implemented')
 \ No newline at end of file
@@ -0,0 +1,12 @@
+from colab.proxy.gitlab.models import *
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
+from django.db.models.fields import DateTimeField
+from dateutil.parser import parse
+import urllib2
+import json
+
+class RedmineDataAPI(ProxyDataAPI):
+
+  def fetchData(self):
+    pass
+
@@ -0,0 +1,12 @@
+from colab.proxy.gitlab.models import *
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
+from django.db.models.fields import DateTimeField
+from dateutil.parser import parse
+import urllib2
+import json
+
+class TracDataAPI(ProxyDataAPI):
+
+  def fetchData(self):
+    pass
+
...	...	@@ -70,6 +70,7 @@ ROBOTS_NOINDEX: false
70	70	# PROXIED_APPS:
71	71	# gitlab:
72	72	# upstream: 'http://localhost:8090/gitlab/'
	73	+# auth_token: ''
73	74	# trac:
74	75	# upstream: 'http://localhost:5000/trac/'
75	76
...	...
...	...	@@ -0,0 +1,51 @@
	1	+from colab.proxy.gitlab.models import *
	2	+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
	3	+from django.db.models.fields import DateTimeField
	4	+from dateutil.parser import parse
	5	+import urllib2
	6	+import json
	7	+from django.conf import settings
	8	+
	9	+class GitlabDataAPI(ProxyDataAPI):
	10	+
	11	+
	12	+ def fetchProjects(self):
	13	+ page = 1
	14	+ projects = []
	15	+
	16	+ proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
	17	+ admin_token = proxy_config.get('auth_token')
	18	+
	19	+ # Iterates throughout all projects pages
	20	+ while(True):
	21	+ data = urllib2.urlopen(proxy_config.get('upstream')+'api/v3/projects/all?private_token={}&per_page=100&page={}'.format(admin_token, page))
	22	+ json_data = json.load(data)
	23	+
	24	+ if len(json_data) == 0:
	25	+ break
	26	+
	27	+ page = page + 1
	28	+
	29	+ for element in json_data:
	30	+ project = GitlabProject()
	31	+
	32	+ for field in GitlabProject._meta.fields:
	33	+ value = element[field.name]
	34	+ value = parse(element[field.name]) if isinstance(field, DateTimeField) else value
	35	+ setattr(project, field.name, value)
	36	+
	37	+ projects.append(project)
	38	+
	39	+ return projects
	40	+
	41	+
	42	+ def fetchData(self):
	43	+ data = self.fetchProjects()
	44	+
	45	+ for datum in data:
	46	+ datum.save()
	47	+
	48	+ @property
	49	+ def app_label(self):
	50	+ return 'gitlab'
	51	+
...	...
...	...	@@ -0,0 +1,28 @@
	1	+# -- coding: utf-8 --
	2	+from __future__ import unicode_literals
	3	+
	4	+from django.db import models, migrations
	5	+
	6	+
	7	+class Migration(migrations.Migration):
	8	+
	9	+ dependencies = [
	10	+ ]
	11	+
	12	+ operations = [
	13	+ migrations.CreateModel(
	14	+ name='GitlabProject',
	15	+ fields=[
	16	+ ('id', models.IntegerField(serialize=False, primary_key=True)),
	17	+ ('description', models.TextField()),
	18	+ ('public', models.BooleanField(default=True)),
	19	+ ('name', models.TextField()),
	20	+ ('name_with_namespace', models.TextField()),
	21	+ ('created_at', models.DateTimeField(blank=True)),
	22	+ ('last_activity_at', models.DateTimeField(blank=True)),
	23	+ ],
	24	+ options={
	25	+ },
	26	+ bases=(models.Model,),
	27	+ ),
	28	+ ]
...	...
1	1	from django.db import models
	2	+from django.conf import settings
	3	+from colab.accounts.models import User
2	4
3		-# Create your models here.
	5	+class GitlabProject(models.Model):
	6	+
	7	+ id = models.IntegerField(primary_key=True)
	8	+ description = models.TextField()
	9	+ public = models.BooleanField(default=True)
	10	+ name = models.TextField()
	11	+ name_with_namespace = models.TextField()
	12	+ created_at = models.DateTimeField(blank=True)
	13	+ last_activity_at = models.DateTimeField(blank=True)
...	...
...	...	@@ -0,0 +1,12 @@
	1	+from colab.proxy.gitlab.models import *
	2	+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
	3	+from django.db.models.fields import DateTimeField
	4	+from dateutil.parser import parse
	5	+import urllib2
	6	+import json
	7	+
	8	+class JenkinsDataAPI(ProxyDataAPI):
	9	+
	10	+ def fetchData(self):
	11	+ pass
	12	+
...	...
...	...	@@ -0,0 +1 @@
	1	+__init__.py
0	2	\ No newline at end of file
...	...
...	...	@@ -0,0 +1,21 @@
	1	+#!/usr/bin/env python
	2	+
	3	+import colab
	4	+from django.core.management.base import BaseCommand
	5	+from colab.super_archives.models import Message
	6	+from django.conf import settings
	7	+modules = [ i for i in settings.INSTALLED_APPS if i.startswith("colab.proxy.") ]
	8	+for module in modules:
	9	+ module += ".data_api"
	10	+ __import__(module, locals(), globals())
	11	+
	12	+class Command(BaseCommand):
	13	+ help = "Import proxy data into colab database"
	14	+
	15	+ def handle(self, args, *kwargs):
	16	+ print "Executing extraction command..."
	17	+
	18	+ for module in modules:
	19	+ extractionClassname = module + ".data_api." + module.split('.')[-1].title() + "DataAPI"
	20	+ api = eval(extractionClassname)()
	21	+ api.fetchData()
...	...
...	...	@@ -0,0 +1,6 @@
	1	+
	2	+class ProxyDataAPI():
	3	+
	4	+
	5	+ def fetchData(self):
	6	+ raise NotImplementedError('fetchData not yet implemented')
0	7	\ No newline at end of file
...	...