Merge branch 'proxy_data_importation'

Sergio Oliveira
2 parents 212b6ed7 e2ef0735
Showing 33 changed files with 178 additions and 57 deletions Show diff stats
colab/management/initconfig.py
colab/proxy/gitlab/admin.py
colab/proxy/gitlab/data_api.py
colab/proxy/gitlab/migrations/0001_initial.py
colab/proxy/gitlab/migrations/__init__.py
colab/proxy/gitlab/models.py
colab/proxy/gitlab/tests.py
colab/proxy/jenkins/admin.py
colab/proxy/jenkins/data_api.py
colab/proxy/jenkins/models.py
colab/proxy/jenkins/tests.py
colab/proxy/jenkins/views.py
colab/proxy/management/__init__.py
colab/proxy/management/commands/__init__.py
colab/proxy/management/commands/import_proxy_data.py
colab/proxy/noosfero/admin.py
colab/proxy/noosfero/data_api.py
colab/proxy/noosfero/models.py
colab/proxy/noosfero/tests.py
colab/proxy/redmine/admin.py
@@ -70,6 +70,7 @@ ROBOTS_NOINDEX: false
 # PROXIED_APPS:
 #   gitlab:
 #     upstream: 'http://localhost:8090/gitlab/'
+#     private_token: ''
 #   trac:
 #     upstream: 'http://localhost:5000/trac/'
@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.
@@ -0,0 +1,69 @@
+
+import json
+import urllib
+import urllib2
+
+from dateutil.parser import parse
+
+from django.conf import settings
+from django.db.models.fields import DateTimeField
+
+from colab.proxy.gitlab.models import GitlabProject
+from colab.proxy.utils.proxy_data_api import ProxyDataAPI
+
+
+class GitlabDataAPI(ProxyDataAPI):
+
+    def get_request_url(self, path, **kwargs):
+        proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
+
+        upstream = proxy_config.get('upstream')
+        kwargs['private_token'] = proxy_config.get('private_token')
+        params = urllib.urlencode(kwargs)
+
+        if upstream[-1] == '/':
+            upstream = upstream[:-1]
+
+        return u'{}{}?{}'.format(upstream, path, params)
+
+    def fetchProjects(self):
+        page = 1
+        projects = []
+
+        # Iterates throughout all projects pages
+        while(True):
+            url = self.get_request_url('/api/v3/projects/all',
+                                       per_page=100,
+                                       page=page)
+            data = urllib2.urlopen(url)
+            json_data = json.load(data)
+
+            if len(json_data) == 0:
+                break
+
+            page = page + 1
+
+            for element in json_data:
+                project = GitlabProject()
+
+                for field in GitlabProject._meta.fields:
+                    if isinstance(field, DateTimeField):
+                        value = parse(element[field.name])
+                    else:
+                        value = element[field.name]
+
+                    setattr(project, field.name, value)
+
+                projects.append(project)
+
+        return projects
+
+    def fetchData(self):
+        data = self.fetchProjects()
+
+        for datum in data:
+            datum.save()
+
+    @property
+    def app_label(self):
+        return 'gitlab'
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GitlabProject',
+            fields=[
+                ('id', models.IntegerField(serialize=False, primary_key=True)),
+                ('description', models.TextField()),
+                ('public', models.BooleanField(default=True)),
+                ('name', models.TextField()),
+                ('name_with_namespace', models.TextField()),
+                ('created_at', models.DateTimeField(blank=True)),
+                ('last_activity_at', models.DateTimeField(blank=True)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+    ]
 from django.db import models
-# Create your models here.
+
+class GitlabProject(models.Model):
+
+    id = models.IntegerField(primary_key=True)
+    description = models.TextField()
+    public = models.BooleanField(default=True)
+    name = models.TextField()
+    name_with_namespace = models.TextField()
+    created_at = models.DateTimeField(blank=True)
+    last_activity_at = models.DateTimeField(blank=True)
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.
@@ -0,0 +1,7 @@
+from colab.proxy.utils.proxy_data_api import ProxyDataAPI
+
+
+class JenkinsDataAPI(ProxyDataAPI):
+
+    def fetchData(self):
+        pass
@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
-from django.conf import settings
-
 from ..utils.views import ColabProxyView
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+import importlib
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
+
+
+class Command(BaseCommand):
+    help = "Import proxy data into colab database"
+
+    def handle(self, *args, **kwargs):
+        print "Executing extraction command..."
+
+        for module_name in settings.PROXIED_APPS.keys():
+            module_path = 'colab.proxy.{}.data_api'.format(module_name)
+            module = importlib.import_module(module_path)
+
+            for module_item_name in dir(module):
+                module_item = getattr(module, module_item_name)
+                if issubclass(module_item, ProxyDataAPI):
+                    if module_item != ProxyDataAPI:
+                        api = module_item()
+                        api.fetchData()
@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.
@@ -0,0 +1,7 @@
+from colab.proxy.utils.proxy_data_api import ProxyDataAPI
+
+
+class NoosferoDataAPI(ProxyDataAPI):
+
+    def fetchData(self):
+        pass
@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.
@@ -0,0 +1,7 @@
+from colab.proxy.utils.proxy_data_api import ProxyDataAPI
+
+
+class RedmineDataAPI(ProxyDataAPI):
+
+    def fetchData(self):
+        pass
@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
-from django.conf import settings
-
 from ..utils.views import ColabProxyView
-from django.contrib import admin
-
-from . import signals
+from . import signals  # noqa
@@ -0,0 +1,7 @@
+from colab.proxy.utils.proxy_data_api import ProxyDataAPI
+
+
+class TracDataAPI(ProxyDataAPI):
+
+    def fetchData(self):
+        pass
@@ -11,7 +11,7 @@ class TracRouter(object):
     def allow_relation(self, obj1, obj2, **hints):
         if obj1._meta.app_label == 'proxy' or \
-            obj2._meta.app_label == 'proxy':
+           obj2._meta.app_label == 'proxy':
             return True
         return None
 # -*- coding: utf-8 -*-
-import math
 import string
 from django.template import loader, Context
-from django.utils.text import slugify
 from haystack import indexes
 from haystack.utils import log as logging
@@ -15,7 +13,7 @@ from .models import Attachment, Ticket, Wiki, Revision
 logger = logging.getLogger('haystack')
 # the string maketrans always return a string encoded with latin1
-# http://stackoverflow.com/questions/1324067/how-do-i-get-str-translate-to-work-with-unicode-strings
+# http://stackoverflow.com/questions/1324067/how-do-i-get-str-translate-to-work-with-unicode-strings # noqa
 table = string.maketrans(
     string.punctuation,
     '.' * len(string.punctuation)
@@ -12,22 +12,22 @@ def change_session_attribute_email(sender, instance, **kwargs):
     cursor.execute(("UPDATE session_attribute SET value=%s "
                     "WHERE name='email' AND sid=%s"),
-                    [instance.email, instance.username])
+                   [instance.email, instance.username])
     cursor.execute(("UPDATE session_attribute SET value=%s "
                     "WHERE name='name' AND sid=%s"),
-                    [instance.get_full_name(), instance.username])
+                   [instance.get_full_name(), instance.username])
     cursor.execute(("INSERT INTO session_attribute "
                     "(sid, authenticated,  name, value) "
                     "SELECT %s, '1', 'email', %s WHERE NOT EXISTS "
                     "(SELECT 1 FROM session_attribute WHERE sid=%s "
                     "AND name='email')"),
-                    [instance.username, instance.email, instance.username])
+                   [instance.username, instance.email, instance.username])
     cursor.execute(("INSERT INTO session_attribute "
                     "(sid, authenticated, name, value) "
                     "SELECT %s, '1', 'name', %s WHERE NOT EXISTS "
                     "(SELECT 1 FROM session_attribute WHERE sid=%s "
                     "AND name='name')"),
-                    [instance.username, instance.get_full_name(),
-                     instance.username])
+                   [instance.username, instance.get_full_name(),
+                    instance.username])
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
-from django.conf import settings
-
 from hitcounter.views import HitCounterViewMixin
 from ..utils.views import ColabProxyView
@@ -0,0 +1,6 @@
+
+
+class ProxyDataAPI(object):
+
+    def fetchData(self):
+        raise NotImplementedError('fetchData not yet implemented')
@@ -320,8 +320,8 @@ if FEEDZILLA_ENABLED:
         'common',
     )
-proxied_apps = locals().get('PROXIED_APPS') or {}
+PROXIED_APPS = locals().get('PROXIED_APPS') or {}
 BROWSERID_ENABLED = locals().get('BROWSERID_ENABLED') or False
-for app_label in proxied_apps.keys():
+for app_label in PROXIED_APPS.keys():
     INSTALLED_APPS += ('colab.proxy.{}'.format(app_label),)
	@@ -70,6 +70,7 @@ ROBOTS_NOINDEX: false		@@ -70,6 +70,7 @@ ROBOTS_NOINDEX: false
70	# PROXIED_APPS:	70	# PROXIED_APPS:
71	# gitlab:	71	# gitlab:
72	# upstream: 'http://localhost:8090/gitlab/'	72	# upstream: 'http://localhost:8090/gitlab/'
		73	+# private_token: ''
73	# trac:	74	# trac:
74	# upstream: 'http://localhost:5000/trac/'	75	# upstream: 'http://localhost:5000/trac/'
75		76
	@@ -1,3 +0,0 @@	@@ -1,3 +0,0 @@
1	-from django.contrib import admin
2	-
3	-# Register your models here.
@@ -0,0 +1,69 @@		@@ -0,0 +1,69 @@
	1	+
	2	+import json
	3	+import urllib
	4	+import urllib2
	5	+
	6	+from dateutil.parser import parse
	7	+
	8	+from django.conf import settings
	9	+from django.db.models.fields import DateTimeField
	10	+
	11	+from colab.proxy.gitlab.models import GitlabProject
	12	+from colab.proxy.utils.proxy_data_api import ProxyDataAPI
	13	+
	14	+
	15	+class GitlabDataAPI(ProxyDataAPI):
	16	+
	17	+ def get_request_url(self, path, **kwargs):
	18	+ proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
	19	+
	20	+ upstream = proxy_config.get('upstream')
	21	+ kwargs['private_token'] = proxy_config.get('private_token')
	22	+ params = urllib.urlencode(kwargs)
	23	+
	24	+ if upstream[-1] == '/':
	25	+ upstream = upstream[:-1]
	26	+
	27	+ return u'{}{}?{}'.format(upstream, path, params)
	28	+
	29	+ def fetchProjects(self):
	30	+ page = 1
	31	+ projects = []
	32	+
	33	+ # Iterates throughout all projects pages
	34	+ while(True):
	35	+ url = self.get_request_url('/api/v3/projects/all',
	36	+ per_page=100,
	37	+ page=page)
	38	+ data = urllib2.urlopen(url)
	39	+ json_data = json.load(data)
	40	+
	41	+ if len(json_data) == 0:
	42	+ break
	43	+
	44	+ page = page + 1
	45	+
	46	+ for element in json_data:
	47	+ project = GitlabProject()
	48	+
	49	+ for field in GitlabProject._meta.fields:
	50	+ if isinstance(field, DateTimeField):
	51	+ value = parse(element[field.name])
	52	+ else:
	53	+ value = element[field.name]
	54	+
	55	+ setattr(project, field.name, value)
	56	+
	57	+ projects.append(project)
	58	+
	59	+ return projects
	60	+
	61	+ def fetchData(self):
	62	+ data = self.fetchProjects()
	63	+
	64	+ for datum in data:
	65	+ datum.save()
	66	+
	67	+ @property
	68	+ def app_label(self):
	69	+ return 'gitlab'
@@ -0,0 +1,28 @@		@@ -0,0 +1,28 @@
	1	+# -- coding: utf-8 --
	2	+from __future__ import unicode_literals
	3	+
	4	+from django.db import models, migrations
	5	+
	6	+
	7	+class Migration(migrations.Migration):
	8	+
	9	+ dependencies = [
	10	+ ]
	11	+
	12	+ operations = [
	13	+ migrations.CreateModel(
	14	+ name='GitlabProject',
	15	+ fields=[
	16	+ ('id', models.IntegerField(serialize=False, primary_key=True)),
	17	+ ('description', models.TextField()),
	18	+ ('public', models.BooleanField(default=True)),
	19	+ ('name', models.TextField()),
	20	+ ('name_with_namespace', models.TextField()),
	21	+ ('created_at', models.DateTimeField(blank=True)),
	22	+ ('last_activity_at', models.DateTimeField(blank=True)),
	23	+ ],
	24	+ options={
	25	+ },
	26	+ bases=(models.Model,),
	27	+ ),
	28	+ ]
1	from django.db import models	1	from django.db import models
2		2
3	-# Create your models here.	3	+
		4	+class GitlabProject(models.Model):
		5	+
		6	+ id = models.IntegerField(primary_key=True)
		7	+ description = models.TextField()
		8	+ public = models.BooleanField(default=True)
		9	+ name = models.TextField()
		10	+ name_with_namespace = models.TextField()
		11	+ created_at = models.DateTimeField(blank=True)
		12	+ last_activity_at = models.DateTimeField(blank=True)
	@@ -1,3 +0,0 @@	@@ -1,3 +0,0 @@
1	-from django.test import TestCase
2	-
3	-# Create your tests here.
@@ -0,0 +1,7 @@		@@ -0,0 +1,7 @@
	1	+from colab.proxy.utils.proxy_data_api import ProxyDataAPI
	2	+
	3	+
	4	+class JenkinsDataAPI(ProxyDataAPI):
	5	+
	6	+ def fetchData(self):
	7	+ pass
	@@ -1,3 +0,0 @@	@@ -1,3 +0,0 @@
1	-from django.db import models
2	-
3	-# Create your models here.
1		1
2	-from django.conf import settings
3	-
4	from ..utils.views import ColabProxyView	2	from ..utils.views import ColabProxyView
5		3
6		4
@@ -0,0 +1,26 @@		@@ -0,0 +1,26 @@
	1	+#!/usr/bin/env python
	2	+
	3	+import importlib
	4	+
	5	+from django.core.management.base import BaseCommand
	6	+from django.conf import settings
	7	+
	8	+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
	9	+
	10	+
	11	+class Command(BaseCommand):
	12	+ help = "Import proxy data into colab database"
	13	+
	14	+ def handle(self, args, *kwargs):
	15	+ print "Executing extraction command..."
	16	+
	17	+ for module_name in settings.PROXIED_APPS.keys():
	18	+ module_path = 'colab.proxy.{}.data_api'.format(module_name)
	19	+ module = importlib.import_module(module_path)
	20	+
	21	+ for module_item_name in dir(module):
	22	+ module_item = getattr(module, module_item_name)
	23	+ if issubclass(module_item, ProxyDataAPI):
	24	+ if module_item != ProxyDataAPI:
	25	+ api = module_item()
	26	+ api.fetchData()