diff --git a/colab/management/initconfig.py b/colab/management/initconfig.py index 142ee45..aa34543 100644 --- a/colab/management/initconfig.py +++ b/colab/management/initconfig.py @@ -70,6 +70,7 @@ ROBOTS_NOINDEX: false # PROXIED_APPS: # gitlab: # upstream: 'http://localhost:8090/gitlab/' +# auth_token: '' # trac: # upstream: 'http://localhost:5000/trac/' diff --git a/colab/proxy/gitlab/data_api.py b/colab/proxy/gitlab/data_api.py new file mode 100644 index 0000000..61aa0e3 --- /dev/null +++ b/colab/proxy/gitlab/data_api.py @@ -0,0 +1,51 @@ +from colab.proxy.gitlab.models import * +from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI +from django.db.models.fields import DateTimeField +from dateutil.parser import parse +import urllib2 +import json +from django.conf import settings + +class GitlabDataAPI(ProxyDataAPI): + + + def fetchProjects(self): + page = 1 + projects = [] + + proxy_config = settings.PROXIED_APPS.get(self.app_label, {}) + admin_token = proxy_config.get('auth_token') + + # Iterates throughout all projects pages + while(True): + data = urllib2.urlopen(proxy_config.get('upstream')+'api/v3/projects/all?private_token={}&per_page=100&page={}'.format(admin_token, page)) + json_data = json.load(data) + + if len(json_data) == 0: + break + + page = page + 1 + + for element in json_data: + project = GitlabProject() + + for field in GitlabProject._meta.fields: + value = element[field.name] + value = parse(element[field.name]) if isinstance(field, DateTimeField) else value + setattr(project, field.name, value) + + projects.append(project) + + return projects + + + def fetchData(self): + data = self.fetchProjects() + + for datum in data: + datum.save() + + @property + def app_label(self): + return 'gitlab' + diff --git a/colab/proxy/gitlab/migrations/0001_initial.py b/colab/proxy/gitlab/migrations/0001_initial.py new file mode 100644 index 0000000..516d858 --- /dev/null +++ b/colab/proxy/gitlab/migrations/0001_initial.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='GitlabProject', + fields=[ + ('id', models.IntegerField(serialize=False, primary_key=True)), + ('description', models.TextField()), + ('public', models.BooleanField(default=True)), + ('name', models.TextField()), + ('name_with_namespace', models.TextField()), + ('created_at', models.DateTimeField(blank=True)), + ('last_activity_at', models.DateTimeField(blank=True)), + ], + options={ + }, + bases=(models.Model,), + ), + ] diff --git a/colab/proxy/gitlab/migrations/__init__.py b/colab/proxy/gitlab/migrations/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/colab/proxy/gitlab/migrations/__init__.py diff --git a/colab/proxy/gitlab/models.py b/colab/proxy/gitlab/models.py index 71a8362..2998b7d 100644 --- a/colab/proxy/gitlab/models.py +++ b/colab/proxy/gitlab/models.py @@ -1,3 +1,13 @@ from django.db import models +from django.conf import settings +from colab.accounts.models import User -# Create your models here. +class GitlabProject(models.Model): + + id = models.IntegerField(primary_key=True) + description = models.TextField() + public = models.BooleanField(default=True) + name = models.TextField() + name_with_namespace = models.TextField() + created_at = models.DateTimeField(blank=True) + last_activity_at = models.DateTimeField(blank=True) diff --git a/colab/proxy/jenkins/data_api.py b/colab/proxy/jenkins/data_api.py new file mode 100644 index 0000000..bec0639 --- /dev/null +++ b/colab/proxy/jenkins/data_api.py @@ -0,0 +1,12 @@ +from colab.proxy.gitlab.models import * +from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI +from django.db.models.fields import DateTimeField +from dateutil.parser import parse +import urllib2 +import json + +class JenkinsDataAPI(ProxyDataAPI): + + def fetchData(self): + pass + diff --git a/colab/proxy/noosfero/data_api.py b/colab/proxy/noosfero/data_api.py new file mode 100644 index 0000000..1099e46 --- /dev/null +++ b/colab/proxy/noosfero/data_api.py @@ -0,0 +1,12 @@ +from colab.proxy.gitlab.models import * +from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI +from django.db.models.fields import DateTimeField +from dateutil.parser import parse +import urllib2 +import json + +class NoosferoDataAPI(ProxyDataAPI): + + def fetchData(self): + pass + diff --git a/colab/proxy/proxybase/__init__.py b/colab/proxy/proxybase/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/colab/proxy/proxybase/__init__.py diff --git a/colab/proxy/proxybase/management/commands/__init__.py b/colab/proxy/proxybase/management/commands/__init__.py new file mode 100644 index 0000000..93f5256 --- /dev/null +++ b/colab/proxy/proxybase/management/commands/__init__.py @@ -0,0 +1 @@ +__init__.py \ No newline at end of file diff --git a/colab/proxy/proxybase/management/commands/import_proxy_data.py b/colab/proxy/proxybase/management/commands/import_proxy_data.py new file mode 100644 index 0000000..acd8f7a --- /dev/null +++ b/colab/proxy/proxybase/management/commands/import_proxy_data.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +import colab +from django.core.management.base import BaseCommand +from colab.super_archives.models import Message +from django.conf import settings +modules = [ i for i in settings.INSTALLED_APPS if i.startswith("colab.proxy.") ] +for module in modules: + module += ".data_api" + __import__(module, locals(), globals()) + +class Command(BaseCommand): + help = "Import proxy data into colab database" + + def handle(self, *args, **kwargs): + print "Executing extraction command..." + + for module in modules: + extractionClassname = module + ".data_api." + module.split('.')[-1].title() + "DataAPI" + api = eval(extractionClassname)() + api.fetchData() diff --git a/colab/proxy/proxybase/proxy_data_api.py b/colab/proxy/proxybase/proxy_data_api.py new file mode 100644 index 0000000..b4fa475 --- /dev/null +++ b/colab/proxy/proxybase/proxy_data_api.py @@ -0,0 +1,6 @@ + +class ProxyDataAPI(): + + + def fetchData(self): + raise NotImplementedError('fetchData not yet implemented') \ No newline at end of file diff --git a/colab/proxy/redmine/data_api.py b/colab/proxy/redmine/data_api.py new file mode 100644 index 0000000..5683519 --- /dev/null +++ b/colab/proxy/redmine/data_api.py @@ -0,0 +1,12 @@ +from colab.proxy.gitlab.models import * +from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI +from django.db.models.fields import DateTimeField +from dateutil.parser import parse +import urllib2 +import json + +class RedmineDataAPI(ProxyDataAPI): + + def fetchData(self): + pass + diff --git a/colab/proxy/trac/data_api.py b/colab/proxy/trac/data_api.py new file mode 100644 index 0000000..f2673c7 --- /dev/null +++ b/colab/proxy/trac/data_api.py @@ -0,0 +1,12 @@ +from colab.proxy.gitlab.models import * +from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI +from django.db.models.fields import DateTimeField +from dateutil.parser import parse +import urllib2 +import json + +class TracDataAPI(ProxyDataAPI): + + def fetchData(self): + pass + -- libgit2 0.21.2