Merge Request #23

← To merge requests From proxy_data_importation into master
Merged
softwarepublico/colab!23
Created by Gustavo Jaruga Cruz

Proxy data importation

Please merge https://beta.softwarepublico.gov.br/gitlab/softwarepublico/colab/merge_requests/18 first, as this assumes proxybase is already an app

Assignee: Sergio Oliveira
Milestone: None

Merged by Sergio Oliveira

Source branch has been removed
Commits (7)
2 participants
Showing 33 changed files   Show diff stats
colab/management/initconfig.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -70,6 +70,7 @@ ROBOTS_NOINDEX: false
70 70 # PROXIED_APPS:
71 71 # gitlab:
72 72 # upstream: 'http://localhost:8090/gitlab/'
  73 +# private_token: ''
73 74 # trac:
74 75 # upstream: 'http://localhost:5000/trac/'
75 76  
... ...
colab/proxy/gitlab/admin.py
View file @dbe18c1
... ... @@ -1,3 +0,0 @@
1   -from django.contrib import admin
2   -
3   -# Register your models here.
colab/proxy/gitlab/data_api.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -0,0 +1,69 @@
  1 +
  2 +import json
  3 +import urllib
  4 +import urllib2
  5 +
  6 +from dateutil.parser import parse
  7 +
  8 +from django.conf import settings
  9 +from django.db.models.fields import DateTimeField
  10 +
  11 +from colab.proxy.gitlab.models import GitlabProject
  12 +from colab.proxy.utils.proxy_data_api import ProxyDataAPI
  13 +
  14 +
  15 +class GitlabDataAPI(ProxyDataAPI):
  16 +
  17 + def get_request_url(self, path, **kwargs):
  18 + proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
  19 +
  20 + upstream = proxy_config.get('upstream')
  21 + kwargs['private_token'] = proxy_config.get('private_token')
  22 + params = urllib.urlencode(kwargs)
  23 +
  24 + if upstream[-1] == '/':
  25 + upstream = upstream[:-1]
  26 +
  27 + return u'{}{}?{}'.format(upstream, path, params)
  28 +
  29 + def fetchProjects(self):
  30 + page = 1
  31 + projects = []
  32 +
  33 + # Iterates throughout all projects pages
  34 + while(True):
  35 + url = self.get_request_url('/api/v3/projects/all',
  36 + per_page=100,
  37 + page=page)
  38 + data = urllib2.urlopen(url)
  39 + json_data = json.load(data)
  40 +
  41 + if len(json_data) == 0:
  42 + break
  43 +
  44 + page = page + 1
  45 +
  46 + for element in json_data:
  47 + project = GitlabProject()
  48 +
  49 + for field in GitlabProject._meta.fields:
  50 + if isinstance(field, DateTimeField):
  51 + value = parse(element[field.name])
  52 + else:
  53 + value = element[field.name]
  54 +
  55 + setattr(project, field.name, value)
  56 +
  57 + projects.append(project)
  58 +
  59 + return projects
  60 +
  61 + def fetchData(self):
  62 + data = self.fetchProjects()
  63 +
  64 + for datum in data:
  65 + datum.save()
  66 +
  67 + @property
  68 + def app_label(self):
  69 + return 'gitlab'
... ...
colab/proxy/gitlab/migrations/0001_initial.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -0,0 +1,28 @@
  1 +# -*- coding: utf-8 -*-
  2 +from __future__ import unicode_literals
  3 +
  4 +from django.db import models, migrations
  5 +
  6 +
  7 +class Migration(migrations.Migration):
  8 +
  9 + dependencies = [
  10 + ]
  11 +
  12 + operations = [
  13 + migrations.CreateModel(
  14 + name='GitlabProject',
  15 + fields=[
  16 + ('id', models.IntegerField(serialize=False, primary_key=True)),
  17 + ('description', models.TextField()),
  18 + ('public', models.BooleanField(default=True)),
  19 + ('name', models.TextField()),
  20 + ('name_with_namespace', models.TextField()),
  21 + ('created_at', models.DateTimeField(blank=True)),
  22 + ('last_activity_at', models.DateTimeField(blank=True)),
  23 + ],
  24 + options={
  25 + },
  26 + bases=(models.Model,),
  27 + ),
  28 + ]
... ...
colab/proxy/gitlab/migrations/__init__.py
  Diff comments   Edit   View file @e2ef073
colab/proxy/gitlab/models.py
  Diff comments   Edit   View file @e2ef073
1 1 from django.db import models
2 2  
3   -# Create your models here.
  3 +
  4 +class GitlabProject(models.Model):
  5 +
  6 + id = models.IntegerField(primary_key=True)
  7 + description = models.TextField()
  8 + public = models.BooleanField(default=True)
  9 + name = models.TextField()
  10 + name_with_namespace = models.TextField()
  11 + created_at = models.DateTimeField(blank=True)
  12 + last_activity_at = models.DateTimeField(blank=True)
... ...
colab/proxy/gitlab/tests.py
View file @dbe18c1
... ... @@ -1,3 +0,0 @@
1   -from django.test import TestCase
2   -
3   -# Create your tests here.
colab/proxy/jenkins/admin.py
View file @dbe18c1
... ... @@ -1,3 +0,0 @@
1   -from django.contrib import admin
2   -
3   -# Register your models here.
colab/proxy/jenkins/data_api.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -0,0 +1,7 @@
  1 +from colab.proxy.utils.proxy_data_api import ProxyDataAPI
  2 +
  3 +
  4 +class JenkinsDataAPI(ProxyDataAPI):
  5 +
  6 + def fetchData(self):
  7 + pass
... ...
colab/proxy/jenkins/models.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -1,3 +0,0 @@
1   -from django.db import models
2   -
3   -# Create your models here.
colab/proxy/jenkins/tests.py
View file @dbe18c1
... ... @@ -1,3 +0,0 @@
1   -from django.test import TestCase
2   -
3   -# Create your tests here.
colab/proxy/jenkins/views.py
  Diff comments   Edit   View file @e2ef073
1 1  
2   -from django.conf import settings
3   -
4 2 from ..utils.views import ColabProxyView
5 3  
6 4  
... ...
colab/proxy/management/__init__.py
  Diff comments   Edit   View file @e2ef073
colab/proxy/management/commands/__init__.py
  Diff comments   Edit   View file @e2ef073
colab/proxy/management/commands/import_proxy_data.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -0,0 +1,26 @@
  1 +#!/usr/bin/env python
  2 +
  3 +import importlib
  4 +
  5 +from django.core.management.base import BaseCommand
  6 +from django.conf import settings
  7 +
  8 +from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
  9 +
  10 +
  11 +class Command(BaseCommand):
  12 + help = "Import proxy data into colab database"
  13 +
  14 + def handle(self, *args, **kwargs):
  15 + print "Executing extraction command..."
  16 +
  17 + for module_name in settings.PROXIED_APPS.keys():
  18 + module_path = 'colab.proxy.{}.data_api'.format(module_name)
  19 + module = importlib.import_module(module_path)
  20 +
  21 + for module_item_name in dir(module):
  22 + module_item = getattr(module, module_item_name)
  23 + if issubclass(module_item, ProxyDataAPI):
  24 + if module_item != ProxyDataAPI:
  25 + api = module_item()
  26 + api.fetchData()
... ...
colab/proxy/noosfero/admin.py
View file @dbe18c1
... ... @@ -1,3 +0,0 @@
1   -from django.contrib import admin
2   -
3   -# Register your models here.
colab/proxy/noosfero/data_api.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -0,0 +1,7 @@
  1 +from colab.proxy.utils.proxy_data_api import ProxyDataAPI
  2 +
  3 +
  4 +class NoosferoDataAPI(ProxyDataAPI):
  5 +
  6 + def fetchData(self):
  7 + pass
... ...
colab/proxy/noosfero/models.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -1,3 +0,0 @@
1   -from django.db import models
2   -
3   -# Create your models here.
colab/proxy/noosfero/tests.py
View file @dbe18c1
... ... @@ -1,3 +0,0 @@
1   -from django.test import TestCase
2   -
3   -# Create your tests here.
colab/proxy/redmine/admin.py
View file @dbe18c1
... ... @@ -1,3 +0,0 @@
1   -from django.contrib import admin
2   -
3   -# Register your models here.
colab/proxy/redmine/data_api.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -0,0 +1,7 @@
  1 +from colab.proxy.utils.proxy_data_api import ProxyDataAPI
  2 +
  3 +
  4 +class RedmineDataAPI(ProxyDataAPI):
  5 +
  6 + def fetchData(self):
  7 + pass
... ...
colab/proxy/redmine/models.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -1,3 +0,0 @@
1   -from django.db import models
2   -
3   -# Create your models here.
colab/proxy/redmine/tests.py
View file @dbe18c1
... ... @@ -1,3 +0,0 @@
1   -from django.test import TestCase
2   -
3   -# Create your tests here.
colab/proxy/redmine/views.py
  Diff comments   Edit   View file @e2ef073
1 1  
2   -from django.conf import settings
3   -
4 2 from ..utils.views import ColabProxyView
5 3  
6 4  
... ...
colab/proxy/trac/admin.py
  Diff comments   Edit   View file @e2ef073
1   -from django.contrib import admin
2   -
3   -from . import signals
  1 +from . import signals # noqa
... ...
colab/proxy/trac/data_api.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -0,0 +1,7 @@
  1 +from colab.proxy.utils.proxy_data_api import ProxyDataAPI
  2 +
  3 +
  4 +class TracDataAPI(ProxyDataAPI):
  5 +
  6 + def fetchData(self):
  7 + pass
... ...
colab/proxy/trac/routers.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -11,7 +11,7 @@ class TracRouter(object):
11 11  
12 12 def allow_relation(self, obj1, obj2, **hints):
13 13 if obj1._meta.app_label == 'proxy' or \
14   - obj2._meta.app_label == 'proxy':
  14 + obj2._meta.app_label == 'proxy':
15 15 return True
16 16 return None
17 17  
... ...
colab/proxy/trac/search_indexes.py
  Diff comments   Edit   View file @e2ef073
1 1 # -*- coding: utf-8 -*-
2 2  
3   -import math
4 3 import string
5 4  
6 5 from django.template import loader, Context
7   -from django.utils.text import slugify
8 6 from haystack import indexes
9 7 from haystack.utils import log as logging
10 8  
... ... @@ -15,7 +13,7 @@ from .models import Attachment, Ticket, Wiki, Revision
15 13 logger = logging.getLogger('haystack')
16 14  
17 15 # the string maketrans always return a string encoded with latin1
18   -# http://stackoverflow.com/questions/1324067/how-do-i-get-str-translate-to-work-with-unicode-strings
  16 +# http://stackoverflow.com/questions/1324067/how-do-i-get-str-translate-to-work-with-unicode-strings # noqa
19 17 table = string.maketrans(
20 18 string.punctuation,
21 19 '.' * len(string.punctuation)
... ...
colab/proxy/trac/signals.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -12,22 +12,22 @@ def change_session_attribute_email(sender, instance, **kwargs):
12 12  
13 13 cursor.execute(("UPDATE session_attribute SET value=%s "
14 14 "WHERE name='email' AND sid=%s"),
15   - [instance.email, instance.username])
  15 + [instance.email, instance.username])
16 16 cursor.execute(("UPDATE session_attribute SET value=%s "
17 17 "WHERE name='name' AND sid=%s"),
18   - [instance.get_full_name(), instance.username])
  18 + [instance.get_full_name(), instance.username])
19 19  
20 20 cursor.execute(("INSERT INTO session_attribute "
21 21 "(sid, authenticated, name, value) "
22 22 "SELECT %s, '1', 'email', %s WHERE NOT EXISTS "
23 23 "(SELECT 1 FROM session_attribute WHERE sid=%s "
24 24 "AND name='email')"),
25   - [instance.username, instance.email, instance.username])
  25 + [instance.username, instance.email, instance.username])
26 26  
27 27 cursor.execute(("INSERT INTO session_attribute "
28 28 "(sid, authenticated, name, value) "
29 29 "SELECT %s, '1', 'name', %s WHERE NOT EXISTS "
30 30 "(SELECT 1 FROM session_attribute WHERE sid=%s "
31 31 "AND name='name')"),
32   - [instance.username, instance.get_full_name(),
33   - instance.username])
  32 + [instance.username, instance.get_full_name(),
  33 + instance.username])
... ...
colab/proxy/trac/tests.py
View file @dbe18c1
... ... @@ -1,3 +0,0 @@
1   -from django.test import TestCase
2   -
3   -# Create your tests here.
colab/proxy/trac/views.py
  Diff comments   Edit   View file @e2ef073
1 1  
2   -from django.conf import settings
3   -
4 2 from hitcounter.views import HitCounterViewMixin
5 3  
6 4 from ..utils.views import ColabProxyView
... ...
colab/proxy/utils/proxy_data_api.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -0,0 +1,6 @@
  1 +
  2 +
  3 +class ProxyDataAPI(object):
  4 +
  5 + def fetchData(self):
  6 + raise NotImplementedError('fetchData not yet implemented')
... ...
colab/settings.py
  Diff comments   Edit   View file @e2ef073
... ... @@ -320,8 +320,8 @@ if FEEDZILLA_ENABLED:
320 320 'common',
321 321 )
322 322  
323   -proxied_apps = locals().get('PROXIED_APPS') or {}
  323 +PROXIED_APPS = locals().get('PROXIED_APPS') or {}
324 324 BROWSERID_ENABLED = locals().get('BROWSERID_ENABLED') or False
325 325  
326   -for app_label in proxied_apps.keys():
  326 +for app_label in PROXIED_APPS.keys():
327 327 INSTALLED_APPS += ('colab.proxy.{}'.format(app_label),)
... ...
    9fe63c7bd60deeb55e409a1d7dd173f5?s=40&d=identicon
    Show/hide discussion
    Sergio Oliveira started a discussion on the outdated diff
    last updated by Sergio Oliveira
    colab/proxy/gitlab/data_api.py
      1 +from colab.proxy.gitlab.models import *
      2 +from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
      3 +from django.db.models.fields import DateTimeField
      4 +from dateutil.parser import parse
      5 +import urllib2
      6 +import json
      7 +from django.conf import settings
      8 +
      9 +class GitlabDataAPI(ProxyDataAPI):
      10 +
      11 +
      12 + def fetchProjects(self):
    1
    9fe63c7bd60deeb55e409a1d7dd173f5?s=40&d=identicon
    Show/hide discussion
    Sergio Oliveira started a discussion on the outdated diff
    last updated by Sergio Oliveira
    colab/proxy/proxybase/proxy_data_api.py
      1 +
      2 +class ProxyDataAPI():
    1
    • 9fe63c7bd60deeb55e409a1d7dd173f5?s=40&d=identicon
      Link here  
      Sergio Oliveira @seocam

      Em python 2 voce precisa extender object ProxyDataAPI(object). Se não fizer isso vai ter o comportamento das classes do Python

      Em python 3 não precisa mais extender object e nesse caso os parenteses nem são necessários.

      Choose File ...   File name...
      Cancel
    9fe63c7bd60deeb55e409a1d7dd173f5?s=40&d=identicon
    Show/hide discussion
    Sergio Oliveira started a discussion on the outdated diff
    last updated by Gustavo Jaruga Cruz
    colab/management/initconfig.py
    66 66 # PROXIED_APPS:
    67 67 # gitlab:
    68 68 # upstream: 'http://localhost:8090/gitlab/'
      69 +# auth_token: ''
    2
    • 9fe63c7bd60deeb55e409a1d7dd173f5?s=40&d=identicon
      Link here  
      Sergio Oliveira @seocam

      A url de request chama de private_token, a variavel que voces utilizam chama admin_token e nos settings auth_token. Porque nao usamos apenas um nome?

      Choose File ...   File name...
      Cancel
    • C6b14af78e51fba6beb90142971240cc?s=40&d=identicon
      Link here  
      Gustavo Jaruga Cruz @darksshades

      Verdade... nem notei quando estava escrevendo

      Choose File ...   File name...
      Cancel
    9fe63c7bd60deeb55e409a1d7dd173f5?s=40&d=identicon
    Show/hide discussion
    Sergio Oliveira started a discussion on the outdated diff
    last updated by Sergio Oliveira
    colab/proxy/jenkins/data_api.py
    ... ... @@ -0,0 +1,12 @@
  • 9fe63c7bd60deeb55e409a1d7dd173f5?s=40&d=identicon
    Link here  
    Sergio Oliveira @seocam

    Roda o flake8 nos arquivos modificados pra pegarmos os erros. Logo logo vamos integrar isso nos testes unitarios ;)

    Choose File ...   File name...
    Cancel