From 52ba4161e6085880ca5749f4645147ec50cb9964 Mon Sep 17 00:00:00 2001 From: Thiago Ribeiro Date: Tue, 21 Jul 2015 11:53:12 -0300 Subject: [PATCH] Add support to noosfero data. --- colab/plugins/noosfero/data_api.py | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- colab/plugins/noosfero/migrations/0001_initial.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ colab/plugins/noosfero/migrations/__init__.py | 0 colab/plugins/noosfero/models.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 colab/plugins/noosfero/migrations/0001_initial.py create mode 100644 colab/plugins/noosfero/migrations/__init__.py diff --git a/colab/plugins/noosfero/data_api.py b/colab/plugins/noosfero/data_api.py index a062b86..23b5b3d 100644 --- a/colab/plugins/noosfero/data_api.py +++ b/colab/plugins/noosfero/data_api.py @@ -1,7 +1,121 @@ +import json +import urllib +import urllib2 +import logging + +from dateutil.parser import parse + +from django.conf import settings +from django.db.models.fields import DateTimeField + +from colab.plugins.noosfero.models import (NoosferoArticle, NoosferoCommunity, + NoosferoCategory) from colab.plugins.utils.proxy_data_api import ProxyDataAPI +LOGGER = logging.getLogger('colab.plugin.debug') + class NoosferoDataAPI(ProxyDataAPI): + def get_request_url(self, path, **kwargs): + proxy_config = settings.PROXIED_APPS.get(self.app_label, {}) + + upstream = proxy_config.get('upstream') + kwargs['private_token'] = proxy_config.get('private_token') + params = urllib.urlencode(kwargs) + + if upstream[-1] == '/': + upstream = upstream[:-1] + + return u'{}{}?{}'.format(upstream, path, params) + + def get_json_data(self, api_url, page, pages=1000): + url = self.get_request_url(api_url, per_page=pages, + page=page) + try: + data = urllib2.urlopen(url, timeout=10) + json_data = json.load(data) + except urllib2.URLError: + LOGGER.exception("Connection timeout: " + url) + json_data = [] + + return json_data + + def fill_object_data(self, element, _object): + for field in _object._meta.fields: + try: + if field.name == "user": + _object.update_user( + element["author"]["name"]) + continue + + if field.name == "profile_identifier": + _object.profile_identifier = \ + element["profile"]["identifier"] + continue + + if isinstance(field, DateTimeField): + value = parse(element[field.name]) + else: + value = element[field.name] + + setattr(_object, field.name, value) + except KeyError: + continue + except TypeError: + continue + + return _object + + def fetch_communities(self): + communities = [] + + json_data = self.get_json_data('/api/v1/communities', 1) + + json_data = json_data['communities'] + for element in json_data: + community = NoosferoCommunity() + self.fill_object_data(element, community) + if element.has_key('categories'): + for category_json in element["categories"]: + category = NoosferoCategory.objects.get_or_create( + id=category_json["id"], name=category_json["name"])[0] + community.categories.add(category.id) + communities.append(community) + + return communities + + def fetch_articles(self): + articles = [] + + json_data = self.get_json_data('/api/v1/articles', 1) + + json_data = json_data['articles'] + + for element in json_data: + article = NoosferoArticle() + self.fill_object_data(element, article) + + for category_json in element["categories"]: + category = NoosferoCategory.objects.get_or_create( + id=category_json["id"], name=category_json["name"])[0] + article.categories.add(category.id) + + articles.append(article) + + return articles + def fetch_data(self): - pass + LOGGER.info("Importing Communities") + communities = self.fetch_communities() + for datum in communities: + datum.save() + + LOGGER.info("Importing Articles") + articles = self.fetch_articles() + for datum in articles: + datum.save() + + @property + def app_label(self): + return 'noosfero' diff --git a/colab/plugins/noosfero/migrations/0001_initial.py b/colab/plugins/noosfero/migrations/0001_initial.py new file mode 100644 index 0000000..0ff2b9e --- /dev/null +++ b/colab/plugins/noosfero/migrations/0001_initial.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import django.db.models.deletion +from django.conf import settings + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='NoosferoArticle', + fields=[ + ('id', models.IntegerField(serialize=False, primary_key=True)), + ('title', models.TextField()), + ('path', models.TextField(null=True, blank=True)), + ('body', models.TextField(null=True, blank=True)), + ('profile_identifier', models.TextField()), + ('created_at', models.DateTimeField(blank=True)), + ], + options={ + 'verbose_name': 'Article', + 'verbose_name_plural': 'Articles', + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='NoosferoCategory', + fields=[ + ('id', models.IntegerField(serialize=False, primary_key=True)), + ('name', models.TextField(null=True, blank=True)), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='NoosferoCommunity', + fields=[ + ('id', models.IntegerField(serialize=False, primary_key=True)), + ('name', models.TextField()), + ('identifier', models.TextField()), + ('description', models.TextField(null=True, blank=True)), + ('created_at', models.DateTimeField(blank=True)), + ('categories', models.ManyToManyField(to='noosfero.NoosferoCategory')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True)), + ], + options={ + 'verbose_name': 'Community', + 'verbose_name_plural': 'Communities', + }, + bases=(models.Model,), + ), + migrations.AddField( + model_name='noosferoarticle', + name='categories', + field=models.ManyToManyField(to='noosfero.NoosferoCategory'), + preserve_default=True, + ), + migrations.AddField( + model_name='noosferoarticle', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True), + preserve_default=True, + ), + ] diff --git a/colab/plugins/noosfero/migrations/__init__.py b/colab/plugins/noosfero/migrations/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/colab/plugins/noosfero/migrations/__init__.py diff --git a/colab/plugins/noosfero/models.py b/colab/plugins/noosfero/models.py index e69de29..13b3354 100644 --- a/colab/plugins/noosfero/models.py +++ b/colab/plugins/noosfero/models.py @@ -0,0 +1,68 @@ +from colab.plugins.utils.models import Collaboration +from django.db import models +from django.utils.translation import ugettext_lazy as _ + + +class NoosferoCategory(models.Model): + + id = models.IntegerField(primary_key=True) + name = models.TextField(null=True, blank=True) + + def __unicode__(self): + return "{}-{}".format(self.id, self.name) + + +class NoosferoCommunity(Collaboration): + + id = models.IntegerField(primary_key=True) + type = u'community' + icon_name = u'file' + name = models.TextField() + identifier = models.TextField() + description = models.TextField(null=True, blank=True) + categories = models.ManyToManyField(NoosferoCategory) + created_at = models.DateTimeField(blank=True) + + @property + def url(self): + return u'/social/profile/{}'.format(self.identifier) + + @property + def modified(self): + return self.created_at + + def __unicode__(self): + return "{}({}) - {}".format(self.name, self.identifier, + self.description) + + class Meta: + verbose_name = _('Community') + verbose_name_plural = _('Communities') + + +class NoosferoArticle(Collaboration): + + id = models.IntegerField(primary_key=True) + type = u'article' + icon_name = u'file' + title = models.TextField() + path = models.TextField(null=True, blank=True) + body = models.TextField(null=True, blank=True) + categories = models.ManyToManyField(NoosferoCategory) + profile_identifier = models.TextField() + created_at = models.DateTimeField(blank=True) + + @property + def url(self): + return u'/social/{}/{}'.format(self.profile_identifier, self.path) + + @property + def modified(self): + return self.created_at + + def __unicode__(self): + return "{}({})".format(self.title, self.path) + + class Meta: + verbose_name = _('Article') + verbose_name_plural = _('Articles') -- libgit2 0.21.2