Add support to noosfero data.

Signed-off-by: Thiago Ribeiro <thiagitosouza@hotmail.com> Signed-off-by: Luciano Prestes <lucianopcbr@gmail.com> Signed-off-by: Gustavo Jaruga <darksshades@gmail.com>

Add support to noosfero data.
Signed-off-by: Thiago Ribeiro <thiagitosouza@hotmail.com> Signed-off-by: Luciano Prestes <lucianopcbr@gmail.com> Signed-off-by: Gustavo Jaruga <darksshades@gmail.com>
Thiago Ribeiro · Sergio Oliveira
1 parent 8068038a
Showing 4 changed files with 254 additions and 1 deletions Show diff stats
colab/plugins/noosfero/data_api.py
colab/plugins/noosfero/migrations/0001_initial.py
colab/plugins/noosfero/migrations/__init__.py
colab/plugins/noosfero/models.py
+import json
+import urllib
+import urllib2
+import logging
+
+from dateutil.parser import parse
+
+from django.conf import settings
+from django.db.models.fields import DateTimeField
+
+from colab.plugins.noosfero.models import (NoosferoArticle, NoosferoCommunity,
+                                          NoosferoCategory)
 from colab.plugins.utils.proxy_data_api import ProxyDataAPI
+LOGGER = logging.getLogger('colab.plugin.debug')
+
 class NoosferoDataAPI(ProxyDataAPI):
+    def get_request_url(self, path, **kwargs):
+        proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
+
+        upstream = proxy_config.get('upstream')
+        kwargs['private_token'] = proxy_config.get('private_token')
+        params = urllib.urlencode(kwargs)
+
+        if upstream[-1] == '/':
+            upstream = upstream[:-1]
+
+        return u'{}{}?{}'.format(upstream, path, params)
+
+    def get_json_data(self, api_url, page, pages=1000):
+        url = self.get_request_url(api_url, per_page=pages,
+                                   page=page)
+        try:
+            data = urllib2.urlopen(url, timeout=10)
+            json_data = json.load(data)
+        except urllib2.URLError:
+            LOGGER.exception("Connection timeout: " + url)
+            json_data = []
+
+        return json_data
+
+    def fill_object_data(self, element, _object):
+        for field in _object._meta.fields:
+            try:
+                if field.name == "user":
+                    _object.update_user(
+                        element["author"]["name"])
+                    continue
+
+                if field.name == "profile_identifier":
+                    _object.profile_identifier = \
+                        element["profile"]["identifier"]
+                    continue
+
+                if isinstance(field, DateTimeField):
+                    value = parse(element[field.name])
+                else:
+                    value = element[field.name]
+
+                setattr(_object, field.name, value)
+            except KeyError:
+                continue
+            except TypeError:
+                continue
+
+        return _object
+
+    def fetch_communities(self):
+        communities = []
+
+        json_data = self.get_json_data('/api/v1/communities', 1)
+
+        json_data = json_data['communities']
+        for element in json_data:
+            community = NoosferoCommunity()
+            self.fill_object_data(element, community)
+            if element.has_key('categories'):
+                for category_json in element["categories"]:
+                    category = NoosferoCategory.objects.get_or_create(
+                        id=category_json["id"], name=category_json["name"])[0]
+                    community.categories.add(category.id)
+            communities.append(community)
+
+        return communities
+
+    def fetch_articles(self):
+        articles = []
+
+        json_data = self.get_json_data('/api/v1/articles', 1)
+
+        json_data = json_data['articles']
+
+        for element in json_data:
+            article = NoosferoArticle()
+            self.fill_object_data(element, article)
+
+            for category_json in element["categories"]:
+                category = NoosferoCategory.objects.get_or_create(
+                    id=category_json["id"], name=category_json["name"])[0]
+                article.categories.add(category.id)
+
+            articles.append(article)
+
+        return articles
+
     def fetch_data(self):
-        pass
+        LOGGER.info("Importing Communities")
+        communities = self.fetch_communities()
+        for datum in communities:
+            datum.save()
+
+        LOGGER.info("Importing Articles")
+        articles = self.fetch_articles()
+        for datum in articles:
+            datum.save()
+
+    @property
+    def app_label(self):
+        return 'noosfero'
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import django.db.models.deletion
+from django.conf import settings
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='NoosferoArticle',
+            fields=[
+                ('id', models.IntegerField(serialize=False, primary_key=True)),
+                ('title', models.TextField()),
+                ('path', models.TextField(null=True, blank=True)),
+                ('body', models.TextField(null=True, blank=True)),
+                ('profile_identifier', models.TextField()),
+                ('created_at', models.DateTimeField(blank=True)),
+            ],
+            options={
+                'verbose_name': 'Article',
+                'verbose_name_plural': 'Articles',
+            },
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='NoosferoCategory',
+            fields=[
+                ('id', models.IntegerField(serialize=False, primary_key=True)),
+                ('name', models.TextField(null=True, blank=True)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='NoosferoCommunity',
+            fields=[
+                ('id', models.IntegerField(serialize=False, primary_key=True)),
+                ('name', models.TextField()),
+                ('identifier', models.TextField()),
+                ('description', models.TextField(null=True, blank=True)),
+                ('created_at', models.DateTimeField(blank=True)),
+                ('categories', models.ManyToManyField(to='noosfero.NoosferoCategory')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True)),
+            ],
+            options={
+                'verbose_name': 'Community',
+                'verbose_name_plural': 'Communities',
+            },
+            bases=(models.Model,),
+        ),
+        migrations.AddField(
+            model_name='noosferoarticle',
+            name='categories',
+            field=models.ManyToManyField(to='noosfero.NoosferoCategory'),
+            preserve_default=True,
+        ),
+        migrations.AddField(
+            model_name='noosferoarticle',
+            name='user',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True),
+            preserve_default=True,
+        ),
+    ]
@@ -0,0 +1,68 @@
+from colab.plugins.utils.models import Collaboration
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+
+class NoosferoCategory(models.Model):
+
+    id = models.IntegerField(primary_key=True)
+    name = models.TextField(null=True, blank=True)
+
+    def __unicode__(self):
+        return "{}-{}".format(self.id, self.name)
+
+
+class NoosferoCommunity(Collaboration):
+
+    id = models.IntegerField(primary_key=True)
+    type = u'community'
+    icon_name = u'file'
+    name = models.TextField()
+    identifier = models.TextField()
+    description = models.TextField(null=True, blank=True)
+    categories = models.ManyToManyField(NoosferoCategory)
+    created_at = models.DateTimeField(blank=True)
+
+    @property
+    def url(self):
+        return u'/social/profile/{}'.format(self.identifier)
+
+    @property
+    def modified(self):
+        return self.created_at
+
+    def __unicode__(self):
+        return "{}({}) - {}".format(self.name, self.identifier,
+                                    self.description)
+
+    class Meta:
+        verbose_name = _('Community')
+        verbose_name_plural = _('Communities')
+
+
+class NoosferoArticle(Collaboration):
+
+    id = models.IntegerField(primary_key=True)
+    type = u'article'
+    icon_name = u'file'
+    title = models.TextField()
+    path = models.TextField(null=True, blank=True)
+    body = models.TextField(null=True, blank=True)
+    categories = models.ManyToManyField(NoosferoCategory)
+    profile_identifier = models.TextField()
+    created_at = models.DateTimeField(blank=True)
+
+    @property
+    def url(self):
+        return u'/social/{}/{}'.format(self.profile_identifier, self.path)
+
+    @property
+    def modified(self):
+        return self.created_at
+
+    def __unicode__(self):
+        return "{}({})".format(self.title, self.path)
+
+    class Meta:
+        verbose_name = _('Article')
+        verbose_name_plural = _('Articles')
		1	+import json
		2	+import urllib
		3	+import urllib2
		4	+import logging
		5	+
		6	+from dateutil.parser import parse
		7	+
		8	+from django.conf import settings
		9	+from django.db.models.fields import DateTimeField
		10	+
		11	+from colab.plugins.noosfero.models import (NoosferoArticle, NoosferoCommunity,
		12	+ NoosferoCategory)
1	from colab.plugins.utils.proxy_data_api import ProxyDataAPI	13	from colab.plugins.utils.proxy_data_api import ProxyDataAPI
2		14
		15	+LOGGER = logging.getLogger('colab.plugin.debug')
		16	+
3		17
4	class NoosferoDataAPI(ProxyDataAPI):	18	class NoosferoDataAPI(ProxyDataAPI):
5		19
		20	+ def get_request_url(self, path, **kwargs):
		21	+ proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
		22	+
		23	+ upstream = proxy_config.get('upstream')
		24	+ kwargs['private_token'] = proxy_config.get('private_token')
		25	+ params = urllib.urlencode(kwargs)
		26	+
		27	+ if upstream[-1] == '/':
		28	+ upstream = upstream[:-1]
		29	+
		30	+ return u'{}{}?{}'.format(upstream, path, params)
		31	+
		32	+ def get_json_data(self, api_url, page, pages=1000):
		33	+ url = self.get_request_url(api_url, per_page=pages,
		34	+ page=page)
		35	+ try:
		36	+ data = urllib2.urlopen(url, timeout=10)
		37	+ json_data = json.load(data)
		38	+ except urllib2.URLError:
		39	+ LOGGER.exception("Connection timeout: " + url)
		40	+ json_data = []
		41	+
		42	+ return json_data
		43	+
		44	+ def fill_object_data(self, element, _object):
		45	+ for field in _object._meta.fields:
		46	+ try:
		47	+ if field.name == "user":
		48	+ _object.update_user(
		49	+ element["author"]["name"])
		50	+ continue
		51	+
		52	+ if field.name == "profile_identifier":
		53	+ _object.profile_identifier = \
		54	+ element["profile"]["identifier"]
		55	+ continue
		56	+
		57	+ if isinstance(field, DateTimeField):
		58	+ value = parse(element[field.name])
		59	+ else:
		60	+ value = element[field.name]
		61	+
		62	+ setattr(_object, field.name, value)
		63	+ except KeyError:
		64	+ continue
		65	+ except TypeError:
		66	+ continue
		67	+
		68	+ return _object
		69	+
		70	+ def fetch_communities(self):
		71	+ communities = []
		72	+
		73	+ json_data = self.get_json_data('/api/v1/communities', 1)
		74	+
		75	+ json_data = json_data['communities']
		76	+ for element in json_data:
		77	+ community = NoosferoCommunity()
		78	+ self.fill_object_data(element, community)
		79	+ if element.has_key('categories'):
		80	+ for category_json in element["categories"]:
		81	+ category = NoosferoCategory.objects.get_or_create(
		82	+ id=category_json["id"], name=category_json["name"])[0]
		83	+ community.categories.add(category.id)
		84	+ communities.append(community)
		85	+
		86	+ return communities
		87	+
		88	+ def fetch_articles(self):
		89	+ articles = []
		90	+
		91	+ json_data = self.get_json_data('/api/v1/articles', 1)
		92	+
		93	+ json_data = json_data['articles']
		94	+
		95	+ for element in json_data:
		96	+ article = NoosferoArticle()
		97	+ self.fill_object_data(element, article)
		98	+
		99	+ for category_json in element["categories"]:
		100	+ category = NoosferoCategory.objects.get_or_create(
		101	+ id=category_json["id"], name=category_json["name"])[0]
		102	+ article.categories.add(category.id)
		103	+
		104	+ articles.append(article)
		105	+
		106	+ return articles
		107	+
6	def fetch_data(self):	108	def fetch_data(self):
7	- pass	109	+ LOGGER.info("Importing Communities")
		110	+ communities = self.fetch_communities()
		111	+ for datum in communities:
		112	+ datum.save()
		113	+
		114	+ LOGGER.info("Importing Articles")
		115	+ articles = self.fetch_articles()
		116	+ for datum in articles:
		117	+ datum.save()
		118	+
		119	+ @property
		120	+ def app_label(self):
		121	+ return 'noosfero'
@@ -0,0 +1,71 @@		@@ -0,0 +1,71 @@
	1	+# -- coding: utf-8 --
	2	+from __future__ import unicode_literals
	3	+
	4	+from django.db import models, migrations
	5	+import django.db.models.deletion
	6	+from django.conf import settings
	7	+
	8	+
	9	+class Migration(migrations.Migration):
	10	+
	11	+ dependencies = [
	12	+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
	13	+ ]
	14	+
	15	+ operations = [
	16	+ migrations.CreateModel(
	17	+ name='NoosferoArticle',
	18	+ fields=[
	19	+ ('id', models.IntegerField(serialize=False, primary_key=True)),
	20	+ ('title', models.TextField()),
	21	+ ('path', models.TextField(null=True, blank=True)),
	22	+ ('body', models.TextField(null=True, blank=True)),
	23	+ ('profile_identifier', models.TextField()),
	24	+ ('created_at', models.DateTimeField(blank=True)),
	25	+ ],
	26	+ options={
	27	+ 'verbose_name': 'Article',
	28	+ 'verbose_name_plural': 'Articles',
	29	+ },
	30	+ bases=(models.Model,),
	31	+ ),
	32	+ migrations.CreateModel(
	33	+ name='NoosferoCategory',
	34	+ fields=[
	35	+ ('id', models.IntegerField(serialize=False, primary_key=True)),
	36	+ ('name', models.TextField(null=True, blank=True)),
	37	+ ],
	38	+ options={
	39	+ },
	40	+ bases=(models.Model,),
	41	+ ),
	42	+ migrations.CreateModel(
	43	+ name='NoosferoCommunity',
	44	+ fields=[
	45	+ ('id', models.IntegerField(serialize=False, primary_key=True)),
	46	+ ('name', models.TextField()),
	47	+ ('identifier', models.TextField()),
	48	+ ('description', models.TextField(null=True, blank=True)),
	49	+ ('created_at', models.DateTimeField(blank=True)),
	50	+ ('categories', models.ManyToManyField(to='noosfero.NoosferoCategory')),
	51	+ ('user', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True)),
	52	+ ],
	53	+ options={
	54	+ 'verbose_name': 'Community',
	55	+ 'verbose_name_plural': 'Communities',
	56	+ },
	57	+ bases=(models.Model,),
	58	+ ),
	59	+ migrations.AddField(
	60	+ model_name='noosferoarticle',
	61	+ name='categories',
	62	+ field=models.ManyToManyField(to='noosfero.NoosferoCategory'),
	63	+ preserve_default=True,
	64	+ ),
	65	+ migrations.AddField(
	66	+ model_name='noosferoarticle',
	67	+ name='user',
	68	+ field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True),
	69	+ preserve_default=True,
	70	+ ),
	71	+ ]
@@ -0,0 +1,68 @@		@@ -0,0 +1,68 @@
	1	+from colab.plugins.utils.models import Collaboration
	2	+from django.db import models
	3	+from django.utils.translation import ugettext_lazy as _
	4	+
	5	+
	6	+class NoosferoCategory(models.Model):
	7	+
	8	+ id = models.IntegerField(primary_key=True)
	9	+ name = models.TextField(null=True, blank=True)
	10	+
	11	+ def __unicode__(self):
	12	+ return "{}-{}".format(self.id, self.name)
	13	+
	14	+
	15	+class NoosferoCommunity(Collaboration):
	16	+
	17	+ id = models.IntegerField(primary_key=True)
	18	+ type = u'community'
	19	+ icon_name = u'file'
	20	+ name = models.TextField()
	21	+ identifier = models.TextField()
	22	+ description = models.TextField(null=True, blank=True)
	23	+ categories = models.ManyToManyField(NoosferoCategory)
	24	+ created_at = models.DateTimeField(blank=True)
	25	+
	26	+ @property
	27	+ def url(self):
	28	+ return u'/social/profile/{}'.format(self.identifier)
	29	+
	30	+ @property
	31	+ def modified(self):
	32	+ return self.created_at
	33	+
	34	+ def __unicode__(self):
	35	+ return "{}({}) - {}".format(self.name, self.identifier,
	36	+ self.description)
	37	+
	38	+ class Meta:
	39	+ verbose_name = _('Community')
	40	+ verbose_name_plural = _('Communities')
	41	+
	42	+
	43	+class NoosferoArticle(Collaboration):
	44	+
	45	+ id = models.IntegerField(primary_key=True)
	46	+ type = u'article'
	47	+ icon_name = u'file'
	48	+ title = models.TextField()
	49	+ path = models.TextField(null=True, blank=True)
	50	+ body = models.TextField(null=True, blank=True)
	51	+ categories = models.ManyToManyField(NoosferoCategory)
	52	+ profile_identifier = models.TextField()
	53	+ created_at = models.DateTimeField(blank=True)
	54	+
	55	+ @property
	56	+ def url(self):
	57	+ return u'/social/{}/{}'.format(self.profile_identifier, self.path)
	58	+
	59	+ @property
	60	+ def modified(self):
	61	+ return self.created_at
	62	+
	63	+ def __unicode__(self):
	64	+ return "{}({})".format(self.title, self.path)
	65	+
	66	+ class Meta:
	67	+ verbose_name = _('Article')
	68	+ verbose_name_plural = _('Articles')