Merge branch 'noosfero_data' into 'master'

Noosfero data Index noosfero data to colab See merge request !85

Merge branch 'noosfero_data' into 'master'
Noosfero data Index noosfero data to colab See merge request !85
Sergio Oliveira
2 parents 8068038a 1d7c2fa4
Showing 7 changed files with 320 additions and 1 deletions Show diff stats
colab/plugins/noosfero/data_api.py
colab/plugins/noosfero/migrations/0001_initial.py
colab/plugins/noosfero/migrations/__init__.py
colab/plugins/noosfero/models.py
colab/plugins/noosfero/search_indexes.py
colab/plugins/noosfero/templates/search/indexes/noosfero/noosferoarticle_text.txt
colab/plugins/noosfero/templates/search/indexes/noosfero/noosferocommunity_text.txt
+import json
+import urllib
+import urllib2
+import logging
+
+from dateutil.parser import parse
+
+from django.conf import settings
+from django.db.models.fields import DateTimeField
+
+from colab.plugins.noosfero.models import (NoosferoArticle, NoosferoCommunity,
+                                           NoosferoCategory)
 from colab.plugins.utils.proxy_data_api import ProxyDataAPI
+LOGGER = logging.getLogger('colab.plugin.debug')
+
 class NoosferoDataAPI(ProxyDataAPI):
+    def get_request_url(self, path, **kwargs):
+        proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
+
+        upstream = proxy_config.get('upstream')
+        kwargs['private_token'] = proxy_config.get('private_token')
+        params = urllib.urlencode(kwargs)
+
+        if upstream[-1] == '/':
+            upstream = upstream[:-1]
+
+        return u'{}{}?{}'.format(upstream, path, params)
+
+    def get_json_data(self, api_url, page, pages=1000):
+        url = self.get_request_url(api_url, per_page=pages,
+                                   page=page)
+        try:
+            data = urllib2.urlopen(url, timeout=10)
+            json_data = json.load(data)
+        except urllib2.URLError:
+            LOGGER.exception("Connection timeout: " + url)
+            json_data = []
+
+        return json_data
+
+    def fill_object_data(self, element, _object):
+        for field in _object._meta.fields:
+            try:
+                if field.name == "user":
+                    _object.update_user(
+                        element["author"]["name"])
+                    continue
+
+                if field.name == "profile_identifier":
+                    _object.profile_identifier = \
+                        element["profile"]["identifier"]
+                    continue
+
+                if isinstance(field, DateTimeField):
+                    value = parse(element[field.name])
+                else:
+                    value = element[field.name]
+
+                setattr(_object, field.name, value)
+            except KeyError:
+                continue
+            except TypeError:
+                continue
+
+        return _object
+
+    def fetch_communities(self):
+        json_data = self.get_json_data('/api/v1/communities', 1)
+
+        json_data = json_data['communities']
+        for element in json_data:
+            community = NoosferoCommunity()
+            self.fill_object_data(element, community)
+            community.save()
+
+            if 'categories' in element:
+                for category_json in element["categories"]:
+                    category = NoosferoCategory.objects.get_or_create(
+                        id=category_json["id"], name=category_json["name"])[0]
+                    community.categories.add(category.id)
+
+    def fetch_articles(self):
+        json_data = self.get_json_data('/api/v1/articles', 1)
+
+        json_data = json_data['articles']
+
+        for element in json_data:
+            article = NoosferoArticle()
+            self.fill_object_data(element, article)
+            article.save()
+
+            for category_json in element["categories"]:
+                category = NoosferoCategory.objects.get_or_create(
+                    id=category_json["id"], name=category_json["name"])[0]
+                article.categories.add(category.id)
+
     def fetch_data(self):
-        pass
+        LOGGER.info("Importing Communities")
+        self.fetch_communities()
+
+        LOGGER.info("Importing Articles")
+        self.fetch_articles()
+
+    @property
+    def app_label(self):
+        return 'noosfero'
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import django.db.models.deletion
+from django.conf import settings
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='NoosferoArticle',
+            fields=[
+                ('id', models.IntegerField(serialize=False, primary_key=True)),
+                ('title', models.TextField()),
+                ('path', models.TextField(null=True, blank=True)),
+                ('body', models.TextField(null=True, blank=True)),
+                ('profile_identifier', models.TextField()),
+                ('created_at', models.DateTimeField(blank=True)),
+            ],
+            options={
+                'verbose_name': 'Article',
+                'verbose_name_plural': 'Articles',
+            },
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='NoosferoCategory',
+            fields=[
+                ('id', models.IntegerField(serialize=False, primary_key=True)),
+                ('name', models.TextField(null=True, blank=True)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='NoosferoCommunity',
+            fields=[
+                ('id', models.IntegerField(serialize=False, primary_key=True)),
+                ('name', models.TextField()),
+                ('identifier', models.TextField()),
+                ('description', models.TextField(null=True, blank=True)),
+                ('created_at', models.DateTimeField(blank=True)),
+                ('categories', models.ManyToManyField(to='noosfero.NoosferoCategory')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True)),
+            ],
+            options={
+                'verbose_name': 'Community',
+                'verbose_name_plural': 'Communities',
+            },
+            bases=(models.Model,),
+        ),
+        migrations.AddField(
+            model_name='noosferoarticle',
+            name='categories',
+            field=models.ManyToManyField(to='noosfero.NoosferoCategory'),
+            preserve_default=True,
+        ),
+        migrations.AddField(
+            model_name='noosferoarticle',
+            name='user',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True),
+            preserve_default=True,
+        ),
+    ]
@@ -0,0 +1,68 @@
+from colab.plugins.utils.models import Collaboration
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+
+class NoosferoCategory(models.Model):
+
+    id = models.IntegerField(primary_key=True)
+    name = models.TextField(null=True, blank=True)
+
+    def __unicode__(self):
+        return "{}-{}".format(self.id, self.name)
+
+
+class NoosferoCommunity(Collaboration):
+
+    id = models.IntegerField(primary_key=True)
+    type = u'community'
+    icon_name = u'file'
+    name = models.TextField()
+    identifier = models.TextField()
+    description = models.TextField(null=True, blank=True)
+    categories = models.ManyToManyField(NoosferoCategory)
+    created_at = models.DateTimeField(blank=True)
+
+    @property
+    def url(self):
+        return u'/social/profile/{}'.format(self.identifier)
+
+    @property
+    def modified(self):
+        return self.created_at
+
+    def __unicode__(self):
+        return "{}({}) - {}".format(self.name, self.identifier,
+                                    self.description)
+
+    class Meta:
+        verbose_name = _('Community')
+        verbose_name_plural = _('Communities')
+
+
+class NoosferoArticle(Collaboration):
+
+    id = models.IntegerField(primary_key=True)
+    type = u'article'
+    icon_name = u'file'
+    title = models.TextField()
+    path = models.TextField(null=True, blank=True)
+    body = models.TextField(null=True, blank=True)
+    categories = models.ManyToManyField(NoosferoCategory)
+    profile_identifier = models.TextField()
+    created_at = models.DateTimeField(blank=True)
+
+    @property
+    def url(self):
+        return u'/social/{}/{}'.format(self.profile_identifier, self.path)
+
+    @property
+    def modified(self):
+        return self.created_at
+
+    def __unicode__(self):
+        return "{}({})".format(self.title, self.path)
+
+    class Meta:
+        verbose_name = _('Article')
+        verbose_name_plural = _('Articles')
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+
+import string
+
+from haystack import indexes
+from haystack.utils import log as logging
+
+from .models import (NoosferoArticle, NoosferoCommunity)
+
+
+logger = logging.getLogger('haystack')
+
+# The string maketrans always return a string encoded with latin1
+# http://stackoverflow.com/questions/1324067/how-do-i-get-str-translate-to-work-with-unicode-strings
+table = string.maketrans(
+    string.punctuation,
+    '.' * len(string.punctuation)
+).decode('latin1')
+
+
+class NoosferoCommunityIndex(indexes.SearchIndex, indexes.Indexable):
+    text = indexes.CharField(document=True, use_template=True, stored=False)
+    title = indexes.CharField(model_attr='name')
+    description = indexes.CharField(model_attr='description', null=True)
+    url = indexes.CharField(model_attr='url', indexed=False)
+    icon_name = indexes.CharField()
+    type = indexes.CharField()
+    modified = indexes.DateTimeField(model_attr='modified', null=True)
+    created_at = indexes.DateTimeField(model_attr='created_at', null=True)
+    category = indexes.MultiValueField()
+
+    def prepare_category(self, obj):
+        return obj.categories.values_list('name', flat=True)
+
+    def prepare_icon_name(self, obj):
+        return u'file'
+
+    def get_ful_name(self):
+        self.objs.name
+
+    def get_model(self):
+        return NoosferoCommunity
+
+    def prepare_type(self, obj):
+        return u'noosfero_community'
+
+
+class NoosferoArticleIndex(indexes.SearchIndex, indexes.Indexable):
+
+    text = indexes.CharField(document=True, use_template=True, stored=False)
+    title = indexes.CharField(model_attr='title')
+    body = indexes.CharField(model_attr='body', null=True)
+    url = indexes.CharField(model_attr='url', indexed=False)
+    icon_name = indexes.CharField()
+    type = indexes.CharField(model_attr='type')
+    modified = indexes.DateTimeField(model_attr='modified', null=True)
+    created_at = indexes.DateTimeField(model_attr='created_at', null=True)
+    category = indexes.MultiValueField()
+
+    def get_model(self):
+        return NoosferoArticle
+
+    def prepare_category(self, obj):
+        return obj.categories.values_list('name', flat=True)
+
+    def prepare_icon_name(self, obj):
+        return u'file'
+
+    def prepare_type(self, obj):
+        return u'noosfero_articles'
@@ -0,0 +1,4 @@
+{{ object.title }}
+{{ object.body }}
+{{ object.url }}
+{{ object.category }}
@@ -0,0 +1,4 @@
+{{ object.name }}
+{{ object.description }}
+{{ object.url }}
+{{ object.category }}
		1	+import json
		2	+import urllib
		3	+import urllib2
		4	+import logging
		5	+
		6	+from dateutil.parser import parse
		7	+
		8	+from django.conf import settings
		9	+from django.db.models.fields import DateTimeField
		10	+
		11	+from colab.plugins.noosfero.models import (NoosferoArticle, NoosferoCommunity,
		12	+ NoosferoCategory)
1	from colab.plugins.utils.proxy_data_api import ProxyDataAPI	13	from colab.plugins.utils.proxy_data_api import ProxyDataAPI
2		14
		15	+LOGGER = logging.getLogger('colab.plugin.debug')
		16	+
3		17
4	class NoosferoDataAPI(ProxyDataAPI):	18	class NoosferoDataAPI(ProxyDataAPI):
5		19
		20	+ def get_request_url(self, path, **kwargs):
		21	+ proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
		22	+
		23	+ upstream = proxy_config.get('upstream')
		24	+ kwargs['private_token'] = proxy_config.get('private_token')
		25	+ params = urllib.urlencode(kwargs)
		26	+
		27	+ if upstream[-1] == '/':
		28	+ upstream = upstream[:-1]
		29	+
		30	+ return u'{}{}?{}'.format(upstream, path, params)
		31	+
		32	+ def get_json_data(self, api_url, page, pages=1000):
		33	+ url = self.get_request_url(api_url, per_page=pages,
		34	+ page=page)
		35	+ try:
		36	+ data = urllib2.urlopen(url, timeout=10)
		37	+ json_data = json.load(data)
		38	+ except urllib2.URLError:
		39	+ LOGGER.exception("Connection timeout: " + url)
		40	+ json_data = []
		41	+
		42	+ return json_data
		43	+
		44	+ def fill_object_data(self, element, _object):
		45	+ for field in _object._meta.fields:
		46	+ try:
		47	+ if field.name == "user":
		48	+ _object.update_user(
		49	+ element["author"]["name"])
		50	+ continue
		51	+
		52	+ if field.name == "profile_identifier":
		53	+ _object.profile_identifier = \
		54	+ element["profile"]["identifier"]
		55	+ continue
		56	+
		57	+ if isinstance(field, DateTimeField):
		58	+ value = parse(element[field.name])
		59	+ else:
		60	+ value = element[field.name]
		61	+
		62	+ setattr(_object, field.name, value)
		63	+ except KeyError:
		64	+ continue
		65	+ except TypeError:
		66	+ continue
		67	+
		68	+ return _object
		69	+
		70	+ def fetch_communities(self):
		71	+ json_data = self.get_json_data('/api/v1/communities', 1)
		72	+
		73	+ json_data = json_data['communities']
		74	+ for element in json_data:
		75	+ community = NoosferoCommunity()
		76	+ self.fill_object_data(element, community)
		77	+ community.save()
		78	+
		79	+ if 'categories' in element:
		80	+ for category_json in element["categories"]:
		81	+ category = NoosferoCategory.objects.get_or_create(
		82	+ id=category_json["id"], name=category_json["name"])[0]
		83	+ community.categories.add(category.id)
		84	+
		85	+ def fetch_articles(self):
		86	+ json_data = self.get_json_data('/api/v1/articles', 1)
		87	+
		88	+ json_data = json_data['articles']
		89	+
		90	+ for element in json_data:
		91	+ article = NoosferoArticle()
		92	+ self.fill_object_data(element, article)
		93	+ article.save()
		94	+
		95	+ for category_json in element["categories"]:
		96	+ category = NoosferoCategory.objects.get_or_create(
		97	+ id=category_json["id"], name=category_json["name"])[0]
		98	+ article.categories.add(category.id)
		99	+
6	def fetch_data(self):	100	def fetch_data(self):
7	- pass	101	+ LOGGER.info("Importing Communities")
		102	+ self.fetch_communities()
		103	+
		104	+ LOGGER.info("Importing Articles")
		105	+ self.fetch_articles()
		106	+
		107	+ @property
		108	+ def app_label(self):
		109	+ return 'noosfero'
@@ -0,0 +1,71 @@		@@ -0,0 +1,71 @@
	1	+# -- coding: utf-8 --
	2	+from __future__ import unicode_literals
	3	+
	4	+from django.db import models, migrations
	5	+import django.db.models.deletion
	6	+from django.conf import settings
	7	+
	8	+
	9	+class Migration(migrations.Migration):
	10	+
	11	+ dependencies = [
	12	+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
	13	+ ]
	14	+
	15	+ operations = [
	16	+ migrations.CreateModel(
	17	+ name='NoosferoArticle',
	18	+ fields=[
	19	+ ('id', models.IntegerField(serialize=False, primary_key=True)),
	20	+ ('title', models.TextField()),
	21	+ ('path', models.TextField(null=True, blank=True)),
	22	+ ('body', models.TextField(null=True, blank=True)),
	23	+ ('profile_identifier', models.TextField()),
	24	+ ('created_at', models.DateTimeField(blank=True)),
	25	+ ],
	26	+ options={
	27	+ 'verbose_name': 'Article',
	28	+ 'verbose_name_plural': 'Articles',
	29	+ },
	30	+ bases=(models.Model,),
	31	+ ),
	32	+ migrations.CreateModel(
	33	+ name='NoosferoCategory',
	34	+ fields=[
	35	+ ('id', models.IntegerField(serialize=False, primary_key=True)),
	36	+ ('name', models.TextField(null=True, blank=True)),
	37	+ ],
	38	+ options={
	39	+ },
	40	+ bases=(models.Model,),
	41	+ ),
	42	+ migrations.CreateModel(
	43	+ name='NoosferoCommunity',
	44	+ fields=[
	45	+ ('id', models.IntegerField(serialize=False, primary_key=True)),
	46	+ ('name', models.TextField()),
	47	+ ('identifier', models.TextField()),
	48	+ ('description', models.TextField(null=True, blank=True)),
	49	+ ('created_at', models.DateTimeField(blank=True)),
	50	+ ('categories', models.ManyToManyField(to='noosfero.NoosferoCategory')),
	51	+ ('user', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True)),
	52	+ ],
	53	+ options={
	54	+ 'verbose_name': 'Community',
	55	+ 'verbose_name_plural': 'Communities',
	56	+ },
	57	+ bases=(models.Model,),
	58	+ ),
	59	+ migrations.AddField(
	60	+ model_name='noosferoarticle',
	61	+ name='categories',
	62	+ field=models.ManyToManyField(to='noosfero.NoosferoCategory'),
	63	+ preserve_default=True,
	64	+ ),
	65	+ migrations.AddField(
	66	+ model_name='noosferoarticle',
	67	+ name='user',
	68	+ field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True),
	69	+ preserve_default=True,
	70	+ ),
	71	+ ]
@@ -0,0 +1,68 @@		@@ -0,0 +1,68 @@
	1	+from colab.plugins.utils.models import Collaboration
	2	+from django.db import models
	3	+from django.utils.translation import ugettext_lazy as _
	4	+
	5	+
	6	+class NoosferoCategory(models.Model):
	7	+
	8	+ id = models.IntegerField(primary_key=True)
	9	+ name = models.TextField(null=True, blank=True)
	10	+
	11	+ def __unicode__(self):
	12	+ return "{}-{}".format(self.id, self.name)
	13	+
	14	+
	15	+class NoosferoCommunity(Collaboration):
	16	+
	17	+ id = models.IntegerField(primary_key=True)
	18	+ type = u'community'
	19	+ icon_name = u'file'
	20	+ name = models.TextField()
	21	+ identifier = models.TextField()
	22	+ description = models.TextField(null=True, blank=True)
	23	+ categories = models.ManyToManyField(NoosferoCategory)
	24	+ created_at = models.DateTimeField(blank=True)
	25	+
	26	+ @property
	27	+ def url(self):
	28	+ return u'/social/profile/{}'.format(self.identifier)
	29	+
	30	+ @property
	31	+ def modified(self):
	32	+ return self.created_at
	33	+
	34	+ def __unicode__(self):
	35	+ return "{}({}) - {}".format(self.name, self.identifier,
	36	+ self.description)
	37	+
	38	+ class Meta:
	39	+ verbose_name = _('Community')
	40	+ verbose_name_plural = _('Communities')
	41	+
	42	+
	43	+class NoosferoArticle(Collaboration):
	44	+
	45	+ id = models.IntegerField(primary_key=True)
	46	+ type = u'article'
	47	+ icon_name = u'file'
	48	+ title = models.TextField()
	49	+ path = models.TextField(null=True, blank=True)
	50	+ body = models.TextField(null=True, blank=True)
	51	+ categories = models.ManyToManyField(NoosferoCategory)
	52	+ profile_identifier = models.TextField()
	53	+ created_at = models.DateTimeField(blank=True)
	54	+
	55	+ @property
	56	+ def url(self):
	57	+ return u'/social/{}/{}'.format(self.profile_identifier, self.path)
	58	+
	59	+ @property
	60	+ def modified(self):
	61	+ return self.created_at
	62	+
	63	+ def __unicode__(self):
	64	+ return "{}({})".format(self.title, self.path)
	65	+
	66	+ class Meta:
	67	+ verbose_name = _('Article')
	68	+ verbose_name_plural = _('Articles')
@@ -0,0 +1,70 @@		@@ -0,0 +1,70 @@
	1	+# -- coding: utf-8 --
	2	+
	3	+import string
	4	+
	5	+from haystack import indexes
	6	+from haystack.utils import log as logging
	7	+
	8	+from .models import (NoosferoArticle, NoosferoCommunity)
	9	+
	10	+
	11	+logger = logging.getLogger('haystack')
	12	+
	13	+# The string maketrans always return a string encoded with latin1
	14	+# http://stackoverflow.com/questions/1324067/how-do-i-get-str-translate-to-work-with-unicode-strings
	15	+table = string.maketrans(
	16	+ string.punctuation,
	17	+ '.' * len(string.punctuation)
	18	+).decode('latin1')
	19	+
	20	+
	21	+class NoosferoCommunityIndex(indexes.SearchIndex, indexes.Indexable):
	22	+ text = indexes.CharField(document=True, use_template=True, stored=False)
	23	+ title = indexes.CharField(model_attr='name')
	24	+ description = indexes.CharField(model_attr='description', null=True)
	25	+ url = indexes.CharField(model_attr='url', indexed=False)
	26	+ icon_name = indexes.CharField()
	27	+ type = indexes.CharField()
	28	+ modified = indexes.DateTimeField(model_attr='modified', null=True)
	29	+ created_at = indexes.DateTimeField(model_attr='created_at', null=True)
	30	+ category = indexes.MultiValueField()
	31	+
	32	+ def prepare_category(self, obj):
	33	+ return obj.categories.values_list('name', flat=True)
	34	+
	35	+ def prepare_icon_name(self, obj):
	36	+ return u'file'
	37	+
	38	+ def get_ful_name(self):
	39	+ self.objs.name
	40	+
	41	+ def get_model(self):
	42	+ return NoosferoCommunity
	43	+
	44	+ def prepare_type(self, obj):
	45	+ return u'noosfero_community'
	46	+
	47	+
	48	+class NoosferoArticleIndex(indexes.SearchIndex, indexes.Indexable):
	49	+
	50	+ text = indexes.CharField(document=True, use_template=True, stored=False)
	51	+ title = indexes.CharField(model_attr='title')
	52	+ body = indexes.CharField(model_attr='body', null=True)
	53	+ url = indexes.CharField(model_attr='url', indexed=False)
	54	+ icon_name = indexes.CharField()
	55	+ type = indexes.CharField(model_attr='type')
	56	+ modified = indexes.DateTimeField(model_attr='modified', null=True)
	57	+ created_at = indexes.DateTimeField(model_attr='created_at', null=True)
	58	+ category = indexes.MultiValueField()
	59	+
	60	+ def get_model(self):
	61	+ return NoosferoArticle
	62	+
	63	+ def prepare_category(self, obj):
	64	+ return obj.categories.values_list('name', flat=True)
	65	+
	66	+ def prepare_icon_name(self, obj):
	67	+ return u'file'
	68	+
	69	+ def prepare_type(self, obj):
	70	+ return u'noosfero_articles'
@@ -0,0 +1,4 @@		@@ -0,0 +1,4 @@
	1	+{{ object.title }}
	2	+{{ object.body }}
	3	+{{ object.url }}
	4	+{{ object.category }}
@@ -0,0 +1,4 @@		@@ -0,0 +1,4 @@
	1	+{{ object.name }}
	2	+{{ object.description }}
	3	+{{ object.url }}
	4	+{{ object.category }}