Merge Request #85
← To merge requests
From
noosfero_data
into
master
Commits (4)
-
- The models was saved in fetch_data method, but now, the models are saved in fetch_communities and fetch_articles methods Signed-off-by: Luciano Prestes Cavalcanti <lucianopcbr@gmail.com> Signed-off-by: Thiago Ribeiro <thiagitosouza@hotmail.com>
-
Add noosfero atributtes to serach_indexes.py Signed-off-by: Thiago Ribeiro <thiagitosouza@hotmail.com> Signed-off-by: Luciano Prestes <lucianopcbr@gmail.com> Signed-off-by: Gustavo Jaruga <darksshades@gmail.com>
-
Signed-off-by: Thiago Ribeiro <thiagitosouza@hotmail.com> Signed-off-by: Luciano Prestes <lucianopcbr@gmail.com> Signed-off-by: Gustavo Jaruga <darksshades@gmail.com>
2 participants
Showing
7 changed files
Show diff stats
colab/plugins/noosfero/data_api.py
| 1 | +import json | ||
| 2 | +import urllib | ||
| 3 | +import urllib2 | ||
| 4 | +import logging | ||
| 5 | + | ||
| 6 | +from dateutil.parser import parse | ||
| 7 | + | ||
| 8 | +from django.conf import settings | ||
| 9 | +from django.db.models.fields import DateTimeField | ||
| 10 | + | ||
| 11 | +from colab.plugins.noosfero.models import (NoosferoArticle, NoosferoCommunity, | ||
| 12 | + NoosferoCategory) | ||
| 1 | from colab.plugins.utils.proxy_data_api import ProxyDataAPI | 13 | from colab.plugins.utils.proxy_data_api import ProxyDataAPI |
| 2 | 14 | ||
| 15 | +LOGGER = logging.getLogger('colab.plugin.debug') | ||
| 16 | + | ||
| 3 | 17 | ||
| 4 | class NoosferoDataAPI(ProxyDataAPI): | 18 | class NoosferoDataAPI(ProxyDataAPI): |
| 5 | 19 | ||
| 20 | + def get_request_url(self, path, **kwargs): | ||
| 21 | + proxy_config = settings.PROXIED_APPS.get(self.app_label, {}) | ||
| 22 | + | ||
| 23 | + upstream = proxy_config.get('upstream') | ||
| 24 | + kwargs['private_token'] = proxy_config.get('private_token') | ||
| 25 | + params = urllib.urlencode(kwargs) | ||
| 26 | + | ||
| 27 | + if upstream[-1] == '/': | ||
| 28 | + upstream = upstream[:-1] | ||
| 29 | + | ||
| 30 | + return u'{}{}?{}'.format(upstream, path, params) | ||
| 31 | + | ||
| 32 | + def get_json_data(self, api_url, page, pages=1000): | ||
| 33 | + url = self.get_request_url(api_url, per_page=pages, | ||
| 34 | + page=page) | ||
| 35 | + try: | ||
| 36 | + data = urllib2.urlopen(url, timeout=10) | ||
| 37 | + json_data = json.load(data) | ||
| 38 | + except urllib2.URLError: | ||
| 39 | + LOGGER.exception("Connection timeout: " + url) | ||
| 40 | + json_data = [] | ||
| 41 | + | ||
| 42 | + return json_data | ||
| 43 | + | ||
| 44 | + def fill_object_data(self, element, _object): | ||
| 45 | + for field in _object._meta.fields: | ||
| 46 | + try: | ||
| 47 | + if field.name == "user": | ||
| 48 | + _object.update_user( | ||
| 49 | + element["author"]["name"]) | ||
| 50 | + continue | ||
| 51 | + | ||
| 52 | + if field.name == "profile_identifier": | ||
| 53 | + _object.profile_identifier = \ | ||
| 54 | + element["profile"]["identifier"] | ||
| 55 | + continue | ||
| 56 | + | ||
| 57 | + if isinstance(field, DateTimeField): | ||
| 58 | + value = parse(element[field.name]) | ||
| 59 | + else: | ||
| 60 | + value = element[field.name] | ||
| 61 | + | ||
| 62 | + setattr(_object, field.name, value) | ||
| 63 | + except KeyError: | ||
| 64 | + continue | ||
| 65 | + except TypeError: | ||
| 66 | + continue | ||
| 67 | + | ||
| 68 | + return _object | ||
| 69 | + | ||
| 70 | + def fetch_communities(self): | ||
| 71 | + json_data = self.get_json_data('/api/v1/communities', 1) | ||
| 72 | + | ||
| 73 | + json_data = json_data['communities'] | ||
| 74 | + for element in json_data: | ||
| 75 | + community = NoosferoCommunity() | ||
| 76 | + self.fill_object_data(element, community) | ||
| 77 | + community.save() | ||
| 78 | + | ||
| 79 | + if 'categories' in element: | ||
| 80 | + for category_json in element["categories"]: | ||
| 81 | + category = NoosferoCategory.objects.get_or_create( | ||
| 82 | + id=category_json["id"], name=category_json["name"])[0] | ||
| 83 | + community.categories.add(category.id) | ||
| 84 | + | ||
| 85 | + def fetch_articles(self): | ||
| 86 | + json_data = self.get_json_data('/api/v1/articles', 1) | ||
| 87 | + | ||
| 88 | + json_data = json_data['articles'] | ||
| 89 | + | ||
| 90 | + for element in json_data: | ||
| 91 | + article = NoosferoArticle() | ||
| 92 | + self.fill_object_data(element, article) | ||
| 93 | + article.save() | ||
| 94 | + | ||
| 95 | + for category_json in element["categories"]: | ||
| 96 | + category = NoosferoCategory.objects.get_or_create( | ||
| 97 | + id=category_json["id"], name=category_json["name"])[0] | ||
| 98 | + article.categories.add(category.id) | ||
| 99 | + | ||
| 6 | def fetch_data(self): | 100 | def fetch_data(self): |
| 7 | - pass | 101 | + LOGGER.info("Importing Communities") |
| 102 | + self.fetch_communities() | ||
| 103 | + | ||
| 104 | + LOGGER.info("Importing Articles") | ||
| 105 | + self.fetch_articles() | ||
| 106 | + | ||
| 107 | + @property | ||
| 108 | + def app_label(self): | ||
| 109 | + return 'noosfero' |
| @@ -0,0 +1,71 @@ | @@ -0,0 +1,71 @@ | ||
| 1 | +# -*- coding: utf-8 -*- | ||
| 2 | +from __future__ import unicode_literals | ||
| 3 | + | ||
| 4 | +from django.db import models, migrations | ||
| 5 | +import django.db.models.deletion | ||
| 6 | +from django.conf import settings | ||
| 7 | + | ||
| 8 | + | ||
| 9 | +class Migration(migrations.Migration): | ||
| 10 | + | ||
| 11 | + dependencies = [ | ||
| 12 | + migrations.swappable_dependency(settings.AUTH_USER_MODEL), | ||
| 13 | + ] | ||
| 14 | + | ||
| 15 | + operations = [ | ||
| 16 | + migrations.CreateModel( | ||
| 17 | + name='NoosferoArticle', | ||
| 18 | + fields=[ | ||
| 19 | + ('id', models.IntegerField(serialize=False, primary_key=True)), | ||
| 20 | + ('title', models.TextField()), | ||
| 21 | + ('path', models.TextField(null=True, blank=True)), | ||
| 22 | + ('body', models.TextField(null=True, blank=True)), | ||
| 23 | + ('profile_identifier', models.TextField()), | ||
| 24 | + ('created_at', models.DateTimeField(blank=True)), | ||
| 25 | + ], | ||
| 26 | + options={ | ||
| 27 | + 'verbose_name': 'Article', | ||
| 28 | + 'verbose_name_plural': 'Articles', | ||
| 29 | + }, | ||
| 30 | + bases=(models.Model,), | ||
| 31 | + ), | ||
| 32 | + migrations.CreateModel( | ||
| 33 | + name='NoosferoCategory', | ||
| 34 | + fields=[ | ||
| 35 | + ('id', models.IntegerField(serialize=False, primary_key=True)), | ||
| 36 | + ('name', models.TextField(null=True, blank=True)), | ||
| 37 | + ], | ||
| 38 | + options={ | ||
| 39 | + }, | ||
| 40 | + bases=(models.Model,), | ||
| 41 | + ), | ||
| 42 | + migrations.CreateModel( | ||
| 43 | + name='NoosferoCommunity', | ||
| 44 | + fields=[ | ||
| 45 | + ('id', models.IntegerField(serialize=False, primary_key=True)), | ||
| 46 | + ('name', models.TextField()), | ||
| 47 | + ('identifier', models.TextField()), | ||
| 48 | + ('description', models.TextField(null=True, blank=True)), | ||
| 49 | + ('created_at', models.DateTimeField(blank=True)), | ||
| 50 | + ('categories', models.ManyToManyField(to='noosfero.NoosferoCategory')), | ||
| 51 | + ('user', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True)), | ||
| 52 | + ], | ||
| 53 | + options={ | ||
| 54 | + 'verbose_name': 'Community', | ||
| 55 | + 'verbose_name_plural': 'Communities', | ||
| 56 | + }, | ||
| 57 | + bases=(models.Model,), | ||
| 58 | + ), | ||
| 59 | + migrations.AddField( | ||
| 60 | + model_name='noosferoarticle', | ||
| 61 | + name='categories', | ||
| 62 | + field=models.ManyToManyField(to='noosfero.NoosferoCategory'), | ||
| 63 | + preserve_default=True, | ||
| 64 | + ), | ||
| 65 | + migrations.AddField( | ||
| 66 | + model_name='noosferoarticle', | ||
| 67 | + name='user', | ||
| 68 | + field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, null=True), | ||
| 69 | + preserve_default=True, | ||
| 70 | + ), | ||
| 71 | + ] |
colab/plugins/noosfero/models.py
| @@ -0,0 +1,68 @@ | @@ -0,0 +1,68 @@ | ||
| 1 | +from colab.plugins.utils.models import Collaboration | ||
| 2 | +from django.db import models | ||
| 3 | +from django.utils.translation import ugettext_lazy as _ | ||
| 4 | + | ||
| 5 | + | ||
| 6 | +class NoosferoCategory(models.Model): | ||
| 7 | + | ||
| 8 | + id = models.IntegerField(primary_key=True) | ||
| 9 | + name = models.TextField(null=True, blank=True) | ||
| 10 | + | ||
| 11 | + def __unicode__(self): | ||
| 12 | + return "{}-{}".format(self.id, self.name) | ||
| 13 | + | ||
| 14 | + | ||
| 15 | +class NoosferoCommunity(Collaboration): | ||
| 16 | + | ||
| 17 | + id = models.IntegerField(primary_key=True) | ||
| 18 | + type = u'community' | ||
| 19 | + icon_name = u'file' | ||
| 20 | + name = models.TextField() | ||
| 21 | + identifier = models.TextField() | ||
| 22 | + description = models.TextField(null=True, blank=True) | ||
| 23 | + categories = models.ManyToManyField(NoosferoCategory) | ||
| 24 | + created_at = models.DateTimeField(blank=True) | ||
| 25 | + | ||
| 26 | + @property | ||
| 27 | + def url(self): | ||
| 28 | + return u'/social/profile/{}'.format(self.identifier) | ||
| 29 | + | ||
| 30 | + @property | ||
| 31 | + def modified(self): | ||
| 32 | + return self.created_at | ||
| 33 | + | ||
| 34 | + def __unicode__(self): | ||
| 35 | + return "{}({}) - {}".format(self.name, self.identifier, | ||
| 36 | + self.description) | ||
| 37 | + | ||
| 38 | + class Meta: | ||
| 39 | + verbose_name = _('Community') | ||
| 40 | + verbose_name_plural = _('Communities') | ||
| 41 | + | ||
| 42 | + | ||
| 43 | +class NoosferoArticle(Collaboration): | ||
| 44 | + | ||
| 45 | + id = models.IntegerField(primary_key=True) | ||
| 46 | + type = u'article' | ||
| 47 | + icon_name = u'file' | ||
| 48 | + title = models.TextField() | ||
| 49 | + path = models.TextField(null=True, blank=True) | ||
| 50 | + body = models.TextField(null=True, blank=True) | ||
| 51 | + categories = models.ManyToManyField(NoosferoCategory) | ||
| 52 | + profile_identifier = models.TextField() | ||
| 53 | + created_at = models.DateTimeField(blank=True) | ||
| 54 | + | ||
| 55 | + @property | ||
| 56 | + def url(self): | ||
| 57 | + return u'/social/{}/{}'.format(self.profile_identifier, self.path) | ||
| 58 | + | ||
| 59 | + @property | ||
| 60 | + def modified(self): | ||
| 61 | + return self.created_at | ||
| 62 | + | ||
| 63 | + def __unicode__(self): | ||
| 64 | + return "{}({})".format(self.title, self.path) | ||
| 65 | + | ||
| 66 | + class Meta: | ||
| 67 | + verbose_name = _('Article') | ||
| 68 | + verbose_name_plural = _('Articles') |
| @@ -0,0 +1,70 @@ | @@ -0,0 +1,70 @@ | ||
| 1 | +# -*- coding: utf-8 -*- | ||
| 2 | + | ||
| 3 | +import string | ||
| 4 | + | ||
| 5 | +from haystack import indexes | ||
| 6 | +from haystack.utils import log as logging | ||
| 7 | + | ||
| 8 | +from .models import (NoosferoArticle, NoosferoCommunity) | ||
| 9 | + | ||
| 10 | + | ||
| 11 | +logger = logging.getLogger('haystack') | ||
| 12 | + | ||
| 13 | +# The string maketrans always return a string encoded with latin1 | ||
| 14 | +# http://stackoverflow.com/questions/1324067/how-do-i-get-str-translate-to-work-with-unicode-strings | ||
| 15 | +table = string.maketrans( | ||
| 16 | + string.punctuation, | ||
| 17 | + '.' * len(string.punctuation) | ||
| 18 | +).decode('latin1') | ||
| 19 | + | ||
| 20 | + | ||
| 21 | +class NoosferoCommunityIndex(indexes.SearchIndex, indexes.Indexable): | ||
| 22 | + text = indexes.CharField(document=True, use_template=True, stored=False) | ||
| 23 | + title = indexes.CharField(model_attr='name') | ||
| 24 | + description = indexes.CharField(model_attr='description', null=True) | ||
| 25 | + url = indexes.CharField(model_attr='url', indexed=False) | ||
| 26 | + icon_name = indexes.CharField() | ||
| 27 | + type = indexes.CharField() | ||
| 28 | + modified = indexes.DateTimeField(model_attr='modified', null=True) | ||
| 29 | + created_at = indexes.DateTimeField(model_attr='created_at', null=True) | ||
| 30 | + category = indexes.MultiValueField() | ||
| 31 | + | ||
| 32 | + def prepare_category(self, obj): | ||
| 33 | + return obj.categories.values_list('name', flat=True) | ||
| 34 | + | ||
| 35 | + def prepare_icon_name(self, obj): | ||
| 36 | + return u'file' | ||
| 37 | + | ||
| 38 | + def get_ful_name(self): | ||
| 39 | + self.objs.name | ||
| 40 | + | ||
| 41 | + def get_model(self): | ||
| 42 | + return NoosferoCommunity | ||
| 43 | + | ||
| 44 | + def prepare_type(self, obj): | ||
| 45 | + return u'noosfero_community' | ||
| 46 | + | ||
| 47 | + | ||
| 48 | +class NoosferoArticleIndex(indexes.SearchIndex, indexes.Indexable): | ||
| 49 | + | ||
| 50 | + text = indexes.CharField(document=True, use_template=True, stored=False) | ||
| 51 | + title = indexes.CharField(model_attr='title') | ||
| 52 | + body = indexes.CharField(model_attr='body', null=True) | ||
| 53 | + url = indexes.CharField(model_attr='url', indexed=False) | ||
| 54 | + icon_name = indexes.CharField() | ||
| 55 | + type = indexes.CharField(model_attr='type') | ||
| 56 | + modified = indexes.DateTimeField(model_attr='modified', null=True) | ||
| 57 | + created_at = indexes.DateTimeField(model_attr='created_at', null=True) | ||
| 58 | + category = indexes.MultiValueField() | ||
| 59 | + | ||
| 60 | + def get_model(self): | ||
| 61 | + return NoosferoArticle | ||
| 62 | + | ||
| 63 | + def prepare_category(self, obj): | ||
| 64 | + return obj.categories.values_list('name', flat=True) | ||
| 65 | + | ||
| 66 | + def prepare_icon_name(self, obj): | ||
| 67 | + return u'file' | ||
| 68 | + | ||
| 69 | + def prepare_type(self, obj): | ||
| 70 | + return u'noosfero_articles' |
colab/plugins/noosfero/templates/search/indexes/noosfero/noosferoarticle_text.txt
0 → 100644
colab/plugins/noosfero/templates/search/indexes/noosfero/noosferocommunity_text.txt
0 → 100644
-
Added 62 new commits:
- 83ef0de7 - Set colab_settings for tests
- c72f8481 - Fixed pep8
- f1e92a83 - Fixed Gitlab tests
- 7c84af70 - Add celery to colab
- 18990d74 - Add celery service
- 50b085db - Fixing for tests
- 692253bb - Merge branch 'add_celery'
- 7f68454c - Remove Sorl from script.
- 5e7a172c - Added epel to centos, because of celery.
- 75f3d7a6 - Improved menu.
- df127fc2 - updated revproxy and removed browserid
- 13d0938f - Removed browserid left-overs
- 6f406e45 - Added requires
- e32b47e8 - Fixed search.
- 166a69fb - Merge branch 'fix_master'
- cbf39547 - 1.10.1 Release
- 31499efd - Merge remote-tracking branch 'origin/match_release'
- c6970710 - Merge branch 'remove-deps' into 'master'
- 41486a1c - Add private_token to plugins to fix api data for plugins
- 9f844060 - Merge branch 'fix_private_token_plugin' into 'master'
- 2b77d361 - Decrease graph label size to fix cutoff
- 391c7119 - Merge branch 'fix_contrib_graph' into 'master'
- 2b5b7cce - Fix super_archives pt_BR translations
- 5505fb3b - Include super_archives translations in MANIFEST.in
- 1c074167 - Merge branch 'fix_locale' into 'master'
- d9198a01 - Fix gitlab links
- 9a166399 - Refactor url method of mr/issue
- a70fae9a - Merge branch 'fix_gitlab_links' into 'master'
- 1ba623cd - Merge branch 'master' of portal.softwarepublico.gov.br:softwarepublico/colab
- 2a43c881 - No need for postgres nor solr
- 3329a8d7 - We don't use apt on tests
- 080cb1eb - Moving to new infrastructure
- fd24ee4c - Fixed flake8
- 97c80b74 - Updated setuptools
- 935957af - Fixing database path
- 0d5e9faf - Set colab_settings for tests
- 58ec1d8d - Fixed pep8
- 47ca4d8d - Fixed Gitlab tests
- b71fe737 - Don't allow auto db conf if debug is False
- 65a70b23 - Test settings validation for db
- 5498660e - Removing old deps from badger
- a2965d3c - remove legacy converse.js stuff
- 121feb9a - Removed i18n_model (no longer needed)
- 6c3931c0 - Reorganized deps on setup.py
- ef767871 - Removed django-mobile
- e373e29d - Readded tastypie
- f83576eb - Removed open-data api
- 87c02305 - Removed old TODO file
- 9c0fc2b4 - Moved voting to super_archives
- 70e66881 - Refactored vagrant install scripts
- f1daeea2 - Ignore whoosh_index
- aaaff48e - Added TODO
- 2f9c5931 - Added missing slash
- 6cb1ec46 - Removed old cron jobs left behind
- 60d43389 - Removed tastypie dep
- 9c651342 - Merge branch 'cleanup-old-deps' into fix_master
- d1bc4536 - Removed precise64 support
- cb01d435 - Added travis notifications on IRC
- 8068038a - Merge branch 'fix_master'
- 52ba4161 - Add support to noosfero data.
- 6befc78f - Noosfero data to haystack.
- 226f9140 - Change location of save model of noosfero import data
-
mentioned in commit ce7c9854500c8ec02fee5b71a8bf53f83c38bd8b
