diff --git a/colab/api/handlers.py b/colab/api/handlers.py index 57a1cf0..a1a02c4 100644 --- a/colab/api/handlers.py +++ b/colab/api/handlers.py @@ -7,6 +7,7 @@ from django.contrib.auth.decorators import login_required from piston.utils import rc from piston.handler import BaseHandler +from colab import solrutils from colab.super_archives.models import Message, PageHit @@ -69,3 +70,34 @@ class CountHandler(BaseHandler): return rc.CREATED +class SearchHandler(BaseHandler): + allowed_methods = ('GET', ) + + def read(self, request): + query = request.GET.get('q') + page = request.GET.get('p', 1) + results_per_page = request.GET.get('n', 50) + order = request.GET.get('o') + + if not query: + return 'Query cannot be empty.' + else: + query = query.encode('utf-8') + + try: + n = int(results_per_page) + except ValueError: + n = 10 + + if 1 > n > 500: + n = 1 + + try: + page = int(page) + except ValueError: + page = 1 + + if page < 1: + page = 1 + + return solrutils.select(query, results_per_page, page, order) diff --git a/colab/api/urls.py b/colab/api/urls.py index c6deb97..f89db6c 100644 --- a/colab/api/urls.py +++ b/colab/api/urls.py @@ -2,13 +2,15 @@ from django.conf.urls.defaults import patterns, include, url from piston.resource import Resource -from colab.api.handlers import VoteHandler, CountHandler +from colab.api.handlers import VoteHandler, CountHandler, SearchHandler vote_handler = Resource(VoteHandler) count_handler = Resource(CountHandler) +search_handler = Resource(SearchHandler) urlpatterns = patterns('', url(r'message/(?P\d+)/vote$', vote_handler), url(r'hit/$', count_handler), -) \ No newline at end of file + url(r'search/$', search_handler), +) diff --git a/colab/settings.py b/colab/settings.py index e0c3fc9..ed36a77 100644 --- a/colab/settings.py +++ b/colab/settings.py @@ -153,7 +153,7 @@ SOLR_HOSTNAME = '10.1.2.154' SOLR_PORT = '8080' SOLR_SELECT_PATH = '/solr/select' -SOLR_COLAB_URI = 'http://colab.interlegis.gov.br' +SOLR_COLAB_URI = 'http://colab.interlegis.leg.br' SOLR_BASE_QUERY = """ ((Type:changeset OR Type:ticket OR Type:wiki OR Type:thread) AND Title:["" TO *]) """ diff --git a/colab/settings_local-dev.py b/colab/settings_local-dev.py index 23e4d48..b06f16b 100644 --- a/colab/settings_local-dev.py +++ b/colab/settings_local-dev.py @@ -14,11 +14,13 @@ DATABASES = { } } +SOLR_COLAB_URI = None + # Make this unique, and don't share it with anybody. SECRET_KEY = ')(jksdfhsjkadfhjkh234ns!8fqu-1186h$vuj' -import socks -SOCKS_TYPE = socks.PROXY_TYPE_SOCKS5 -SOCKS_SERVER = '127.0.0.1' -SOCKS_PORT = 9050 +#import socks +#SOCKS_TYPE = socks.PROXY_TYPE_SOCKS5 +#SOCKS_SERVER = '127.0.0.1' +#SOCKS_PORT = 9050 diff --git a/colab/solrutils.py b/colab/solrutils.py index 3282cf1..8466e7d 100644 --- a/colab/solrutils.py +++ b/colab/solrutils.py @@ -61,7 +61,7 @@ def get_document_url(doc): doc_type = doc.get('Type') url = '' - if doc_type in ('ticket', 'wiki', 'changeset'): + if settings.SOLR_COLAB_URI: url += settings.SOLR_COLAB_URI url += doc.get('path_string', '') diff --git a/colab/static/img/opendata3.png b/colab/static/img/opendata3.png new file mode 100644 index 0000000..dc12759 Binary files /dev/null and b/colab/static/img/opendata3.png differ diff --git a/colab/super_archives/admin.py b/colab/super_archives/admin.py index 37e4055..12d9c15 100644 --- a/colab/super_archives/admin.py +++ b/colab/super_archives/admin.py @@ -3,7 +3,7 @@ from django.contrib import admin from colab.super_archives.models import Message, Thread, UserProfile class MessageAdmin(admin.ModelAdmin): - list_filter = ('spam', 'mailinglist', 'received_time', ) + list_filter = ('spam', 'thread__mailinglist', 'received_time', ) search_fields = ( 'id', 'subject', diff --git a/colab/super_archives/management/commands/import_emails.py b/colab/super_archives/management/commands/import_emails.py index 34a6ec8..9c68096 100644 --- a/colab/super_archives/management/commands/import_emails.py +++ b/colab/super_archives/management/commands/import_emails.py @@ -130,14 +130,14 @@ class Command(BaseCommand, object): for index, msg in self.parse_emails(mbox_path, n_msgs): yield mailinglist_name, msg, index - def get_thread(self, email): + def get_thread(self, email, mailinglist): """Group messages by thread looking for similar subjects""" subject_slug = slugify(email.subject_clean) thread = self.THREAD_CACHE.get(subject_slug) if thread is None: thread = Thread.objects.get_or_create( - mailinglist=email.mailinglist, + mailinglist=mailinglist, subject_token=subject_slug )[0] @@ -156,7 +156,10 @@ class Command(BaseCommand, object): try: # If the message is already at the database don't do anything - Message.objects.get(message_id=email_msg.get('Message-ID')) + message = Message.objects.get( + message_id=email_msg.get('Message-ID')) + if message.thread.mailinglist.name != mailinglist.name: + raise ObjectDoesNotExist except ObjectDoesNotExist: self.create_email(mailinglist, email_msg) @@ -181,13 +184,12 @@ class Command(BaseCommand, object): email = Message.objects.create( message_id=email_msg.get('Message-ID'), from_address=email_addr, - mailinglist=mailinglist, subject=subject, subject_clean=self.RE_SUBJECT_CLEAN.sub('', subject).strip(), body=email_msg.get_body(), received_time=email_msg.get_received_datetime(), ) - email.thread = self.get_thread(email) + email.thread = self.get_thread(email, mailinglist) email.save() @transaction.commit_manually @@ -245,4 +247,4 @@ class Command(BaseCommand, object): options.get('all'), options.get('exclude_lists')) os.remove(lock_file) - \ No newline at end of file + diff --git a/colab/super_archives/migrations/0009_auto__del_field_message_mailinglist.py b/colab/super_archives/migrations/0009_auto__del_field_message_mailinglist.py new file mode 100644 index 0000000..f68774e --- /dev/null +++ b/colab/super_archives/migrations/0009_auto__del_field_message_mailinglist.py @@ -0,0 +1,136 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Deleting field 'Message.mailinglist' + db.delete_column('super_archives_message', 'mailinglist_id') + + + def backwards(self, orm): + + # User chose to not deal with backwards NULL issues for 'Message.mailinglist' + raise RuntimeError("Cannot reverse this migration. 'Message.mailinglist' and its values cannot be restored.") + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'super_archives.emailaddress': { + 'Meta': {'object_name': 'EmailAddress'}, + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'md5': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}), + 'real_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '64', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails'", 'null': 'True', 'to': "orm['auth.User']"}) + }, + 'super_archives.mailinglist': { + 'Meta': {'object_name': 'MailingList'}, + 'description': ('django.db.models.fields.TextField', [], {}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_imported_index': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'logo': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '80'}) + }, + 'super_archives.mailinglistmembership': { + 'Meta': {'object_name': 'MailingListMembership'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.MailingList']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'super_archives.message': { + 'Meta': {'object_name': 'Message'}, + 'body': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'from_address': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.EmailAddress']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'received_time': ('django.db.models.fields.DateTimeField', [], {}), + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), + 'subject_clean': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Thread']", 'null': 'True'}) + }, + 'super_archives.messagemetadata': { + 'Message': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Message']"}), + 'Meta': {'object_name': 'MessageMetadata'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'super_archives.pagehit': { + 'Meta': {'object_name': 'PageHit'}, + 'hit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'url_path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'}) + }, + 'super_archives.thread': { + 'Meta': {'unique_together': "(('subject_token', 'mailinglist'),)", 'object_name': 'Thread'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'latest_message': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'+'", 'unique': 'True', 'null': 'True', 'to': "orm['super_archives.Message']"}), + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.MailingList']"}), + 'score': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subject_token': ('django.db.models.fields.CharField', [], {'max_length': '512'}) + }, + 'super_archives.userprofile': { + 'Meta': {'object_name': 'UserProfile'}, + 'facebook': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'google_talk': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'institution': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'role': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'twitter': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'}), + 'verification_hash': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}), + 'webpage': ('django.db.models.fields.CharField', [], {'max_length': '256'}) + }, + 'super_archives.vote': { + 'Meta': {'unique_together': "(('user', 'message'),)", 'object_name': 'Vote'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Message']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + } + } + + complete_apps = ['super_archives'] diff --git a/colab/super_archives/models.py b/colab/super_archives/models.py index 88062c2..195fe63 100644 --- a/colab/super_archives/models.py +++ b/colab/super_archives/models.py @@ -194,6 +194,14 @@ class Message(models.Model): return '(%s) %s: %s' % (self.id, self.from_address.get_full_name(), self.subject_clean) + + @property + def mailinglist(self): + if not self.thread or not self.thread.mailinglist: + return None + + return self.thread.mailinglist + def vote_list(self): """Return a list of user that voted in this message.""" diff --git a/colab/super_archives/queries.py b/colab/super_archives/queries.py index ecaba3e..75bae64 100644 --- a/colab/super_archives/queries.py +++ b/colab/super_archives/queries.py @@ -28,7 +28,7 @@ def get_messages_by_voted(): def get_first_message_in_thread(mailinglist, thread_token): query = get_messages_by_date() - query = query.filter(mailinglist__name=mailinglist) + query = query.filter(thread__mailinglist__name=mailinglist) query = query.filter(thread__subject_token=thread_token)[0] return query diff --git a/colab/super_archives/views.py b/colab/super_archives/views.py index 5a3d30a..b37adfb 100644 --- a/colab/super_archives/views.py +++ b/colab/super_archives/views.py @@ -18,7 +18,7 @@ def thread(request, mailinglist, thread_token): msgs_query = queries.get_messages_by_date() msgs_query = msgs_query.filter(thread__subject_token=thread_token) - msgs_query = msgs_query.filter(mailinglist__name=mailinglist) + msgs_query = msgs_query.filter(thread__mailinglist__name=mailinglist) emails = msgs_query.exclude(id=first_message.id) total_votes = first_message.votes_count() diff --git a/colab/templates/base.html b/colab/templates/base.html index eeb4f97..e041920 100644 --- a/colab/templates/base.html +++ b/colab/templates/base.html @@ -137,6 +137,7 @@