From 435ccefcd9379390e8aef891e2c21e4ec39affcd Mon Sep 17 00:00:00 2001 From: Sergio Oliveira Date: Tue, 15 Oct 2013 14:31:13 -0300 Subject: [PATCH] Moving find blocks to utils --- requirements.txt | 1 + src/static/css/screen.css | 5 +++++ src/super_archives/migrations/0017_auto__add_keyword.py | 151 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/super_archives/migrations/0018_auto__add_index_message_received_time.py | 145 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/super_archives/models.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++- src/super_archives/templates/message-thread.html | 11 ++++++++++- src/super_archives/templates/superarchives/tags/display_message.html | 10 +++------- src/super_archives/templatetags/superarchives.py | 78 ++---------------------------------------------------------------------------- src/super_archives/utils.py | 19 ------------------- src/super_archives/utils/__init__.py | 0 src/super_archives/utils/blocks.py | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/super_archives/utils/email.py | 19 +++++++++++++++++++ src/super_archives/utils/etiquetador.py | 5 +++++ src/super_archives/views.py | 11 ++++++----- 14 files changed, 499 insertions(+), 109 deletions(-) create mode 100644 src/super_archives/migrations/0017_auto__add_keyword.py create mode 100644 src/super_archives/migrations/0018_auto__add_index_message_received_time.py delete mode 100644 src/super_archives/utils.py create mode 100644 src/super_archives/utils/__init__.py create mode 100644 src/super_archives/utils/blocks.py create mode 100644 src/super_archives/utils/email.py create mode 100644 src/super_archives/utils/etiquetador.py diff --git a/requirements.txt b/requirements.txt index 794f106..fea3a07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ chardet==1.0.1 python-dateutil==1.5 django-cliauth==0.9 django-mobile==0.3.0 +etiquetando==0.1 html2text gunicorn diff --git a/src/static/css/screen.css b/src/static/css/screen.css index 96c47ee..36868a6 100644 --- a/src/static/css/screen.css +++ b/src/static/css/screen.css @@ -335,6 +335,11 @@ ul.unstyled-list .glyphicon-chevron-right { margin-top: 0; } +.tag-cloud { + text-align: justify; + margin-top: 6px; +} + .tag-cloud .size-1 { font-size: 0.9em; } diff --git a/src/super_archives/migrations/0017_auto__add_keyword.py b/src/super_archives/migrations/0017_auto__add_keyword.py new file mode 100644 index 0000000..29aeebb --- /dev/null +++ b/src/super_archives/migrations/0017_auto__add_keyword.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'Keyword' + db.create_table(u'super_archives_keyword', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('keyword', self.gf('django.db.models.fields.CharField')(max_length='128')), + ('weight', self.gf('django.db.models.fields.IntegerField')(default=0)), + ('thread', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['super_archives.Thread'])), + )) + db.send_create_signal(u'super_archives', ['Keyword']) + + + def backwards(self, orm): + # Deleting model 'Keyword' + db.delete_table(u'super_archives_keyword') + + + models = { + u'accounts.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75', 'blank': 'True'}), + 'facebook': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'google_talk': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'institution': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'role': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'twitter': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}), + 'verification_hash': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'}), + 'webpage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'null': 'True', 'blank': 'True'}) + }, + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'super_archives.emailaddress': { + 'Meta': {'ordering': "('id',)", 'object_name': 'EmailAddress'}, + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'md5': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}), + 'real_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '64', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': u"orm['accounts.User']"}) + }, + u'super_archives.emailaddressvalidation': { + 'Meta': {'unique_together': "(('user', 'address'),)", 'object_name': 'EmailAddressValidation'}, + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails_not_validated'", 'null': 'True', 'to': u"orm['accounts.User']"}), + 'validation_key': ('django.db.models.fields.CharField', [], {'default': "'e2d18f1f8d6445fc91c04d17acdbe23a'", 'max_length': '32', 'null': 'True'}) + }, + u'super_archives.keyword': { + 'Meta': {'object_name': 'Keyword'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'keyword': ('django.db.models.fields.CharField', [], {'max_length': "'128'"}), + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Thread']"}), + 'weight': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + u'super_archives.mailinglist': { + 'Meta': {'object_name': 'MailingList'}, + 'description': ('django.db.models.fields.TextField', [], {}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_imported_index': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'logo': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '80'}) + }, + u'super_archives.mailinglistmembership': { + 'Meta': {'object_name': 'MailingListMembership'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.MailingList']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['accounts.User']"}) + }, + u'super_archives.message': { + 'Meta': {'ordering': "('received_time',)", 'unique_together': "(('thread', 'message_id'),)", 'object_name': 'Message'}, + 'body': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'from_address': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.EmailAddress']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'received_time': ('django.db.models.fields.DateTimeField', [], {}), + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), + 'subject_clean': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Thread']", 'null': 'True'}) + }, + u'super_archives.messagemetadata': { + 'Message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Message']"}), + 'Meta': {'object_name': 'MessageMetadata'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + u'super_archives.pagehit': { + 'Meta': {'object_name': 'PageHit'}, + 'hit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'url_path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'}) + }, + u'super_archives.thread': { + 'Meta': {'unique_together': "(('subject_token', 'mailinglist'),)", 'object_name': 'Thread'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'latest_message': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'+'", 'unique': 'True', 'null': 'True', 'to': u"orm['super_archives.Message']"}), + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.MailingList']"}), + 'score': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subject_token': ('django.db.models.fields.CharField', [], {'max_length': '512'}) + }, + u'super_archives.vote': { + 'Meta': {'unique_together': "(('user', 'message'),)", 'object_name': 'Vote'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Message']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['accounts.User']"}) + } + } + + complete_apps = ['super_archives'] \ No newline at end of file diff --git a/src/super_archives/migrations/0018_auto__add_index_message_received_time.py b/src/super_archives/migrations/0018_auto__add_index_message_received_time.py new file mode 100644 index 0000000..63dd7cd --- /dev/null +++ b/src/super_archives/migrations/0018_auto__add_index_message_received_time.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding index on 'Message', fields ['received_time'] + db.create_index(u'super_archives_message', ['received_time']) + + + def backwards(self, orm): + # Removing index on 'Message', fields ['received_time'] + db.delete_index(u'super_archives_message', ['received_time']) + + + models = { + u'accounts.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75', 'blank': 'True'}), + 'facebook': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'google_talk': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'institution': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'role': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'twitter': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}), + 'verification_hash': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'}), + 'webpage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'null': 'True', 'blank': 'True'}) + }, + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'super_archives.emailaddress': { + 'Meta': {'ordering': "('id',)", 'object_name': 'EmailAddress'}, + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'md5': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}), + 'real_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '64', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': u"orm['accounts.User']"}) + }, + u'super_archives.emailaddressvalidation': { + 'Meta': {'unique_together': "(('user', 'address'),)", 'object_name': 'EmailAddressValidation'}, + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails_not_validated'", 'null': 'True', 'to': u"orm['accounts.User']"}), + 'validation_key': ('django.db.models.fields.CharField', [], {'default': "'1775e2bdc5e341a9b9b8a6346eec1024'", 'max_length': '32', 'null': 'True'}) + }, + u'super_archives.keyword': { + 'Meta': {'ordering': "('?',)", 'object_name': 'Keyword'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'keyword': ('django.db.models.fields.CharField', [], {'max_length': "'128'"}), + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Thread']"}), + 'weight': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + u'super_archives.mailinglist': { + 'Meta': {'object_name': 'MailingList'}, + 'description': ('django.db.models.fields.TextField', [], {}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_imported_index': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'logo': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '80'}) + }, + u'super_archives.mailinglistmembership': { + 'Meta': {'object_name': 'MailingListMembership'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.MailingList']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['accounts.User']"}) + }, + u'super_archives.message': { + 'Meta': {'ordering': "('received_time',)", 'unique_together': "(('thread', 'message_id'),)", 'object_name': 'Message'}, + 'body': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'from_address': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.EmailAddress']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'received_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), + 'subject_clean': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Thread']", 'null': 'True'}) + }, + u'super_archives.messagemetadata': { + 'Message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Message']"}), + 'Meta': {'object_name': 'MessageMetadata'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + u'super_archives.pagehit': { + 'Meta': {'object_name': 'PageHit'}, + 'hit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'url_path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'}) + }, + u'super_archives.thread': { + 'Meta': {'unique_together': "(('subject_token', 'mailinglist'),)", 'object_name': 'Thread'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'latest_message': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'+'", 'unique': 'True', 'null': 'True', 'to': u"orm['super_archives.Message']"}), + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.MailingList']"}), + 'score': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subject_token': ('django.db.models.fields.CharField', [], {'max_length': '512'}) + }, + u'super_archives.vote': { + 'Meta': {'unique_together': "(('user', 'message'),)", 'object_name': 'Vote'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Message']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['accounts.User']"}) + } + } + + complete_apps = ['super_archives'] \ No newline at end of file diff --git a/src/super_archives/models.py b/src/super_archives/models.py index 5f7e165..c87a16e 100644 --- a/src/super_archives/models.py +++ b/src/super_archives/models.py @@ -6,10 +6,16 @@ from hashlib import md5 from django.db import models from django.conf import settings from django.utils import timezone +from django.core.cache import cache from django.contrib.auth import get_user_model from django.core.urlresolvers import reverse, NoReverseMatch from django.utils.translation import ugettext_lazy as _ +from html2text import html2text + +from .utils import blocks +from .utils.etiquetador import etiquetador + User = get_user_model() @@ -84,6 +90,18 @@ class MailingListMembership(models.Model): return '%s on %s' % (self.user.email, self.mailinglist.name) +class Keyword(models.Model): + keyword = models.CharField(max_length='128') + weight = models.IntegerField(default=0) + thread = models.ForeignKey('Thread') + + class Meta: + ordering = ('?', ) # random order + + def __unicode__(self): + return self.keyword + + class Thread(models.Model): subject_token = models.CharField(max_length=512) @@ -105,6 +123,31 @@ class Thread(models.Model): verbose_name_plural = _(u"Threads") unique_together = ('subject_token', 'mailinglist') + def gen_tags(self): + blocks = [] + for message in self.message_set.iterator(): + blocks.extend([block for block in message.blocks() + if not block.is_reply]) + text = u'\n'.join(map(unicode, blocks)) + tags = etiquetador(html2text(text)) + return tags + + def save(self, *args, **kwargs): + super(Thread, self).save(*args, **kwargs) + tags = self.gen_tags() + + for tag, weight in tags: + keyword, created = Keyword.objects.get_or_create(thread=self, + keyword=tag) + if created or keyword.weight != weight: + keyword.weight = weight + keyword.save() + + # removing old tags + qs = Keyword.objects.filter(thread=self) + qs = qs.exclude(keyword__in=zip(*tags)[0]) + qs.delete() + def __unicode__(self): return '%s - %s (%s)' % (self.id, self.subject_token, @@ -190,7 +233,7 @@ class Message(models.Model): body = models.TextField(default='', verbose_name=_(u"Message body"), help_text=_(u"Please enter a message body")) - received_time = models.DateTimeField() + received_time = models.DateTimeField(db_index=True) message_id = models.CharField(max_length=512) spam = models.BooleanField(default=False) @@ -201,12 +244,21 @@ class Message(models.Model): verbose_name = _(u"Message") verbose_name_plural = _(u"Messages") unique_together = ('thread', 'message_id') + ordering = ('received_time', ) def __unicode__(self): return '(%s) %s: %s' % (self.id, self.from_address.get_full_name(), self.subject_clean) + def blocks(self): + cache_key = 'email-blocks-{}'.format(self.pk) + blks = cache.get(cache_key) + if not blks: + blks = blocks.EmailBlockParser(self) + cache.set(cache_key, blks) + return blks + @property def mailinglist(self): if not self.thread or not self.thread.mailinglist: diff --git a/src/super_archives/templates/message-thread.html b/src/super_archives/templates/message-thread.html index 8e32ee1..9d8b7c5 100644 --- a/src/super_archives/templates/message-thread.html +++ b/src/super_archives/templates/message-thread.html @@ -140,7 +140,7 @@
- {% display_message email emails %} + {% display_message email %}
@@ -189,6 +189,15 @@ {% trans "voted" %}
{{ total_votes }} {% trans "times" %}
+
  • + + {% trans "tags" %} +
    + {% for keyword in thread.keyword_set.iterator %} + {{ keyword|escape }} + {% endfor %} +
    +
    • diff --git a/src/super_archives/templates/superarchives/tags/display_message.html b/src/super_archives/templates/superarchives/tags/display_message.html index f98ad16..c8a673b 100644 --- a/src/super_archives/templates/superarchives/tags/display_message.html +++ b/src/super_archives/templates/superarchives/tags/display_message.html @@ -1,9 +1,5 @@ -{% load cache %} - -{% cache cache_timeout 'display_message' cache_key %} -{% for message, class in messages %} - {% if class == 'reply' %} +{% for block in blocks %} + {% if block.is_reply %} -
    {% else %}
    {% endif %}{{ message|safe|linebreaksbr }}
    +
    {% else %}
    {% endif %}{{ block|safe|linebreaksbr }}
    {% endfor %} -{% endcache %} diff --git a/src/super_archives/templatetags/superarchives.py b/src/super_archives/templatetags/superarchives.py index eec4878..6e66f64 100644 --- a/src/super_archives/templatetags/superarchives.py +++ b/src/super_archives/templatetags/superarchives.py @@ -1,85 +1,11 @@ -import re - from django import template -from django.core.cache import cache - -from html2text import html2text register = template.Library() TEMPLATE_PATH = 'superarchives/tags/' -EXTENDED_PUNCTUATION = '!"#$%&\'()*+,-./:;=?@[\\]^_`{|}~ \t\n\r\x0b\x0c' -RE_WRAPPED_BY_HTML = re.compile(r'^<[a-z]+[^>]*>.*]*>$', - re.MULTILINE|re.IGNORECASE|re.DOTALL) -RE_LINKS = re.compile(r'(?Phttps?://[^ \t\r\n\<]+)') - - -def find_links(block): - links = RE_LINKS.finditer(block) - - block, n = RE_LINKS.subn(r'\g', - block) - - return block - - -def join(block): - block_txt = u''.join(block) - - if RE_WRAPPED_BY_HTML.match(block_txt.strip()): - block = html2text(block_txt) - else: - block = block_txt - - return find_links(block) - -def is_reply(line, message, thread): - clean_line = line.strip() - if clean_line.startswith('>'): - return True - - for other_msg in thread: - if other_msg == message: - return False - - clean_body = other_msg.body.replace('\r', ' ')\ - .replace('\n', ' ') - if clean_line.strip('> ') in clean_body: - return True - - return False - - @register.inclusion_tag(TEMPLATE_PATH + 'display_message.html') -def display_message(email, thread): - message = email.body - messages = [] - - block = [] - reply_block = False - - for line in message.split('\n'): - if line.strip(EXTENDED_PUNCTUATION): - if is_reply(line, email, thread): - if not reply_block: - reply_block = True - messages.append((join(block), 'normal')) - block = [] - elif reply_block: - reply_block = False - messages.append((join(block), 'reply')) - block = [] - - block.append(line.rstrip() + '\n') - - if reply_block: - messages.append((join(block), 'reply')) - else: - messages.append((join(block), 'normal')) - - return {'messages': messages, - 'cache_key': email.pk, - 'cache_timeout': cache.default_timeout} +def display_message(email): + return {'blocks': email.blocks()} diff --git a/src/super_archives/utils.py b/src/super_archives/utils.py deleted file mode 100644 index 9095406..0000000 --- a/src/super_archives/utils.py +++ /dev/null @@ -1,19 +0,0 @@ - -from django.core import mail -from django.conf import settings -from django.template import Context, loader -from django.utils.translation import ugettext as _ - - -def colab_send_email(subject, message, to): - from_email = settings.COLAB_FROM_ADDRESS - return mail.send_mail(subject, message, from_email, [to]) - - -def send_verification_email(to, user, validation_key): - subject = _('Please verify your email ') + u'{}'.format(to) - msg_tmpl = loader.get_template('superarchives/emails/email_verification.txt') - message = msg_tmpl.render(Context({'to': to, 'user': user, - 'key': validation_key, - 'SITE_URL': settings.SITE_URL})) - return colab_send_email(subject, message, to) diff --git a/src/super_archives/utils/__init__.py b/src/super_archives/utils/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/super_archives/utils/__init__.py diff --git a/src/super_archives/utils/blocks.py b/src/super_archives/utils/blocks.py new file mode 100644 index 0000000..ffadac6 --- /dev/null +++ b/src/super_archives/utils/blocks.py @@ -0,0 +1,99 @@ + +import re + +from django.utils.html import strip_tags + +from html2text import html2text + + +EXTENDED_PUNCTUATION = '!"#$%&\'()*+,-./:;=?@[\\]^_`{|}~ \t\n\r\x0b\x0c' +RE_WRAPPED_BY_HTML = re.compile(r'^<[a-z]+[^>]*>.*]*>$', + re.MULTILINE|re.IGNORECASE|re.DOTALL) +RE_LINKS = re.compile(r'(?Phttps?://[^ \t\r\n\<]+)') +LINK_MARKUP = u'\g' + +RE_REPLY_LINE = re.compile(r'^[\s\t>]*>[\s\t]*') + +RE_BR_TO_LINEBREAK = re.compile(r'<\s*/?\s*br\s*/?\s*>') + +class EmailBlock(list): + def __init__(self, is_reply=False, mark_links=True, html2text=True): + self.mark_links = mark_links + self.html2text = html2text + self.is_reply = is_reply + + def _html2text(self, text): + if RE_WRAPPED_BY_HTML.match(text.strip()): + return html2text(text) + + text, n = RE_BR_TO_LINEBREAK.subn('\n', text) + text = strip_tags(text) + return text + + def _mark_links(self, text): + text, n = RE_LINKS.subn(LINK_MARKUP, text) + return text + + @property + def text(self): + block = u''.join(self) + + if self.html2text: + block = self._html2text(block) + + if self.mark_links: + block = self._mark_links(block) + + return block + + def __unicode__(self): + return self.text + + def __str__(self): + return self.text + + +class EmailBlockParser(list): + def __init__(self, email): + self.email = email + self.thread_emails = email.thread.message_set + + message = email.body + block = EmailBlock() + + for line in message.split('\n'): + if self.context_switch(line, block): + self.append(block) + new_block_context = not block.is_reply + block = EmailBlock(is_reply=new_block_context) + + block.append(line + '\n') + + self.append(block) + + def context_switch(self, line, block): + if line.strip(EXTENDED_PUNCTUATION): + if self.is_reply(line): + if not block.is_reply: + return True + + else: + if block.is_reply: + return True + + return False + + def is_reply(self, line): + stripped_line = line.strip() + if stripped_line.startswith('>') or RE_REPLY_LINE.match(line): + return True + + clean_line = RE_REPLY_LINE.subn('', stripped_line)[0] + queryset = self.thread_emails.filter( + received_time__lt=self.email.received_time, + body__contains=clean_line).order_by('-received_time') + + if queryset[:1]: + return True + + return False diff --git a/src/super_archives/utils/email.py b/src/super_archives/utils/email.py new file mode 100644 index 0000000..9095406 --- /dev/null +++ b/src/super_archives/utils/email.py @@ -0,0 +1,19 @@ + +from django.core import mail +from django.conf import settings +from django.template import Context, loader +from django.utils.translation import ugettext as _ + + +def colab_send_email(subject, message, to): + from_email = settings.COLAB_FROM_ADDRESS + return mail.send_mail(subject, message, from_email, [to]) + + +def send_verification_email(to, user, validation_key): + subject = _('Please verify your email ') + u'{}'.format(to) + msg_tmpl = loader.get_template('superarchives/emails/email_verification.txt') + message = msg_tmpl.render(Context({'to': to, 'user': user, + 'key': validation_key, + 'SITE_URL': settings.SITE_URL})) + return colab_send_email(subject, message, to) diff --git a/src/super_archives/utils/etiquetador.py b/src/super_archives/utils/etiquetador.py new file mode 100644 index 0000000..c08c956 --- /dev/null +++ b/src/super_archives/utils/etiquetador.py @@ -0,0 +1,5 @@ + +from etiquetando import Etiquetador + +etiquetador = Etiquetador(word_min_size=3, max_tags=24, + weight_range=(1, 4)) diff --git a/src/super_archives/views.py b/src/super_archives/views.py index 5aa49f5..0333685 100644 --- a/src/super_archives/views.py +++ b/src/super_archives/views.py @@ -14,8 +14,8 @@ from django.contrib.auth.decorators import login_required from django.shortcuts import render, redirect from . import queries -from .utils import send_verification_email from .decorators import count_hit +from .utils.email import send_verification_email from .models import MailingList, Thread, EmailAddress, EmailAddressValidation @@ -42,18 +42,19 @@ def thread(request, mailinglist, thread_token): total_votes += email.votes_count() # Update relevance score - query = Thread.objects.filter(mailinglist__name=mailinglist) - thread = query.get(subject_token=thread_token) + thread = Thread.objects.get(subject_token=thread_token, + mailinglist__name=mailinglist) thread.update_score() - template_data = { + context = { 'first_msg': first_message, 'emails': [first_message] + list(emails), 'pagehits': queries.get_page_hits(request.path_info), 'total_votes': total_votes, + 'thread': thread, } - return render(request, 'message-thread.html', template_data) + return render(request, 'message-thread.html', context) def list_messages(request): -- libgit2 0.21.2