Download as
Browse Code »

Commit eaa3aa9f1d2aaa77f0306f7966f9bd0768e800de

Authored by Sergio Oliveira
1 parent 435ccefc
Exists in master and in 39 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, check-port, colab-signals, colab-vcard, colab_search, colab_tag, colab_tag_merge, community_association, detach_super_archives, fix-dashboard, fix-message-preview, fix_app_label, fix_cache_count_spb, fix_header, fixed_gitlab_import, go_to_profile_panel, header_footer, layout-fix, mobile_user_scalable, move_out_plugins, paginate-threads, profile_integration, refactor-data-import, remove-gitlab-plugin, removing_namespace, search_block, search_filters, spb-release/3.0, split_screen, stable, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd, workin_whoosh_temp

Using email blocks from db

Showing 4 changed files with 197 additions and 18 deletions   Show diff stats
src/super_archives/migrations/0019_auto__add_messageblock.py 0 → 100644
  Diff comments   View file @eaa3aa9
... ... @@ -0,0 +1,160 @@
  1 +# -*- coding: utf-8 -*-
  2 +import datetime
  3 +from south.db import db
  4 +from south.v2 import SchemaMigration
  5 +from django.db import models
  6 +
  7 +
  8 +class Migration(SchemaMigration):
  9 +
  10 + def forwards(self, orm):
  11 + # Adding model 'MessageBlock'
  12 + db.create_table(u'super_archives_messageblock', (
  13 + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
  14 + ('message', self.gf('django.db.models.fields.related.ForeignKey')(related_name='blocks', to=orm['super_archives.Message'])),
  15 + ('text', self.gf('django.db.models.fields.TextField')()),
  16 + ('is_reply', self.gf('django.db.models.fields.BooleanField')()),
  17 + ('order', self.gf('django.db.models.fields.IntegerField')()),
  18 + ))
  19 + db.send_create_signal(u'super_archives', ['MessageBlock'])
  20 +
  21 +
  22 + def backwards(self, orm):
  23 + # Deleting model 'MessageBlock'
  24 + db.delete_table(u'super_archives_messageblock')
  25 +
  26 +
  27 + models = {
  28 + u'accounts.user': {
  29 + 'Meta': {'object_name': 'User'},
  30 + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
  31 + 'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75', 'blank': 'True'}),
  32 + 'facebook': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
  33 + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
  34 + 'google_talk': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
  35 + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
  36 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  37 + 'institution': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
  38 + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
  39 + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
  40 + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
  41 + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
  42 + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
  43 + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
  44 + 'role': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
  45 + 'twitter': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}),
  46 + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
  47 + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}),
  48 + 'verification_hash': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'}),
  49 + 'webpage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'null': 'True', 'blank': 'True'})
  50 + },
  51 + u'auth.group': {
  52 + 'Meta': {'object_name': 'Group'},
  53 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  54 + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
  55 + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
  56 + },
  57 + u'auth.permission': {
  58 + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
  59 + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
  60 + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
  61 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  62 + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
  63 + },
  64 + u'contenttypes.contenttype': {
  65 + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
  66 + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
  67 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  68 + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
  69 + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
  70 + },
  71 + u'super_archives.emailaddress': {
  72 + 'Meta': {'ordering': "('id',)", 'object_name': 'EmailAddress'},
  73 + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}),
  74 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  75 + 'md5': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}),
  76 + 'real_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '64', 'blank': 'True'}),
  77 + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': u"orm['accounts.User']"})
  78 + },
  79 + u'super_archives.emailaddressvalidation': {
  80 + 'Meta': {'unique_together': "(('user', 'address'),)", 'object_name': 'EmailAddressValidation'},
  81 + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}),
  82 + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
  83 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  84 + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails_not_validated'", 'null': 'True', 'to': u"orm['accounts.User']"}),
  85 + 'validation_key': ('django.db.models.fields.CharField', [], {'default': "'f7c4c15797f34834bf5a8b9bd84fabee'", 'max_length': '32', 'null': 'True'})
  86 + },
  87 + u'super_archives.keyword': {
  88 + 'Meta': {'ordering': "('?',)", 'object_name': 'Keyword'},
  89 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  90 + 'keyword': ('django.db.models.fields.CharField', [], {'max_length': "'128'"}),
  91 + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Thread']"}),
  92 + 'weight': ('django.db.models.fields.IntegerField', [], {'default': '0'})
  93 + },
  94 + u'super_archives.mailinglist': {
  95 + 'Meta': {'object_name': 'MailingList'},
  96 + 'description': ('django.db.models.fields.TextField', [], {}),
  97 + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}),
  98 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  99 + 'last_imported_index': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
  100 + 'logo': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
  101 + 'name': ('django.db.models.fields.CharField', [], {'max_length': '80'})
  102 + },
  103 + u'super_archives.mailinglistmembership': {
  104 + 'Meta': {'object_name': 'MailingListMembership'},
  105 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  106 + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.MailingList']"}),
  107 + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['accounts.User']"})
  108 + },
  109 + u'super_archives.message': {
  110 + 'Meta': {'ordering': "('received_time',)", 'unique_together': "(('thread', 'message_id'),)", 'object_name': 'Message'},
  111 + 'body': ('django.db.models.fields.TextField', [], {'default': "''"}),
  112 + 'from_address': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.EmailAddress']"}),
  113 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  114 + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
  115 + 'received_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
  116 + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
  117 + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}),
  118 + 'subject_clean': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}),
  119 + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Thread']", 'null': 'True'})
  120 + },
  121 + u'super_archives.messageblock': {
  122 + 'Meta': {'ordering': "('order',)", 'object_name': 'MessageBlock'},
  123 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  124 + 'is_reply': ('django.db.models.fields.BooleanField', [], {}),
  125 + 'message': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'blocks'", 'to': u"orm['super_archives.Message']"}),
  126 + 'order': ('django.db.models.fields.IntegerField', [], {}),
  127 + 'text': ('django.db.models.fields.TextField', [], {})
  128 + },
  129 + u'super_archives.messagemetadata': {
  130 + 'Message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Message']"}),
  131 + 'Meta': {'object_name': 'MessageMetadata'},
  132 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  133 + 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
  134 + 'value': ('django.db.models.fields.TextField', [], {})
  135 + },
  136 + u'super_archives.pagehit': {
  137 + 'Meta': {'object_name': 'PageHit'},
  138 + 'hit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
  139 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  140 + 'url_path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'})
  141 + },
  142 + u'super_archives.thread': {
  143 + 'Meta': {'unique_together': "(('subject_token', 'mailinglist'),)", 'object_name': 'Thread'},
  144 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  145 + 'latest_message': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'+'", 'unique': 'True', 'null': 'True', 'to': u"orm['super_archives.Message']"}),
  146 + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.MailingList']"}),
  147 + 'score': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
  148 + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
  149 + 'subject_token': ('django.db.models.fields.CharField', [], {'max_length': '512'})
  150 + },
  151 + u'super_archives.vote': {
  152 + 'Meta': {'unique_together': "(('user', 'message'),)", 'object_name': 'Vote'},
  153 + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
  154 + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  155 + 'message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Message']"}),
  156 + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['accounts.User']"})
  157 + }
  158 + }
  159 +
  160 + complete_apps = ['super_archives']
0 161 \ No newline at end of file
... ...
src/super_archives/models.py
  Diff comments   View file @eaa3aa9
... ... @@ -123,18 +123,12 @@ class Thread(models.Model):
123 123 verbose_name_plural = _(u"Threads")
124 124 unique_together = ('subject_token', 'mailinglist')
125 125  
126   - def gen_tags(self):
127   - blocks = []
128   - for message in self.message_set.iterator():
129   - blocks.extend([block for block in message.blocks()
130   - if not block.is_reply])
  126 + def update_keywords(self):
  127 + blocks = MessageBlock.objects.filter(message__thread__pk=self.pk,
  128 + is_reply=False)
  129 +
131 130 text = u'\n'.join(map(unicode, blocks))
132 131 tags = etiquetador(html2text(text))
133   - return tags
134   -
135   - def save(self, *args, **kwargs):
136   - super(Thread, self).save(*args, **kwargs)
137   - tags = self.gen_tags()
138 132  
139 133 for tag, weight in tags:
140 134 keyword, created = Keyword.objects.get_or_create(thread=self,
... ... @@ -251,13 +245,12 @@ class Message(models.Model):
251 245 self.from_address.get_full_name(),
252 246 self.subject_clean)
253 247  
254   - def blocks(self):
255   - cache_key = 'email-blocks-{}'.format(self.pk)
256   - blks = cache.get(cache_key)
257   - if not blks:
258   - blks = blocks.EmailBlockParser(self)
259   - cache.set(cache_key, blks)
260   - return blks
  248 + def update_blocks(self):
  249 + # delete all blocks for that message
  250 + self.blocks.all().delete()
  251 +
  252 + for i, block in enumerate(blocks.EmailBlockParser(self)):
  253 + MessageBlock.from_emailblock(block, self, i)
261 254  
262 255 @property
263 256 def mailinglist(self):
... ... @@ -307,6 +300,28 @@ class Message(models.Model):
307 300 return self.received_time
308 301  
309 302  
  303 +class MessageBlock(models.Model):
  304 + message = models.ForeignKey(Message, related_name='blocks')
  305 + text = models.TextField()
  306 + is_reply = models.BooleanField()
  307 + order = models.IntegerField()
  308 +
  309 + def __unicode__(self):
  310 + return self.text
  311 +
  312 + class Meta:
  313 + ordering = ('order', )
  314 +
  315 + @classmethod
  316 + def from_emailblock(klass, emailblock, message, order):
  317 + obj = klass.objects.create(text=emailblock.text,
  318 + is_reply=emailblock.is_reply,
  319 + message=message,
  320 + order=order)
  321 + return obj
  322 +
  323 +
  324 +
310 325 class MessageMetadata(models.Model):
311 326 Message = models.ForeignKey(Message)
312 327 # Same problem here than on subjects. Read comment above
... ...
src/super_archives/templatetags/superarchives.py
  Diff comments   View file @eaa3aa9
... ... @@ -8,4 +8,7 @@ TEMPLATE_PATH = 'superarchives/tags/'
8 8  
9 9 @register.inclusion_tag(TEMPLATE_PATH + 'display_message.html')
10 10 def display_message(email):
11   - return {'blocks': email.blocks()}
  11 + if not email.blocks.count():
  12 + email.update_blocks()
  13 +
  14 + return { 'blocks': email.blocks.all }
... ...
src/super_archives/utils/blocks.py
  Diff comments   View file @eaa3aa9
... ... @@ -16,6 +16,7 @@ RE_REPLY_LINE = re.compile(r'^[\s\t>]*>[\s\t]*')
16 16  
17 17 RE_BR_TO_LINEBREAK = re.compile(r'<\s*/?\s*br\s*/?\s*>')
18 18  
  19 +
19 20 class EmailBlock(list):
20 21 def __init__(self, is_reply=False, mark_links=True, html2text=True):
21 22 self.mark_links = mark_links
... ...