Commit eaa3aa9f1d2aaa77f0306f7966f9bd0768e800de
1 parent
435ccefc
Exists in
master
and in
39 other branches
Using email blocks from db
Showing
4 changed files
with
197 additions
and
18 deletions
Show diff stats
src/super_archives/migrations/0019_auto__add_messageblock.py
0 → 100644
@@ -0,0 +1,160 @@ | @@ -0,0 +1,160 @@ | ||
1 | +# -*- coding: utf-8 -*- | ||
2 | +import datetime | ||
3 | +from south.db import db | ||
4 | +from south.v2 import SchemaMigration | ||
5 | +from django.db import models | ||
6 | + | ||
7 | + | ||
8 | +class Migration(SchemaMigration): | ||
9 | + | ||
10 | + def forwards(self, orm): | ||
11 | + # Adding model 'MessageBlock' | ||
12 | + db.create_table(u'super_archives_messageblock', ( | ||
13 | + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), | ||
14 | + ('message', self.gf('django.db.models.fields.related.ForeignKey')(related_name='blocks', to=orm['super_archives.Message'])), | ||
15 | + ('text', self.gf('django.db.models.fields.TextField')()), | ||
16 | + ('is_reply', self.gf('django.db.models.fields.BooleanField')()), | ||
17 | + ('order', self.gf('django.db.models.fields.IntegerField')()), | ||
18 | + )) | ||
19 | + db.send_create_signal(u'super_archives', ['MessageBlock']) | ||
20 | + | ||
21 | + | ||
22 | + def backwards(self, orm): | ||
23 | + # Deleting model 'MessageBlock' | ||
24 | + db.delete_table(u'super_archives_messageblock') | ||
25 | + | ||
26 | + | ||
27 | + models = { | ||
28 | + u'accounts.user': { | ||
29 | + 'Meta': {'object_name': 'User'}, | ||
30 | + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), | ||
31 | + 'email': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75', 'blank': 'True'}), | ||
32 | + 'facebook': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), | ||
33 | + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), | ||
34 | + 'google_talk': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), | ||
35 | + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), | ||
36 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
37 | + 'institution': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), | ||
38 | + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), | ||
39 | + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
40 | + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
41 | + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), | ||
42 | + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), | ||
43 | + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), | ||
44 | + 'role': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), | ||
45 | + 'twitter': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), | ||
46 | + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), | ||
47 | + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}), | ||
48 | + 'verification_hash': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'}), | ||
49 | + 'webpage': ('django.db.models.fields.CharField', [], {'max_length': '256', 'null': 'True', 'blank': 'True'}) | ||
50 | + }, | ||
51 | + u'auth.group': { | ||
52 | + 'Meta': {'object_name': 'Group'}, | ||
53 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
54 | + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), | ||
55 | + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) | ||
56 | + }, | ||
57 | + u'auth.permission': { | ||
58 | + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, | ||
59 | + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), | ||
60 | + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), | ||
61 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
62 | + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) | ||
63 | + }, | ||
64 | + u'contenttypes.contenttype': { | ||
65 | + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, | ||
66 | + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), | ||
67 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
68 | + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), | ||
69 | + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) | ||
70 | + }, | ||
71 | + u'super_archives.emailaddress': { | ||
72 | + 'Meta': {'ordering': "('id',)", 'object_name': 'EmailAddress'}, | ||
73 | + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}), | ||
74 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
75 | + 'md5': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}), | ||
76 | + 'real_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '64', 'blank': 'True'}), | ||
77 | + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': u"orm['accounts.User']"}) | ||
78 | + }, | ||
79 | + u'super_archives.emailaddressvalidation': { | ||
80 | + 'Meta': {'unique_together': "(('user', 'address'),)", 'object_name': 'EmailAddressValidation'}, | ||
81 | + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}), | ||
82 | + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), | ||
83 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
84 | + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails_not_validated'", 'null': 'True', 'to': u"orm['accounts.User']"}), | ||
85 | + 'validation_key': ('django.db.models.fields.CharField', [], {'default': "'f7c4c15797f34834bf5a8b9bd84fabee'", 'max_length': '32', 'null': 'True'}) | ||
86 | + }, | ||
87 | + u'super_archives.keyword': { | ||
88 | + 'Meta': {'ordering': "('?',)", 'object_name': 'Keyword'}, | ||
89 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
90 | + 'keyword': ('django.db.models.fields.CharField', [], {'max_length': "'128'"}), | ||
91 | + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Thread']"}), | ||
92 | + 'weight': ('django.db.models.fields.IntegerField', [], {'default': '0'}) | ||
93 | + }, | ||
94 | + u'super_archives.mailinglist': { | ||
95 | + 'Meta': {'object_name': 'MailingList'}, | ||
96 | + 'description': ('django.db.models.fields.TextField', [], {}), | ||
97 | + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), | ||
98 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
99 | + 'last_imported_index': ('django.db.models.fields.IntegerField', [], {'default': '0'}), | ||
100 | + 'logo': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), | ||
101 | + 'name': ('django.db.models.fields.CharField', [], {'max_length': '80'}) | ||
102 | + }, | ||
103 | + u'super_archives.mailinglistmembership': { | ||
104 | + 'Meta': {'object_name': 'MailingListMembership'}, | ||
105 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
106 | + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.MailingList']"}), | ||
107 | + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['accounts.User']"}) | ||
108 | + }, | ||
109 | + u'super_archives.message': { | ||
110 | + 'Meta': {'ordering': "('received_time',)", 'unique_together': "(('thread', 'message_id'),)", 'object_name': 'Message'}, | ||
111 | + 'body': ('django.db.models.fields.TextField', [], {'default': "''"}), | ||
112 | + 'from_address': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.EmailAddress']"}), | ||
113 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
114 | + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '512'}), | ||
115 | + 'received_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), | ||
116 | + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
117 | + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), | ||
118 | + 'subject_clean': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), | ||
119 | + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Thread']", 'null': 'True'}) | ||
120 | + }, | ||
121 | + u'super_archives.messageblock': { | ||
122 | + 'Meta': {'ordering': "('order',)", 'object_name': 'MessageBlock'}, | ||
123 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
124 | + 'is_reply': ('django.db.models.fields.BooleanField', [], {}), | ||
125 | + 'message': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'blocks'", 'to': u"orm['super_archives.Message']"}), | ||
126 | + 'order': ('django.db.models.fields.IntegerField', [], {}), | ||
127 | + 'text': ('django.db.models.fields.TextField', [], {}) | ||
128 | + }, | ||
129 | + u'super_archives.messagemetadata': { | ||
130 | + 'Message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Message']"}), | ||
131 | + 'Meta': {'object_name': 'MessageMetadata'}, | ||
132 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
133 | + 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}), | ||
134 | + 'value': ('django.db.models.fields.TextField', [], {}) | ||
135 | + }, | ||
136 | + u'super_archives.pagehit': { | ||
137 | + 'Meta': {'object_name': 'PageHit'}, | ||
138 | + 'hit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), | ||
139 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
140 | + 'url_path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'}) | ||
141 | + }, | ||
142 | + u'super_archives.thread': { | ||
143 | + 'Meta': {'unique_together': "(('subject_token', 'mailinglist'),)", 'object_name': 'Thread'}, | ||
144 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
145 | + 'latest_message': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'+'", 'unique': 'True', 'null': 'True', 'to': u"orm['super_archives.Message']"}), | ||
146 | + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.MailingList']"}), | ||
147 | + 'score': ('django.db.models.fields.IntegerField', [], {'default': '0'}), | ||
148 | + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
149 | + 'subject_token': ('django.db.models.fields.CharField', [], {'max_length': '512'}) | ||
150 | + }, | ||
151 | + u'super_archives.vote': { | ||
152 | + 'Meta': {'unique_together': "(('user', 'message'),)", 'object_name': 'Vote'}, | ||
153 | + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), | ||
154 | + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
155 | + 'message': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['super_archives.Message']"}), | ||
156 | + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['accounts.User']"}) | ||
157 | + } | ||
158 | + } | ||
159 | + | ||
160 | + complete_apps = ['super_archives'] | ||
0 | \ No newline at end of file | 161 | \ No newline at end of file |
src/super_archives/models.py
@@ -123,18 +123,12 @@ class Thread(models.Model): | @@ -123,18 +123,12 @@ class Thread(models.Model): | ||
123 | verbose_name_plural = _(u"Threads") | 123 | verbose_name_plural = _(u"Threads") |
124 | unique_together = ('subject_token', 'mailinglist') | 124 | unique_together = ('subject_token', 'mailinglist') |
125 | 125 | ||
126 | - def gen_tags(self): | ||
127 | - blocks = [] | ||
128 | - for message in self.message_set.iterator(): | ||
129 | - blocks.extend([block for block in message.blocks() | ||
130 | - if not block.is_reply]) | 126 | + def update_keywords(self): |
127 | + blocks = MessageBlock.objects.filter(message__thread__pk=self.pk, | ||
128 | + is_reply=False) | ||
129 | + | ||
131 | text = u'\n'.join(map(unicode, blocks)) | 130 | text = u'\n'.join(map(unicode, blocks)) |
132 | tags = etiquetador(html2text(text)) | 131 | tags = etiquetador(html2text(text)) |
133 | - return tags | ||
134 | - | ||
135 | - def save(self, *args, **kwargs): | ||
136 | - super(Thread, self).save(*args, **kwargs) | ||
137 | - tags = self.gen_tags() | ||
138 | 132 | ||
139 | for tag, weight in tags: | 133 | for tag, weight in tags: |
140 | keyword, created = Keyword.objects.get_or_create(thread=self, | 134 | keyword, created = Keyword.objects.get_or_create(thread=self, |
@@ -251,13 +245,12 @@ class Message(models.Model): | @@ -251,13 +245,12 @@ class Message(models.Model): | ||
251 | self.from_address.get_full_name(), | 245 | self.from_address.get_full_name(), |
252 | self.subject_clean) | 246 | self.subject_clean) |
253 | 247 | ||
254 | - def blocks(self): | ||
255 | - cache_key = 'email-blocks-{}'.format(self.pk) | ||
256 | - blks = cache.get(cache_key) | ||
257 | - if not blks: | ||
258 | - blks = blocks.EmailBlockParser(self) | ||
259 | - cache.set(cache_key, blks) | ||
260 | - return blks | 248 | + def update_blocks(self): |
249 | + # delete all blocks for that message | ||
250 | + self.blocks.all().delete() | ||
251 | + | ||
252 | + for i, block in enumerate(blocks.EmailBlockParser(self)): | ||
253 | + MessageBlock.from_emailblock(block, self, i) | ||
261 | 254 | ||
262 | @property | 255 | @property |
263 | def mailinglist(self): | 256 | def mailinglist(self): |
@@ -307,6 +300,28 @@ class Message(models.Model): | @@ -307,6 +300,28 @@ class Message(models.Model): | ||
307 | return self.received_time | 300 | return self.received_time |
308 | 301 | ||
309 | 302 | ||
303 | +class MessageBlock(models.Model): | ||
304 | + message = models.ForeignKey(Message, related_name='blocks') | ||
305 | + text = models.TextField() | ||
306 | + is_reply = models.BooleanField() | ||
307 | + order = models.IntegerField() | ||
308 | + | ||
309 | + def __unicode__(self): | ||
310 | + return self.text | ||
311 | + | ||
312 | + class Meta: | ||
313 | + ordering = ('order', ) | ||
314 | + | ||
315 | + @classmethod | ||
316 | + def from_emailblock(klass, emailblock, message, order): | ||
317 | + obj = klass.objects.create(text=emailblock.text, | ||
318 | + is_reply=emailblock.is_reply, | ||
319 | + message=message, | ||
320 | + order=order) | ||
321 | + return obj | ||
322 | + | ||
323 | + | ||
324 | + | ||
310 | class MessageMetadata(models.Model): | 325 | class MessageMetadata(models.Model): |
311 | Message = models.ForeignKey(Message) | 326 | Message = models.ForeignKey(Message) |
312 | # Same problem here than on subjects. Read comment above | 327 | # Same problem here than on subjects. Read comment above |
src/super_archives/templatetags/superarchives.py
@@ -8,4 +8,7 @@ TEMPLATE_PATH = 'superarchives/tags/' | @@ -8,4 +8,7 @@ TEMPLATE_PATH = 'superarchives/tags/' | ||
8 | 8 | ||
9 | @register.inclusion_tag(TEMPLATE_PATH + 'display_message.html') | 9 | @register.inclusion_tag(TEMPLATE_PATH + 'display_message.html') |
10 | def display_message(email): | 10 | def display_message(email): |
11 | - return {'blocks': email.blocks()} | 11 | + if not email.blocks.count(): |
12 | + email.update_blocks() | ||
13 | + | ||
14 | + return { 'blocks': email.blocks.all } |
src/super_archives/utils/blocks.py
@@ -16,6 +16,7 @@ RE_REPLY_LINE = re.compile(r'^[\s\t>]*>[\s\t]*') | @@ -16,6 +16,7 @@ RE_REPLY_LINE = re.compile(r'^[\s\t>]*>[\s\t]*') | ||
16 | 16 | ||
17 | RE_BR_TO_LINEBREAK = re.compile(r'<\s*/?\s*br\s*/?\s*>') | 17 | RE_BR_TO_LINEBREAK = re.compile(r'<\s*/?\s*br\s*/?\s*>') |
18 | 18 | ||
19 | + | ||
19 | class EmailBlock(list): | 20 | class EmailBlock(list): |
20 | def __init__(self, is_reply=False, mark_links=True, html2text=True): | 21 | def __init__(self, is_reply=False, mark_links=True, html2text=True): |
21 | self.mark_links = mark_links | 22 | self.mark_links = mark_links |