Commit 48ac76aa76306c6a092d4d0b2edef83cdd531270
1 parent
18791ad7
Exists in
master
and in
39 other branches
Merge com https://bitbucket.org/seocam/atu-colab/src/d0fa6287a857
git-svn-id: http://repositorio.interlegis.gov.br/colab/trunk@6285 bee1b3ed-c3eb-0310-9994-b88e04532788
Showing
3 changed files
with
163 additions
and
18 deletions
Show diff stats
colab/super_archives/management/commands/import_emails.py
@@ -133,14 +133,16 @@ class Command(BaseCommand, object): | @@ -133,14 +133,16 @@ class Command(BaseCommand, object): | ||
133 | """Group messages by thread looking for similar subjects""" | 133 | """Group messages by thread looking for similar subjects""" |
134 | 134 | ||
135 | subject_slug = slugify(email.subject_clean) | 135 | subject_slug = slugify(email.subject_clean) |
136 | - thread = self.THREAD_CACHE.get(subject_slug) | 136 | + thread = self.THREAD_CACHE.get(subject_slug, {}).get(mailinglist.id) |
137 | if thread is None: | 137 | if thread is None: |
138 | thread = Thread.objects.get_or_create( | 138 | thread = Thread.objects.get_or_create( |
139 | mailinglist=mailinglist, | 139 | mailinglist=mailinglist, |
140 | subject_token=subject_slug | 140 | subject_token=subject_slug |
141 | )[0] | 141 | )[0] |
142 | - | ||
143 | - self.THREAD_CACHE[subject_slug] = thread | 142 | + |
143 | + if self.THREAD_CACHE.get(subject_slug) is None: | ||
144 | + self.THREAD_CACHE[subject_slug] = dict() | ||
145 | + self.THREAD_CACHE[subject_slug][mailinglist.id] = thread | ||
144 | 146 | ||
145 | thread.latest_message = email | 147 | thread.latest_message = email |
146 | thread.save() | 148 | thread.save() |
@@ -150,20 +152,24 @@ class Command(BaseCommand, object): | @@ -150,20 +152,24 @@ class Command(BaseCommand, object): | ||
150 | """Save email message into the database.""" | 152 | """Save email message into the database.""" |
151 | 153 | ||
152 | # Update last imported message into the DB | 154 | # Update last imported message into the DB |
153 | - mailinglist = MailingList.objects.get_or_create(name=list_name)[0] | 155 | + mailinglist, created = MailingList.objects.get_or_create(name=list_name) |
154 | mailinglist.last_imported_index = index | 156 | mailinglist.last_imported_index = index |
155 | - | ||
156 | - # If the message is already at the database don't do anything | ||
157 | - messages = Message.objects.filter( | ||
158 | - message_id=email_msg.get('Message-ID')) | ||
159 | - create = False | ||
160 | - if not messages: | ||
161 | - create = True | ||
162 | - elif messages[0].thread.mailinglist.name != mailinglist.name: | ||
163 | - create = True | ||
164 | - | ||
165 | - if create: | 157 | + |
158 | + if created: | ||
159 | + # if the mailinglist is newly created it's sure that the message | ||
160 | + # is not in the DB yet. | ||
166 | self.create_email(mailinglist, email_msg) | 161 | self.create_email(mailinglist, email_msg) |
162 | + | ||
163 | + else: | ||
164 | + # If the message is already at the database don't do anything | ||
165 | + try: | ||
166 | + messages = Message.objects.get( | ||
167 | + message_id=email_msg.get('Message-ID'), | ||
168 | + thread__mailinglist=mailinglist | ||
169 | + ) | ||
170 | + | ||
171 | + except Message.DoesNotExist: | ||
172 | + self.create_email(mailinglist, email_msg) | ||
167 | 173 | ||
168 | mailinglist.save() | 174 | mailinglist.save() |
169 | 175 |
colab/super_archives/migrations/0010_auto__add_unique_message_message_id_thread.py
0 → 100644
@@ -0,0 +1,136 @@ | @@ -0,0 +1,136 @@ | ||
1 | +# encoding: utf-8 | ||
2 | +import datetime | ||
3 | +from south.db import db | ||
4 | +from south.v2 import SchemaMigration | ||
5 | +from django.db import models | ||
6 | + | ||
7 | +class Migration(SchemaMigration): | ||
8 | + | ||
9 | + def forwards(self, orm): | ||
10 | + | ||
11 | + # Adding unique constraint on 'Message', fields ['message_id', 'thread'] | ||
12 | + db.create_unique('super_archives_message', ['message_id', 'thread_id']) | ||
13 | + | ||
14 | + | ||
15 | + def backwards(self, orm): | ||
16 | + | ||
17 | + # Removing unique constraint on 'Message', fields ['message_id', 'thread'] | ||
18 | + db.delete_unique('super_archives_message', ['message_id', 'thread_id']) | ||
19 | + | ||
20 | + | ||
21 | + models = { | ||
22 | + 'auth.group': { | ||
23 | + 'Meta': {'object_name': 'Group'}, | ||
24 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
25 | + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), | ||
26 | + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) | ||
27 | + }, | ||
28 | + 'auth.permission': { | ||
29 | + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, | ||
30 | + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), | ||
31 | + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), | ||
32 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
33 | + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) | ||
34 | + }, | ||
35 | + 'auth.user': { | ||
36 | + 'Meta': {'object_name': 'User'}, | ||
37 | + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), | ||
38 | + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), | ||
39 | + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), | ||
40 | + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), | ||
41 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
42 | + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), | ||
43 | + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
44 | + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
45 | + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), | ||
46 | + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), | ||
47 | + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), | ||
48 | + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), | ||
49 | + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) | ||
50 | + }, | ||
51 | + 'contenttypes.contenttype': { | ||
52 | + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, | ||
53 | + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), | ||
54 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
55 | + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), | ||
56 | + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) | ||
57 | + }, | ||
58 | + 'super_archives.emailaddress': { | ||
59 | + 'Meta': {'object_name': 'EmailAddress'}, | ||
60 | + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}), | ||
61 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
62 | + 'md5': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}), | ||
63 | + 'real_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '64', 'blank': 'True'}), | ||
64 | + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails'", 'null': 'True', 'to': "orm['auth.User']"}) | ||
65 | + }, | ||
66 | + 'super_archives.mailinglist': { | ||
67 | + 'Meta': {'object_name': 'MailingList'}, | ||
68 | + 'description': ('django.db.models.fields.TextField', [], {}), | ||
69 | + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), | ||
70 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
71 | + 'last_imported_index': ('django.db.models.fields.IntegerField', [], {'default': '0'}), | ||
72 | + 'logo': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), | ||
73 | + 'name': ('django.db.models.fields.CharField', [], {'max_length': '80'}) | ||
74 | + }, | ||
75 | + 'super_archives.mailinglistmembership': { | ||
76 | + 'Meta': {'object_name': 'MailingListMembership'}, | ||
77 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
78 | + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.MailingList']"}), | ||
79 | + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) | ||
80 | + }, | ||
81 | + 'super_archives.message': { | ||
82 | + 'Meta': {'unique_together': "(('thread', 'message_id'),)", 'object_name': 'Message'}, | ||
83 | + 'body': ('django.db.models.fields.TextField', [], {'default': "''"}), | ||
84 | + 'from_address': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.EmailAddress']"}), | ||
85 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
86 | + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '512'}), | ||
87 | + 'received_time': ('django.db.models.fields.DateTimeField', [], {}), | ||
88 | + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
89 | + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), | ||
90 | + 'subject_clean': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), | ||
91 | + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Thread']", 'null': 'True'}) | ||
92 | + }, | ||
93 | + 'super_archives.messagemetadata': { | ||
94 | + 'Message': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Message']"}), | ||
95 | + 'Meta': {'object_name': 'MessageMetadata'}, | ||
96 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
97 | + 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}), | ||
98 | + 'value': ('django.db.models.fields.TextField', [], {}) | ||
99 | + }, | ||
100 | + 'super_archives.pagehit': { | ||
101 | + 'Meta': {'object_name': 'PageHit'}, | ||
102 | + 'hit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), | ||
103 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
104 | + 'url_path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'}) | ||
105 | + }, | ||
106 | + 'super_archives.thread': { | ||
107 | + 'Meta': {'unique_together': "(('subject_token', 'mailinglist'),)", 'object_name': 'Thread'}, | ||
108 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
109 | + 'latest_message': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'+'", 'unique': 'True', 'null': 'True', 'to': "orm['super_archives.Message']"}), | ||
110 | + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.MailingList']"}), | ||
111 | + 'score': ('django.db.models.fields.IntegerField', [], {'default': '0'}), | ||
112 | + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
113 | + 'subject_token': ('django.db.models.fields.CharField', [], {'max_length': '512'}) | ||
114 | + }, | ||
115 | + 'super_archives.userprofile': { | ||
116 | + 'Meta': {'object_name': 'UserProfile'}, | ||
117 | + 'facebook': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), | ||
118 | + 'google_talk': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True'}), | ||
119 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
120 | + 'institution': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), | ||
121 | + 'role': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), | ||
122 | + 'twitter': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), | ||
123 | + 'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'}), | ||
124 | + 'verification_hash': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}), | ||
125 | + 'webpage': ('django.db.models.fields.CharField', [], {'max_length': '256'}) | ||
126 | + }, | ||
127 | + 'super_archives.vote': { | ||
128 | + 'Meta': {'unique_together': "(('user', 'message'),)", 'object_name': 'Vote'}, | ||
129 | + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), | ||
130 | + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
131 | + 'message': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Message']"}), | ||
132 | + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) | ||
133 | + } | ||
134 | + } | ||
135 | + | ||
136 | + complete_apps = ['super_archives'] |
colab/super_archives/models.py
@@ -88,8 +88,6 @@ class MailingListMembership(models.Model): | @@ -88,8 +88,6 @@ class MailingListMembership(models.Model): | ||
88 | 88 | ||
89 | 89 | ||
90 | class Thread(models.Model): | 90 | class Thread(models.Model): |
91 | - class Meta: | ||
92 | - unique_together = ('subject_token', 'mailinglist') | ||
93 | 91 | ||
94 | subject_token = models.CharField(max_length=512) | 92 | subject_token = models.CharField(max_length=512) |
95 | mailinglist = models.ForeignKey(MailingList) | 93 | mailinglist = models.ForeignKey(MailingList) |
@@ -100,6 +98,9 @@ class Thread(models.Model): | @@ -100,6 +98,9 @@ class Thread(models.Model): | ||
100 | 98 | ||
101 | all_objects = models.Manager() | 99 | all_objects = models.Manager() |
102 | objects = NotSpamManager() | 100 | objects = NotSpamManager() |
101 | + | ||
102 | + class Meta: | ||
103 | + unique_together = ('subject_token', 'mailinglist') | ||
103 | 104 | ||
104 | def __unicode__(self): | 105 | def __unicode__(self): |
105 | return '%s - %s (%s)' % (self.id, | 106 | return '%s - %s (%s)' % (self.id, |
@@ -175,7 +176,6 @@ class Vote(models.Model): | @@ -175,7 +176,6 @@ class Vote(models.Model): | ||
175 | class Message(models.Model): | 176 | class Message(models.Model): |
176 | 177 | ||
177 | from_address = models.ForeignKey(EmailAddress, db_index=True) | 178 | from_address = models.ForeignKey(EmailAddress, db_index=True) |
178 | - mailinglist = models.ForeignKey(MailingList) | ||
179 | thread = models.ForeignKey(Thread, null=True, db_index=True) | 179 | thread = models.ForeignKey(Thread, null=True, db_index=True) |
180 | # RFC 2822 recommends to use 78 chars + CRLF (so 80 chars) for | 180 | # RFC 2822 recommends to use 78 chars + CRLF (so 80 chars) for |
181 | # the max_length of a subject but most of implementations | 181 | # the max_length of a subject but most of implementations |
@@ -190,6 +190,9 @@ class Message(models.Model): | @@ -190,6 +190,9 @@ class Message(models.Model): | ||
190 | all_objects = models.Manager() | 190 | all_objects = models.Manager() |
191 | objects = NotSpamManager() | 191 | objects = NotSpamManager() |
192 | 192 | ||
193 | + class Meta: | ||
194 | + unique_together = ('thread', 'message_id') | ||
195 | + | ||
193 | def __unicode__(self): | 196 | def __unicode__(self): |
194 | return '(%s) %s: %s' % (self.id, | 197 | return '(%s) %s: %s' % (self.id, |
195 | self.from_address.get_full_name(), | 198 | self.from_address.get_full_name(), |