Download as
Browse Code »

Commit 48ac76aa76306c6a092d4d0b2edef83cdd531270

Authored by seocam
1 parent 18791ad7
Exists in master and in 39 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, check-port, colab-signals, colab-vcard, colab_search, colab_tag, colab_tag_merge, community_association, detach_super_archives, fix-dashboard, fix-message-preview, fix_app_label, fix_cache_count_spb, fix_header, fixed_gitlab_import, go_to_profile_panel, header_footer, layout-fix, mobile_user_scalable, move_out_plugins, paginate-threads, profile_integration, refactor-data-import, remove-gitlab-plugin, removing_namespace, search_block, search_filters, spb-release/3.0, split_screen, stable, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd, workin_whoosh_temp

Merge com https://bitbucket.org/seocam/atu-colab/src/d0fa6287a857 


git-svn-id: http://repositorio.interlegis.gov.br/colab/trunk@6285 bee1b3ed-c3eb-0310-9994-b88e04532788
Showing 3 changed files with 163 additions and 18 deletions   Show diff stats
colab/super_archives/management/commands/import_emails.py
  Diff comments   View file @48ac76a
@@ -133,14 +133,16 @@ class Command(BaseCommand, object): @@ -133,14 +133,16 @@ class Command(BaseCommand, object):
133 """Group messages by thread looking for similar subjects""" 133 """Group messages by thread looking for similar subjects"""
134 134
135 subject_slug = slugify(email.subject_clean) 135 subject_slug = slugify(email.subject_clean)
136 - thread = self.THREAD_CACHE.get(subject_slug) 136 + thread = self.THREAD_CACHE.get(subject_slug, {}).get(mailinglist.id)
137 if thread is None: 137 if thread is None:
138 thread = Thread.objects.get_or_create( 138 thread = Thread.objects.get_or_create(
139 mailinglist=mailinglist, 139 mailinglist=mailinglist,
140 subject_token=subject_slug 140 subject_token=subject_slug
141 )[0] 141 )[0]
142 -  
143 - self.THREAD_CACHE[subject_slug] = thread 142 +
  143 + if self.THREAD_CACHE.get(subject_slug) is None:
  144 + self.THREAD_CACHE[subject_slug] = dict()
  145 + self.THREAD_CACHE[subject_slug][mailinglist.id] = thread
144 146
145 thread.latest_message = email 147 thread.latest_message = email
146 thread.save() 148 thread.save()
@@ -150,20 +152,24 @@ class Command(BaseCommand, object): @@ -150,20 +152,24 @@ class Command(BaseCommand, object):
150 """Save email message into the database.""" 152 """Save email message into the database."""
151 153
152 # Update last imported message into the DB 154 # Update last imported message into the DB
153 - mailinglist = MailingList.objects.get_or_create(name=list_name)[0] 155 + mailinglist, created = MailingList.objects.get_or_create(name=list_name)
154 mailinglist.last_imported_index = index 156 mailinglist.last_imported_index = index
155 -  
156 - # If the message is already at the database don't do anything  
157 - messages = Message.objects.filter(  
158 - message_id=email_msg.get('Message-ID'))  
159 - create = False  
160 - if not messages:  
161 - create = True  
162 - elif messages[0].thread.mailinglist.name != mailinglist.name:  
163 - create = True  
164 -  
165 - if create: 157 +
  158 + if created:
  159 + # if the mailinglist is newly created it's sure that the message
  160 + # is not in the DB yet.
166 self.create_email(mailinglist, email_msg) 161 self.create_email(mailinglist, email_msg)
  162 +
  163 + else:
  164 + # If the message is already at the database don't do anything
  165 + try:
  166 + messages = Message.objects.get(
  167 + message_id=email_msg.get('Message-ID'),
  168 + thread__mailinglist=mailinglist
  169 + )
  170 +
  171 + except Message.DoesNotExist:
  172 + self.create_email(mailinglist, email_msg)
167 173
168 mailinglist.save() 174 mailinglist.save()
169 175
colab/super_archives/migrations/0010_auto__add_unique_message_message_id_thread.py 0 → 100644
  Diff comments   View file @48ac76a
@@ -0,0 +1,136 @@ @@ -0,0 +1,136 @@
  1 +# encoding: utf-8
  2 +import datetime
  3 +from south.db import db
  4 +from south.v2 import SchemaMigration
  5 +from django.db import models
  6 +
  7 +class Migration(SchemaMigration):
  8 +
  9 + def forwards(self, orm):
  10 +
  11 + # Adding unique constraint on 'Message', fields ['message_id', 'thread']
  12 + db.create_unique('super_archives_message', ['message_id', 'thread_id'])
  13 +
  14 +
  15 + def backwards(self, orm):
  16 +
  17 + # Removing unique constraint on 'Message', fields ['message_id', 'thread']
  18 + db.delete_unique('super_archives_message', ['message_id', 'thread_id'])
  19 +
  20 +
  21 + models = {
  22 + 'auth.group': {
  23 + 'Meta': {'object_name': 'Group'},
  24 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  25 + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
  26 + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
  27 + },
  28 + 'auth.permission': {
  29 + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
  30 + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
  31 + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
  32 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  33 + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
  34 + },
  35 + 'auth.user': {
  36 + 'Meta': {'object_name': 'User'},
  37 + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
  38 + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
  39 + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
  40 + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
  41 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  42 + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
  43 + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
  44 + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
  45 + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
  46 + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
  47 + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
  48 + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
  49 + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
  50 + },
  51 + 'contenttypes.contenttype': {
  52 + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
  53 + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
  54 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  55 + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
  56 + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
  57 + },
  58 + 'super_archives.emailaddress': {
  59 + 'Meta': {'object_name': 'EmailAddress'},
  60 + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}),
  61 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  62 + 'md5': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}),
  63 + 'real_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '64', 'blank': 'True'}),
  64 + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails'", 'null': 'True', 'to': "orm['auth.User']"})
  65 + },
  66 + 'super_archives.mailinglist': {
  67 + 'Meta': {'object_name': 'MailingList'},
  68 + 'description': ('django.db.models.fields.TextField', [], {}),
  69 + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}),
  70 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  71 + 'last_imported_index': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
  72 + 'logo': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
  73 + 'name': ('django.db.models.fields.CharField', [], {'max_length': '80'})
  74 + },
  75 + 'super_archives.mailinglistmembership': {
  76 + 'Meta': {'object_name': 'MailingListMembership'},
  77 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  78 + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.MailingList']"}),
  79 + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
  80 + },
  81 + 'super_archives.message': {
  82 + 'Meta': {'unique_together': "(('thread', 'message_id'),)", 'object_name': 'Message'},
  83 + 'body': ('django.db.models.fields.TextField', [], {'default': "''"}),
  84 + 'from_address': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.EmailAddress']"}),
  85 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  86 + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
  87 + 'received_time': ('django.db.models.fields.DateTimeField', [], {}),
  88 + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
  89 + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}),
  90 + 'subject_clean': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}),
  91 + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Thread']", 'null': 'True'})
  92 + },
  93 + 'super_archives.messagemetadata': {
  94 + 'Message': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Message']"}),
  95 + 'Meta': {'object_name': 'MessageMetadata'},
  96 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  97 + 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}),
  98 + 'value': ('django.db.models.fields.TextField', [], {})
  99 + },
  100 + 'super_archives.pagehit': {
  101 + 'Meta': {'object_name': 'PageHit'},
  102 + 'hit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
  103 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  104 + 'url_path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'})
  105 + },
  106 + 'super_archives.thread': {
  107 + 'Meta': {'unique_together': "(('subject_token', 'mailinglist'),)", 'object_name': 'Thread'},
  108 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  109 + 'latest_message': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'+'", 'unique': 'True', 'null': 'True', 'to': "orm['super_archives.Message']"}),
  110 + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.MailingList']"}),
  111 + 'score': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
  112 + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
  113 + 'subject_token': ('django.db.models.fields.CharField', [], {'max_length': '512'})
  114 + },
  115 + 'super_archives.userprofile': {
  116 + 'Meta': {'object_name': 'UserProfile'},
  117 + 'facebook': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}),
  118 + 'google_talk': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True'}),
  119 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  120 + 'institution': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}),
  121 + 'role': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}),
  122 + 'twitter': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}),
  123 + 'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'}),
  124 + 'verification_hash': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}),
  125 + 'webpage': ('django.db.models.fields.CharField', [], {'max_length': '256'})
  126 + },
  127 + 'super_archives.vote': {
  128 + 'Meta': {'unique_together': "(('user', 'message'),)", 'object_name': 'Vote'},
  129 + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
  130 + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
  131 + 'message': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Message']"}),
  132 + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
  133 + }
  134 + }
  135 +
  136 + complete_apps = ['super_archives']
colab/super_archives/models.py
  Diff comments   View file @48ac76a
@@ -88,8 +88,6 @@ class MailingListMembership(models.Model): @@ -88,8 +88,6 @@ class MailingListMembership(models.Model):
88 88
89 89
90 class Thread(models.Model): 90 class Thread(models.Model):
91 - class Meta:  
92 - unique_together = ('subject_token', 'mailinglist')  
93 91
94 subject_token = models.CharField(max_length=512) 92 subject_token = models.CharField(max_length=512)
95 mailinglist = models.ForeignKey(MailingList) 93 mailinglist = models.ForeignKey(MailingList)
@@ -100,6 +98,9 @@ class Thread(models.Model): @@ -100,6 +98,9 @@ class Thread(models.Model):
100 98
101 all_objects = models.Manager() 99 all_objects = models.Manager()
102 objects = NotSpamManager() 100 objects = NotSpamManager()
  101 +
  102 + class Meta:
  103 + unique_together = ('subject_token', 'mailinglist')
103 104
104 def __unicode__(self): 105 def __unicode__(self):
105 return '%s - %s (%s)' % (self.id, 106 return '%s - %s (%s)' % (self.id,
@@ -175,7 +176,6 @@ class Vote(models.Model): @@ -175,7 +176,6 @@ class Vote(models.Model):
175 class Message(models.Model): 176 class Message(models.Model):
176 177
177 from_address = models.ForeignKey(EmailAddress, db_index=True) 178 from_address = models.ForeignKey(EmailAddress, db_index=True)
178 - mailinglist = models.ForeignKey(MailingList)  
179 thread = models.ForeignKey(Thread, null=True, db_index=True) 179 thread = models.ForeignKey(Thread, null=True, db_index=True)
180 # RFC 2822 recommends to use 78 chars + CRLF (so 80 chars) for 180 # RFC 2822 recommends to use 78 chars + CRLF (so 80 chars) for
181 # the max_length of a subject but most of implementations 181 # the max_length of a subject but most of implementations
@@ -190,6 +190,9 @@ class Message(models.Model): @@ -190,6 +190,9 @@ class Message(models.Model):
190 all_objects = models.Manager() 190 all_objects = models.Manager()
191 objects = NotSpamManager() 191 objects = NotSpamManager()
192 192
  193 + class Meta:
  194 + unique_together = ('thread', 'message_id')
  195 +
193 def __unicode__(self): 196 def __unicode__(self):
194 return '(%s) %s: %s' % (self.id, 197 return '(%s) %s: %s' % (self.id,
195 self.from_address.get_full_name(), 198 self.from_address.get_full_name(),