From 48ac76aa76306c6a092d4d0b2edef83cdd531270 Mon Sep 17 00:00:00 2001 From: seocam Date: Thu, 3 May 2012 01:53:01 +0000 Subject: [PATCH] Merge com https://bitbucket.org/seocam/atu-colab/src/d0fa6287a857 --- colab/super_archives/management/commands/import_emails.py | 36 +++++++++++++++++++++--------------- colab/super_archives/migrations/0010_auto__add_unique_message_message_id_thread.py | 136 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ colab/super_archives/models.py | 9 ++++++--- 3 files changed, 163 insertions(+), 18 deletions(-) create mode 100644 colab/super_archives/migrations/0010_auto__add_unique_message_message_id_thread.py diff --git a/colab/super_archives/management/commands/import_emails.py b/colab/super_archives/management/commands/import_emails.py index ebeddb2..cfffb5a 100644 --- a/colab/super_archives/management/commands/import_emails.py +++ b/colab/super_archives/management/commands/import_emails.py @@ -133,14 +133,16 @@ class Command(BaseCommand, object): """Group messages by thread looking for similar subjects""" subject_slug = slugify(email.subject_clean) - thread = self.THREAD_CACHE.get(subject_slug) + thread = self.THREAD_CACHE.get(subject_slug, {}).get(mailinglist.id) if thread is None: thread = Thread.objects.get_or_create( mailinglist=mailinglist, subject_token=subject_slug )[0] - - self.THREAD_CACHE[subject_slug] = thread + + if self.THREAD_CACHE.get(subject_slug) is None: + self.THREAD_CACHE[subject_slug] = dict() + self.THREAD_CACHE[subject_slug][mailinglist.id] = thread thread.latest_message = email thread.save() @@ -150,20 +152,24 @@ class Command(BaseCommand, object): """Save email message into the database.""" # Update last imported message into the DB - mailinglist = MailingList.objects.get_or_create(name=list_name)[0] + mailinglist, created = MailingList.objects.get_or_create(name=list_name) mailinglist.last_imported_index = index - - # If the message is already at the database don't do anything - messages = Message.objects.filter( - message_id=email_msg.get('Message-ID')) - create = False - if not messages: - create = True - elif messages[0].thread.mailinglist.name != mailinglist.name: - create = True - - if create: + + if created: + # if the mailinglist is newly created it's sure that the message + # is not in the DB yet. self.create_email(mailinglist, email_msg) + + else: + # If the message is already at the database don't do anything + try: + messages = Message.objects.get( + message_id=email_msg.get('Message-ID'), + thread__mailinglist=mailinglist + ) + + except Message.DoesNotExist: + self.create_email(mailinglist, email_msg) mailinglist.save() diff --git a/colab/super_archives/migrations/0010_auto__add_unique_message_message_id_thread.py b/colab/super_archives/migrations/0010_auto__add_unique_message_message_id_thread.py new file mode 100644 index 0000000..ef2efb4 --- /dev/null +++ b/colab/super_archives/migrations/0010_auto__add_unique_message_message_id_thread.py @@ -0,0 +1,136 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding unique constraint on 'Message', fields ['message_id', 'thread'] + db.create_unique('super_archives_message', ['message_id', 'thread_id']) + + + def backwards(self, orm): + + # Removing unique constraint on 'Message', fields ['message_id', 'thread'] + db.delete_unique('super_archives_message', ['message_id', 'thread_id']) + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'super_archives.emailaddress': { + 'Meta': {'object_name': 'EmailAddress'}, + 'address': ('django.db.models.fields.EmailField', [], {'unique': 'True', 'max_length': '75'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'md5': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}), + 'real_name': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '64', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'emails'", 'null': 'True', 'to': "orm['auth.User']"}) + }, + 'super_archives.mailinglist': { + 'Meta': {'object_name': 'MailingList'}, + 'description': ('django.db.models.fields.TextField', [], {}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_imported_index': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'logo': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '80'}) + }, + 'super_archives.mailinglistmembership': { + 'Meta': {'object_name': 'MailingListMembership'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.MailingList']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + }, + 'super_archives.message': { + 'Meta': {'unique_together': "(('thread', 'message_id'),)", 'object_name': 'Message'}, + 'body': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'from_address': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.EmailAddress']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'received_time': ('django.db.models.fields.DateTimeField', [], {}), + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), + 'subject_clean': ('django.db.models.fields.CharField', [], {'max_length': '512', 'db_index': 'True'}), + 'thread': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Thread']", 'null': 'True'}) + }, + 'super_archives.messagemetadata': { + 'Message': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Message']"}), + 'Meta': {'object_name': 'MessageMetadata'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '512'}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'super_archives.pagehit': { + 'Meta': {'object_name': 'PageHit'}, + 'hit_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'url_path': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '2048', 'db_index': 'True'}) + }, + 'super_archives.thread': { + 'Meta': {'unique_together': "(('subject_token', 'mailinglist'),)", 'object_name': 'Thread'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'latest_message': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'+'", 'unique': 'True', 'null': 'True', 'to': "orm['super_archives.Message']"}), + 'mailinglist': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.MailingList']"}), + 'score': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'spam': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'subject_token': ('django.db.models.fields.CharField', [], {'max_length': '512'}) + }, + 'super_archives.userprofile': { + 'Meta': {'object_name': 'UserProfile'}, + 'facebook': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'google_talk': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'institution': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'role': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'twitter': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'user': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['auth.User']", 'unique': 'True'}), + 'verification_hash': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True'}), + 'webpage': ('django.db.models.fields.CharField', [], {'max_length': '256'}) + }, + 'super_archives.vote': { + 'Meta': {'unique_together': "(('user', 'message'),)", 'object_name': 'Vote'}, + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['super_archives.Message']"}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) + } + } + + complete_apps = ['super_archives'] diff --git a/colab/super_archives/models.py b/colab/super_archives/models.py index 195fe63..39ba60a 100644 --- a/colab/super_archives/models.py +++ b/colab/super_archives/models.py @@ -88,8 +88,6 @@ class MailingListMembership(models.Model): class Thread(models.Model): - class Meta: - unique_together = ('subject_token', 'mailinglist') subject_token = models.CharField(max_length=512) mailinglist = models.ForeignKey(MailingList) @@ -100,6 +98,9 @@ class Thread(models.Model): all_objects = models.Manager() objects = NotSpamManager() + + class Meta: + unique_together = ('subject_token', 'mailinglist') def __unicode__(self): return '%s - %s (%s)' % (self.id, @@ -175,7 +176,6 @@ class Vote(models.Model): class Message(models.Model): from_address = models.ForeignKey(EmailAddress, db_index=True) - mailinglist = models.ForeignKey(MailingList) thread = models.ForeignKey(Thread, null=True, db_index=True) # RFC 2822 recommends to use 78 chars + CRLF (so 80 chars) for # the max_length of a subject but most of implementations @@ -190,6 +190,9 @@ class Message(models.Model): all_objects = models.Manager() objects = NotSpamManager() + class Meta: + unique_together = ('thread', 'message_id') + def __unicode__(self): return '(%s) %s: %s' % (self.id, self.from_address.get_full_name(), -- libgit2 0.21.2