Remove haystack search from dashboard an profile

-Generalizes a structure for proxies to define its collaboration models -Remove haystack search from profile and search pages in pro of the colab own database

Remove haystack search from dashboard an profile
-Generalizes a structure for proxies to define its collaboration models -Remove haystack search from profile and search pages in pro of the colab own database
Rodrigo Siqueira de Melo · Gust
1 parent 19615e85
Showing 10 changed files with 169 additions and 71 deletions Show diff stats
colab/accounts/views.py
colab/home/views.py
colab/proxy/gitlab/apps.py
colab/proxy/jenkins/apps.py
colab/proxy/noosfero/apps.py
colab/proxy/redmine/apps.py
colab/proxy/trac/apps.py
colab/search/preview_block.py
colab/search/templates/search/search-message-preview.html
colab/search/utils.py
 #!/usr/bin/env python
 # encoding: utf-8
+import importlib
+import inspect
 from collections import OrderedDict
@@ -15,6 +17,7 @@ from django.shortcuts import render, redirect, get_object_or_404
 from django.core.urlresolvers import reverse
 from django.core.exceptions import PermissionDenied
 from django.views.generic import DetailView, UpdateView, TemplateView
+from django.apps import apps
 from conversejs import xmpp
 from conversejs.models import XMPPAccount
@@ -22,7 +25,7 @@ from haystack.query import SearchQuerySet
 from colab.super_archives.models import (EmailAddress, Message,
                                          EmailAddressValidation)
-from colab.search.utils import trans
+from colab.search.utils import trans, getCollaborationData
 # from proxy.trac.models import WikiCollabCount, TicketCollabCount
 from .forms import (UserCreationForm, UserForm, ListsForm,
                     UserUpdateForm, ChangeXMPPPasswordForm)
@@ -71,48 +74,19 @@ class UserProfileDetailView(UserProfileBaseMixin, DetailView):
         count_types = OrderedDict()
-        fields_or_lookup = (
-            {'collaborators__contains': user.username},
-            {'fullname_and_username__contains': user.username},
-        )
+        # TODO: remove when mailman becomes a proxied plugin
+        messages = Message.objects.filter(from_address__user__pk=user.pk)
+        count_types[trans('thread')] = messages.count()
-        counter_class = {}
-        # {
-        #     'wiki': WikiCollabCount,
-        #     'ticket': TicketCollabCount,
-        # }
+        collaborations, count_types_extras = getCollaborationData(user)
+        collaborations.extend(messages)
-        types = ['thread']
-        # types.extend(['ticket', 'wiki', 'changeset', 'attachment'])
+        collaborations = sorted(collaborations, key=lambda elem : elem.modified, reverse=True)
-        messages = Message.objects.filter(from_address__user__pk=user.pk)
-        for type in types:
-            CounterClass = counter_class.get(type)
-            if CounterClass:
-                try:
-                    counter = CounterClass.objects.get(author=user.username)
-                except CounterClass.DoesNotExist:
-                    count_types[trans(type)] = 0
-                else:
-                    count_types[trans(type)] = counter.count
-            elif type == 'thread':
-                count_types[trans(type)] = messages.count()
-            else:
-                sqs = SearchQuerySet()
-                for filter_or in fields_or_lookup:
-                    sqs = sqs.filter_or(type=type, **filter_or)
-                count_types[trans(type)] = sqs.count()
+        count_types.update(count_types_extras)
         context['type_count'] = count_types
-
-        sqs = SearchQuerySet()
-        for filter_or in fields_or_lookup:
-            sqs = sqs.filter_or(**filter_or).exclude(type='thread')
-
-        try:
-            context['results'] = sqs.order_by('-modified', '-created')[:10]
-        except SearchBackendError:
-            context['results'] = sqs.order_by('-modified')[:10]
+        context['results'] = collaborations[:10]
         email_pks = [addr.pk for addr in user.emails.iterator()]
         query = Message.objects.filter(from_address__in=email_pks)
@@ -127,7 +101,6 @@ class UserProfileDetailView(UserProfileBaseMixin, DetailView):
         context.update(kwargs)
         return super(UserProfileDetailView, self).get_context_data(**context)
-
 def signup(request):
     BROWSERID_ENABLED = getattr(settings, 'BROWSERID_ENABLED', False)
-
 from collections import OrderedDict
 from django.conf import settings
@@ -8,53 +7,49 @@ from django.http import HttpResponse, Http404
 from haystack.query import SearchQuerySet
+<<<<<<< HEAD
 # from proxy.trac.models import WikiCollabCount, TicketCollabCount
 from colab.search.utils import trans
 from colab.super_archives.models import Thread
+=======
+from colab.search.utils import trans, getCollaborationData
+from colab.super_archives.models import Thread, Message
+from colab.search.preview_block import PreviewBlock
+from colab.accounts.models import User
+>>>>>>> Remove haystack search from dashboard an profile
 def dashboard(request):
     """Dashboard page"""
+
     latest_threads = Thread.objects.all()[:6]
-    hottest_threads = Thread.highest_score.from_haystack()[:6]
-
-    count_types = cache.get('home_chart')
-    if count_types is None:
-        count_types = OrderedDict()
-        count_types['thread'] = SearchQuerySet().filter(
-            type='thread',
-        ).count()
-        # TODO: this section should be inside trac app and only use it here
-        # if settings.TRAC_ENABLED:
-        #     for type in ['changeset', 'attachment']:
-        #         count_types[type] = SearchQuerySet().filter(
-        #             type=type,
-        #         ).count()
-
-        #     count_types['ticket'] = sum([
-        #         ticket.count for ticket in TicketCollabCount.objects.all()
-        #     ])
-
-        #     count_types['wiki'] = sum([
-        #         wiki.count for wiki in WikiCollabCount.objects.all()
-        #     ])
-
-        cache.set('home_chart', count_types)
+    highest_score_threads = Thread.highest_score.all()[:6]
+
+    hottest_threads = []
+    for thread in highest_score_threads:
+        hottest_threads.append(thread.latest_message)
+
+    latest_results, count_types = getCollaborationData()
+    threads = Thread.objects.all()
+    messages = []
+    for t in threads:
+        messages.append(t.latest_message)
+
+    latest_results.extend(messages)
+    latest_results = sorted(latest_results,
+                            key=lambda elem : elem.modified, reverse=True)
     for key in count_types.keys():
         count_types[trans(key)] = count_types.pop(key)
     context = {
         'hottest_threads': hottest_threads[:6],
-        'latest_threads': latest_threads,
+        'latest_threads': latest_threads[:6],
         'type_count': count_types,
-        'latest_results': SearchQuerySet().all().order_by(
-            '-modified', '-created'
-        )[:6],
+        'latest_results': latest_results[:6],
     }
     return render(request, 'home.html', context)
-
 def robots(request):
     if getattr(settings, 'ROBOTS_NOINDEX', False):
         return HttpResponse('User-agent: *\nDisallow: /',
@@ -3,8 +3,36 @@ from django.utils.translation import ugettext_lazy as _
 from ..utils.apps import ColabProxiedAppConfig
-
 class ProxyGitlabAppConfig(ColabProxiedAppConfig):
+    '''
+    You can define a collaboration_models list to tell colab which
+    models and what values should be displayed as collaborations.
+
+    See the example bellow:
+
+    Field model refers to the model to be displayed.
+    Field model_verbose is the human name to be displayed in charts.
+    Field collaborator_username tells which user(username) is associated with this collaboration.
+
+    The value of the hashes maps the attribute or method of the model to be put in those positions.
+
+        collaboration_models = [
+            {
+                'model' : 'User',
+                'model_verbose' : 'User',
+                'tag' : '',
+                'title' : 'username',
+                'description' : 'get_full_name',
+                'fullname' : '',
+                'modified' : 'modified',
+                'modified_by' : '',
+                'modified_by_url' : '',
+                'url' : '',
+                'type' : '',
+                'collaborator_username' : 'username',
+            },
+        ]
+    '''
     name = 'colab.proxy.gitlab'
     verbose_name = 'Gitlab Proxy'
@@ -23,3 +51,5 @@ class ProxyGitlabAppConfig(ColabProxiedAppConfig):
         ),
     }
+
+    collaboration_models = []
@@ -14,3 +14,5 @@ class ProxyJenkinsAppConfig(ColabProxiedAppConfig):
             (_('Continuos Integration'), ''),
         ),
     }
+
+    collaboration_models = []
@@ -19,3 +19,5 @@ class ProxyNoosferoAppConfig(ColabProxiedAppConfig):
             (_('Control panel'), 'myprofile'),
         ),
     }
+
+    collaboration_models = []
@@ -5,3 +5,5 @@ from ..utils.apps import ColabProxiedAppConfig
 class ProxyRedmineAppConfig(ColabProxiedAppConfig):
     name = 'colab.proxy.redmine'
     verbose_name = 'Redmine Proxy'
+
+    collaboration_models = []
@@ -22,3 +22,5 @@ class ProxyTracAppConfig(ColabProxiedAppConfig):
             (_('New Wiki Page'), 'wiki/WikiNewPage'),
         ),
     }
+
+    collaboration_models = []
@@ -0,0 +1,15 @@
+class PreviewBlock():
+    tag = None
+    title = None
+    description = None
+    fullname = None
+    modified = None
+    modified_by = None
+    url = None 
+    type = None
+    modified_by_url = None
+    collaborator_username = None
+
+    def __init__(self, **kwargs):
+        for key, value in kwargs.items():
+            setattr(self, key, value)
@@ -11,7 +11,6 @@
 <span class="subject">
   <a href="{{ result.url }}#msg-{{ result.pk }}"
     title="{% filter striptags|truncatewords:50 %}{{ result.description|escape }}{% endfilter %}">
-    {{ result.title }}
   </a>
 </span>
+import importlib
+import inspect
+from collections import OrderedDict
+
+from django.core.cache import cache
 from django.utils.translation import ugettext as _
+from django.apps import apps
+from django.conf import settings
+from colab.super_archives.models import Thread, Message
+from colab.search.preview_block import PreviewBlock
 def trans(key):
@@ -11,4 +20,73 @@ def trans(key):
         'attachment': _('Attachments'),
     }
+    app_names = settings.PROXIED_APPS.keys()
+
+    for app_name in app_names:
+        collaboration_models = apps.get_app_config(app_name).collaboration_models
+
+        for collaboration in collaboration_models:
+            module = importlib.import_module('colab.proxy.{}.models'.format(app_name))
+            elements = eval("module." + collaboration['model']).objects.all()
+            translations[ collaboration['model'].lower() ] = collaboration['model_verbose']
+
     return translations.get(key, key)
+
+def getCollaborationData(filter_by_user = None):
+
+    latest_results = []
+    count_types = cache.get('home_chart')
+    populate_count_types = False
+
+    if count_types is None:
+        populate_count_types = True
+        count_types = OrderedDict()
+        count_types['thread'] = Thread.objects.count()
+
+    app_names = settings.PROXIED_APPS.keys()
+
+    for app_name in app_names:
+        collaboration_models = apps.get_app_config(app_name).collaboration_models
+
+        for collaboration in collaboration_models:
+            module = importlib.import_module('colab.proxy.{}.models'.format(app_name))
+            elements = eval("module." + collaboration['model']).objects
+
+            if filter_by_user:
+                dic = {}
+                dic[collaboration['collaborator_username']] = filter_by_user
+                elements = elements.filter(**dic)
+            else:
+                elements = elements.all()
+
+            latest_results.extend(parsePreviewBlock(elements, collaboration))
+
+            if populate_count_types:
+                count_types[ collaboration['model'].lower() ] = elements.count()
+
+    if populate_count_types:
+        cache.set('home_chart', count_types)
+
+    for key in count_types.keys():
+        count_types[trans(key)] = count_types.pop(key)
+
+    return latest_results, count_types
+
+def parsePreviewBlock(elements, collaboration):
+    results = []
+    for element in elements:
+        previewblock = PreviewBlock()
+        attributes = collaboration.keys()
+
+        for keyname in attributes:
+            if keyname == 'model' or keyname == 'model_verbose' or len(collaboration[keyname].strip()) == 0:
+                continue
+            value = getattr(element, collaboration[keyname])
+            if(inspect.ismethod(value)):
+                setattr(previewblock, keyname, value() )
+            else:
+                setattr(previewblock, keyname, value )
+
+        results.append(previewblock)
+
+    return results
 \ No newline at end of file
	@@ -3,8 +3,36 @@ from django.utils.translation import ugettext_lazy as _		@@ -3,8 +3,36 @@ from django.utils.translation import ugettext_lazy as _
3		3
4	from ..utils.apps import ColabProxiedAppConfig	4	from ..utils.apps import ColabProxiedAppConfig
5		5
6	-
7	class ProxyGitlabAppConfig(ColabProxiedAppConfig):	6	class ProxyGitlabAppConfig(ColabProxiedAppConfig):
		7	+ '''
		8	+ You can define a collaboration_models list to tell colab which
		9	+ models and what values should be displayed as collaborations.
		10	+
		11	+ See the example bellow:
		12	+
		13	+ Field model refers to the model to be displayed.
		14	+ Field model_verbose is the human name to be displayed in charts.
		15	+ Field collaborator_username tells which user(username) is associated with this collaboration.
		16	+
		17	+ The value of the hashes maps the attribute or method of the model to be put in those positions.
		18	+
		19	+ collaboration_models = [
		20	+ {
		21	+ 'model' : 'User',
		22	+ 'model_verbose' : 'User',
		23	+ 'tag' : '',
		24	+ 'title' : 'username',
		25	+ 'description' : 'get_full_name',
		26	+ 'fullname' : '',
		27	+ 'modified' : 'modified',
		28	+ 'modified_by' : '',
		29	+ 'modified_by_url' : '',
		30	+ 'url' : '',
		31	+ 'type' : '',
		32	+ 'collaborator_username' : 'username',
		33	+ },
		34	+ ]
		35	+ '''
8	name = 'colab.proxy.gitlab'	36	name = 'colab.proxy.gitlab'
9	verbose_name = 'Gitlab Proxy'	37	verbose_name = 'Gitlab Proxy'
10		38
	@@ -23,3 +51,5 @@ class ProxyGitlabAppConfig(ColabProxiedAppConfig):		@@ -23,3 +51,5 @@ class ProxyGitlabAppConfig(ColabProxiedAppConfig):
23		51
24	),	52	),
25	}	53	}
		54	+
		55	+ collaboration_models = []
	@@ -14,3 +14,5 @@ class ProxyJenkinsAppConfig(ColabProxiedAppConfig):		@@ -14,3 +14,5 @@ class ProxyJenkinsAppConfig(ColabProxiedAppConfig):
14	(_('Continuos Integration'), ''),	14	(_('Continuos Integration'), ''),
15	),	15	),
16	}	16	}
		17	+
		18	+ collaboration_models = []
	@@ -19,3 +19,5 @@ class ProxyNoosferoAppConfig(ColabProxiedAppConfig):		@@ -19,3 +19,5 @@ class ProxyNoosferoAppConfig(ColabProxiedAppConfig):
19	(_('Control panel'), 'myprofile'),	19	(_('Control panel'), 'myprofile'),
20	),	20	),
21	}	21	}
		22	+
		23	+ collaboration_models = []
	@@ -5,3 +5,5 @@ from ..utils.apps import ColabProxiedAppConfig		@@ -5,3 +5,5 @@ from ..utils.apps import ColabProxiedAppConfig
5	class ProxyRedmineAppConfig(ColabProxiedAppConfig):	5	class ProxyRedmineAppConfig(ColabProxiedAppConfig):
6	name = 'colab.proxy.redmine'	6	name = 'colab.proxy.redmine'
7	verbose_name = 'Redmine Proxy'	7	verbose_name = 'Redmine Proxy'
		8	+
		9	+ collaboration_models = []
	@@ -22,3 +22,5 @@ class ProxyTracAppConfig(ColabProxiedAppConfig):		@@ -22,3 +22,5 @@ class ProxyTracAppConfig(ColabProxiedAppConfig):
22	(_('New Wiki Page'), 'wiki/WikiNewPage'),	22	(_('New Wiki Page'), 'wiki/WikiNewPage'),
23	),	23	),
24	}	24	}
		25	+
		26	+ collaboration_models = []
@@ -0,0 +1,15 @@		@@ -0,0 +1,15 @@
	1	+class PreviewBlock():
	2	+ tag = None
	3	+ title = None
	4	+ description = None
	5	+ fullname = None
	6	+ modified = None
	7	+ modified_by = None
	8	+ url = None
	9	+ type = None
	10	+ modified_by_url = None
	11	+ collaborator_username = None
	12	+
	13	+ def __init__(self, **kwargs):
	14	+ for key, value in kwargs.items():
	15	+ setattr(self, key, value)
	@@ -11,7 +11,6 @@		@@ -11,7 +11,6 @@
11	<span class="subject">	11	<span class="subject">
12	<a href="{{ result.url }}#msg-{{ result.pk }}"	12	<a href="{{ result.url }}#msg-{{ result.pk }}"
13	title="{% filter striptags\|truncatewords:50 %}{{ result.description\|escape }}{% endfilter %}">	13	title="{% filter striptags\|truncatewords:50 %}{{ result.description\|escape }}{% endfilter %}">
14	- {{ result.title }}
15	</a>	14	</a>
16	</span>	15	</span>
17		16
		1	+import importlib
		2	+import inspect
1		3
		4	+from collections import OrderedDict
		5	+
		6	+from django.core.cache import cache
2	from django.utils.translation import ugettext as _	7	from django.utils.translation import ugettext as _
		8	+from django.apps import apps
		9	+from django.conf import settings
		10	+from colab.super_archives.models import Thread, Message
		11	+from colab.search.preview_block import PreviewBlock
3		12
4		13
5	def trans(key):	14	def trans(key):
	@@ -11,4 +20,73 @@ def trans(key):		@@ -11,4 +20,73 @@ def trans(key):
11	'attachment': _('Attachments'),	20	'attachment': _('Attachments'),
12	}	21	}
13		22
		23	+ app_names = settings.PROXIED_APPS.keys()
		24	+
		25	+ for app_name in app_names:
		26	+ collaboration_models = apps.get_app_config(app_name).collaboration_models
		27	+
		28	+ for collaboration in collaboration_models:
		29	+ module = importlib.import_module('colab.proxy.{}.models'.format(app_name))
		30	+ elements = eval("module." + collaboration['model']).objects.all()
		31	+ translations[ collaboration['model'].lower() ] = collaboration['model_verbose']
		32	+
14	return translations.get(key, key)	33	return translations.get(key, key)
		34	+
		35	+def getCollaborationData(filter_by_user = None):
		36	+
		37	+ latest_results = []
		38	+ count_types = cache.get('home_chart')
		39	+ populate_count_types = False
		40	+
		41	+ if count_types is None:
		42	+ populate_count_types = True
		43	+ count_types = OrderedDict()
		44	+ count_types['thread'] = Thread.objects.count()
		45	+
		46	+ app_names = settings.PROXIED_APPS.keys()
		47	+
		48	+ for app_name in app_names:
		49	+ collaboration_models = apps.get_app_config(app_name).collaboration_models
		50	+
		51	+ for collaboration in collaboration_models:
		52	+ module = importlib.import_module('colab.proxy.{}.models'.format(app_name))
		53	+ elements = eval("module." + collaboration['model']).objects
		54	+
		55	+ if filter_by_user:
		56	+ dic = {}
		57	+ dic[collaboration['collaborator_username']] = filter_by_user
		58	+ elements = elements.filter(**dic)
		59	+ else:
		60	+ elements = elements.all()
		61	+
		62	+ latest_results.extend(parsePreviewBlock(elements, collaboration))
		63	+
		64	+ if populate_count_types:
		65	+ count_types[ collaboration['model'].lower() ] = elements.count()
		66	+
		67	+ if populate_count_types:
		68	+ cache.set('home_chart', count_types)
		69	+
		70	+ for key in count_types.keys():
		71	+ count_types[trans(key)] = count_types.pop(key)
		72	+
		73	+ return latest_results, count_types
		74	+
		75	+def parsePreviewBlock(elements, collaboration):
		76	+ results = []
		77	+ for element in elements:
		78	+ previewblock = PreviewBlock()
		79	+ attributes = collaboration.keys()
		80	+
		81	+ for keyname in attributes:
		82	+ if keyname == 'model' or keyname == 'model_verbose' or len(collaboration[keyname].strip()) == 0:
		83	+ continue
		84	+ value = getattr(element, collaboration[keyname])
		85	+ if(inspect.ismethod(value)):
		86	+ setattr(previewblock, keyname, value() )
		87	+ else:
		88	+ setattr(previewblock, keyname, value )
		89	+
		90	+ results.append(previewblock)
		91	+
		92	+ return results
15	\ No newline at end of file	93	\ No newline at end of file