Merge branch 'master' into haystack

Luan
2 parents 5a17864f fd7bc7d3
Showing 18 changed files with 112 additions and 116 deletions Show diff stats
fabfile.py
puppet/modules/colab/manifests/init.pp
puppet/modules/colab/manifests/requirements.pp
requirements.txt
src/accounts/search_indexes.py
src/colab/custom_settings.py
src/colab/local_settings-dev.py
src/planet/templates/feedzilla/index.html
src/proxy/diazo/trac.xml
src/proxy/models.py
src/super_archives/search_indexes.py
src/super_archives/templates/message-list.html
src/super_archives/templatetags/superarchives.py
src/super_archives/templatetags/urlutils.py
src/super_archives/utils/url.py
src/super_archives/views.py
src/templates/search.html
src/templates/search/search.html
@@ -19,6 +19,11 @@ environments = {
         'hosts': [], #TODO
         'key_filename': '~/.ssh/id_rsa',
     },
+    'demo': {
+        'hosts': ['colab-demo.tracy.com.br'],
+        'key_filename': '~/.ssh/id_rsa',
+        'port': 22,
+    },
 }
@@ -27,8 +32,8 @@ WORKON_COLAB = &#39;{} &amp;&amp; workon colab&#39;.format(SOURCE_VENV)
 def environment(name):
-     env.update(environments[name])
-     env.environment = name
+    env.update(environments[name])
+    env.environment = name
 environment('dev')
@@ -48,6 +53,15 @@ def install(local_settings=None):
     if local_settings:
         put(local_settings, '~/colab/src/colab/local_settings.py')
+    if not exists('~/apache-solr-3.6.2/'):
+        run('wget http://archive.apache.org/dist/lucene/solr/3.6.2/apache-solr-3.6.2.tgz')
+        run('tar xzf apache-solr-3.6.2.tgz')
+        run('rm apache-solr-3.6.2.tgz')
+
+    with cd('~/apache-solr-3.6.2/example/solr/conf/'):
+        if not exists('stopwords_en.txt'):
+            run('cp stopwords.txt stopwords_en.txt')
+
     if env_created:
         update_requirements()
@@ -56,7 +70,7 @@ def install(local_settings=None):
 def update_requirements():
     with cd('~/colab'), prefix(WORKON_COLAB):
-        run('pip install -r requirements.txt')
+        run('pip install -U -r requirements.txt')
 def deploy(update=False):
@@ -71,10 +85,20 @@ def deploy(update=False):
         run('python manage.py syncdb')
         run('python manage.py migrate')
         run('python manage.py collectstatic --noinput')
+        run('python manage.py build_solr_schema -f ~/apache-solr-3.6.2/example/solr/conf/schema.xml')
     sudo('supervisorctl restart all')
+def rebuild_index(age=None):
+    with cd('~/colab/src/'), prefix(WORKON_COLAB):
+        age_arg = ''
+        if age:
+            age_arg = '--age={}'.format(age)
+
+        run('python manage.py rebuild_index {}'.format(age_arg))
+
+
 @with_settings(user='vagrant')
 def runserver(update_requirements=False):
     env_created = mkvirtualenv()
@@ -56,6 +56,12 @@ class colab {
     user      => 'colab',
   }
+  supervisor::app { 'solr':
+    command   => 'java -jar start.jar',
+    directory => '/home/colab/apache-solr-3.6.2/example',
+    user      => 'colab',
+  }
+
   nginx::config { 'nginx':
     content => template('colab/nginx/extra_conf.erb'),
   }
@@ -42,7 +42,7 @@ class colab::requirements {
     provider => pip,
     require  => [Package['python-pip'], Package['python-dev'], Package['build-essential']],
   }
-  
+
   # links virtualenvwrapper to load automaticaly
   file { '/etc/bash_completion.d/virtualenvwrapper.sh':
     ensure => link,
@@ -58,7 +58,7 @@ class colab::requirements {
   package { 'python-dev':
     ensure => installed,
   }
-  
+
   # req by gvent
   package { 'libevent-dev':
     ensure => installed,
@@ -83,4 +83,9 @@ class colab::requirements {
   package { 'gettext':
     ensure => installed,
   }
+
+  # req by solr
+  package { 'openjdk-7-jre':
+    ensure => installed,
+  }
 }
@@ -9,6 +9,7 @@ python-dateutil==1.5
 django-cliauth==0.9
 django-mobile==0.3.0
 django-haystack==2.1
+pysolr==2.1
 etiquetando==0.1
 html2text
 django-taggit
@@ -30,9 +30,8 @@ class UserIndex(indexes.SearchIndex, indexes.Indexable):
         return 'date_joined'
     def prepare_description(self, obj):
-        return u'{}\n{}\n{}\n{}\n{}\n{}'.format(
-            obj.institution, obj.role, obj.username, obj.get_full_name(),
-            obj.google_talk, obj.webpage
+        return u'{}\n{}\n{}\n{}'.format(
+            obj.institution, obj.role, obj.username, obj.get_full_name()
         )
     def prepare_icon_name(self, obj):
@@ -56,8 +56,8 @@ DATABASES = {
     },
     'trac': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': 'trac',
-        'USER': 'trac',
+        'NAME': 'trac_colab',
+        'USER': 'colab',
         'PASSWORD': os.environ.get('COLAB_TRAC_DB_PWD'),
         'HOST': os.environ.get('COLAB_TRAC_DB_HOST'),
     }
@@ -97,7 +97,7 @@ LOGGING = {
     'disable_existing_loggers': False,
     'root': {
         'level': 'WARNING',
-        'handlers': ['sentry'],
+        'handlers': ['sentry', 'console'],
     },
     'formatters': {
         'verbose': {
@@ -209,6 +209,8 @@ LOCALE_PATHS = (
 AUTH_USER_MODEL = 'accounts.User'
+ALLOWED_HOSTS = ['colab.interlegis.leg.br']
+
 from django.contrib.messages import constants as messages
 MESSAGE_TAGS = {
     messages.INFO: 'alert-info',
@@ -227,6 +229,8 @@ FEEDZILLA_SITE_DESCRIPTION = gettext(u&#39;Colab blog aggregator&#39;)
 ### BrowserID / Persona
 SITE_URL = 'https://colab.interlegis.leg.br'
+BROWSERID_AUDIENCES = [SITE_URL]
+
 LOGIN_URL = '/'
 LOGIN_REDIRECT_URL = '/'
@@ -235,27 +239,20 @@ LOGOUT_REDIRECT_URL = &#39;/&#39;
 BROWSERID_CREATE_USER = False
-### Apache Solr
-#SOLR_HOSTNAME = 'solr.interlegis.leg.br'
-SOLR_HOSTNAME = '10.1.2.154'
-SOLR_PORT = '8080'
-SOLR_SELECT_PATH = '/solr/select'
-
-SOLR_COLAB_URI = 'http://colab.interlegis.leg.br'
-SOLR_BASE_QUERY = """
-    ((Type:changeset OR Type:ticket OR Type:wiki OR Type:thread) AND Title:["" TO *])
-"""
-
+## Proxy settings
 COLAB_TRAC_URL = 'http://colab-backend.interlegis.leg.br/'
 COLAB_CI_URL = 'http://jenkins.interlegis.leg.br:8080/ci/'
 REVPROXY_ADD_REMOTE_USER = True
+
+## Converse.js settings
 # This URL must use SSL in order to keep chat sessions secure
 CONVERSEJS_BOSH_SERVICE_URL = SITE_URL + '/http-bind'
 CONVERSEJS_AUTO_REGISTER = 'mensageiro.interlegis.gov.br'
+
 try:
     from local_settings import *
 except ImportError:
@@ -9,15 +9,32 @@ ADMINS = (
 MANAGERS = ADMINS
-SOLR_COLAB_URI = None
-SOLR_HOSTNAME = None
+EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
+#EMAIL_HOST = ''
+#EMAIL_PORT = 587
+EMAIL_SUBJECT_PREFIX = ''
 # Make this unique, and don't share it with anybody.
 SECRET_KEY = ')(jksdfhsjkadfhjkh234ns!8fqu-1186h$vuj'
 SITE_URL = 'http://localhost:8000'
+ALLOWED_HOSTS = ['*']
+
 INTERNAL_IPS = ('127.0.0.1', )
 CONVERSEJS_BOSH_SERVICE_URL = 'http://localhost:5280/http-bind'
+DATABASES['default']['PASSWORD'] = 'colab'
+DATABASES['default']['HOST'] = 'localhost'
+DATABASES['trac']['PASSWORD'] = 'colab'
+DATABASES['trac']['HOST'] = 'localhost'
+
+HAYSTACK_CONNECTIONS['default']['URL'] = 'http://localhost:8983/solr/'
+
+COLAB_TRAC_URL = 'http://localhost:5000/'
+COLAB_CI_URL = 'http://localhost:8080/ci/'
+
+CONVERSEJS_ENABLED = False
+
+DIAZO_THEME = SITE_URL
@@ -8,9 +8,11 @@
   <hr>
   {% empty %}
     <h2>{% trans 'There is no RSS registered' %}</h2>
-    <h3>{% trans 'Please' %}
- href="{% url "feedzilla_submit_blog" %}">{% trans 'click here' %}</a>
+    <h3>
+      {% trans 'Please' %}
+      <a href="{% url "feedzilla_submit_blog" %}">{% trans 'click here' %}</a>
     {% trans 'to submit a blog' %}</h3>
+    </h3>
 {% endfor %}
 {% include "common/pagination.html" %}
@@ -14,7 +14,6 @@
           .navbar .nav li { border: 0; padding: 0; white-space: normal; display: list-item;}
           :link:not(.btn),
           :visited:not(.btn) { border: 0; color: rgb(66, 139, 202); }
-          :link, :visited { border: 0; }
           h1 { font-size: 24px; margin: 0.15em 1em 0.5em 0px; }
           h2 { font-size: 20px }
           h3 { font-size: 16px }
@@ -73,7 +73,7 @@ class Wiki(models.Model):
         db_table = 'wiki_view'
     def get_absolute_url(self):
-        return u'/ticket/{}'.format(self.name)
+        return u'/wiki/{}'.format(self.name)
     def get_author(self):
         try:
@@ -29,7 +29,7 @@ class ThreadIndex(indexes.SearchIndex, indexes.Indexable):
         return Thread
     def get_updated_field(self):
-        return 'received_time'
+        return 'latest_message__received_time'
     def prepare_author(self, obj):
         return obj.message_set.first().from_address.get_full_name()
 {% extends "base.html" %}
-{% load i18n %}
-{% load append_to_get %}
+{% load i18n superarchives %}
 {% block main-content %}
 <div>
   <h2>{% trans "Discussions" %}</h2>
 from django import template
+from super_archives.utils import url
+
 register = template.Library()
 TEMPLATE_PATH = 'superarchives/tags/'
@@ -12,3 +14,21 @@ def display_message(email):
         email.update_blocks()
     return { 'blocks': email.blocks.all }
+
+
+@register.simple_tag(takes_context=True)
+def append_to_get(context, **kwargs):
+    return url.append_to_get(
+        context['request'].META['PATH_INFO'],
+        context['request'].META['QUERY_STRING'],
+        **kwargs
+    )
+
+
+@register.simple_tag(takes_context=True)
+def pop_from_get(context, **kwargs):
+    return url.pop_from_get(
+        context['request'].META['PATH_INFO'],
+        context['request'].META['QUERY_STRING'],
+        **kwargs
+    )
@@ -1,24 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from django import template
-
-from super_archives.utils import url
-
-register = template.Library()
-
-
-@register.simple_tag(takes_context=True)
-def append_to_get(context, **kwargs):
-    return url.append_to_get(
-        context['request'].META['PATH_INFO'],
-        context['request'].META['QUERY_STRING'],
-        **kwargs
-    )
-
-@register.simple_tag(takes_context=True)
-def pop_from_get(context, **kwargs):
-    return url.pop_from_get(
-        context['request'].META['PATH_INFO'],
-        context['request'].META['QUERY_STRING'],
-        **kwargs
-    )
 # -*- coding: utf-8 -*-
-def append_to_get(path, query=None, **kwargs):
-# Getting the path with the query
-    current_url = u'{}?{}'.format(
-        path,
-        query,
-    )
+import urllib
+import urlparse
-    if kwargs and query:
-        current_url += '&'
+def append_to_get(path, query=None, **kwargs):
+    query_dict = dict(urlparse.parse_qsl(query))
     for key, value in kwargs.items():
-        # get the key, value to check if the pair exists in the query
-        new = u'{}={}'.format(key, value)
-
-        if new in current_url:
-            continue
-
-        if key not in current_url:
-            current_url += u'{}={}&'.format(key, value)
-            continue
-
-        parse_url = current_url.split(key)
-
-        if len(parse_url) > 2:
-            continue
-
-        if unicode(value) in parse_url[1][1:]:
-            continue
-
-        check_kwargs_values = [
-            False for value in kwargs.values()
-            if unicode(value) not in parse_url[1]
-        ]
-
-        if not all(check_kwargs_values):
-            list_remaining = parse_url[1][1:].split('&')
-            real_remaining = u''
-
-            if len(list_remaining) >= 2:
-                real_remaining = u'&'.join(list_remaining[1:])
-
-            current_url = u'{url}{key}={value}&{remaining}'.format(
-                url=parse_url[0],
-                key=key,
-                value=value,
-                remaining=real_remaining,
-            )
-            continue
-
-        current_url = u'{url}{key}={value}+{remaining_get}'.format(
-            url=parse_url[0],
-            key=key,
-            value=value,
-            remaining_get=parse_url[1][1:],
-        )
-    if current_url[-1] == '&':
-        return current_url[:-1]
-    return current_url
+        query_dict[key] = value
+    return u'{}?{}'.format(path, urllib.urlencode(query_dict))
 def pop_from_get(path, query=None, **kwargs):
 # -*- coding: utf-8 -*-
 import smtplib
+import logging
 from django import http
 from django.conf import settings
@@ -212,6 +213,7 @@ class EmailValidationView(View):
             send_verification_email(email_addr, request.user,
                                     email.validation_key)
         except smtplib.SMTPException:
+            logging.exception('Error sending validation email')
             return http.HttpResponseServerError()
         return http.HttpResponse(status=204)
 {% extends "base.html" %}
-{% load i18n %}
-{% load append_to_get %}
+{% load i18n superarchives %}
+
 {% block main-content %}
   <div class="row">
     <div class="col-lg-2">
 {% extends "base.html" %}
-{% load i18n %}
-{% load urlutils %}
-{% load highlight %}
+{% load i18n highlight superarchives %}
 {% block main-content %}
   <div class="row">
@@ -72,7 +70,7 @@
       {% endfor %}
       </ul>
-      {% if query and page.has_other_pages %}
+      {% if page.has_other_pages %}
         <div class="text-center">
           <span>
             {% if page.has_previous %}
	@@ -56,6 +56,12 @@ class colab {		@@ -56,6 +56,12 @@ class colab {
56	user => 'colab',	56	user => 'colab',
57	}	57	}
58		58
		59	+ supervisor::app { 'solr':
		60	+ command => 'java -jar start.jar',
		61	+ directory => '/home/colab/apache-solr-3.6.2/example',
		62	+ user => 'colab',
		63	+ }
		64	+
59	nginx::config { 'nginx':	65	nginx::config { 'nginx':
60	content => template('colab/nginx/extra_conf.erb'),	66	content => template('colab/nginx/extra_conf.erb'),
61	}	67	}
	@@ -9,15 +9,32 @@ ADMINS = (		@@ -9,15 +9,32 @@ ADMINS = (
9		9
10	MANAGERS = ADMINS	10	MANAGERS = ADMINS
11		11
12	-SOLR_COLAB_URI = None
13	-SOLR_HOSTNAME = None	12	+EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
		13	+#EMAIL_HOST = ''
		14	+#EMAIL_PORT = 587
		15	+EMAIL_SUBJECT_PREFIX = ''
14		16
15	# Make this unique, and don't share it with anybody.	17	# Make this unique, and don't share it with anybody.
16	SECRET_KEY = ')(jksdfhsjkadfhjkh234ns!8fqu-1186h$vuj'	18	SECRET_KEY = ')(jksdfhsjkadfhjkh234ns!8fqu-1186h$vuj'
17		19
18	SITE_URL = 'http://localhost:8000'	20	SITE_URL = 'http://localhost:8000'
19		21
		22	+ALLOWED_HOSTS = ['*']
		23	+
20	INTERNAL_IPS = ('127.0.0.1', )	24	INTERNAL_IPS = ('127.0.0.1', )
21		25
22	CONVERSEJS_BOSH_SERVICE_URL = 'http://localhost:5280/http-bind'	26	CONVERSEJS_BOSH_SERVICE_URL = 'http://localhost:5280/http-bind'
23		27
		28	+DATABASES['default']['PASSWORD'] = 'colab'
		29	+DATABASES['default']['HOST'] = 'localhost'
		30	+DATABASES['trac']['PASSWORD'] = 'colab'
		31	+DATABASES['trac']['HOST'] = 'localhost'
		32	+
		33	+HAYSTACK_CONNECTIONS['default']['URL'] = 'http://localhost:8983/solr/'
		34	+
		35	+COLAB_TRAC_URL = 'http://localhost:5000/'
		36	+COLAB_CI_URL = 'http://localhost:8080/ci/'
		37	+
		38	+CONVERSEJS_ENABLED = False
		39	+
		40	+DIAZO_THEME = SITE_URL
	@@ -73,7 +73,7 @@ class Wiki(models.Model):		@@ -73,7 +73,7 @@ class Wiki(models.Model):
73	db_table = 'wiki_view'	73	db_table = 'wiki_view'
74		74
75	def get_absolute_url(self):	75	def get_absolute_url(self):
76	- return u'/ticket/{}'.format(self.name)	76	+ return u'/wiki/{}'.format(self.name)
77		77
78	def get_author(self):	78	def get_author(self):
79	try:	79	try:
	@@ -29,7 +29,7 @@ class ThreadIndex(indexes.SearchIndex, indexes.Indexable):		@@ -29,7 +29,7 @@ class ThreadIndex(indexes.SearchIndex, indexes.Indexable):
29	return Thread	29	return Thread
30		30
31	def get_updated_field(self):	31	def get_updated_field(self):
32	- return 'received_time'	32	+ return 'latest_message__received_time'
33		33
34	def prepare_author(self, obj):	34	def prepare_author(self, obj):
35	return obj.message_set.first().from_address.get_full_name()	35	return obj.message_set.first().from_address.get_full_name()
1	{% extends "base.html" %}	1	{% extends "base.html" %}
2	-{% load i18n %}
3	-{% load append_to_get %}	2	+{% load i18n superarchives %}
4	{% block main-content %}	3	{% block main-content %}
5	<div>	4	<div>
6	<h2>{% trans "Discussions" %}</h2>	5	<h2>{% trans "Discussions" %}</h2>
1		1
2	from django import template	2	from django import template
3		3
		4	+from super_archives.utils import url
		5	+
4		6
5	register = template.Library()	7	register = template.Library()
6	TEMPLATE_PATH = 'superarchives/tags/'	8	TEMPLATE_PATH = 'superarchives/tags/'
	@@ -12,3 +14,21 @@ def display_message(email):		@@ -12,3 +14,21 @@ def display_message(email):
12	email.update_blocks()	14	email.update_blocks()
13		15
14	return { 'blocks': email.blocks.all }	16	return { 'blocks': email.blocks.all }
		17	+
		18	+
		19	+@register.simple_tag(takes_context=True)
		20	+def append_to_get(context, **kwargs):
		21	+ return url.append_to_get(
		22	+ context['request'].META['PATH_INFO'],
		23	+ context['request'].META['QUERY_STRING'],
		24	+ **kwargs
		25	+ )
		26	+
		27	+
		28	+@register.simple_tag(takes_context=True)
		29	+def pop_from_get(context, **kwargs):
		30	+ return url.pop_from_get(
		31	+ context['request'].META['PATH_INFO'],
		32	+ context['request'].META['QUERY_STRING'],
		33	+ **kwargs
		34	+ )
	@@ -1,24 +0,0 @@	@@ -1,24 +0,0 @@
1	-# -- coding: utf-8 --
2	-
3	-from django import template
4	-
5	-from super_archives.utils import url
6	-
7	-register = template.Library()
8	-
9	-
10	-@register.simple_tag(takes_context=True)
11	-def append_to_get(context, **kwargs):
12	- return url.append_to_get(
13	- context['request'].META['PATH_INFO'],
14	- context['request'].META['QUERY_STRING'],
15	- **kwargs
16	- )
17	-
18	-@register.simple_tag(takes_context=True)
19	-def pop_from_get(context, **kwargs):
20	- return url.pop_from_get(
21	- context['request'].META['PATH_INFO'],
22	- context['request'].META['QUERY_STRING'],
23	- **kwargs
24	- )
1	# -- coding: utf-8 --	1	# -- coding: utf-8 --
2		2
3	-def append_to_get(path, query=None, **kwargs):
4	-# Getting the path with the query
5	- current_url = u'{}?{}'.format(
6	- path,
7	- query,
8	- )	3	+import urllib
		4	+import urlparse
9		5
10	- if kwargs and query:
11	- current_url += '&'
12		6
		7	+def append_to_get(path, query=None, **kwargs):
		8	+ query_dict = dict(urlparse.parse_qsl(query))
13	for key, value in kwargs.items():	9	for key, value in kwargs.items():
14	- # get the key, value to check if the pair exists in the query
15	- new = u'{}={}'.format(key, value)
16	-
17	- if new in current_url:
18	- continue
19	-
20	- if key not in current_url:
21	- current_url += u'{}={}&'.format(key, value)
22	- continue
23	-
24	- parse_url = current_url.split(key)
25	-
26	- if len(parse_url) > 2:
27	- continue
28	-
29	- if unicode(value) in parse_url[1][1:]:
30	- continue
31	-
32	- check_kwargs_values = [
33	- False for value in kwargs.values()
34	- if unicode(value) not in parse_url[1]
35	- ]
36	-
37	- if not all(check_kwargs_values):
38	- list_remaining = parse_url[1][1:].split('&')
39	- real_remaining = u''
40	-
41	- if len(list_remaining) >= 2:
42	- real_remaining = u'&'.join(list_remaining[1:])
43	-
44	- current_url = u'{url}{key}={value}&{remaining}'.format(
45	- url=parse_url[0],
46	- key=key,
47	- value=value,
48	- remaining=real_remaining,
49	- )
50	- continue
51	-
52	- current_url = u'{url}{key}={value}+{remaining_get}'.format(
53	- url=parse_url[0],
54	- key=key,
55	- value=value,
56	- remaining_get=parse_url[1][1:],
57	- )
58	- if current_url[-1] == '&':
59	- return current_url[:-1]
60	- return current_url	10	+ query_dict[key] = value
		11	+ return u'{}?{}'.format(path, urllib.urlencode(query_dict))
61		12
62		13
63	def pop_from_get(path, query=None, **kwargs):	14	def pop_from_get(path, query=None, **kwargs):