Merge branch 'master' from interlegis/master

Gust
2 parents 4b0aace8 8360aceb
Showing 2 changed files with 243 additions and 36 deletions Show diff stats
install.rst
src/colab/custom_settings.py
@@ -145,6 +145,227 @@ Finally, trac can be run by
 Use port 8000 and access it on port 8080 in your local machine to see it works if you'd like(vagrant has port redirects)
+Installing Solr and indexing colab's schemas
+==========================================
+In Progress of making it readable
+  #http servlet jetty
+  sudo apt-get install jetty
+
+  #solr
+  wget http://ftp.unicamp.br/pub/apache/lucene/solr/4.6.1/solr-4.6.1.tgz
+  tar xvzf solr-4.6.1.tgz
+  sudo mv solr-4.6.1 /usr/share/solr
+
+  sudo cp /usr/share/solr/example/webapps/solr.war /usr/share/solr/example/solr/solr.war
+
+  python manage.py build_solr_schema >> schema.xml
+    #existe mais de uma referencia a stopwords_en
+    trocar no schema.xml stopwords_en.txt por lang/stopwords_en.txt
+
+  sudo cp schema.xml /usr/share/solr/example/solr/collection1/conf
+
+  #inclusive a tag
+  Remova <updateLog> em solrconfig.xml localizado em /usr/share/solr/example/solr/collection1/conf
+
+  #executa servidor
+  cd /usr/share/solr/example/; java -jar start.jar;
+
+  #indexa
+  python manage.py update_index
+  
+  
+Installing Mailman with Nginx
+=============================
+
+Nginx Configuration:
+
+In order to let nginx serve the Mailman web interface, we need to have the fcgiwrap package installed:
+  
+.. code-block::
+
+  sudo apt-get install fcgiwrap
+  
+Setting the vhost:
+  
+Create the www.example.com web site root as follows:
+
+.. code-block::
+  
+  mkdir -p /var/www/www.example.com/web
+  
+Next create a basic nginx vhost configuration editing the .vhost file (editor /etc/nginx/sites-available/www.example.com.vhost):
+  
+.. code-block::
+  
+  server {
+       listen 8000;
+       server_name www.example.com example.com;
+       root /var/www/www.example.com/web;
+       if ($http_host != "localhost:8080") {
+                 rewrite ^ localhost$request_uri permanent;
+       }
+       index index.php index.html;
+       location = /favicon.ico {
+                log_not_found off;
+                access_log off;
+       }
+       location = /robots.txt {
+                allow all;
+                log_not_found off;
+                access_log off;
+       }
+       # Deny all attempts to access hidden files such as .htaccess, .htpasswd, .DS_Store (Mac).
+       location ~ /\. {
+                deny all;
+                access_log off;
+                log_not_found off;
+       }
+       location ~*  \.(jpg|jpeg|png|gif|css|js|ico)$ {
+                expires max;
+                log_not_found off;
+       }
+  }
+  
+To enable the vhost, we create a symlink to it from the /etc/nginx/sites-enabled/ directory:
+
+.. code-block::
+
+  cd /etc/nginx/sites-enabled/
+  ln -s /etc/nginx/sites-available/www.example.com.vhost www.example.com.vhost
+
+Reload nginx for the changes to take effect:
+
+.. code-block::
+
+  /etc/init.d/nginx reload
+  
+Mailman Configuration:
+
+.. code-block::
+
+  sudo apt-get install mailman
+
+Create a new mailing list:
+
+.. code-block::
+
+  sudo newlist mailman
+  
+Open the /etc/aliases (editor /etc/aliases) and add the following:
+
+.. code-block::
+
+  mailman:              "|/var/lib/mailman/mail/mailman post mailman"
+  mailman-admin:        "|/var/lib/mailman/mail/mailman admin mailman"
+  mailman-bounces:      "|/var/lib/mailman/mail/mailman bounces mailman"
+  mailman-confirm:      "|/var/lib/mailman/mail/mailman confirm mailman"
+  mailman-join:         "|/var/lib/mailman/mail/mailman join mailman"
+  mailman-leave:        "|/var/lib/mailman/mail/mailman leave mailman"
+  mailman-owner:        "|/var/lib/mailman/mail/mailman owner mailman"
+  mailman-request:      "|/var/lib/mailman/mail/mailman request mailman"
+  mailman-subscribe:    "|/var/lib/mailman/mail/mailman subscribe mailman"
+  mailman-unsubscribe:  "|/var/lib/mailman/mail/mailman unsubscribe mailman"
+
+Run:
+
+.. code-block::
+
+  sudo newaliases
+
+Now restart postfix and mailman:
+
+.. code-block::
+
+  sudo /etc/init.d/postfix restart
+  sudo /etc/init.d/mailman start
+  
+Open /etc/nginx/sites-available/www.example.com.vhost and add the following part to the server {} container:
+
+.. code-block::
+
+  server {
+  [...]
+          location /cgi-bin/mailman {
+                 root /usr/lib/;
+                 fastcgi_split_path_info (^/cgi-bin/mailman/[^/]*)(.*)$;
+                 include /etc/nginx/fastcgi_params;
+                 fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+                 fastcgi_param PATH_INFO $fastcgi_path_info;
+                 fastcgi_param PATH_TRANSLATED $document_root$fastcgi_path_info;
+                 fastcgi_intercept_errors on;
+                 fastcgi_pass unix:/var/run/fcgiwrap.socket;
+          }
+          location /images/mailman {
+                 alias /usr/share/images/mailman;
+          }
+          location /pipermail {
+                 alias /var/lib/mailman/archives/public;
+                 autoindex on;
+          }
+  [...]
+  }
+
+Now create the fastcgi_params file and add the following:
+
+.. code-block::
+
+  fastcgi_param	QUERY_STRING		$query_string;
+  fastcgi_param	REQUEST_METHOD		$request_method;
+  fastcgi_param	CONTENT_TYPE		$content_type;
+  fastcgi_param	CONTENT_LENGTH		$content_length;
+  
+  fastcgi_param	SCRIPT_FILENAME		$document_root$fastcgi_script_name;
+  fastcgi_param	SCRIPT_NAME		$fastcgi_script_name;
+  fastcgi_param	REQUEST_URI		$request_uri;
+  fastcgi_param	DOCUMENT_URI		$document_uri;
+  fastcgi_param	DOCUMENT_ROOT		$document_root;
+  fastcgi_param	SERVER_PROTOCOL		$server_protocol;
+  
+  fastcgi_param	GATEWAY_INTERFACE	CGI/1.1;
+  fastcgi_param	SERVER_SOFTWARE		nginx;
+  
+  fastcgi_param	REMOTE_ADDR		$remote_addr;
+  fastcgi_param	REMOTE_PORT		$remote_port;
+  fastcgi_param	SERVER_ADDR		$server_addr;
+  fastcgi_param	SERVER_PORT		$server_port;
+  fastcgi_param	SERVER_NAME		$server_name;
+
+Now open the mm_config.py file (sudo editor /etc/mailman/mm_config.py) and modify the lines below to:
+
+.. code-block::
+
+  DEFAULT_EMAIL_HOST = 'localhost'
+  DEFAULT_URL_HOST = 'localhost:8080'
+
+To these modifications have effect do the following:
+
+.. code-block::
+
+  sudo withlist -l -a -r fix_url 
+  
+Also tt's needed to add permission to mailman to access the name_of_the_list.mbox because it needs to archive the emails:
+
+.. code-block::
+
+  sudo chown -R root:list /var/lib/mailman/archives
+  sudo chown -R root:list /var/lib/mailman/data/aliases
+  sudo chown -R root:list /etc/aliases
+  sudo chmod -R u+rwX /var/lib/mailman/archives
+  
+Now restart fcgi, mailman and nginx:
+
+.. code-block::
+  
+  sudo /etc/init.d/fcgiwrap restart
+  sudo /etc/init.d/mailman restart
+  sudo /etc/init.d/nginx reload
+  
+To see if it's working:
+
+.. code-block::
+  
+  http://localhost:8080/cgi-bin/mailman/listinfo/mailman
+  
 Setting up Colab
 =========================
@@ -160,7 +381,19 @@ Change COLAB_TRAC_URL to
 Or the port you're using to Trac
-Use fabric at colab's root to update the requirements to your VM
+After this it's needed to import the e-mails from Mailman:
+  
+.. code-block::
+  
+  sudo python manage.py import_emails
+  
+And also update Solr:
+
+.. code-block::
+
+  sudo python manage.py update_index
+
+Then, use fabric at colab's root to update the requirements to your VM
 .. code-block::
@@ -171,44 +404,18 @@ Use fabric at colab&#39;s root to update the requirements to your VM
   The fabric installation on ubuntu through 'pip install' might not be added to the path so you'll need to find
   where it was installed if thats your case
-Now simply run fabric again to run the server (it'll also sync and migrate colab's database)
+Lastly, run fabric again to run the server (it'll also sync and migrate colab's database) and open new tabs to run trac and solr:
 .. code-block::
-  fabric runserver
+  fab runserver
+  fab trac
+  fab solr
 You should be able to see colab with its cms service already working at port 8000(because of vagrant redirects)
 *NOTE*
-  In case login doesn't work, change the SITE_URL in src/colab/local_settings.py the reflect the django's port
+  In case login doesn't work, change the SITE_URL in src/colab/local_settings.py and in src/colab/custom_settings.py the reflect the django's port
   Also, add the following line riht bellow it
     BROWSERID_AUDIENCES = [SITE_URL, SITE_URL.replace('https', 'http')]
-
-Installing Solr and indexing colab's schemas
-==========================================
-In Progress of making it readable
-  #http servlet jetty
-  sudo apt-get install jetty
-
-  #solr
-  wget http://ftp.unicamp.br/pub/apache/lucene/solr/4.6.1/solr-4.6.1.tgz
-  tar xvzf solr-4.6.1.tgz
-  sudo mv solr-4.6.1 /usr/share/solr
-
-  sudo cp /usr/share/solr/example/webapps/solr.war /usr/share/solr/example/solr/solr.war
-
-  python manage.py build_solr_schema >> schema.xml
-    #existe mais de uma referencia a stopwords_en
-    trocar no schema.xml stopwords_en.txt por lang/stopwords_en.txt
-
-  sudo cp schema.xml /usr/share/solr/example/solr/collection1/conf
-
-  #inclusive a tag
-  Remova <updateLog> em solrconfig.xml localizado em /usr/share/solr/example/solr/collection1/conf
-
-  #executa servidor
-  cd /usr/share/solr/example/; java -jar start.jar;
-
-  #indexa
-  python manage.py update_index
 \ No newline at end of file
@@ -268,11 +268,11 @@ FEEDZILLA_SITE_DESCRIPTION = _(u&#39;Colab blog aggregator&#39;)
 ### Mailman API settings
-MAILMAN_API_URL = 'http://listas.interlegis.gov.br:8000'
+MAILMAN_API_URL = 'localhost:8000'
 ### BrowserID / Persona
-SITE_URL = 'https://colab.interlegis.leg.br'
+SITE_URL = 'localhost:8000'
 BROWSERID_AUDIENCES = [SITE_URL, SITE_URL.replace('https', 'http')]
@@ -284,8 +284,8 @@ BROWSERID_CREATE_USER = False
 ## Proxy settings
-COLAB_TRAC_URL = 'http://colab-backend.interlegis.leg.br/'
-COLAB_CI_URL = 'http://jenkins.interlegis.leg.br:8080/ci/'
+COLAB_TRAC_URL = 'localhost:5000/trac/'
+COLAB_CI_URL = 'localhost:9000/ci/'
 REVPROXY_ADD_REMOTE_USER = True
	@@ -145,6 +145,227 @@ Finally, trac can be run by		@@ -145,6 +145,227 @@ Finally, trac can be run by
145		145
146	Use port 8000 and access it on port 8080 in your local machine to see it works if you'd like(vagrant has port redirects)	146	Use port 8000 and access it on port 8080 in your local machine to see it works if you'd like(vagrant has port redirects)
147		147
		148	+Installing Solr and indexing colab's schemas
		149	+==========================================
		150	+In Progress of making it readable
		151	+ #http servlet jetty
		152	+ sudo apt-get install jetty
		153	+
		154	+ #solr
		155	+ wget http://ftp.unicamp.br/pub/apache/lucene/solr/4.6.1/solr-4.6.1.tgz
		156	+ tar xvzf solr-4.6.1.tgz
		157	+ sudo mv solr-4.6.1 /usr/share/solr
		158	+
		159	+ sudo cp /usr/share/solr/example/webapps/solr.war /usr/share/solr/example/solr/solr.war
		160	+
		161	+ python manage.py build_solr_schema >> schema.xml
		162	+ #existe mais de uma referencia a stopwords_en
		163	+ trocar no schema.xml stopwords_en.txt por lang/stopwords_en.txt
		164	+
		165	+ sudo cp schema.xml /usr/share/solr/example/solr/collection1/conf
		166	+
		167	+ #inclusive a tag
		168	+ Remova <updateLog> em solrconfig.xml localizado em /usr/share/solr/example/solr/collection1/conf
		169	+
		170	+ #executa servidor
		171	+ cd /usr/share/solr/example/; java -jar start.jar;
		172	+
		173	+ #indexa
		174	+ python manage.py update_index
		175	+
		176	+
		177	+Installing Mailman with Nginx
		178	+=============================
		179	+
		180	+Nginx Configuration:
		181	+
		182	+In order to let nginx serve the Mailman web interface, we need to have the fcgiwrap package installed:
		183	+
		184	+.. code-block::
		185	+
		186	+ sudo apt-get install fcgiwrap
		187	+
		188	+Setting the vhost:
		189	+
		190	+Create the www.example.com web site root as follows:
		191	+
		192	+.. code-block::
		193	+
		194	+ mkdir -p /var/www/www.example.com/web
		195	+
		196	+Next create a basic nginx vhost configuration editing the .vhost file (editor /etc/nginx/sites-available/www.example.com.vhost):
		197	+
		198	+.. code-block::
		199	+
		200	+ server {
		201	+ listen 8000;
		202	+ server_name www.example.com example.com;
		203	+ root /var/www/www.example.com/web;
		204	+ if ($http_host != "localhost:8080") {
		205	+ rewrite ^ localhost$request_uri permanent;
		206	+ }
		207	+ index index.php index.html;
		208	+ location = /favicon.ico {
		209	+ log_not_found off;
		210	+ access_log off;
		211	+ }
		212	+ location = /robots.txt {
		213	+ allow all;
		214	+ log_not_found off;
		215	+ access_log off;
		216	+ }
		217	+ # Deny all attempts to access hidden files such as .htaccess, .htpasswd, .DS_Store (Mac).
		218	+ location ~ /\. {
		219	+ deny all;
		220	+ access_log off;
		221	+ log_not_found off;
		222	+ }
		223	+ location ~* \.(jpg\|jpeg\|png\|gif\|css\|js\|ico)$ {
		224	+ expires max;
		225	+ log_not_found off;
		226	+ }
		227	+ }
		228	+
		229	+To enable the vhost, we create a symlink to it from the /etc/nginx/sites-enabled/ directory:
		230	+
		231	+.. code-block::
		232	+
		233	+ cd /etc/nginx/sites-enabled/
		234	+ ln -s /etc/nginx/sites-available/www.example.com.vhost www.example.com.vhost
		235	+
		236	+Reload nginx for the changes to take effect:
		237	+
		238	+.. code-block::
		239	+
		240	+ /etc/init.d/nginx reload
		241	+
		242	+Mailman Configuration:
		243	+
		244	+.. code-block::
		245	+
		246	+ sudo apt-get install mailman
		247	+
		248	+Create a new mailing list:
		249	+
		250	+.. code-block::
		251	+
		252	+ sudo newlist mailman
		253	+
		254	+Open the /etc/aliases (editor /etc/aliases) and add the following:
		255	+
		256	+.. code-block::
		257	+
		258	+ mailman: "\|/var/lib/mailman/mail/mailman post mailman"
		259	+ mailman-admin: "\|/var/lib/mailman/mail/mailman admin mailman"
		260	+ mailman-bounces: "\|/var/lib/mailman/mail/mailman bounces mailman"
		261	+ mailman-confirm: "\|/var/lib/mailman/mail/mailman confirm mailman"
		262	+ mailman-join: "\|/var/lib/mailman/mail/mailman join mailman"
		263	+ mailman-leave: "\|/var/lib/mailman/mail/mailman leave mailman"
		264	+ mailman-owner: "\|/var/lib/mailman/mail/mailman owner mailman"
		265	+ mailman-request: "\|/var/lib/mailman/mail/mailman request mailman"
		266	+ mailman-subscribe: "\|/var/lib/mailman/mail/mailman subscribe mailman"
		267	+ mailman-unsubscribe: "\|/var/lib/mailman/mail/mailman unsubscribe mailman"
		268	+
		269	+Run:
		270	+
		271	+.. code-block::
		272	+
		273	+ sudo newaliases
		274	+
		275	+Now restart postfix and mailman:
		276	+
		277	+.. code-block::
		278	+
		279	+ sudo /etc/init.d/postfix restart
		280	+ sudo /etc/init.d/mailman start
		281	+
		282	+Open /etc/nginx/sites-available/www.example.com.vhost and add the following part to the server {} container:
		283	+
		284	+.. code-block::
		285	+
		286	+ server {
		287	+ [...]
		288	+ location /cgi-bin/mailman {
		289	+ root /usr/lib/;
		290	+ fastcgi_split_path_info (^/cgi-bin/mailman/[^/])(.)$;
		291	+ include /etc/nginx/fastcgi_params;
		292	+ fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
		293	+ fastcgi_param PATH_INFO $fastcgi_path_info;
		294	+ fastcgi_param PATH_TRANSLATED $document_root$fastcgi_path_info;
		295	+ fastcgi_intercept_errors on;
		296	+ fastcgi_pass unix:/var/run/fcgiwrap.socket;
		297	+ }
		298	+ location /images/mailman {
		299	+ alias /usr/share/images/mailman;
		300	+ }
		301	+ location /pipermail {
		302	+ alias /var/lib/mailman/archives/public;
		303	+ autoindex on;
		304	+ }
		305	+ [...]
		306	+ }
		307	+
		308	+Now create the fastcgi_params file and add the following:
		309	+
		310	+.. code-block::
		311	+
		312	+ fastcgi_param QUERY_STRING $query_string;
		313	+ fastcgi_param REQUEST_METHOD $request_method;
		314	+ fastcgi_param CONTENT_TYPE $content_type;
		315	+ fastcgi_param CONTENT_LENGTH $content_length;
		316	+
		317	+ fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
		318	+ fastcgi_param SCRIPT_NAME $fastcgi_script_name;
		319	+ fastcgi_param REQUEST_URI $request_uri;
		320	+ fastcgi_param DOCUMENT_URI $document_uri;
		321	+ fastcgi_param DOCUMENT_ROOT $document_root;
		322	+ fastcgi_param SERVER_PROTOCOL $server_protocol;
		323	+
		324	+ fastcgi_param GATEWAY_INTERFACE CGI/1.1;
		325	+ fastcgi_param SERVER_SOFTWARE nginx;
		326	+
		327	+ fastcgi_param REMOTE_ADDR $remote_addr;
		328	+ fastcgi_param REMOTE_PORT $remote_port;
		329	+ fastcgi_param SERVER_ADDR $server_addr;
		330	+ fastcgi_param SERVER_PORT $server_port;
		331	+ fastcgi_param SERVER_NAME $server_name;
		332	+
		333	+Now open the mm_config.py file (sudo editor /etc/mailman/mm_config.py) and modify the lines below to:
		334	+
		335	+.. code-block::
		336	+
		337	+ DEFAULT_EMAIL_HOST = 'localhost'
		338	+ DEFAULT_URL_HOST = 'localhost:8080'
		339	+
		340	+To these modifications have effect do the following:
		341	+
		342	+.. code-block::
		343	+
		344	+ sudo withlist -l -a -r fix_url
		345	+
		346	+Also tt's needed to add permission to mailman to access the name_of_the_list.mbox because it needs to archive the emails:
		347	+
		348	+.. code-block::
		349	+
		350	+ sudo chown -R root:list /var/lib/mailman/archives
		351	+ sudo chown -R root:list /var/lib/mailman/data/aliases
		352	+ sudo chown -R root:list /etc/aliases
		353	+ sudo chmod -R u+rwX /var/lib/mailman/archives
		354	+
		355	+Now restart fcgi, mailman and nginx:
		356	+
		357	+.. code-block::
		358	+
		359	+ sudo /etc/init.d/fcgiwrap restart
		360	+ sudo /etc/init.d/mailman restart
		361	+ sudo /etc/init.d/nginx reload
		362	+
		363	+To see if it's working:
		364	+
		365	+.. code-block::
		366	+
		367	+ http://localhost:8080/cgi-bin/mailman/listinfo/mailman
		368	+
148	Setting up Colab	369	Setting up Colab
149	=========================	370	=========================
150		371
	@@ -160,7 +381,19 @@ Change COLAB_TRAC_URL to		@@ -160,7 +381,19 @@ Change COLAB_TRAC_URL to
160		381
161	Or the port you're using to Trac	382	Or the port you're using to Trac
162		383
163	-Use fabric at colab's root to update the requirements to your VM	384	+After this it's needed to import the e-mails from Mailman:
		385	+
		386	+.. code-block::
		387	+
		388	+ sudo python manage.py import_emails
		389	+
		390	+And also update Solr:
		391	+
		392	+.. code-block::
		393	+
		394	+ sudo python manage.py update_index
		395	+
		396	+Then, use fabric at colab's root to update the requirements to your VM
164		397
165	.. code-block::	398	.. code-block::
166		399
	@@ -171,44 +404,18 @@ Use fabric at colab's root to update the requirements to your VM		@@ -171,44 +404,18 @@ Use fabric at colab's root to update the requirements to your VM
171	The fabric installation on ubuntu through 'pip install' might not be added to the path so you'll need to find	404	The fabric installation on ubuntu through 'pip install' might not be added to the path so you'll need to find
172	where it was installed if thats your case	405	where it was installed if thats your case
173		406
174	-Now simply run fabric again to run the server (it'll also sync and migrate colab's database)	407	+Lastly, run fabric again to run the server (it'll also sync and migrate colab's database) and open new tabs to run trac and solr:
175		408
176	.. code-block::	409	.. code-block::
177		410
178	- fabric runserver	411	+ fab runserver
		412	+ fab trac
		413	+ fab solr
179		414
180	You should be able to see colab with its cms service already working at port 8000(because of vagrant redirects)	415	You should be able to see colab with its cms service already working at port 8000(because of vagrant redirects)
181		416
182	NOTE	417	NOTE
183		418
184	- In case login doesn't work, change the SITE_URL in src/colab/local_settings.py the reflect the django's port	419	+ In case login doesn't work, change the SITE_URL in src/colab/local_settings.py and in src/colab/custom_settings.py the reflect the django's port
185	Also, add the following line riht bellow it	420	Also, add the following line riht bellow it
186	BROWSERID_AUDIENCES = [SITE_URL, SITE_URL.replace('https', 'http')]	421	BROWSERID_AUDIENCES = [SITE_URL, SITE_URL.replace('https', 'http')]
187	-
188	-Installing Solr and indexing colab's schemas
189	-==========================================
190	-In Progress of making it readable
191	- #http servlet jetty
192	- sudo apt-get install jetty
193	-
194	- #solr
195	- wget http://ftp.unicamp.br/pub/apache/lucene/solr/4.6.1/solr-4.6.1.tgz
196	- tar xvzf solr-4.6.1.tgz
197	- sudo mv solr-4.6.1 /usr/share/solr
198	-
199	- sudo cp /usr/share/solr/example/webapps/solr.war /usr/share/solr/example/solr/solr.war
200	-
201	- python manage.py build_solr_schema >> schema.xml
202	- #existe mais de uma referencia a stopwords_en
203	- trocar no schema.xml stopwords_en.txt por lang/stopwords_en.txt
204	-
205	- sudo cp schema.xml /usr/share/solr/example/solr/collection1/conf
206	-
207	- #inclusive a tag
208	- Remova <updateLog> em solrconfig.xml localizado em /usr/share/solr/example/solr/collection1/conf
209	-
210	- #executa servidor
211	- cd /usr/share/solr/example/; java -jar start.jar;
212	-
213	- #indexa
214	- python manage.py update_index
215	\ No newline at end of file	422	\ No newline at end of file