Merge branch 'master' of github.com:colab/colab

Sergio Oliveira
2 parents fe35cecc 60aef4af
Showing 22 changed files with 349 additions and 45 deletions Show diff stats
colab/__init__.py
colab/celery.py
colab/exceptions.py
colab/home/apps.py
colab/plugins/__init__.py
colab/plugins/apps.py
colab/plugins/gitlab/apps.py
colab/plugins/gitlab/tasks.py
colab/plugins/utils/apps.py
colab/plugins/utils/signals.py
colab/plugins/utils/tests/__init__.py
colab/plugins/utils/tests/test_signals.py
colab/settings.py
colab/signals/__init__.py
colab/signals/exceptions.py
colab/signals/signals.py
colab/signals/tests/__init__.py
colab/signals/tests/test_signals.py
colab/utils/conf.py
docs/source/plugindev.rst
@@ -0,0 +1,5 @@
+from __future__ import absolute_import
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app  # noqa
@@ -14,13 +14,6 @@ app = Celery(&#39;colab&#39;)
 app.config_from_object('django.conf:settings')
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
-app.conf.update(
-    CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend',
-)
-app.conf.update(
-    CELERY_RESULT_BACKEND='djcelery.backends.cache:CacheBackend',
-)
-
 @app.task(bind=True)
 def debug_task(self):
@@ -0,0 +1,3 @@
+
+class ColabException(Exception):
+    """Base class for all exceptions raised by Colab"""
@@ -4,6 +4,3 @@ from django.apps import AppConfig
 class HomeConfig(AppConfig):
     name = 'colab.home'
-
-    def ready(self):
-        from ..celery import app  # noqa
@@ -0,0 +1,2 @@
+
+default_app_config = 'colab.plugins.apps.PluginAppConfig'
@@ -0,0 +1,12 @@
+
+from django.apps import AppConfig
+
+from .utils.signals import connect_signal, register_signal
+
+
+class PluginAppConfig(AppConfig):
+    name = 'colab.plugins'
+
+    def ready(self):
+        register_signal()
+        connect_signal()
 from ..utils.apps import ColabProxiedAppConfig
+from colab.plugins.gitlab.tasks import handling_method
+from colab.signals.signals import register_signal, connect_signal
 class ProxyGitlabAppConfig(ColabProxiedAppConfig):
     name = 'colab.plugins.gitlab'
     verbose_name = 'Gitlab Plugin'
+    short_name = 'gitlab'
+
+    signals_list = ['gitlab_create_project']
+
+    def register_signal(self):
+        register_signal(self.short_name, self.signals_list)
+
+    def connect_signal(self):
+        connect_signal(self.signals_list[0], self.short_name, handling_method)
@@ -0,0 +1,10 @@
+
+from colab.celery import app
+
+
+@app.task(bind=True)
+def handling_method(self, **kwargs):
+    f = open('/vagrant/test_plugin', 'wb')
+    f.write(str(kwargs))
+    f.close()
+    return 5
@@ -4,3 +4,9 @@ from django.apps import AppConfig
 class ColabProxiedAppConfig(AppConfig):
     colab_proxied_app = True
+
+    def register_signals(self):
+        pass
+
+    def connect_signals(self):
+        pass
@@ -0,0 +1,22 @@
+
+from django.apps import apps
+
+
+def _init_signals(method_name):
+    for app in apps.get_app_configs():
+        # Try to get the method with `method_name`.
+        #   If it exists call it using `app` as the first parameter.
+        #   This is required because methods take `self` as first
+        #   parameter and as we are calling it as a function python
+        #   won't send it explicitly.
+        # If the method doesn't exist we return a dummy function that
+        #   won't do anything.
+        getattr(app, method_name, lambda: None)()
+
+
+def register_signal():
+    _init_signals('register_signal')
+
+
+def connect_signal():
+    _init_signals('connect_signal')
@@ -0,0 +1,20 @@
+from django.test import TestCase
+from mock import patch, MagicMock
+from colab.plugins.utils import signals
+
+
+class SignalsTest(TestCase):
+    @patch("colab.plugins.utils.signals.apps.get_app_configs")
+    def test_init_signals(self, mock_app):
+        method_name = 'test'
+
+        app_mock = MagicMock()
+
+        apps_list = ['a', 'b', app_mock]
+
+        mock_app.return_value = apps_list
+        signals._init_signals(method_name)
+
+        app_mock.test.assert_called_with()
+        self.assertEqual(1, app_mock.test.call_count)
+        self.assertTrue(app_mock.test.called)
@@ -47,7 +47,6 @@ INSTALLED_APPS = (
     'haystack',
     'hitcounter',
     'taggit',
-    'djcelery',
     # Own apps
     'colab.home',
@@ -57,6 +56,7 @@ INSTALLED_APPS = (
     'colab.search',
     'colab.tz',
     'colab.utils',
+    'colab.signals',
 )
 ROOT_URLCONF = 'colab.urls'
@@ -253,8 +253,8 @@ from .utils import conf
 SOCIAL_NETWORK_ENABLED = locals().get('SOCIAL_NETWORK_ENABLED') or False
-locals().update(conf.load_colab_apps())
 locals().update(conf.load_py_settings())
+locals().update(conf.load_colab_apps())
 COLAB_APPS = locals().get('COLAB_APPS') or {}
 PROXIED_APPS = {}
@@ -0,0 +1,6 @@
+
+from ..exceptions import ColabException
+
+
+class SignalDoesNotExist(ColabException):
+    """Expcetion raised when signal does not exist"""
@@ -0,0 +1,47 @@
+
+from django.dispatch import Signal
+
+from .exceptions import SignalDoesNotExist
+
+registered_signals = {}
+signal_instances = {}
+
+
+class ColabSignal(Signal):
+    def __reduce__(self):
+        """
+
+        In order to send a signal to a celery task, it is necessary to pickle
+        the objects that will be used as parameters. However,
+        django.dispatch.Signal has an instance of threading.Lock, which is an
+        object that cannot be pickled. Therefore, this function changes the
+        pickle behaviour of Signal, making that only the providind_args of
+        Signal to be pickled."""
+
+        return (ColabSignal, (self.providing_args,))
+
+
+def register_signal(plugin_name, list_signals):
+    for signal in list_signals:
+        if signal in registered_signals:
+            if plugin_name not in registered_signals[signal]:
+                registered_signals[signal].append(plugin_name)
+        else:
+            registered_signals[signal] = []
+            registered_signals[signal].append(plugin_name)
+            signal_instances[signal] = ColabSignal()
+
+
+def connect_signal(signal_name, sender, handling_method):
+    if signal_name in signal_instances:
+        signal_instances[signal_name].connect(handling_method.delay,
+                                              sender=sender)
+    else:
+        raise SignalDoesNotExist
+
+
+def send(signal_name, sender, **kwargs):
+    if signal_name in signal_instances:
+        signal_instances[signal_name].send(sender=sender, **kwargs)
+    else:
+        raise SignalDoesNotExist
@@ -0,0 +1,76 @@
+"""
+Test Signals class.
+Objective: Test parameters, and behavior.
+"""
+
+from django.test import TestCase
+
+from mock import patch, MagicMock, PropertyMock
+
+from ..signals import registered_signals, register_signal, connect_signal, send
+from ..exceptions import SignalDoesNotExist
+
+
+class SignalsTest(TestCase):
+
+    def setUp(self):
+        self.list_signal = ['a', 'b', 'c']
+        self.plugin_name = 'test_signal'
+
+    def test_register_signal_(self):
+        register_signal(self.plugin_name, self.list_signal)
+        signal_name = 'a'
+        signal_list = ['test_signal']
+        self.assertEqual(len(registered_signals[signal_name]), 1)
+        self.assertEqual(registered_signals[signal_name], signal_list)
+
+    def test_register_signal_already_registered(self):
+        signal_name = 'a'
+        signal_list = ['test_signal']
+
+        register_signal(self.plugin_name, self.list_signal)
+        self.assertEqual(len(registered_signals[signal_name]), 1)
+
+        register_signal(self.plugin_name, self.list_signal)
+        self.assertEqual(len(registered_signals[signal_name]), 1)
+        self.assertEqual(registered_signals[signal_name], signal_list)
+
+    def test_connect_non_registered_signal(self):
+        sender = 'Test'
+        handling_method = 'Test'
+        signal_name = 'Test'
+
+        self.assertRaises(SignalDoesNotExist, connect_signal, signal_name,
+                          sender, handling_method)
+
+    @patch('colab.signals.signals.Signal.connect')
+    def test_connect_already_registered_signal(self, mock):
+        sender = 'Test'
+        handling_method = MagicMock()
+        type(handling_method).delay = PropertyMock(return_value='Test')
+        signal_name = 'a'
+
+        register_signal(self.plugin_name, self.list_signal)
+
+        connect_signal(signal_name, sender, handling_method)
+        args, kwargs = mock.call_args
+
+        self.assertEqual(args[0], handling_method.delay)
+        self.assertEqual(kwargs['sender'], sender)
+        self.assertTrue(mock.is_called)
+
+    @patch('colab.signals.signals.Signal.send')
+    def test_send_signal(self, mock):
+        sender = 'Test'
+        signal_name = 'a'
+
+        register_signal(self.plugin_name, self.list_signal)
+        send(signal_name, sender)
+
+        args, kwargs = mock.call_args
+
+        self.assertEqual(kwargs['sender'], sender)
+        self.assertTrue(mock.is_called)
+
+    def test_send_signal_not_registered(self):
+        self.assertRaises(SignalDoesNotExist, send, 'test_signal', 'test')
 import os
 import sys
+import logging
 import importlib
 import warnings
 from django.core.exceptions import ImproperlyConfigured
+logger = logging.getLogger('colab.init')
+if os.environ.get('COLAB_DEBUG'):
+    logger.addHandler(logging.StreamHandler())
+    logger.setLevel(logging.INFO)
+
 class InaccessibleSettings(ImproperlyConfigured):
     """Settings.py is Inaccessible.
@@ -48,57 +54,71 @@ def _load_py_file(py_path, path):
 def load_py_settings():
-    settings_dir = '/etc/colab/settings.d'
     settings_file = os.getenv('COLAB_SETTINGS', '/etc/colab/settings.py')
     settings_module = settings_file.split('.')[-2].split('/')[-1]
     py_path = "/".join(settings_file.split('/')[:-1])
+    logger.info('Settings file: %s', settings_file)
+
     if not os.path.exists(py_path):
         msg = "The py file {} does not exist".format(py_path)
         raise InaccessibleSettings(msg)
     py_settings = _load_py_file(settings_module, py_path)
-    # Try to read settings from settings.d
+    # Read settings from settings.d
+    settings_dir = '/etc/colab/settings.d'
+    logger.info('Settings directory: %s', settings_dir)
+
+    if not os.path.exists(settings_dir):
+        return py_settings
+
+    for file_name in os.listdir(settings_dir):
+        if not file_name.endswith('.py'):
+            continue
-    if os.path.exists(settings_dir):
-        for file_name in os.listdir(settings_dir):
-            if file_name.endswith('.py'):
-                file_module = file_name.split('.')[0]
-                py_settings_d = _load_py_file(file_module, settings_dir)
-                py_settings.update(py_settings_d)
+        file_module = file_name.split('.')[0]
+        py_settings_d = _load_py_file(file_module, settings_dir)
+        py_settings.update(py_settings_d)
+        logger.info('Loaded %s/%s', settings_dir, file_name)
     return py_settings
 def load_colab_apps():
     plugins_dir = os.getenv('COLAB_PLUGINS', '/etc/colab/plugins.d/')
+    logger.info('Plugin settings directory: %s', plugins_dir)
     COLAB_APPS = {}
     # Try to read settings from plugins.d
-    if os.path.exists(plugins_dir):
-        for file_name in os.listdir(plugins_dir):
-            if file_name.endswith('.py'):
-                file_module = file_name.split('.')[0]
-                py_settings_d = _load_py_file(file_module, plugins_dir)
-                fields = ['verbose_name', 'upstream', 'urls',
-                          'menu_urls', 'middlewares', 'dependencies',
-                          'context_processors', 'private_token']
-
-                app_name = py_settings_d.get('name')
-                if not app_name:
-                    warnings.warn("Plugin missing name variable")
-                    continue
-
-                COLAB_APPS[app_name] = {}
-                COLAB_APPS[app_name]['menu_title'] = \
-                    py_settings_d.get('menu_title')
-
-                for key in fields:
-                    value = py_settings_d.get(key)
-                    if value:
-                        COLAB_APPS[app_name][key] = value
+    if not os.path.exists(plugins_dir):
+        return {'COLAB_APPS': COLAB_APPS}
+
+    for file_name in os.listdir(plugins_dir):
+        if not file_name.endswith('.py'):
+            continue
+
+        file_module = file_name.split('.')[0]
+        py_settings_d = _load_py_file(file_module, plugins_dir)
+        logger.info('Loaded plugin settings: %s/%s', plugins_dir, file_name)
+
+        app_name = py_settings_d.get('name')
+        if not app_name:
+            warnings.warn("Plugin missing name variable")
+            continue
+
+        COLAB_APPS[app_name] = {}
+        COLAB_APPS[app_name]['menu_title'] = py_settings_d.get('menu_title')
+
+        fields = ['verbose_name', 'upstream', 'urls',
+                  'menu_urls', 'middlewares', 'dependencies',
+                  'context_processors', 'private_token']
+
+        for key in fields:
+            value = py_settings_d.get(key)
+            if value:
+                COLAB_APPS[app_name][key] = value
     return {'COLAB_APPS': COLAB_APPS}
-.. _plugin-dev: 
+.. _plugin-dev:
 Plugin Developer Documentation
 ====================================
 Getting Started
 ---------------
-.. TODO
+
+Signals
+----------
+Implement signals in plugins is optional! You may follow this steps only if you
+want to communicate with another plugins.
+
+In order to configure a plugin to able to listen and send signals using Colab
+signals structure, some steps are required:
+
+* In the apps.py file it is necessary to declare a list variable containing all
+  the signals that the plugin will dispatch. It is suggested to name the
+  variable "registered_signals", but that nomenclature not strictly necessary.
+* It is also necessary to declare a variable containing the name of the plugin
+  that will send the signal. It must be said that the name of the plugin cannot
+  contain any special character, such as dot or comma. It is suggested to name
+  the variable "short_name", but that nomenclature is not strictly
+  necessary.
+* In order to actually register the signals, it is necessary to implement the
+  method register_signal, which require the name of the plugin that is
+  registering the signals and a list of signals to be registered as parameters.
+  You must not call this method nowhere.
+* In order to listen for a given signal, it is required to create a handling
+  method. This method should be located at a file named tasks.py in the same
+  directory as the plugins files. It also must be said that this method need to
+  receive at least a \*\*kwargs parameter. An example of a handling method can
+  be seen below:
+
+.. code-block:: python
+   from colab.celery import app
+
+   @app.task(bind=True)
+   def handling_method(self, **kwargs):
+       # DO SOMETHING
+
+* With signals registered and handling method defined you must connect them.
+  To do it you must call connect_signal passing signal name, sender and handling
+  method as arguments. These should be implemented on plugin's apps.py. It must
+  be said that the plugin app class must extend ColabProxiedAppConfig. An
+  example of this configuration can be seen below:
+
+
+.. code-block:: python
+   from colab.plugins.utils.apps import ColabProxiedAppConfig
+   from colab.signals.signals import register_signal, connect_signal
+   from colab.plugins.PLUGIN.tasks import HANDLING_METHOD
+
+   class PluginApps(ColabProxiedAppConfig):
+        short_name = PLUGIN_NAME
+        signals_list = [SIGNAL1, SIGNAL2]
+
+        def registered_signal(self):
+            register_signal(self.short_name, self.signals_list)
+
+        def connect_signal(self):
+            connect_signal(self.signals_list[0], self.short_name,
+              HANDLING_METHOD)
+            connect_signal(self.signals_list[1], self.short_name,
+              HANDLING_METHOD)
+
+
+* To send a broadcast signal you must call send method anywhere passing signal
+  name and sender as arguments. If necessary you can pass another parameters in
+  \*\*kwargs. As you can see below:
+
+.. code-block:: python
+   from colab.signals.signals import send
+
+   send(signal_name, sender)
+
+* If you want to run celery manually to make some tests, you should execute:
+
+.. code-block:: shell
+   celery -A colab worker --loglevel=debug
@@ -16,7 +16,7 @@ REQUIREMENTS = [
     'diazo>=1.0.5',
     # Async Signals
-    'django-celery==3.1.16',
+    'celery>=3.1',
     ### Move out of colab (as plugins):
@@ -46,3 +46,5 @@ colab-admin loaddata /vagrant/tests/test_data.json
 sudo cp $basedir/vagrant/misc/etc/init.d/celeryd /etc/init.d/
 sudo cp $basedir/vagrant/misc/etc/default/celeryd /etc/default/
 sudo service celeryd start
+
+colab-admin rebuild_index --noinput
@@ -0,0 +1,5 @@		@@ -0,0 +1,5 @@
	1	+from __future__ import absolute_import
	2	+
	3	+# This will make sure the app is always imported when
	4	+# Django starts so that shared_task will use this app.
	5	+from .celery import app as celery_app # noqa
	@@ -14,13 +14,6 @@ app = Celery('colab')		@@ -14,13 +14,6 @@ app = Celery('colab')
14	app.config_from_object('django.conf:settings')	14	app.config_from_object('django.conf:settings')
15	app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)	15	app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
16		16
17	-app.conf.update(
18	- CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend',
19	-)
20	-app.conf.update(
21	- CELERY_RESULT_BACKEND='djcelery.backends.cache:CacheBackend',
22	-)
23	-
24		17
25	@app.task(bind=True)	18	@app.task(bind=True)
26	def debug_task(self):	19	def debug_task(self):
@@ -0,0 +1,3 @@		@@ -0,0 +1,3 @@
	1	+
	2	+class ColabException(Exception):
	3	+ """Base class for all exceptions raised by Colab"""
	@@ -4,6 +4,3 @@ from django.apps import AppConfig		@@ -4,6 +4,3 @@ from django.apps import AppConfig
4		4
5	class HomeConfig(AppConfig):	5	class HomeConfig(AppConfig):
6	name = 'colab.home'	6	name = 'colab.home'
7	-
8	- def ready(self):
9	- from ..celery import app # noqa
@@ -0,0 +1,2 @@		@@ -0,0 +1,2 @@
	1	+
	2	+default_app_config = 'colab.plugins.apps.PluginAppConfig'
@@ -0,0 +1,12 @@		@@ -0,0 +1,12 @@
	1	+
	2	+from django.apps import AppConfig
	3	+
	4	+from .utils.signals import connect_signal, register_signal
	5	+
	6	+
	7	+class PluginAppConfig(AppConfig):
	8	+ name = 'colab.plugins'
	9	+
	10	+ def ready(self):
	11	+ register_signal()
	12	+ connect_signal()
1		1
2	from ..utils.apps import ColabProxiedAppConfig	2	from ..utils.apps import ColabProxiedAppConfig
		3	+from colab.plugins.gitlab.tasks import handling_method
		4	+from colab.signals.signals import register_signal, connect_signal
3		5
4		6
5	class ProxyGitlabAppConfig(ColabProxiedAppConfig):	7	class ProxyGitlabAppConfig(ColabProxiedAppConfig):
6	name = 'colab.plugins.gitlab'	8	name = 'colab.plugins.gitlab'
7	verbose_name = 'Gitlab Plugin'	9	verbose_name = 'Gitlab Plugin'
		10	+ short_name = 'gitlab'
		11	+
		12	+ signals_list = ['gitlab_create_project']
		13	+
		14	+ def register_signal(self):
		15	+ register_signal(self.short_name, self.signals_list)
		16	+
		17	+ def connect_signal(self):
		18	+ connect_signal(self.signals_list[0], self.short_name, handling_method)
@@ -0,0 +1,10 @@		@@ -0,0 +1,10 @@
	1	+
	2	+from colab.celery import app
	3	+
	4	+
	5	+@app.task(bind=True)
	6	+def handling_method(self, **kwargs):
	7	+ f = open('/vagrant/test_plugin', 'wb')
	8	+ f.write(str(kwargs))
	9	+ f.close()
	10	+ return 5
	@@ -4,3 +4,9 @@ from django.apps import AppConfig		@@ -4,3 +4,9 @@ from django.apps import AppConfig
4		4
5	class ColabProxiedAppConfig(AppConfig):	5	class ColabProxiedAppConfig(AppConfig):
6	colab_proxied_app = True	6	colab_proxied_app = True
		7	+
		8	+ def register_signals(self):
		9	+ pass
		10	+
		11	+ def connect_signals(self):
		12	+ pass
@@ -0,0 +1,22 @@		@@ -0,0 +1,22 @@
	1	+
	2	+from django.apps import apps
	3	+
	4	+
	5	+def _init_signals(method_name):
	6	+ for app in apps.get_app_configs():
	7	+ # Try to get the method with `method_name`.
	8	+ # If it exists call it using `app` as the first parameter.
	9	+ # This is required because methods take `self` as first
	10	+ # parameter and as we are calling it as a function python
	11	+ # won't send it explicitly.
	12	+ # If the method doesn't exist we return a dummy function that
	13	+ # won't do anything.
	14	+ getattr(app, method_name, lambda: None)()
	15	+
	16	+
	17	+def register_signal():
	18	+ _init_signals('register_signal')
	19	+
	20	+
	21	+def connect_signal():
	22	+ _init_signals('connect_signal')