Download as
Browse Code »

Commit fd7476715230d54e084c3090e36bb6fd83e7286d

Authored by Sergio Oliveira
2 parents fe35cecc 60aef4af
Exists in master and in 29 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, colab-vcard, colab_search, colab_tag, colab_tag_merge, detach_super_archives, fix-dashboard, fix-message-preview, fix_app_label, fix_header, fixed_gitlab_import, go_to_profile_panel, mobile_user_scalable, paginate-threads, profile_integration, refactor-data-import, remove-gitlab-plugin, removing_namespace, search_block, search_filters, split_screen, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd

Merge branch 'master' of github.com:colab/colab

Showing 22 changed files with 349 additions and 45 deletions   Show diff stats
colab/__init__.py
  Diff comments   View file @fd74767
... ... @@ -0,0 +1,5 @@
  1 +from __future__ import absolute_import
  2 +
  3 +# This will make sure the app is always imported when
  4 +# Django starts so that shared_task will use this app.
  5 +from .celery import app as celery_app # noqa
... ...
colab/celery.py
  Diff comments   View file @fd74767
... ... @@ -14,13 +14,6 @@ app = Celery('colab')
14 14 app.config_from_object('django.conf:settings')
15 15 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
16 16  
17   -app.conf.update(
18   - CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend',
19   -)
20   -app.conf.update(
21   - CELERY_RESULT_BACKEND='djcelery.backends.cache:CacheBackend',
22   -)
23   -
24 17  
25 18 @app.task(bind=True)
26 19 def debug_task(self):
... ...
colab/exceptions.py 0 → 100644
  Diff comments   View file @fd74767
... ... @@ -0,0 +1,3 @@
  1 +
  2 +class ColabException(Exception):
  3 + """Base class for all exceptions raised by Colab"""
... ...
colab/home/apps.py
  Diff comments   View file @fd74767
... ... @@ -4,6 +4,3 @@ from django.apps import AppConfig
4 4  
5 5 class HomeConfig(AppConfig):
6 6 name = 'colab.home'
7   -
8   - def ready(self):
9   - from ..celery import app # noqa
... ...
colab/plugins/__init__.py
  Diff comments   View file @fd74767
... ... @@ -0,0 +1,2 @@
  1 +
  2 +default_app_config = 'colab.plugins.apps.PluginAppConfig'
... ...
colab/plugins/apps.py 0 → 100644
  Diff comments   View file @fd74767
... ... @@ -0,0 +1,12 @@
  1 +
  2 +from django.apps import AppConfig
  3 +
  4 +from .utils.signals import connect_signal, register_signal
  5 +
  6 +
  7 +class PluginAppConfig(AppConfig):
  8 + name = 'colab.plugins'
  9 +
  10 + def ready(self):
  11 + register_signal()
  12 + connect_signal()
... ...
colab/plugins/gitlab/apps.py
  Diff comments   View file @fd74767
1 1  
2 2 from ..utils.apps import ColabProxiedAppConfig
  3 +from colab.plugins.gitlab.tasks import handling_method
  4 +from colab.signals.signals import register_signal, connect_signal
3 5  
4 6  
5 7 class ProxyGitlabAppConfig(ColabProxiedAppConfig):
6 8 name = 'colab.plugins.gitlab'
7 9 verbose_name = 'Gitlab Plugin'
  10 + short_name = 'gitlab'
  11 +
  12 + signals_list = ['gitlab_create_project']
  13 +
  14 + def register_signal(self):
  15 + register_signal(self.short_name, self.signals_list)
  16 +
  17 + def connect_signal(self):
  18 + connect_signal(self.signals_list[0], self.short_name, handling_method)
... ...
colab/plugins/gitlab/tasks.py 0 → 100644
  Diff comments   View file @fd74767
... ... @@ -0,0 +1,10 @@
  1 +
  2 +from colab.celery import app
  3 +
  4 +
  5 +@app.task(bind=True)
  6 +def handling_method(self, **kwargs):
  7 + f = open('/vagrant/test_plugin', 'wb')
  8 + f.write(str(kwargs))
  9 + f.close()
  10 + return 5
... ...
colab/plugins/utils/apps.py
  Diff comments   View file @fd74767
... ... @@ -4,3 +4,9 @@ from django.apps import AppConfig
4 4  
5 5 class ColabProxiedAppConfig(AppConfig):
6 6 colab_proxied_app = True
  7 +
  8 + def register_signals(self):
  9 + pass
  10 +
  11 + def connect_signals(self):
  12 + pass
... ...
colab/plugins/utils/signals.py 0 → 100644
  Diff comments   View file @fd74767
... ... @@ -0,0 +1,22 @@
  1 +
  2 +from django.apps import apps
  3 +
  4 +
  5 +def _init_signals(method_name):
  6 + for app in apps.get_app_configs():
  7 + # Try to get the method with `method_name`.
  8 + # If it exists call it using `app` as the first parameter.
  9 + # This is required because methods take `self` as first
  10 + # parameter and as we are calling it as a function python
  11 + # won't send it explicitly.
  12 + # If the method doesn't exist we return a dummy function that
  13 + # won't do anything.
  14 + getattr(app, method_name, lambda: None)()
  15 +
  16 +
  17 +def register_signal():
  18 + _init_signals('register_signal')
  19 +
  20 +
  21 +def connect_signal():
  22 + _init_signals('connect_signal')
... ...
colab/plugins/utils/tests/__init__.py 0 → 100644
  Diff comments   View file @fd74767
colab/plugins/utils/tests/test_signals.py 0 → 100644
  Diff comments   View file @fd74767
... ... @@ -0,0 +1,20 @@
  1 +from django.test import TestCase
  2 +from mock import patch, MagicMock
  3 +from colab.plugins.utils import signals
  4 +
  5 +
  6 +class SignalsTest(TestCase):
  7 + @patch("colab.plugins.utils.signals.apps.get_app_configs")
  8 + def test_init_signals(self, mock_app):
  9 + method_name = 'test'
  10 +
  11 + app_mock = MagicMock()
  12 +
  13 + apps_list = ['a', 'b', app_mock]
  14 +
  15 + mock_app.return_value = apps_list
  16 + signals._init_signals(method_name)
  17 +
  18 + app_mock.test.assert_called_with()
  19 + self.assertEqual(1, app_mock.test.call_count)
  20 + self.assertTrue(app_mock.test.called)
... ...
colab/settings.py
  Diff comments   View file @fd74767
... ... @@ -47,7 +47,6 @@ INSTALLED_APPS = (
47 47 'haystack',
48 48 'hitcounter',
49 49 'taggit',
50   - 'djcelery',
51 50  
52 51 # Own apps
53 52 'colab.home',
... ... @@ -57,6 +56,7 @@ INSTALLED_APPS = (
57 56 'colab.search',
58 57 'colab.tz',
59 58 'colab.utils',
  59 + 'colab.signals',
60 60 )
61 61  
62 62 ROOT_URLCONF = 'colab.urls'
... ... @@ -253,8 +253,8 @@ from .utils import conf
253 253  
254 254 SOCIAL_NETWORK_ENABLED = locals().get('SOCIAL_NETWORK_ENABLED') or False
255 255  
256   -locals().update(conf.load_colab_apps())
257 256 locals().update(conf.load_py_settings())
  257 +locals().update(conf.load_colab_apps())
258 258  
259 259 COLAB_APPS = locals().get('COLAB_APPS') or {}
260 260 PROXIED_APPS = {}
... ...
colab/signals/__init__.py 0 → 100644
  Diff comments   View file @fd74767
colab/signals/exceptions.py 0 → 100644
  Diff comments   View file @fd74767
... ... @@ -0,0 +1,6 @@
  1 +
  2 +from ..exceptions import ColabException
  3 +
  4 +
  5 +class SignalDoesNotExist(ColabException):
  6 + """Expcetion raised when signal does not exist"""
... ...
colab/signals/signals.py 0 → 100644
  Diff comments   View file @fd74767
... ... @@ -0,0 +1,47 @@
  1 +
  2 +from django.dispatch import Signal
  3 +
  4 +from .exceptions import SignalDoesNotExist
  5 +
  6 +registered_signals = {}
  7 +signal_instances = {}
  8 +
  9 +
  10 +class ColabSignal(Signal):
  11 + def __reduce__(self):
  12 + """
  13 +
  14 + In order to send a signal to a celery task, it is necessary to pickle
  15 + the objects that will be used as parameters. However,
  16 + django.dispatch.Signal has an instance of threading.Lock, which is an
  17 + object that cannot be pickled. Therefore, this function changes the
  18 + pickle behaviour of Signal, making that only the providind_args of
  19 + Signal to be pickled."""
  20 +
  21 + return (ColabSignal, (self.providing_args,))
  22 +
  23 +
  24 +def register_signal(plugin_name, list_signals):
  25 + for signal in list_signals:
  26 + if signal in registered_signals:
  27 + if plugin_name not in registered_signals[signal]:
  28 + registered_signals[signal].append(plugin_name)
  29 + else:
  30 + registered_signals[signal] = []
  31 + registered_signals[signal].append(plugin_name)
  32 + signal_instances[signal] = ColabSignal()
  33 +
  34 +
  35 +def connect_signal(signal_name, sender, handling_method):
  36 + if signal_name in signal_instances:
  37 + signal_instances[signal_name].connect(handling_method.delay,
  38 + sender=sender)
  39 + else:
  40 + raise SignalDoesNotExist
  41 +
  42 +
  43 +def send(signal_name, sender, **kwargs):
  44 + if signal_name in signal_instances:
  45 + signal_instances[signal_name].send(sender=sender, **kwargs)
  46 + else:
  47 + raise SignalDoesNotExist
... ...
colab/signals/tests/__init__.py 0 → 100644
  Diff comments   View file @fd74767
colab/signals/tests/test_signals.py 0 → 100644
  Diff comments   View file @fd74767
... ... @@ -0,0 +1,76 @@
  1 +"""
  2 +Test Signals class.
  3 +Objective: Test parameters, and behavior.
  4 +"""
  5 +
  6 +from django.test import TestCase
  7 +
  8 +from mock import patch, MagicMock, PropertyMock
  9 +
  10 +from ..signals import registered_signals, register_signal, connect_signal, send
  11 +from ..exceptions import SignalDoesNotExist
  12 +
  13 +
  14 +class SignalsTest(TestCase):
  15 +
  16 + def setUp(self):
  17 + self.list_signal = ['a', 'b', 'c']
  18 + self.plugin_name = 'test_signal'
  19 +
  20 + def test_register_signal_(self):
  21 + register_signal(self.plugin_name, self.list_signal)
  22 + signal_name = 'a'
  23 + signal_list = ['test_signal']
  24 + self.assertEqual(len(registered_signals[signal_name]), 1)
  25 + self.assertEqual(registered_signals[signal_name], signal_list)
  26 +
  27 + def test_register_signal_already_registered(self):
  28 + signal_name = 'a'
  29 + signal_list = ['test_signal']
  30 +
  31 + register_signal(self.plugin_name, self.list_signal)
  32 + self.assertEqual(len(registered_signals[signal_name]), 1)
  33 +
  34 + register_signal(self.plugin_name, self.list_signal)
  35 + self.assertEqual(len(registered_signals[signal_name]), 1)
  36 + self.assertEqual(registered_signals[signal_name], signal_list)
  37 +
  38 + def test_connect_non_registered_signal(self):
  39 + sender = 'Test'
  40 + handling_method = 'Test'
  41 + signal_name = 'Test'
  42 +
  43 + self.assertRaises(SignalDoesNotExist, connect_signal, signal_name,
  44 + sender, handling_method)
  45 +
  46 + @patch('colab.signals.signals.Signal.connect')
  47 + def test_connect_already_registered_signal(self, mock):
  48 + sender = 'Test'
  49 + handling_method = MagicMock()
  50 + type(handling_method).delay = PropertyMock(return_value='Test')
  51 + signal_name = 'a'
  52 +
  53 + register_signal(self.plugin_name, self.list_signal)
  54 +
  55 + connect_signal(signal_name, sender, handling_method)
  56 + args, kwargs = mock.call_args
  57 +
  58 + self.assertEqual(args[0], handling_method.delay)
  59 + self.assertEqual(kwargs['sender'], sender)
  60 + self.assertTrue(mock.is_called)
  61 +
  62 + @patch('colab.signals.signals.Signal.send')
  63 + def test_send_signal(self, mock):
  64 + sender = 'Test'
  65 + signal_name = 'a'
  66 +
  67 + register_signal(self.plugin_name, self.list_signal)
  68 + send(signal_name, sender)
  69 +
  70 + args, kwargs = mock.call_args
  71 +
  72 + self.assertEqual(kwargs['sender'], sender)
  73 + self.assertTrue(mock.is_called)
  74 +
  75 + def test_send_signal_not_registered(self):
  76 + self.assertRaises(SignalDoesNotExist, send, 'test_signal', 'test')
... ...
colab/utils/conf.py
  Diff comments   View file @fd74767
1 1  
2 2 import os
3 3 import sys
  4 +import logging
4 5 import importlib
5 6 import warnings
6 7  
7 8 from django.core.exceptions import ImproperlyConfigured
8 9  
  10 +logger = logging.getLogger('colab.init')
  11 +if os.environ.get('COLAB_DEBUG'):
  12 + logger.addHandler(logging.StreamHandler())
  13 + logger.setLevel(logging.INFO)
  14 +
9 15  
10 16 class InaccessibleSettings(ImproperlyConfigured):
11 17 """Settings.py is Inaccessible.
... ... @@ -48,57 +54,71 @@ def _load_py_file(py_path, path):
48 54  
49 55  
50 56 def load_py_settings():
51   - settings_dir = '/etc/colab/settings.d'
52 57 settings_file = os.getenv('COLAB_SETTINGS', '/etc/colab/settings.py')
53 58 settings_module = settings_file.split('.')[-2].split('/')[-1]
54 59 py_path = "/".join(settings_file.split('/')[:-1])
55 60  
  61 + logger.info('Settings file: %s', settings_file)
  62 +
56 63 if not os.path.exists(py_path):
57 64 msg = "The py file {} does not exist".format(py_path)
58 65 raise InaccessibleSettings(msg)
59 66  
60 67 py_settings = _load_py_file(settings_module, py_path)
61 68  
62   - # Try to read settings from settings.d
  69 + # Read settings from settings.d
  70 + settings_dir = '/etc/colab/settings.d'
  71 + logger.info('Settings directory: %s', settings_dir)
  72 +
  73 + if not os.path.exists(settings_dir):
  74 + return py_settings
  75 +
  76 + for file_name in os.listdir(settings_dir):
  77 + if not file_name.endswith('.py'):
  78 + continue
63 79  
64   - if os.path.exists(settings_dir):
65   - for file_name in os.listdir(settings_dir):
66   - if file_name.endswith('.py'):
67   - file_module = file_name.split('.')[0]
68   - py_settings_d = _load_py_file(file_module, settings_dir)
69   - py_settings.update(py_settings_d)
  80 + file_module = file_name.split('.')[0]
  81 + py_settings_d = _load_py_file(file_module, settings_dir)
  82 + py_settings.update(py_settings_d)
  83 + logger.info('Loaded %s/%s', settings_dir, file_name)
70 84  
71 85 return py_settings
72 86  
73 87  
74 88 def load_colab_apps():
75 89 plugins_dir = os.getenv('COLAB_PLUGINS', '/etc/colab/plugins.d/')
  90 + logger.info('Plugin settings directory: %s', plugins_dir)
76 91  
77 92 COLAB_APPS = {}
78 93  
79 94 # Try to read settings from plugins.d
80   - if os.path.exists(plugins_dir):
81   - for file_name in os.listdir(plugins_dir):
82   - if file_name.endswith('.py'):
83   - file_module = file_name.split('.')[0]
84   - py_settings_d = _load_py_file(file_module, plugins_dir)
85   - fields = ['verbose_name', 'upstream', 'urls',
86   - 'menu_urls', 'middlewares', 'dependencies',
87   - 'context_processors', 'private_token']
88   -
89   - app_name = py_settings_d.get('name')
90   - if not app_name:
91   - warnings.warn("Plugin missing name variable")
92   - continue
93   -
94   - COLAB_APPS[app_name] = {}
95   - COLAB_APPS[app_name]['menu_title'] = \
96   - py_settings_d.get('menu_title')
97   -
98   - for key in fields:
99   - value = py_settings_d.get(key)
100   - if value:
101   - COLAB_APPS[app_name][key] = value
  95 + if not os.path.exists(plugins_dir):
  96 + return {'COLAB_APPS': COLAB_APPS}
  97 +
  98 + for file_name in os.listdir(plugins_dir):
  99 + if not file_name.endswith('.py'):
  100 + continue
  101 +
  102 + file_module = file_name.split('.')[0]
  103 + py_settings_d = _load_py_file(file_module, plugins_dir)
  104 + logger.info('Loaded plugin settings: %s/%s', plugins_dir, file_name)
  105 +
  106 + app_name = py_settings_d.get('name')
  107 + if not app_name:
  108 + warnings.warn("Plugin missing name variable")
  109 + continue
  110 +
  111 + COLAB_APPS[app_name] = {}
  112 + COLAB_APPS[app_name]['menu_title'] = py_settings_d.get('menu_title')
  113 +
  114 + fields = ['verbose_name', 'upstream', 'urls',
  115 + 'menu_urls', 'middlewares', 'dependencies',
  116 + 'context_processors', 'private_token']
  117 +
  118 + for key in fields:
  119 + value = py_settings_d.get(key)
  120 + if value:
  121 + COLAB_APPS[app_name][key] = value
102 122  
103 123 return {'COLAB_APPS': COLAB_APPS}
104 124  
... ...
docs/source/plugindev.rst
  Diff comments   View file @fd74767
1 1  
2   -.. _plugin-dev:
  2 +.. _plugin-dev:
3 3  
4 4 Plugin Developer Documentation
5 5 ====================================
6 6  
7 7 Getting Started
8 8 ---------------
9   -.. TODO
  9 +
  10 +Signals
  11 +----------
  12 +Implement signals in plugins is optional! You may follow this steps only if you
  13 +want to communicate with another plugins.
  14 +
  15 +In order to configure a plugin to able to listen and send signals using Colab
  16 +signals structure, some steps are required:
  17 +
  18 +* In the apps.py file it is necessary to declare a list variable containing all
  19 + the signals that the plugin will dispatch. It is suggested to name the
  20 + variable "registered_signals", but that nomenclature not strictly necessary.
  21 +* It is also necessary to declare a variable containing the name of the plugin
  22 + that will send the signal. It must be said that the name of the plugin cannot
  23 + contain any special character, such as dot or comma. It is suggested to name
  24 + the variable "short_name", but that nomenclature is not strictly
  25 + necessary.
  26 +* In order to actually register the signals, it is necessary to implement the
  27 + method register_signal, which require the name of the plugin that is
  28 + registering the signals and a list of signals to be registered as parameters.
  29 + You must not call this method nowhere.
  30 +* In order to listen for a given signal, it is required to create a handling
  31 + method. This method should be located at a file named tasks.py in the same
  32 + directory as the plugins files. It also must be said that this method need to
  33 + receive at least a \*\*kwargs parameter. An example of a handling method can
  34 + be seen below:
  35 +
  36 +.. code-block:: python
  37 + from colab.celery import app
  38 +
  39 + @app.task(bind=True)
  40 + def handling_method(self, **kwargs):
  41 + # DO SOMETHING
  42 +
  43 +* With signals registered and handling method defined you must connect them.
  44 + To do it you must call connect_signal passing signal name, sender and handling
  45 + method as arguments. These should be implemented on plugin's apps.py. It must
  46 + be said that the plugin app class must extend ColabProxiedAppConfig. An
  47 + example of this configuration can be seen below:
  48 +
  49 +
  50 +.. code-block:: python
  51 + from colab.plugins.utils.apps import ColabProxiedAppConfig
  52 + from colab.signals.signals import register_signal, connect_signal
  53 + from colab.plugins.PLUGIN.tasks import HANDLING_METHOD
  54 +
  55 + class PluginApps(ColabProxiedAppConfig):
  56 + short_name = PLUGIN_NAME
  57 + signals_list = [SIGNAL1, SIGNAL2]
  58 +
  59 + def registered_signal(self):
  60 + register_signal(self.short_name, self.signals_list)
  61 +
  62 + def connect_signal(self):
  63 + connect_signal(self.signals_list[0], self.short_name,
  64 + HANDLING_METHOD)
  65 + connect_signal(self.signals_list[1], self.short_name,
  66 + HANDLING_METHOD)
  67 +
  68 +
  69 +* To send a broadcast signal you must call send method anywhere passing signal
  70 + name and sender as arguments. If necessary you can pass another parameters in
  71 + \*\*kwargs. As you can see below:
  72 +
  73 +.. code-block:: python
  74 + from colab.signals.signals import send
  75 +
  76 + send(signal_name, sender)
  77 +
  78 +* If you want to run celery manually to make some tests, you should execute:
  79 +
  80 +.. code-block:: shell
  81 + celery -A colab worker --loglevel=debug
... ...
setup.py
  Diff comments   View file @fd74767
... ... @@ -16,7 +16,7 @@ REQUIREMENTS = [
16 16 'diazo>=1.0.5',
17 17  
18 18 # Async Signals
19   - 'django-celery==3.1.16',
  19 + 'celery>=3.1',
20 20  
21 21 ### Move out of colab (as plugins):
22 22  
... ...
vagrant/provision.sh
  Diff comments   View file @fd74767
... ... @@ -46,3 +46,5 @@ colab-admin loaddata /vagrant/tests/test_data.json
46 46 sudo cp $basedir/vagrant/misc/etc/init.d/celeryd /etc/init.d/
47 47 sudo cp $basedir/vagrant/misc/etc/default/celeryd /etc/default/
48 48 sudo service celeryd start
  49 +
  50 +colab-admin rebuild_index --noinput
... ...