data.py 818 Bytes
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 
#!/usr/bin/env python

import importlib

from django.conf import settings

from colab.celery import app
from proxy_data_api import ProxyDataAPI


TASKS = set()


def register_tasks():

    global TASKS

    for app_name in settings.INSTALLED_APPS:

        try:
            module = importlib.import_module('{}.data_api'.format(app_name))
        except ImportError as e:
            continue

        for item_name in dir(module):
            item = getattr(module, item_name)
            if item is ProxyDataAPI:
                continue

            if callable(getattr(item, 'fetch_data', None)):
                instance = item()
                task = app.task(bind=True)(instance.fetch_data)
                TASKS.add(task)

    return TASKS


def data_import(self):
    for task in TASKS:
        task.delay()