Download as
Browse Code »

Commit 1efa7b87c6463e9e647beb29ec3799d7a4b98a1e

Authored by Sergio Oliveira
1 parent d19c324f
Exists in master and in 29 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, colab-vcard, colab_search, colab_tag, colab_tag_merge, detach_super_archives, fix-dashboard, fix-message-preview, fix_app_label, fix_header, fixed_gitlab_import, go_to_profile_panel, mobile_user_scalable, paginate-threads, profile_integration, refactor-data-import, remove-gitlab-plugin, removing_namespace, search_block, search_filters, split_screen, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd

Splitted gitlab imports to allow async workers 

Signed-off-by: Sergio Oliveira <sergio@tracy.com.br>
Showing 2 changed files with 25 additions and 10 deletions   Show diff stats
colab/plugins/gitlab/data_api.py
  Diff comments   View file @1efa7b8
... ... @@ -5,23 +5,22 @@ import logging
5 5  
6 6 from dateutil.parser import parse
7 7  
8   -from django.conf import settings
9 8 from django.db.models.fields import DateTimeField
10 9  
11 10 from colab.plugins.gitlab.models import (GitlabProject, GitlabMergeRequest,
12 11 GitlabComment, GitlabIssue)
13 12 from colab.plugins.utils.proxy_data_api import ProxyDataAPI
14 13  
  14 +
15 15 LOGGER = logging.getLogger('colab.plugin.gitlab')
16 16  
17 17  
18   -class GitlabDataAPI(ProxyDataAPI):
  18 +class GitlabDataImporter(ProxyDataAPI):
  19 + app_label = 'gitlab'
19 20  
20 21 def get_request_url(self, path, **kwargs):
21   - proxy_config = settings.COLAB_APPS.get(self.app_label, {})
22   -
23   - upstream = proxy_config.get('upstream')
24   - kwargs['private_token'] = proxy_config.get('private_token')
  22 + upstream = self.config.get('upstream')
  23 + kwargs['private_token'] = self.config.get('private_token')
25 24 params = urllib.urlencode(kwargs)
26 25  
27 26 if upstream[-1] == '/':
... ... @@ -179,27 +178,38 @@ class GitlabDataAPI(ProxyDataAPI):
179 178  
180 179 return all_comments
181 180  
  181 +
  182 +class GitlabProjectImporter(GitlabDataImporter):
  183 +
182 184 def fetch_data(self):
183 185 LOGGER.info("Importing Projects")
184 186 projects = self.fetch_projects()
185 187 for datum in projects:
186 188 datum.save()
187 189  
  190 +
  191 +class GitlabMergeRequestImporter(GitlabDataImporter):
  192 +
  193 + def fetch_data(self):
188 194 LOGGER.info("Importing Merge Requests")
189 195 merge_request_list = self.fetch_merge_request(projects)
190 196 for datum in merge_request_list:
191 197 datum.save()
192 198  
  199 +
  200 +class GitlabIssueImporter(GitlabDataImporter):
  201 +
  202 + def fetch_data(self):
193 203 LOGGER.info("Importing Issues")
194 204 issue_list = self.fetch_issue(projects)
195 205 for datum in issue_list:
196 206 datum.save()
197 207  
  208 +
  209 +class GitlabCommentImporter(GitlabDataImporter):
  210 +
  211 + def fetch_data(self):
198 212 LOGGER.info("Importing Comments")
199 213 comments_list = self.fetch_comments()
200 214 for datum in comments_list:
201 215 datum.save()
202   -
203   - @property
204   - def app_label(self):
205   - return 'gitlab'
... ...
colab/plugins/utils/proxy_data_api.py
  Diff comments   View file @1efa7b8
1 1  
2 2 import abc
3 3  
  4 +from django.conf import settings
  5 +
4 6  
5 7 class ProxyDataAPI(object):
6 8  
  9 + def __init__(self):
  10 + self.config = settings.COLAB_APPS.get(self.app_label, {})
  11 +
7 12 @abc.abstractmethod
8 13 def fetch_data(self):
9 14 raise NotImplementedError('fetchData not yet implemented')
... ...