Download as
Browse Code »

Commit a2cad50a7c5fd9fb0c9acde0621d73ebe3038bf1

Authored by Matheus Fernandes
Committed by Lucas Kanashiro
1 parent fb0a796d
Exists in master and in 27 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, colab-vcard, colab_search, colab_tag, colab_tag_merge, detach_super_archives, fix-dashboard, fix-message-preview, fix_header, fixed_gitlab_import, go_to_profile_panel, mobile_user_scalable, paginate-threads, profile_integration, remove-gitlab-plugin, removing_namespace, search_block, search_filters, split_screen, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd

Refactored data_importer 

Signed-off-by: Matheus Fernandes <matheus.souza.fernandes@gmail.com>
Signed-off-by: Lucas Moura <lucas.moura128@gmail.com>
Showing 1 changed file with 47 additions and 85 deletions   Show diff stats
colab/plugins/gitlab/data_importer.py
  Diff comments   View file @a2cad50
... ... @@ -9,7 +9,7 @@ from django.db.models.fields import DateTimeField
9 9 from colab.plugins.data import PluginDataImporter
10 10  
11 11 from .models import (GitlabProject, GitlabMergeRequest,
12   - GitlabComment, GitlabIssue)
  12 + GitlabComment, GitlabIssue, GitlabGroup)
13 13  
14 14  
15 15 LOGGER = logging.getLogger('colab.plugin.gitlab')
... ... @@ -63,120 +63,82 @@ class GitlabDataImporter(PluginDataImporter):
63 63  
64 64 return _object
65 65  
66   - def fetch_projects(self):
  66 + def fetch(self, url, gitlab_class):
67 67 page = 1
68   - projects = []
  68 + obj_list = []
69 69  
70 70 while True:
71   - json_data = self.get_json_data('/api/v3/projects/all', page)
  71 + json_data = self.get_json_data(url, page)
72 72 page = page + 1
73 73  
74 74 if not len(json_data):
75 75 break
76 76  
77 77 for element in json_data:
78   - project = GitlabProject()
79   - self.fill_object_data(element, project)
80   - projects.append(project)
  78 + obj = gitlab_class()
  79 + self.fill_object_data(element, obj)
  80 + obj_list.append(obj)
81 81  
82   - return projects
  82 + return obj_list
83 83  
84   - def fetch_merge_request(self, projects):
85   - all_merge_request = []
  84 + def fetch_comments(self, url, parent_class, issue_comment):
  85 + all_comments = []
  86 + all_parent_objects = parent_class.objects.all()
86 87  
87   - for project in projects:
  88 + for parent_obj in all_parent_objects:
88 89 page = 1
89 90 while True:
90   - url = '/api/v3/projects/{}/merge_requests'.format(project.id)
91   - json_data_mr = self.get_json_data(url, page)
  91 + formated_url = url.format(parent_obj.project_id, parent_obj.id)
  92 + json_data = self.get_json_data(formated_url, page)
92 93 page = page + 1
93 94  
94   - if len(json_data_mr) == 0:
  95 + if len(json_data) == 0:
95 96 break
96 97  
97   - for element in json_data_mr:
98   - single_merge_request = GitlabMergeRequest()
99   - self.fill_object_data(element, single_merge_request)
100   - all_merge_request.append(single_merge_request)
101   -
102   - return all_merge_request
  98 + for element in json_data:
  99 + single_comment = GitlabComment()
  100 + self.fill_object_data(element, single_comment)
  101 + single_comment.project = parent_obj.project
  102 + single_comment.issue_comment = issue_comment
  103 + single_comment.parent_id = parent_obj.id
  104 + all_comments.append(single_comment)
103 105  
104   - def fetch_issue(self, projects):
105   - all_issues = []
  106 + return all_comments
106 107  
107   - for project in projects:
108   - page = 1
109   - while True:
110   - url = '/api/v3/projects/{}/issues'.format(project.id)
111   - json_data_issue = self.get_json_data(url, page)
112   - page = page + 1
  108 + def fetch_projects(self):
  109 + return self.fetch('/api/v3/projects/all', GitlabProject)
113 110  
114   - if len(json_data_issue) == 0:
115   - break
  111 + def fetch_groups(self):
  112 + return self.fetch('/api/v3/groups', GitlabGroup)
116 113  
117   - for element in json_data_issue:
118   - single_issue = GitlabIssue()
119   - self.fill_object_data(element, single_issue)
120   - all_issues.append(single_issue)
  114 + def fetch_merge_request(self, projects):
  115 + merge_requests = []
  116 + for project in projects:
  117 + url = '/api/v3/projects/{}/merge_requests'.format(project.id)
  118 + merge_requests.extend(self.fetch(url, GitlabMergeRequest))
  119 + return merge_requests
121 120  
122   - return all_issues
  121 + def fetch_issue(self, projects):
  122 + issues = []
  123 + for project in projects:
  124 + url = '/api/v3/projects/{}/issues'.format(project.id)
  125 + issues.extend(self.fetch(url, GitlabIssue))
  126 + return issues
123 127  
124   - def fetch_comments(self):
  128 + def fetch_all_comments(self):
125 129 all_comments = []
126   - all_comments.extend(self.fetch_comments_MR())
  130 + all_comments.extend(self.fetch_comments_mr())
127 131 all_comments.extend(self.fetch_comments_issues())
128 132  
129 133 return all_comments
130 134  
131   - def fetch_comments_MR(self):
132   - all_comments = []
133   - all_merge_requests = GitlabMergeRequest.objects.all()
134   -
135   - for merge_request in all_merge_requests:
136   - page = 1
137   - while True:
138   - url = '/api/v3/projects/{}/merge_requests/{}/notes'.format(
139   - merge_request.project_id, merge_request.id)
140   - json_data_mr = self.get_json_data(url, page)
141   - page = page + 1
142   -
143   - if len(json_data_mr) == 0:
144   - break
145   -
146   - for element in json_data_mr:
147   - single_comment = GitlabComment()
148   - self.fill_object_data(element, single_comment)
149   - single_comment.project = merge_request.project
150   - single_comment.issue_comment = False
151   - single_comment.parent_id = merge_request.id
152   - all_comments.append(single_comment)
153   -
154   - return all_comments
  135 + def fetch_comments_mr(self):
  136 + url = '/api/v3/projects/{}/merge_requests/{}/notes'
  137 + return self.fetch_comments(url, GitlabMergeRequest, False)
155 138  
156 139 def fetch_comments_issues(self):
157   - all_comments = []
158   - all_issues = GitlabIssue.objects.all()
159   -
160   - for issue in all_issues:
161   - page = 1
162   - while True:
163   - url = '/api/v3/projects/{}/issues/{}/notes'.format(
164   - issue.project_id, issue.id)
165   - json_data_mr = self.get_json_data(url, page)
166   - page = page + 1
167   -
168   - if len(json_data_mr) == 0:
169   - break
170   -
171   - for element in json_data_mr:
172   - single_comment = GitlabComment()
173   - self.fill_object_data(element, single_comment)
174   - single_comment.project = issue.project
175   - single_comment.issue_comment = True
176   - single_comment.parent_id = issue.id
177   - all_comments.append(single_comment)
178   -
179   - return all_comments
  140 + url = '/api/v3/projects/{}/issues/{}/notes'
  141 + return self.fetch_comments(url, GitlabIssue, True)
180 142  
181 143  
182 144 class GitlabProjectImporter(GitlabDataImporter):
... ... @@ -212,6 +174,6 @@ class GitlabCommentImporter(GitlabDataImporter):
212 174  
213 175 def fetch_data(self):
214 176 LOGGER.info("Importing Comments")
215   - comments_list = self.fetch_comments()
  177 + comments_list = self.fetch_all_comments()
216 178 for datum in comments_list:
217 179 datum.save()
... ...