Download as
Browse Code »

Commit a2cad50a7c5fd9fb0c9acde0621d73ebe3038bf1

Authored by Matheus Fernandes
Committed by Lucas Kanashiro
1 parent fb0a796d
Exists in master and in 27 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, colab-vcard, colab_search, colab_tag, colab_tag_merge, detach_super_archives, fix-dashboard, fix-message-preview, fix_header, fixed_gitlab_import, go_to_profile_panel, mobile_user_scalable, paginate-threads, profile_integration, remove-gitlab-plugin, removing_namespace, search_block, search_filters, split_screen, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd

Refactored data_importer 

Signed-off-by: Matheus Fernandes <matheus.souza.fernandes@gmail.com>
Signed-off-by: Lucas Moura <lucas.moura128@gmail.com>
Showing 1 changed file with 47 additions and 85 deletions   Show diff stats
colab/plugins/gitlab/data_importer.py
  Diff comments   View file @a2cad50
@@ -9,7 +9,7 @@ from django.db.models.fields import DateTimeField @@ -9,7 +9,7 @@ from django.db.models.fields import DateTimeField
9 from colab.plugins.data import PluginDataImporter 9 from colab.plugins.data import PluginDataImporter
10 10
11 from .models import (GitlabProject, GitlabMergeRequest, 11 from .models import (GitlabProject, GitlabMergeRequest,
12 - GitlabComment, GitlabIssue) 12 + GitlabComment, GitlabIssue, GitlabGroup)
13 13
14 14
15 LOGGER = logging.getLogger('colab.plugin.gitlab') 15 LOGGER = logging.getLogger('colab.plugin.gitlab')
@@ -63,120 +63,82 @@ class GitlabDataImporter(PluginDataImporter): @@ -63,120 +63,82 @@ class GitlabDataImporter(PluginDataImporter):
63 63
64 return _object 64 return _object
65 65
66 - def fetch_projects(self): 66 + def fetch(self, url, gitlab_class):
67 page = 1 67 page = 1
68 - projects = [] 68 + obj_list = []
69 69
70 while True: 70 while True:
71 - json_data = self.get_json_data('/api/v3/projects/all', page) 71 + json_data = self.get_json_data(url, page)
72 page = page + 1 72 page = page + 1
73 73
74 if not len(json_data): 74 if not len(json_data):
75 break 75 break
76 76
77 for element in json_data: 77 for element in json_data:
78 - project = GitlabProject()  
79 - self.fill_object_data(element, project)  
80 - projects.append(project) 78 + obj = gitlab_class()
  79 + self.fill_object_data(element, obj)
  80 + obj_list.append(obj)
81 81
82 - return projects 82 + return obj_list
83 83
84 - def fetch_merge_request(self, projects):  
85 - all_merge_request = [] 84 + def fetch_comments(self, url, parent_class, issue_comment):
  85 + all_comments = []
  86 + all_parent_objects = parent_class.objects.all()
86 87
87 - for project in projects: 88 + for parent_obj in all_parent_objects:
88 page = 1 89 page = 1
89 while True: 90 while True:
90 - url = '/api/v3/projects/{}/merge_requests'.format(project.id)  
91 - json_data_mr = self.get_json_data(url, page) 91 + formated_url = url.format(parent_obj.project_id, parent_obj.id)
  92 + json_data = self.get_json_data(formated_url, page)
92 page = page + 1 93 page = page + 1
93 94
94 - if len(json_data_mr) == 0: 95 + if len(json_data) == 0:
95 break 96 break
96 97
97 - for element in json_data_mr:  
98 - single_merge_request = GitlabMergeRequest()  
99 - self.fill_object_data(element, single_merge_request)  
100 - all_merge_request.append(single_merge_request)  
101 -  
102 - return all_merge_request 98 + for element in json_data:
  99 + single_comment = GitlabComment()
  100 + self.fill_object_data(element, single_comment)
  101 + single_comment.project = parent_obj.project
  102 + single_comment.issue_comment = issue_comment
  103 + single_comment.parent_id = parent_obj.id
  104 + all_comments.append(single_comment)
103 105
104 - def fetch_issue(self, projects):  
105 - all_issues = [] 106 + return all_comments
106 107
107 - for project in projects:  
108 - page = 1  
109 - while True:  
110 - url = '/api/v3/projects/{}/issues'.format(project.id)  
111 - json_data_issue = self.get_json_data(url, page)  
112 - page = page + 1 108 + def fetch_projects(self):
  109 + return self.fetch('/api/v3/projects/all', GitlabProject)
113 110
114 - if len(json_data_issue) == 0:  
115 - break 111 + def fetch_groups(self):
  112 + return self.fetch('/api/v3/groups', GitlabGroup)
116 113
117 - for element in json_data_issue:  
118 - single_issue = GitlabIssue()  
119 - self.fill_object_data(element, single_issue)  
120 - all_issues.append(single_issue) 114 + def fetch_merge_request(self, projects):
  115 + merge_requests = []
  116 + for project in projects:
  117 + url = '/api/v3/projects/{}/merge_requests'.format(project.id)
  118 + merge_requests.extend(self.fetch(url, GitlabMergeRequest))
  119 + return merge_requests
121 120
122 - return all_issues 121 + def fetch_issue(self, projects):
  122 + issues = []
  123 + for project in projects:
  124 + url = '/api/v3/projects/{}/issues'.format(project.id)
  125 + issues.extend(self.fetch(url, GitlabIssue))
  126 + return issues
123 127
124 - def fetch_comments(self): 128 + def fetch_all_comments(self):
125 all_comments = [] 129 all_comments = []
126 - all_comments.extend(self.fetch_comments_MR()) 130 + all_comments.extend(self.fetch_comments_mr())
127 all_comments.extend(self.fetch_comments_issues()) 131 all_comments.extend(self.fetch_comments_issues())
128 132
129 return all_comments 133 return all_comments
130 134
131 - def fetch_comments_MR(self):  
132 - all_comments = []  
133 - all_merge_requests = GitlabMergeRequest.objects.all()  
134 -  
135 - for merge_request in all_merge_requests:  
136 - page = 1  
137 - while True:  
138 - url = '/api/v3/projects/{}/merge_requests/{}/notes'.format(  
139 - merge_request.project_id, merge_request.id)  
140 - json_data_mr = self.get_json_data(url, page)  
141 - page = page + 1  
142 -  
143 - if len(json_data_mr) == 0:  
144 - break  
145 -  
146 - for element in json_data_mr:  
147 - single_comment = GitlabComment()  
148 - self.fill_object_data(element, single_comment)  
149 - single_comment.project = merge_request.project  
150 - single_comment.issue_comment = False  
151 - single_comment.parent_id = merge_request.id  
152 - all_comments.append(single_comment)  
153 -  
154 - return all_comments 135 + def fetch_comments_mr(self):
  136 + url = '/api/v3/projects/{}/merge_requests/{}/notes'
  137 + return self.fetch_comments(url, GitlabMergeRequest, False)
155 138
156 def fetch_comments_issues(self): 139 def fetch_comments_issues(self):
157 - all_comments = []  
158 - all_issues = GitlabIssue.objects.all()  
159 -  
160 - for issue in all_issues:  
161 - page = 1  
162 - while True:  
163 - url = '/api/v3/projects/{}/issues/{}/notes'.format(  
164 - issue.project_id, issue.id)  
165 - json_data_mr = self.get_json_data(url, page)  
166 - page = page + 1  
167 -  
168 - if len(json_data_mr) == 0:  
169 - break  
170 -  
171 - for element in json_data_mr:  
172 - single_comment = GitlabComment()  
173 - self.fill_object_data(element, single_comment)  
174 - single_comment.project = issue.project  
175 - single_comment.issue_comment = True  
176 - single_comment.parent_id = issue.id  
177 - all_comments.append(single_comment)  
178 -  
179 - return all_comments 140 + url = '/api/v3/projects/{}/issues/{}/notes'
  141 + return self.fetch_comments(url, GitlabIssue, True)
180 142
181 143
182 class GitlabProjectImporter(GitlabDataImporter): 144 class GitlabProjectImporter(GitlabDataImporter):
@@ -212,6 +174,6 @@ class GitlabCommentImporter(GitlabDataImporter): @@ -212,6 +174,6 @@ class GitlabCommentImporter(GitlabDataImporter):
212 174
213 def fetch_data(self): 175 def fetch_data(self):
214 LOGGER.info("Importing Comments") 176 LOGGER.info("Importing Comments")
215 - comments_list = self.fetch_comments() 177 + comments_list = self.fetch_all_comments()
216 for datum in comments_list: 178 for datum in comments_list:
217 datum.save() 179 datum.save()