Download as
Browse Code »

Commit 8825336b337a2186f5ef61c64a774b7ddff7ce9c

Authored by Gust
Committed by Sergio Oliveira
1 parent 15e644b6
Exists in master and in 39 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, check-port, colab-signals, colab-vcard, colab_search, colab_tag, colab_tag_merge, community_association, detach_super_archives, fix-dashboard, fix-message-preview, fix_app_label, fix_cache_count_spb, fix_header, fixed_gitlab_import, go_to_profile_panel, header_footer, layout-fix, mobile_user_scalable, move_out_plugins, paginate-threads, profile_integration, refactor-data-import, remove-gitlab-plugin, removing_namespace, search_block, search_filters, spb-release/3.0, split_screen, stable, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd, workin_whoosh_temp

Refactory 

Sign-off-by: Rodrigo Siqueira <rodrigosiqueiramelo@gmail.com>
Showing 1 changed file with 41 additions and 78 deletions   Show diff stats
colab/proxy/gitlab/data_api.py
  Diff comments   View file @8825336
@@ -25,34 +25,50 @@ class GitlabDataAPI(ProxyDataAPI): @@ -25,34 +25,50 @@ class GitlabDataAPI(ProxyDataAPI):
25 25
26 return u'{}{}?{}'.format(upstream, path, params) 26 return u'{}{}?{}'.format(upstream, path, params)
27 27
  28 + def get_json_data(self, api_url, page=1, pages=1000):
  29 + url = self.get_request_url(api_url, per_page=pages,
  30 + page=page)
  31 + data = urllib2.urlopen(url)
  32 + json_data = json.load(data)
  33 +
  34 + return json_data
  35 +
  36 + def fill_object_data(self, element, _object):
  37 + for field in _object._meta.fields:
  38 + if field.name == "user":
  39 + _object.update_user(
  40 + element["author"]["username"])
  41 + continue
  42 + if field.name == "project":
  43 + _object.project_id = \
  44 + element["project_id"]
  45 + continue
  46 +
  47 + if isinstance(field, DateTimeField):
  48 + value = parse(element["created_at"])
  49 + else:
  50 + value = element[field.name]
  51 +
  52 + setattr(_object, field.name, value)
  53 +
  54 + return _object
  55 +
28 def fetch_projects(self): 56 def fetch_projects(self):
29 page = 1 57 page = 1
30 projects = [] 58 projects = []
31 59
32 # Iterates throughout all projects pages 60 # Iterates throughout all projects pages
33 - while(True):  
34 - url = self.get_request_url('/api/v3/projects/all',  
35 - per_page=1000,  
36 - page=page)  
37 - data = urllib2.urlopen(url)  
38 - json_data = json.load(data)  
39 -  
40 - if len(json_data) == 0: 61 + while True:
  62 + json_data = self.get_json_data('/api/v3/projects/all', page)
  63 +
  64 + if not len(json_data):
41 break 65 break
42 66
43 page = page + 1 67 page = page + 1
44 68
45 for element in json_data: 69 for element in json_data:
46 project = GitlabProject() 70 project = GitlabProject()
47 -  
48 - for field in GitlabProject._meta.fields:  
49 - if isinstance(field, DateTimeField):  
50 - value = parse(element[field.name])  
51 - else:  
52 - value = element[field.name]  
53 -  
54 - setattr(project, field.name, value)  
55 - 71 + self.fill_object_data(element, project)
56 projects.append(project) 72 projects.append(project)
57 73
58 return projects 74 return projects
@@ -66,12 +82,7 @@ class GitlabDataAPI(ProxyDataAPI): @@ -66,12 +82,7 @@ class GitlabDataAPI(ProxyDataAPI):
66 while(True): 82 while(True):
67 merge_request_url = \ 83 merge_request_url = \
68 '/api/v3/projects/{}/merge_requests'.format(project.id) 84 '/api/v3/projects/{}/merge_requests'.format(project.id)
69 - url = self.get_request_url(merge_request_url,  
70 - per_page=1000,  
71 - page=page)  
72 -  
73 - data = urllib2.urlopen(url)  
74 - json_data_mr = json.load(data) 85 + json_data_mr = self.get_json_data(merge_request_url, page)
75 86
76 if len(json_data_mr) == 0: 87 if len(json_data_mr) == 0:
77 break 88 break
@@ -79,24 +90,7 @@ class GitlabDataAPI(ProxyDataAPI): @@ -79,24 +90,7 @@ class GitlabDataAPI(ProxyDataAPI):
79 page = page + 1 90 page = page + 1
80 for element in json_data_mr: 91 for element in json_data_mr:
81 single_merge_request = GitlabMergeRequest() 92 single_merge_request = GitlabMergeRequest()
82 -  
83 - for field in GitlabMergeRequest._meta.fields:  
84 - if field.name == "user":  
85 - single_merge_request.update_user(  
86 - element["author"]["username"])  
87 - continue  
88 - if field.name == "project":  
89 - single_merge_request.project_id = \  
90 - element["project_id"]  
91 - continue  
92 -  
93 - if isinstance(field, DateTimeField):  
94 - value = parse(element["created_at"])  
95 - else:  
96 - value = element[field.name]  
97 -  
98 - setattr(single_merge_request, field.name, value)  
99 - 93 + self.fill_object_data(element, single_merge_request)
100 all_merge_request.append(single_merge_request) 94 all_merge_request.append(single_merge_request)
101 95
102 return all_merge_request 96 return all_merge_request
@@ -111,11 +105,8 @@ class GitlabDataAPI(ProxyDataAPI): @@ -111,11 +105,8 @@ class GitlabDataAPI(ProxyDataAPI):
111 while(True): 105 while(True):
112 issue_url = \ 106 issue_url = \
113 '/api/v3/projects/{}/issues'.format(project.id) 107 '/api/v3/projects/{}/issues'.format(project.id)
114 - url = self.get_request_url(issue_url, per_page=1000,  
115 - page=page)  
116 108
117 - data = urllib2.urlopen(url)  
118 - json_data_issue = json.load(data) 109 + json_data_issue = self.get_json_data(issue_url, page)
119 110
120 if len(json_data_issue) == 0: 111 if len(json_data_issue) == 0:
121 break 112 break
@@ -123,30 +114,13 @@ class GitlabDataAPI(ProxyDataAPI): @@ -123,30 +114,13 @@ class GitlabDataAPI(ProxyDataAPI):
123 page = page + 1 114 page = page + 1
124 for element in json_data_issue: 115 for element in json_data_issue:
125 single_issue = GitlabIssue() 116 single_issue = GitlabIssue()
126 -  
127 - for field in GitlabIssue._meta.fields:  
128 - if field.name == "project":  
129 - single_issue.project_id = element["project_id"]  
130 - continue  
131 - if field.name == "user":  
132 - single_issue.update_user(  
133 - element["author"]["username"])  
134 - continue  
135 -  
136 - if isinstance(field, DateTimeField):  
137 - value = parse(element["created_at"])  
138 - else:  
139 - value = element[field.name]  
140 -  
141 - setattr(single_issue, field.name, value)  
142 - 117 + self.fill_object_data(element, single_issue)
143 all_issues.append(single_issue) 118 all_issues.append(single_issue)
144 119
145 return all_issues 120 return all_issues
146 121
147 def fetch_comments(self): 122 def fetch_comments(self):
148 all_comments = [] 123 all_comments = []
149 -  
150 all_comments.extend(self.fetch_comments_MR()) 124 all_comments.extend(self.fetch_comments_MR())
151 all_comments.extend(self.fetch_comments_issues()) 125 all_comments.extend(self.fetch_comments_issues())
152 126
@@ -160,15 +134,9 @@ class GitlabDataAPI(ProxyDataAPI): @@ -160,15 +134,9 @@ class GitlabDataAPI(ProxyDataAPI):
160 page = 1 134 page = 1
161 # Iterate under all comments of MR inside project 135 # Iterate under all comments of MR inside project
162 while(True): 136 while(True):
163 - merge_request_url = \  
164 - '/api/v3/projects/{}/merge_requests/{}/notes' \  
165 - .format(merge_request.project_id, merge_request.id)  
166 - url = self.get_request_url(merge_request_url,  
167 - per_page=1000,  
168 - page=page)  
169 -  
170 - data = urllib2.urlopen(url)  
171 - json_data_mr = json.load(data) 137 + mr_url = '/api/v3/projects/{}/merge_requests/{}/notes'.format(
  138 + merge_request.project_id, merge_request.id)
  139 + json_data_mr = self.get_json_data(mr_url, page)
172 140
173 if len(json_data_mr) == 0: 141 if len(json_data_mr) == 0:
174 break 142 break
@@ -217,12 +185,7 @@ class GitlabDataAPI(ProxyDataAPI): @@ -217,12 +185,7 @@ class GitlabDataAPI(ProxyDataAPI):
217 issue_comments_request_url = \ 185 issue_comments_request_url = \
218 '/api/v3/projects/{}/issues/{}/notes' \ 186 '/api/v3/projects/{}/issues/{}/notes' \
219 .format(issue.project_id, issue.id) 187 .format(issue.project_id, issue.id)
220 - url = self.get_request_url(issue_comments_request_url,  
221 - per_page=1000,  
222 - page=page)  
223 -  
224 - data = urllib2.urlopen(url)  
225 - json_data_mr = json.load(data) 188 + json_data_mr = self.get_json_data(issue_comments_request_url, page)
226 189
227 if len(json_data_mr) == 0: 190 if len(json_data_mr) == 0:
228 break 191 break