Download as
Browse Code »

Commit 8825336b337a2186f5ef61c64a774b7ddff7ce9c

Authored by Gust
Committed by Sergio Oliveira
1 parent 15e644b6
Exists in master and in 39 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, check-port, colab-signals, colab-vcard, colab_search, colab_tag, colab_tag_merge, community_association, detach_super_archives, fix-dashboard, fix-message-preview, fix_app_label, fix_cache_count_spb, fix_header, fixed_gitlab_import, go_to_profile_panel, header_footer, layout-fix, mobile_user_scalable, move_out_plugins, paginate-threads, profile_integration, refactor-data-import, remove-gitlab-plugin, removing_namespace, search_block, search_filters, spb-release/3.0, split_screen, stable, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd, workin_whoosh_temp

Refactory 

Sign-off-by: Rodrigo Siqueira <rodrigosiqueiramelo@gmail.com>
Showing 1 changed file with 41 additions and 78 deletions   Show diff stats
colab/proxy/gitlab/data_api.py
  Diff comments   View file @8825336
... ... @@ -25,34 +25,50 @@ class GitlabDataAPI(ProxyDataAPI):
25 25  
26 26 return u'{}{}?{}'.format(upstream, path, params)
27 27  
  28 + def get_json_data(self, api_url, page=1, pages=1000):
  29 + url = self.get_request_url(api_url, per_page=pages,
  30 + page=page)
  31 + data = urllib2.urlopen(url)
  32 + json_data = json.load(data)
  33 +
  34 + return json_data
  35 +
  36 + def fill_object_data(self, element, _object):
  37 + for field in _object._meta.fields:
  38 + if field.name == "user":
  39 + _object.update_user(
  40 + element["author"]["username"])
  41 + continue
  42 + if field.name == "project":
  43 + _object.project_id = \
  44 + element["project_id"]
  45 + continue
  46 +
  47 + if isinstance(field, DateTimeField):
  48 + value = parse(element["created_at"])
  49 + else:
  50 + value = element[field.name]
  51 +
  52 + setattr(_object, field.name, value)
  53 +
  54 + return _object
  55 +
28 56 def fetch_projects(self):
29 57 page = 1
30 58 projects = []
31 59  
32 60 # Iterates throughout all projects pages
33   - while(True):
34   - url = self.get_request_url('/api/v3/projects/all',
35   - per_page=1000,
36   - page=page)
37   - data = urllib2.urlopen(url)
38   - json_data = json.load(data)
39   -
40   - if len(json_data) == 0:
  61 + while True:
  62 + json_data = self.get_json_data('/api/v3/projects/all', page)
  63 +
  64 + if not len(json_data):
41 65 break
42 66  
43 67 page = page + 1
44 68  
45 69 for element in json_data:
46 70 project = GitlabProject()
47   -
48   - for field in GitlabProject._meta.fields:
49   - if isinstance(field, DateTimeField):
50   - value = parse(element[field.name])
51   - else:
52   - value = element[field.name]
53   -
54   - setattr(project, field.name, value)
55   -
  71 + self.fill_object_data(element, project)
56 72 projects.append(project)
57 73  
58 74 return projects
... ... @@ -66,12 +82,7 @@ class GitlabDataAPI(ProxyDataAPI):
66 82 while(True):
67 83 merge_request_url = \
68 84 '/api/v3/projects/{}/merge_requests'.format(project.id)
69   - url = self.get_request_url(merge_request_url,
70   - per_page=1000,
71   - page=page)
72   -
73   - data = urllib2.urlopen(url)
74   - json_data_mr = json.load(data)
  85 + json_data_mr = self.get_json_data(merge_request_url, page)
75 86  
76 87 if len(json_data_mr) == 0:
77 88 break
... ... @@ -79,24 +90,7 @@ class GitlabDataAPI(ProxyDataAPI):
79 90 page = page + 1
80 91 for element in json_data_mr:
81 92 single_merge_request = GitlabMergeRequest()
82   -
83   - for field in GitlabMergeRequest._meta.fields:
84   - if field.name == "user":
85   - single_merge_request.update_user(
86   - element["author"]["username"])
87   - continue
88   - if field.name == "project":
89   - single_merge_request.project_id = \
90   - element["project_id"]
91   - continue
92   -
93   - if isinstance(field, DateTimeField):
94   - value = parse(element["created_at"])
95   - else:
96   - value = element[field.name]
97   -
98   - setattr(single_merge_request, field.name, value)
99   -
  93 + self.fill_object_data(element, single_merge_request)
100 94 all_merge_request.append(single_merge_request)
101 95  
102 96 return all_merge_request
... ... @@ -111,11 +105,8 @@ class GitlabDataAPI(ProxyDataAPI):
111 105 while(True):
112 106 issue_url = \
113 107 '/api/v3/projects/{}/issues'.format(project.id)
114   - url = self.get_request_url(issue_url, per_page=1000,
115   - page=page)
116 108  
117   - data = urllib2.urlopen(url)
118   - json_data_issue = json.load(data)
  109 + json_data_issue = self.get_json_data(issue_url, page)
119 110  
120 111 if len(json_data_issue) == 0:
121 112 break
... ... @@ -123,30 +114,13 @@ class GitlabDataAPI(ProxyDataAPI):
123 114 page = page + 1
124 115 for element in json_data_issue:
125 116 single_issue = GitlabIssue()
126   -
127   - for field in GitlabIssue._meta.fields:
128   - if field.name == "project":
129   - single_issue.project_id = element["project_id"]
130   - continue
131   - if field.name == "user":
132   - single_issue.update_user(
133   - element["author"]["username"])
134   - continue
135   -
136   - if isinstance(field, DateTimeField):
137   - value = parse(element["created_at"])
138   - else:
139   - value = element[field.name]
140   -
141   - setattr(single_issue, field.name, value)
142   -
  117 + self.fill_object_data(element, single_issue)
143 118 all_issues.append(single_issue)
144 119  
145 120 return all_issues
146 121  
147 122 def fetch_comments(self):
148 123 all_comments = []
149   -
150 124 all_comments.extend(self.fetch_comments_MR())
151 125 all_comments.extend(self.fetch_comments_issues())
152 126  
... ... @@ -160,15 +134,9 @@ class GitlabDataAPI(ProxyDataAPI):
160 134 page = 1
161 135 # Iterate under all comments of MR inside project
162 136 while(True):
163   - merge_request_url = \
164   - '/api/v3/projects/{}/merge_requests/{}/notes' \
165   - .format(merge_request.project_id, merge_request.id)
166   - url = self.get_request_url(merge_request_url,
167   - per_page=1000,
168   - page=page)
169   -
170   - data = urllib2.urlopen(url)
171   - json_data_mr = json.load(data)
  137 + mr_url = '/api/v3/projects/{}/merge_requests/{}/notes'.format(
  138 + merge_request.project_id, merge_request.id)
  139 + json_data_mr = self.get_json_data(mr_url, page)
172 140  
173 141 if len(json_data_mr) == 0:
174 142 break
... ... @@ -217,12 +185,7 @@ class GitlabDataAPI(ProxyDataAPI):
217 185 issue_comments_request_url = \
218 186 '/api/v3/projects/{}/issues/{}/notes' \
219 187 .format(issue.project_id, issue.id)
220   - url = self.get_request_url(issue_comments_request_url,
221   - per_page=1000,
222   - page=page)
223   -
224   - data = urllib2.urlopen(url)
225   - json_data_mr = json.load(data)
  188 + json_data_mr = self.get_json_data(issue_comments_request_url, page)
226 189  
227 190 if len(json_data_mr) == 0:
228 191 break
... ...