Download as
Browse Code »

Commit 6dffc007011e8de85b4540260ba9441cca42962b

Authored by Gust
Committed by Sergio Oliveira
1 parent 7feb4c10
Exists in master and in 39 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, check-port, colab-signals, colab-vcard, colab_search, colab_tag, colab_tag_merge, community_association, detach_super_archives, fix-dashboard, fix-message-preview, fix_app_label, fix_cache_count_spb, fix_header, fixed_gitlab_import, go_to_profile_panel, header_footer, layout-fix, mobile_user_scalable, move_out_plugins, paginate-threads, profile_integration, refactor-data-import, remove-gitlab-plugin, removing_namespace, search_block, search_filters, spb-release/3.0, split_screen, stable, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd, workin_whoosh_temp

Add fetch comment

Showing 2 changed files with 156 additions and 54 deletions   Show diff stats
colab/proxy/gitlab/data_api.py
  Diff comments   View file @6dffc00
... ... @@ -7,7 +7,7 @@ from dateutil.parser import parse
7 7 from django.conf import settings
8 8 from django.db.models.fields import DateTimeField
9 9  
10   -from colab.proxy.gitlab.models import GitlabProject, GitlabMergeRequest
  10 +from colab.proxy.gitlab.models import GitlabProject, GitlabMergeRequest, GitlabComment, GitlabIssue
11 11 from colab.proxy.utils.proxy_data_api import ProxyDataAPI
12 12  
13 13  
... ... @@ -57,57 +57,57 @@ class GitlabDataAPI(ProxyDataAPI):
57 57  
58 58 return projects
59 59  
60   -# def fetch_merge_request(self, projects):
61   -# page = 1
62   -# all_merge_request = []
63   -#
64   -# # Iterate under all projects
65   -# for project in projects:
66   -# # Iterate under all MR inside project
67   -# while(True):
68   -# merge_request_url = \
69   -# '/api/v3/projects/{}/merge_requests'.format(project.id)
70   -# url = self.get_request_url(merge_request_url,
71   -# per_page=1000,
72   -# page=page)
73   -#
74   -# data = urllib2.urlopen(url)
75   -# json_data_mr = json.load(data)
76   -#
77   -# if len(json_data_mr) == 0:
78   -# break
79   -#
80   -# page = page + 1
81   -# for element in json_data_mr:
82   -# single_merge_request = GitlabMergeRequest()
83   -#
84   -# for field in GitlabMergeRequest._meta.fields:
85   -# if field.name == "user":
86   -# single_merge_request.update_user(
87   -# element["author"]["username"])
88   -# continue
89   -# if field.name == "project":
90   -# single_merge_request.project_id = \
91   -# element["project_id"]
92   -# continue
93   -#
94   -# if isinstance(field, DateTimeField):
95   -# value = parse(element[field.name])
96   -# else:
97   -# value = element[field.name]
98   -#
99   -# setattr(single_merge_request, field.name, value)
100   -#
101   -# all_merge_request.append(single_merge_request)
102   -#
103   -# return all_merge_request
  60 + def fetch_merge_request(self, projects):
  61 + all_merge_request = []
  62 +
  63 + # Iterate under all projects
  64 + for project in projects:
  65 + page = 1
  66 + # Iterate under all MR inside project
  67 + while(True):
  68 + merge_request_url = \
  69 + '/api/v3/projects/{}/merge_requests'.format(project.id)
  70 + url = self.get_request_url(merge_request_url,
  71 + per_page=1000,
  72 + page=page)
  73 +
  74 + data = urllib2.urlopen(url)
  75 + json_data_mr = json.load(data)
  76 +
  77 + if len(json_data_mr) == 0:
  78 + break
  79 +
  80 + page = page + 1
  81 + for element in json_data_mr:
  82 + single_merge_request = GitlabMergeRequest()
  83 +
  84 + for field in GitlabMergeRequest._meta.fields:
  85 + if field.name == "user":
  86 + single_merge_request.update_user(
  87 + element["author"]["username"])
  88 + continue
  89 + if field.name == "project":
  90 + single_merge_request.project_id = \
  91 + element["project_id"]
  92 + continue
  93 +
  94 + if isinstance(field, DateTimeField):
  95 + value = parse(element[field.name])
  96 + else:
  97 + value = element[field.name]
  98 +
  99 + setattr(single_merge_request, field.name, value)
  100 +
  101 + all_merge_request.append(single_merge_request)
  102 +
  103 + return all_merge_request
104 104  
105 105 def fetch_issue(self, projects):
106   - page = 1
107 106 all_issues = []
108 107  
109 108 # Iterate under all projects
110 109 for project in projects:
  110 + page = 1
111 111 # Iterate under all Issues inside project
112 112 while(True):
113 113 issue_url = \
... ... @@ -129,6 +129,10 @@ class GitlabDataAPI(ProxyDataAPI):
129 129 if field.name == "project":
130 130 single_issue.project_id = element["project_id"]
131 131 continue
  132 + if field.name == "user":
  133 + single_issue.update_user(
  134 + element["author"]["username"])
  135 + continue
132 136  
133 137 if isinstance(field, DateTimeField):
134 138 value = parse(element[field.name])
... ... @@ -139,20 +143,120 @@ class GitlabDataAPI(ProxyDataAPI):
139 143  
140 144 all_issues.append(single_issue)
141 145  
142   - return all_issues
  146 + return all_issues
143 147  
144   - def fetch_data(self):
145   - data = self.fetch_projects()
146   - #merge_request_list = self.fetch_merge_request(data)
147   - issue_list = self.fetch_issue(data)
  148 + def fetch_comments(self):
  149 + all_comments = []
  150 +
  151 + all_comments.extend(self.fetch_comments_MR())
  152 + all_comments.extend(self.fetch_comments_issues())
  153 +
  154 + return all_comments
  155 +
  156 + def fetch_comments_MR(self):
  157 + all_comments = []
  158 + all_merge_requests = GitlabMergeRequest.objects.all()
  159 +
  160 + for merge_request in all_merge_requests:
  161 + page = 1
  162 + # Iterate under all comments of MR inside project
  163 + while(True):
  164 + merge_request_url = \
  165 + '/api/v3/projects/{}/merge_requests/{}/notes' \
  166 + .format(merge_request.project_id, merge_request.id)
  167 + url = self.get_request_url(merge_request_url,
  168 + per_page=1000,
  169 + page=page)
  170 +
  171 + data = urllib2.urlopen(url)
  172 + json_data_mr = json.load(data)
  173 +
  174 + if len(json_data_mr) == 0:
  175 + break
  176 +
  177 + page = page + 1
148 178  
149   - for datum in data:
  179 + for element in json_data_mr:
  180 + single_comment = GitlabComment()
  181 +
  182 + for field in GitlabComment._meta.fields:
  183 + if field.name == "user":
  184 + single_comment.update_user(
  185 + element["author"]["username"])
  186 + continue
  187 +
  188 + if isinstance(field, DateTimeField):
  189 + value = parse(element[field.name])
  190 + else:
  191 + value = element[field.name]
  192 +
  193 + setattr(single_comment, field.name, value)
  194 +
  195 + all_comments.append(single_comment)
  196 +
  197 + return all_comments
  198 +
  199 +
  200 + def fetch_comments_issues(self):
  201 + all_comments = []
  202 + all_issues = GitlabIssue.objects.all()
  203 +
  204 + for issue in all_issues:
  205 + page = 1
  206 + # Iterate under all comments of MR inside project
  207 + while(True):
  208 + issue_comments_request_url = \
  209 + '/api/v3/projects/{}/issues/{}/notes' \
  210 + .format(issue.project_id, issue.id)
  211 + url = self.get_request_url(issue_comments_request_url,
  212 + per_page=1000,
  213 + page=page)
  214 +
  215 + data = urllib2.urlopen(url)
  216 + json_data_mr = json.load(data)
  217 +
  218 + if len(json_data_mr) == 0:
  219 + break
  220 +
  221 + page = page + 1
  222 + for element in json_data_mr:
  223 + single_comment = GitlabComment()
  224 +
  225 + for field in GitlabComment._meta.fields:
  226 + if field.name == "user":
  227 + single_comment.update_user(
  228 + element["author"]["username"])
  229 + continue
  230 +
  231 + if isinstance(field, DateTimeField):
  232 + value = parse(element[field.name])
  233 + else:
  234 + value = element[field.name]
  235 +
  236 + setattr(single_comment, field.name, value)
  237 +
  238 + all_comments.append(single_comment)
  239 +
  240 + return all_comments
  241 +
  242 + def fetch_data(self):
  243 + projects = self.fetch_projects()
  244 + for datum in projects:
150 245 datum.save()
  246 +
  247 +# merge_request_list = self.fetch_merge_request(projects)
151 248 # for datum in merge_request_list:
152 249 # datum.save()
  250 +
  251 + issue_list = self.fetch_issue(projects)
153 252 for datum in issue_list:
154 253 datum.save()
155 254  
  255 + comments_list = self.fetch_comments()
  256 + for datum in comments_list:
  257 + datum.save()
  258 +
  259 +
156 260 @property
157 261 def app_label(self):
158 262 return 'gitlab'
... ...
colab/proxy/gitlab/models.py
  Diff comments   View file @6dffc00
... ... @@ -53,8 +53,6 @@ class GitlabComment(Collaboration):
53 53  
54 54 id = models.IntegerField(primary_key=True)
55 55 body = models.TextField()
56   - author_email = models.EmailField(null=True, blank=True)
57   - author_name = models.TextField()
58 56 created_at = models.DateTimeField(blank=True)
59 57  
60 58 class Meta:
... ...