From 6dffc007011e8de85b4540260ba9441cca42962b Mon Sep 17 00:00:00 2001 From: Gust Date: Thu, 5 Feb 2015 09:57:11 -0200 Subject: [PATCH] Add fetch comment --- colab/proxy/gitlab/data_api.py | 208 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------------- colab/proxy/gitlab/models.py | 2 -- 2 files changed, 156 insertions(+), 54 deletions(-) diff --git a/colab/proxy/gitlab/data_api.py b/colab/proxy/gitlab/data_api.py index ac88184..2c81b7e 100644 --- a/colab/proxy/gitlab/data_api.py +++ b/colab/proxy/gitlab/data_api.py @@ -7,7 +7,7 @@ from dateutil.parser import parse from django.conf import settings from django.db.models.fields import DateTimeField -from colab.proxy.gitlab.models import GitlabProject, GitlabMergeRequest +from colab.proxy.gitlab.models import GitlabProject, GitlabMergeRequest, GitlabComment, GitlabIssue from colab.proxy.utils.proxy_data_api import ProxyDataAPI @@ -57,57 +57,57 @@ class GitlabDataAPI(ProxyDataAPI): return projects -# def fetch_merge_request(self, projects): -# page = 1 -# all_merge_request = [] -# -# # Iterate under all projects -# for project in projects: -# # Iterate under all MR inside project -# while(True): -# merge_request_url = \ -# '/api/v3/projects/{}/merge_requests'.format(project.id) -# url = self.get_request_url(merge_request_url, -# per_page=1000, -# page=page) -# -# data = urllib2.urlopen(url) -# json_data_mr = json.load(data) -# -# if len(json_data_mr) == 0: -# break -# -# page = page + 1 -# for element in json_data_mr: -# single_merge_request = GitlabMergeRequest() -# -# for field in GitlabMergeRequest._meta.fields: -# if field.name == "user": -# single_merge_request.update_user( -# element["author"]["username"]) -# continue -# if field.name == "project": -# single_merge_request.project_id = \ -# element["project_id"] -# continue -# -# if isinstance(field, DateTimeField): -# value = parse(element[field.name]) -# else: -# value = element[field.name] -# -# setattr(single_merge_request, field.name, value) -# -# all_merge_request.append(single_merge_request) -# -# return all_merge_request + def fetch_merge_request(self, projects): + all_merge_request = [] + + # Iterate under all projects + for project in projects: + page = 1 + # Iterate under all MR inside project + while(True): + merge_request_url = \ + '/api/v3/projects/{}/merge_requests'.format(project.id) + url = self.get_request_url(merge_request_url, + per_page=1000, + page=page) + + data = urllib2.urlopen(url) + json_data_mr = json.load(data) + + if len(json_data_mr) == 0: + break + + page = page + 1 + for element in json_data_mr: + single_merge_request = GitlabMergeRequest() + + for field in GitlabMergeRequest._meta.fields: + if field.name == "user": + single_merge_request.update_user( + element["author"]["username"]) + continue + if field.name == "project": + single_merge_request.project_id = \ + element["project_id"] + continue + + if isinstance(field, DateTimeField): + value = parse(element[field.name]) + else: + value = element[field.name] + + setattr(single_merge_request, field.name, value) + + all_merge_request.append(single_merge_request) + + return all_merge_request def fetch_issue(self, projects): - page = 1 all_issues = [] # Iterate under all projects for project in projects: + page = 1 # Iterate under all Issues inside project while(True): issue_url = \ @@ -129,6 +129,10 @@ class GitlabDataAPI(ProxyDataAPI): if field.name == "project": single_issue.project_id = element["project_id"] continue + if field.name == "user": + single_issue.update_user( + element["author"]["username"]) + continue if isinstance(field, DateTimeField): value = parse(element[field.name]) @@ -139,20 +143,120 @@ class GitlabDataAPI(ProxyDataAPI): all_issues.append(single_issue) - return all_issues + return all_issues - def fetch_data(self): - data = self.fetch_projects() - #merge_request_list = self.fetch_merge_request(data) - issue_list = self.fetch_issue(data) + def fetch_comments(self): + all_comments = [] + + all_comments.extend(self.fetch_comments_MR()) + all_comments.extend(self.fetch_comments_issues()) + + return all_comments + + def fetch_comments_MR(self): + all_comments = [] + all_merge_requests = GitlabMergeRequest.objects.all() + + for merge_request in all_merge_requests: + page = 1 + # Iterate under all comments of MR inside project + while(True): + merge_request_url = \ + '/api/v3/projects/{}/merge_requests/{}/notes' \ + .format(merge_request.project_id, merge_request.id) + url = self.get_request_url(merge_request_url, + per_page=1000, + page=page) + + data = urllib2.urlopen(url) + json_data_mr = json.load(data) + + if len(json_data_mr) == 0: + break + + page = page + 1 - for datum in data: + for element in json_data_mr: + single_comment = GitlabComment() + + for field in GitlabComment._meta.fields: + if field.name == "user": + single_comment.update_user( + element["author"]["username"]) + continue + + if isinstance(field, DateTimeField): + value = parse(element[field.name]) + else: + value = element[field.name] + + setattr(single_comment, field.name, value) + + all_comments.append(single_comment) + + return all_comments + + + def fetch_comments_issues(self): + all_comments = [] + all_issues = GitlabIssue.objects.all() + + for issue in all_issues: + page = 1 + # Iterate under all comments of MR inside project + while(True): + issue_comments_request_url = \ + '/api/v3/projects/{}/issues/{}/notes' \ + .format(issue.project_id, issue.id) + url = self.get_request_url(issue_comments_request_url, + per_page=1000, + page=page) + + data = urllib2.urlopen(url) + json_data_mr = json.load(data) + + if len(json_data_mr) == 0: + break + + page = page + 1 + for element in json_data_mr: + single_comment = GitlabComment() + + for field in GitlabComment._meta.fields: + if field.name == "user": + single_comment.update_user( + element["author"]["username"]) + continue + + if isinstance(field, DateTimeField): + value = parse(element[field.name]) + else: + value = element[field.name] + + setattr(single_comment, field.name, value) + + all_comments.append(single_comment) + + return all_comments + + def fetch_data(self): + projects = self.fetch_projects() + for datum in projects: datum.save() + +# merge_request_list = self.fetch_merge_request(projects) # for datum in merge_request_list: # datum.save() + + issue_list = self.fetch_issue(projects) for datum in issue_list: datum.save() + comments_list = self.fetch_comments() + for datum in comments_list: + datum.save() + + @property def app_label(self): return 'gitlab' diff --git a/colab/proxy/gitlab/models.py b/colab/proxy/gitlab/models.py index 5e9beca..162d0c9 100644 --- a/colab/proxy/gitlab/models.py +++ b/colab/proxy/gitlab/models.py @@ -53,8 +53,6 @@ class GitlabComment(Collaboration): id = models.IntegerField(primary_key=True) body = models.TextField() - author_email = models.EmailField(null=True, blank=True) - author_name = models.TextField() created_at = models.DateTimeField(blank=True) class Meta: -- libgit2 0.21.2