Add fetch comment

Gust · Sergio Oliveira
1 parent 7feb4c10
Showing 2 changed files with 156 additions and 54 deletions Show diff stats
colab/proxy/gitlab/data_api.py
colab/proxy/gitlab/models.py
@@ -7,7 +7,7 @@ from dateutil.parser import parse
 from django.conf import settings
 from django.db.models.fields import DateTimeField
-from colab.proxy.gitlab.models import GitlabProject, GitlabMergeRequest
+from colab.proxy.gitlab.models import GitlabProject, GitlabMergeRequest, GitlabComment, GitlabIssue
 from colab.proxy.utils.proxy_data_api import ProxyDataAPI
@@ -57,57 +57,57 @@ class GitlabDataAPI(ProxyDataAPI):
         return projects
-#    def fetch_merge_request(self, projects):
-#        page = 1
-#        all_merge_request = []
-#
-#        # Iterate under all projects
-#        for project in projects:
-#            # Iterate under all MR inside project
-#            while(True):
-#                merge_request_url = \
-#                    '/api/v3/projects/{}/merge_requests'.format(project.id)
-#                url = self.get_request_url(merge_request_url,
-#                                           per_page=1000,
-#                                           page=page)
-#
-#                data = urllib2.urlopen(url)
-#                json_data_mr = json.load(data)
-#
-#                if len(json_data_mr) == 0:
-#                    break
-#
-#                page = page + 1
-#                for element in json_data_mr:
-#                    single_merge_request = GitlabMergeRequest()
-#
-#                    for field in GitlabMergeRequest._meta.fields:
-#                        if field.name == "user":
-#                            single_merge_request.update_user(
-#                                element["author"]["username"])
-#                            continue
-#                        if field.name == "project":
-#                            single_merge_request.project_id = \
-#                                element["project_id"]
-#                            continue
-#
-#                        if isinstance(field, DateTimeField):
-#                            value = parse(element[field.name])
-#                        else:
-#                            value = element[field.name]
-#
-#                        setattr(single_merge_request, field.name, value)
-#
-#                    all_merge_request.append(single_merge_request)
-#
-#        return all_merge_request
+    def fetch_merge_request(self, projects):
+        all_merge_request = []
+
+        # Iterate under all projects
+        for project in projects:
+            page = 1
+            # Iterate under all MR inside project
+            while(True):
+                merge_request_url = \
+                    '/api/v3/projects/{}/merge_requests'.format(project.id)
+                url = self.get_request_url(merge_request_url,
+                                           per_page=1000,
+                                           page=page)
+
+                data = urllib2.urlopen(url)
+                json_data_mr = json.load(data)
+
+                if len(json_data_mr) == 0:
+                    break
+
+                page = page + 1
+                for element in json_data_mr:
+                    single_merge_request = GitlabMergeRequest()
+
+                    for field in GitlabMergeRequest._meta.fields:
+                        if field.name == "user":
+                            single_merge_request.update_user(
+                                element["author"]["username"])
+                            continue
+                        if field.name == "project":
+                            single_merge_request.project_id = \
+                                element["project_id"]
+                            continue
+
+                        if isinstance(field, DateTimeField):
+                            value = parse(element[field.name])
+                        else:
+                            value = element[field.name]
+
+                        setattr(single_merge_request, field.name, value)
+
+                    all_merge_request.append(single_merge_request)
+
+        return all_merge_request
     def fetch_issue(self, projects):
-        page = 1
         all_issues = []
         # Iterate under all projects
         for project in projects:
+            page = 1
             # Iterate under all Issues inside project
             while(True):
                 issue_url = \
@@ -129,6 +129,10 @@ class GitlabDataAPI(ProxyDataAPI):
                         if field.name == "project":
                             single_issue.project_id = element["project_id"]
                             continue
+                        if field.name == "user":
+                            single_issue.update_user(
+                                element["author"]["username"])
+                            continue
                         if isinstance(field, DateTimeField):
                             value = parse(element[field.name])
@@ -139,20 +143,120 @@ class GitlabDataAPI(ProxyDataAPI):
                     all_issues.append(single_issue)
-            return all_issues
+        return all_issues
-    def fetch_data(self):
-        data = self.fetch_projects()
-        #merge_request_list = self.fetch_merge_request(data)
-        issue_list = self.fetch_issue(data)
+    def fetch_comments(self):
+        all_comments = []
+
+        all_comments.extend(self.fetch_comments_MR())
+        all_comments.extend(self.fetch_comments_issues())
+
+        return all_comments
+
+    def fetch_comments_MR(self):
+        all_comments = []
+        all_merge_requests = GitlabMergeRequest.objects.all()
+
+        for merge_request in all_merge_requests:
+            page = 1
+            # Iterate under all comments of MR inside project
+            while(True):
+                merge_request_url = \
+                    '/api/v3/projects/{}/merge_requests/{}/notes' \
+                    .format(merge_request.project_id, merge_request.id)
+                url = self.get_request_url(merge_request_url,
+                                           per_page=1000,
+                                           page=page)
+
+                data = urllib2.urlopen(url)
+                json_data_mr = json.load(data)
+
+                if len(json_data_mr) == 0:
+                    break
+
+                page = page + 1
-        for datum in data:
+                for element in json_data_mr:
+                    single_comment = GitlabComment()
+
+                    for field in GitlabComment._meta.fields:
+                        if field.name == "user":
+                            single_comment.update_user(
+                                element["author"]["username"])
+                            continue
+
+                        if isinstance(field, DateTimeField):
+                            value = parse(element[field.name])
+                        else:
+                            value = element[field.name]
+
+                        setattr(single_comment, field.name, value)
+
+                    all_comments.append(single_comment)
+
+        return all_comments
+
+
+    def fetch_comments_issues(self):
+        all_comments = []
+        all_issues = GitlabIssue.objects.all()
+
+        for issue in all_issues:
+            page = 1
+            # Iterate under all comments of MR inside project
+            while(True):
+                issue_comments_request_url = \
+                    '/api/v3/projects/{}/issues/{}/notes' \
+                    .format(issue.project_id, issue.id)
+                url = self.get_request_url(issue_comments_request_url,
+                                           per_page=1000,
+                                           page=page)
+
+                data = urllib2.urlopen(url)
+                json_data_mr = json.load(data)
+
+                if len(json_data_mr) == 0:
+                    break
+
+                page = page + 1
+                for element in json_data_mr:
+                    single_comment = GitlabComment()
+
+                    for field in GitlabComment._meta.fields:
+                        if field.name == "user":
+                            single_comment.update_user(
+                                element["author"]["username"])
+                            continue
+
+                        if isinstance(field, DateTimeField):
+                            value = parse(element[field.name])
+                        else:
+                            value = element[field.name]
+
+                        setattr(single_comment, field.name, value)
+
+                    all_comments.append(single_comment)
+
+        return all_comments
+
+    def fetch_data(self):
+        projects = self.fetch_projects()
+        for datum in projects:
             datum.save()
+
+#        merge_request_list = self.fetch_merge_request(projects)
 #        for datum in merge_request_list:
 #            datum.save()
+
+        issue_list = self.fetch_issue(projects)
         for datum in issue_list:
             datum.save()
+        comments_list = self.fetch_comments()
+        for datum in comments_list:
+            datum.save()
+
+
     @property
     def app_label(self):
         return 'gitlab'
@@ -53,8 +53,6 @@ class GitlabComment(Collaboration):
     id = models.IntegerField(primary_key=True)
     body = models.TextField()
-    author_email = models.EmailField(null=True, blank=True)
-    author_name = models.TextField()
     created_at = models.DateTimeField(blank=True)
     class Meta:
	@@ -7,7 +7,7 @@ from dateutil.parser import parse		@@ -7,7 +7,7 @@ from dateutil.parser import parse
7	from django.conf import settings	7	from django.conf import settings
8	from django.db.models.fields import DateTimeField	8	from django.db.models.fields import DateTimeField
9		9
10	-from colab.proxy.gitlab.models import GitlabProject, GitlabMergeRequest	10	+from colab.proxy.gitlab.models import GitlabProject, GitlabMergeRequest, GitlabComment, GitlabIssue
11	from colab.proxy.utils.proxy_data_api import ProxyDataAPI	11	from colab.proxy.utils.proxy_data_api import ProxyDataAPI
12		12
13		13
	@@ -57,57 +57,57 @@ class GitlabDataAPI(ProxyDataAPI):		@@ -57,57 +57,57 @@ class GitlabDataAPI(ProxyDataAPI):
57		57
58	return projects	58	return projects
59		59
60	-# def fetch_merge_request(self, projects):
61	-# page = 1
62	-# all_merge_request = []
63	-#
64	-# # Iterate under all projects
65	-# for project in projects:
66	-# # Iterate under all MR inside project
67	-# while(True):
68	-# merge_request_url = \
69	-# '/api/v3/projects/{}/merge_requests'.format(project.id)
70	-# url = self.get_request_url(merge_request_url,
71	-# per_page=1000,
72	-# page=page)
73	-#
74	-# data = urllib2.urlopen(url)
75	-# json_data_mr = json.load(data)
76	-#
77	-# if len(json_data_mr) == 0:
78	-# break
79	-#
80	-# page = page + 1
81	-# for element in json_data_mr:
82	-# single_merge_request = GitlabMergeRequest()
83	-#
84	-# for field in GitlabMergeRequest._meta.fields:
85	-# if field.name == "user":
86	-# single_merge_request.update_user(
87	-# element["author"]["username"])
88	-# continue
89	-# if field.name == "project":
90	-# single_merge_request.project_id = \
91	-# element["project_id"]
92	-# continue
93	-#
94	-# if isinstance(field, DateTimeField):
95	-# value = parse(element[field.name])
96	-# else:
97	-# value = element[field.name]
98	-#
99	-# setattr(single_merge_request, field.name, value)
100	-#
101	-# all_merge_request.append(single_merge_request)
102	-#
103	-# return all_merge_request	60	+ def fetch_merge_request(self, projects):
		61	+ all_merge_request = []
		62	+
		63	+ # Iterate under all projects
		64	+ for project in projects:
		65	+ page = 1
		66	+ # Iterate under all MR inside project
		67	+ while(True):
		68	+ merge_request_url = \
		69	+ '/api/v3/projects/{}/merge_requests'.format(project.id)
		70	+ url = self.get_request_url(merge_request_url,
		71	+ per_page=1000,
		72	+ page=page)
		73	+
		74	+ data = urllib2.urlopen(url)
		75	+ json_data_mr = json.load(data)
		76	+
		77	+ if len(json_data_mr) == 0:
		78	+ break
		79	+
		80	+ page = page + 1
		81	+ for element in json_data_mr:
		82	+ single_merge_request = GitlabMergeRequest()
		83	+
		84	+ for field in GitlabMergeRequest._meta.fields:
		85	+ if field.name == "user":
		86	+ single_merge_request.update_user(
		87	+ element["author"]["username"])
		88	+ continue
		89	+ if field.name == "project":
		90	+ single_merge_request.project_id = \
		91	+ element["project_id"]
		92	+ continue
		93	+
		94	+ if isinstance(field, DateTimeField):
		95	+ value = parse(element[field.name])
		96	+ else:
		97	+ value = element[field.name]
		98	+
		99	+ setattr(single_merge_request, field.name, value)
		100	+
		101	+ all_merge_request.append(single_merge_request)
		102	+
		103	+ return all_merge_request
104		104
105	def fetch_issue(self, projects):	105	def fetch_issue(self, projects):
106	- page = 1
107	all_issues = []	106	all_issues = []
108		107
109	# Iterate under all projects	108	# Iterate under all projects
110	for project in projects:	109	for project in projects:
		110	+ page = 1
111	# Iterate under all Issues inside project	111	# Iterate under all Issues inside project
112	while(True):	112	while(True):
113	issue_url = \	113	issue_url = \
	@@ -129,6 +129,10 @@ class GitlabDataAPI(ProxyDataAPI):		@@ -129,6 +129,10 @@ class GitlabDataAPI(ProxyDataAPI):
129	if field.name == "project":	129	if field.name == "project":
130	single_issue.project_id = element["project_id"]	130	single_issue.project_id = element["project_id"]
131	continue	131	continue
		132	+ if field.name == "user":
		133	+ single_issue.update_user(
		134	+ element["author"]["username"])
		135	+ continue
132		136
133	if isinstance(field, DateTimeField):	137	if isinstance(field, DateTimeField):
134	value = parse(element[field.name])	138	value = parse(element[field.name])
	@@ -139,20 +143,120 @@ class GitlabDataAPI(ProxyDataAPI):		@@ -139,20 +143,120 @@ class GitlabDataAPI(ProxyDataAPI):
139		143
140	all_issues.append(single_issue)	144	all_issues.append(single_issue)
141		145
142	- return all_issues	146	+ return all_issues
143		147
144	- def fetch_data(self):
145	- data = self.fetch_projects()
146	- #merge_request_list = self.fetch_merge_request(data)
147	- issue_list = self.fetch_issue(data)	148	+ def fetch_comments(self):
		149	+ all_comments = []
		150	+
		151	+ all_comments.extend(self.fetch_comments_MR())
		152	+ all_comments.extend(self.fetch_comments_issues())
		153	+
		154	+ return all_comments
		155	+
		156	+ def fetch_comments_MR(self):
		157	+ all_comments = []
		158	+ all_merge_requests = GitlabMergeRequest.objects.all()
		159	+
		160	+ for merge_request in all_merge_requests:
		161	+ page = 1
		162	+ # Iterate under all comments of MR inside project
		163	+ while(True):
		164	+ merge_request_url = \
		165	+ '/api/v3/projects/{}/merge_requests/{}/notes' \
		166	+ .format(merge_request.project_id, merge_request.id)
		167	+ url = self.get_request_url(merge_request_url,
		168	+ per_page=1000,
		169	+ page=page)
		170	+
		171	+ data = urllib2.urlopen(url)
		172	+ json_data_mr = json.load(data)
		173	+
		174	+ if len(json_data_mr) == 0:
		175	+ break
		176	+
		177	+ page = page + 1
148		178
149	- for datum in data:	179	+ for element in json_data_mr:
		180	+ single_comment = GitlabComment()
		181	+
		182	+ for field in GitlabComment._meta.fields:
		183	+ if field.name == "user":
		184	+ single_comment.update_user(
		185	+ element["author"]["username"])
		186	+ continue
		187	+
		188	+ if isinstance(field, DateTimeField):
		189	+ value = parse(element[field.name])
		190	+ else:
		191	+ value = element[field.name]
		192	+
		193	+ setattr(single_comment, field.name, value)
		194	+
		195	+ all_comments.append(single_comment)
		196	+
		197	+ return all_comments
		198	+
		199	+
		200	+ def fetch_comments_issues(self):
		201	+ all_comments = []
		202	+ all_issues = GitlabIssue.objects.all()
		203	+
		204	+ for issue in all_issues:
		205	+ page = 1
		206	+ # Iterate under all comments of MR inside project
		207	+ while(True):
		208	+ issue_comments_request_url = \
		209	+ '/api/v3/projects/{}/issues/{}/notes' \
		210	+ .format(issue.project_id, issue.id)
		211	+ url = self.get_request_url(issue_comments_request_url,
		212	+ per_page=1000,
		213	+ page=page)
		214	+
		215	+ data = urllib2.urlopen(url)
		216	+ json_data_mr = json.load(data)
		217	+
		218	+ if len(json_data_mr) == 0:
		219	+ break
		220	+
		221	+ page = page + 1
		222	+ for element in json_data_mr:
		223	+ single_comment = GitlabComment()
		224	+
		225	+ for field in GitlabComment._meta.fields:
		226	+ if field.name == "user":
		227	+ single_comment.update_user(
		228	+ element["author"]["username"])
		229	+ continue
		230	+
		231	+ if isinstance(field, DateTimeField):
		232	+ value = parse(element[field.name])
		233	+ else:
		234	+ value = element[field.name]
		235	+
		236	+ setattr(single_comment, field.name, value)
		237	+
		238	+ all_comments.append(single_comment)
		239	+
		240	+ return all_comments
		241	+
		242	+ def fetch_data(self):
		243	+ projects = self.fetch_projects()
		244	+ for datum in projects:
150	datum.save()	245	datum.save()
		246	+
		247	+# merge_request_list = self.fetch_merge_request(projects)
151	# for datum in merge_request_list:	248	# for datum in merge_request_list:
152	# datum.save()	249	# datum.save()
		250	+
		251	+ issue_list = self.fetch_issue(projects)
153	for datum in issue_list:	252	for datum in issue_list:
154	datum.save()	253	datum.save()
155		254
		255	+ comments_list = self.fetch_comments()
		256	+ for datum in comments_list:
		257	+ datum.save()
		258	+
		259	+
156	@property	260	@property
157	def app_label(self):	261	def app_label(self):
158	return 'gitlab'	262	return 'gitlab'
	@@ -53,8 +53,6 @@ class GitlabComment(Collaboration):		@@ -53,8 +53,6 @@ class GitlabComment(Collaboration):
53		53
54	id = models.IntegerField(primary_key=True)	54	id = models.IntegerField(primary_key=True)
55	body = models.TextField()	55	body = models.TextField()
56	- author_email = models.EmailField(null=True, blank=True)
57	- author_name = models.TextField()
58	created_at = models.DateTimeField(blank=True)	56	created_at = models.DateTimeField(blank=True)
59		57
60	class Meta:	58	class Meta: