diff --git a/colab/proxy/gitlab/data_api.py b/colab/proxy/gitlab/data_api.py index 42d0773..3c9a433 100644 --- a/colab/proxy/gitlab/data_api.py +++ b/colab/proxy/gitlab/data_api.py @@ -7,7 +7,8 @@ from dateutil.parser import parse from django.conf import settings from django.db.models.fields import DateTimeField -from colab.proxy.gitlab.models import GitlabProject, GitlabMergeRequest, GitlabComment, GitlabIssue +from colab.proxy.gitlab.models import (GitlabProject, GitlabMergeRequest, + GitlabComment, GitlabIssue) from colab.proxy.utils.proxy_data_api import ProxyDataAPI @@ -25,47 +26,52 @@ class GitlabDataAPI(ProxyDataAPI): return u'{}{}?{}'.format(upstream, path, params) - def get_json_data(self, api_url, page=1, pages=1000): + def get_json_data(self, api_url, page, pages=1000): url = self.get_request_url(api_url, per_page=pages, page=page) - data = urllib2.urlopen(url) - json_data = json.load(data) + + try: + data = urllib2.urlopen(url, timeout=10) + json_data = json.load(data) + except urllib2.URLError: + print "Connection timeout: " + url + json_data = [] return json_data def fill_object_data(self, element, _object): for field in _object._meta.fields: - if field.name == "user": - _object.update_user( - element["author"]["username"]) - continue - if field.name == "project": - _object.project_id = \ - element["project_id"] + try: + if field.name == "user": + _object.update_user( + element["author"]["username"]) + continue + if field.name == "project": + _object.project_id = element["project_id"] + continue + + if isinstance(field, DateTimeField): + value = parse(element[field.name]) + else: + value = element[field.name] + + setattr(_object, field.name, value) + except KeyError: continue - if isinstance(field, DateTimeField): - value = parse(element["created_at"]) - else: - value = element[field.name] - - setattr(_object, field.name, value) - return _object def fetch_projects(self): page = 1 projects = [] - # Iterates throughout all projects pages while True: json_data = self.get_json_data('/api/v3/projects/all', page) + page = page + 1 if not len(json_data): break - page = page + 1 - for element in json_data: project = GitlabProject() self.fill_object_data(element, project) @@ -75,19 +81,17 @@ class GitlabDataAPI(ProxyDataAPI): def fetch_merge_request(self, projects): all_merge_request = [] - # Iterate under all projects + for project in projects: page = 1 - # Iterate under all MR inside project - while(True): - merge_request_url = \ - '/api/v3/projects/{}/merge_requests'.format(project.id) - json_data_mr = self.get_json_data(merge_request_url, page) + while True: + url = '/api/v3/projects/{}/merge_requests'.format(project.id) + json_data_mr = self.get_json_data(url, page) + page = page + 1 if len(json_data_mr) == 0: break - page = page + 1 for element in json_data_mr: single_merge_request = GitlabMergeRequest() self.fill_object_data(element, single_merge_request) @@ -98,20 +102,16 @@ class GitlabDataAPI(ProxyDataAPI): def fetch_issue(self, projects): all_issues = [] - # Iterate under all projects for project in projects: page = 1 - # Iterate under all Issues inside project - while(True): - issue_url = \ - '/api/v3/projects/{}/issues'.format(project.id) - - json_data_issue = self.get_json_data(issue_url, page) + while True: + url = '/api/v3/projects/{}/issues'.format(project.id) + json_data_issue = self.get_json_data(url, page) + page = page + 1 if len(json_data_issue) == 0: break - page = page + 1 for element in json_data_issue: single_issue = GitlabIssue() self.fill_object_data(element, single_issue) @@ -132,91 +132,46 @@ class GitlabDataAPI(ProxyDataAPI): for merge_request in all_merge_requests: page = 1 - # Iterate under all comments of MR inside project - while(True): - mr_url = '/api/v3/projects/{}/merge_requests/{}/notes'.format( + while True: + url = '/api/v3/projects/{}/merge_requests/{}/notes'.format( merge_request.project_id, merge_request.id) - json_data_mr = self.get_json_data(mr_url, page) + json_data_mr = self.get_json_data(url, page) + page = page + 1 if len(json_data_mr) == 0: break - page = page + 1 - for element in json_data_mr: single_comment = GitlabComment() - - for field in GitlabComment._meta.fields: - if field.name == "user": - single_comment.update_user( - element["author"]["username"]) - continue - if field.name == "project": - single_comment.project = \ - merge_request.project - continue - if field.name == "issue_comment": - single_comment.issue_comment = False - continue - if field.name == "parent_id": - single_comment.parent_id = merge_request.id - continue - - if isinstance(field, DateTimeField): - value = parse(element["created_at"]) - else: - value = element[field.name] - - setattr(single_comment, field.name, value) - + self.fill_object_data(element, single_comment) + single_comment.project = merge_request.project + single_comment.issue_comment = False + single_comment.parent_id = merge_request.id all_comments.append(single_comment) return all_comments - def fetch_comments_issues(self): all_comments = [] all_issues = GitlabIssue.objects.all() for issue in all_issues: page = 1 - # Iterate under all comments of MR inside project - while(True): - issue_comments_request_url = \ - '/api/v3/projects/{}/issues/{}/notes' \ - .format(issue.project_id, issue.id) - json_data_mr = self.get_json_data(issue_comments_request_url, page) + while True: + url = '/api/v3/projects/{}/issues/{}/notes'.format( + issue.project_id, issue.id) + json_data_mr = self.get_json_data(url, page) + page = page + 1 if len(json_data_mr) == 0: break - page = page + 1 for element in json_data_mr: single_comment = GitlabComment() - - for field in GitlabComment._meta.fields: - if field.name == "user": - single_comment.update_user( - element["author"]["username"]) - continue - if field.name == "project": - single_comment.project = \ - issue.project - continue - if field.name == "issue_comment": - single_comment.issue_comment = True - continue - if field.name == "parent_id": - single_comment.parent_id = issue.id - continue - - if isinstance(field, DateTimeField): - value = parse(element["created_at"]) - else: - value = element[field.name] - - setattr(single_comment, field.name, value) - + self.fill_object_data(element, single_comment) + single_comment.project = issue.project + single_comment.issue_comment = True + single_comment.parent_id = issue.id all_comments.append(single_comment) return all_comments @@ -242,7 +197,6 @@ class GitlabDataAPI(ProxyDataAPI): for datum in comments_list: datum.save() - @property def app_label(self): return 'gitlab' diff --git a/colab/proxy/gitlab/migrations/0003_auto_20150211_1203.py b/colab/proxy/gitlab/migrations/0003_auto_20150211_1203.py new file mode 100644 index 0000000..4d49bab --- /dev/null +++ b/colab/proxy/gitlab/migrations/0003_auto_20150211_1203.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('gitlab', '0002_auto_20150205_1220'), + ] + + operations = [ + migrations.AlterModelOptions( + name='gitlabissue', + options={'verbose_name': 'Gitlab Issue', 'verbose_name_plural': 'Gitlab Issues'}, + ), + migrations.AddField( + model_name='gitlabcomment', + name='issue_comment', + field=models.BooleanField(default=True), + preserve_default=True, + ), + migrations.AddField( + model_name='gitlabcomment', + name='parent_id', + field=models.IntegerField(null=True), + preserve_default=True, + ), + migrations.AddField( + model_name='gitlabcomment', + name='project', + field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to='gitlab.GitlabProject', null=True), + preserve_default=True, + ), + migrations.AddField( + model_name='gitlabissue', + name='created_at', + field=models.DateTimeField(null=True, blank=True), + preserve_default=True, + ), + migrations.AddField( + model_name='gitlabmergerequest', + name='created_at', + field=models.DateTimeField(null=True, blank=True), + preserve_default=True, + ), + migrations.AddField( + model_name='gitlabproject', + name='path_with_namespace', + field=models.TextField(null=True, blank=True), + preserve_default=True, + ), + migrations.AlterField( + model_name='gitlabcomment', + name='created_at', + field=models.DateTimeField(null=True, blank=True), + preserve_default=True, + ), + ] diff --git a/colab/proxy/gitlab/models.py b/colab/proxy/gitlab/models.py index 36b43de..05c40b2 100644 --- a/colab/proxy/gitlab/models.py +++ b/colab/proxy/gitlab/models.py @@ -3,7 +3,6 @@ from django.utils.translation import ugettext_lazy as _ from colab.proxy.utils.models import Collaboration from hitcounter.models import HitCounterModelMixin -import datetime class GitlabProject(models.Model, HitCounterModelMixin): @@ -35,7 +34,11 @@ class GitlabMergeRequest(Collaboration): description = models.TextField() title = models.TextField() state = models.TextField() - modified = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField(blank=True, null=True) + + @property + def modified(self): + return self.created_at @property def tag(self): @@ -46,10 +49,11 @@ class GitlabMergeRequest(Collaboration): @property def url(self): - return u'/gitlab/{}/merge_requests/{}'.format(self.project.path_with_namespace, self.id) + return u'/gitlab/{}/merge_requests/{}'.format( + self.project.path_with_namespace, self.id) def get_author(self): - return user + return self.user class Meta: verbose_name = _('Gitlab Merge Request') @@ -60,20 +64,24 @@ class GitlabIssue(Collaboration): id = models.IntegerField(primary_key=True) project = models.ForeignKey(GitlabProject, null=True, - on_delete=models.SET_NULL) + on_delete=models.SET_NULL) title = models.TextField() description = models.TextField() state = models.TextField() - modified = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField(blank=True, null=True) icon_name = u'align-right' type = u'gitlab_issue' @property - def url(self): - return u'/gitlab/{}/issues/{}'.format(self.project.path_with_namespace, self.id) + def modified(self): + return self.created_at + @property + def url(self): + return u'/gitlab/{}/issues/{}'.format( + self.project.path_with_namespace, self.id) class Meta: verbose_name = _('Gitlab Issue') @@ -84,8 +92,7 @@ class GitlabComment(Collaboration): id = models.IntegerField(primary_key=True) body = models.TextField() - created_at = models.DateTimeField(blank=True) - modified = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField(blank=True, null=True) issue_comment = models.BooleanField(default=True) project = models.ForeignKey(GitlabProject, null=True, @@ -95,6 +102,10 @@ class GitlabComment(Collaboration): type = u'comment' @property + def modified(self): + return self.created_at + + @property def title(self): if self.issue_comment: issue = GitlabIssue.objects.get(id=self.parent_id) @@ -127,7 +138,6 @@ class GitlabComment(Collaboration): return u'/gitlab/{}/merge_requests/{}#notes_{}'.format( self.project.path_with_namespace, self.parent_id, self.id) - class Meta: verbose_name = _('Gitlab Comments') verbose_name_plural = _('Gitlab Comments') diff --git a/colab/proxy/gitlab/search_indexes.py b/colab/proxy/gitlab/search_indexes.py index 3f81a81..b590835 100644 --- a/colab/proxy/gitlab/search_indexes.py +++ b/colab/proxy/gitlab/search_indexes.py @@ -1,34 +1,32 @@ # -*- coding: utf-8 -*- -import math import string -from django.template import loader, Context -from django.utils.text import slugify from haystack import indexes from haystack.utils import log as logging -from colab.search.base_indexes import BaseIndex -from .models import GitlabProject, GitlabMergeRequest, GitlabIssue, GitlabComment +from .models import (GitlabProject, GitlabMergeRequest, + GitlabIssue, GitlabComment) logger = logging.getLogger('haystack') -# the string maketrans always return a string encoded with latin1 +# The string maketrans always return a string encoded with latin1 # http://stackoverflow.com/questions/1324067/how-do-i-get-str-translate-to-work-with-unicode-strings table = string.maketrans( string.punctuation, '.' * len(string.punctuation) ).decode('latin1') + class GitlabProjectIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=False, stored=False) title = indexes.CharField(model_attr='name') description = indexes.CharField(model_attr='description', null=True) + tag = indexes.CharField() url = indexes.CharField(model_attr='url', indexed=False) - type = indexes.CharField() icon_name = indexes.CharField() - tag = indexes.CharField() + type = indexes.CharField() created = indexes.DateTimeField(model_attr='created_at', null=True) def prepare_tag(self, obj): @@ -43,26 +41,24 @@ class GitlabProjectIndex(indexes.SearchIndex, indexes.Indexable): def get_model(self): return GitlabProject - def prepare_icon_name(self, obj): - return u'book' - def prepare_type(self, obj): return u'gitlab' + class GitlabMergeRequestIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=False, stored=False) - description = indexes.CharField(model_attr='description') title = indexes.CharField(model_attr='title') + description = indexes.CharField(model_attr='description') tag = indexes.CharField(model_attr='state') url = indexes.CharField(model_attr='url', indexed=False) icon_name = indexes.CharField() type = indexes.CharField(model_attr='type') modified_by = indexes.CharField(model_attr='modified_by', null=True) - modified_by_url = indexes.CharField(model_attr='modified_by_url', null=True) - modified = indexes.DateTimeField(model_attr='modified', null=True) - + modified_by_url = indexes.CharField(model_attr='modified_by_url', + null=True) + modified = indexes.DateTimeField(model_attr='created_at', null=True) def get_model(self): return GitlabMergeRequest @@ -73,20 +69,21 @@ class GitlabMergeRequestIndex(indexes.SearchIndex, indexes.Indexable): def prepare_type(self, obj): return u'merge_request' + class GitlabIssueIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=False, stored=False) - description = indexes.CharField(model_attr='description') title = indexes.CharField(model_attr='title') + description = indexes.CharField(model_attr='description') tag = indexes.CharField(model_attr='state') url = indexes.CharField(model_attr='url', indexed=False) icon_name = indexes.CharField() type = indexes.CharField(model_attr='type') modified_by = indexes.CharField(model_attr='modified_by', null=True) - modified_by_url = indexes.CharField(model_attr='modified_by_url', null=True) - modified = indexes.DateTimeField(model_attr='modified', null=True) - + modified_by_url = indexes.CharField(model_attr='modified_by_url', + null=True) + modified = indexes.DateTimeField(model_attr='created_at', null=True) def get_model(self): return GitlabIssue @@ -97,25 +94,28 @@ class GitlabIssueIndex(indexes.SearchIndex, indexes.Indexable): def prepare_type(self, obj): return u'merge_request' + class GitlabCommentIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField(document=True, use_template=False, stored=False) - description = indexes.CharField(model_attr='description') title = indexes.CharField(model_attr='title') + description = indexes.CharField(model_attr='description') tag = indexes.CharField() url = indexes.CharField(model_attr='url', indexed=False) icon_name = indexes.CharField() type = indexes.CharField(model_attr='type') modified_by = indexes.CharField(model_attr='modified_by', null=True) - modified_by_url = indexes.CharField(model_attr='modified_by_url', null=True) - modified = indexes.DateTimeField(model_attr='modified', null=True) - - def prepare_tag(self, obj): - return obj.tag + modified_by_url = indexes.CharField(model_attr='modified_by_url', + null=True) + modified = indexes.DateTimeField(model_attr='created_at', null=True) def get_model(self): return GitlabComment def prepare_icon_name(self, obj): - return u'align-right' \ No newline at end of file + return u'align-right' + + def prepare_tag(self, obj): + return obj.tag + diff --git a/colab/proxy/gitlab/tests/test_gitlab.py b/colab/proxy/gitlab/tests/test_gitlab.py index 75afb0f..1c3a69c 100644 --- a/colab/proxy/gitlab/tests/test_gitlab.py +++ b/colab/proxy/gitlab/tests/test_gitlab.py @@ -41,14 +41,13 @@ class GitlabTest(TestCase): '/gitlab/softwarepublico/colab/issues/1') def test_comment_on_mr_url(self): - self.assertEqual(GitlabComment.objects.get(id=1).url, - '/gitlab/softwarepublico/colab/merge_requests/1#notes_1') + url = '/gitlab/softwarepublico/colab/merge_requests/1#notes_1' + self.assertEqual(GitlabComment.objects.get(id=1).url, url) def test_comment_on_issue_url(self): self.assertEqual(GitlabComment.objects.get(id=2).url, '/gitlab/softwarepublico/colab/issues/1#notes_2') - def create_gitlab_data(self): g = GitlabProject() g.id = 1 @@ -113,4 +112,3 @@ class GitlabTest(TestCase): user.save() return user - -- libgit2 0.21.2