Download as
Browse Code »

Commit e460fd681b0ae5df1ff5ee26e2eaabe22ca76c93

Authored by Gust
Committed by Sergio Oliveira
1 parent 8825336b
Exists in master and in 39 other branches 1.12.x, add_requests_as_dep, celery_service, change-passwd-signal, check-port, colab-signals, colab-vcard, colab_search, colab_tag, colab_tag_merge, community_association, detach_super_archives, fix-dashboard, fix-message-preview, fix_app_label, fix_cache_count_spb, fix_header, fixed_gitlab_import, go_to_profile_panel, header_footer, layout-fix, mobile_user_scalable, move_out_plugins, paginate-threads, profile_integration, refactor-data-import, remove-gitlab-plugin, removing_namespace, search_block, search_filters, spb-release/3.0, split_screen, stable, startplugin, timestamp_plugins, translation_review, user_regex, validate-passwd, workin_whoosh_temp

Refactored data_api and uncorrected flake8 . 

Signed-off-by: Gustavo Jaruga <darksshades@gmail.com>
Signed-off-by: Rodrigo Siqueira <rodrigosiqueiramelo@gmail.com>
Showing 5 changed files with 163 additions and 140 deletions   Show diff stats
colab/proxy/gitlab/data_api.py
  Diff comments   View file @e460fd6
... ... @@ -7,7 +7,8 @@ from dateutil.parser import parse
7 7 from django.conf import settings
8 8 from django.db.models.fields import DateTimeField
9 9  
10   -from colab.proxy.gitlab.models import GitlabProject, GitlabMergeRequest, GitlabComment, GitlabIssue
  10 +from colab.proxy.gitlab.models import (GitlabProject, GitlabMergeRequest,
  11 + GitlabComment, GitlabIssue)
11 12 from colab.proxy.utils.proxy_data_api import ProxyDataAPI
12 13  
13 14  
... ... @@ -25,47 +26,52 @@ class GitlabDataAPI(ProxyDataAPI):
25 26  
26 27 return u'{}{}?{}'.format(upstream, path, params)
27 28  
28   - def get_json_data(self, api_url, page=1, pages=1000):
  29 + def get_json_data(self, api_url, page, pages=1000):
29 30 url = self.get_request_url(api_url, per_page=pages,
30 31 page=page)
31   - data = urllib2.urlopen(url)
32   - json_data = json.load(data)
  32 +
  33 + try:
  34 + data = urllib2.urlopen(url, timeout=10)
  35 + json_data = json.load(data)
  36 + except urllib2.URLError:
  37 + print "Connection timeout: " + url
  38 + json_data = []
33 39  
34 40 return json_data
35 41  
36 42 def fill_object_data(self, element, _object):
37 43 for field in _object._meta.fields:
38   - if field.name == "user":
39   - _object.update_user(
40   - element["author"]["username"])
41   - continue
42   - if field.name == "project":
43   - _object.project_id = \
44   - element["project_id"]
  44 + try:
  45 + if field.name == "user":
  46 + _object.update_user(
  47 + element["author"]["username"])
  48 + continue
  49 + if field.name == "project":
  50 + _object.project_id = element["project_id"]
  51 + continue
  52 +
  53 + if isinstance(field, DateTimeField):
  54 + value = parse(element[field.name])
  55 + else:
  56 + value = element[field.name]
  57 +
  58 + setattr(_object, field.name, value)
  59 + except KeyError:
45 60 continue
46 61  
47   - if isinstance(field, DateTimeField):
48   - value = parse(element["created_at"])
49   - else:
50   - value = element[field.name]
51   -
52   - setattr(_object, field.name, value)
53   -
54 62 return _object
55 63  
56 64 def fetch_projects(self):
57 65 page = 1
58 66 projects = []
59 67  
60   - # Iterates throughout all projects pages
61 68 while True:
62 69 json_data = self.get_json_data('/api/v3/projects/all', page)
  70 + page = page + 1
63 71  
64 72 if not len(json_data):
65 73 break
66 74  
67   - page = page + 1
68   -
69 75 for element in json_data:
70 76 project = GitlabProject()
71 77 self.fill_object_data(element, project)
... ... @@ -75,19 +81,17 @@ class GitlabDataAPI(ProxyDataAPI):
75 81  
76 82 def fetch_merge_request(self, projects):
77 83 all_merge_request = []
78   - # Iterate under all projects
  84 +
79 85 for project in projects:
80 86 page = 1
81   - # Iterate under all MR inside project
82   - while(True):
83   - merge_request_url = \
84   - '/api/v3/projects/{}/merge_requests'.format(project.id)
85   - json_data_mr = self.get_json_data(merge_request_url, page)
  87 + while True:
  88 + url = '/api/v3/projects/{}/merge_requests'.format(project.id)
  89 + json_data_mr = self.get_json_data(url, page)
  90 + page = page + 1
86 91  
87 92 if len(json_data_mr) == 0:
88 93 break
89 94  
90   - page = page + 1
91 95 for element in json_data_mr:
92 96 single_merge_request = GitlabMergeRequest()
93 97 self.fill_object_data(element, single_merge_request)
... ... @@ -98,20 +102,16 @@ class GitlabDataAPI(ProxyDataAPI):
98 102 def fetch_issue(self, projects):
99 103 all_issues = []
100 104  
101   - # Iterate under all projects
102 105 for project in projects:
103 106 page = 1
104   - # Iterate under all Issues inside project
105   - while(True):
106   - issue_url = \
107   - '/api/v3/projects/{}/issues'.format(project.id)
108   -
109   - json_data_issue = self.get_json_data(issue_url, page)
  107 + while True:
  108 + url = '/api/v3/projects/{}/issues'.format(project.id)
  109 + json_data_issue = self.get_json_data(url, page)
  110 + page = page + 1
110 111  
111 112 if len(json_data_issue) == 0:
112 113 break
113 114  
114   - page = page + 1
115 115 for element in json_data_issue:
116 116 single_issue = GitlabIssue()
117 117 self.fill_object_data(element, single_issue)
... ... @@ -132,91 +132,46 @@ class GitlabDataAPI(ProxyDataAPI):
132 132  
133 133 for merge_request in all_merge_requests:
134 134 page = 1
135   - # Iterate under all comments of MR inside project
136   - while(True):
137   - mr_url = '/api/v3/projects/{}/merge_requests/{}/notes'.format(
  135 + while True:
  136 + url = '/api/v3/projects/{}/merge_requests/{}/notes'.format(
138 137 merge_request.project_id, merge_request.id)
139   - json_data_mr = self.get_json_data(mr_url, page)
  138 + json_data_mr = self.get_json_data(url, page)
  139 + page = page + 1
140 140  
141 141 if len(json_data_mr) == 0:
142 142 break
143 143  
144   - page = page + 1
145   -
146 144 for element in json_data_mr:
147 145 single_comment = GitlabComment()
148   -
149   - for field in GitlabComment._meta.fields:
150   - if field.name == "user":
151   - single_comment.update_user(
152   - element["author"]["username"])
153   - continue
154   - if field.name == "project":
155   - single_comment.project = \
156   - merge_request.project
157   - continue
158   - if field.name == "issue_comment":
159   - single_comment.issue_comment = False
160   - continue
161   - if field.name == "parent_id":
162   - single_comment.parent_id = merge_request.id
163   - continue
164   -
165   - if isinstance(field, DateTimeField):
166   - value = parse(element["created_at"])
167   - else:
168   - value = element[field.name]
169   -
170   - setattr(single_comment, field.name, value)
171   -
  146 + self.fill_object_data(element, single_comment)
  147 + single_comment.project = merge_request.project
  148 + single_comment.issue_comment = False
  149 + single_comment.parent_id = merge_request.id
172 150 all_comments.append(single_comment)
173 151  
174 152 return all_comments
175 153  
176   -
177 154 def fetch_comments_issues(self):
178 155 all_comments = []
179 156 all_issues = GitlabIssue.objects.all()
180 157  
181 158 for issue in all_issues:
182 159 page = 1
183   - # Iterate under all comments of MR inside project
184   - while(True):
185   - issue_comments_request_url = \
186   - '/api/v3/projects/{}/issues/{}/notes' \
187   - .format(issue.project_id, issue.id)
188   - json_data_mr = self.get_json_data(issue_comments_request_url, page)
  160 + while True:
  161 + url = '/api/v3/projects/{}/issues/{}/notes'.format(
  162 + issue.project_id, issue.id)
  163 + json_data_mr = self.get_json_data(url, page)
  164 + page = page + 1
189 165  
190 166 if len(json_data_mr) == 0:
191 167 break
192 168  
193   - page = page + 1
194 169 for element in json_data_mr:
195 170 single_comment = GitlabComment()
196   -
197   - for field in GitlabComment._meta.fields:
198   - if field.name == "user":
199   - single_comment.update_user(
200   - element["author"]["username"])
201   - continue
202   - if field.name == "project":
203   - single_comment.project = \
204   - issue.project
205   - continue
206   - if field.name == "issue_comment":
207   - single_comment.issue_comment = True
208   - continue
209   - if field.name == "parent_id":
210   - single_comment.parent_id = issue.id
211   - continue
212   -
213   - if isinstance(field, DateTimeField):
214   - value = parse(element["created_at"])
215   - else:
216   - value = element[field.name]
217   -
218   - setattr(single_comment, field.name, value)
219   -
  171 + self.fill_object_data(element, single_comment)
  172 + single_comment.project = issue.project
  173 + single_comment.issue_comment = True
  174 + single_comment.parent_id = issue.id
220 175 all_comments.append(single_comment)
221 176  
222 177 return all_comments
... ... @@ -242,7 +197,6 @@ class GitlabDataAPI(ProxyDataAPI):
242 197 for datum in comments_list:
243 198 datum.save()
244 199  
245   -
246 200 @property
247 201 def app_label(self):
248 202 return 'gitlab'
... ...
colab/proxy/gitlab/migrations/0003_auto_20150211_1203.py 0 → 100644
  Diff comments   View file @e460fd6
... ... @@ -0,0 +1,61 @@
  1 +# -*- coding: utf-8 -*-
  2 +from __future__ import unicode_literals
  3 +
  4 +from django.db import models, migrations
  5 +import django.db.models.deletion
  6 +
  7 +
  8 +class Migration(migrations.Migration):
  9 +
  10 + dependencies = [
  11 + ('gitlab', '0002_auto_20150205_1220'),
  12 + ]
  13 +
  14 + operations = [
  15 + migrations.AlterModelOptions(
  16 + name='gitlabissue',
  17 + options={'verbose_name': 'Gitlab Issue', 'verbose_name_plural': 'Gitlab Issues'},
  18 + ),
  19 + migrations.AddField(
  20 + model_name='gitlabcomment',
  21 + name='issue_comment',
  22 + field=models.BooleanField(default=True),
  23 + preserve_default=True,
  24 + ),
  25 + migrations.AddField(
  26 + model_name='gitlabcomment',
  27 + name='parent_id',
  28 + field=models.IntegerField(null=True),
  29 + preserve_default=True,
  30 + ),
  31 + migrations.AddField(
  32 + model_name='gitlabcomment',
  33 + name='project',
  34 + field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, to='gitlab.GitlabProject', null=True),
  35 + preserve_default=True,
  36 + ),
  37 + migrations.AddField(
  38 + model_name='gitlabissue',
  39 + name='created_at',
  40 + field=models.DateTimeField(null=True, blank=True),
  41 + preserve_default=True,
  42 + ),
  43 + migrations.AddField(
  44 + model_name='gitlabmergerequest',
  45 + name='created_at',
  46 + field=models.DateTimeField(null=True, blank=True),
  47 + preserve_default=True,
  48 + ),
  49 + migrations.AddField(
  50 + model_name='gitlabproject',
  51 + name='path_with_namespace',
  52 + field=models.TextField(null=True, blank=True),
  53 + preserve_default=True,
  54 + ),
  55 + migrations.AlterField(
  56 + model_name='gitlabcomment',
  57 + name='created_at',
  58 + field=models.DateTimeField(null=True, blank=True),
  59 + preserve_default=True,
  60 + ),
  61 + ]
... ...
colab/proxy/gitlab/models.py
  Diff comments   View file @e460fd6
... ... @@ -3,7 +3,6 @@ from django.utils.translation import ugettext_lazy as _
3 3 from colab.proxy.utils.models import Collaboration
4 4 from hitcounter.models import HitCounterModelMixin
5 5  
6   -import datetime
7 6  
8 7 class GitlabProject(models.Model, HitCounterModelMixin):
9 8  
... ... @@ -35,7 +34,11 @@ class GitlabMergeRequest(Collaboration):
35 34 description = models.TextField()
36 35 title = models.TextField()
37 36 state = models.TextField()
38   - modified = models.DateTimeField(blank=True, null=True)
  37 + created_at = models.DateTimeField(blank=True, null=True)
  38 +
  39 + @property
  40 + def modified(self):
  41 + return self.created_at
39 42  
40 43 @property
41 44 def tag(self):
... ... @@ -46,10 +49,11 @@ class GitlabMergeRequest(Collaboration):
46 49  
47 50 @property
48 51 def url(self):
49   - return u'/gitlab/{}/merge_requests/{}'.format(self.project.path_with_namespace, self.id)
  52 + return u'/gitlab/{}/merge_requests/{}'.format(
  53 + self.project.path_with_namespace, self.id)
50 54  
51 55 def get_author(self):
52   - return user
  56 + return self.user
53 57  
54 58 class Meta:
55 59 verbose_name = _('Gitlab Merge Request')
... ... @@ -60,20 +64,24 @@ class GitlabIssue(Collaboration):
60 64  
61 65 id = models.IntegerField(primary_key=True)
62 66 project = models.ForeignKey(GitlabProject, null=True,
63   - on_delete=models.SET_NULL)
  67 + on_delete=models.SET_NULL)
64 68 title = models.TextField()
65 69 description = models.TextField()
66 70  
67 71 state = models.TextField()
68   - modified = models.DateTimeField(blank=True, null=True)
  72 + created_at = models.DateTimeField(blank=True, null=True)
69 73  
70 74 icon_name = u'align-right'
71 75 type = u'gitlab_issue'
72 76  
73 77 @property
74   - def url(self):
75   - return u'/gitlab/{}/issues/{}'.format(self.project.path_with_namespace, self.id)
  78 + def modified(self):
  79 + return self.created_at
76 80  
  81 + @property
  82 + def url(self):
  83 + return u'/gitlab/{}/issues/{}'.format(
  84 + self.project.path_with_namespace, self.id)
77 85  
78 86 class Meta:
79 87 verbose_name = _('Gitlab Issue')
... ... @@ -84,8 +92,7 @@ class GitlabComment(Collaboration):
84 92  
85 93 id = models.IntegerField(primary_key=True)
86 94 body = models.TextField()
87   - created_at = models.DateTimeField(blank=True)
88   - modified = models.DateTimeField(blank=True, null=True)
  95 + created_at = models.DateTimeField(blank=True, null=True)
89 96 issue_comment = models.BooleanField(default=True)
90 97  
91 98 project = models.ForeignKey(GitlabProject, null=True,
... ... @@ -95,6 +102,10 @@ class GitlabComment(Collaboration):
95 102 type = u'comment'
96 103  
97 104 @property
  105 + def modified(self):
  106 + return self.created_at
  107 +
  108 + @property
98 109 def title(self):
99 110 if self.issue_comment:
100 111 issue = GitlabIssue.objects.get(id=self.parent_id)
... ... @@ -127,7 +138,6 @@ class GitlabComment(Collaboration):
127 138 return u'/gitlab/{}/merge_requests/{}#notes_{}'.format(
128 139 self.project.path_with_namespace, self.parent_id, self.id)
129 140  
130   -
131 141 class Meta:
132 142 verbose_name = _('Gitlab Comments')
133 143 verbose_name_plural = _('Gitlab Comments')
... ...
colab/proxy/gitlab/search_indexes.py
  Diff comments   View file @e460fd6
1 1 # -*- coding: utf-8 -*-
2 2  
3   -import math
4 3 import string
5 4  
6   -from django.template import loader, Context
7   -from django.utils.text import slugify
8 5 from haystack import indexes
9 6 from haystack.utils import log as logging
10 7  
11   -from colab.search.base_indexes import BaseIndex
12   -from .models import GitlabProject, GitlabMergeRequest, GitlabIssue, GitlabComment
  8 +from .models import (GitlabProject, GitlabMergeRequest,
  9 + GitlabIssue, GitlabComment)
13 10  
14 11  
15 12 logger = logging.getLogger('haystack')
16 13  
17   -# the string maketrans always return a string encoded with latin1
  14 +# The string maketrans always return a string encoded with latin1
18 15 # http://stackoverflow.com/questions/1324067/how-do-i-get-str-translate-to-work-with-unicode-strings
19 16 table = string.maketrans(
20 17 string.punctuation,
21 18 '.' * len(string.punctuation)
22 19 ).decode('latin1')
23 20  
  21 +
24 22 class GitlabProjectIndex(indexes.SearchIndex, indexes.Indexable):
25 23 text = indexes.CharField(document=True, use_template=False, stored=False)
26 24 title = indexes.CharField(model_attr='name')
27 25 description = indexes.CharField(model_attr='description', null=True)
  26 + tag = indexes.CharField()
28 27 url = indexes.CharField(model_attr='url', indexed=False)
29   - type = indexes.CharField()
30 28 icon_name = indexes.CharField()
31   - tag = indexes.CharField()
  29 + type = indexes.CharField()
32 30 created = indexes.DateTimeField(model_attr='created_at', null=True)
33 31  
34 32 def prepare_tag(self, obj):
... ... @@ -43,26 +41,24 @@ class GitlabProjectIndex(indexes.SearchIndex, indexes.Indexable):
43 41 def get_model(self):
44 42 return GitlabProject
45 43  
46   - def prepare_icon_name(self, obj):
47   - return u'book'
48   -
49 44 def prepare_type(self, obj):
50 45 return u'gitlab'
51 46  
  47 +
52 48 class GitlabMergeRequestIndex(indexes.SearchIndex, indexes.Indexable):
53 49  
54 50 text = indexes.CharField(document=True, use_template=False, stored=False)
55   - description = indexes.CharField(model_attr='description')
56 51 title = indexes.CharField(model_attr='title')
  52 + description = indexes.CharField(model_attr='description')
57 53 tag = indexes.CharField(model_attr='state')
58 54 url = indexes.CharField(model_attr='url', indexed=False)
59 55 icon_name = indexes.CharField()
60 56 type = indexes.CharField(model_attr='type')
61 57  
62 58 modified_by = indexes.CharField(model_attr='modified_by', null=True)
63   - modified_by_url = indexes.CharField(model_attr='modified_by_url', null=True)
64   - modified = indexes.DateTimeField(model_attr='modified', null=True)
65   -
  59 + modified_by_url = indexes.CharField(model_attr='modified_by_url',
  60 + null=True)
  61 + modified = indexes.DateTimeField(model_attr='created_at', null=True)
66 62  
67 63 def get_model(self):
68 64 return GitlabMergeRequest
... ... @@ -73,20 +69,21 @@ class GitlabMergeRequestIndex(indexes.SearchIndex, indexes.Indexable):
73 69 def prepare_type(self, obj):
74 70 return u'merge_request'
75 71  
  72 +
76 73 class GitlabIssueIndex(indexes.SearchIndex, indexes.Indexable):
77 74  
78 75 text = indexes.CharField(document=True, use_template=False, stored=False)
79   - description = indexes.CharField(model_attr='description')
80 76 title = indexes.CharField(model_attr='title')
  77 + description = indexes.CharField(model_attr='description')
81 78 tag = indexes.CharField(model_attr='state')
82 79 url = indexes.CharField(model_attr='url', indexed=False)
83 80 icon_name = indexes.CharField()
84 81 type = indexes.CharField(model_attr='type')
85 82  
86 83 modified_by = indexes.CharField(model_attr='modified_by', null=True)
87   - modified_by_url = indexes.CharField(model_attr='modified_by_url', null=True)
88   - modified = indexes.DateTimeField(model_attr='modified', null=True)
89   -
  84 + modified_by_url = indexes.CharField(model_attr='modified_by_url',
  85 + null=True)
  86 + modified = indexes.DateTimeField(model_attr='created_at', null=True)
90 87  
91 88 def get_model(self):
92 89 return GitlabIssue
... ... @@ -97,25 +94,28 @@ class GitlabIssueIndex(indexes.SearchIndex, indexes.Indexable):
97 94 def prepare_type(self, obj):
98 95 return u'merge_request'
99 96  
  97 +
100 98 class GitlabCommentIndex(indexes.SearchIndex, indexes.Indexable):
101 99  
102 100 text = indexes.CharField(document=True, use_template=False, stored=False)
103   - description = indexes.CharField(model_attr='description')
104 101 title = indexes.CharField(model_attr='title')
  102 + description = indexes.CharField(model_attr='description')
105 103 tag = indexes.CharField()
106 104 url = indexes.CharField(model_attr='url', indexed=False)
107 105 icon_name = indexes.CharField()
108 106 type = indexes.CharField(model_attr='type')
109 107  
110 108 modified_by = indexes.CharField(model_attr='modified_by', null=True)
111   - modified_by_url = indexes.CharField(model_attr='modified_by_url', null=True)
112   - modified = indexes.DateTimeField(model_attr='modified', null=True)
113   -
114   - def prepare_tag(self, obj):
115   - return obj.tag
  109 + modified_by_url = indexes.CharField(model_attr='modified_by_url',
  110 + null=True)
  111 + modified = indexes.DateTimeField(model_attr='created_at', null=True)
116 112  
117 113 def get_model(self):
118 114 return GitlabComment
119 115  
120 116 def prepare_icon_name(self, obj):
121   - return u'align-right'
122 117 \ No newline at end of file
  118 + return u'align-right'
  119 +
  120 + def prepare_tag(self, obj):
  121 + return obj.tag
  122 +
... ...
colab/proxy/gitlab/tests/test_gitlab.py
  Diff comments   View file @e460fd6
... ... @@ -41,14 +41,13 @@ class GitlabTest(TestCase):
41 41 '/gitlab/softwarepublico/colab/issues/1')
42 42  
43 43 def test_comment_on_mr_url(self):
44   - self.assertEqual(GitlabComment.objects.get(id=1).url,
45   - '/gitlab/softwarepublico/colab/merge_requests/1#notes_1')
  44 + url = '/gitlab/softwarepublico/colab/merge_requests/1#notes_1'
  45 + self.assertEqual(GitlabComment.objects.get(id=1).url, url)
46 46  
47 47 def test_comment_on_issue_url(self):
48 48 self.assertEqual(GitlabComment.objects.get(id=2).url,
49 49 '/gitlab/softwarepublico/colab/issues/1#notes_2')
50 50  
51   -
52 51 def create_gitlab_data(self):
53 52 g = GitlabProject()
54 53 g.id = 1
... ... @@ -113,4 +112,3 @@ class GitlabTest(TestCase):
113 112 user.save()
114 113  
115 114 return user
116   -
... ...