Refactored gitlab data_api

Sergio Oliveira · Gust
1 parent 16da24cf
Showing 3 changed files with 53 additions and 36 deletions Show diff stats
colab/management/initconfig.py
colab/proxy/gitlab/data_api.py
colab/proxy/proxybase/proxy_data_api.py
@@ -70,7 +70,7 @@ ROBOTS_NOINDEX: false
 # PROXIED_APPS:
 #   gitlab:
 #     upstream: 'http://localhost:8090/gitlab/'
-#     auth_token: ''
+#     private_token: ''
 #   trac:
 #     upstream: 'http://localhost:5000/trac/'
  
-from colab.proxy.gitlab.models import *
-from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
-from django.db.models.fields import DateTimeField
-from dateutil.parser import parse
-import urllib2
+
 import json
+import urllib
+import urllib2
+import urlparse
+
+from dateutil.parser import parse
+
 from django.conf import settings
+from django.db.models.fields import DateTimeField
+
+from colab.proxy.gitlab.models import GitlabProject
+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
+
  
 class GitlabDataAPI(ProxyDataAPI):
  
+    def get_request_url(self, path, **kwargs):
+        proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
  
-  def fetchProjects(self):
-    page = 1
-    projects = []
+        upstream = proxy_config.get('upstream')
+        kwargs['private_token'] = proxy_config.get('private_token')
+        params = urllib.urlencode(kwargs)
  
-    proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
-    admin_token = proxy_config.get('auth_token')
+        url = u'{}?{}'.format(path, params)
+        return urlparse.urljoin(upstream, url)
  
-    # Iterates throughout all projects pages
-    while(True):
-      data = urllib2.urlopen(proxy_config.get('upstream')+'api/v3/projects/all?private_token={}&per_page=100&page={}'.format(admin_token, page))
-      json_data = json.load(data)
+    def fetchProjects(self):
+        page = 1
+        projects = []
  
-      if len(json_data) == 0:
-        break
+        # Iterates throughout all projects pages
+        while(True):
+            url = self.get_request_url('/api/v3/projects/all',
+                                       per_page=100,
+                                       page=page)
+            data = urllib2.urlopen(url)
+            json_data = json.load(data)
  
-      page = page + 1
+            if len(json_data) == 0:
+                break
  
-      for element in json_data:
-        project = GitlabProject()
+            page = page + 1
  
-        for field in GitlabProject._meta.fields:
-          value = element[field.name]
-          value = parse(element[field.name]) if isinstance(field, DateTimeField) else value
-          setattr(project, field.name, value)
+            for element in json_data:
+                project = GitlabProject()
  
-        projects.append(project)
+                for field in GitlabProject._meta.fields:
+                    if isinstance(field, DateTimeField):
+                        value = parse(element[field.name])
+                    else:
+                        value = element[field.name]
  
-    return projects
+                    setattr(project, field.name, value)
  
+                projects.append(project)
  
-  def fetchData(self):
-    data = self.fetchProjects()
+        return projects
  
-    for datum in data:
-      datum.save()
+    def fetchData(self):
+        data = self.fetchProjects()
  
-  @property
-  def app_label(self):
-    return 'gitlab'
+        for datum in data:
+            datum.save()
  
+    @property
+    def app_label(self):
+        return 'gitlab'
  
-class ProxyDataAPI():
  
+class ProxyDataAPI(object):
  
-  def fetchData(self):
-    raise NotImplementedError('fetchData not yet implemented')
 \ No newline at end of file
+    def fetchData(self):
+        raise NotImplementedError('fetchData not yet implemented')
...	...	@@ -70,7 +70,7 @@ ROBOTS_NOINDEX: false
70	70	# PROXIED_APPS:
71	71	# gitlab:
72	72	# upstream: 'http://localhost:8090/gitlab/'
73		-# auth_token: ''
	73	+# private_token: ''
74	74	# trac:
75	75	# upstream: 'http://localhost:5000/trac/'
76	76
...	...
1		-from colab.proxy.gitlab.models import *
2		-from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
3		-from django.db.models.fields import DateTimeField
4		-from dateutil.parser import parse
5		-import urllib2
	1	+
6	2	import json
	3	+import urllib
	4	+import urllib2
	5	+import urlparse
	6	+
	7	+from dateutil.parser import parse
	8	+
7	9	from django.conf import settings
	10	+from django.db.models.fields import DateTimeField
	11	+
	12	+from colab.proxy.gitlab.models import GitlabProject
	13	+from colab.proxy.proxybase.proxy_data_api import ProxyDataAPI
	14	+
8	15
9	16	class GitlabDataAPI(ProxyDataAPI):
10	17
	18	+ def get_request_url(self, path, **kwargs):
	19	+ proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
11	20
12		- def fetchProjects(self):
13		- page = 1
14		- projects = []
	21	+ upstream = proxy_config.get('upstream')
	22	+ kwargs['private_token'] = proxy_config.get('private_token')
	23	+ params = urllib.urlencode(kwargs)
15	24
16		- proxy_config = settings.PROXIED_APPS.get(self.app_label, {})
17		- admin_token = proxy_config.get('auth_token')
	25	+ url = u'{}?{}'.format(path, params)
	26	+ return urlparse.urljoin(upstream, url)
18	27
19		- # Iterates throughout all projects pages
20		- while(True):
21		- data = urllib2.urlopen(proxy_config.get('upstream')+'api/v3/projects/all?private_token={}&per_page=100&page={}'.format(admin_token, page))
22		- json_data = json.load(data)
	28	+ def fetchProjects(self):
	29	+ page = 1
	30	+ projects = []
23	31
24		- if len(json_data) == 0:
25		- break
	32	+ # Iterates throughout all projects pages
	33	+ while(True):
	34	+ url = self.get_request_url('/api/v3/projects/all',
	35	+ per_page=100,
	36	+ page=page)
	37	+ data = urllib2.urlopen(url)
	38	+ json_data = json.load(data)
26	39
27		- page = page + 1
	40	+ if len(json_data) == 0:
	41	+ break
28	42
29		- for element in json_data:
30		- project = GitlabProject()
	43	+ page = page + 1
31	44
32		- for field in GitlabProject._meta.fields:
33		- value = element[field.name]
34		- value = parse(element[field.name]) if isinstance(field, DateTimeField) else value
35		- setattr(project, field.name, value)
	45	+ for element in json_data:
	46	+ project = GitlabProject()
36	47
37		- projects.append(project)
	48	+ for field in GitlabProject._meta.fields:
	49	+ if isinstance(field, DateTimeField):
	50	+ value = parse(element[field.name])
	51	+ else:
	52	+ value = element[field.name]
38	53
39		- return projects
	54	+ setattr(project, field.name, value)
40	55
	56	+ projects.append(project)
41	57
42		- def fetchData(self):
43		- data = self.fetchProjects()
	58	+ return projects
44	59
45		- for datum in data:
46		- datum.save()
	60	+ def fetchData(self):
	61	+ data = self.fetchProjects()
47	62
48		- @property
49		- def app_label(self):
50		- return 'gitlab'
	63	+ for datum in data:
	64	+ datum.save()
51	65
	66	+ @property
	67	+ def app_label(self):
	68	+ return 'gitlab'
...	...
1	1
2		-class ProxyDataAPI():
3	2
	3	+class ProxyDataAPI(object):
4	4
5		- def fetchData(self):
6		- raise NotImplementedError('fetchData not yet implemented')
7	5	\ No newline at end of file
	6	+ def fetchData(self):
	7	+ raise NotImplementedError('fetchData not yet implemented')
...	...