Download as
Browse Code »

Commit ca3ad95878bbeb76f00b3a39cd52ee60a7fba2e7

Authored by Tássia Camões Araújo
1 parent b6c8c0a6
Exists in master and in 1 other branch add_vagrant

AppRecommender server refactoring.

Showing 1 changed file with 73 additions and 246 deletions   Show diff stats
src/web/server.py
  Diff comments   View file @ca3ad95
1 1 #!/usr/bin/env python
2 2  
  3 +import os
3 4 import web
4 5 from web import form
5 6 import tempfile
... ... @@ -7,263 +8,84 @@ import sys
7 8 import simplejson as json
8 9 import apt
9 10 import re
  11 +import socket
10 12  
11   -sys.path.insert(0,"../")
  13 +sys.path.insert(0,"/var/www/AppRecommender/src/")
12 14  
13   -from config import *
  15 +from config import Config
14 16 from recommender import *
15 17 from user import *
  18 +from data import DebianPackage
16 19  
17 20 import urllib
18 21  
19   -#class FeedbackForm(form.Form):
20   -# def __init__(self,selected_strategies):
21   -# desc_dict = {"cb": "Content-based", "cbt": "Content-based",
22   -# "cbd": "Content-based", "col": "Collaborative",
23   -# "hybrid": "Hybrib"}
24   -# fields = []
25   -# for strategy in selected_strategies:
26   -# fields.append(form.Radio(desc_dict[strategy],
27   -# [('1','1 '),('2','2 '),('3','3'),('4','4 '),('5','5')]))
28   -# form.Form.__init__(self, *fields, validators = [])
  22 +# avoid "RuntimeError: maximum recursion depth exceeded"
  23 +sys.setrecursionlimit(50000)
  24 +
  25 +class Fake:
  26 + def GET(self):
  27 + return render_plain.fake()
29 28  
30 29 class Index:
31 30 def GET(self):
32   - if Config().survey_mode:
33   - return render.survey_index()
34   - else:
35 31 return render.index()
36 32  
37 33 class About:
38 34 def GET(self):
39 35 return render.about()
40 36  
41   -#class Support:
42   -# def GET(self):
43   -# return render.support()
44   -
45   -class Thanks:
46   - def POST(self):
47   - return render.thanks()
48   -
49 37 class Package:
50   - def GET(self, pkg):
51   - result = self.get_details_from_dde(pkg)
52   - return render_plain.package(result)
53   -
54   - def get_details_from_dde(self, pkg):
55   - json_source = Config().dde_url % pkg
56   - json_data = json.load(urllib.urlopen(json_source))
57   - # parse tags
58   - tags = self._debtags_list_to_dict(json_data['r']['tag'])
59   - json_data['r']['tag'] = tags
60   - # format long description
61   - json_data['r']['long_description'] = json_data['r']['long_description'].replace(' .\n','').replace('\n','<br />')
62   - return json_data['r']
63   -
64   - def _debtags_list_to_dict(self, debtags_list):
65   - """ in:
66   - ['use::editing',
67   - 'works-with-format::gif',
68   - 'works-with-format::jpg',
69   - 'works-with-format::pdf']
70   - out:
71   - {'use': [editing],
72   - 'works-with-format': ['gif', 'jpg', 'pdf']'
73   - }
74   - """
75   - debtags = {}
76   - subtags = []
77   - for tag in debtags_list:
78   - match = re.search(r'^(.*)::(.*)$', tag)
79   - if not match:
80   - log.error("Could not parse debtags format from tag: %s", tag)
81   - facet, subtag = match.groups()
82   - subtags.append(subtag)
83   - if facet not in debtags:
84   - debtags[facet] = subtags
85   - else:
86   - debtags[facet].append(subtag)
87   - subtags = []
88   - return debtags
89   -
90   -class Request:
91   - def __init__(self,web_input):
92   - submissions_dir = "./submissions/"
93   - if not os.path.exists(submissions_dir):
94   - os.makedirs(submissions_dir)
95   - outputdir = tempfile.mkdtemp(prefix='',dir=submissions_dir)
96   - user_id = outputdir.lstrip(submissions_dir)
97   - self.user_id = user_id
98   - self.storage = web_input
99   -
100   - self.pkgs_list = []
101   - if web_input.has_key('pkgs_list'):
102   - self.pkgs_list = web_input['pkgs_list'].encode('utf8').split()
103   - if web_input['pkgs_file'].value:
104   - f = open(outputdir + "/packages_list", "wb")
105   - lines = web_input['pkgs_file'].file.readlines()
106   - if lines[0].startswith('POPULARITY-CONTEST'):
107   - del lines[0]
108   - del lines[-1]
109   - package_name_field = 2
110   - else:
111   - package_name_field = 0
112   - for line in lines:
113   - self.pkgs_list.append(line.split()[package_name_field])
114   - f.write(line)
115   - f.close()
116   -
117   - if web_input.has_key('limit'):
118   - self.limit = int(web_input['limit'])
119   - if web_input.has_key('profile_size'):
120   - self.profile_size = int(web_input['profile_size'])
121   - if web_input.has_key('weight'):
122   - self.weight = web_input['weight'].encode('utf8')
123   -
124   - self.selected_strategies = []
125   - if web_input.has_key('strategy'):
126   - if (web_input['strategy'].encode('utf8') == "content" or
127   - web_input['strategy'].encode('utf8') == "hybrid"):
128   - if (web_input.has_key('tag') and web_input.has_key('desc')):
129   - self.selected_strategies.append("cb")
130   - elif web_input.has_key('desc'):
131   - self.selected_strategies.append("cbd")
132   - else:
133   - self.selected_strategies.append("cbt")
134   - if (web_input['strategy'].encode('utf8') == "collab" or
135   - web_input['strategy'].encode('utf8') == "hybrid"):
136   - self.selected_strategies.append("col")
137   -
138   - if web_input.has_key('cluster'):
139   - self.cluster = web_input['cluster'].encode('utf8')
140   -
141   - if web_input.has_key('neighbours'):
142   - self.neighbours = int(web_input['neighbours'])
143   -
144   - self.profiles_set = set()
145   - if web_input.has_key('profile_desktop'):
146   - self.profiles_set.add("desktop")
147   - if web_input.has_key('profile_admin'):
148   - self.profiles_set.add("admin")
149   - if web_input.has_key('profile_devel'):
150   - self.profiles_set.add("devel")
151   - if web_input.has_key('profile_science'):
152   - self.profiles_set.add("science")
153   - if web_input.has_key('profile_arts'):
154   - self.profiles_set.add("arts")
155   -
156   - def __str__(self):
157   - return self.storage
158   -
159   - def validates(self):
160   - self.errors = []
161   - if not self.pkgs_list:
162   - self.errors.append("No upload file or packages list was provided.")
163   - if not self.selected_strategies:
164   - self.errors.append("No strategy was selected.")
165   - if self.errors:
166   - return False
167   - return True
168   -
169   - def get_details(self):
170   - details = {}
171   - details['User id'] = self.user_id
172   - if len(self.pkgs_list)>10:
173   - details['Packages list'] = (" ".join(self.pkgs_list[:5])
174   - +" ... ("+str(len(self.pkgs_list)-10)
175   - +" more)")
176   - else:
177   - details['Packages list'] = " ".join(self.pkgs_list)
178   -
179   - if self.storage.has_key('pkgs_list') and self.storage['pkgs_file']:
180   - details['User profile'] = "from upload file and packages list"
181   - elif self.storage.has_key('pkgs_list'):
182   - details['User profile'] = "from packages list"
183   - else:
184   - details['User profile'] = "from upload file"
185   -
186   - if self.storage.has_key('limit'):
187   - details['Recommendation size'] = self.limit
188   - if self.storage.has_key('profile_size'):
189   - details['Profile size'] = self.profile_size
190   - if self.storage.has_key('weight'):
191   - if self.weight == "trad":
192   - details['weight'] = "traditional"
193   - else:
194   - details['weight'] = "BM25"
195   -
196   - if self.profiles_set:
197   - details['Personal profile'] = " ".join(list(self.profiles_set))
198   -
199   - if len(self.selected_strategies) > 1: details['strategy'] = "Hybrid"
200   - for strategy in self.selected_strategies:
201   - if strategy == "col":
202   - if not details.has_key('strategy'):
203   - details['Strategy'] = "Collaborative"
204   - if self.storage.has_key('cluster'):
205   - details['Cluster'] = self.cluster
206   - if self.storage.has_key('neighbours'):
207   - details['Neighbours'] = self.neighbours
208   - if not details.has_key('strategy'):
209   - details['Strategy'] = "Collaborative"
210   - else:
211   - if not details.has_key('strategy'):
212   - details['Strategy'] = "Content-based"
213   - if "cb" in self.selected_strategies:
214   - details['Content representation'] = "tags and descriptions"
215   - elif "cbt" in self.selected_strategies:
216   - details['Content representation'] = "packages tags"
217   - elif "cbd" in self.selected_strategies:
218   - details['Content representation'] = "packages descriptions"
219   -
220   - print details
221   - return details
222   -
223   -#class RandomRequest(Request):
224   -# def __init__(self):
225   -# pass
226   -# #self.storage = web.Storage()
  38 + def GET(self, pkg_name):
  39 + cfg = Config()
  40 + pkg = DebianPackage(pkg_name)
  41 + pkg.load_details()
  42 + return render_plain.package(pkg)
227 43  
228 44 class AppRecommender:
229 45 def __init__(self):
  46 + logging.info("Setting up AppRecommender...")
230 47 self.cfg = Config()
231 48 self.rec = Recommender(self.cfg)
  49 + self.requests_dir = "/var/www/AppRecommender/src/web/requests/"
  50 + if not os.path.exists(self.requests_dir):
  51 + os.makedirs(self.requests_dir)
232 52  
233 53 def POST(self):
234   - request = Request(web.input(pkgs_file={}))
235   - if not request.validates():
236   - return render.error(request.errors)
  54 + web_input = web.input(pkgs_file={})
  55 + user_dir = tempfile.mkdtemp(prefix='',dir=self.requests_dir)
  56 + user_id = user_dir.split("/")[-1]
  57 + uploaded_file = os.path.join(user_dir,"uploaded_file")
  58 + if web_input['pkgs_file'].value:
  59 + lines = web_input['pkgs_file'].file.readlines()
  60 + with open(uploaded_file, "w") as uploaded:
  61 + uploaded.writelines(lines)
  62 + with open(uploaded_file) as uploaded:
  63 + if uploaded.readline().startswith('POPULARITY-CONTEST'):
  64 + user = PopconSystem(uploaded_file,user_id)
  65 + else:
  66 + user = PkgsListSystem(uploaded_file,user_id)
  67 + if len(user.pkg_profile) < 10:
  68 + return render.error(["Could not extract profile from uploaded file. It must have at least 10 applications."],
  69 + "/","RECOMMENDATION")
237 70 else:
238   - recommendation = self._recommends(request)
239   - ### Getting package summary (short description) ###
240   - pkg_summaries = {}
241   - pkg_details = []
242   - cache = apt.Cache()
243   - for strategy, result in recommendation.items():
244   - for pkg in result:
245   - try:
246   - pkg_summaries[pkg] = cache[pkg].candidate.summary
247   - pkg_details.append(Package().get_details_from_dde(pkg))
248   - except:
249   - pkg_summaries[pkg] = ""
250   - if Config().survey_mode:
251   - return render.survey(recommendation, pkg_details,
252   - FeedbackForm(request.selected_strategies))
  71 + self.rec.set_strategy("knn_eset")
  72 + user.maximal_pkg_profile()
  73 + prediction = self.rec.get_recommendation(user,12).get_prediction()
  74 + logging.info("Prediction for user %s" % user.user_id)
  75 + logging.info(str(prediction))
  76 + recommendation = [result[0] for result in prediction]
  77 + pkgs_details = []
  78 + for pkg_name in recommendation:
  79 + logging.info("Getting details of package %s" % pkg_name)
  80 + pkg = DebianPackage(pkg_name)
  81 + pkg.load_summary()
  82 + pkgs_details.append(pkg)
  83 + if pkgs_details:
  84 + logging.info("Rendering recommendation...")
  85 + return render.apprec(pkgs_details)
253 86 else:
254   - return render.apprec(recommendation, pkg_summaries, request)
  87 + return render.error(["No recommendation produced for the uploaded file."],"/","RECOMMENDATION")
255 88  
256   - def _recommends(self,request):
257   - user = User(dict.fromkeys(request.pkgs_list,1),request.user_id)
258   - user.maximal_pkg_profile()
259   - results = dict()
260   - for strategy_str in request.selected_strategies:
261   - self.rec.set_strategy(strategy_str)
262   - prediction = self.rec.get_recommendation(user,10).get_prediction()
263   - print prediction
264   - results[strategy_str] = \
265   - [result[0] for result in prediction]
266   - return results
267 89  
268 90 # parsing json from screenshots - can be usefull in the future...
269 91 # def _packages_attrs(self, recommends): #recommends is result of _recommends()
... ... @@ -279,28 +101,33 @@ class AppRecommender:
279 101 # recommended_pkgs_attrs[pkg_attrs_dict['name']] = pkg_attrs_dict
280 102 # return recommended_pkgs_attrs
281 103  
282   -def add_global_hook():
283   - g = web.storage({"counter": "1"})
284   - def _wrapper(handler):
285   - web.ctx.globals = g
286   - return handler()
287   - return _wrapper
  104 +# to be used if it is not under apache
  105 +#def add_global_hook():
  106 +# g = web.storage({"counter": "1"})
  107 +# def _wrapper(handler):
  108 +# web.ctx.globals = g
  109 +# return handler()
  110 +# return _wrapper
288 111  
289   -render = web.template.render('templates/', base='layout')
290   -render_plain = web.template.render('templates/')
  112 +render = web.template.render('/var/www/AppRecommender/src/web/templates/', base='layout', globals={'hasattr':hasattr})
  113 +render_plain = web.template.render('/var/www/AppRecommender/src/web/templates/', globals={'hasattr':hasattr})
291 114  
292   -urls = ('/', 'Index',
293   - '/apprec', 'AppRecommender',
294   - '/thanks', 'Thanks',
  115 +urls = ('/', 'Index',
  116 + '/index', 'Index',
  117 + '/apprec', 'AppRecommender',
295 118 '/support', 'Support',
296 119 '/about', 'About',
297   - '/package/(.*)', 'Package'
  120 + '/package/(.*)', 'Package'
298 121 )
299 122  
300 123 web.webapi.internalerror = web.debugerror
301 124  
302   -if __name__ == "__main__":
303   - apprec = web.application(urls, globals())
304   - apprec.add_processor(add_global_hook())
305   - apprec.run()
  125 +#if __name__ == "__main__":
  126 +cfg = Config()
  127 +app = web.application(urls, globals(), autoreload=False)
  128 +application = app.wsgifunc()
  129 +
  130 +# apprec = web.application(urls, globals())
  131 +# apprec.add_processor(add_global_hook())
  132 +# apprec.run()
306 133  
... ...