Download as
Browse Code »

Commit ca3ad95878bbeb76f00b3a39cd52ee60a7fba2e7

Authored by Tássia Camões Araújo
1 parent b6c8c0a6
Exists in master and in 1 other branch add_vagrant

AppRecommender server refactoring.

Showing 1 changed file with 73 additions and 246 deletions   Show diff stats
src/web/server.py
  Diff comments   View file @ca3ad95
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 2
  3 +import os
3 import web 4 import web
4 from web import form 5 from web import form
5 import tempfile 6 import tempfile
@@ -7,263 +8,84 @@ import sys @@ -7,263 +8,84 @@ import sys
7 import simplejson as json 8 import simplejson as json
8 import apt 9 import apt
9 import re 10 import re
  11 +import socket
10 12
11 -sys.path.insert(0,"../") 13 +sys.path.insert(0,"/var/www/AppRecommender/src/")
12 14
13 -from config import * 15 +from config import Config
14 from recommender import * 16 from recommender import *
15 from user import * 17 from user import *
  18 +from data import DebianPackage
16 19
17 import urllib 20 import urllib
18 21
19 -#class FeedbackForm(form.Form):  
20 -# def __init__(self,selected_strategies):  
21 -# desc_dict = {"cb": "Content-based", "cbt": "Content-based",  
22 -# "cbd": "Content-based", "col": "Collaborative",  
23 -# "hybrid": "Hybrib"}  
24 -# fields = []  
25 -# for strategy in selected_strategies:  
26 -# fields.append(form.Radio(desc_dict[strategy],  
27 -# [('1','1 '),('2','2 '),('3','3'),('4','4 '),('5','5')]))  
28 -# form.Form.__init__(self, *fields, validators = []) 22 +# avoid "RuntimeError: maximum recursion depth exceeded"
  23 +sys.setrecursionlimit(50000)
  24 +
  25 +class Fake:
  26 + def GET(self):
  27 + return render_plain.fake()
29 28
30 class Index: 29 class Index:
31 def GET(self): 30 def GET(self):
32 - if Config().survey_mode:  
33 - return render.survey_index()  
34 - else:  
35 return render.index() 31 return render.index()
36 32
37 class About: 33 class About:
38 def GET(self): 34 def GET(self):
39 return render.about() 35 return render.about()
40 36
41 -#class Support:  
42 -# def GET(self):  
43 -# return render.support()  
44 -  
45 -class Thanks:  
46 - def POST(self):  
47 - return render.thanks()  
48 -  
49 class Package: 37 class Package:
50 - def GET(self, pkg):  
51 - result = self.get_details_from_dde(pkg)  
52 - return render_plain.package(result)  
53 -  
54 - def get_details_from_dde(self, pkg):  
55 - json_source = Config().dde_url % pkg  
56 - json_data = json.load(urllib.urlopen(json_source))  
57 - # parse tags  
58 - tags = self._debtags_list_to_dict(json_data['r']['tag'])  
59 - json_data['r']['tag'] = tags  
60 - # format long description  
61 - json_data['r']['long_description'] = json_data['r']['long_description'].replace(' .\n','').replace('\n','<br />')  
62 - return json_data['r']  
63 -  
64 - def _debtags_list_to_dict(self, debtags_list):  
65 - """ in:  
66 - ['use::editing',  
67 - 'works-with-format::gif',  
68 - 'works-with-format::jpg',  
69 - 'works-with-format::pdf']  
70 - out:  
71 - {'use': [editing],  
72 - 'works-with-format': ['gif', 'jpg', 'pdf']'  
73 - }  
74 - """  
75 - debtags = {}  
76 - subtags = []  
77 - for tag in debtags_list:  
78 - match = re.search(r'^(.*)::(.*)$', tag)  
79 - if not match:  
80 - log.error("Could not parse debtags format from tag: %s", tag)  
81 - facet, subtag = match.groups()  
82 - subtags.append(subtag)  
83 - if facet not in debtags:  
84 - debtags[facet] = subtags  
85 - else:  
86 - debtags[facet].append(subtag)  
87 - subtags = []  
88 - return debtags  
89 -  
90 -class Request:  
91 - def __init__(self,web_input):  
92 - submissions_dir = "./submissions/"  
93 - if not os.path.exists(submissions_dir):  
94 - os.makedirs(submissions_dir)  
95 - outputdir = tempfile.mkdtemp(prefix='',dir=submissions_dir)  
96 - user_id = outputdir.lstrip(submissions_dir)  
97 - self.user_id = user_id  
98 - self.storage = web_input  
99 -  
100 - self.pkgs_list = []  
101 - if web_input.has_key('pkgs_list'):  
102 - self.pkgs_list = web_input['pkgs_list'].encode('utf8').split()  
103 - if web_input['pkgs_file'].value:  
104 - f = open(outputdir + "/packages_list", "wb")  
105 - lines = web_input['pkgs_file'].file.readlines()  
106 - if lines[0].startswith('POPULARITY-CONTEST'):  
107 - del lines[0]  
108 - del lines[-1]  
109 - package_name_field = 2  
110 - else:  
111 - package_name_field = 0  
112 - for line in lines:  
113 - self.pkgs_list.append(line.split()[package_name_field])  
114 - f.write(line)  
115 - f.close()  
116 -  
117 - if web_input.has_key('limit'):  
118 - self.limit = int(web_input['limit'])  
119 - if web_input.has_key('profile_size'):  
120 - self.profile_size = int(web_input['profile_size'])  
121 - if web_input.has_key('weight'):  
122 - self.weight = web_input['weight'].encode('utf8')  
123 -  
124 - self.selected_strategies = []  
125 - if web_input.has_key('strategy'):  
126 - if (web_input['strategy'].encode('utf8') == "content" or  
127 - web_input['strategy'].encode('utf8') == "hybrid"):  
128 - if (web_input.has_key('tag') and web_input.has_key('desc')):  
129 - self.selected_strategies.append("cb")  
130 - elif web_input.has_key('desc'):  
131 - self.selected_strategies.append("cbd")  
132 - else:  
133 - self.selected_strategies.append("cbt")  
134 - if (web_input['strategy'].encode('utf8') == "collab" or  
135 - web_input['strategy'].encode('utf8') == "hybrid"):  
136 - self.selected_strategies.append("col")  
137 -  
138 - if web_input.has_key('cluster'):  
139 - self.cluster = web_input['cluster'].encode('utf8')  
140 -  
141 - if web_input.has_key('neighbours'):  
142 - self.neighbours = int(web_input['neighbours'])  
143 -  
144 - self.profiles_set = set()  
145 - if web_input.has_key('profile_desktop'):  
146 - self.profiles_set.add("desktop")  
147 - if web_input.has_key('profile_admin'):  
148 - self.profiles_set.add("admin")  
149 - if web_input.has_key('profile_devel'):  
150 - self.profiles_set.add("devel")  
151 - if web_input.has_key('profile_science'):  
152 - self.profiles_set.add("science")  
153 - if web_input.has_key('profile_arts'):  
154 - self.profiles_set.add("arts")  
155 -  
156 - def __str__(self):  
157 - return self.storage  
158 -  
159 - def validates(self):  
160 - self.errors = []  
161 - if not self.pkgs_list:  
162 - self.errors.append("No upload file or packages list was provided.")  
163 - if not self.selected_strategies:  
164 - self.errors.append("No strategy was selected.")  
165 - if self.errors:  
166 - return False  
167 - return True  
168 -  
169 - def get_details(self):  
170 - details = {}  
171 - details['User id'] = self.user_id  
172 - if len(self.pkgs_list)>10:  
173 - details['Packages list'] = (" ".join(self.pkgs_list[:5])  
174 - +" ... ("+str(len(self.pkgs_list)-10)  
175 - +" more)")  
176 - else:  
177 - details['Packages list'] = " ".join(self.pkgs_list)  
178 -  
179 - if self.storage.has_key('pkgs_list') and self.storage['pkgs_file']:  
180 - details['User profile'] = "from upload file and packages list"  
181 - elif self.storage.has_key('pkgs_list'):  
182 - details['User profile'] = "from packages list"  
183 - else:  
184 - details['User profile'] = "from upload file"  
185 -  
186 - if self.storage.has_key('limit'):  
187 - details['Recommendation size'] = self.limit  
188 - if self.storage.has_key('profile_size'):  
189 - details['Profile size'] = self.profile_size  
190 - if self.storage.has_key('weight'):  
191 - if self.weight == "trad":  
192 - details['weight'] = "traditional"  
193 - else:  
194 - details['weight'] = "BM25"  
195 -  
196 - if self.profiles_set:  
197 - details['Personal profile'] = " ".join(list(self.profiles_set))  
198 -  
199 - if len(self.selected_strategies) > 1: details['strategy'] = "Hybrid"  
200 - for strategy in self.selected_strategies:  
201 - if strategy == "col":  
202 - if not details.has_key('strategy'):  
203 - details['Strategy'] = "Collaborative"  
204 - if self.storage.has_key('cluster'):  
205 - details['Cluster'] = self.cluster  
206 - if self.storage.has_key('neighbours'):  
207 - details['Neighbours'] = self.neighbours  
208 - if not details.has_key('strategy'):  
209 - details['Strategy'] = "Collaborative"  
210 - else:  
211 - if not details.has_key('strategy'):  
212 - details['Strategy'] = "Content-based"  
213 - if "cb" in self.selected_strategies:  
214 - details['Content representation'] = "tags and descriptions"  
215 - elif "cbt" in self.selected_strategies:  
216 - details['Content representation'] = "packages tags"  
217 - elif "cbd" in self.selected_strategies:  
218 - details['Content representation'] = "packages descriptions"  
219 -  
220 - print details  
221 - return details  
222 -  
223 -#class RandomRequest(Request):  
224 -# def __init__(self):  
225 -# pass  
226 -# #self.storage = web.Storage() 38 + def GET(self, pkg_name):
  39 + cfg = Config()
  40 + pkg = DebianPackage(pkg_name)
  41 + pkg.load_details()
  42 + return render_plain.package(pkg)
227 43
228 class AppRecommender: 44 class AppRecommender:
229 def __init__(self): 45 def __init__(self):
  46 + logging.info("Setting up AppRecommender...")
230 self.cfg = Config() 47 self.cfg = Config()
231 self.rec = Recommender(self.cfg) 48 self.rec = Recommender(self.cfg)
  49 + self.requests_dir = "/var/www/AppRecommender/src/web/requests/"
  50 + if not os.path.exists(self.requests_dir):
  51 + os.makedirs(self.requests_dir)
232 52
233 def POST(self): 53 def POST(self):
234 - request = Request(web.input(pkgs_file={}))  
235 - if not request.validates():  
236 - return render.error(request.errors) 54 + web_input = web.input(pkgs_file={})
  55 + user_dir = tempfile.mkdtemp(prefix='',dir=self.requests_dir)
  56 + user_id = user_dir.split("/")[-1]
  57 + uploaded_file = os.path.join(user_dir,"uploaded_file")
  58 + if web_input['pkgs_file'].value:
  59 + lines = web_input['pkgs_file'].file.readlines()
  60 + with open(uploaded_file, "w") as uploaded:
  61 + uploaded.writelines(lines)
  62 + with open(uploaded_file) as uploaded:
  63 + if uploaded.readline().startswith('POPULARITY-CONTEST'):
  64 + user = PopconSystem(uploaded_file,user_id)
  65 + else:
  66 + user = PkgsListSystem(uploaded_file,user_id)
  67 + if len(user.pkg_profile) < 10:
  68 + return render.error(["Could not extract profile from uploaded file. It must have at least 10 applications."],
  69 + "/","RECOMMENDATION")
237 else: 70 else:
238 - recommendation = self._recommends(request)  
239 - ### Getting package summary (short description) ###  
240 - pkg_summaries = {}  
241 - pkg_details = []  
242 - cache = apt.Cache()  
243 - for strategy, result in recommendation.items():  
244 - for pkg in result:  
245 - try:  
246 - pkg_summaries[pkg] = cache[pkg].candidate.summary  
247 - pkg_details.append(Package().get_details_from_dde(pkg))  
248 - except:  
249 - pkg_summaries[pkg] = ""  
250 - if Config().survey_mode:  
251 - return render.survey(recommendation, pkg_details,  
252 - FeedbackForm(request.selected_strategies)) 71 + self.rec.set_strategy("knn_eset")
  72 + user.maximal_pkg_profile()
  73 + prediction = self.rec.get_recommendation(user,12).get_prediction()
  74 + logging.info("Prediction for user %s" % user.user_id)
  75 + logging.info(str(prediction))
  76 + recommendation = [result[0] for result in prediction]
  77 + pkgs_details = []
  78 + for pkg_name in recommendation:
  79 + logging.info("Getting details of package %s" % pkg_name)
  80 + pkg = DebianPackage(pkg_name)
  81 + pkg.load_summary()
  82 + pkgs_details.append(pkg)
  83 + if pkgs_details:
  84 + logging.info("Rendering recommendation...")
  85 + return render.apprec(pkgs_details)
253 else: 86 else:
254 - return render.apprec(recommendation, pkg_summaries, request) 87 + return render.error(["No recommendation produced for the uploaded file."],"/","RECOMMENDATION")
255 88
256 - def _recommends(self,request):  
257 - user = User(dict.fromkeys(request.pkgs_list,1),request.user_id)  
258 - user.maximal_pkg_profile()  
259 - results = dict()  
260 - for strategy_str in request.selected_strategies:  
261 - self.rec.set_strategy(strategy_str)  
262 - prediction = self.rec.get_recommendation(user,10).get_prediction()  
263 - print prediction  
264 - results[strategy_str] = \  
265 - [result[0] for result in prediction]  
266 - return results  
267 89
268 # parsing json from screenshots - can be usefull in the future... 90 # parsing json from screenshots - can be usefull in the future...
269 # def _packages_attrs(self, recommends): #recommends is result of _recommends() 91 # def _packages_attrs(self, recommends): #recommends is result of _recommends()
@@ -279,28 +101,33 @@ class AppRecommender: @@ -279,28 +101,33 @@ class AppRecommender:
279 # recommended_pkgs_attrs[pkg_attrs_dict['name']] = pkg_attrs_dict 101 # recommended_pkgs_attrs[pkg_attrs_dict['name']] = pkg_attrs_dict
280 # return recommended_pkgs_attrs 102 # return recommended_pkgs_attrs
281 103
282 -def add_global_hook():  
283 - g = web.storage({"counter": "1"})  
284 - def _wrapper(handler):  
285 - web.ctx.globals = g  
286 - return handler()  
287 - return _wrapper 104 +# to be used if it is not under apache
  105 +#def add_global_hook():
  106 +# g = web.storage({"counter": "1"})
  107 +# def _wrapper(handler):
  108 +# web.ctx.globals = g
  109 +# return handler()
  110 +# return _wrapper
288 111
289 -render = web.template.render('templates/', base='layout')  
290 -render_plain = web.template.render('templates/') 112 +render = web.template.render('/var/www/AppRecommender/src/web/templates/', base='layout', globals={'hasattr':hasattr})
  113 +render_plain = web.template.render('/var/www/AppRecommender/src/web/templates/', globals={'hasattr':hasattr})
291 114
292 -urls = ('/', 'Index',  
293 - '/apprec', 'AppRecommender',  
294 - '/thanks', 'Thanks', 115 +urls = ('/', 'Index',
  116 + '/index', 'Index',
  117 + '/apprec', 'AppRecommender',
295 '/support', 'Support', 118 '/support', 'Support',
296 '/about', 'About', 119 '/about', 'About',
297 - '/package/(.*)', 'Package' 120 + '/package/(.*)', 'Package'
298 ) 121 )
299 122
300 web.webapi.internalerror = web.debugerror 123 web.webapi.internalerror = web.debugerror
301 124
302 -if __name__ == "__main__":  
303 - apprec = web.application(urls, globals())  
304 - apprec.add_processor(add_global_hook())  
305 - apprec.run() 125 +#if __name__ == "__main__":
  126 +cfg = Config()
  127 +app = web.application(urls, globals(), autoreload=False)
  128 +application = app.wsgifunc()
  129 +
  130 +# apprec = web.application(urls, globals())
  131 +# apprec.add_processor(add_global_hook())
  132 +# apprec.run()
306 133