Software Público Brasileiro

#!/usr/bin/env python

import os
import web
from web import form
import tempfile
import sys
import simplejson as json
import apt
import re
import urllib
import socket
import csv
import datetime
import stat

sys.path.insert(0,"/var/www/AppRecommender/src/")

import logging
from config import Config
from recommender import *
from user import *
from data import DebianPackage

# avoid "RuntimeError: maximum recursion depth exceeded"
sys.setrecursionlimit(50000)

class Index:
    def GET(self):
        return render.index_survey()

class About:
    def GET(self):
        return render.about_survey()

class Thanks:
    def POST(self):
        web_input = web.input()
        user_id = web_input['user_id'].encode('utf8')
        personal_file = open("/var/www/AppRecommender/src/web/submissions/%s/personal" % user_id,'w')
        personal = {}
        for key in ["name","email","user_habits","comments"]:
            if web_input.has_key(key):
                personal[key] = web_input[key].encode("utf-8")
            else:
                personal[key] = ""
        try:
            writer = csv.writer(personal_file)
            writer.writerow(("user","name","email","habits_id","comments"))
            writer.writerow((user_id,personal["name"],personal["email"],
                             personal["user_habits"],personal["comments"]))
        except:
            error_msg = "Could not save optional information."
            logging.critical("Could not save optional information.")
            return render.error([error_msg],"/survey/","START")
        finally:
            personal_file.close()
        return render.thanks_id()

class Fake:
    def GET(self):
        return render_plain.fake()

class Save:
    def POST(self):
        web_input = web.input()
        logging.info("Saving user evaluation...")
        logging.info(web_input)
        user_id = web_input['user_id'].encode('utf8')
        user_dir = "/var/www/AppRecommender/src/web/submissions/%s" % user_id
        strategy = web_input['strategy']
        strategy_dir = os.path.join(user_dir,strategy)
        logging.debug("Saving evaluation for user %s strategy %s."
                      % (user_id,strategy))
        summary = {}
        summary["poor"] = 0
        summary["redundant"] = 0
        summary["useful"] = 0
        summary["surprising"] = 0

        # Save evaluation
        try:
            prediction_file = open(os.path.join(strategy_dir,"prediction"),'r')
            evaluation_file = open(os.path.join(strategy_dir,"evaluation"),'w')
            reader = csv.DictReader(prediction_file)
            writer = csv.DictWriter(evaluation_file,fieldnames=reader.fieldnames)
            headers = dict( (n,n) for n in reader.fieldnames )
            writer.writerow(headers)
            for key, value in web_input.items():
                if key.startswith("evaluation-"):
                    ranking = key.lstrip("evaluation-")
                    for row in reader:
                        if row['ranking'] == ranking:
                            evaluation = value.encode('utf8')
                            row['evaluation'] = evaluation
                            writer.writerow(row)
                            summary[evaluation] += 1
                            break
                    prediction_file.seek(0)
            prediction_file.close()
            evaluation_file.close()
            os.remove(os.path.join(strategy_dir,"prediction"))
        except:
            error_msg = "Could not write evaluation to file."
            logging.critical(error_msg)
            if not os.path.exists(os.path.join(strategy_dir,"evaluation")):
                return render.error([error_msg], "/survey/","START")
        finally:
            with open(os.path.join(strategy_dir,"end"),'w') as end:
                end_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
                end.write(end_time)

        # Save report
        try:
            report = os.path.join(user_dir,"report")
            report_file = open(os.path.join(user_dir,"report"),'a')
            writer = csv.writer(report_file)
            if os.path.getsize(report) == 0:
                fieldnames = ('user','strategy','start','end','poor',
                              'redundant','useful','surprising','comments')
                writer.writerow(fieldnames)
            with open(os.path.join(strategy_dir,"start"),'r') as start:
                start_time = start.readline().strip()
            if web_input.has_key("comments"):
                comments =  web_input['comments'].encode("utf-8")
            else:
                comments = ""
            writer.writerow((user_id,strategy,start_time,end_time,summary["poor"],
                             summary["redundant"],summary["useful"],
                             summary["surprising"],comments))
            report_file.close()
        except:
            error_msg = "Could not save evaluation report."
            logging.critical(error_msg)
            if not os.path.exists(os.path.join(user_dir,"report")):
                return render.error([error_msg], "/survey/","START")

        if web_input.has_key('continue_button'):
            return Survey().POST()
        elif web_input.has_key('finish_button'):
            return render.thanks(user_id)
        else:
            return render.index_survey()

class Instruction:
    def POST(self):
        web_input = web.input(pkgs_file={})
        submissions_dir = "/var/www/AppRecommender/src/web/submissions/"
        user_dir = tempfile.mkdtemp(prefix='',dir=submissions_dir)
        os.chmod(user_dir,stat.S_IRWXU|stat.S_IXOTH|stat.S_IROTH)
        user_id = user_dir.split("/")[-1]
        logging.info("Request from user %s" % user_id)
        logging.debug("Created dir %s" % user_dir)
        uploaded_file = os.path.join(user_dir,"uploaded_file")
        if not os.path.exists(uploaded_file):
            if web_input['pkgs_file'].value:
                lines = web_input['pkgs_file'].file.readlines()
                with open(uploaded_file, "w") as uploaded:
                    uploaded.writelines(lines)
            else:
                # saving empty files
                with open(uploaded_file, "w") as uploaded:
                    pass
        return render.intro(user_id)

class Survey:
    def __init__(self):
        logging.info("Setting up survey...")
        self.cfg = Config()
        self.rec = Recommender(self.cfg)
        self.submissions_dir = "/var/www/AppRecommender/src/web/submissions/"
        if not os.path.exists(self.submissions_dir):
            os.makedirs(self.submissions_dir)
        self.strategies = ["cbh","cbh_eset",
                           "knn","knn_eset","knn_plus",
                           "knnco"]
    def POST(self):
        web_input = web.input(pkgs_file={})
        if web_input.has_key('user_id'):
            user_id = web_input['user_id'].encode('utf8')
            user_dir = os.path.join(self.submissions_dir, user_id)
            logging.info("New recommendation for user %s" % user_id)
        
        uploaded_file = os.path.join(user_dir,"uploaded_file")
        with open(uploaded_file) as uploaded:
            if uploaded.readline().startswith('POPULARITY-CONTEST'):
                user = PopconSystem(uploaded_file,user_id)
            else:
                user = PkgsListSystem(uploaded_file,user_id)
        user.maximal_pkg_profile()
        if len(user.pkg_profile)<10:
            error_msg = "Could not extract profile from uploaded file. It must have at least 10 applications."
            logging.critical(error_msg)
            return render.error([error_msg], "/survey/","START")
        else:
            # Check the remaining strategies and select a new one
            old_strategies = [dirs for root, dirs, files in
                              os.walk(os.path.join(self.submissions_dir,
                                                   user_id))]
            if old_strategies:
                strategies = [s for s in self.strategies if s not in old_strategies[0]]
                logging.info("Already used strategies %s" % old_strategies[0])
            else:
                strategies = self.strategies
            if not strategies:
                return render.thanks(user_id)
            selected_strategy = random.choice(strategies)
            logging.info("Selected \'%s\' from %s" % (selected_strategy,strategies))
            self.set_rec_strategy(selected_strategy)
            prediction = self.rec.get_recommendation(user,10).get_prediction()
            logging.info("Prediction for user %s" % user_id)
            logging.info(str(prediction))
            self.save_prediction(user_id,selected_strategy,prediction)

            # Load packages details
            recommendation = [result[0] for result in prediction]
            pkgs_details = []
            for pkg_name in recommendation:
                logging.info("Getting details of package %s" % pkg_name)
                pkg = DebianPackage(pkg_name)
                pkg.load_details()
                pkgs_details.append(pkg)

            if pkgs_details:
                logging.info("Rendering survey slide...")
                return render.survey(pkgs_details, user_id, selected_strategy, len(strategies))
            else:
                return render.error(["No recommendation produced for the uploaded file."],"/survey/","START")

    def set_rec_strategy(self,selected_strategy):
        k=10
        n=20
        if selected_strategy == "cbh":
            pass
        if selected_strategy == "cbh_eset":
            pass
        if selected_strategy == "knn":
            pass
        if selected_strategy == "knn_eset":
            pass
        if selected_strategy == "knn_plus":
            pass
        if selected_strategy == "knnco":
            pass
        self.rec.set_strategy(selected_strategy,k,n)
        return selected_strategy

    def save_prediction(self,user_id,strategy,prediction):
        strategy_dir = os.path.join(self.submissions_dir,user_id,strategy)
        if not os.path.exists(strategy_dir):
            os.makedirs(strategy_dir)
        ranking = 0
        prediction_file = open(os.path.join(strategy_dir,"prediction"),"w")
        try:
            writer = csv.writer(prediction_file)
            fieldnames = ('ranking','rating','package','evaluation')
            writer.writerow(fieldnames)
            for pkg,rating in prediction:
                writer.writerow((ranking,"%.4f"%rating,pkg,""))
                ranking += 1
        except:
            error_msg = "Error to write prediction to file."
            logging.critical(error_msg)
            return render.error([error_msg], "/survey/","START")
        finally:
            prediction_file.close()
        with open(os.path.join(strategy_dir,"start"),'w') as start:
            now = datetime.datetime.now()
            start.write(now.strftime("%Y%m%d%H%M%S"))
        logging.debug("Saved prediction to file at %s/%s" %
                      (user_id,strategy))
#def add_global_hook():
#    g = web.storage({"counter": "1"})
#    def _wrapper(handler):
#        web.ctx.globals = g
#        return handler()
#    return _wrapper

render = web.template.render('/var/www/AppRecommender/src/web/templates/', base='layout', globals={'hasattr':hasattr})
render_plain = web.template.render('/var/www/AppRecommender/src/web/templates/', globals={'hasattr':hasattr})

urls = ('/apprec',         'Instruction',
        '/evaluation',     'Survey',
        '/thanks',   	   'Thanks',
        '/save',   	   'Save',
        '/about',          'About',
        '/index',          'Index',
        '/', 	           'Index',
#        '/', 	           'Fake',
       )

web.webapi.internalerror = web.debugerror

cfg = Config()
app = web.application(urls, globals(), autoreload=False)
application = app.wsgifunc()