Download as
Browse Code »

Commit e2be2c33b1e481ae26daffa35f532e6e4c7dd336

Authored by Tássia Camões Araújo
1 parent 0c315647
Exists in master and in 1 other branch add_vagrant

Up-to-date roc testes.

Showing 2 changed files with 509 additions and 0 deletions   Show diff stats
src/experiments/roc-sample.py 0 → 100755
  Diff comments   View file @e2be2c3
... ... @@ -0,0 +1,240 @@
  1 +#!/usr/bin/env python
  2 +"""
  3 + recommender suite - recommender experiments suite
  4 +"""
  5 +__author__ = "Tassia Camoes Araujo <tassia@gmail.com>"
  6 +__copyright__ = "Copyright (C) 2011 Tassia Camoes Araujo"
  7 +__license__ = """
  8 + This program is free software: you can redistribute it and/or modify
  9 + it under the terms of the GNU General Public License as published by
  10 + the Free Software Foundation, either version 3 of the License, or
  11 + (at your option) any later version.
  12 +
  13 + This program is distributed in the hope that it will be useful,
  14 + but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16 + GNU General Public License for more details.
  17 +
  18 + You should have received a copy of the GNU General Public License
  19 + along with this program. If not, see <http://www.gnu.org/licenses/>.
  20 +"""
  21 +
  22 +import sys
  23 +sys.path.insert(0,'../')
  24 +from config import Config
  25 +from data import PopconXapianIndex, PopconSubmission
  26 +from recommender import Recommender
  27 +from user import LocalSystem, User
  28 +from evaluation import *
  29 +import logging
  30 +import random
  31 +import Gnuplot
  32 +import numpy
  33 +import shutil
  34 +
  35 +def plot_roc(results,log_file,mean=0):
  36 + g = Gnuplot.Gnuplot()
  37 + g('set style data lines')
  38 + g.xlabel('False Positive Rate')
  39 + g.ylabel('True Positive Rate')
  40 + g('set xrange [0:1.0]')
  41 + g('set yrange [0:1.0]')
  42 + g.title("Setup: %s" % log_file.split("/")[-1])
  43 + g('set label "C %.4f" at 0.68,0.2' % results.coverage())
  44 + g('set label "AUC %.4f" at 0.68,0.15' % results.get_auc())
  45 + g('set label "P(10) %.2f +- %.2f" at 0.68,0.10' % (numpy.mean(results.precision[10]),numpy.std(results.precision[10])))
  46 + g('set label "F05(100) %.2f +- %.2f" at 0.68,0.05' % (numpy.mean(results.f05[100]),numpy.std(results.f05[100])))
  47 + if mean==1:
  48 + g.plot(Gnuplot.Data(results.get_roc_points(),title="mean ROC"),
  49 + Gnuplot.Data([[0,0],[1,1]],with_="lines lt 7"))
  50 + g.hardcopy(log_file+"-roc-mean.png",terminal="png")
  51 + g.hardcopy(log_file+"-roc-mean.ps",terminal="postscript",enhanced=1,color=1)
  52 + else:
  53 + g.plot(Gnuplot.Data(results.get_roc_points(),title="ROC",with_="xyerrorbars"),
  54 + Gnuplot.Data([[0,0],[1,1]],with_="lines lt 7"))
  55 + g.hardcopy(log_file+"-roc.png",terminal="png")
  56 + g.hardcopy(log_file+"-roc.ps",terminal="postscript",enhanced=1,color=1)
  57 +
  58 +def get_label(cfg):
  59 + label = {}
  60 + if cfg.strategy in content_based:
  61 + label["description"] = "strategy-profile"
  62 + label["values"] = ("%s-profile%.3d" %
  63 + (cfg.strategy,cfg.profile_size))
  64 + elif cfg.strategy in collaborative:
  65 + label["description"] = "strategy-knn"
  66 + label["values"] = ("%s-k%.3d" %
  67 + (cfg.strategy,cfg.k_neighbors))
  68 + elif cfg.strategy in hybrid:
  69 + label["description"] = "strategy-knn-profile"
  70 + label["values"] = ("%s-k%.3d-profile%.3d" %
  71 + (cfg.strategy,cfg.k_neighbors,cfg.profile_size))
  72 + return label
  73 +
  74 +class ExperimentResults:
  75 + def __init__(self,repo_size):
  76 + self.repository_size = repo_size
  77 + self.precision = {}
  78 + self.recall = {}
  79 + self.fpr = {}
  80 + self.f05 = {}
  81 + self.recommended = {}
  82 + self.thresholds = [1]+range(10,self.repository_size,10)
  83 + for size in self.thresholds:
  84 + self.precision[size] = []
  85 + self.recall[size] = []
  86 + self.fpr[size] = []
  87 + self.f05[size] = []
  88 + self.recommended[size] = set()
  89 +
  90 + def add_result(self,ranking,sample):
  91 + for size in self.thresholds:
  92 + recommendation = ranking[:size]
  93 + self.recommended[size] = self.recommended[size].union(recommendation)
  94 + predicted = RecommendationResult(dict.fromkeys(recommendation,1))
  95 + real = RecommendationResult(sample)
  96 + evaluation = Evaluation(predicted,real,self.repository_size)
  97 + self.precision[size].append(evaluation.run(Precision()))
  98 + self.recall[size].append(evaluation.run(Recall()))
  99 + self.f05[size].append(evaluation.run(F_score(0.5)))
  100 + self.fpr[size].append(evaluation.run(FPR()))
  101 +
  102 + def precision_summary(self):
  103 + return [[size,numpy.mean(self.precision[size])] for size in self.thresholds]
  104 +
  105 + def recall_summary(self):
  106 + return [[size,numpy.mean(self.recall[size])] for size in self.thresholds]
  107 +
  108 + def f05_summary(self):
  109 + return [[size,numpy.mean(self.f05[size])] for size in self.thresholds]
  110 +
  111 + def coverage_summary(self):
  112 + return [[size,self.coverage(size)] for size in self.thresholds]
  113 +
  114 + def coverage(self,size=0):
  115 + if not size:
  116 + size = self.thresholds[-1]
  117 + return len(self.recommended[size])/float(self.repository_size)
  118 +
  119 + def precision(self,size):
  120 + return numpy.mean(results.precision[size])
  121 +
  122 + def get_auc(self):
  123 + roc_points = self.get_roc_points()
  124 + x_roc = [p[0] for p in roc_points]
  125 + y_roc = [p[1] for p in roc_points]
  126 + x_roc.insert(0,0)
  127 + y_roc.insert(0,0)
  128 + x_roc.append(1)
  129 + y_roc.append(1)
  130 + return numpy.trapz(y=y_roc, x=x_roc)
  131 +
  132 + # Average ROC by threshold (= size of recommendation)
  133 + def get_roc_points(self):
  134 + points = []
  135 + for size in self.recall.keys():
  136 + tpr = self.recall[size]
  137 + fpr = self.fpr[size]
  138 + points.append([numpy.mean(fpr),numpy.mean(tpr),numpy.std(fpr),numpy.std(tpr)])
  139 + return sorted(points)
  140 +
  141 +def run_strategy(cfg,sample_file):
  142 + rec = Recommender(cfg)
  143 + repo_size = rec.items_repository.get_doccount()
  144 + results = ExperimentResults(repo_size)
  145 + label = get_label(cfg)
  146 + population_sample = []
  147 + sample_str = sample_file.split('/')[-1]
  148 + with open(sample_file,'r') as f:
  149 + for line in f.readlines():
  150 + user_id = line.strip('\n')
  151 + population_sample.append(os.path.join(cfg.popcon_dir,user_id[:2],user_id))
  152 + sample_dir = ("results/roc-sample/%s" % sample_str)
  153 + if not os.path.exists(sample_dir):
  154 + os.makedirs(sample_dir)
  155 + log_file = os.path.join(sample_dir,label["values"])
  156 +
  157 + # n iterations per population user
  158 + for submission_file in population_sample:
  159 + user = PopconSystem(submission_file)
  160 + user.filter_pkg_profile(cfg.pkgs_filter)
  161 + user.maximal_pkg_profile()
  162 + for n in range(iterations):
  163 + # Fill sample profile
  164 + profile_len = len(user.pkg_profile)
  165 + item_score = {}
  166 + for pkg in user.pkg_profile:
  167 + item_score[pkg] = user.item_score[pkg]
  168 + sample = {}
  169 + sample_size = int(profile_len*0.9)
  170 + for i in range(sample_size):
  171 + key = random.choice(item_score.keys())
  172 + sample[key] = item_score.pop(key)
  173 + iteration_user = User(item_score)
  174 + recommendation = rec.get_recommendation(iteration_user,repo_size)
  175 + if hasattr(recommendation,"ranking"):
  176 + results.add_result(recommendation.ranking,sample)
  177 +
  178 + plot_roc(results,log_file)
  179 + plot_roc(results,log_file,1)
  180 + with open(log_file+"-roc.jpg.comment",'w') as f:
  181 + f.write("# %s\n# %s\n\n" %
  182 + (label["description"],label["values"]))
  183 + f.write("# roc AUC\n%.4f\n\n"%results.get_auc())
  184 + f.write("# threshold\tmean_fpr\tdev_fpr\t\tmean_tpr\tdev_tpr\t\tcoverage\n")
  185 + for size in results.thresholds:
  186 + f.write("%4d\t\t%.4f\t\t%.4f\t\t%.4f\t\t%.4f\t\t%.4f\n" %
  187 + (size,numpy.mean(results.fpr[size]),
  188 + numpy.std(results.fpr[size]),
  189 + numpy.mean(results.recall[size]),
  190 + numpy.std(results.recall[size]),
  191 + numpy.mean(results.coverage(size))))
  192 +
  193 +def run_content(cfg,sample_file):
  194 + for size in profile_size:
  195 + cfg.profile_size = size
  196 + run_strategy(cfg,sample_file)
  197 +
  198 +def run_collaborative(cfg,sample_file):
  199 + for k in neighbors:
  200 + cfg.k_neighbors = k
  201 + run_strategy(cfg,sample_file)
  202 +
  203 +def run_hybrid(cfg,sample_file):
  204 + for k in neighbors:
  205 + cfg.k_neighbors = k
  206 + for size in profile_size:
  207 + cfg.profile_size = size
  208 + run_strategy(cfg,sample_file)
  209 +
  210 +if __name__ == '__main__':
  211 + if len(sys.argv)<2:
  212 + print "Usage: sample-roc strategy_str [popcon_sample_path]"
  213 + exit(1)
  214 +
  215 + #iterations = 3
  216 + #content_based = ['cb']
  217 + #collaborative = ['knn_eset']
  218 + #hybrid = ['knnco']
  219 + #profile_size = [50,100]
  220 + #neighbors = [50]
  221 + iterations = 20
  222 + content_based = ['cb','cbt','cbd','cbh','cb_eset','cbt_eset','cbd_eset','cbh_eset']
  223 + collaborative = ['knn_eset','knn','knn_plus']
  224 + hybrid = ['knnco','knnco_eset']
  225 + profile_size = [10,20,50,100,200]
  226 + neighbors = [200]
  227 + #neighbors = [3,10,50,100,200]
  228 + #profile_size = [10,20,40,60,80,100,140,170,200,240]
  229 + #neighbors = [3,5,10,20,30,50,70,100,150,200]
  230 +
  231 + cfg = Config()
  232 + cfg.strategy = sys.argv[1]
  233 + sample_file = sys.argv[2]
  234 +
  235 + if cfg.strategy in content_based:
  236 + run_content(cfg,sample_file)
  237 + if cfg.strategy in collaborative:
  238 + run_collaborative(cfg,sample_file)
  239 + if cfg.strategy in hybrid:
  240 + run_hybrid(cfg,sample_file)
... ...
src/experiments/roc-single.py 0 → 100755
  Diff comments   View file @e2be2c3
... ... @@ -0,0 +1,269 @@
  1 +#!/usr/bin/env python
  2 +"""
  3 + recommender suite - recommender experiments suite
  4 +"""
  5 +__author__ = "Tassia Camoes Araujo <tassia@gmail.com>"
  6 +__copyright__ = "Copyright (C) 2011 Tassia Camoes Araujo"
  7 +__license__ = """
  8 + This program is free software: you can redistribute it and/or modify
  9 + it under the terms of the GNU General Public License as published by
  10 + the Free Software Foundation, either version 3 of the License, or
  11 + (at your option) any later version.
  12 +
  13 + This program is distributed in the hope that it will be useful,
  14 + but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16 + GNU General Public License for more details.
  17 +
  18 + You should have received a copy of the GNU General Public License
  19 + along with this program. If not, see <http://www.gnu.org/licenses/>.
  20 +"""
  21 +
  22 +import sys
  23 +sys.path.insert(0,'../')
  24 +from config import Config
  25 +from data import PopconXapianIndex, PopconSubmission
  26 +from recommender import Recommender
  27 +from user import LocalSystem, User
  28 +from evaluation import *
  29 +import logging
  30 +import random
  31 +import Gnuplot
  32 +import numpy
  33 +import shutil
  34 +
  35 +def write_recall_log(label,n,sample,recommendation,profile_size,repo_size,log_file):
  36 + # Write recall log
  37 + output = open(("%s-%.2d" % (log_file,n)),'w')
  38 + output.write("# %s-n\n" % label["description"])
  39 + output.write("# %s-%.2d\n" % (label["values"],n))
  40 + output.write("\n# repository profile sample\n%d %d %d\n" % \
  41 + (repo_size,profile_size,len(sample)))
  42 + if hasattr(recommendation,"ranking"):
  43 + notfound = []
  44 + ranks = []
  45 + for pkg in sample.keys():
  46 + if pkg in recommendation.ranking:
  47 + ranks.append(recommendation.ranking.index(pkg))
  48 + else:
  49 + notfound.append(pkg)
  50 + for r in sorted(ranks):
  51 + output.write(str(r)+"\n")
  52 + if notfound:
  53 + output.write("# out of recommendation:\n")
  54 + for pkg in notfound:
  55 + output.write(pkg+"\n")
  56 + output.close()
  57 +
  58 +def plot_summary(results,log_file):
  59 + # Plot metrics summary
  60 + g = Gnuplot.Gnuplot()
  61 + g('set style data lines')
  62 + g('set yrange [0:1.0]')
  63 + g.xlabel('Threshold (recommendation size)')
  64 + g.title("Setup: %s" % log_file.split("/")[-1])
  65 + g.plot(Gnuplot.Data(results.precision_summary(),title="Precision"),
  66 + Gnuplot.Data(results.recall_summary(),title="Recall"),
  67 + Gnuplot.Data(results.f05_summary(),title="F05"),
  68 + Gnuplot.Data(results.coverage_summary(),title="Coverage"))
  69 + g.hardcopy(log_file+".png",terminal="png")
  70 + g.hardcopy(log_file+".ps",terminal="postscript",enhanced=1,color=1)
  71 + g('set logscale x')
  72 + g('replot')
  73 + g.hardcopy(log_file+"-logscale.png",terminal="png")
  74 + g.hardcopy(log_file+"-logscale.ps",terminal="postscript",enhanced=1,color=1)
  75 +
  76 +def plot_roc(results,log_file):
  77 + g = Gnuplot.Gnuplot()
  78 + g('set style data lines')
  79 + g.xlabel('False Positive Rate')
  80 + g.ylabel('True Positive Rate')
  81 + g('set xrange [0:1.0]')
  82 + g('set yrange [0:1.0]')
  83 + g.title("Setup: %s" % log_file.split("/")[-1])
  84 + g('set label "C %.2f" at 0.8,0.25' % results.coverage())
  85 + g('set label "AUC %.2f" at 0.8,0.2' % results.get_auc())
  86 + g('set label "P(10) %.2f" at 0.8,0.15' % numpy.mean(results.precision[10]))
  87 + g('set label "P(20) %.2f" at 0.8,0.10' % numpy.mean(results.precision[20]))
  88 + g('set label "F05(100) %.2f" at 0.8,0.05' % numpy.mean(results.f05[100]))
  89 + g.plot(Gnuplot.Data(results.get_roc_points(),title="ROC"),
  90 + Gnuplot.Data([[0,0],[1,1]],with_="lines lt 7"))
  91 + #Gnuplot.Data([roc_points[-1],[1,1]],with_="lines lt 6"))
  92 + g.hardcopy(log_file+"-roc.png",terminal="png")
  93 + g.hardcopy(log_file+"-roc.ps",terminal="postscript",enhanced=1,color=1)
  94 +
  95 +def get_label(cfg):
  96 + label = {}
  97 + if cfg.strategy in content_based:
  98 + label["description"] = "strategy-profile"
  99 + label["values"] = ("%s-profile%.3d" %
  100 + (cfg.strategy,cfg.profile_size))
  101 + elif cfg.strategy in collaborative:
  102 + label["description"] = "strategy-knn"
  103 + label["values"] = ("%s-k%.3d" %
  104 + (cfg.strategy,cfg.k_neighbors))
  105 + elif cfg.strategy in hybrid:
  106 + label["description"] = "strategy-knn-profile"
  107 + label["values"] = ("%s-k%.3d-profile%.3d" %
  108 + (cfg.strategy,cfg.k_neighbors,cfg.profile_size))
  109 + return label
  110 +
  111 +class ExperimentResults:
  112 + def __init__(self,repo_size):
  113 + self.repository_size = repo_size
  114 + self.precision = {}
  115 + self.recall = {}
  116 + self.fpr = {}
  117 + self.f05 = {}
  118 + self.recommended = {}
  119 + self.thresholds = [1]+range(10,self.repository_size,10)
  120 + for size in self.thresholds:
  121 + self.precision[size] = []
  122 + self.recall[size] = []
  123 + self.fpr[size] = []
  124 + self.f05[size] = []
  125 + self.recommended[size] = set()
  126 +
  127 + def add_result(self,ranking,sample):
  128 + for size in self.thresholds:
  129 + recommendation = ranking[:size]
  130 + self.recommended[size] = self.recommended[size].union(recommendation)
  131 + predicted = RecommendationResult(dict.fromkeys(recommendation,1))
  132 + real = RecommendationResult(sample)
  133 + evaluation = Evaluation(predicted,real,self.repository_size)
  134 + print evaluation.run(Precision())
  135 + self.precision[size].append(evaluation.run(Precision()))
  136 + self.recall[size].append(evaluation.run(Recall()))
  137 + self.f05[size].append(evaluation.run(F_score(0.5)))
  138 + self.fpr[size].append(evaluation.run(FPR()))
  139 +
  140 + def precision_summary(self):
  141 + return [[size,numpy.mean(self.precision[size])] for size in self.thresholds]
  142 +
  143 + def recall_summary(self):
  144 + return [[size,numpy.mean(self.recall[size])] for size in self.thresholds]
  145 +
  146 + def f05_summary(self):
  147 + return [[size,numpy.mean(self.f05[size])] for size in self.thresholds]
  148 +
  149 + def coverage_summary(self):
  150 + return [[size,self.coverage(size)] for size in self.thresholds]
  151 +
  152 + def coverage(self,size=0):
  153 + if not size:
  154 + size = self.thresholds[-1]
  155 + return len(self.recommended[size])/float(self.repository_size)
  156 +
  157 + def precision(self,size):
  158 + return numpy.mean(results.precision[size])
  159 +
  160 + def get_auc(self):
  161 + roc_points = self.get_roc_points()
  162 + x_roc = [p[0] for p in roc_points]
  163 + y_roc = [p[1] for p in roc_points]
  164 + x_roc.insert(0,0)
  165 + y_roc.insert(0,0)
  166 + x_roc.append(1)
  167 + y_roc.append(1)
  168 + return numpy.trapz(y=y_roc, x=x_roc)
  169 +
  170 + # Average ROC by threshold (= size of recommendation)
  171 + def get_roc_points(self):
  172 + points = []
  173 + for size in self.recall.keys():
  174 + tpr = self.recall[size]
  175 + fpr = self.fpr[size]
  176 + points.append([sum(fpr)/len(fpr),sum(tpr)/len(tpr)])
  177 + return sorted(points)
  178 +
  179 +def run_strategy(cfg,user):
  180 + rec = Recommender(cfg)
  181 + repo_size = rec.items_repository.get_doccount()
  182 + results = ExperimentResults(repo_size)
  183 + label = get_label(cfg)
  184 + user_dir = ("results/roc-suite/%s/%s" % (user.user_id[:8],cfg.strategy))
  185 + if not os.path.exists(user_dir):
  186 + os.makedirs(user_dir)
  187 + log_file = os.path.join(user_dir,label["values"])
  188 + for n in range(iterations):
  189 + # Fill sample profile
  190 + profile_len = len(user.pkg_profile)
  191 + item_score = {}
  192 + for pkg in user.pkg_profile:
  193 + item_score[pkg] = user.item_score[pkg]
  194 + sample = {}
  195 + sample_size = int(profile_len*0.9)
  196 + for i in range(sample_size):
  197 + key = random.choice(item_score.keys())
  198 + sample[key] = item_score.pop(key)
  199 + iteration_user = User(item_score)
  200 + recommendation = rec.get_recommendation(iteration_user,repo_size)
  201 + write_recall_log(label,n,sample,recommendation,profile_len,repo_size,log_file)
  202 + if hasattr(recommendation,"ranking"):
  203 + results.add_result(recommendation.ranking,sample)
  204 + with open(log_file+"-roc.jpg.comment",'w') as f:
  205 + f.write("# %s\n# %s\n\n" %
  206 + (label["description"],label["values"]))
  207 + f.write("# roc AUC\n%.4f\n\n"%results.get_auc())
  208 + f.write("# threshold\tprecision\trecall\t\tf05\t\tcoverage\n")
  209 + for size in results.thresholds:
  210 + f.write("%4d\t\t%.4f\t\t%.4f\t\t%.4f\t\t%.4f\n" %
  211 + (size,numpy.mean(results.precision[size]),
  212 + numpy.mean(results.recall[size]),
  213 + numpy.mean(results.f05[size]),
  214 + numpy.mean(results.coverage(size))))
  215 + shutil.copy(log_file+"-roc.jpg.comment",log_file+".jpg.comment")
  216 + shutil.copy(log_file+"-roc.jpg.comment",log_file+"-logscale.jpg.comment")
  217 + plot_roc(results,log_file)
  218 + plot_summary(results,log_file)
  219 +
  220 +def run_content(user,cfg):
  221 + for size in profile_size:
  222 + cfg.profile_size = size
  223 + run_strategy(cfg,user)
  224 +
  225 +def run_collaborative(user,cfg):
  226 + for k in neighbors:
  227 + cfg.k_neighbors = k
  228 + run_strategy(cfg,user)
  229 +
  230 +def run_hybrid(user,cfg):
  231 + for k in neighbors:
  232 + cfg.k_neighbors = k
  233 + for size in profile_size:
  234 + cfg.profile_size = size
  235 + run_strategy(cfg,user)
  236 +
  237 +if __name__ == '__main__':
  238 + if len(sys.argv)<2:
  239 + print "Usage: roc-suite strategy_str [popcon_submission_path]"
  240 + exit(1)
  241 +
  242 + #iterations = 3
  243 + #content_based = ['cb']
  244 + #collaborative = ['knn_eset']
  245 + #hybrid = ['knnco']
  246 + #profile_size = [50,100]
  247 + #neighbors = [50]
  248 + iterations = 20
  249 + content_based = ['cb','cbt','cbd','cbh','cb_eset','cbt_eset','cbd_eset','cbh_eset']
  250 + collaborative = ['knn_eset','knn','knn_plus']
  251 + hybrid = ['knnco','knnco_eset']
  252 + profile_size = [10,20,40,60,80,100,140,170,200,240]
  253 + neighbors = [3,5,10,20,30,50,70,100,150,200]
  254 +
  255 + cfg = Config()
  256 + cfg.strategy = sys.argv[1]
  257 +
  258 + #user = PopconSystem("/root/.app-recommender/popcon-entries/4a/4a67a295ec14826db2aa1d90be2f1623")
  259 + user = PopconSystem("/root/.app-recommender/popcon-entries/8b/8b44fcdbcf676e711a153d5db09979d7")
  260 + #user = PopconSystem(sys.argv[1])
  261 + user.filter_pkg_profile(cfg.pkgs_filter)
  262 + user.maximal_pkg_profile()
  263 +
  264 + if cfg.strategy in content_based:
  265 + run_content(user,cfg)
  266 + if cfg.strategy in collaborative:
  267 + run_collaborative(user,cfg)
  268 + if cfg.strategy in hybrid:
  269 + run_hybrid(user,cfg)
... ...