Download as
Browse Code »

Commit ccd4ef5568d421a430f37975baff0bacf775b91c

Authored by Tássia Camões Araújo
1 parent e2be2c33
Exists in master and in 1 other branch add_vagrant

Renamed files.

Showing 3 changed files with 0 additions and 691 deletions   Show diff stats
src/experiments/k-suite.py
View file @e2be2c3
... ... @@ -1,186 +0,0 @@
1   -#!/usr/bin/env python
2   -"""
3   - k-suite - experiment different neighborhood sizes
4   -"""
5   -__author__ = "Tassia Camoes Araujo <tassia@gmail.com>"
6   -__copyright__ = "Copyright (C) 2011 Tassia Camoes Araujo"
7   -__license__ = """
8   - This program is free software: you can redistribute it and/or modify
9   - it under the terms of the GNU General Public License as published by
10   - the Free Software Foundation, either version 3 of the License, or
11   - (at your option) any later version.
12   -
13   - This program is distributed in the hope that it will be useful,
14   - but WITHOUT ANY WARRANTY; without even the implied warranty of
15   - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16   - GNU General Public License for more details.
17   -
18   - You should have received a copy of the GNU General Public License
19   - along with this program. If not, see <http://www.gnu.org/licenses/>.
20   -"""
21   -
22   -import sys
23   -sys.path.insert(0,'../')
24   -from config import Config
25   -from data import PopconXapianIndex, PopconSubmission
26   -from recommender import Recommender
27   -from user import LocalSystem, User
28   -from evaluation import *
29   -import logging
30   -import random
31   -import Gnuplot
32   -import numpy
33   -
34   -def plot_roc(k,roc_points,log_file):
35   - g = Gnuplot.Gnuplot()
36   - g('set style data points')
37   - g.xlabel('False Positive Rate')
38   - g.ylabel('True Positive Rate')
39   - g('set xrange [0:1.0]')
40   - g('set yrange [0:1.0]')
41   - g.title("Setup: %s-k%d" % (log_file.split("/")[-1],k))
42   - g.plot(Gnuplot.Data([[0,0],[1,1]],with_="lines lt 7"),
43   - Gnuplot.Data(roc_points))
44   - g.hardcopy(log_file+("-k%.3d.png"%k),terminal="png")
45   - g.hardcopy(log_file+("-k%.3d.ps"%k),terminal="postscript",enhanced=1,color=1)
46   -
47   -def plot_summary(precision,f05,mcc,log_file):
48   - g = Gnuplot.Gnuplot()
49   - g('set style data lines')
50   - g.xlabel('Neighborhood (k)')
51   - g.title("Setup: %s-size20" % (log_file.split("/")[-1]))
52   - g.plot(Gnuplot.Data([[k,sum(precision[k])/len(precision[k])] for k in precision.keys()],title="P"),
53   - Gnuplot.Data([[k,sum(f05[k])/len(f05[k])] for k in f05.keys()],title="F05"),
54   - Gnuplot.Data([[k,sum(mcc[k])/len(mcc[k])] for k in mcc.keys()],title="MCC"))
55   - g.hardcopy(log_file+(".png"),terminal="png")
56   - g.hardcopy(log_file+(".ps"),terminal="postscript",enhanced=1,color=1)
57   -
58   -class ExperimentResults:
59   - def __init__(self,repo_size):
60   - self.repository_size = repo_size
61   - self.precision = []
62   - self.recall = []
63   - self.fpr = []
64   - self.f05 = []
65   - self.mcc = []
66   -
67   - def add_result(self,ranking,sample):
68   - predicted = RecommendationResult(dict.fromkeys(ranking,1))
69   - real = RecommendationResult(sample)
70   - evaluation = Evaluation(predicted,real,self.repository_size)
71   - self.precision.append(evaluation.run(Precision()))
72   - self.recall.append(evaluation.run(Recall()))
73   - self.fpr.append(evaluation.run(FPR()))
74   - self.f05.append(evaluation.run(F_score(0.5)))
75   - self.mcc.append(evaluation.run(MCC()))
76   -
77   - def get_roc_point(self):
78   - tpr = self.recall
79   - fpr = self.fpr
80   - if not tpr or not fpr:
81   - return [0,0]
82   - return [sum(fpr)/len(fpr),sum(tpr)/len(tpr)]
83   -
84   - def get_precision_summary(self):
85   - if not self.precision: return 0
86   - return sum(self.precision)/len(self.precision)
87   -
88   - def get_f05_summary(self):
89   - if not self.f05: return 0
90   - return sum(self.f05)/len(self.f05)
91   -
92   - def get_mcc_summary(self):
93   - if not self.mcc: return 0
94   - return sum(self.mcc)/len(self.mcc)
95   -
96   -if __name__ == '__main__':
97   - if len(sys.argv)<3:
98   - print "Usage: k-suite strategy_str sample_file"
99   - exit(1)
100   - threshold = 20
101   - iterations = 30
102   - neighbors = [3,5,10,50,100,150,200,300,400,500]
103   - cfg = Config()
104   - cfg.strategy = sys.argv[1]
105   - sample_file = sys.argv[2]
106   - population_sample = []
107   - with open(sample_file,'r') as f:
108   - for line in f.readlines():
109   - user_id = line.strip('\n')
110   - population_sample.append(os.path.join(cfg.popcon_dir,user_id[:2],user_id))
111   - # setup dictionaries and files
112   - roc_summary = {}
113   - recommended = {}
114   - precision_summary = {}
115   - f05_summary = {}
116   - mcc_summary = {}
117   - sample_dir = ("results/k-suite/%s" % sample_file.split('/')[-1])
118   - if not os.path.exists(sample_dir):
119   - os.makedirs(sample_dir)
120   - log_file = os.path.join(sample_dir,cfg.strategy)
121   - with open(log_file,'w') as f:
122   - f.write("# %s\n\n" % sample_file.split('/')[-1])
123   - f.write("# strategy %s recommendation_size %d iterations %d\n\n" %
124   - (cfg.strategy,threshold,iterations))
125   - f.write("# k coverage \tprecision \tf05 \tmcc\n\n")
126   -
127   - for k in neighbors:
128   - roc_summary[k] = []
129   - recommended[k] = set()
130   - precision_summary[k] = []
131   - f05_summary[k] = []
132   - mcc_summary[k] = []
133   - with open(log_file+"-k%.3d"%k,'w') as f:
134   - f.write("# %s\n\n" % sample_file.split('/')[-1])
135   - f.write("# strategy-k %s-k%.3d\n\n" % (cfg.strategy,k))
136   - f.write("# roc_point \tprecision \tf05 \tmcc\n\n")
137   -
138   - # main loop per user
139   - for submission_file in population_sample:
140   - user = PopconSystem(submission_file)
141   - user.filter_pkg_profile(cfg.pkgs_filter)
142   - user.maximal_pkg_profile()
143   - for k in neighbors:
144   - cfg.k_neighbors = k
145   - rec = Recommender(cfg)
146   - repo_size = rec.items_repository.get_doccount()
147   - results = ExperimentResults(repo_size)
148   - # n iterations for same recommender and user
149   - for n in range(iterations):
150   - # Fill sample profile
151   - profile_len = len(user.pkg_profile)
152   - item_score = {}
153   - for pkg in user.pkg_profile:
154   - item_score[pkg] = user.item_score[pkg]
155   - sample = {}
156   - sample_size = int(profile_len*0.9)
157   - for i in range(sample_size):
158   - key = random.choice(item_score.keys())
159   - sample[key] = item_score.pop(key)
160   - iteration_user = User(item_score)
161   - recommendation = rec.get_recommendation(iteration_user,threshold)
162   - if hasattr(recommendation,"ranking"):
163   - results.add_result(recommendation.ranking,sample)
164   - recommended[k] = recommended[k].union(recommendation.ranking)
165   - # save summary
166   - roc_point = results.get_roc_point()
167   - roc_summary[k].append(roc_point)
168   - precision = results.get_precision_summary()
169   - precision_summary[k].append(precision)
170   - f05 = results.get_f05_summary()
171   - f05_summary[k].append(f05)
172   - mcc = results.get_mcc_summary()
173   - mcc_summary[k].append(mcc)
174   - with open(log_file+"-k%.3d"%k,'a') as f:
175   - f.write("[%.2f,%.2f] \t%.4f \t%.4f \t%.4f\n" %
176   - (roc_point[0],roc_point[1],precision,f05,mcc))
177   - # back to main flow
178   - with open(log_file,'a') as f:
179   - plot_summary(precision_summary,f05_summary,mcc_summary,log_file)
180   - for k in neighbors:
181   - coverage = len(recommended[size])/float(repo_size)
182   - f.write("%3d \t%.2f \t%.4f \t%.4f \t%.4f\n" %
183   - (k,coverage,float(sum(precision_summary[k]))/len(precision_summary[k]),
184   - float(sum(f05_summary[k]))/len(f05_summary[k]),
185   - float(sum(mcc_summary[k]))/len(mcc_summary[k])))
186   - plot_roc(k,roc_summary[k],log_file)
src/experiments/roc-suite.py
View file @e2be2c3
... ... @@ -1,231 +0,0 @@
1   -#!/usr/bin/env python
2   -"""
3   - recommender suite - recommender experiments suite
4   -"""
5   -__author__ = "Tassia Camoes Araujo <tassia@gmail.com>"
6   -__copyright__ = "Copyright (C) 2011 Tassia Camoes Araujo"
7   -__license__ = """
8   - This program is free software: you can redistribute it and/or modify
9   - it under the terms of the GNU General Public License as published by
10   - the Free Software Foundation, either version 3 of the License, or
11   - (at your option) any later version.
12   -
13   - This program is distributed in the hope that it will be useful,
14   - but WITHOUT ANY WARRANTY; without even the implied warranty of
15   - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16   - GNU General Public License for more details.
17   -
18   - You should have received a copy of the GNU General Public License
19   - along with this program. If not, see <http://www.gnu.org/licenses/>.
20   -"""
21   -
22   -import sys
23   -sys.path.insert(0,'../')
24   -from config import Config
25   -from data import PopconXapianIndex, PopconSubmission
26   -from recommender import Recommender
27   -from user import LocalSystem, User
28   -from evaluation import *
29   -import logging
30   -import random
31   -import Gnuplot
32   -import numpy
33   -
34   -#iterations = 3
35   -#sample_proportions = [0.9]
36   -#weighting = [('bm25',1.2)]
37   -#collaborative = ['knn_eset']
38   -#content_based = ['cb']
39   -#hybrid = ['knnco']
40   -#profile_size = [50,100]
41   -#popcon_size = ["1000"]
42   -#neighbors = [50]
43   -
44   -iterations = 30
45   -sample_proportions = [0.9]
46   -weighting = [('bm25',1.0)]
47   -content_based = ['cb','cbt','cbd','cbh','cb_eset','cbt_eset','cbd_eset','cbh_eset']
48   -collaborative = ['knn_eset','knn','knn_plus']
49   -hybrid = ['knnco','knnco_eset']
50   -profile_size = range(20,200,40)
51   -neighbors = range(10,510,50)
52   -
53   -def write_recall_log(label,n,sample,recommendation,profile_size,repo_size,log_file):
54   - # Write recall log
55   - output = open(("%s-%.2d" % (log_file,n)),'w')
56   - output.write("# %s-n\n" % label["description"])
57   - output.write("# %s-%.2d\n" % (label["values"],n))
58   - output.write("\n# repository profile sample\n%d %d %d\n" % \
59   - (repo_size,profile_size,len(sample)))
60   - if hasattr(recommendation,"ranking"):
61   - notfound = []
62   - ranks = []
63   - for pkg in sample.keys():
64   - if pkg in recommendation.ranking:
65   - ranks.append(recommendation.ranking.index(pkg))
66   - else:
67   - notfound.append(pkg)
68   - for r in sorted(ranks):
69   - output.write(str(r)+"\n")
70   - if notfound:
71   - output.write("# out of recommendation:\n")
72   - for pkg in notfound:
73   - output.write(pkg+"\n")
74   - output.close()
75   -
76   -def plot_roc(roc_points,eauc,c,p,log_file):
77   - g = Gnuplot.Gnuplot()
78   - g('set style data lines')
79   - g.xlabel('False Positive Rate')
80   - g.ylabel('True Positive Rate')
81   - g('set xrange [0:1.0]')
82   - g('set yrange [0:1.0]')
83   - g.title("Setup: %s" % log_file.split("/")[-1])
84   - g('set label "C %.2f" at 0.8,0.25' % c)
85   - g('set label "P(20) %.2f" at 0.8,0.2' % p)
86   - g('set label "AUC %.4f" at 0.8,0.15' % eauc)
87   - g.plot(Gnuplot.Data(roc_points,title="ROC"),
88   - Gnuplot.Data([[0,0],[1,1]],with_="lines lt 7"))
89   - #Gnuplot.Data([roc_points[-1],[1,1]],with_="lines lt 6"))
90   - g.hardcopy(log_file+"-roc.png",terminal="png")
91   - g.hardcopy(log_file+"-roc.ps",terminal="postscript",enhanced=1,color=1)
92   -
93   -def get_label(cfg,sample_proportion):
94   - label = {}
95   - if cfg.strategy in content_based:
96   - label["description"] = "strategy-profile"
97   - label["values"] = ("%s-profile%.3d" %
98   - (cfg.strategy,cfg.profile_size))
99   - elif cfg.strategy in collaborative:
100   - label["description"] = "strategy-knn"
101   - label["values"] = ("%s-k%.3d" %
102   - (cfg.strategy,cfg.k_neighbors))
103   - elif cfg.strategy in hybrid:
104   - label["description"] = "strategy-knn-profile"
105   - label["values"] = ("%s-k%.3d-profile%.3d" %
106   - (cfg.strategy,cfg.k_neighbors,cfg.profile_size))
107   - else:
108   - print "Unknown strategy"
109   - return label
110   -
111   -class ExperimentResults:
112   - def __init__(self,repo_size):
113   - self.repository_size = repo_size
114   - self.precision = {}
115   - self.recall = {}
116   - self.fpr = {}
117   - points = [1]+range(10,self.repository_size,10)
118   - self.recommended = set()
119   - for size in points:
120   - self.precision[size] = []
121   - self.recall[size] = []
122   - self.fpr[size] = []
123   -
124   - def add_result(self,ranking,sample):
125   - self.recommended = self.recommended.union(ranking)
126   - # get data only for point
127   - for size in self.precision.keys():
128   - predicted = RecommendationResult(dict.fromkeys(ranking[:size],1))
129   - real = RecommendationResult(sample)
130   - evaluation = Evaluation(predicted,real,self.repository_size)
131   - self.precision[size].append(evaluation.run(Precision()))
132   - self.recall[size].append(evaluation.run(Recall()))
133   - self.fpr[size].append(evaluation.run(FPR()))
134   -
135   - # Average ROC by threshold (= size of recommendation)
136   - def get_roc_points(self):
137   - points = []
138   - for size in self.recall.keys():
139   - tpr = self.recall[size]
140   - fpr = self.fpr[size]
141   - points.append([sum(fpr)/len(fpr),sum(tpr)/len(tpr)])
142   - return sorted(points)
143   -
144   -def run_strategy(cfg,user):
145   - for weight in weighting:
146   - cfg.weight = weight[0]
147   - cfg.bm25_k1 = weight[1]
148   - rec = Recommender(cfg)
149   - repo_size = rec.items_repository.get_doccount()
150   - for proportion in sample_proportions:
151   - results = ExperimentResults(repo_size)
152   - label = get_label(cfg,proportion)
153   - user_dir = ("results/roc-suite/%s" % user.user_id[:8])
154   - if not os.path.exists(user_dir):
155   - os.mkdir(user_dir)
156   - log_file = os.path.join(user_dir,label["values"])
157   - for n in range(iterations):
158   - # Fill sample profile
159   - profile_len = len(user.pkg_profile)
160   - item_score = {}
161   - for pkg in user.pkg_profile:
162   - item_score[pkg] = user.item_score[pkg]
163   - sample = {}
164   - sample_size = int(profile_len*proportion)
165   - for i in range(sample_size):
166   - key = random.choice(item_score.keys())
167   - sample[key] = item_score.pop(key)
168   - iteration_user = User(item_score)
169   - recommendation = rec.get_recommendation(iteration_user,repo_size)
170   - write_recall_log(label,n,sample,recommendation,profile_len,repo_size,log_file)
171   - if hasattr(recommendation,"ranking"):
172   - results.add_result(recommendation.ranking,sample)
173   - with open(log_file,'w') as f:
174   - roc_points = results.get_roc_points()
175   - x_coord = [p[0] for p in roc_points]
176   - y_coord = [p[1] for p in roc_points]
177   - auc = numpy.trapz(y=y_coord, x=x_coord)
178   - eauc = (auc+
179   - numpy.trapz(y=[0,roc_points[0][1]],x=[0,roc_points[0][0]])+
180   - numpy.trapz(y=[roc_points[-1][1],1],x=[roc_points[-1][0],1]))
181   - precision_20 = sum(results.precision[10])/len(results.precision[10])
182   - coverage = len(results.recommended)/float(repo_size)
183   - f.write("# %s\n# %s\n\n" %
184   - (label["description"],label["values"]))
185   - f.write("# coverage \tp(20) \tauc \teauc\n\t%.2f \t%.2f \t%.4f \t%.4f\n\n" %
186   - (coverage,precision_20,auc,eauc))
187   - plot_roc(roc_points,eauc,coverage,precision_20,log_file)
188   -
189   -def run_content(user,cfg):
190   - for strategy in content_based:
191   - cfg.strategy = strategy
192   - for size in profile_size:
193   - cfg.profile_size = size
194   - run_strategy(cfg,user)
195   -
196   -def run_collaborative(user,cfg):
197   - popcon_desktopapps = cfg.popcon_desktopapps
198   - popcon_programs = cfg.popcon_programs
199   - for strategy in collaborative:
200   - cfg.strategy = strategy
201   - for k in neighbors:
202   - cfg.k_neighbors = k
203   - run_strategy(cfg,user)
204   -
205   -def run_hybrid(user,cfg):
206   - popcon_desktopapps = cfg.popcon_desktopapps
207   - popcon_programs = cfg.popcon_programs
208   - for strategy in hybrid:
209   - cfg.strategy = strategy
210   - for k in neighbors:
211   - cfg.k_neighbors = k
212   - for size in profile_size:
213   - cfg.profile_size = size
214   - run_strategy(cfg,user)
215   -
216   -if __name__ == '__main__':
217   - if len(sys.argv)<2:
218   - print "Usage: roc-suite popcon_submission_path [content|collaborative|hybrid]"
219   - exit(1)
220   -
221   - cfg = Config()
222   - user = PopconSystem(sys.argv[1])
223   - user.filter_pkg_profile(cfg.pkgs_filter)
224   - user.maximal_pkg_profile()
225   -
226   - if "content" in sys.argv or len(sys.argv)<3:
227   - run_content(user,cfg)
228   - if "collaborative" in sys.argv or len(sys.argv)<3:
229   - run_collaborative(user,cfg)
230   - if "hybrid" in sys.argv or len(sys.argv)<3:
231   - run_hybrid(user,cfg)
src/experiments/strategies-suite.py
View file @e2be2c3
... ... @@ -1,274 +0,0 @@
1   -#!/usr/bin/env python
2   -"""
3   - recommender suite - recommender experiments suite
4   -"""
5   -__author__ = "Tassia Camoes Araujo <tassia@gmail.com>"
6   -__copyright__ = "Copyright (C) 2011 Tassia Camoes Araujo"
7   -__license__ = """
8   - This program is free software: you can redistribute it and/or modify
9   - it under the terms of the GNU General Public License as published by
10   - the Free Software Foundation, either version 3 of the License, or
11   - (at your option) any later version.
12   -
13   - This program is distributed in the hope that it will be useful,
14   - but WITHOUT ANY WARRANTY; without even the implied warranty of
15   - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16   - GNU General Public License for more details.
17   -
18   - You should have received a copy of the GNU General Public License
19   - along with this program. If not, see <http://www.gnu.org/licenses/>.
20   -"""
21   -
22   -import sys
23   -sys.path.insert(0,'../')
24   -from config import Config
25   -from data import PopconXapianIndex, PopconSubmission, AppAptXapianIndex
26   -from recommender import Recommender
27   -from user import LocalSystem, User
28   -from evaluation import *
29   -import logging
30   -import random
31   -import Gnuplot
32   -
33   -#iterations = 3
34   -#sample_proportions = [0.9]
35   -#weighting = [('bm25',1.2)]
36   -#collaborative = ['knn']
37   -#content_based = []
38   -#hybrid = ['knnco']
39   -#profile_size = [50,100]
40   -#popcon_size = ["1000"]
41   -#neighbors = [50]
42   -
43   -iterations = 10
44   -sample_proportions = [0.5, 0.6, 0.7, 0.8, 0.9]
45   -weighting = [('bm25',1.2), ('bm25',1.6), ('bm25',2.0), ('trad',0)]
46   -content_based = ['cb','cbt','cbd','cbh','cb_eset','cbt_eset','cbd_eset','cbh_eset']
47   -collaborative = ['knn_eset','knn','knn_plus']
48   -hybrid = ['knnco','knnco_eset']
49   -
50   -profile_size = range(20,100,20)
51   -#popcon_size = [1000,10000,50000,'full']
52   -neighbors = range(10,510,50)
53   -
54   -def write_recall_log(label,n,sample,recommendation,profile_size,repo_size,log_file):
55   - # Write recall log
56   - output = open(("%s-%d" % (log_file,n)),'w')
57   - output.write("# %s-n\n" % label["description"])
58   - output.write("# %s-%d\n" % (label["values"],n))
59   - output.write("\n%d %d %d\n" % \
60   - (repo_size,profile_size,len(sample)))
61   - if hasattr(recommendation,"ranking"):
62   - notfound = []
63   - ranks = []
64   - for pkg in sample.keys():
65   - if pkg in recommendation.ranking:
66   - ranks.append(recommendation.ranking.index(pkg))
67   - else:
68   - notfound.append(pkg)
69   - for r in sorted(ranks):
70   - output.write(str(r)+"\n")
71   - if notfound:
72   - output.write("Out of recommendation:\n")
73   - for pkg in notfound:
74   - output.write(pkg+"\n")
75   - output.close()
76   -
77   -def plot_summary(precision,recall,f1,f05,accuracy,log_file):
78   - # Plot metrics summary
79   - g = Gnuplot.Gnuplot()
80   - g('set style data lines')
81   - g.xlabel('Recommendation size')
82   - g.title("Setup: %s" % log_file.split("/")[-1])
83   - g.plot(Gnuplot.Data(accuracy,title="Accuracy"),
84   - Gnuplot.Data(precision,title="Precision"),
85   - Gnuplot.Data(recall,title="Recall"),
86   - Gnuplot.Data(f1,title="F_1"),
87   - Gnuplot.Data(f05,title="F_0.5"))
88   - g.hardcopy(log_file+".png",terminal="png")
89   - g.hardcopy(log_file+".ps",terminal="postscript",enhanced=1,color=1)
90   - g('set logscale x')
91   - g('replot')
92   - g.hardcopy(log_file+"-logscale.png",terminal="png")
93   - g.hardcopy(log_file+"-logscale.ps",terminal="postscript",enhanced=1,color=1)
94   -
95   -def get_label(cfg,sample_proportion):
96   - label = {}
97   - if cfg.strategy in content_based:
98   - label["description"] = "strategy-filter-profile-k1_bm25-sample"
99   - label["values"] = ("%s-profile%d-%s-kbm%.1f-sample%.1f" %
100   - (cfg.strategy,cfg.profile_size,
101   - cfg.pkgs_filter.split("/")[-1],
102   - cfg.bm25_k1,sample_proportion))
103   - elif cfg.strategy in collaborative:
104   - label["description"] = "strategy-knn-filter-k1_bm25-sample"
105   - label["values"] = ("%s-k%d-%s-kbm%.1f-sample%.1f" %
106   - (cfg.strategy,cfg.k_neighbors,
107   - cfg.pkgs_filter.split("/")[-1],
108   - cfg.bm25_k1,sample_proportion))
109   - elif cfg.strategy in hybrid:
110   - label["description"] = "strategy-knn-filter-profile-k1_bm25-sample"
111   - label["values"] = ("%s-k%d-profile%d-%s-kbm%.1f-sample%.1f" %
112   - (cfg.strategy,cfg.k_neighbors,cfg.profile_size,
113   - cfg.pkgs_filter.split("/")[-1],
114   - cfg.bm25_k1,sample_proportion))
115   - else:
116   - print "Unknown strategy"
117   - return label
118   -
119   -class ExperimentResults:
120   - def __init__(self,repo_size):
121   - self.repository_size = repo_size
122   - self.accuracy = {}
123   - self.precision = {}
124   - self.recall = {}
125   - self.f1 = {}
126   - self.f05 = {}
127   - points = [1]+range(10,200,10)+range(200,self.repository_size,100)
128   - for size in points:
129   - self.accuracy[size] = []
130   - self.precision[size] = []
131   - self.recall[size] = []
132   - self.f1[size] = []
133   - self.f05[size] = []
134   -
135   - def add_result(self,ranking,sample):
136   - for size in self.accuracy.keys():
137   - predicted = RecommendationResult(dict.fromkeys(ranking[:size],1))
138   - real = RecommendationResult(sample)
139   - evaluation = Evaluation(predicted,real,self.repository_size)
140   - self.accuracy[size].append(evaluation.run(Accuracy()))
141   - self.precision[size].append(evaluation.run(Precision()))
142   - self.recall[size].append(evaluation.run(Recall()))
143   - self.f1[size].append(evaluation.run(F_score(1)))
144   - self.f05[size].append(evaluation.run(F_score(0.5)))
145   -
146   - def get_precision_summary(self):
147   - summary = [[size,sum(values)/len(values)] for size,values in self.precision.items()]
148   - return sorted(summary)
149   -
150   - def get_recall_summary(self):
151   - summary = [[size,sum(values)/len(values)] for size,values in self.recall.items()]
152   - return sorted(summary)
153   -
154   - def get_f1_summary(self):
155   - summary = [[size,sum(values)/len(values)] for size,values in self.f1.items()]
156   - return sorted(summary)
157   -
158   - def get_f05_summary(self):
159   - summary = [[size,sum(values)/len(values)] for size,values in self.f05.items()]
160   - return sorted(summary)
161   -
162   - def get_accuracy_summary(self):
163   - summary = [[size,sum(values)/len(values)] for size,values in self.accuracy.items()]
164   - return sorted(summary)
165   -
166   - def best_precision(self):
167   - size = max(self.precision, key = lambda x: max(self.precision[x]))
168   - return (size,max(self.precision[size]))
169   -
170   - def best_f1(self):
171   - size = max(self.f1, key = lambda x: max(self.f1[x]))
172   - return (size,max(self.f1[size]))
173   -
174   - def best_f05(self):
175   - size = max(self.f05, key = lambda x: max(self.f05[x]))
176   - return (size,max(self.f05[size]))
177   -
178   -def run_strategy(cfg,user):
179   - for weight in weighting:
180   - cfg.weight = weight[0]
181   - cfg.bm25_k1 = weight[1]
182   - rec = Recommender(cfg)
183   - repo_size = rec.items_repository.get_doccount()
184   - for proportion in sample_proportions:
185   - results = ExperimentResults(repo_size)
186   - label = get_label(cfg,proportion)
187   - log_file = "results/strategies/"+label["values"]
188   - for n in range(iterations):
189   - # Fill sample profile
190   - profile_size = len(user.pkg_profile)
191   - item_score = {}
192   - for pkg in user.pkg_profile:
193   - item_score[pkg] = user.item_score[pkg]
194   - sample = {}
195   - sample_size = int(profile_size*proportion)
196   - for i in range(sample_size):
197   - key = random.choice(item_score.keys())
198   - sample[key] = item_score.pop(key)
199   - iteration_user = User(item_score)
200   - recommendation = rec.get_recommendation(iteration_user,repo_size)
201   - write_recall_log(label,n,sample,recommendation,profile_size,repo_size,log_file)
202   - if hasattr(recommendation,"ranking"):
203   - results.add_result(recommendation.ranking,sample)
204   - with open(log_file,'w') as f:
205   - precision_10 = sum(results.precision[10])/len(results.precision[10])
206   - f1_10 = sum(results.f1[10])/len(results.f1[10])
207   - f05_10 = sum(results.f05[10])/len(results.f05[10])
208   - f.write("# %s\n# %s\n\ncoverage %d\n\n" %
209   - (label["description"],label["values"],recommendation.size))
210   - f.write("# best results (recommendation size; metric)\n")
211   - f.write("precision (%d; %.2f)\nf1 (%d; %.2f)\nf05 (%d; %.2f)\n\n" %
212   - (results.best_precision()[0],results.best_precision()[1],
213   - results.best_f1()[0],results.best_f1()[1],
214   - results.best_f05()[0],results.best_f05()[1]))
215   - f.write("# recommendation size 10\nprecision (10; %.2f)\nf1 (10; %.2f)\nf05 (10; %.2f)" %
216   - (precision_10,f1_10,f05_10))
217   - precision = results.get_precision_summary()
218   - recall = results.get_recall_summary()
219   - f1 = results.get_f1_summary()
220   - f05 = results.get_f05_summary()
221   - accuracy = results.get_accuracy_summary()
222   - plot_summary(precision,recall,f1,f05,accuracy,log_file)
223   -
224   -def run_content(user,cfg):
225   - for strategy in content_based:
226   - cfg.strategy = strategy
227   - for size in profile_size:
228   - cfg.profile_size = size
229   - run_strategy(cfg,user)
230   -
231   -def run_collaborative(user,cfg):
232   - popcon_desktopapps = cfg.popcon_desktopapps
233   - popcon_programs = cfg.popcon_programs
234   - for strategy in collaborative:
235   - cfg.strategy = strategy
236   - for k in neighbors:
237   - cfg.k_neighbors = k
238   - #for size in popcon_size:
239   - # if size:
240   - # cfg.popcon_desktopapps = popcon_desktopapps+"_"+size
241   - # cfg.popcon_programs = popcon_programs+"_"+size
242   - run_strategy(cfg,user)
243   -
244   -def run_hybrid(user,cfg):
245   - popcon_desktopapps = cfg.popcon_desktopapps
246   - popcon_programs = cfg.popcon_programs
247   - for strategy in hybrid:
248   - cfg.strategy = strategy
249   - for k in neighbors:
250   - cfg.k_neighbors = k
251   - #for size in popcon_size:
252   - # if size:
253   - # cfg.popcon_desktopapps = popcon_desktopapps+"_"+size
254   - # cfg.popcon_programs = popcon_programs+"_"+size
255   - for size in profile_size:
256   - cfg.profile_size = size
257   - run_strategy(cfg,user)
258   -
259   -if __name__ == '__main__':
260   - #user = LocalSystem()
261   - #user = RandomPopcon(cfg.popcon_dir,os.path.join(cfg.filters_dir,"desktopapps"))
262   -
263   - cfg = Config()
264   - user = PopconSystem("/root/.app-recommender/popcon-entries/8b/8b44fcdbcf676e711a153d5db09979d7")
265   - #user = PopconSystem("/root/.app-recommender/popcon-entries/4a/4a67a295ec14826db2aa1d90be2f1623")
266   - user.filter_pkg_profile(cfg.pkgs_filter)
267   - user.maximal_pkg_profile()
268   -
269   - if "content" in sys.argv or len(sys.argv)<2:
270   - run_content(user,cfg)
271   - if "collaborative" in sys.argv or len(sys.argv)<2:
272   - run_collaborative(user,cfg)
273   - if "hybrid" in sys.argv or len(sys.argv)<2:
274   - run_hybrid(user,cfg)