Merge remote branch 'upstream/master'

Conflicts: src/data.py

Merge remote branch 'upstream/master'
Conflicts: src/data.py
Tássia Camões Araújo
2 parents e3176f2b 9e602af3
Showing 15 changed files with 198 additions and 122 deletions Show diff stats
src/config.py
src/data.py
src/evaluation.py
src/recommender.py
src/tests/data_tests.py
src/tests/test_data/popcon_dir/test_popcon_0
src/tests/test_data/popcon_dir/test_popcon_1
src/tests/test_data/popcon_dir/test_popcon_2
src/tests/test_data/popcon_dir/test_popcon_3
src/tests/test_data/popcon_dir/test_popcon_4
src/tests/test_data/popcon_dir/test_popcon_5
src/tests/test_data/popcon_dir/test_popcon_6
src/tests/test_data/popcon_dir/test_popcon_7
src/tests/test_data/popcon_dir/test_popcon_8
src/user.py
@@ -44,7 +44,8 @@ class Config():
         self.popcon_index = os.path.expanduser("~/.app-recommender/popcon_index")
         self.popcon_dir = os.path.expanduser("~/.app-recommender/popcon_dir")
         self.clusters_dir = os.path.expanduser("~/.app-recommender/clusters_dir")
-        self.index_mode = "0"     # use old index
+        self.k_medoids = 100
+        self.index_mode = "old"
         self.strategy = "cb"
         self.weight = "bm25"
         self.load_options()
@@ -65,8 +66,9 @@ class Config():
         print "  -a, --axi=PATH          Path to Apt-xapian-index"
         print "  -p, --popconindex=PATH  Path to popcon dedicated index"
         print "  -m, --popcondir=PATH    Path to popcon submissions dir"
-        print "  -u, --index_mode=       0: old, 1:reindex, 11:clustered_index"
+        print "  -u, --indexmode=        old, reindex, cluster, recluster"
         print "  -l, --clustersdir=PATH  Path to popcon clusters dir"
+        print "  -e, --medoids=k         Number of medoids for clustering"          
         print "  -w, --weight=OPTION     Search weighting scheme"
         print "  -s, --strategy=OPTION   Recommendation strategy"
         print ""
@@ -115,13 +117,14 @@ class Config():
         self.popcon_dir = self.read_option('recommender', 'popcon_dir')
         self.index_mode = self.read_option('recommender', 'index_mode')
         self.clusters_dir = self.read_option('recommender', 'clusters_dir')
+        self.k_medoids = self.read_option('recommender', 'k_medoids')
         self.weight = self.read_option('recommender', 'weight')
         self.strategy = self.read_option('recommender', 'strategy')
-        short_options = "hdvo:c:a:p:m:ul:w:s:"
+        short_options = "hdvo:c:a:p:m:ul:e:w:s:"
         long_options = ["help", "debug", "verbose", "output=", "config=",
-                        "axi=", "popconindex=", "popcondir=", "index_mode=",
-                        "clusters_dir=", "weight=", "strategy="]
+                        "axi=", "popconindex=", "popcondir=", "indexmode=",
+                        "clustersdir=", "kmedoids=", "weight=", "strategy="]
         try:
             opts, args = getopt.getopt(sys.argv[1:], short_options,
                                        long_options)
@@ -154,6 +157,8 @@ class Config():
                 self.index_mode = p
             elif o in ("-l", "--clustersdir"):
                 self.clusters_dir = p
+            elif o in ("-e", "--kmedoids"):
+                self.k_medoids = p
             elif o in ("-w", "--weight"):
                 self.weight = p
             elif o in ("-s", "--strategy"):
@@ -129,31 +129,36 @@ class PopconXapianIndex(xapian.WritableDatabase):
         """
         self.axi = xapian.Database(cfg.axi)
         self.path = os.path.expanduser(cfg.popcon_index)
-        if cfg.index_mode.startswith("1") or not self.load_index():
+        if not cfg.index_mode == "old" or not self.load_index():
             if not os.path.exists(cfg.popcon_dir):
                 os.makedirs(cfg.popcon_dir)
             if not os.listdir(cfg.popcon_dir):
                 logging.critical("Popcon dir seems to be empty.")
                 raise Error
-            if cfg.index_mode == "10":
+            if cfg.index_mode == "reindex":
                 self.source_dir = os.path.expanduser(cfg.popcon_dir)
             else:
                 self.source_dir = os.path.expanduser(cfg.clusters_dir)
                 if not os.path.exists(cfg.clusters_dir):
                     os.makedirs(cfg.clusters_dir)
-                if not os.listdir(cfg.clusters_dir):
-                    distance = JaccardDistance()
+                if not os.listdir(cfg.clusters_dir) or \
+                    cfg.index_mode == "recluster":
+                    shutil.rmtree(cfg.clusters_dir,1)
+                    os.makedirs(cfg.clusters_dir)
                     logging.info("Clustering popcon submissions from \'%s\'"
                                  % cfg.popcon_dir)
                     logging.info("Clusters will be placed at \'%s\'"
                                  % cfg.clusters_dir)
+                    distance = JaccardDistance()
                     data = self.get_submissions(cfg.popcon_dir)
-                    if cfg.clustering == "Hierarchical":
-                        self.hierarchical_clustering(data,cfg.clusters_dir,
-                                                     distance)
-                    else:
-                        self.kmedoids_clustering(data,cfg.clusters_dir,
-                                                 distance)
+                    self.cluster_dispersion = \
+                        self.kmedoids_clustering(data, cfg.clusters_dir,
+                                                 distance, cfg.k_medoids)
+                    logging.info("Clusters dispersion: %f.2",
+                                 self.cluster_dispersion)
+                else:
+                    logging.info("Using clusters from \'%s\'" %
+                                 cfg.clusters_dir)
             self.build_index()
     def __str__(self):
@@ -167,10 +172,9 @@ class PopconXapianIndex(xapian.WritableDatabase):
             logging.info("Opening existing popcon xapian index at \'%s\'"
                           % self.path)
             xapian.Database.__init__(self,self.path)
-            return True
+            return 1
         except xapian.DatabaseError:
             logging.info("Could not open popcon index.")
-            return True
             return 0
     def build_index(self):
@@ -224,35 +228,23 @@ class PopconXapianIndex(xapian.WritableDatabase):
                 submissions.append(submission)
         return submissions
-    def hierarchical_clustering(self,data,clusters_dir,distance,k=10):
-        """
-        Select popcon submissions from popcon_dir and place them at clusters_dir
-        """
-        cl = cluster.HierarchicalClustering(data, lambda x,y:
-                                                  distance(x.packages.keys(),
-                                                           y.packages.keys()))
-        clusters = cl.getlevel(0.5)
-        for c in clusters:
-            print "cluster"
-            for submission in c:
-                print submission.user_id
-
-    def kmedoids_clustering(self,data,clusters_dir,distance,k=10):
+    def kmedoids_clustering(self,data,clusters_dir,distance,k_medoids):
         clusters = KMedoidsClustering(data,lambda x,y:
                                            distance(x.packages.keys(),
                                                     y.packages.keys()))
-        medoids = clusters.getMedoids(2)
+        medoids,dispersion = clusters.getMedoids(k_medoids)
         for submission in medoids:
             shutil.copyfile(submission.path,os.path.join(clusters_dir,
                                                          submission.user_id))
+        return dispersion
 class KMedoidsClustering(cluster.KMeansClustering):
-    def __init__(self,data,distance):
-        if len(data)<100:
+    def __init__(self,data,distance,max_data=100):
+        if len(data)<max_data:
             data_sample = data
         else:
-            data_sample = random.sample(data,100)
+            data_sample = random.sample(data,max_data)
         cluster.KMeansClustering.__init__(self, data_sample, distance)
         self.distanceMatrix = {}
         for submission in self._KMeansClustering__data:
@@ -287,7 +279,7 @@ class KMedoidsClustering(cluster.KMeansClustering):
             logging.debug("medoidDistance: %f" % medoidDistance)
         logging.debug("Cluster medoid: [%d] %s" % (medoid,
                                                    cluster[medoid].user_id))
-        return cluster[medoid]
+        return (cluster[medoid],medoidDistance)
     def assign_item(self, item, origin):
         """
@@ -295,7 +287,8 @@ class KMedoidsClustering(cluster.KMeansClustering):
         """
         closest_cluster = origin
         for cluster in self._KMeansClustering__clusters:
-            if self.distance(item,self.getMedoid(cluster)) < self.distance(item,self.getMedoid(closest_cluster)):
+            if self.distance(item,self.getMedoid(cluster)[0]) < \
+                self.distance(item,self.getMedoid(closest_cluster)[0]):
                 closest_cluster = cluster
         if closest_cluster != origin:
@@ -309,6 +302,8 @@ class KMedoidsClustering(cluster.KMeansClustering):
         """
         Generate n clusters and return their medoids.
         """
-        medoids = [self.getMedoid(cluster) for cluster in self.getclusters(n)]
-        logging.info("Clustering completed and the following centroids were found: %s" % [c.user_id for c in medoids])
-        return medoids
+        medoids_distances = [self.getMedoid(cluster) for cluster in self.getclusters(n)]
+        medoids = [m[0] for m in medoids_distances]
+        dispersion = sum([m[1] for m in medoids_distances])
+        logging.info("Clustering completed and the following medoids were found: %s" % [c.user_id for c in medoids])
+        return medoids,dispersion
@@ -49,6 +49,45 @@ class Metric(Singleton):
                           evaluation.real_item_scores[k]))
         return errors
+
+class SimpleAccuracy(Metric):
+    """
+    Classification accuracy metric which consider classes sizes.
+    """
+    def __init__(self):
+        """
+        Set metric description.
+        """
+        self.desc = "  S_Accuracy  "
+
+    def run(self,evaluation):
+        """
+        Compute metric.
+        """
+        return float((evaluation.repository_size-
+                      len(evaluation.false_positive))-
+                      len(evaluation.false_negative))/evaluation.repository_size
+
+class Accuracy(Metric):
+    """
+    Classification accuracy metric which consider classes sizes.
+    """
+    def __init__(self):
+        """
+        Set metric description.
+        """
+        self.desc = "    Accuracy  "
+
+    def run(self,evaluation):
+        """
+        Compute metric.
+        """
+        error_1 = (float(len(evaluation.false_positive))/
+               (evaluation.repository_size-len(evaluation.real_relevant)))
+        error_2 = (float(len(evaluation.false_negative))/len(evaluation.real_relevant))
+        accuracy = 1-(float(error_1+error_2)/2)
+        return accuracy
+
 class Precision(Metric):
     """
     Classification accuracy metric defined as the percentage of relevant itens
@@ -64,7 +103,7 @@ class Precision(Metric):
         """
         Compute metric.
         """
-        return float(len(evaluation.predicted_real))/len(evaluation.predicted_relevant)
+        return float(len(evaluation.true_positive))/len(evaluation.predicted_relevant)
 class Recall(Metric):
     """
@@ -81,7 +120,7 @@ class Recall(Metric):
         """
         Compute metric.
         """
-        return float(len(evaluation.predicted_real))/len(evaluation.real_relevant)
+        return float(len(evaluation.true_positive))/len(evaluation.real_relevant)
 class F1(Metric):
     """
@@ -100,7 +139,10 @@ class F1(Metric):
         """
         p = Precision().run(evaluation)
         r = Recall().run(evaluation)
-        return float((2*p*r))/(p+r)
+        if (p+r)>0:
+            return float((2*p*r))/(p+r)
+        else:
+            return 0
 class MAE(Metric):
     """
@@ -158,43 +200,47 @@ class Coverage(Metric):
     Evaluation metric defined as the percentage of itens covered by the
     recommender (have been recommended at least once).
     """
-    def __init__(self,repository_size):
+    def __init__(self):
         """
         Set initial parameters.
         """
         self.desc = "   Coverage  "
-        self.repository_size = repository_size
-        self.covered = set()
-
-    def save_covered(self,recommended_list):
-        """
-        Register that a list of itens has been recommended.
-        """
-        self.covered.update(set(recommended_list))
-    def run(self,evaluation):
+    def run(self,evaluations_set):
         """
         Compute metric.
         """
-        return float(self.covered.size)/self.repository_size
+        covered = set()
+        for evaluation in evaluations_set:
+            covered.update(set(evaluation.predicted_relevant))
+        return float(len(covered))/evaluation.repository_size
 class Evaluation:
     """
     Class designed to perform prediction evaluation, given data and metric.
     """
-    def __init__(self,predicted_result,real_result):
+    def __init__(self,predicted,real,repository_size):
         """
         Set initial parameters.
         """
-        self.predicted_item_scores = predicted_result.item_score
-        self.predicted_relevant = predicted_result.get_prediction()
-        self.real_item_scores = real_result.item_score
-        self.real_relevant = real_result.get_prediction()
-        self.predicted_real = [v for v in self.predicted_relevant if v in
-                               self.real_relevant]
-        #print len(self.predicted_relevant)
-        #print len(self.real_relevant)
-        #print len(self.predicted_real)
+        self.repository_size = repository_size
+        self.predicted_item_scores = predicted.item_score
+        self.predicted_relevant = predicted.get_prediction()
+        self.real_item_scores = real.item_score
+        self.real_relevant = real.get_prediction()
+
+        self.true_positive = [v[0] for v in self.predicted_relevant if v[0] in
+                              [w[0] for w in self.real_relevant]]
+        self.false_positive = [v[0] for v in self.predicted_relevant if not v[0] in
+                               [w[0] for w in self.real_relevant]]
+        self.false_negative = [v[0] for v in self.real_relevant if not v[0] in
+                               [w[0] for w in self.predicted_relevant]]
+
+        logging.debug("TP: %d" % len(self.true_positive))
+        logging.debug("FP: %d" % len(self.false_positive))
+        logging.debug("FN: %d" % len(self.false_negative))
+        logging.debug("Repo_size: %d" % self.repository_size)
+        logging.debug("Relevant: %d" % len(self.real_relevant))
     def run(self,metric):
         """
@@ -206,7 +252,7 @@ class CrossValidation:
     """
     Class designed to perform cross-validation process.
     """
-    def __init__(self,partition_proportion,rounds,rec,metrics_list):
+    def __init__(self,partition_proportion,rounds,rec,metrics_list,result_proportion):
         """
         Set initial parameters.
         """
@@ -219,34 +265,13 @@ class CrossValidation:
         self.recommender = rec
         self.metrics_list = metrics_list
         self.cross_results = defaultdict(list)
-
-    def __str__(self):
-        """
-        String representation of the object.
-        """
-        str = "\n"
-        metrics_desc = ""
-        for metric in self.metrics_list:
-            metrics_desc += "%s|" % (metric.desc)
-        str += "| Round |%s\n" % metrics_desc
-        for r in range(self.rounds):
-            metrics_result = ""
-            for metric in self.metrics_list:
-                metrics_result += ("     %2.1f%%    |" %
-                                   (self.cross_results[metric.desc][r]*100))
-            str += "|   %d   |%s\n" % (r,metrics_result)
-        metrics_mean = ""
-        for metric in self.metrics_list:
-            mean = float(sum(self.cross_results[metric.desc]) /
-                         len(self.cross_results[metric.desc]))
-            metrics_mean += "     %2.1f%%    |" % (mean*100)
-        str += "|  Mean |%s\n" % (metrics_mean)
-        return str
+        self.result_proportion = result_proportion
     def run(self,user):
         """
         Perform cross-validation.
         """
+        #
         cross_item_score = dict.fromkeys(user.pkg_profile,1)
         partition_size = int(len(cross_item_score)*self.partition_proportion)
         for r in range(self.rounds):
@@ -258,10 +283,17 @@ class CrossValidation:
                     logging.critical("Empty cross_item_score.")
                     raise Error
                 round_partition[random_key] = cross_item_score.pop(random_key)
+            #logging.debug("Round partition: %s",str(round_partition))
+            #logging.debug("Cross item-score: %s",str(cross_item_score))
             round_user = User(cross_item_score)
-            predicted_result = self.recommender.get_recommendation(round_user)
-            real_result = RecommendationResult(round_partition,len(round_partition))
-            evaluation = Evaluation(predicted_result,real_result)
+            result_size = int(self.recommender.items_repository.get_doccount()*
+                              self.result_proportion)
+            predicted_result = self.recommender.get_recommendation(round_user,result_size)
+            print len(round_partition)
+            real_result = RecommendationResult(round_partition)
+            #logging.debug("Predicted result: %s",predicted_result)
+            evaluation = Evaluation(predicted_result,real_result,
+                                    self.recommender.items_repository.get_doccount())
             for metric in self.metrics_list:
                 result = evaluation.run(metric)
                 self.cross_results[metric.desc].append(result)
@@ -269,3 +301,26 @@ class CrossValidation:
                 item,score = round_partition.popitem()
                 cross_item_score[item] = score
+    def __str__(self):
+        """
+        String representation of the object.
+        """
+        str = "\n"
+        metrics_desc = ""
+        for metric in self.metrics_list:
+            metrics_desc += "%s|" % (metric.desc)
+        str += "| Round |%s\n" % metrics_desc
+        for r in range(self.rounds):
+            metrics_result = ""
+            for metric in self.metrics_list:
+                metrics_result += ("     %2.1f%%    |" %
+                                   (self.cross_results[metric.desc][r]*100))
+            str += "|   %d   |%s\n" % (r,metrics_result)
+        metrics_mean = ""
+        for metric in self.metrics_list:
+            mean = float(sum(self.cross_results[metric.desc]) /
+                         len(self.cross_results[metric.desc]))
+            metrics_mean += "     %2.1f%%    |" % (mean*100)
+        str += "|  Mean |%s\n" % (metrics_mean)
+        return str
+
@@ -45,13 +45,15 @@ class RecommendationResult:
             str += "%2d: %s\n" % (i,result[i][0])
         return str
-    def get_prediction(self,limit=20):
+    def get_prediction(self,limit=0):
         """
         Return prediction based on recommendation size (number of items).
         """
-        if limit > self.size: limit = self.size
         sorted_result = sorted(self.item_score.items(),
                                key=operator.itemgetter(1))
+        if not limit or limit > self.size:
+            limit = self.size
+
         return list(reversed(sorted_result[-limit:]))
 class Recommender:
@@ -63,13 +65,12 @@ class Recommender:
         Set initial parameters.
         """
         self.items_repository = xapian.Database(cfg.axi)
-        self.users_repository = data.PopconXapianIndex(cfg)
-        #self.clustered_users_repository = data.PopconXapianIndex(cfg)
         self.set_strategy(cfg.strategy)
         if cfg.weight == "bm25":
             self.weight = xapian.BM25Weight()
         else:
             self.weight = xapian.TradWeight()
+        self.cfg = cfg
     def set_strategy(self,strategy_str):
         """
@@ -83,6 +84,7 @@ class Recommender:
             self.strategy = strategy.ContentBasedStrategy("desc")
         if strategy_str == "col":
             self.strategy = strategy.CollaborativeStrategy(20)
+            self.users_repository = data.PopconXapianIndex(self.cfg)
     def get_recommendation(self,user,result_size=20):
         """
@@ -71,13 +71,13 @@ class PopconXapianIndexTests(unittest2.TestCase):
     def test_reindex(self):
         # force reindex with no clustering
-        self.cfg.index_mode = "10"
+        self.cfg.index_mode = "reindex"
         pxi = PopconXapianIndex(self.cfg)
         self.assertEqual(pxi.get_metadata("old"),"")
     def test_clustering(self):
         # force reindex with clustering
-        self.cfg.index_mode = "11"
+        self.cfg.index_mode = "cluster"
         pxi = PopconXapianIndex(self.cfg)
         self.assertEqual(pxi.source_dir,self.cfg.clusters_dir)
         all_submissions = [submissions for (root, dirs, submissions) in
@@ -95,6 +95,13 @@ class PopconXapianIndexTests(unittest2.TestCase):
                          sum([len(submissions) for submissions in
                               all_submissions]))
+    def test_recluster(self):
+        # force reindexing and clustering
+        self.cfg.index_mode = "recluster"
+        self.cfg.k_medoids = 2
+        pxi = PopconXapianIndex(self.cfg)
+        self.assertEqual(pxi.source_dir,self.cfg.clusters_dir)
+        self.assertEqual(pxi.get_doccount(),2)
 if __name__ == '__main__':
         unittest2.main()
 POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_0 ARCH:i386 POPCONVER:1.52
-1309407475 1303670994 perl-base /usr/bin/perl
-1309407451 1303670982 libc6-i686 /lib/i686/cmov/libc-2.11.2.so
-1309407450 1303670973 libc6 /lib/ld-2.11.2.so
+1309407475 1303670994 gimp /usr/bin/perl
+1309407451 1303670982 inkscape /lib/i686/cmov/libc-2.11.2.so
+1309407450 1303670973 imagination /lib/ld-2.11.2.so
 1309407434 1295654294 dash /bin/dash
 0 0 libusbmuxd1 <NOFILES>
 END-POPULARITY-CONTEST-0 TIME:1309407492
 POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_1 ARCH:i386 POPCONVER:1.52
-1309407475 1303670994 perl-base /usr/bin/perl
-1309407451 1303670982 libc6-i686 /lib/i686/cmov/libc-2.11.2.so
-1309407450 1303670973 libc6 /lib/ld-2.11.2.so
+1309407475 1303670994 gimp /usr/bin/perl
 1309407434 1295654294 dash /bin/dash
 0 0 libusbmuxd1 <NOFILES>
 END-POPULARITY-CONTEST-0 TIME:1309407492
 POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_2 ARCH:i386 POPCONVER:1.52
-1309407475 1303670994 perl-base /usr/bin/perl
-1309407451 1303670982 libc6-i686 /lib/i686/cmov/libc-2.11.2.so
+1309407475 1303670994 iceweasel /usr/bin/perl
+1309407451 1303670982 python /lib/i686/cmov/libc-2.11.2.so
 1309407450 1303670973 libc6 /lib/ld-2.11.2.so
 1309407434 1295654294 dash /bin/dash
 0 0 libusbmuxd1 <NOFILES>
 POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_3 ARCH:i386 POPCONVER:1.52
-1309407475 1303670994 perl-base /usr/bin/perl
-1309407451 1303670982 libc6-i686 /lib/i686/cmov/libc-2.11.2.so
+1309407475 1303670994 eog /usr/bin/perl
+1309407451 1303670982 nautilus /lib/i686/cmov/libc-2.11.2.so
 1309407450 1303670973 libc6 /lib/ld-2.11.2.so
-1309407434 1295654294 dash /bin/dash
 0 0 libusbmuxd1 <NOFILES>
 END-POPULARITY-CONTEST-0 TIME:1309407492
 POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_4 ARCH:i386 POPCONVER:1.52
-1309407475 1303670994 perl-base /usr/bin/perl
-1309407451 1303670982 libc6-i686 /lib/i686/cmov/libc-2.11.2.so
+1309407475 1303670994 konqueror /usr/bin/perl
+1309407451 1303670982 kedit /lib/i686/cmov/libc-2.11.2.so
 1309407450 1303670973 libc6 /lib/ld-2.11.2.so
-1309407434 1295654294 dash /bin/dash
 0 0 libusbmuxd1 <NOFILES>
 END-POPULARITY-CONTEST-0 TIME:1309407492
 POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_5 ARCH:i386 POPCONVER:1.52
-1309407475 1303670994 perl-base /usr/bin/perl
-1309407451 1303670982 libc6-i686 /lib/i686/cmov/libc-2.11.2.so
-1309407450 1303670973 libc6 /lib/ld-2.11.2.so
+1309407475 1303670994 konqueror /usr/bin/perl
 1309407434 1295654294 dash /bin/dash
 0 0 libusbmuxd1 <NOFILES>
 END-POPULARITY-CONTEST-0 TIME:1309407492
 POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_6 ARCH:i386 POPCONVER:1.52
 1309407475 1303670994 perl-base /usr/bin/perl
-1309407451 1303670982 libc6-i686 /lib/i686/cmov/libc-2.11.2.so
-1309407450 1303670973 libc6 /lib/ld-2.11.2.so
+1309407451 1303670982 eog /lib/i686/cmov/libc-2.11.2.so
+1309407450 1303670973 nautilus /lib/ld-2.11.2.so
 1309407434 1295654294 dash /bin/dash
 0 0 libusbmuxd1 <NOFILES>
 END-POPULARITY-CONTEST-0 TIME:1309407492
 POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_7 ARCH:i386 POPCONVER:1.52
-1309407475 1303670994 perl-base /usr/bin/perl
-1309407451 1303670982 libc6-i686 /lib/i686/cmov/libc-2.11.2.so
-1309407450 1303670973 libc6 /lib/ld-2.11.2.so
+1309407475 1303670994 apticron /usr/bin/perl
+1309407451 1303670982 aptitude /lib/i686/cmov/libc-2.11.2.so
+1309407450 1303670973 apt /lib/ld-2.11.2.so
 1309407434 1295654294 dash /bin/dash
 0 0 libusbmuxd1 <NOFILES>
 END-POPULARITY-CONTEST-0 TIME:1309407492
 POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_8 ARCH:i386 POPCONVER:1.52
-1309407475 1303670994 perl-base /usr/bin/perl
-1309407451 1303670982 libc6-i686 /lib/i686/cmov/libc-2.11.2.so
-1309407450 1303670973 libc6 /lib/ld-2.11.2.so
+1309407475 1303670994 apticron /usr/bin/perl
+1309407451 1303670982 eog /lib/i686/cmov/libc-2.11.2.so
+1309407450 1303670973 nautilus /lib/ld-2.11.2.so
 1309407434 1295654294 dash /bin/dash
 0 0 libusbmuxd1 <NOFILES>
 END-POPULARITY-CONTEST-0 TIME:1309407492
@@ -152,6 +152,24 @@ class User:
         desc_profile = self.desc_profile(items_repository,size)[:size/2]
         return tag_profile+desc_profile
+    def app_pkg_profile(self,axi):
+        """
+        Return list of packages that are applications.
+        """
+        old_profile_size = len(self.pkg_profile)
+        for p in self.pkg_profile[:]:     #iterate list copy
+            tags = data.axi_search_pkg_tags(axi,p)
+            try:
+
+                if not "XTrole::program" in tags:
+                    self.pkg_profile.remove(p)
+            except:
+                logging.debug("Package not found in axi: %s" % p)
+        profile_size = len(self.pkg_profile)
+        logging.debug("App package profile: reduced packages profile size \
+                       from %d to %d." % (old_profile_size, profile_size))
+        return self.pkg_profile
+
     def maximal_pkg_profile(self):
         """
         Return list of packages that are not dependence of any other package in
1	POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_1 ARCH:i386 POPCONVER:1.52	1	POPULARITY-CONTEST-0 TIME:1309407492 ID:8b44fcdbcf676e711a153d5db0test_1 ARCH:i386 POPCONVER:1.52
2	-1309407475 1303670994 perl-base /usr/bin/perl
3	-1309407451 1303670982 libc6-i686 /lib/i686/cmov/libc-2.11.2.so
4	-1309407450 1303670973 libc6 /lib/ld-2.11.2.so	2	+1309407475 1303670994 gimp /usr/bin/perl
5	1309407434 1295654294 dash /bin/dash	3	1309407434 1295654294 dash /bin/dash
6	0 0 libusbmuxd1 <NOFILES>	4	0 0 libusbmuxd1 <NOFILES>
7	END-POPULARITY-CONTEST-0 TIME:1309407492	5	END-POPULARITY-CONTEST-0 TIME:1309407492
	@@ -152,6 +152,24 @@ class User:		@@ -152,6 +152,24 @@ class User:
152	desc_profile = self.desc_profile(items_repository,size)[:size/2]	152	desc_profile = self.desc_profile(items_repository,size)[:size/2]
153	return tag_profile+desc_profile	153	return tag_profile+desc_profile
154		154
		155	+ def app_pkg_profile(self,axi):
		156	+ """
		157	+ Return list of packages that are applications.
		158	+ """
		159	+ old_profile_size = len(self.pkg_profile)
		160	+ for p in self.pkg_profile[:]: #iterate list copy
		161	+ tags = data.axi_search_pkg_tags(axi,p)
		162	+ try:
		163	+
		164	+ if not "XTrole::program" in tags:
		165	+ self.pkg_profile.remove(p)
		166	+ except:
		167	+ logging.debug("Package not found in axi: %s" % p)
		168	+ profile_size = len(self.pkg_profile)
		169	+ logging.debug("App package profile: reduced packages profile size \
		170	+ from %d to %d." % (old_profile_size, profile_size))
		171	+ return self.pkg_profile
		172	+
155	def maximal_pkg_profile(self):	173	def maximal_pkg_profile(self):
156	"""	174	"""
157	Return list of packages that are not dependence of any other package in	175	Return list of packages that are not dependence of any other package in