Clustering initial implementation.

Tássia Camões Araújo
1 parent 6c99e7cd
Showing 2 changed files with 197 additions and 0 deletions Show diff stats
src/clustering.py
src/data.py
@@ -0,0 +1,46 @@
+#!/usr/bin/python
+
+#  Clustering - a python script to perform clustering of popcon data.
+#
+#  Copyright (C) 2010  Tassia Camoes <tassia@gmail.com>
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import logging
+import datetime
+from datetime import timedelta
+
+from config import *
+from data import *
+from similarity import *
+from error import Error
+
+if __name__ == '__main__':
+    try:
+        cfg = Config()
+        begin_time = datetime.datetime.now()
+        logging.info("Clustering computation started at %s" % begin_time)
+
+        cl = PopconClusteredData(cfg)
+
+        end_time = datetime.datetime.now()
+        logging.info("Clustering computation completed at %s" % end_time)
+        delta = end_time - begin_time
+        logging.info("Time elapsed: %d seconds." % delta.seconds)
+
+    except Error:
+        logging.critical("Aborting proccess. Use '--debug' for more details.")
+
@@ -29,6 +29,8 @@ import hashlib
 from error import Error
 from singleton import Singleton
+import cluster
+from similarity import *
 class Item:
     """
@@ -258,3 +260,152 @@ class PopconXapianIndex(xapian.WritableDatabase,Singleton):
         	gc.collect()
         # flush to disk database changes
         self.flush()
+
+class PopconSubmission():
+    def __init__(self,submission_hash):
+        self.hash = submission_hash
+        self.pkgs_list = []
+
+    def add_pkg(self,pkg):
+        self.pkgs_list.append(pkg)
+
+class PopconClusteredData(Singleton):
+    """
+    Data source for popcon submissions defined as a singleton xapian database.
+    """
+    def __init__(self,cfg):
+        """
+        Set initial attributes.
+        """
+        self.popcon_dir = os.path.expanduser(cfg.popcon_dir)
+        self.clusters_dir = os.path.expanduser(cfg.clusters_dir)
+        self.submissions = []
+        self.clustering()
+
+    def parse_submission(self,submission_path,binary=1):
+    	"""
+    	Parse a popcon submission, generating the names of the valid packages
+        in the vote.
+    	"""
+        submission_file = open(submission_path)
+    	for line in submission_file:
+            if not line.startswith("POPULARITY"):
+                if not line.startswith("END-POPULARITY"):
+                    data = line[:-1].split(" ")
+                    if len(data) > 3:
+                        if binary:
+                            # every installed package has the same weight
+                            yield data[2], 1
+                        elif data[3] == '<NOFILES>':
+                            # No executable files to track
+                            yield data[2], 1
+                        elif len(data) == 4:
+                            # Recently used packages
+                            yield data[2], 10
+                        elif data[4] == '<OLD>':
+                            # Unused packages
+                            yield data[2], 3
+                        elif data[4] == '<RECENT-CTIME>':
+                            # Recently installed packages
+                            yield data[2], 8
+
+    def clustering(self):
+        """
+        called by init
+        Create a xapian index for popcon submissions at 'popcon_dir' and
+        place it at 'self.path'.
+        """
+        if not os.path.exists(self.clusters_dir):
+            os.makedirs(self.clusters_dir)
+
+        logging.info("Clustering popcon submissions from \'%s\'" %
+                     self.popcon_dir)
+        logging.info("Clusters will be placed at \'%s\'" % self.clusters_dir)
+
+        for root, dirs, files in os.walk(self.popcon_dir):
+            for submission_hash in files:
+                s = PopconSubmission(submission_hash)
+                submission_path = os.path.join(root, submission_hash)
+                logging.debug("Parsing popcon submission \'%s\'" %
+                              submission_hash)
+                for pkg, freq in self.parse_submission(submission_path):
+                    s.add_pkg(pkg)
+                self.submissions.append(s)
+
+        distanceFunction = JaccardIndex()
+        cl = cluster.HierarchicalClustering(self.submissions,lambda x,y: distanceFunction(x.pkgs_list,y.pkgs_list))
+        clusters = cl.getlevel(0.5)
+        for c in clusters:
+            print "cluster"
+            for submission in c:
+                print submission.hash
+        #cl = KMeansClusteringPopcon(self.submissions,
+        #                            lambda x,y: distanceFunction(x.pkgs_list,y.pkgs_list))
+        #clusters = cl.getclusters(2)
+        #medoids = cl.getMedoids(2)
+
+class KMedoidsClusteringPopcon(cluster.KMeansClustering):
+
+    def __init__(self,data,distance):
+        cluster.KMeansClustering.__init__(self, data, distance)
+        self.distanceMatrix = {}
+        for submission in self._KMeansClustering__data:
+            self.distanceMatrix[submission.hash] = {}
+
+    def loadDistanceMatrix(self,cluster):
+        for i in range(len(cluster)-1):
+            for j in range(i+1,len(cluster)):
+                try:
+                    d = self.distanceMatrix[cluster[i].hash][cluster[j].hash]
+                    logging.debug("Using d[%d,%d]" % (i,j))
+                except:
+                    d = self.distance(cluster[i],cluster[j])
+                    self.distanceMatrix[cluster[i].hash][cluster[j].hash] = d
+                    self.distanceMatrix[cluster[j].hash][cluster[i].hash] = d
+                    logging.debug("d[%d,%d] = %.2f" % (i,j,d))
+
+    def getMedoid(self,cluster):
+        """
+        Return the medoid popcon submission of a given a cluster, based on
+        the distance function.
+        """
+        logging.debug("Cluster size: %d" % len(cluster))
+        self.loadDistanceMatrix(cluster)
+        medoidDistance = sys.maxint
+        for i in range(len(cluster)):
+            totalDistance = sum(self.distanceMatrix[cluster[i].hash].values())
+            print "totalDistance[",i,"]=",totalDistance
+            if totalDistance < centroidDistance:
+                medoidDistance = totalDistance
+                medoid = i
+            print "medoidDistance:",medoidDistance
+        logging.debug("Cluster medoid: [%d] %s" % (medoid, cluster[medoid].hash))
+        return cluster[medoid]
+
+    def assign_item(self, item, origin):
+        """
+        Assigns an item from a given cluster to the closest located cluster
+
+        PARAMETERS
+           item   - the item to be moved
+           origin - the originating cluster
+        """
+        closest_cluster = origin
+        for cluster in self._KMeansClustering__clusters:
+            if self.distance(item,self.getMedoid(cluster)) < self.distance(item,self.getMedoid(closest_cluster)):
+                closest_cluster = cluster
+
+        if closest_cluster != origin:
+            self.move_item(item, origin, closest_cluster)
+            logging.debug("Item changed cluster: %s" % item.hash)
+            return True
+        else:
+            return False
+
+    def getMedoids(self,n):
+        """
+        Generate n clusters and return their medoids.
+        """
+        medoids = [self.getMedoid(cluster) for cluster in self.getclusters(n)]
+        logging.info("Clustering completed and the following centroids were found: %s" % [c.hash for c in medoids])
+        return medoids
@@ -0,0 +1,46 @@		@@ -0,0 +1,46 @@
	1	+#!/usr/bin/python
	2	+
	3	+# Clustering - a python script to perform clustering of popcon data.
	4	+#
	5	+# Copyright (C) 2010 Tassia Camoes <tassia@gmail.com>
	6	+#
	7	+# This program is free software: you can redistribute it and/or modify
	8	+# it under the terms of the GNU General Public License as published by
	9	+# the Free Software Foundation, either version 3 of the License, or
	10	+# (at your option) any later version.
	11	+#
	12	+# This program is distributed in the hope that it will be useful,
	13	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	+# GNU General Public License for more details.
	16	+#
	17	+# You should have received a copy of the GNU General Public License
	18	+# along with this program. If not, see <http://www.gnu.org/licenses/>.
	19	+
	20	+import os
	21	+import sys
	22	+import logging
	23	+import datetime
	24	+from datetime import timedelta
	25	+
	26	+from config import *
	27	+from data import *
	28	+from similarity import *
	29	+from error import Error
	30	+
	31	+if __name__ == '__main__':
	32	+ try:
	33	+ cfg = Config()
	34	+ begin_time = datetime.datetime.now()
	35	+ logging.info("Clustering computation started at %s" % begin_time)
	36	+
	37	+ cl = PopconClusteredData(cfg)
	38	+
	39	+ end_time = datetime.datetime.now()
	40	+ logging.info("Clustering computation completed at %s" % end_time)
	41	+ delta = end_time - begin_time
	42	+ logging.info("Time elapsed: %d seconds." % delta.seconds)
	43	+
	44	+ except Error:
	45	+ logging.critical("Aborting proccess. Use '--debug' for more details.")
	46	+
	@@ -29,6 +29,8 @@ import hashlib		@@ -29,6 +29,8 @@ import hashlib
29		29
30	from error import Error	30	from error import Error
31	from singleton import Singleton	31	from singleton import Singleton
		32	+import cluster
		33	+from similarity import *
32		34
33	class Item:	35	class Item:
34	"""	36	"""
	@@ -258,3 +260,152 @@ class PopconXapianIndex(xapian.WritableDatabase,Singleton):		@@ -258,3 +260,152 @@ class PopconXapianIndex(xapian.WritableDatabase,Singleton):
258	gc.collect()	260	gc.collect()
259	# flush to disk database changes	261	# flush to disk database changes
260	self.flush()	262	self.flush()
		263	+
		264	+class PopconSubmission():
		265	+ def __init__(self,submission_hash):
		266	+ self.hash = submission_hash
		267	+ self.pkgs_list = []
		268	+
		269	+ def add_pkg(self,pkg):
		270	+ self.pkgs_list.append(pkg)
		271	+
		272	+class PopconClusteredData(Singleton):
		273	+ """
		274	+ Data source for popcon submissions defined as a singleton xapian database.
		275	+ """
		276	+ def __init__(self,cfg):
		277	+ """
		278	+ Set initial attributes.
		279	+ """
		280	+ self.popcon_dir = os.path.expanduser(cfg.popcon_dir)
		281	+ self.clusters_dir = os.path.expanduser(cfg.clusters_dir)
		282	+ self.submissions = []
		283	+ self.clustering()
		284	+
		285	+ def parse_submission(self,submission_path,binary=1):
		286	+ """
		287	+ Parse a popcon submission, generating the names of the valid packages
		288	+ in the vote.
		289	+ """
		290	+ submission_file = open(submission_path)
		291	+ for line in submission_file:
		292	+ if not line.startswith("POPULARITY"):
		293	+ if not line.startswith("END-POPULARITY"):
		294	+ data = line[:-1].split(" ")
		295	+ if len(data) > 3:
		296	+ if binary:
		297	+ # every installed package has the same weight
		298	+ yield data[2], 1
		299	+ elif data[3] == '<NOFILES>':
		300	+ # No executable files to track
		301	+ yield data[2], 1
		302	+ elif len(data) == 4:
		303	+ # Recently used packages
		304	+ yield data[2], 10
		305	+ elif data[4] == '<OLD>':
		306	+ # Unused packages
		307	+ yield data[2], 3
		308	+ elif data[4] == '<RECENT-CTIME>':
		309	+ # Recently installed packages
		310	+ yield data[2], 8
		311	+
		312	+ def clustering(self):
		313	+ """
		314	+ called by init
		315	+ Create a xapian index for popcon submissions at 'popcon_dir' and
		316	+ place it at 'self.path'.
		317	+ """
		318	+ if not os.path.exists(self.clusters_dir):
		319	+ os.makedirs(self.clusters_dir)
		320	+
		321	+ logging.info("Clustering popcon submissions from \'%s\'" %
		322	+ self.popcon_dir)
		323	+ logging.info("Clusters will be placed at \'%s\'" % self.clusters_dir)
		324	+
		325	+ for root, dirs, files in os.walk(self.popcon_dir):
		326	+ for submission_hash in files:
		327	+ s = PopconSubmission(submission_hash)
		328	+ submission_path = os.path.join(root, submission_hash)
		329	+ logging.debug("Parsing popcon submission \'%s\'" %
		330	+ submission_hash)
		331	+ for pkg, freq in self.parse_submission(submission_path):
		332	+ s.add_pkg(pkg)
		333	+ self.submissions.append(s)
		334	+
		335	+ distanceFunction = JaccardIndex()
		336	+ cl = cluster.HierarchicalClustering(self.submissions,lambda x,y: distanceFunction(x.pkgs_list,y.pkgs_list))
		337	+ clusters = cl.getlevel(0.5)
		338	+ for c in clusters:
		339	+ print "cluster"
		340	+ for submission in c:
		341	+ print submission.hash
		342	+ #cl = KMeansClusteringPopcon(self.submissions,
		343	+ # lambda x,y: distanceFunction(x.pkgs_list,y.pkgs_list))
		344	+ #clusters = cl.getclusters(2)
		345	+ #medoids = cl.getMedoids(2)
		346	+
		347	+class KMedoidsClusteringPopcon(cluster.KMeansClustering):
		348	+
		349	+ def __init__(self,data,distance):
		350	+ cluster.KMeansClustering.__init__(self, data, distance)
		351	+ self.distanceMatrix = {}
		352	+ for submission in self._KMeansClustering__data:
		353	+ self.distanceMatrix[submission.hash] = {}
		354	+
		355	+ def loadDistanceMatrix(self,cluster):
		356	+ for i in range(len(cluster)-1):
		357	+ for j in range(i+1,len(cluster)):
		358	+ try:
		359	+ d = self.distanceMatrix[cluster[i].hash][cluster[j].hash]
		360	+ logging.debug("Using d[%d,%d]" % (i,j))
		361	+ except:
		362	+ d = self.distance(cluster[i],cluster[j])
		363	+ self.distanceMatrix[cluster[i].hash][cluster[j].hash] = d
		364	+ self.distanceMatrix[cluster[j].hash][cluster[i].hash] = d
		365	+ logging.debug("d[%d,%d] = %.2f" % (i,j,d))
		366	+
		367	+ def getMedoid(self,cluster):
		368	+ """
		369	+ Return the medoid popcon submission of a given a cluster, based on
		370	+ the distance function.
		371	+ """
		372	+ logging.debug("Cluster size: %d" % len(cluster))
		373	+ self.loadDistanceMatrix(cluster)
		374	+ medoidDistance = sys.maxint
		375	+ for i in range(len(cluster)):
		376	+ totalDistance = sum(self.distanceMatrix[cluster[i].hash].values())
		377	+ print "totalDistance[",i,"]=",totalDistance
		378	+ if totalDistance < centroidDistance:
		379	+ medoidDistance = totalDistance
		380	+ medoid = i
		381	+ print "medoidDistance:",medoidDistance
		382	+ logging.debug("Cluster medoid: [%d] %s" % (medoid, cluster[medoid].hash))
		383	+ return cluster[medoid]
		384	+
		385	+ def assign_item(self, item, origin):
		386	+ """
		387	+ Assigns an item from a given cluster to the closest located cluster
		388	+
		389	+ PARAMETERS
		390	+ item - the item to be moved
		391	+ origin - the originating cluster
		392	+ """
		393	+ closest_cluster = origin
		394	+ for cluster in self._KMeansClustering__clusters:
		395	+ if self.distance(item,self.getMedoid(cluster)) < self.distance(item,self.getMedoid(closest_cluster)):
		396	+ closest_cluster = cluster
		397	+
		398	+ if closest_cluster != origin:
		399	+ self.move_item(item, origin, closest_cluster)
		400	+ logging.debug("Item changed cluster: %s" % item.hash)
		401	+ return True
		402	+ else:
		403	+ return False
		404	+
		405	+ def getMedoids(self,n):
		406	+ """
		407	+ Generate n clusters and return their medoids.
		408	+ """
		409	+ medoids = [self.getMedoid(cluster) for cluster in self.getclusters(n)]
		410	+ logging.info("Clustering completed and the following centroids were found: %s" % [c.hash for c in medoids])
		411	+ return medoids