Merge branch 'master' of github.com:tassia/AppRecommender

Tássia Camões Araújo
2 parents c1d556c0 08c59678
Showing 3 changed files with 31 additions and 12 deletions Show diff stats
src/config.py
src/data.py
src/web/survey.py
@@ -40,7 +40,7 @@ class Config():
         self.output = "/dev/null"
         self.survey_mode = 0
         self.axi = "/var/lib/apt-xapian-index/index"
-#        self.dde_url = "http://dde.debian.net/dde/q/udd/packs/all/%s?t=json"
+        #self.dde_url = "http://dde.debian.net/dde/q/udd/packs/all/%s?t=json"
         self.dde_url = "http://46.4.235.200:8000/q/udd/packages/all/%s?t=json"
         self.popcon_index = os.path.expanduser("~/.app-recommender/popcon_index")
         self.popcon_dir = os.path.expanduser("~/.app-recommender/popcon_dir")
@@ -51,6 +51,9 @@ class Config():
         self.strategy = "cb"
         self.weight = "bm25"
         self.profile_size = 50
+        # options: maximal, voted, desktop
+        self.profiling = "maximal"
+        self.k_neighbors = 100
         self.load_options()
         self.set_logger()
@@ -77,7 +80,9 @@ class Config():
         print " [ recommender ]"
         print "  -w, --weight=OPTION        Search weighting scheme"
         print "  -s, --strategy=OPTION      Recommendation strategy"
-        print "  -z, --profile_size=SIZE    Size of user profile"
+        print "  -z, --profilesize=k        Size of user profile"
+        print "  -f, --profiling=OPTION     Profile filter strategy"
+        print "  -n, --neighbors=k          Size of neighborhood for collaboration"
         print ""
         print " [ weight options ] "
         print "  trad = traditional probabilistic weighting"
@@ -132,12 +137,15 @@ class Config():
         self.strategy = self.read_option('recommender', 'strategy')
         self.profile_size = int(self.read_option('recommender',
                                                  'profile_size'))
+        self.profiling = self.read_option('recommender', 'profiling')
+        self.k_neighbors = int(self.read_option('recommender',
+                                                 'k_neighbors'))
-        short_options = "hdvo:a:e:p:m:ul:c:x:w:s:z:"
+        short_options = "hdvo:a:e:p:m:ul:c:x:w:s:z:f:n:"
         long_options = ["help", "debug", "verbose", "output=",
                         "axi=", "dde=", "popconindex=", "popcondir=", "indexmode=",
-                        "clustersdir=", "kmedoids=", "max_popcon=", "weight=", "strategy=",
-                        "profile_size="]
+                        "clustersdir=", "kmedoids=", "maxpopcon=", "weight=", "strategy=",
+                        "profile_size=", "profiling=", "neighbors="]
         try:
             opts, args = getopt.getopt(sys.argv[1:], short_options,
                                        long_options)
@@ -178,7 +186,11 @@ class Config():
             elif o in ("-s", "--strategy"):
                 self.strategy = p
             elif o in ("-z", "--profile_size"):
-                self.strategy = int(p)
+                self.profile_size = int(p)
+            elif o in ("-z", "--profiling"):
+                self.profiling = p
+            elif o in ("-n", "--neighbors"):
+                self.k_neighbors = int(p)
             else:
                 assert False, "unhandled option"
@@ -45,7 +45,7 @@ def axi_search_pkg_tags(axi,pkg):
     enquire.set_query(xapian.Query("XP"+pkg))
     matches = enquire.get_mset(0,1)
     if not matches:
-        logging.debug("Package %s not found in items repository" % pkg)
+        #logging.debug("Package %s not found in items repository" % pkg)
         return []
     for m in matches:
         tags = [term.term for term in axi.get_document(m.docid).termlist() if
@@ -158,13 +158,14 @@ class PopconXapianIndex(xapian.WritableDatabase):
         self.axi = xapian.Database(cfg.axi)
         self.path = os.path.expanduser(cfg.popcon_index)
         self.source_dir = os.path.expanduser(cfg.popcon_dir)
+        self.max_popcon = cfg.max_popcon
         if not cfg.index_mode == "old" or not self.load_index():
             if not os.path.exists(cfg.popcon_dir):
                 os.makedirs(cfg.popcon_dir)
             if not os.listdir(cfg.popcon_dir):
                 logging.critical("Popcon dir seems to be empty.")
                 raise Error
-            if cfg.index_mode == "reindex":
+            if cfg.index_mode == "reindex" or cfg.index_mode == "old":
                 self.source_dir = os.path.expanduser(cfg.popcon_dir)
                 logging.debug(self.source_dir)
             else:
@@ -229,8 +230,13 @@ class PopconXapianIndex(xapian.WritableDatabase):
             logging.critical(str(e))
             raise Error
+        doc_count = 0
         for root, dirs, files in os.walk(self.source_dir):
+            if doc_count == self.max_popcon:
+                break
             for popcon_file in files:
+                if doc_count == self.max_popcon:
+                    break
                 submission = PopconSubmission(os.path.join(root, popcon_file))
                 doc = xapian.Document()
                 doc.set_data(submission.user_id)
@@ -238,10 +244,11 @@ class PopconXapianIndex(xapian.WritableDatabase):
                               submission.user_id)
                 for pkg, freq in submission.packages.items():
                     doc.add_term("XP"+pkg,freq)
-                    if axi_search_pkg_tags(self.axi,pkg):
-                        for tag in axi_search_pkg_tags(self.axi,pkg):
-                            doc.add_term(tag,freq)
+                    #if axi_search_pkg_tags(self.axi,pkg):
+                    #    for tag in axi_search_pkg_tags(self.axi,pkg):
+                    #        doc.add_term(tag,freq)
                 doc_id = self.add_document(doc)
+                doc_count += 1
                 logging.debug("Popcon Xapian: Indexing doc %d" % doc_id)
             # python garbage collector
         	gc.collect()
@@ -32,7 +32,7 @@ class Thanks:
         with open("./submissions/%s/ident" % user_id,'w') as ident:
             for key in ["name","email","country","public","comments"]:
                 if web_input.has_key(key):
-                    ident.write("%s: %s\n" % (key,web_input[key]))
+                    ident.write("%s: %s\n" % (key,web_input[key].encode("utf-8")))
         return render.thanks_id()
 class Package:
	@@ -32,7 +32,7 @@ class Thanks:		@@ -32,7 +32,7 @@ class Thanks:
32	with open("./submissions/%s/ident" % user_id,'w') as ident:	32	with open("./submissions/%s/ident" % user_id,'w') as ident:
33	for key in ["name","email","country","public","comments"]:	33	for key in ["name","email","country","public","comments"]:
34	if web_input.has_key(key):	34	if web_input.has_key(key):
35	- ident.write("%s: %s\n" % (key,web_input[key]))	35	+ ident.write("%s: %s\n" % (key,web_input[key].encode("utf-8")))
36	return render.thanks_id()	36	return render.thanks_id()
37		37
38	class Package:	38	class Package: