diff --git a/public/robots.txt b/public/robots.txt index 4ab9e89..fca1bb9 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1 +1,7 @@ -# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file \ No newline at end of file +# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file +# See also http://en.wikipedia.org/wiki/Robots_exclusion_standard for extensions + +User-agent: * +# block searches' queries +Disallow: /search*query=* +Disallow: /cat/* -- libgit2 0.21.2