From 4743c28c85f265b8f8e50122dbc9afa1f7bdf48b Mon Sep 17 00:00:00 2001 From: Braulio Bhavamitra Date: Wed, 23 Apr 2014 20:40:57 -0300 Subject: [PATCH] Block robots on search --- public/robots.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/public/robots.txt b/public/robots.txt index 4ab9e89..fca1bb9 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1 +1,7 @@ -# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file \ No newline at end of file +# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file +# See also http://en.wikipedia.org/wiki/Robots_exclusion_standard for extensions + +User-agent: * +# block searches' queries +Disallow: /search*query=* +Disallow: /cat/* -- libgit2 0.21.2