# # robots.txt for http://www.w3.org/ # # $Id: robots.txt,v 1.41 2006/02/22 16:31:58 ted Exp $ # # For use by search.w3.org User-agent: W3C-gsa Disallow: /Out-Of-Date # W3C Link checker User-agent: W3C-checklink Disallow: # exclude some access-controlled areas User-agent: * Disallow: /cgi-bin/ Disallow:indexmidden.htm # allow:index.html