# robots.txt default # # Allow all # User-agent: htdig Disallow: User-agent: W3C-checklink Disallow: User-agent: * Disallow: /