robots.txt 95 B

12345
  1. # all crawlers
  2. User-agent: *
  3. # disallow nothing; i.e. allow all pages to be crawled
  4. Disallow: