# Allow all indexing # match all user-agents User-agent: * # request a delay of 5 seconds between pages Crawl-delay: 5 # 1 page per 5 seconds (for robots obeying the new extended standard) Request-rate: 1/5 # disallow nothing, so everything is OK Disallow: