user-agent: *  # directed to all spiders, not just Scooter
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /redesign/
Disallow: /search/
Hit-rate: 30
# wait 30 seconds before starting a new URL request default=30
Visiting-hours: 01:00EST-05:00EST
# index this site between 1AM - 5AM EST
Concurrent-hits: 2
# limit concurrent active URLs to 2 for each index server
Disallow: /srch/
Disallow: /dodsrch/
Disallow: /dbsrch/