#-----
# Google is of course permitted here.
# The wildcard entry at the bottom should
# take care of this, but this entry is
# insurance.
User-agent: Googlebot*
Disallow:
#-----

#-----
# These robots do not honor <base /> tags,
# so they generate lots of errors in the log.
# Solution: ban them.

# Also blocked the IP ranges of this one.
User-agent: Gigabot*
Disallow: /

User-agent: BecomeBot 
Crawl-Delay: 60 

# Also blocked the IP ranges of this one.
User-agent: NPBot*
Disallow: /

# Also blocked the IPs of this one.
User-agent: appie*
Disallow: /

User-agent: Nutch*
Disallow: /
#-----

#-----
# Allow every other robot to index the site.
User-agent: *
Crawl-delay: 100
Disallow:
#-----