# Invite only major search engines into the site.

# User-agent: *
# Disallow: /

User-agent: gsa-crawler
# Crawl-delay: 30
Disallow: /cgi-bin/
Disallow: /old/
Disallow: /library/
Disallow: /pdf/
Disallow: /private/
Disallow: /test/
Disallow: /stats/
Disallow: /coverm/
Disallow: /covermt/

User-agent: Googlebot
Crawl-delay: 60
Disallow: /cgi-bin/
Disallow: /old/
Disallow: /library/
Disallow: /pdf/
Disallow: /private/
Disallow: /test/
Disallow: /stats/
Disallow: /coverm/
Disallow: /covermt/
Disallow: /oldcms/

Disallow: *q=cache*

# Disallow: *combo=*


User-agent: Slurp
Crawl-delay: 60
Disallow: /cgi-bin/
Disallow: /old/
Disallow: /library/
Disallow: /pdf/
Disallow: /private/
Disallow: /test/
Disallow: /stats/
Disallow: /coverm/
Disallow: /covermt/


User-agent: Teoma
Crawl-delay: 60
Disallow: /cgi-bin/
Disallow: /old/
Disallow: /library/
Disallow: /pdf/
Disallow: /private/
Disallow: /test/
Disallow: /stats/
Disallow: /coverm/
Disallow: /covermt/


User-agent: msnbot 
Crawl-delay: 60
Disallow: /cgi-bin/
Disallow: /old/
Disallow: /library/
Disallow: /pdf/
Disallow: /private/
Disallow: /test/
Disallow: /stats/
Disallow: /coverm/
Disallow: /covermt/


User-agent: msnbot-Products
Crawl-delay: 60
Disallow: /cgi-bin/
Disallow: /old/
Disallow: /library/
Disallow: /pdf/
Disallow: /private/
Disallow: /test/
Disallow: /stats/
Disallow: /coverm/
Disallow: /covermt/


User-agent: AdsBot-Google
Disallow: /

User-agent: BecomeBot
Disallow: /

User-agent: FatBot
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: Girafabot
Disallow: /

User-agent: Googlebot-Image
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: ichiro
Disallow: /

User-agent: LapozzBot
Disallow: /

User-agent: mattie
Disallow: /

User-agent: msnbot-media
Disallow: /

User-agent: SBIder
Disallow: /

User-agent: ShopWiki
Disallow: /

User-agent: Slurp China
Disallow: /

User-agent: Speedy
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: webGobbler
Disallow: /

User-agent: wordpress
Disallow: /

User-agent: yacy
Disallow: /