# Robots file listing html directories NOT # to search User-agent: * Disallow: /cgi-bin/ Disallow: /class Disallow: /classes Disallow: /email/ User-agent: Googlebot-Image Disallow: / User-agent: Bloodhound Disallow: / User-agent: DA Disallow: / User-agent: eCatch Disallow: / User-agent: e-collector Disallow: / User-agent: fast Disallow: / User-agent: GetRight Disallow: / User-agent: Go!Zilla Disallow: / User-agent: LeechFTP Disallow: / User-agent: Mass Downloader Disallow: / User-agent: Prozilla Disallow: / User-agent: Offline Explorer Disallow: / User-agent: RealDownload Disallow: / User-agent: Scooter_trk3-3.0.3 Disallow: / User-agent: Scooter-W3-1.0 Disallow: / User-agent: Scooter Disallow: / User-agent: SiteSnagger Disallow: / User-agent: Teleport Pro Disallow: / User-agent: WebCopier Disallow: / User-agent: WebDownloader Disallow: / User-agent: Web Downloader Disallow: / User-agent: webfetcher Disallow: / User-agent: WebFountain Disallow: / User-agent: Wget Disallow: / User-agent: WebCapture Disallow: / User-agent: WebMirror Disallow: / User-agent: WebReaper Disallow: / User-agent: WebSauger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebZIP Disallow: /