User-agent: dotbot
Disallow: / 

User-agent: NPBot
Disallow: /

User-agent: psbot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: exabot-images
Disallow: /

User-agent: Fasterfox
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: ingrid/3.0
Disallow: /

User-agent: linksmanager_bot
Disallow: /

User-agent: aipbot
Disallow: /

User-agent: sbider
Disallow: /

User-agent: ScoutJet
Crawl-Delay: 20

User-agent: VoilaBot
Crawl-Delay: 20

User-agent: Yandex
Crawl-Delay: 20

User-agent: *
Disallow: /offer/
Disallow: /wishlist/
Disallow: /cookie_usage.html
Disallow: /cookie_usage.php
Disallow: /index.html
Disallow: /index.php
Disallow: /links.php
Disallow: /links_submit.php
Disallow: /login.php
Disallow: /make_an_offer.php
Disallow: /product_thumb.php
Disallow: /password.html
Disallow: /payments-accepted.php
Disallow: /shopping_cart.php
Disallow: /login.html
Disallow: /downloads.php
Disallow: /product-sitemap.php
Disallow: /shopping_cart.html
Disallow: /unsubs.php
Disallow: /unsubscribe.php