# Google adsense bot(s)
User-agent: Mediapartners-Google*
Disallow: /rss.php
Disallow: /alltags.php
Disallow: /jsJavascript.php
Disallow: /tagmos.css
Disallow: /images
Disallow: /thumbs
Disallow: /thumbs1
Disallow: /thumbs2
Disallow: /thumbs3
Disallow: /thumbs4
Disallow: /popular.php

# Google index bot
User-Agent: Googlebot
Crawl-delay: 10
Disallow: /rss.php
Disallow: /alltags.php
Disallow: /jsJavascript.php
Disallow: /tagmos.css
Disallow: /images
Disallow: /thumbs
Disallow: /thumbs1
Disallow: /thumbs2
Disallow: /thumbs3
Disallow: /thumbs4
Disallow: /popular.php

# Yahoo
# 18feb09: Following this http://en.wikipedia.org/wiki/Robots.txt#Allow_directive
# so only http://www.tagmos.nl/bookmarks.php/marco1 is crawled
User-Agent: Slurp
Crawl-delay: 15
Allow: /bookmarks.php/marco1
Disallow: /
# 18feb09: Disallow: /rss.php
# 18feb09: Disallow: /alltags.php
# 18feb09: Disallow: /jsJavascript.php
# 18feb09: Disallow: /tagmos.css
# 18feb09: Disallow: /images
# 18feb09: Disallow: /thumbs
# 18feb09: Disallow: /popular.php

# Microsoft
# 18feb09: Following this http://en.wikipedia.org/wiki/Robots.txt#Allow_directive
# so only http://www.tagmos.nl/bookmarks.php/marco1 is crawled
# And this says MS honors it too: http://googlewebmastercentral.blogspot.com/2008/06/improving-on-robots-exclusion-protocol.html
User-Agent: msnbot
Crawl-delay: 60
Allow: /bookmarks.php/marco1
Disallow: /


# Disallow all by default. Exceptions for the ones mentioned above.
User-Agent: *
Disallow: /