User-agent: *
Crawl-delay: 10
# CSS, JS, Images
Allow: /misc/*.css$
Allow: /misc/*.css?
Allow: /misc/*.js$
Allow: /misc/*.js?
Allow: /misc/*.gif
Allow: /misc/*.jpg
Allow: /misc/*.jpeg
Allow: /misc/*.png
Allow: /modules/*.css$
Allow: /modules/*.css?
Allow: /modules/*.js$
Allow: /modules/*.js?
Allow: /modules/*.gif
Allow: /modules/*.jpg
Allow: /modules/*.jpeg
Allow: /modules/*.png
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /themes/*.css$
Allow: /themes/*.css?
Allow: /themes/*.js$
Allow: /themes/*.js?
Allow: /themes/*.gif
Allow: /themes/*.jpg
Allow: /themes/*.jpeg
Allow: /themes/*.png
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /sites/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /contact/
Disallow: /logout/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /cn/user/login
Disallow: /de/user/login
Disallow: /en/user/login
Disallow: /en_GB/user/login
Disallow: /en_US/user/login
Disallow: /es/user/login
Disallow: /fr/user/login
Disallow: /he/user/login
Disallow: /it/user/login
Disallow: /ja/user/login
Disallow: /jp/user/login
Disallow: /nl/user/login
Disallow: /tr/user/login
#Old pages
Disallow: /cn/user
Disallow: /de/user
Disallow: /en/user
Disallow: /en_GB/user
Disallow: /en_US/user
Disallow: /es/user
Disallow: /fr/user
Disallow: /he/user
Disallow: /it/user
Disallow: /ja/user
Disallow: /jp/user
Disallow: /nl/user
Disallow: /tr/user
Disallow: /user
# Others
Disallow: /Images/
Disallow: /images/
Disallow: /files/
# Wildcards should be supported by Google and Yahoo, at least
Disallow: /node/*/pdf
Disallow: /cn/taxonomy/*
Disallow: /de/taxonomy/*
Disallow: /en/taxonomy/*
Disallow: /en_GB/taxonomy/*
Disallow: /en_US/taxonomy/*
Disallow: /es/taxonomy/*
Disallow: /fr/taxonomy/*
Disallow: /he/taxonomy/*
Disallow: /it/taxonomy/*
Disallow: /ja/taxonomy/*
Disallow: /jp/taxonomy/*
Disallow: /nl/taxonomy/*
Disallow: /tr/taxonomy/*
# Avoid duplicates in forum
Disallow: */*sort=
Disallow: */forum/active*
Disallow: */forum/*order
Disallow: */forum/unanswered*