You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
72 lines
2.0 KiB
72 lines
2.0 KiB
# |
|
# robots.txt |
|
# |
|
# This file is to prevent the crawling and indexing of certain parts |
|
# of your site by web crawlers and spiders run by sites like Yahoo! |
|
# and Google. By telling these "robots" where not to go on your site, |
|
# you save bandwidth and server resources. |
|
# |
|
# This file will be ignored unless it is at the root of your host: |
|
# Used: http://example.com/robots.txt |
|
# Ignored: http://example.com/site/robots.txt |
|
# |
|
# For more information about the robots.txt standard, see: |
|
# http://www.robotstxt.org/robotstxt.html |
|
|
|
User-agent: * |
|
# CSS, JS, Images |
|
Allow: /core/*.css$ |
|
Allow: /core/*.css? |
|
Allow: /core/*.js$ |
|
Allow: /core/*.js? |
|
Allow: /core/*.gif |
|
Allow: /core/*.jpg |
|
Allow: /core/*.jpeg |
|
Allow: /core/*.png |
|
Allow: /core/*.svg |
|
Allow: /profiles/*.css$ |
|
Allow: /profiles/*.css? |
|
Allow: /profiles/*.js$ |
|
Allow: /profiles/*.js? |
|
Allow: /profiles/*.gif |
|
Allow: /profiles/*.jpg |
|
Allow: /profiles/*.jpeg |
|
Allow: /profiles/*.png |
|
Allow: /profiles/*.svg |
|
# Directories |
|
Disallow: /core/ |
|
Disallow: /profiles/ |
|
# Files |
|
Disallow: /README.md |
|
Disallow: /composer/Metapackage/README.txt |
|
Disallow: /composer/Plugin/ProjectMessage/README.md |
|
Disallow: /composer/Plugin/Scaffold/README.md |
|
Disallow: /composer/Plugin/VendorHardening/README.txt |
|
Disallow: /composer/Template/README.txt |
|
Disallow: /modules/README.txt |
|
Disallow: /sites/README.txt |
|
Disallow: /themes/README.txt |
|
# Paths (clean URLs) |
|
Disallow: /admin/ |
|
Disallow: /comment/reply/ |
|
Disallow: /filter/tips |
|
Disallow: /node/add/ |
|
Disallow: /search/ |
|
Disallow: /user/register |
|
Disallow: /user/password |
|
Disallow: /user/login |
|
Disallow: /user/logout |
|
Disallow: /media/oembed |
|
Disallow: /*/media/oembed |
|
# Paths (no clean URLs) |
|
Disallow: /index.php/admin/ |
|
Disallow: /index.php/comment/reply/ |
|
Disallow: /index.php/filter/tips |
|
Disallow: /index.php/node/add/ |
|
Disallow: /index.php/search/ |
|
Disallow: /index.php/user/password |
|
Disallow: /index.php/user/register |
|
Disallow: /index.php/user/login |
|
Disallow: /index.php/user/logout |
|
Disallow: /index.php/media/oembed |
|
Disallow: /index.php/*/media/oembed
|
|
|