# CRELoaded Generated Robots.txt
# Robot Exclusion File -- robots.txt
# Author:  CRELoaded Team 
# Last Updated : September 11th, 2009

User-Agent: *
Disallow: /themes/
Disallow: /config/
Disallow: /huadmin/
Disallow: /modules/
Disallow: /content/

#To block main page due to size from bots
#Disallow: /index.php

#Block out things that are secure or login oriented
#These are based on not using any rewrite rules
Disallow: /address.php
Disallow: /cart.php
Disallow: /email.php
Disallow: /order.php
Disallow: /category.php
Disallow: /password.php
Disallow: /order-detail.php
Disallow: /cms.php
Disallow: /product.php
Disallow: /category.php



# IF YOU DO NOT WISH TO HAVE THE GOOGLE IMAGE BOT SCAN YOUR DOMAIN FOR IMAGES
# THEN YOU CAN INCLUDE THE FOLLOWING IN YOUR ROBOTS FILE.
# I FOUND THAT MY BANDWIDTH USAGE DROPPED BY A MASSIVE AMOUNT AFTER I GOT RID
# OF THE GOOGLE IMAGE BOT. ALL I HAD WAS IMAGE HUNTERS STEALING PRODUCT SHOTS
# AND NOT EVEN BROWSING THE SITE.
#User-agent: Googlebot-Image
#Disallow: /

User-agent: Slurp
Crawl-delay: 8

User-agent: msnbot
Crawl-delay: 5

User-agent: nutch
Crawl-delay: 5

User-agent: yeti
Crawl-delay: 8

User-agent: Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)
Crawl-delay: 8

User-agent: naverbot
Crawl-delay: 8

User-agent: googlebot
Crawl-delay: 5

User-agent: dotbot
Crawl-delay: 5

User-agent: twiceler
Crawl-delay: 8

User-agent: yandex
Crawl-delay: 5

User-agent: Teoma
Crawl-delay: 5

User-agent: BoardReader
Crawl-delay: 8

User-agent: Exabot
Crawl-delay: 10