User-agent: DataForSeoBot
Crawl-delay: 10
Disallow: /stores/store/redirect
Disallow: /stores/store/switch
Disallow: /customer/account/login/referer
Disallow: *?atg_collection*
Disallow: *&atg_collection*

User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: GPTBot
User-agent: BLEXBot
User-agent: SeekportBot
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Crawl-delay: 10
Disallow: /stores/store/redirect
Disallow: /stores/store/switch
Disallow: /customer/account/login/referer

User-agent: SiteAuditBot
Crawl-delay: 10

User-agent: Googlebot-Image
Crawl-delay: 5
Disallow: /

Allow: /media/catalog/product/
Allow: /images/web/
 

# Crawlers Setup

User-agent: *
Crawl-delay: 5
Disallow: /*?price*

# Directories

User-agent: *

Disallow: /app/

Disallow: /cgi-bin/

Disallow: /downloader/

Disallow: /includes/

Disallow: /lib/

Disallow: /pkginfo/

Disallow: /shell/

Disallow: /skin/

Disallow: /var/


Disallow: /*.php$

Disallow: /report/
Disallow: /var/
Disallow: /review/
Disallow: /*SID= 





# Paths (clean URLs)

User-agent: *
Crawl-delay: 5
Disallow: /index.php/

Disallow: /catalog/product_compare/

Disallow: /catalog/category/view/

Disallow: /catalog/product/view/

Disallow: /catalogsearch/

Disallow: /checkout/

Disallow: /control/

Disallow: /customer/

Disallow: /customize/

Disallow: /review/

Disallow: /sendfriend/

Disallow: /wishlist/

 

# Files

User-agent: *

Disallow: /cron.php

Disallow: /cron.sh

Disallow: /error_log

Disallow: /install.php

Disallow: /LICENSE.html

Disallow: /LICENSE.txt

Disallow: /LICENSE_AFL.txt

Disallow: /api.php

Disallow: /get.php

Disallow: /mage

 

 

# Paths (no clean URLs)

User-agent: *
Crawl-delay: 5

Disallow: /*.php$

Disallow: /*?p=*&

Disallow: /*?SID=
Disallow: *?atg_collection*
Disallow: *&atg_collection*
Disallow: *&brand*
Disallow: *?brand*
Disallow: *&gender*
Disallow: *&price*
Disallow: *&band_type*
Disallow: *&case_shape_family*
Disallow: *&case_width*
Disallow: *&movement_type_family*
Disallow: *&feature_family*
Disallow: *&band_color_family*
Disallow: *&hsww_prod_type*
Disallow: *&frame_color*
Disallow: *&lens_color*
Disallow: *&shipping*
Disallow: *?price*
Disallow: *?band_type*
Disallow: *?case_shape_family*
Disallow: *?case_width*
Disallow: *?movement_type_family*
Disallow: *?feature_family*
Disallow: *?band_color_family*
Disallow: *?frame_color*
Disallow: *?lens_color*
Disallow: *?shipping*
Disallow: *product_list_order*
Disallow: *?dial_color*
Disallow: *&dial_color*
Disallow: *?fragrance_size*
Disallow: *&fragrance_size*
Disallow: *?polarization*
Disallow: *&polarization*
Disallow: *amasty_xsearch*
Disallow: */banner/ajax/* 
Disallow: *ReCaptcha* 
Disallow: *recaptcha* 
Disallow: *?collection*
Disallow: *&collection*
Disallow: *?lens_width*
Disallow: *&lens_width*
Disallow: *?lens_protection*
Disallow: *&lens_protection*



Allow: *&CAWELAID=*
Allow: *&gclid=*
Allow: *&msclkid=*
Allow: *&utm*


Sitemap: https://www.ashford.com/media/us_sitemap.xml