User-agent: * # ------------------------------------------------------- # Allow essential resources (needed for rendering/indexing) # ------------------------------------------------------- Allow: /pub/media/ Allow: /pub/static/ Allow: /media/ Allow: /static/ # ------------------------------------------------------- # Directories # ------------------------------------------------------- Disallow: /app/ Disallow: /bin/ Disallow: /dev/ Disallow: /lib/ Disallow: /phpserver/ Disallow: /pkginfo/ Disallow: /report/ Disallow: /setup/ Disallow: /update/ Disallow: /var/ Disallow: /vendor/ # ------------------------------------------------------- # Paths (clean URLs) # ------------------------------------------------------- Disallow: /index.php/ Disallow: /catalog/product_compare/ Disallow: /catalog/category/view/ Disallow: /catalog/product/view/ Disallow: /catalogsearch/ Disallow: /checkout/ Disallow: /cart/ Disallow: /control/ Disallow: /contacts/ Disallow: /customer/ Disallow: /customize/ Disallow: /newsletter/ Disallow: /review/ Disallow: /sendfriend/ Disallow: /wishlist/ Disallow: /customer/account/ Disallow: /customer/account/login/ Disallow: /onestepcheckout/ Disallow: /sales/order/history/ Disallow: /sales/guest/ Disallow: /pricematch/ Disallow: /gdpr/ Disallow: /vault/ Disallow: /paypal/ Disallow: /xnotif/ Disallow: /ugiftcert/ # ------------------------------------------------------- # Files # ------------------------------------------------------- Disallow: /composer.json Disallow: /composer.lock Disallow: /CONTRIBUTING.md Disallow: /CONTRIBUTOR_LICENSE_AGREEMENT.html Disallow: /COPYING.txt Disallow: /Gruntfile.js Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt Disallow: /nginx.conf.sample Disallow: /package.json Disallow: /php.ini.sample Disallow: /RELEASE_NOTES.txt # ------------------------------------------------------- # URL parameters - layered navigation, filters & sorting # ------------------------------------------------------- Disallow: /*?*product_list_mode= Disallow: /*?*product_list_order= Disallow: /*?*product_list_limit= Disallow: /*?*product_list_dir= Disallow: /*?dir= Disallow: /*?dir=desc Disallow: /*?dir=asc Disallow: /*?limit=all Disallow: /*?mode= Disallow: /*?price= Disallow: /*?ml= Disallow: /*?color= Disallow: /*?size= Disallow: /*?cat= Disallow: /*?q= Disallow: /*?order= Disallow: /*?___store= Disallow: /*?___from_store= # ------------------------------------------------------- # Session ID # ------------------------------------------------------- Disallow: /*?SID= Disallow: /*.php$ # ------------------------------------------------------- # CVS, SVN directory and dump files # ------------------------------------------------------- Disallow: /*.CVS Disallow: /*.Zip$ Disallow: /*.Svn$ Disallow: /*.Idea$ Disallow: /*.Sql$ Disallow: /*.Tgz$ # ------------------------------------------------------- # Disallow Duplicate Content # ------------------------------------------------------- Disallow: /tag/ # ------------------------------------------------------- # Sitemaps # ------------------------------------------------------- Sitemap: https://www.beautycos.dk/pub/sitemap/google_sitemap_dk.xml Sitemap: https://www.beautycos.co.uk/pub/sitemap/google_sitemap_gb.xml Sitemap: https://www.beautycos.de/pub/sitemap/google_sitemap_de.xml Sitemap: https://www.beautycos.se/pub/sitemap/google_sitemap_se.xml Sitemap: https://www.beautycos.no/pub/sitemap/google_sitemap_no.xml Sitemap: https://www.beautycos.eu/pub/sitemap/google_sitemap_eu.xml Sitemap: https://www.beautycos.fr/pub/sitemap/google_sitemap_fr.xml Sitemap: https://www.beautycos.es/pub/sitemap/google_sitemap_es.xml Sitemap: https://www.beautycos.it/pub/sitemap/google_sitemap_it.xml Sitemap: https://www.beautycos.fi/pub/sitemap/google_sitemap_fi.xml Sitemap: https://www.beautycos.nl/pub/sitemap/google_sitemap_nl.xml Sitemap: https://www.beautycos.be/pub/sitemap/google_sitemap_be_fr.xml Sitemap: https://www.beautycos.be/pub/sitemap/google_sitemap_be_nl.xml Sitemap: https://www.beautycos.ie/pub/sitemap/google_sitemap_ie.xml Sitemap: https://www.beautycos.at/pub/sitemap/google_sitemap_at.xml Sitemap: https://www.beautycos.pl/pub/sitemap/google_sitemap_pl.xml Sitemap: https://www.beautycos.cz/pub/sitemap/google_sitemap_cz.xml Sitemap: https://www.beautycos.pt/pub/sitemap/google_sitemap_pt.xml Sitemap: https://www.beautycos.hu/pub/sitemap/google_sitemap_hu.xml Sitemap: https://www.beautycos.sk/pub/sitemap/google_sitemap_sk.xml Sitemap: https://www.beautycos.bg/pub/sitemap/google_sitemap_bg.xml Sitemap: https://www.beautycos.hr/pub/sitemap/google_sitemap_hr.xml Sitemap: https://www.beautycos.ee/pub/sitemap/google_sitemap_ee.xml Sitemap: https://www.beautycos.gr/pub/sitemap/google_sitemap_gr.xml Sitemap: https://www.beautycos.lv/pub/sitemap/google_sitemap_lt.xml Sitemap: https://www.beautycos.lt/pub/sitemap/google_sitemap_lv.xml Sitemap: https://www.beautycos.ro/pub/sitemap/google_sitemap_ro.xml Sitemap: https://www.beautycos.si/pub/sitemap/google_sitemap_si.xml Sitemap: https://www.beautycos.dk/sitemap/google_sitemap_blog_dk.xml Sitemap: https://www.beautycos.co.uk/sitemap/google_sitemap_blog_co_uk.xml Sitemap: https://www.beautycos.de/sitemap/google_sitemap_blog_de.xml Sitemap: https://www.beautycos.se/sitemap/google_sitemap_blog_sv.xml Sitemap: https://www.beautycos.no/sitemap/google_sitemap_blog_no.xml Sitemap: https://www.beautycos.eu/sitemap/google_sitemap_blog_eu.xml Sitemap: https://www.beautycos.fr/sitemap/google_sitemap_blog_fr.xml Sitemap: https://www.beautycos.es/sitemap/google_sitemap_blog_es.xml Sitemap: https://www.beautycos.it/sitemap/google_sitemap_blog_it.xml Sitemap: https://www.beautycos.fi/sitemap/google_sitemap_blog_fi.xml Sitemap: https://www.beautycos.nl/sitemap/google_sitemap_blog_nl.xml Sitemap: https://www.beautycos.be/sitemap/google_sitemap_blog_be_fr.xml Sitemap: https://www.beautycos.be/sitemap/google_sitemap_blog_be_nl.xml Sitemap: https://www.beautycos.ie/sitemap/google_sitemap_blog_ie.xml Sitemap: https://www.beautycos.at/sitemap/google_sitemap_blog_at.xml # ------------------------------------------------------- # Crawl-delay for known heavy crawlers # ------------------------------------------------------- User-agent: AhrefsBot Crawl-Delay: 1 User-agent: Bingbot Crawl-delay: 1 User-agent: SemrushBot Crawl-Delay: 5 User-agent: MJ12bot Crawl-Delay: 10 User-agent: DotBot Crawl-Delay: 10 # ------------------------------------------------------- # Block AI training crawlers # (these scrape content for model training without attribution) # ------------------------------------------------------- User-agent: GPTBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Google-Extended Disallow: / User-agent: CCBot Disallow: / User-agent: Bytespider Disallow: / User-agent: Applebot-Extended Disallow: / # ------------------------------------------------------- # Allow AI search & retrieval crawlers # (these power AI search results and can send traffic) # ------------------------------------------------------- User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / User-agent: Claude-User Allow: / User-agent: Claude-SearchBot Allow: / User-agent: PerplexityBot Allow: /