User-agent: * Sitemap: https://www.luxhub.com/sitemap.xml ############## # custom # ############## # robots.txt for https://luxhub.com/ (Odoo Website) # Last updated: 2026-10-22 # # NOTE: A crawler follows ONLY the most specific User-agent group that matches it, # so the Odoo-specific rules are repeated in each group that allows crawling. # For Google/Bing, the longest matching rule wins, so the Allow lines below # override the broader Disallow: /web/ rule. # --------------------------------------------------------------- # 1. Default rules for all crawlers # --------------------------------------------------------------- User-agent: * # Assets and media needed to render pages and index images Allow: /web/assets/ Allow: /web/image/ Allow: /web/content/ Allow: /web/static/ Allow: /website/static/ Allow: /wp-content/uploads/ # Backend, login and session endpoints Disallow: /web/ Disallow: /web/login Disallow: /web/signup Disallow: /web/reset_password Disallow: /web/session/ Disallow: /web/database/ Disallow: /web/webclient/ Disallow: /odoo Disallow: /jsonrpc Disallow: /xmlrpc Disallow: /xmlrpc/ Disallow: /longpolling/ Disallow: /websocket # Portal / account areas Disallow: /my Disallow: /my/ # Website editor, tools and internal helpers Disallow: /website/ Disallow: /website_form/ Disallow: /website/info Disallow: /website/translations Disallow: /website/lang/ Disallow: /mail/ Disallow: /survey/ Disallow: /im_livechat/ Disallow: /livechat/ Disallow: /r/ Disallow: /gamification/ # Shop / checkout flows (harmless if unused) Disallow: /shop/cart Disallow: /shop/checkout Disallow: /shop/payment Disallow: /shop/confirm_order Disallow: /shop/address # Internal search and tracking / parameter URLs Disallow: /website/search Disallow: /search Disallow: /*?search= Disallow: /*&search= Disallow: /*?utm_ Disallow: /*&utm_ Disallow: /*?fbclid= Disallow: /*?gclid= Disallow: /*?debug= Disallow: /*?enable_editor= Disallow: /*?edit_translations= Disallow: /*?replytocom= # --------------------------------------------------------------- # 2. Major search engines (full access to public content) # --------------------------------------------------------------- User-agent: Googlebot User-agent: Googlebot-Image User-agent: Googlebot-News User-agent: Googlebot-Video User-agent: Bingbot User-agent: DuckDuckBot User-agent: Applebot User-agent: YandexBot User-agent: Qwantbot User-agent: Ecosia User-agent: Baiduspider User-agent: Slurp Allow: /web/assets/ Allow: /web/image/ Allow: /web/content/ Allow: /web/static/ Allow: /website/static/ Allow: /wp-content/uploads/ Disallow: /web/ Disallow: /odoo Disallow: /jsonrpc Disallow: /xmlrpc Disallow: /longpolling/ Disallow: /websocket Disallow: /my Disallow: /website/ Disallow: /website_form/ Disallow: /mail/ Disallow: /survey/ Disallow: /im_livechat/ Disallow: /livechat/ Disallow: /r/ Disallow: /shop/cart Disallow: /shop/checkout Disallow: /shop/payment Disallow: /search Disallow: /*?search= Disallow: /*?debug= Disallow: /*?enable_editor= Disallow: /*?edit_translations= # --------------------------------------------------------------- # 3. AI search / answer-engine / user-triggered agents (ALLOWED) # These surface and cite LUXHUB content in AI answers, which is # useful for B2B visibility (PSD2, VOP, CESOP, CEDR, Wero...). # --------------------------------------------------------------- User-agent: OAI-SearchBot User-agent: ChatGPT-User User-agent: Claude-SearchBot User-agent: Claude-User User-agent: PerplexityBot User-agent: Perplexity-User User-agent: DuckAssistBot User-agent: MistralAI-User User-agent: Meta-ExternalFetcher Allow: /web/image/ Allow: /web/content/ Allow: /wp-content/uploads/ Disallow: /web/ Disallow: /odoo Disallow: /jsonrpc Disallow: /xmlrpc Disallow: /websocket Disallow: /my Disallow: /website/ Disallow: /website_form/ Disallow: /mail/ Disallow: /survey/ Disallow: /im_livechat/ Disallow: /livechat/ Disallow: /r/ Disallow: /shop/cart Disallow: /shop/checkout Disallow: /search Disallow: /*?search= # --------------------------------------------------------------- # 4. AI TRAINING / bulk dataset crawlers (BLOCKED) # Remove any entry below if you are happy for that company to # use the site's content for model training. # --------------------------------------------------------------- User-agent: GPTBot User-agent: ClaudeBot User-agent: anthropic-ai User-agent: Google-Extended User-agent: Google-CloudVertexBot User-agent: Applebot-Extended User-agent: CCBot User-agent: Bytespider User-agent: TikTokSpider User-agent: Meta-ExternalAgent User-agent: FacebookBot User-agent: Amazonbot User-agent: cohere-ai User-agent: cohere-training-data-crawler User-agent: Diffbot User-agent: ImagesiftBot User-agent: Omgilibot User-agent: Omgili User-agent: PetalBot User-agent: Timpibot User-agent: VelenPublicWebCrawler User-agent: Webzio User-agent: Webzio-Extended User-agent: ICC-Crawler User-agent: Kangaroo Bot User-agent: AI2Bot User-agent: Ai2Bot-Dolma User-agent: img2dataset User-agent: PanguBot User-agent: Scrapy User-agent: FirecrawlAgent User-agent: Crawl4AI Disallow: / # --------------------------------------------------------------- # 4b. Semrush Site Audit (ALLOWED - full access) # --------------------------------------------------------------- User-agent: SiteAuditBot Allow: / # --------------------------------------------------------------- # 5. SEO / marketing scrapers and aggressive bots (BLOCKED) # Reduces server load; does not affect search visibility. # --------------------------------------------------------------- User-agent: AhrefsBot User-agent: SemrushBot User-agent: SemrushBot-SA User-agent: MJ12bot User-agent: DotBot User-agent: BLEXBot User-agent: DataForSeoBot User-agent: serpstatbot User-agent: Barkrowler User-agent: SeznamBot User-agent: MegaIndex User-agent: ZoominfoBot User-agent: Screaming Frog SEO Spider Disallow: / # --------------------------------------------------------------- # 6. Sitemap (Odoo generates this automatically) # --------------------------------------------------------------- Sitemap: https://luxhub.com/sitemap.xml