# robots.txt for ONE-DC # Last updated: 2025-12-16 # =========================================== # Default rules for all bots # =========================================== User-agent: * # Global disallows Disallow: /order.html Disallow: /maintenance.html Disallow: /ao Disallow: /actions* # Search pages (all languages) Disallow: /nl/search* Disallow: /en/search* Disallow: /de/search* Disallow: /fr/search* Disallow: /es/search* Disallow: /pl/search* Disallow: /it/search* Disallow: /us/search* # User account pages (all languages) Disallow: /nl/mijn_overzicht* Disallow: /de/meine_uebersicht* Disallow: /en/my_overview* Disallow: /fr/my_overview* Disallow: /es/my_overview* Disallow: /pl/my_overview* Disallow: /it/my_overview* Disallow: /us/my_overview* # Download pages (all languages) Disallow: /nl/download* Disallow: /de/download* Disallow: /en/download* Disallow: /fr/download* Disallow: /es/download* Disallow: /pl/download* Disallow: /it/download* Disallow: /us/download* # Login pages (all languages) Disallow: /nl/login.html Disallow: /de/login.html Disallow: /en/login.html Disallow: /fr/login.html Disallow: /es/login.html Disallow: /pl/login.html Disallow: /it/login.html Disallow: /us/login.html # Signup success (all languages) Disallow: /nl/aanmeldensucces* Disallow: /de/anmeldenkomplett* Disallow: /en/signupsuccess* Disallow: /fr/signupsuccess* Disallow: /es/signupsuccess* Disallow: /pl/signupsuccess* Disallow: /it/signupsuccess* Disallow: /us/signupsuccess* # Cancel request pages (all languages) Disallow: /nl/annuleringsverzoek* Disallow: /de/stornierung* Disallow: /en/cancelrequest* Disallow: /fr/cancelrequest* Disallow: /es/cancelrequest* Disallow: /pl/cancelrequest* Disallow: /it/cancelrequest* Disallow: /us/cancelrequest* # Partial delivery pages (all languages) Disallow: /nl/deelleveringtoestaan* Disallow: /de/teil-lieferung-erlauben* Disallow: /en/allow-partial-delivery* Disallow: /fr/allow-partial-delivery* Disallow: /es/allow-partial-delivery* Disallow: /pl/allow-partial-delivery* Disallow: /it/allow-partial-delivery* Disallow: /us/allow-partial-delivery* # Favourites (all languages) Disallow: /nl/favorieten* Disallow: /de/favoriten* Disallow: /en/favourites* Disallow: /fr/favourites* Disallow: /es/favourites* Disallow: /pl/favourites* Disallow: /it/favourites* Disallow: /us/favourites* # Email change (all languages) Disallow: /nl/mailchange* Disallow: /de/mailchange* Disallow: /en/mailchange* Disallow: /fr/mailchange* Disallow: /es/mailchange* Disallow: /pl/mailchange* Disallow: /it/mailchange* Disallow: /us/mailchange* # RMA pages (all languages) Disallow: /nl/rma* Disallow: /de/rma* Disallow: /en/rma* Disallow: /fr/rma* Disallow: /es/rma* Disallow: /pl/rma* Disallow: /it/rma* Disallow: /us/rma* # Password update (all languages) Disallow: /nl/wachtwoord_updaten* Disallow: /de/kennwort_aktualisieren* Disallow: /en/update_password* Disallow: /fr/update_password* Disallow: /es/update_password* Disallow: /pl/update_password* Disallow: /it/update_password* Disallow: /us/update_password* # Forgot password (all languages) Disallow: /nl/wachtwoordvergeten* Disallow: /de/passwortvergessen* Disallow: /en/forgotpassword* Disallow: /fr/forgotpassword* Disallow: /es/forgotpassword* Disallow: /pl/forgotpassword* Disallow: /it/forgotpassword* Disallow: /us/forgotpassword* # Submit pages (all languages) Disallow: /nl/submit* Disallow: /de/submit* Disallow: /en/submit* Disallow: /fr/submit* Disallow: /es/submit* Disallow: /pl/submit* Disallow: /it/submit* Disallow: /us/submit* # Feeds (all languages) Disallow: /nl/feeds* Disallow: /de/feeds* Disallow: /en/feeds* Disallow: /fr/feeds* Disallow: /es/feeds* Disallow: /pl/feeds* Disallow: /it/feeds* Disallow: /us/feeds* # Checkout flow - Dutch Disallow: /nl/afrekenen* Disallow: /nl/ordertype* Disallow: /nl/adresgegevens* Disallow: /nl/selecteerbetaalmethode* Disallow: /nl/afronden* Disallow: /nl/kassa* Disallow: /nl/payment* Disallow: /nl/docdata* Disallow: /nl/prepaidcontant* Disallow: /nl/vzordertoevoegen* Disallow: /nl/paypal* Disallow: /nl/aoafrekenen* # Checkout flow - German Disallow: /de/bezahlen* Disallow: /de/bestelltyp* Disallow: /de/adressdaten* Disallow: /de/zahlungsmethodeauswaehlen* Disallow: /de/abchliessen* Disallow: /de/kasse* Disallow: /de/payment* Disallow: /de/docdata* Disallow: /de/prepaidkontant* Disallow: /de/sbestellunghinzufuegen* Disallow: /de/paypal* Disallow: /de/aobezahlen* # Checkout flow - English Disallow: /en/checkout* Disallow: /en/ordertype* Disallow: /en/addressdetails* Disallow: /en/selectpaymentmethod* Disallow: /en/finalize* Disallow: /en/checkout_step* Disallow: /en/payment* Disallow: /en/docdata* Disallow: /en/prepaidcash* Disallow: /en/paypal* Disallow: /en/aocheckout* # Checkout flow - French Disallow: /fr/checkout* Disallow: /fr/ordertype* Disallow: /fr/addressdetails* Disallow: /fr/selectpaymentmethod* Disallow: /fr/finalize* Disallow: /fr/checkout_step* Disallow: /fr/payment* Disallow: /fr/docdata* Disallow: /fr/prepaidcash* Disallow: /fr/paypal* Disallow: /fr/aocheckout* # Checkout flow - Spanish Disallow: /es/checkout* Disallow: /es/ordertype* Disallow: /es/addressdetails* Disallow: /es/selectpaymentmethod* Disallow: /es/finalize* Disallow: /es/checkout_step* Disallow: /es/payment* Disallow: /es/docdata* Disallow: /es/prepaidcash* Disallow: /es/paypal* Disallow: /es/aocheckout* # Checkout flow - Polish Disallow: /pl/checkout* Disallow: /pl/ordertype* Disallow: /pl/addressdetails* Disallow: /pl/selectpaymentmethod* Disallow: /pl/finalize* Disallow: /pl/checkout_step* Disallow: /pl/payment* Disallow: /pl/docdata* Disallow: /pl/prepaidcash* Disallow: /pl/paypal* Disallow: /pl/aocheckout* # Checkout flow - Italian Disallow: /it/checkout* Disallow: /it/ordertype* Disallow: /it/addressdetails* Disallow: /it/selectpaymentmethod* Disallow: /it/finalize* Disallow: /it/checkout_step* Disallow: /it/payment* Disallow: /it/docdata* Disallow: /it/prepaidcash* Disallow: /it/paypal* Disallow: /it/aocheckout* # Checkout flow - US Disallow: /us/checkout* Disallow: /us/ordertype* Disallow: /us/addressdetails* Disallow: /us/selectpaymentmethod* Disallow: /us/finalize* Disallow: /us/checkout_step* Disallow: /us/payment* Disallow: /us/docdata* Disallow: /us/prepaidcash* Disallow: /us/paypal* Disallow: /us/aocheckout* # =========================================== # Sitemaps # =========================================== Sitemap: https://www.one-dc.com/nl/download/sitemap Sitemap: https://www.one-dc.com/en/download/sitemap Sitemap: https://www.one-dc.com/de/download/sitemap Sitemap: https://www.one-dc.com/fr/download/sitemap Sitemap: https://www.one-dc.com/es/download/sitemap Sitemap: https://www.one-dc.com/pl/download/sitemap Sitemap: https://www.one-dc.com/it/download/sitemap Sitemap: https://www.one-dc.com/us/download/sitemap # =========================================== # AI Training Bots - Block all # =========================================== # OpenAI User-agent: GPTBot Disallow: / User-agent: ChatGPT-User Disallow: / User-agent: OAI-SearchBot Disallow: / # Anthropic User-agent: anthropic-ai Disallow: / User-agent: Claude-Web Disallow: / User-agent: ClaudeBot Disallow: / # Google AI (not regular search) User-agent: Google-Extended Disallow: / # Common Crawl (used for AI training) User-agent: CCBot Disallow: / # Perplexity AI User-agent: PerplexityBot Disallow: / # Meta AI User-agent: FacebookBot Disallow: / User-agent: Meta-ExternalAgent Disallow: / User-agent: Meta-ExternalFetcher Disallow: / # Cohere AI User-agent: cohere-ai Disallow: / # ByteDance / TikTok User-agent: Bytespider Disallow: / User-agent: ByteDance Disallow: / # Apple AI User-agent: Applebot-Extended Disallow: / # Amazon AI User-agent: Amazonbot Disallow: / # Diffbot User-agent: Diffbot Disallow: / # Omgili / Webz.io User-agent: omgili Disallow: / User-agent: omgilibot Disallow: / # Youbot (You.com) User-agent: YouBot Disallow: / # Siri / Apple User-agent: Applebot Crawl-delay: 10 # img2dataset (AI image scraper) User-agent: img2dataset Disallow: / # Other AI scrapers User-agent: AI2Bot Disallow: / User-agent: Ai2Bot-Dolma Disallow: / User-agent: Scrapy Disallow: / User-agent: iaskspider Disallow: / User-agent: Timpibot Disallow: / User-agent: VelenPublicWebCrawler Disallow: / User-agent: Webzio-Extended Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: ISSCyberRiskCrawler Disallow: / User-agent: PiplBot Disallow: / # =========================================== # SEO / Marketing Bots - Block or limit # =========================================== User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: DotBot Disallow: / User-agent: dotbot Disallow: / User-agent: MJ12bot Disallow: / User-agent: BLEXBot Disallow: / User-agent: PetalBot Disallow: / User-agent: SeznamBot Disallow: / User-agent: Exabot Disallow: / # =========================================== # Spam Bots and Scrapers - Block # =========================================== User-agent: TwengaBot Disallow: / User-agent: TwengaBot-2.0 Disallow: / User-agent: Yandex Disallow: / User-agent: YandexBot Disallow: / User-agent: YandexImages Disallow: / User-agent: Baiduspider Disallow: / User-agent: Baiduspider-image Disallow: / User-agent: Baiduspider-video Disallow: / User-agent: CazoodleBot Disallow: / User-agent: Fasterfox Disallow: / User-agent: Jyxobot Disallow: / User-agent: ShopWiki Disallow: / User-agent: webcrawler Disallow: / User-agent: spbot Disallow: / User-agent: Aboundexbot Disallow: / User-agent: ZoominfoBot Disallow: / User-agent: MegaIndex Disallow: / User-agent: MegaIndex.ru Disallow: / User-agent: serpstatbot Disallow: / User-agent: Sogou Disallow: / User-agent: sogou spider Disallow: / User-agent: BLP_bbot Disallow: / User-agent: Cliqzbot Disallow: / User-agent: AspiegelBot Disallow: / User-agent: Seekport Disallow: / User-agent: Buck Disallow: / User-agent: Linguee Disallow: / User-agent: proximic Disallow: / User-agent: Cocolyzebot Disallow: / User-agent: Clickagy Disallow: / User-agent: AwarioBot Disallow: / User-agent: AwarioSmartBot Disallow: / User-agent: trendictionbot Disallow: / # =========================================== # Good Bots - Allow with optional rate limits # =========================================== # Bing - allow with rate limit User-Agent: msnbot Crawl-Delay: 10 User-Agent: bingbot Crawl-Delay: 5 # Yahoo - allow with rate limit User-Agent: Slurp Crawl-delay: 10 User-Agent: Yahoo! Slurp Crawl-delay: 5 # DuckDuckGo User-agent: DuckDuckBot Allow: / # Google (standard search) - fully allowed User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Video Allow: / User-agent: Googlebot-News Allow: /