# ========================================================================== # ROBOTS.TXT — wikioo.org # # GENERATED FILE — DO NOT EDIT BY HAND. # source : apps/core/robots_txt_policy.py # rebuild: python manage.py generate_robots_txt # guard : apps/core/tests/test_robots_txt_policy.py (fails on drift) # # Every user-agent group repeats the full rule set on purpose: RFC 9309 # groups do NOT inherit from "*". Editing one group here fixes nothing — # change the policy module and regenerate all 8 files. # ========================================================================== # Default rules — applies ONLY to agents with no group of their own User-agent: * # On-demand WebP variants (apps/core/views_media_resize.py). # Every distinct width mints a fresh PIL encode + cache entry, and the # variants carry no content the originals under /media/artworks/ do not # already expose at full resolution. Search and image crawlers are NOT # given this rule — see robots_txt_policy.py for why. Disallow: /media-resize/ # Administration, staff tooling and user-owned content. # # EVERY private prefix is emitted TWICE — bare and language-prefixed. # Rule shapes are matched against the URL as served, and this codebase # serves both shapes (2026-08-28 three-layer audit): # - The Django admin is mounted INSIDE i18n_patterns, so `admin:index` # reverses to /en/admin/ and every one of the 397 admin routes carries # a language prefix. The bare `Disallow: /admin/` matched none of them. # - The allauth OAuth entrypoints (/accounts/google/login/…) are OUTSIDE # i18n_patterns, so the `/*/accounts/` rule matched none of THOSE. # The two rules were exact mirror images of each other. Emitting both # shapes is the only form that cannot be wrong for a given route. Disallow: /admin/ Disallow: /*/admin/ Disallow: /manage/ Disallow: /*/manage/ Disallow: /users/ Disallow: /*/users/ Disallow: /accounts/ Disallow: /*/accounts/ Disallow: /*/orders/ # Staff surfaces that do NOT live under /admin/ and so were never covered # by the rule above (2026-08-28 audit). Prefix match, no trailing slash on # admin_status: it covers all eight admin_status_* dashboards at once. Disallow: /*/admin_status Disallow: /*/artwork_addition/ Disallow: /*/marketing/ Disallow: /*/marketing_emails/ # One customer's order, keyed by sequential order id. The view itself is # restricted to staff and the buyer, but the URL sits at /order/… and # /order-customer-dashboard/… — NOT under /orders/ — so the transactional # Disallow above never covered it and crawlers were free to walk order ids. Disallow: /*/order/ Disallow: /*/order-customer-dashboard/ # Per-order artefacts reached by an unguessable signed token: the public # Certificate of Authenticity and the finished-image download. Anonymous # access is deliberate (a buyer verifies a physical painting), but a token # that leaks into a shared link should not end up in an index. Disallow: /*/order-item-certificate/ Disallow: /*/order-item-image-download/ # Machine endpoints: payment/delivery webhooks and the chatbot + currency # JSON APIs. /api/ was bare-only, so the language-prefixed /en/api/… routes # that actually exist were crawlable — the same shape bug as /admin/. Disallow: /stripe/ Disallow: /17track/ Disallow: /chatbot/ # …but /orders/ is not only cart and checkout: the public service landing # pages are routed under it too (apps/orders/urls.py), and all 33 of them # are listed in the sitemap on purpose. Until 2026-08-25 the blanket # Disallow above blocked every one, so the sitemap advertised 33 URLs per # language that no crawler was allowed to fetch. Prefix Allows carve them # out (longest-match wins, RFC 9309 §2.2.2); the sub-pages of the two # families inherit from their parent prefix. Cart, checkout and every # other /orders/ route stay blocked. Allow: /*/orders/art-appraisal/ Allow: /*/orders/art-consultation/ Allow: /*/orders/custom-reproduction/ Allow: /*/orders/photo-to-painting/ Allow: /*/orders/photo_to_restyled_masterpiece/ # Site-wide bot honeypot. /core/ hosts the invisible trap link rendered on # every page; anything that follows it is flagged as a bot for 24 h. Every # group needs this line — a named group does NOT inherit it from *. Disallow: /core/ # Internal endpoints and developer surfaces Disallow: /api/ Disallow: /*/api/ Disallow: /i18n/ Disallow: /*/i18n/ Disallow: /ckeditor5/ Disallow: /debug/ Disallow: /__debug__/ Disallow: /misc_utils_tags/ # Media: artwork and product imagery only. Everything else under /media/ # is user-supplied (appraisal photos, custom-reproduction photos, avatars). Disallow: /media/ Allow: /media/artworks/ Allow: /media/products/ # Static assets: no crawl budget on CSS/JS bundles Disallow: /static/css/ Disallow: /static/js/ Disallow: /staticfiles/ Allow: /staticfiles/images/ # State-changing and AJAX endpoints Disallow: /*/set_country/ Disallow: /*/set_currency/ Disallow: /*/set_language/ Disallow: /*/process-newsletter/ Disallow: /*/process-art-suggestions/ Disallow: /*/favorite/addajax/ Disallow: /*/favorite/removeajax/ # Search-result and filter URLs — infinite duplicate-content surface. # Named filter params (?movement=, ?origin=, …) stay crawlable on purpose: # they are the "See All" targets of the groupby pages and carry ItemList # structured data. Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?filter= # Low-value / terminal pages Disallow: /*/search-not-found/ Disallow: /*/error/ Disallow: /*/test/ Disallow: /*/empty/ # Artwork utility routes (the /art// detail + buy pages stay open) Disallow: /*/art/diaporama/ Disallow: /*/art/ecard/ Disallow: /*/art/sell/ Disallow: /*/art/favorite/ Disallow: /*/art/error-not-found/ Disallow: /*/art/search-not-found/ Disallow: /*/art/*/?collection= # Derivative per-artwork pages with a heavy per-request cost. /art/similar/ # is the Plate of Affinities (CLIP/FAISS affinity work per render); a crawler # walking it once per artwork is a CPU storm (2026-08-18: a crawl wave held # d3 at ~90% CPU EMA largely via these routes). The canonical /art// # pages stay fully crawlable. # # /art/stats/ was in this group and came back out (2026-09-01, opened with # the rest of the derivative surface below). It is now class A — # artworks:artwork_visits_stats is in SEO_INDEXABLE_VIEW_NAMES. Disallow: /*/art/similar/ # Same argument, found by the 2026-08-28 traffic audit: these were the rest # of the per-artwork derivative surface and had no rule at all. In one 6 h # window a scraper walking them accounted for 1,692 (/art/preview-ar/) and # 1,506 (/art/preview/) hits, each a full render, across ~30 languages. # Neither carries content the canonical /art// page does not have. # # /art/student-guide/ was in this group and came back out (2026-08-28). # It took the largest share of that window at 5,000 hits, but unlike the # other three it is a document in its own right — a printable Student # Artwork Guide, not a re-rendering of the detail page — so it is worth # indexing on its own terms. Its anchors in artwork_fav_icon_actions.html # and artwork_share_classroom_pack.html stay ordinary hrefs, and # artworks:artwork_factsheet is in SEO_INDEXABLE_VIEW_NAMES. # # /art/underthelight/ left this group on 2026-09-01 for the same reason — # see the class-A block further down. Disallow: /*/art/preview/ Disallow: /*/art/preview-ar/ # --------------------------------------------------------------------- # OPENED 2026-09-01 (Laurent). Nine views the 2026-08-28 audit had closed # as class C are now class A — pages worth ranking on their own terms: # # /art/underthelight/ /art/stats/ /art/download/ # /art/search_by_color_wheel/ # /artists/atlas/ /artists/timeline/ /artists/spectral-index/ # /artists/genome/ /artists/mood-room/ /artists/3DMuseum/ # # Their Disallow lines are gone from this module, the matching view names # moved to SEO_INDEXABLE_VIEW_NAMES (apps/core/seo_indexing.py), and the # robot_hidden flags came off their anchors so a crawler can reach them. # # The Cloudflare edge was reconciled the same day: the six per-artist # paths came out of the Managed Challenge rule, and the per-artwork paths # never had an edge rule to begin with (the 5-rule set the docs described # was never deployed — all 8 zones carry 4). See DOC_HTTP_CLOUDFLARE.md. # --------------------------------------------------------------------- # The artist-side derivative surface keeps only its two JSON feeds closed. # Neither is fetched by the page it belongs to — artist_timeline.html embeds # its data with json_script and artist_atlas.html with an inline ATLAS_DATA # literal — so a rendering crawler needs neither, and they carry no page. Disallow: /*/artists/atlas_nodes.json/ Disallow: /*/artists/artist_timeline_data.json/ # The rest of the same family, added 2026-09-01 when the pages that used # to inline these payloads started fetching them instead. Each is the data # behind one visualisation, not a page: no title, no prose, nothing an # index can show. The pages themselves stay class A and crawlable — only # their data feeds are closed, exactly as for the two rules above. Disallow: /*/artists/genome_data.json/ Disallow: /*/artists/mood_room_data.json/ Disallow: /*/artists/spectral_data.json/ Disallow: /*/artists/3DMuseum_scene.xml/ Disallow: /*/art/stats_data.json/ # The Download & Share page at /art/download// is class A since # 2026-09-01, but its two sub-endpoints are not pages and stay closed. The # old rule was a bare `/*/art/download/` prefix, which covered all three; # these two mid-path wildcards (RFC 9309 §2.2.2) keep the sub-endpoints out # while leaving the page itself crawlable. `…/file/` returns image bytes; # `…/embed/` is the chrome-less iframe body — an image-only duplicate of # /art// that must never compete with it in the index (it also # hardcodes its own noindex meta tag, since it does not extend the master # template). Longest-match gives these the win over nothing — the page # prefix is simply not disallowed any more. Disallow: /*/art/download/*/file/ Disallow: /*/art/download/*/embed/ # The Embed builder page at /art/embed// (split off Download & # Share 2026-09-15) is class A like its sibling; its three iframe bodies # are chrome-less duplicates of /art// and stay closed, same # mid-path-wildcard pattern as the two lines above. Disallow: /*/art/embed/*/image/ Disallow: /*/art/embed/*/viewer/ Disallow: /*/art/embed/*/card/ # Branded QR PNGs (apps/core/views_qrcode_endpoints.py + the artwork-info QR). # Same reasoning as /media-resize/: every uncached URL is a fresh PIL render, # so a crawler walking them per artwork is an encode storm — and the images # only re-encode a page URL the crawler already has. Disallow: /qrcode/ # QR-code and terminal routes on the other three entity types. # # The three QR rules read /*/artists/*/qr/ until the 2026-08-28 audit, i.e. # they expected /artists//qr/. The real route is /artists/qr/.png # (apps/artists/urls.py), so all three matched NOTHING while /*/art/qr/ — # written in the correct shape — worked. Only the artwork QR was protected; # the artist and museum encoders were open to every crawler. (No article QR # route exists yet; the rule is kept so one cannot land unprotected.) # ONE rule covers every entity: a robots.txt "*" matches across "/" too, # so "/*/qr/" reads as "/", anything, "/qr/" and catches /en/art/qr/, # /en/artists/qr/, /en/museums/qr/ and any /qr/ route added later. The # four per-entity rules it replaces are what let the bug survive: three of # them were written /*/artists/*/qr/ and matched nothing at all. Disallow: /*/qr/ Disallow: /*/artists/error-not-found/ Disallow: /*/artists/search-not-found/ Disallow: /*/museums/error-not-found/ Disallow: /*/museums/search-not-found/ Disallow: /*/articles/error-not-found/ Disallow: /*/articles/search-not-found/ # Developer and diagnostic pages. /*/test/ and /*/empty/ were written as # exact segments and matched only /en/test/ and a /en/empty/ that does not # exist; the real dev surfaces are named test-error, test-tmp, # test-upload-media and empty_page, and all four were crawlable. Disallow: /*/test-error/ Disallow: /*/test-tmp/ Disallow: /*/test-upload-media/ Disallow: /*/empty_page/ # --------------------------------------------------------------------- # Routes that seo_indexing.py marks noindex,nofollow and that cost real # work per hit. Added by the 2026-08-28 three-layer audit to close the # "noindex but crawlable" gap: a crawler was free to spend budget on # pages we then told it not to index. See DOC_ROBOTS_TXT_STRATEGY.md. # --------------------------------------------------------------------- # FTS redirect entrypoints: /art/show// and its three twins run a # full-text search per request only to 301 onto the canonical detail URL. Disallow: /*/art/show/ Disallow: /*/artists/show/ Disallow: /*/museums/show/ Disallow: /*/articles/show/ Disallow: /*/museums/ofworld/ # AJAX and infinite-scroll endpoints — data feeds for the browser, not pages. Disallow: /*/art/infinite_scroll/ Disallow: /*/art/favorites/check/ Disallow: /*/art/auction/ Disallow: /*/art/buy-texts.js # Thin/duplicate variants of pages that are already crawlable in canonical # form: the "simple" buy pages and the per-artist commission form. # # /*/art/search_by_color_wheel/ was here as "the colour-wheel twin of # search_by_color". Opened 2026-09-01: the two are different tools over the # same catalogue rather than one page and its duplicate, and the wheel is a # single URL per language, not a per-entity surface. Disallow: /*/art/*/simple/ Disallow: /*/artists/*/commission-artwork/ # --------------------------------------------------------------------- # Labyrinth decoy suffixes (2026-09-02). Every model detail page renders # five INVISIBLE rel=nofollow links appending one of these segments to # its own URL (core/includes/labyrinth_trap_links.html); the target 302s # into the maximum-difficulty captcha and flags the IP as a bot # (views_captcha_challenge.labyrinth_trap_decoy_view). These rules are # guardrail #1 and MUST stay in EVERY group — same argument as /core/ # above: a named group does not inherit them from *, and a trap that a # polite crawler (Googlebot, the GEO-strategy AI crawlers) may enter is # armed against exactly the crawlers we invite. Only robots.txt-ignoring # scrapers ever follow them. The suffix always terminates the URL, so # the mid-path wildcard shape '/*/<name>/' matches every decoy. Disallow: /*/Skynet/ Disallow: /*/Proteus/ Disallow: /*/Mechagodzilla/ Disallow: /*/Talos/ Disallow: /*/Cavil/ # NOT blocked on purpose: the four merchant product feeds under # /*/art/feed/ -- google-shopping.xml, microsoft-shopping.xml, # meta-catalog.xml, pinterest-catalog.xml. Google Merchant Center, # Microsoft Merchant Center, Meta Commerce Manager and Pinterest all # fetch their feed as a normal crawler and honour robots.txt, so a # Disallow here would silently kill the feed on all four. They are # noindex in seo_indexing.py, which is the correct pairing for a # machine-readable file that must stay fetchable. # NOT blocked on purpose: /*/articles/feed/rss.xml. Aggregators that # ingest the articles feed (Flipboard first) fetch it as ordinary # crawlers and honour robots.txt, so a Disallow here would silently # kill the feed. It carries X-Robots-Tag: noindex instead -- same # fetchable-but-not-indexed pairing as the shopping feed above. Crawl-delay: 0.75 # Google + the AI crawlers — full public access, no crawl delay (GEO strategy) User-agent: Googlebot User-agent: LinkedInBot User-agent: GPTBot User-agent: ChatGPT-User User-agent: OAI-SearchBot User-agent: ClaudeBot User-agent: Claude-Web User-agent: Claude-SearchBot User-agent: anthropic-ai User-agent: Google-Extended User-agent: GoogleOther User-agent: BingChatbot User-agent: PerplexityBot User-agent: Perplexity-User User-agent: Applebot-Extended User-agent: FacebookBot User-agent: Meta-ExternalAgent User-agent: YouBot User-agent: MistralAI-User Disallow: /admin/ Disallow: /*/admin/ Disallow: /manage/ Disallow: /*/manage/ Disallow: /users/ Disallow: /*/users/ Disallow: /accounts/ Disallow: /*/accounts/ Disallow: /*/orders/ Disallow: /*/admin_status Disallow: /*/artwork_addition/ Disallow: /*/marketing/ Disallow: /*/marketing_emails/ Disallow: /*/order/ Disallow: /*/order-customer-dashboard/ Disallow: /*/order-item-certificate/ Disallow: /*/order-item-image-download/ Disallow: /stripe/ Disallow: /17track/ Disallow: /chatbot/ Allow: /*/orders/art-appraisal/ Allow: /*/orders/art-consultation/ Allow: /*/orders/custom-reproduction/ Allow: /*/orders/photo-to-painting/ Allow: /*/orders/photo_to_restyled_masterpiece/ Disallow: /core/ Disallow: /api/ Disallow: /*/api/ Disallow: /i18n/ Disallow: /*/i18n/ Disallow: /ckeditor5/ Disallow: /debug/ Disallow: /__debug__/ Disallow: /misc_utils_tags/ Disallow: /media/ Allow: /media/artworks/ Allow: /media/products/ Disallow: /static/css/ Disallow: /static/js/ Disallow: /staticfiles/ Allow: /staticfiles/images/ Disallow: /*/set_country/ Disallow: /*/set_currency/ Disallow: /*/set_language/ Disallow: /*/process-newsletter/ Disallow: /*/process-art-suggestions/ Disallow: /*/favorite/addajax/ Disallow: /*/favorite/removeajax/ Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?filter= Disallow: /*/search-not-found/ Disallow: /*/error/ Disallow: /*/test/ Disallow: /*/empty/ Disallow: /*/art/diaporama/ Disallow: /*/art/ecard/ Disallow: /*/art/sell/ Disallow: /*/art/favorite/ Disallow: /*/art/error-not-found/ Disallow: /*/art/search-not-found/ Disallow: /*/art/*/?collection= Disallow: /*/art/similar/ Disallow: /*/art/preview/ Disallow: /*/art/preview-ar/ Disallow: /*/artists/atlas_nodes.json/ Disallow: /*/artists/artist_timeline_data.json/ Disallow: /*/artists/genome_data.json/ Disallow: /*/artists/mood_room_data.json/ Disallow: /*/artists/spectral_data.json/ Disallow: /*/artists/3DMuseum_scene.xml/ Disallow: /*/art/stats_data.json/ Disallow: /*/art/download/*/file/ Disallow: /*/art/download/*/embed/ Disallow: /*/art/embed/*/image/ Disallow: /*/art/embed/*/viewer/ Disallow: /*/art/embed/*/card/ Disallow: /qrcode/ Disallow: /*/qr/ Disallow: /*/artists/error-not-found/ Disallow: /*/artists/search-not-found/ Disallow: /*/museums/error-not-found/ Disallow: /*/museums/search-not-found/ Disallow: /*/articles/error-not-found/ Disallow: /*/articles/search-not-found/ Disallow: /*/test-error/ Disallow: /*/test-tmp/ Disallow: /*/test-upload-media/ Disallow: /*/empty_page/ Disallow: /*/art/show/ Disallow: /*/artists/show/ Disallow: /*/museums/show/ Disallow: /*/articles/show/ Disallow: /*/museums/ofworld/ Disallow: /*/art/infinite_scroll/ Disallow: /*/art/favorites/check/ Disallow: /*/art/auction/ Disallow: /*/art/buy-texts.js Disallow: /*/art/*/simple/ Disallow: /*/artists/*/commission-artwork/ Disallow: /*/Skynet/ Disallow: /*/Proteus/ Disallow: /*/Mechagodzilla/ Disallow: /*/Talos/ Disallow: /*/Cavil/ # Image and link-preview crawlers — artwork imagery explicitly allowed User-agent: Googlebot-Image User-agent: facebookexternalhit User-agent: Twitterbot Allow: /media/artworks/ Disallow: /admin/ Disallow: /*/admin/ Disallow: /manage/ Disallow: /*/manage/ Disallow: /users/ Disallow: /*/users/ Disallow: /accounts/ Disallow: /*/accounts/ Disallow: /*/orders/ Disallow: /*/admin_status Disallow: /*/artwork_addition/ Disallow: /*/marketing/ Disallow: /*/marketing_emails/ Disallow: /*/order/ Disallow: /*/order-customer-dashboard/ Disallow: /*/order-item-certificate/ Disallow: /*/order-item-image-download/ Disallow: /stripe/ Disallow: /17track/ Disallow: /chatbot/ Allow: /*/orders/art-appraisal/ Allow: /*/orders/art-consultation/ Allow: /*/orders/custom-reproduction/ Allow: /*/orders/photo-to-painting/ Allow: /*/orders/photo_to_restyled_masterpiece/ Disallow: /core/ Disallow: /api/ Disallow: /*/api/ Disallow: /i18n/ Disallow: /*/i18n/ Disallow: /ckeditor5/ Disallow: /debug/ Disallow: /__debug__/ Disallow: /misc_utils_tags/ Disallow: /media/ Allow: /media/artworks/ Allow: /media/products/ Disallow: /static/css/ Disallow: /static/js/ Disallow: /staticfiles/ Allow: /staticfiles/images/ Disallow: /*/set_country/ Disallow: /*/set_currency/ Disallow: /*/set_language/ Disallow: /*/process-newsletter/ Disallow: /*/process-art-suggestions/ Disallow: /*/favorite/addajax/ Disallow: /*/favorite/removeajax/ Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?filter= Disallow: /*/search-not-found/ Disallow: /*/error/ Disallow: /*/test/ Disallow: /*/empty/ Disallow: /*/art/diaporama/ Disallow: /*/art/ecard/ Disallow: /*/art/sell/ Disallow: /*/art/favorite/ Disallow: /*/art/error-not-found/ Disallow: /*/art/search-not-found/ Disallow: /*/art/*/?collection= Disallow: /*/art/similar/ Disallow: /*/art/preview/ Disallow: /*/art/preview-ar/ Disallow: /*/artists/atlas_nodes.json/ Disallow: /*/artists/artist_timeline_data.json/ Disallow: /*/artists/genome_data.json/ Disallow: /*/artists/mood_room_data.json/ Disallow: /*/artists/spectral_data.json/ Disallow: /*/artists/3DMuseum_scene.xml/ Disallow: /*/art/stats_data.json/ Disallow: /*/art/download/*/file/ Disallow: /*/art/download/*/embed/ Disallow: /*/art/embed/*/image/ Disallow: /*/art/embed/*/viewer/ Disallow: /*/art/embed/*/card/ Disallow: /qrcode/ Disallow: /*/qr/ Disallow: /*/artists/error-not-found/ Disallow: /*/artists/search-not-found/ Disallow: /*/museums/error-not-found/ Disallow: /*/museums/search-not-found/ Disallow: /*/articles/error-not-found/ Disallow: /*/articles/search-not-found/ Disallow: /*/test-error/ Disallow: /*/test-tmp/ Disallow: /*/test-upload-media/ Disallow: /*/empty_page/ Disallow: /*/art/show/ Disallow: /*/artists/show/ Disallow: /*/museums/show/ Disallow: /*/articles/show/ Disallow: /*/museums/ofworld/ Disallow: /*/art/infinite_scroll/ Disallow: /*/art/favorites/check/ Disallow: /*/art/auction/ Disallow: /*/art/buy-texts.js Disallow: /*/art/*/simple/ Disallow: /*/artists/*/commission-artwork/ Disallow: /*/Skynet/ Disallow: /*/Proteus/ Disallow: /*/Mechagodzilla/ Disallow: /*/Talos/ Disallow: /*/Cavil/ # WhatsApp link previews User-agent: WhatsApp Allow: /media/artworks/ Disallow: /admin/ Disallow: /*/admin/ Disallow: /manage/ Disallow: /*/manage/ Disallow: /users/ Disallow: /*/users/ Disallow: /accounts/ Disallow: /*/accounts/ Disallow: /*/orders/ Disallow: /*/admin_status Disallow: /*/artwork_addition/ Disallow: /*/marketing/ Disallow: /*/marketing_emails/ Disallow: /*/order/ Disallow: /*/order-customer-dashboard/ Disallow: /*/order-item-certificate/ Disallow: /*/order-item-image-download/ Disallow: /stripe/ Disallow: /17track/ Disallow: /chatbot/ Allow: /*/orders/art-appraisal/ Allow: /*/orders/art-consultation/ Allow: /*/orders/custom-reproduction/ Allow: /*/orders/photo-to-painting/ Allow: /*/orders/photo_to_restyled_masterpiece/ Disallow: /core/ Disallow: /api/ Disallow: /*/api/ Disallow: /i18n/ Disallow: /*/i18n/ Disallow: /ckeditor5/ Disallow: /debug/ Disallow: /__debug__/ Disallow: /misc_utils_tags/ Disallow: /media/ Allow: /media/artworks/ Allow: /media/products/ Disallow: /static/css/ Disallow: /static/js/ Disallow: /staticfiles/ Allow: /staticfiles/images/ Disallow: /*/set_country/ Disallow: /*/set_currency/ Disallow: /*/set_language/ Disallow: /*/process-newsletter/ Disallow: /*/process-art-suggestions/ Disallow: /*/favorite/addajax/ Disallow: /*/favorite/removeajax/ Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?filter= Disallow: /*/search-not-found/ Disallow: /*/error/ Disallow: /*/test/ Disallow: /*/empty/ Disallow: /*/art/diaporama/ Disallow: /*/art/ecard/ Disallow: /*/art/sell/ Disallow: /*/art/favorite/ Disallow: /*/art/error-not-found/ Disallow: /*/art/search-not-found/ Disallow: /*/art/*/?collection= Disallow: /*/art/similar/ Disallow: /*/art/preview/ Disallow: /*/art/preview-ar/ Disallow: /*/artists/atlas_nodes.json/ Disallow: /*/artists/artist_timeline_data.json/ Disallow: /*/artists/genome_data.json/ Disallow: /*/artists/mood_room_data.json/ Disallow: /*/artists/spectral_data.json/ Disallow: /*/artists/3DMuseum_scene.xml/ Disallow: /*/art/stats_data.json/ Disallow: /*/art/download/*/file/ Disallow: /*/art/download/*/embed/ Disallow: /*/art/embed/*/image/ Disallow: /*/art/embed/*/viewer/ Disallow: /*/art/embed/*/card/ Disallow: /qrcode/ Disallow: /*/qr/ Disallow: /*/artists/error-not-found/ Disallow: /*/artists/search-not-found/ Disallow: /*/museums/error-not-found/ Disallow: /*/museums/search-not-found/ Disallow: /*/articles/error-not-found/ Disallow: /*/articles/search-not-found/ Disallow: /*/test-error/ Disallow: /*/test-tmp/ Disallow: /*/test-upload-media/ Disallow: /*/empty_page/ Disallow: /*/art/show/ Disallow: /*/artists/show/ Disallow: /*/museums/show/ Disallow: /*/articles/show/ Disallow: /*/museums/ofworld/ Disallow: /*/art/infinite_scroll/ Disallow: /*/art/favorites/check/ Disallow: /*/art/auction/ Disallow: /*/art/buy-texts.js Disallow: /*/art/*/simple/ Disallow: /*/artists/*/commission-artwork/ Disallow: /*/Skynet/ Disallow: /*/Proteus/ Disallow: /*/Mechagodzilla/ Disallow: /*/Talos/ Disallow: /*/Cavil/ # Pinterest pins User-agent: Pinterestbot Allow: /media/artworks/ Disallow: /admin/ Disallow: /*/admin/ Disallow: /manage/ Disallow: /*/manage/ Disallow: /users/ Disallow: /*/users/ Disallow: /accounts/ Disallow: /*/accounts/ Disallow: /*/orders/ Disallow: /*/admin_status Disallow: /*/artwork_addition/ Disallow: /*/marketing/ Disallow: /*/marketing_emails/ Disallow: /*/order/ Disallow: /*/order-customer-dashboard/ Disallow: /*/order-item-certificate/ Disallow: /*/order-item-image-download/ Disallow: /stripe/ Disallow: /17track/ Disallow: /chatbot/ Allow: /*/orders/art-appraisal/ Allow: /*/orders/art-consultation/ Allow: /*/orders/custom-reproduction/ Allow: /*/orders/photo-to-painting/ Allow: /*/orders/photo_to_restyled_masterpiece/ Disallow: /core/ Disallow: /api/ Disallow: /*/api/ Disallow: /i18n/ Disallow: /*/i18n/ Disallow: /ckeditor5/ Disallow: /debug/ Disallow: /__debug__/ Disallow: /misc_utils_tags/ Disallow: /media/ Allow: /media/artworks/ Allow: /media/products/ Disallow: /static/css/ Disallow: /static/js/ Disallow: /staticfiles/ Allow: /staticfiles/images/ Disallow: /*/set_country/ Disallow: /*/set_currency/ Disallow: /*/set_language/ Disallow: /*/process-newsletter/ Disallow: /*/process-art-suggestions/ Disallow: /*/favorite/addajax/ Disallow: /*/favorite/removeajax/ Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?filter= Disallow: /*/search-not-found/ Disallow: /*/error/ Disallow: /*/test/ Disallow: /*/empty/ Disallow: /*/art/diaporama/ Disallow: /*/art/ecard/ Disallow: /*/art/sell/ Disallow: /*/art/favorite/ Disallow: /*/art/error-not-found/ Disallow: /*/art/search-not-found/ Disallow: /*/art/*/?collection= Disallow: /*/art/similar/ Disallow: /*/art/preview/ Disallow: /*/art/preview-ar/ Disallow: /*/artists/atlas_nodes.json/ Disallow: /*/artists/artist_timeline_data.json/ Disallow: /*/artists/genome_data.json/ Disallow: /*/artists/mood_room_data.json/ Disallow: /*/artists/spectral_data.json/ Disallow: /*/artists/3DMuseum_scene.xml/ Disallow: /*/art/stats_data.json/ Disallow: /*/art/download/*/file/ Disallow: /*/art/download/*/embed/ Disallow: /*/art/embed/*/image/ Disallow: /*/art/embed/*/viewer/ Disallow: /*/art/embed/*/card/ Disallow: /qrcode/ Disallow: /*/qr/ Disallow: /*/artists/error-not-found/ Disallow: /*/artists/search-not-found/ Disallow: /*/museums/error-not-found/ Disallow: /*/museums/search-not-found/ Disallow: /*/articles/error-not-found/ Disallow: /*/articles/search-not-found/ Disallow: /*/test-error/ Disallow: /*/test-tmp/ Disallow: /*/test-upload-media/ Disallow: /*/empty_page/ Disallow: /*/art/show/ Disallow: /*/artists/show/ Disallow: /*/museums/show/ Disallow: /*/articles/show/ Disallow: /*/museums/ofworld/ Disallow: /*/art/infinite_scroll/ Disallow: /*/art/favorites/check/ Disallow: /*/art/auction/ Disallow: /*/art/buy-texts.js Disallow: /*/art/*/simple/ Disallow: /*/artists/*/commission-artwork/ Disallow: /*/Skynet/ Disallow: /*/Proteus/ Disallow: /*/Mechagodzilla/ Disallow: /*/Talos/ Disallow: /*/Cavil/ Crawl-delay: 1 # TinEye reverse-image index — artwork imagery explicitly allowed User-agent: TinEyeBot Allow: /media/artworks/ Disallow: /admin/ Disallow: /*/admin/ Disallow: /manage/ Disallow: /*/manage/ Disallow: /users/ Disallow: /*/users/ Disallow: /accounts/ Disallow: /*/accounts/ Disallow: /*/orders/ Disallow: /*/admin_status Disallow: /*/artwork_addition/ Disallow: /*/marketing/ Disallow: /*/marketing_emails/ Disallow: /*/order/ Disallow: /*/order-customer-dashboard/ Disallow: /*/order-item-certificate/ Disallow: /*/order-item-image-download/ Disallow: /stripe/ Disallow: /17track/ Disallow: /chatbot/ Allow: /*/orders/art-appraisal/ Allow: /*/orders/art-consultation/ Allow: /*/orders/custom-reproduction/ Allow: /*/orders/photo-to-painting/ Allow: /*/orders/photo_to_restyled_masterpiece/ Disallow: /core/ Disallow: /api/ Disallow: /*/api/ Disallow: /i18n/ Disallow: /*/i18n/ Disallow: /ckeditor5/ Disallow: /debug/ Disallow: /__debug__/ Disallow: /misc_utils_tags/ Disallow: /media/ Allow: /media/artworks/ Allow: /media/products/ Disallow: /static/css/ Disallow: /static/js/ Disallow: /staticfiles/ Allow: /staticfiles/images/ Disallow: /*/set_country/ Disallow: /*/set_currency/ Disallow: /*/set_language/ Disallow: /*/process-newsletter/ Disallow: /*/process-art-suggestions/ Disallow: /*/favorite/addajax/ Disallow: /*/favorite/removeajax/ Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?filter= Disallow: /*/search-not-found/ Disallow: /*/error/ Disallow: /*/test/ Disallow: /*/empty/ Disallow: /*/art/diaporama/ Disallow: /*/art/ecard/ Disallow: /*/art/sell/ Disallow: /*/art/favorite/ Disallow: /*/art/error-not-found/ Disallow: /*/art/search-not-found/ Disallow: /*/art/*/?collection= Disallow: /*/art/similar/ Disallow: /*/art/preview/ Disallow: /*/art/preview-ar/ Disallow: /*/artists/atlas_nodes.json/ Disallow: /*/artists/artist_timeline_data.json/ Disallow: /*/artists/genome_data.json/ Disallow: /*/artists/mood_room_data.json/ Disallow: /*/artists/spectral_data.json/ Disallow: /*/artists/3DMuseum_scene.xml/ Disallow: /*/art/stats_data.json/ Disallow: /*/art/download/*/file/ Disallow: /*/art/download/*/embed/ Disallow: /*/art/embed/*/image/ Disallow: /*/art/embed/*/viewer/ Disallow: /*/art/embed/*/card/ Disallow: /qrcode/ Disallow: /*/qr/ Disallow: /*/artists/error-not-found/ Disallow: /*/artists/search-not-found/ Disallow: /*/museums/error-not-found/ Disallow: /*/museums/search-not-found/ Disallow: /*/articles/error-not-found/ Disallow: /*/articles/search-not-found/ Disallow: /*/test-error/ Disallow: /*/test-tmp/ Disallow: /*/test-upload-media/ Disallow: /*/empty_page/ Disallow: /*/art/show/ Disallow: /*/artists/show/ Disallow: /*/museums/show/ Disallow: /*/articles/show/ Disallow: /*/museums/ofworld/ Disallow: /*/art/infinite_scroll/ Disallow: /*/art/favorites/check/ Disallow: /*/art/auction/ Disallow: /*/art/buy-texts.js Disallow: /*/art/*/simple/ Disallow: /*/artists/*/commission-artwork/ Disallow: /*/Skynet/ Disallow: /*/Proteus/ Disallow: /*/Mechagodzilla/ Disallow: /*/Talos/ Disallow: /*/Cavil/ Crawl-delay: 1 # Bing and DuckDuckGo User-agent: bingbot User-agent: DuckDuckBot Disallow: /admin/ Disallow: /*/admin/ Disallow: /manage/ Disallow: /*/manage/ Disallow: /users/ Disallow: /*/users/ Disallow: /accounts/ Disallow: /*/accounts/ Disallow: /*/orders/ Disallow: /*/admin_status Disallow: /*/artwork_addition/ Disallow: /*/marketing/ Disallow: /*/marketing_emails/ Disallow: /*/order/ Disallow: /*/order-customer-dashboard/ Disallow: /*/order-item-certificate/ Disallow: /*/order-item-image-download/ Disallow: /stripe/ Disallow: /17track/ Disallow: /chatbot/ Allow: /*/orders/art-appraisal/ Allow: /*/orders/art-consultation/ Allow: /*/orders/custom-reproduction/ Allow: /*/orders/photo-to-painting/ Allow: /*/orders/photo_to_restyled_masterpiece/ Disallow: /core/ Disallow: /api/ Disallow: /*/api/ Disallow: /i18n/ Disallow: /*/i18n/ Disallow: /ckeditor5/ Disallow: /debug/ Disallow: /__debug__/ Disallow: /misc_utils_tags/ Disallow: /media/ Allow: /media/artworks/ Allow: /media/products/ Disallow: /static/css/ Disallow: /static/js/ Disallow: /staticfiles/ Allow: /staticfiles/images/ Disallow: /*/set_country/ Disallow: /*/set_currency/ Disallow: /*/set_language/ Disallow: /*/process-newsletter/ Disallow: /*/process-art-suggestions/ Disallow: /*/favorite/addajax/ Disallow: /*/favorite/removeajax/ Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?filter= Disallow: /*/search-not-found/ Disallow: /*/error/ Disallow: /*/test/ Disallow: /*/empty/ Disallow: /*/art/diaporama/ Disallow: /*/art/ecard/ Disallow: /*/art/sell/ Disallow: /*/art/favorite/ Disallow: /*/art/error-not-found/ Disallow: /*/art/search-not-found/ Disallow: /*/art/*/?collection= Disallow: /*/art/similar/ Disallow: /*/art/preview/ Disallow: /*/art/preview-ar/ Disallow: /*/artists/atlas_nodes.json/ Disallow: /*/artists/artist_timeline_data.json/ Disallow: /*/artists/genome_data.json/ Disallow: /*/artists/mood_room_data.json/ Disallow: /*/artists/spectral_data.json/ Disallow: /*/artists/3DMuseum_scene.xml/ Disallow: /*/art/stats_data.json/ Disallow: /*/art/download/*/file/ Disallow: /*/art/download/*/embed/ Disallow: /*/art/embed/*/image/ Disallow: /*/art/embed/*/viewer/ Disallow: /*/art/embed/*/card/ Disallow: /qrcode/ Disallow: /*/qr/ Disallow: /*/artists/error-not-found/ Disallow: /*/artists/search-not-found/ Disallow: /*/museums/error-not-found/ Disallow: /*/museums/search-not-found/ Disallow: /*/articles/error-not-found/ Disallow: /*/articles/search-not-found/ Disallow: /*/test-error/ Disallow: /*/test-tmp/ Disallow: /*/test-upload-media/ Disallow: /*/empty_page/ Disallow: /*/art/show/ Disallow: /*/artists/show/ Disallow: /*/museums/show/ Disallow: /*/articles/show/ Disallow: /*/museums/ofworld/ Disallow: /*/art/infinite_scroll/ Disallow: /*/art/favorites/check/ Disallow: /*/art/auction/ Disallow: /*/art/buy-texts.js Disallow: /*/art/*/simple/ Disallow: /*/artists/*/commission-artwork/ Disallow: /*/Skynet/ Disallow: /*/Proteus/ Disallow: /*/Mechagodzilla/ Disallow: /*/Talos/ Disallow: /*/Cavil/ Crawl-delay: 1 # Regional search engines — slower crawl User-agent: Yandex User-agent: Baiduspider User-agent: Sogou User-agent: 360Spider Disallow: /admin/ Disallow: /*/admin/ Disallow: /manage/ Disallow: /*/manage/ Disallow: /users/ Disallow: /*/users/ Disallow: /accounts/ Disallow: /*/accounts/ Disallow: /*/orders/ Disallow: /*/admin_status Disallow: /*/artwork_addition/ Disallow: /*/marketing/ Disallow: /*/marketing_emails/ Disallow: /*/order/ Disallow: /*/order-customer-dashboard/ Disallow: /*/order-item-certificate/ Disallow: /*/order-item-image-download/ Disallow: /stripe/ Disallow: /17track/ Disallow: /chatbot/ Allow: /*/orders/art-appraisal/ Allow: /*/orders/art-consultation/ Allow: /*/orders/custom-reproduction/ Allow: /*/orders/photo-to-painting/ Allow: /*/orders/photo_to_restyled_masterpiece/ Disallow: /core/ Disallow: /api/ Disallow: /*/api/ Disallow: /i18n/ Disallow: /*/i18n/ Disallow: /ckeditor5/ Disallow: /debug/ Disallow: /__debug__/ Disallow: /misc_utils_tags/ Disallow: /media/ Allow: /media/artworks/ Allow: /media/products/ Disallow: /static/css/ Disallow: /static/js/ Disallow: /staticfiles/ Allow: /staticfiles/images/ Disallow: /*/set_country/ Disallow: /*/set_currency/ Disallow: /*/set_language/ Disallow: /*/process-newsletter/ Disallow: /*/process-art-suggestions/ Disallow: /*/favorite/addajax/ Disallow: /*/favorite/removeajax/ Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?filter= Disallow: /*/search-not-found/ Disallow: /*/error/ Disallow: /*/test/ Disallow: /*/empty/ Disallow: /*/art/diaporama/ Disallow: /*/art/ecard/ Disallow: /*/art/sell/ Disallow: /*/art/favorite/ Disallow: /*/art/error-not-found/ Disallow: /*/art/search-not-found/ Disallow: /*/art/*/?collection= Disallow: /*/art/similar/ Disallow: /*/art/preview/ Disallow: /*/art/preview-ar/ Disallow: /*/artists/atlas_nodes.json/ Disallow: /*/artists/artist_timeline_data.json/ Disallow: /*/artists/genome_data.json/ Disallow: /*/artists/mood_room_data.json/ Disallow: /*/artists/spectral_data.json/ Disallow: /*/artists/3DMuseum_scene.xml/ Disallow: /*/art/stats_data.json/ Disallow: /*/art/download/*/file/ Disallow: /*/art/download/*/embed/ Disallow: /*/art/embed/*/image/ Disallow: /*/art/embed/*/viewer/ Disallow: /*/art/embed/*/card/ Disallow: /qrcode/ Disallow: /*/qr/ Disallow: /*/artists/error-not-found/ Disallow: /*/artists/search-not-found/ Disallow: /*/museums/error-not-found/ Disallow: /*/museums/search-not-found/ Disallow: /*/articles/error-not-found/ Disallow: /*/articles/search-not-found/ Disallow: /*/test-error/ Disallow: /*/test-tmp/ Disallow: /*/test-upload-media/ Disallow: /*/empty_page/ Disallow: /*/art/show/ Disallow: /*/artists/show/ Disallow: /*/museums/show/ Disallow: /*/articles/show/ Disallow: /*/museums/ofworld/ Disallow: /*/art/infinite_scroll/ Disallow: /*/art/favorites/check/ Disallow: /*/art/auction/ Disallow: /*/art/buy-texts.js Disallow: /*/art/*/simple/ Disallow: /*/artists/*/commission-artwork/ Disallow: /*/Skynet/ Disallow: /*/Proteus/ Disallow: /*/Mechagodzilla/ Disallow: /*/Talos/ Disallow: /*/Cavil/ Crawl-delay: 2 # SEO/backlink crawlers — allowed, but throttled hard User-agent: MJ12bot User-agent: DotBot Disallow: /static/ Disallow: /media-resize/ Disallow: /admin/ Disallow: /*/admin/ Disallow: /manage/ Disallow: /*/manage/ Disallow: /users/ Disallow: /*/users/ Disallow: /accounts/ Disallow: /*/accounts/ Disallow: /*/orders/ Disallow: /*/admin_status Disallow: /*/artwork_addition/ Disallow: /*/marketing/ Disallow: /*/marketing_emails/ Disallow: /*/order/ Disallow: /*/order-customer-dashboard/ Disallow: /*/order-item-certificate/ Disallow: /*/order-item-image-download/ Disallow: /stripe/ Disallow: /17track/ Disallow: /chatbot/ Allow: /*/orders/art-appraisal/ Allow: /*/orders/art-consultation/ Allow: /*/orders/custom-reproduction/ Allow: /*/orders/photo-to-painting/ Allow: /*/orders/photo_to_restyled_masterpiece/ Disallow: /core/ Disallow: /api/ Disallow: /*/api/ Disallow: /i18n/ Disallow: /*/i18n/ Disallow: /ckeditor5/ Disallow: /debug/ Disallow: /__debug__/ Disallow: /misc_utils_tags/ Disallow: /media/ Allow: /media/artworks/ Allow: /media/products/ Disallow: /static/css/ Disallow: /static/js/ Disallow: /staticfiles/ Allow: /staticfiles/images/ Disallow: /*/set_country/ Disallow: /*/set_currency/ Disallow: /*/set_language/ Disallow: /*/process-newsletter/ Disallow: /*/process-art-suggestions/ Disallow: /*/favorite/addajax/ Disallow: /*/favorite/removeajax/ Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?filter= Disallow: /*/search-not-found/ Disallow: /*/error/ Disallow: /*/test/ Disallow: /*/empty/ Disallow: /*/art/diaporama/ Disallow: /*/art/ecard/ Disallow: /*/art/sell/ Disallow: /*/art/favorite/ Disallow: /*/art/error-not-found/ Disallow: /*/art/search-not-found/ Disallow: /*/art/*/?collection= Disallow: /*/art/similar/ Disallow: /*/art/preview/ Disallow: /*/art/preview-ar/ Disallow: /*/artists/atlas_nodes.json/ Disallow: /*/artists/artist_timeline_data.json/ Disallow: /*/artists/genome_data.json/ Disallow: /*/artists/mood_room_data.json/ Disallow: /*/artists/spectral_data.json/ Disallow: /*/artists/3DMuseum_scene.xml/ Disallow: /*/art/stats_data.json/ Disallow: /*/art/download/*/file/ Disallow: /*/art/download/*/embed/ Disallow: /*/art/embed/*/image/ Disallow: /*/art/embed/*/viewer/ Disallow: /*/art/embed/*/card/ Disallow: /qrcode/ Disallow: /*/qr/ Disallow: /*/artists/error-not-found/ Disallow: /*/artists/search-not-found/ Disallow: /*/museums/error-not-found/ Disallow: /*/museums/search-not-found/ Disallow: /*/articles/error-not-found/ Disallow: /*/articles/search-not-found/ Disallow: /*/test-error/ Disallow: /*/test-tmp/ Disallow: /*/test-upload-media/ Disallow: /*/empty_page/ Disallow: /*/art/show/ Disallow: /*/artists/show/ Disallow: /*/museums/show/ Disallow: /*/articles/show/ Disallow: /*/museums/ofworld/ Disallow: /*/art/infinite_scroll/ Disallow: /*/art/favorites/check/ Disallow: /*/art/auction/ Disallow: /*/art/buy-texts.js Disallow: /*/art/*/simple/ Disallow: /*/artists/*/commission-artwork/ Disallow: /*/Skynet/ Disallow: /*/Proteus/ Disallow: /*/Mechagodzilla/ Disallow: /*/Talos/ Disallow: /*/Cavil/ Crawl-delay: 10 # Internet Archive User-agent: ia_archiver Disallow: /media-resize/ Disallow: /admin/ Disallow: /*/admin/ Disallow: /manage/ Disallow: /*/manage/ Disallow: /users/ Disallow: /*/users/ Disallow: /accounts/ Disallow: /*/accounts/ Disallow: /*/orders/ Disallow: /*/admin_status Disallow: /*/artwork_addition/ Disallow: /*/marketing/ Disallow: /*/marketing_emails/ Disallow: /*/order/ Disallow: /*/order-customer-dashboard/ Disallow: /*/order-item-certificate/ Disallow: /*/order-item-image-download/ Disallow: /stripe/ Disallow: /17track/ Disallow: /chatbot/ Allow: /*/orders/art-appraisal/ Allow: /*/orders/art-consultation/ Allow: /*/orders/custom-reproduction/ Allow: /*/orders/photo-to-painting/ Allow: /*/orders/photo_to_restyled_masterpiece/ Disallow: /core/ Disallow: /api/ Disallow: /*/api/ Disallow: /i18n/ Disallow: /*/i18n/ Disallow: /ckeditor5/ Disallow: /debug/ Disallow: /__debug__/ Disallow: /misc_utils_tags/ Disallow: /media/ Allow: /media/artworks/ Allow: /media/products/ Disallow: /static/css/ Disallow: /static/js/ Disallow: /staticfiles/ Allow: /staticfiles/images/ Disallow: /*/set_country/ Disallow: /*/set_currency/ Disallow: /*/set_language/ Disallow: /*/process-newsletter/ Disallow: /*/process-art-suggestions/ Disallow: /*/favorite/addajax/ Disallow: /*/favorite/removeajax/ Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?filter= Disallow: /*/search-not-found/ Disallow: /*/error/ Disallow: /*/test/ Disallow: /*/empty/ Disallow: /*/art/diaporama/ Disallow: /*/art/ecard/ Disallow: /*/art/sell/ Disallow: /*/art/favorite/ Disallow: /*/art/error-not-found/ Disallow: /*/art/search-not-found/ Disallow: /*/art/*/?collection= Disallow: /*/art/similar/ Disallow: /*/art/preview/ Disallow: /*/art/preview-ar/ Disallow: /*/artists/atlas_nodes.json/ Disallow: /*/artists/artist_timeline_data.json/ Disallow: /*/artists/genome_data.json/ Disallow: /*/artists/mood_room_data.json/ Disallow: /*/artists/spectral_data.json/ Disallow: /*/artists/3DMuseum_scene.xml/ Disallow: /*/art/stats_data.json/ Disallow: /*/art/download/*/file/ Disallow: /*/art/download/*/embed/ Disallow: /*/art/embed/*/image/ Disallow: /*/art/embed/*/viewer/ Disallow: /*/art/embed/*/card/ Disallow: /qrcode/ Disallow: /*/qr/ Disallow: /*/artists/error-not-found/ Disallow: /*/artists/search-not-found/ Disallow: /*/museums/error-not-found/ Disallow: /*/museums/search-not-found/ Disallow: /*/articles/error-not-found/ Disallow: /*/articles/search-not-found/ Disallow: /*/test-error/ Disallow: /*/test-tmp/ Disallow: /*/test-upload-media/ Disallow: /*/empty_page/ Disallow: /*/art/show/ Disallow: /*/artists/show/ Disallow: /*/museums/show/ Disallow: /*/articles/show/ Disallow: /*/museums/ofworld/ Disallow: /*/art/infinite_scroll/ Disallow: /*/art/favorites/check/ Disallow: /*/art/auction/ Disallow: /*/art/buy-texts.js Disallow: /*/art/*/simple/ Disallow: /*/artists/*/commission-artwork/ Disallow: /*/Skynet/ Disallow: /*/Proteus/ Disallow: /*/Mechagodzilla/ Disallow: /*/Talos/ Disallow: /*/Cavil/ Crawl-delay: 5 # Scanners and abusive crawlers — blocked outright User-agent: MauiBot User-agent: SurveyBot User-agent: masscan User-agent: nmap User-agent: sqlmap User-agent: CCBot User-agent: Amazonbot User-agent: Bytespider User-agent: cohere-ai User-agent: Diffbot User-agent: Omgilibot User-agent: AhrefsBot User-agent: SemrushBot Disallow: / # -------------------------------------------------------------------------- # SITEMAPS — absolute URLs are mandatory; a relative path is silently # discarded by Google and Bing (this is why the file is per-brand). # -------------------------------------------------------------------------- Sitemap: https://wikioo.org/sitemap.xml Sitemap: https://wikioo.org/en/sitemap-index.xml Sitemap: https://wikioo.org/fr/sitemap-index.xml Sitemap: https://wikioo.org/it/sitemap-index.xml Sitemap: https://wikioo.org/es/sitemap-index.xml Sitemap: https://wikioo.org/pt/sitemap-index.xml Sitemap: https://wikioo.org/de/sitemap-index.xml Sitemap: https://wikioo.org/pl/sitemap-index.xml Sitemap: https://wikioo.org/ro/sitemap-index.xml Sitemap: https://wikioo.org/sv/sitemap-index.xml Sitemap: https://wikioo.org/nl/sitemap-index.xml Sitemap: https://wikioo.org/uk/sitemap-index.xml Sitemap: https://wikioo.org/hu/sitemap-index.xml Sitemap: https://wikioo.org/el/sitemap-index.xml Sitemap: https://wikioo.org/cs/sitemap-index.xml Sitemap: https://wikioo.org/sr/sitemap-index.xml Sitemap: https://wikioo.org/bg/sitemap-index.xml Sitemap: https://wikioo.org/da/sitemap-index.xml Sitemap: https://wikioo.org/hr/sitemap-index.xml Sitemap: https://wikioo.org/fi/sitemap-index.xml Sitemap: https://wikioo.org/lt/sitemap-index.xml Sitemap: https://wikioo.org/sl/sitemap-index.xml Sitemap: https://wikioo.org/et/sitemap-index.xml Sitemap: https://wikioo.org/sk/sitemap-index.xml Sitemap: https://wikioo.org/no/sitemap-index.xml Sitemap: https://wikioo.org/zh/sitemap-index.xml Sitemap: https://wikioo.org/hi/sitemap-index.xml Sitemap: https://wikioo.org/ar/sitemap-index.xml Sitemap: https://wikioo.org/ja/sitemap-index.xml Sitemap: https://wikioo.org/tr/sitemap-index.xml Sitemap: https://wikioo.org/ko/sitemap-index.xml Sitemap: https://wikioo.org/ru/sitemap-index.xml Sitemap: https://wikioo.org/ka/sitemap-index.xml