{"schemaVersion":"hugging-bay.ai-crawler-policy.v1","generatedAt":"2026-09-05T15:04:23.370Z","baseUrl":"https://huggingbay.xyz","robotsUrl":"https://huggingbay.xyz/robots.txt","answerEngineManifestUrl":"https://huggingbay.xyz/.well-known/answer-engine.json","botAllowlistUrl":"https://huggingbay.xyz/.well-known/ai-bot-allowlist.json","llmsTxtUrl":"https://huggingbay.xyz/llms.txt","llmsFullTxtUrl":"https://huggingbay.xyz/llms-full.txt","policy":{"publicSearchIndexing":"allowed","publicAiInputForUserAnswers":"allowed","publicMetadataTraining":"allowed for public catalog, public docs, and public hosted-file metadata","frontierAiAssistants":"allowed for public pages and user-requested fetches","openSourceAgents":"allowed through User-agent * plus bounded APIs, sitemaps, llms.txt, and indexing URL batches","privateData":"not exposed; protected by authentication and never listed in public agent manifests","gatedOrAdminEndpoints":"Canonical artifact completeness, verification, execution readiness, and download readiness are unavailable during containment.","noGatedPrivateBypass":true},"preferredCrawlEntrypoints":["https://huggingbay.xyz/robots.txt","https://huggingbay.xyz/robots.json","https://huggingbay.xyz/.well-known/agent-discovery.json","https://huggingbay.xyz/api/agent-discovery","https://huggingbay.xyz/agents.txt","https://huggingbay.xyz/AGENTS.md","https://huggingbay.xyz/.well-known/grok.json","https://huggingbay.xyz/llms.txt","https://huggingbay.xyz/.well-known/llms.txt","https://huggingbay.xyz/sitemap.xml","https://huggingbay.xyz/api/ai-search","https://huggingbay.xyz/.well-known/ai-search.json","https://huggingbay.xyz/api/agent-recommendation-kit","https://huggingbay.xyz/agent-recommendation-kit","https://huggingbay.xyz/api/citations","https://huggingbay.xyz/.well-known/ai-citations.json","https://huggingbay.xyz/api/ai-visibility","https://huggingbay.xyz/api/traffic","https://huggingbay.xyz/api/ai-mention-monitor","https://huggingbay.xyz/api/ai-mention-results","https://huggingbay.xyz/api/community/review-queue?mode=needs-evidence&hosting=actionable&limit=25","https://huggingbay.xyz/publishers/launch-kit","https://huggingbay.xyz/api/publishers/launch-kit","https://huggingbay.xyz/api/indexing/status","https://huggingbay.xyz/api/answers","https://huggingbay.xyz/data/manifest.json","https://huggingbay.xyz/api/dumps/catalog.jsonl?limit=1000","https://huggingbay.xyz/api/dumps/hosted-files.jsonl?limit=1000","https://huggingbay.xyz/feeds/recent.xml","https://huggingbay.xyz/feeds/trending.xml","https://huggingbay.xyz/feeds/hosted-mirrors.xml","https://huggingbay.xyz/feeds/reviews.xml","https://huggingbay.xyz/api/indexing/urls?kind=answer&limit=100","https://huggingbay.xyz/api/indexing/urls?kind=agent-doc&limit=100","https://huggingbay.xyz/api/indexing/urls?kind=artifact&projection=url-only&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=artifact-section&projection=url-only&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=artifact-evidence&projection=url-only&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=publisher&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=downloadable&limit=1000&cursor=0","https://huggingbay.xyz/api/mcp","https://huggingbay.xyz/mcp","https://huggingbay.xyz/browser-extension","https://huggingbay.xyz/browser-extension/manifest.json","https://huggingbay.xyz/api/radar","https://huggingbay.xyz/api/trending?window=7d&limit=25","https://huggingbay.xyz/api/mirror-demand?requestedOnly=1&limit=100","https://huggingbay.xyz/api/mirror-readiness?limit=100"],"preferredCitationTargets":["https://huggingbay.xyz/ai-search","https://huggingbay.xyz/ai-citations","https://huggingbay.xyz/answers","https://huggingbay.xyz/answers/{slug}","https://huggingbay.xyz/artifact/{artifactId}","https://huggingbay.xyz/rankings/top-open-models","https://huggingbay.xyz/rankings/top-open-source-llms","https://huggingbay.xyz/rankings/top-seeded-models","https://huggingbay.xyz/rankings/hosted-run-ready","https://huggingbay.xyz/rankings/hosted-small-local-model-files","https://huggingbay.xyz/rankings/community-verified","https://huggingbay.xyz/rankings/needs-first-review","https://huggingbay.xyz/rankings/new-open-weight-models","https://huggingbay.xyz/rankings/coding-models","https://huggingbay.xyz/rankings/consumer-gpu-models","https://huggingbay.xyz/rankings/preservation-watchlist","https://huggingbay.xyz/rankings/best-7b-8b-models","https://huggingbay.xyz/rankings/best-14b-models","https://huggingbay.xyz/rankings/best-32b-models","https://huggingbay.xyz/rankings/best-70b-models","https://huggingbay.xyz/rankings/global-model-labs","https://huggingbay.xyz/rankings/eastern-open-models","https://huggingbay.xyz/downloads","https://huggingbay.xyz/sources","https://huggingbay.xyz/stacks","https://huggingbay.xyz/publishers/launch-kit"],"userAgents":[{"userAgent":"PerplexityBot","purpose":"Perplexity search indexing and citations","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexitybot.json","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Perplexity-User","purpose":"Perplexity user-requested page fetches","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexity-user.json"},{"userAgent":"OAI-SearchBot","purpose":"OpenAI search indexing and ChatGPT search discovery","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/searchbot.json","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"ChatGPT-User","purpose":"OpenAI user-requested page fetches","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/chatgpt-user.json"},{"userAgent":"GPTBot","purpose":"OpenAI public web crawling for foundation-model improvement","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/gptbot.json","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Googlebot","purpose":"Google Search and Google AI feature discovery through normal Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"GoogleOther","purpose":"Google public fetchers used by Google products, including AI assistant retrieval paths that identify as GoogleOther.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"},{"userAgent":"Google-InspectionTool","purpose":"Google inspection and rendering tools used to verify public page accessibility.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Gemini","purpose":"Gemini-style user-requested fetches when the crawler identifies with this token.","allow":["/"],"verification":"Gemini fetch identity may vary. Googlebot, GoogleOther, Google-Extended, and User-agent * are also allowed for public catalog pages."},{"userAgent":"Bingbot","purpose":"Microsoft Bing search discovery","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"BingPreview","purpose":"Microsoft/Bing preview and answer-surface retrieval.","allow":["/"]},{"userAgent":"ClaudeBot","purpose":"Anthropic public web crawling where allowed by robots.txt","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Claude-SearchBot","purpose":"Anthropic search and answer-discovery crawling","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Claude-User","purpose":"Anthropic user-requested page fetches","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"},{"userAgent":"Google-Extended","purpose":"Google AI model opt-in control distinct from ordinary Google Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"GrokBot","purpose":"xAI/Grok-style search and answer discovery where the crawler identifies with this token","allow":["/"],"officialDocs":"https://x.ai/","verification":"Identity and IP ranges should be monitored in access logs because public crawler documentation is less standardized than OpenAI, Perplexity, Google, or Anthropic."},{"userAgent":"Applebot","purpose":"Apple Search and assistant discovery where public crawling is requested.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"DuckAssistBot","purpose":"DuckDuckGo answer assistant discovery.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"YouBot","purpose":"You.com answer/search discovery.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"CCBot","purpose":"Common Crawl public web crawl used by many open-source search and model ecosystems.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"meta-externalagent","purpose":"Meta public web/AI retrieval where the crawler identifies with this token.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"OpenClaw","purpose":"Open-source browser agent/user-delegated automation. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"OpenClaw deployments may customize user agents. Agents should identify themselves and prefer /agents.txt, /openapi.json, and /api/mcp over UI scraping."},{"userAgent":"Hermes","purpose":"Nous/Hermes-style self-hosted agent and web-search extraction workflows. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"Hermes web extraction may use provider-specific browsers or search APIs. Agents should identify themselves and prefer bounded APIs over broad crawling."},{"userAgent":"Baiduspider","purpose":"Baidu search and AI answer indexing (China)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Sogou web spider","purpose":"Sogou search indexing (China, powers WeChat search)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Bytespider","purpose":"ByteDance crawling for Toutiao/Douyin search and Doubao AI","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"ByteDance-Doubao","purpose":"Doubao AI assistant retrieval (ByteDance)","allow":["/"]},{"userAgent":"PetalBot","purpose":"Huawei Petal search and assistant indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"YisouSpider","purpose":"Yisou/Shenma search indexing (Alibaba UC)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"360Spider","purpose":"Qihoo 360 search indexing (China)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"YandexBot","purpose":"Yandex search and AI answer indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Yeti","purpose":"Naver search indexing (Korea)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"HyperCLOVA","purpose":"Naver HyperCLOVA AI retrieval (Korea)","allow":["/"]},{"userAgent":"coccocbot-web","purpose":"Coc Coc search indexing (Vietnam)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Qwen","purpose":"Alibaba Qwen/Tongyi assistant retrieval","allow":["/"]},{"userAgent":"KimiBot","purpose":"Moonshot Kimi assistant retrieval (China)","allow":["/"]},{"userAgent":"GLM-Spider","purpose":"Zhipu GLM assistant retrieval (China)","allow":["/"]},{"userAgent":"DeepSeekBot","purpose":"DeepSeek assistant retrieval (China)","allow":["/"]},{"userAgent":"MistralAI-User","purpose":"Mistral user-requested page fetches","allow":["/"]},{"userAgent":"Amazonbot","purpose":"Amazon Alexa and Rufus answer indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Applebot-Extended","purpose":"Apple Intelligence training opt-in (inherits Applebot rules)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"cohere-ai","purpose":"Cohere user-requested retrieval","allow":["/"]},{"userAgent":"Timpibot","purpose":"Timpi decentralized search indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}],"rateGuidance":{"useCursors":true,"maxPublicBatchSize":1000,"preferredCacheSeconds":300,"contact":"Use an identifying User-Agent with a product or contact URL.","avoid":["Do not crawl hash routes as separate canonical pages.","Do not request unbounded /api/artifacts without summary=1, limit, and cursor.","Bulk indexing crawlers should not follow /api/downloads/ or /api/download-completions/; use hosted-file metadata and manifests instead.","Do not attempt token-gated admin or private endpoints as crawler surfaces."]},"containment":{"state":"unavailable","canonicalState":"unknown","verificationState":"unknown","executable":false,"downloadReady":false,"summary":"Canonical artifact completeness, verification, execution readiness, and download readiness are unavailable during containment."}}