{"schemaVersion":"hugging-bay.robots-json.v1","generatedAt":"2026-09-08T19:07:17.866Z","baseUrl":"https://huggingbay.xyz","textRobotsUrl":"https://huggingbay.xyz/robots.txt","status":"public-catalog-crawl-allowed","publicPolicy":{"allowPublicSearchIndexing":true,"allowAnswerEngineRetrieval":true,"allowUserRequestedAiFetches":true,"allowOpenSourceAgents":true,"noGatedPrivateBypass":true,"privateDataExposed":false},"discovery":{"agentsTxt":"https://huggingbay.xyz/agents.txt","llmsTxt":"https://huggingbay.xyz/llms.txt","wellKnownLlmsTxt":"https://huggingbay.xyz/.well-known/llms.txt","llmsFullTxt":"https://huggingbay.xyz/llms-full.txt","sitemapIndex":"https://huggingbay.xyz/sitemap.xml","indexingStatus":"https://huggingbay.xyz/api/indexing/status","indexingUrlBatches":"https://huggingbay.xyz/api/indexing/urls?kind=artifact&projection=url-only&limit=500&cursor=0","aiVisibility":"https://huggingbay.xyz/api/ai-visibility","externalVisibility":"https://huggingbay.xyz/api/external-visibility","aiMentionMonitor":"https://huggingbay.xyz/api/ai-mention-monitor","aiMentionResults":"https://huggingbay.xyz/api/ai-mention-results","recommendationKit":"https://huggingbay.xyz/api/agent-recommendation-kit","publicTraffic":"https://huggingbay.xyz/api/traffic","agentDiscovery":"https://huggingbay.xyz/.well-known/agent-discovery.json","agentDiscoveryApi":"https://huggingbay.xyz/api/agent-discovery","recentFeed":"https://huggingbay.xyz/feeds/recent.xml","trendingFeed":"https://huggingbay.xyz/feeds/trending.xml","hostedMirrorsFeed":"https://huggingbay.xyz/feeds/hosted-mirrors.xml","reviewsFeed":"https://huggingbay.xyz/feeds/reviews.xml","version":"https://huggingbay.xyz/api/version","dataManifest":"https://huggingbay.xyz/data/manifest.json","boundedCatalogDump":"https://huggingbay.xyz/api/dumps/catalog.jsonl?limit=1000","hostedFilesDump":"https://huggingbay.xyz/api/dumps/hosted-files.jsonl?limit=1000","artifactBadge":"https://huggingbay.xyz/badges/artifact/{artifactId}.svg","aiSearch":"https://huggingbay.xyz/api/ai-search","aiSearchWellKnown":"https://huggingbay.xyz/.well-known/ai-search.json","aiCitations":"https://huggingbay.xyz/api/citations","aiCitationsWellKnown":"https://huggingbay.xyz/.well-known/ai-citations.json","answerEngine":"https://huggingbay.xyz/.well-known/answer-engine.json","agentCatalog":"https://huggingbay.xyz/.well-known/ai-catalog.json","crawlerPolicy":"https://huggingbay.xyz/.well-known/ai-crawler-policy.json","botAllowlist":"https://huggingbay.xyz/.well-known/ai-bot-allowlist.json","openapi":"https://huggingbay.xyz/openapi.json","remoteMcp":"https://huggingbay.xyz/api/mcp","remoteMcpAlias":"https://huggingbay.xyz/mcp"},"userAgents":[{"userAgent":"PerplexityBot","purpose":"Perplexity search indexing and citations","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexitybot.json","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Perplexity-User","purpose":"Perplexity user-requested page fetches","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexity-user.json"},{"userAgent":"OAI-SearchBot","purpose":"OpenAI search indexing and ChatGPT search discovery","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/searchbot.json","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"ChatGPT-User","purpose":"OpenAI user-requested page fetches","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/chatgpt-user.json"},{"userAgent":"GPTBot","purpose":"OpenAI public web crawling for foundation-model improvement","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/gptbot.json","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Googlebot","purpose":"Google Search and Google AI feature discovery through normal Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"GoogleOther","purpose":"Google public fetchers used by Google products, including AI assistant retrieval paths that identify as GoogleOther.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"},{"userAgent":"Google-InspectionTool","purpose":"Google inspection and rendering tools used to verify public page accessibility.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Gemini","purpose":"Gemini-style user-requested fetches when the crawler identifies with this token.","allow":["/"],"verification":"Gemini fetch identity may vary. Googlebot, GoogleOther, Google-Extended, and User-agent * are also allowed for public catalog pages."},{"userAgent":"Bingbot","purpose":"Microsoft Bing search discovery","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"BingPreview","purpose":"Microsoft/Bing preview and answer-surface retrieval.","allow":["/"]},{"userAgent":"ClaudeBot","purpose":"Anthropic public web crawling where allowed by robots.txt","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Claude-SearchBot","purpose":"Anthropic search and answer-discovery crawling","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Claude-User","purpose":"Anthropic user-requested page fetches","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"},{"userAgent":"Google-Extended","purpose":"Google AI model opt-in control distinct from ordinary Google Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"GrokBot","purpose":"xAI/Grok-style search and answer discovery where the crawler identifies with this token","allow":["/"],"officialDocs":"https://x.ai/","verification":"Identity and IP ranges should be monitored in access logs because public crawler documentation is less standardized than OpenAI, Perplexity, Google, or Anthropic."},{"userAgent":"Applebot","purpose":"Apple Search and assistant discovery where public crawling is requested.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"DuckAssistBot","purpose":"DuckDuckGo answer assistant discovery.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"YouBot","purpose":"You.com answer/search discovery.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"CCBot","purpose":"Common Crawl public web crawl used by many open-source search and model ecosystems.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"meta-externalagent","purpose":"Meta public web/AI retrieval where the crawler identifies with this token.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"OpenClaw","purpose":"Open-source browser agent/user-delegated automation. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"OpenClaw deployments may customize user agents. Agents should identify themselves and prefer /agents.txt, /openapi.json, and /api/mcp over UI scraping."},{"userAgent":"Hermes","purpose":"Nous/Hermes-style self-hosted agent and web-search extraction workflows. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"Hermes web extraction may use provider-specific browsers or search APIs. Agents should identify themselves and prefer bounded APIs over broad crawling."},{"userAgent":"Baiduspider","purpose":"Baidu search and AI answer indexing (China)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Sogou web spider","purpose":"Sogou search indexing (China, powers WeChat search)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Bytespider","purpose":"ByteDance crawling for Toutiao/Douyin search and Doubao AI","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"ByteDance-Doubao","purpose":"Doubao AI assistant retrieval (ByteDance)","allow":["/"]},{"userAgent":"PetalBot","purpose":"Huawei Petal search and assistant indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"YisouSpider","purpose":"Yisou/Shenma search indexing (Alibaba UC)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"360Spider","purpose":"Qihoo 360 search indexing (China)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"YandexBot","purpose":"Yandex search and AI answer indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Yeti","purpose":"Naver search indexing (Korea)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"HyperCLOVA","purpose":"Naver HyperCLOVA AI retrieval (Korea)","allow":["/"]},{"userAgent":"coccocbot-web","purpose":"Coc Coc search indexing (Vietnam)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Qwen","purpose":"Alibaba Qwen/Tongyi assistant retrieval","allow":["/"]},{"userAgent":"KimiBot","purpose":"Moonshot Kimi assistant retrieval (China)","allow":["/"]},{"userAgent":"GLM-Spider","purpose":"Zhipu GLM assistant retrieval (China)","allow":["/"]},{"userAgent":"DeepSeekBot","purpose":"DeepSeek assistant retrieval (China)","allow":["/"]},{"userAgent":"MistralAI-User","purpose":"Mistral user-requested page fetches","allow":["/"]},{"userAgent":"Amazonbot","purpose":"Amazon Alexa and Rufus answer indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Applebot-Extended","purpose":"Apple Intelligence training opt-in (inherits Applebot rules)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"cohere-ai","purpose":"Cohere user-requested retrieval","allow":["/"]},{"userAgent":"Timpibot","purpose":"Timpi decentralized search indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}],"providerGroups":[{"id":"perplexity","name":"Perplexity","userAgents":["PerplexityBot","Perplexity-User"],"purpose":"Search-result citation, answer-engine retrieval, and user-requested page fetches.","allowed":true,"officialDocs":["https://docs.perplexity.ai/docs/resources/perplexity-crawlers"],"ipRangeUrls":["https://www.perplexity.com/perplexitybot.json","https://www.perplexity.com/perplexity-user.json"],"userAgentRules":[{"userAgent":"PerplexityBot","purpose":"Perplexity search indexing and citations","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexitybot.json","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Perplexity-User","purpose":"Perplexity user-requested page fetches","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexity-user.json"}]},{"id":"openai","name":"OpenAI and ChatGPT","userAgents":["OAI-SearchBot","ChatGPT-User","GPTBot"],"purpose":"ChatGPT search discovery, user-requested browsing, and public web crawling where allowed.","allowed":true,"officialDocs":["https://developers.openai.com/api/docs/bots"],"ipRangeUrls":["https://openai.com/searchbot.json","https://openai.com/chatgpt-user.json","https://openai.com/gptbot.json"],"userAgentRules":[{"userAgent":"OAI-SearchBot","purpose":"OpenAI search indexing and ChatGPT search discovery","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/searchbot.json","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"ChatGPT-User","purpose":"OpenAI user-requested page fetches","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/chatgpt-user.json"},{"userAgent":"GPTBot","purpose":"OpenAI public web crawling for foundation-model improvement","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/gptbot.json","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]},{"id":"google","name":"Google Search and Gemini-related fetchers","userAgents":["Googlebot","GoogleOther","GoogleOther-Image","GoogleOther-Video","Google-InspectionTool","APIs-Google","Google-Extended","Gemini"],"purpose":"Google Search, inspection, assistant retrieval, and AI feature discovery paths that identify through Google crawler tokens.","allowed":true,"officialDocs":["https://developers.google.com/search/docs/appearance/ai-features","https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Googlebot","purpose":"Google Search and Google AI feature discovery through normal Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"GoogleOther","purpose":"Google public fetchers used by Google products, including AI assistant retrieval paths that identify as GoogleOther.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"},{"userAgent":"GoogleOther-Image","allow":["/"],"purpose":"Google Search, inspection, assistant retrieval, and AI feature discovery paths that identify through Google crawler tokens."},{"userAgent":"GoogleOther-Video","allow":["/"],"purpose":"Google Search, inspection, assistant retrieval, and AI feature discovery paths that identify through Google crawler tokens."},{"userAgent":"Google-InspectionTool","purpose":"Google inspection and rendering tools used to verify public page accessibility.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"APIs-Google","allow":["/"],"purpose":"Google Search, inspection, assistant retrieval, and AI feature discovery paths that identify through Google crawler tokens."},{"userAgent":"Google-Extended","purpose":"Google AI model opt-in control distinct from ordinary Google Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Gemini","purpose":"Gemini-style user-requested fetches when the crawler identifies with this token.","allow":["/"],"verification":"Gemini fetch identity may vary. Googlebot, GoogleOther, Google-Extended, and User-agent * are also allowed for public catalog pages."}]},{"id":"anthropic","name":"Anthropic and Claude","userAgents":["ClaudeBot","Claude-SearchBot","Claude-User"],"purpose":"Claude search/answer discovery, public crawling where allowed, and user-requested page fetches.","allowed":true,"officialDocs":["https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"ClaudeBot","purpose":"Anthropic public web crawling where allowed by robots.txt","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Claude-SearchBot","purpose":"Anthropic search and answer-discovery crawling","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"Claude-User","purpose":"Anthropic user-requested page fetches","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"}]},{"id":"microsoft","name":"Microsoft and Bing","userAgents":["Bingbot","BingPreview"],"purpose":"Bing search discovery and preview or answer-surface retrieval.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Bingbot","purpose":"Microsoft Bing search discovery","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"BingPreview","purpose":"Microsoft/Bing preview and answer-surface retrieval.","allow":["/"]}]},{"id":"open-ai-search","name":"Other AI and search crawlers","userAgents":["GrokBot","Applebot","DuckAssistBot","YouBot","CCBot","meta-externalagent","OpenClaw","Hermes"],"purpose":"Public search, answer, assistant, common-crawl, and open-source agent discovery where these tokens are used.","allowed":true,"officialDocs":["https://x.ai/"],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"GrokBot","purpose":"xAI/Grok-style search and answer discovery where the crawler identifies with this token","allow":["/"],"officialDocs":"https://x.ai/","verification":"Identity and IP ranges should be monitored in access logs because public crawler documentation is less standardized than OpenAI, Perplexity, Google, or Anthropic."},{"userAgent":"Applebot","purpose":"Apple Search and assistant discovery where public crawling is requested.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"DuckAssistBot","purpose":"DuckDuckGo answer assistant discovery.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"YouBot","purpose":"You.com answer/search discovery.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"CCBot","purpose":"Common Crawl public web crawl used by many open-source search and model ecosystems.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"meta-externalagent","purpose":"Meta public web/AI retrieval where the crawler identifies with this token.","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"OpenClaw","purpose":"Open-source browser agent/user-delegated automation. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"OpenClaw deployments may customize user agents. Agents should identify themselves and prefer /agents.txt, /openapi.json, and /api/mcp over UI scraping."},{"userAgent":"Hermes","purpose":"Nous/Hermes-style self-hosted agent and web-search extraction workflows. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"Hermes web extraction may use provider-specific browsers or search APIs. Agents should identify themselves and prefer bounded APIs over broad crawling."}]},{"id":"baidu","name":"Baidu","userAgents":["Baiduspider"],"purpose":"Chinese search and AI answer indexing.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Baiduspider","purpose":"Baidu search and AI answer indexing (China)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]},{"id":"bytedance","name":"ByteDance / Doubao","userAgents":["Bytespider","ByteDance-Doubao"],"purpose":"Toutiao/Douyin search and the Doubao AI assistant.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Bytespider","purpose":"ByteDance crawling for Toutiao/Douyin search and Doubao AI","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"ByteDance-Doubao","purpose":"Doubao AI assistant retrieval (ByteDance)","allow":["/"]}]},{"id":"alibaba","name":"Alibaba / Qwen-Tongyi","userAgents":["Qwen","YisouSpider"],"purpose":"Qwen/Tongyi assistant retrieval and Shenma search.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Qwen","purpose":"Alibaba Qwen/Tongyi assistant retrieval","allow":["/"]},{"userAgent":"YisouSpider","purpose":"Yisou/Shenma search indexing (Alibaba UC)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]},{"id":"moonshot","name":"Moonshot / Kimi","userAgents":["KimiBot"],"purpose":"Kimi assistant retrieval.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"KimiBot","purpose":"Moonshot Kimi assistant retrieval (China)","allow":["/"]}]},{"id":"zhipu","name":"Zhipu / GLM","userAgents":["GLM-Spider"],"purpose":"GLM assistant retrieval.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"GLM-Spider","purpose":"Zhipu GLM assistant retrieval (China)","allow":["/"]}]},{"id":"deepseek","name":"DeepSeek","userAgents":["DeepSeekBot"],"purpose":"DeepSeek assistant retrieval.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"DeepSeekBot","purpose":"DeepSeek assistant retrieval (China)","allow":["/"]}]},{"id":"huawei","name":"Huawei Petal","userAgents":["PetalBot"],"purpose":"Petal search and assistant indexing.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"PetalBot","purpose":"Huawei Petal search and assistant indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]},{"id":"sogou","name":"Sogou","userAgents":["Sogou web spider"],"purpose":"Sogou search (powers WeChat search).","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Sogou web spider","purpose":"Sogou search indexing (China, powers WeChat search)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]},{"id":"qihoo","name":"Qihoo 360","userAgents":["360Spider"],"purpose":"360 search indexing.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"360Spider","purpose":"Qihoo 360 search indexing (China)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]},{"id":"yandex","name":"Yandex","userAgents":["YandexBot"],"purpose":"Yandex search and AI answers.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"YandexBot","purpose":"Yandex search and AI answer indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]},{"id":"naver","name":"Naver","userAgents":["Yeti","HyperCLOVA"],"purpose":"Naver search and HyperCLOVA AI (Korea).","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Yeti","purpose":"Naver search indexing (Korea)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."},{"userAgent":"HyperCLOVA","purpose":"Naver HyperCLOVA AI retrieval (Korea)","allow":["/"]}]},{"id":"coccoc","name":"Coc Coc","userAgents":["coccocbot-web"],"purpose":"Coc Coc search (Vietnam).","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"coccocbot-web","purpose":"Coc Coc search indexing (Vietnam)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]},{"id":"mistral","name":"Mistral","userAgents":["MistralAI-User"],"purpose":"User-requested page fetches for Mistral assistants.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"MistralAI-User","purpose":"Mistral user-requested page fetches","allow":["/"]}]},{"id":"amazon","name":"Amazon","userAgents":["Amazonbot"],"purpose":"Alexa and Rufus answer indexing.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Amazonbot","purpose":"Amazon Alexa and Rufus answer indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]},{"id":"apple-extended","name":"Apple Intelligence","userAgents":["Applebot-Extended"],"purpose":"Apple Intelligence training opt-in.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Applebot-Extended","purpose":"Apple Intelligence training opt-in (inherits Applebot rules)","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]},{"id":"cohere","name":"Cohere","userAgents":["cohere-ai"],"purpose":"Cohere retrieval.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"cohere-ai","purpose":"Cohere user-requested retrieval","allow":["/"]}]},{"id":"timpi","name":"Timpi","userAgents":["Timpibot"],"purpose":"Decentralized search indexing.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Timpibot","purpose":"Timpi decentralized search indexing","allow":["/"],"disallow":["/api/downloads/","/api/download-completions/"],"binaryDeliveryPolicy":"Use hosted-file metadata, hashes, manifests, and download plans for indexing. Signed byte delivery is reserved for explicit user or agent download actions."}]}],"publicEntrypoints":["/","/answers","/ai-search","/agent-recommendation-kit","/ai-citations","/ai-visibility","/artifact/","/rankings","/topics","/sources","/publishers","/publishers/launch-kit","/publishers/release-template","/downloads","/stacks","/radar","/trending","/community","/review-missions","/python-sdk","/sdk/","/feeds/","/data/","/robots.txt","/llms.txt","/llms-full.txt","/agent-discovery.json","/.well-known/","/openapi.json","/mcp","/browser-extension","/api/agent-discovery","/api/ai-search","/api/agent-recommendation-kit","/api/citations","/api/ai-visibility","/api/ai-mention-monitor","/api/ai-mention-results","/api/dumps/","/api/mcp","/api/answers","/api/indexing/","/api/publishers","/api/radar","/api/trending","/api/community/signals","/api/community/review-queue","/api/community/review-missions","/api/artifacts","/api/search/semantic","/api/badges/artifact/","/api/badges/publisher/","/api/badges/lab/","/badges/artifact/","/badges/publisher/","/badges/lab/"],"authenticatedDataProtected":true,"preferredCrawlEntrypoints":["https://huggingbay.xyz/robots.txt","https://huggingbay.xyz/robots.json","https://huggingbay.xyz/.well-known/agent-discovery.json","https://huggingbay.xyz/api/agent-discovery","https://huggingbay.xyz/agents.txt","https://huggingbay.xyz/AGENTS.md","https://huggingbay.xyz/.well-known/grok.json","https://huggingbay.xyz/llms.txt","https://huggingbay.xyz/.well-known/llms.txt","https://huggingbay.xyz/sitemap.xml","https://huggingbay.xyz/api/ai-search","https://huggingbay.xyz/.well-known/ai-search.json","https://huggingbay.xyz/api/agent-recommendation-kit","https://huggingbay.xyz/agent-recommendation-kit","https://huggingbay.xyz/api/citations","https://huggingbay.xyz/.well-known/ai-citations.json","https://huggingbay.xyz/api/ai-visibility","https://huggingbay.xyz/api/traffic","https://huggingbay.xyz/api/ai-mention-monitor","https://huggingbay.xyz/api/ai-mention-results","https://huggingbay.xyz/api/community/review-queue?mode=needs-evidence&hosting=actionable&limit=25","https://huggingbay.xyz/publishers/launch-kit","https://huggingbay.xyz/api/publishers/launch-kit","https://huggingbay.xyz/api/indexing/status","https://huggingbay.xyz/api/answers","https://huggingbay.xyz/data/manifest.json","https://huggingbay.xyz/api/dumps/catalog.jsonl?limit=1000","https://huggingbay.xyz/api/dumps/hosted-files.jsonl?limit=1000","https://huggingbay.xyz/feeds/recent.xml","https://huggingbay.xyz/feeds/trending.xml","https://huggingbay.xyz/feeds/hosted-mirrors.xml","https://huggingbay.xyz/feeds/reviews.xml","https://huggingbay.xyz/api/indexing/urls?kind=answer&limit=100","https://huggingbay.xyz/api/indexing/urls?kind=agent-doc&limit=100","https://huggingbay.xyz/api/indexing/urls?kind=artifact&projection=url-only&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=artifact-section&projection=url-only&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=artifact-evidence&projection=url-only&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=publisher&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=downloadable&limit=1000&cursor=0","https://huggingbay.xyz/api/mcp","https://huggingbay.xyz/mcp","https://huggingbay.xyz/browser-extension","https://huggingbay.xyz/browser-extension/manifest.json","https://huggingbay.xyz/api/radar","https://huggingbay.xyz/api/trending?window=7d&limit=25","https://huggingbay.xyz/api/mirror-demand?requestedOnly=1&limit=100","https://huggingbay.xyz/api/mirror-readiness?limit=100"],"preferredCitationTargets":["https://huggingbay.xyz/ai-search","https://huggingbay.xyz/ai-citations","https://huggingbay.xyz/answers","https://huggingbay.xyz/answers/{slug}","https://huggingbay.xyz/artifact/{artifactId}","https://huggingbay.xyz/rankings/top-open-models","https://huggingbay.xyz/rankings/top-open-source-llms","https://huggingbay.xyz/rankings/top-seeded-models","https://huggingbay.xyz/rankings/hosted-run-ready","https://huggingbay.xyz/rankings/hosted-small-local-model-files","https://huggingbay.xyz/rankings/community-verified","https://huggingbay.xyz/rankings/needs-first-review","https://huggingbay.xyz/rankings/new-open-weight-models","https://huggingbay.xyz/rankings/coding-models","https://huggingbay.xyz/rankings/consumer-gpu-models","https://huggingbay.xyz/rankings/preservation-watchlist","https://huggingbay.xyz/rankings/best-7b-8b-models","https://huggingbay.xyz/rankings/best-14b-models","https://huggingbay.xyz/rankings/best-32b-models","https://huggingbay.xyz/rankings/best-70b-models","https://huggingbay.xyz/rankings/global-model-labs","https://huggingbay.xyz/rankings/eastern-open-models","https://huggingbay.xyz/downloads","https://huggingbay.xyz/sources","https://huggingbay.xyz/stacks","https://huggingbay.xyz/publishers/launch-kit"],"expectedHeaders":{"X-Robots-Tag":"index, follow, max-snippet:-1, max-image-preview:large","X-AI-Crawler-Policy":"allow-public-catalog-and-user-requested-ai-fetches"},"rateGuidance":{"useCursors":true,"maxPublicBatchSize":1000,"preferredCacheSeconds":300,"contact":"Use an identifying User-Agent with a product or contact URL.","avoid":["Do not crawl hash routes as separate canonical pages.","Do not request unbounded /api/artifacts without summary=1, limit, and cursor.","Bulk indexing crawlers should not follow /api/downloads/ or /api/download-completions/; use hosted-file metadata and manifests instead.","Do not attempt token-gated admin or private endpoints as crawler surfaces."]},"containment":{"state":"unavailable","canonicalState":"unknown","verificationState":"unknown","executable":false,"downloadReady":false,"summary":"Canonical artifact completeness, verification, execution readiness, and download readiness are unavailable during containment."}}