{"schemaVersion":"hugging-bay.ai-bot-allowlist.v1","generatedAt":"2026-07-07T22:55:30.327Z","baseUrl":"https://huggingbay.xyz","status":"public-catalog-crawl-allowed","policySummary":"Hugging Bay intentionally allows public search engines, answer engines, user-requested AI fetchers, frontier AI assistants, and open-source agents to crawl public catalog, answer, ranking, topic, source, download, and machine-readable API surfaces. Authenticated, gated, private, admin, and publisher-token data is not exposed.","discovery":{"robots":"https://huggingbay.xyz/robots.txt","robotsJson":"https://huggingbay.xyz/robots.json","agentsTxt":"https://huggingbay.xyz/agents.txt","llmsTxt":"https://huggingbay.xyz/llms.txt","wellKnownLlmsTxt":"https://huggingbay.xyz/.well-known/llms.txt","llmsFullTxt":"https://huggingbay.xyz/llms-full.txt","agentDiscovery":"https://huggingbay.xyz/.well-known/agent-discovery.json","agentDiscoveryApi":"https://huggingbay.xyz/api/agent-discovery","answerEngine":"https://huggingbay.xyz/.well-known/answer-engine.json","aiSearch":"https://huggingbay.xyz/api/ai-search","aiSearchWellKnown":"https://huggingbay.xyz/.well-known/ai-search.json","aiCitations":"https://huggingbay.xyz/api/citations","aiCitationsWellKnown":"https://huggingbay.xyz/.well-known/ai-citations.json","activation":"https://huggingbay.xyz/api/activation","hostedLocalModels":"https://huggingbay.xyz/api/hosted-local-models?limit=100","communityReviewQueue":"https://huggingbay.xyz/api/community/review-queue?mode=needs-evidence&hosting=actionable&limit=25","communityReviewActions":"https://huggingbay.xyz/api/community/review-actions","publisherLaunchKit":"https://huggingbay.xyz/api/publishers/launch-kit","hfOverlayRelease":"https://huggingbay.xyz/browser-extension/release.json","hfOverlayInstall":"https://huggingbay.xyz/browser-extension/install.json","hfOverlayStoreKit":"https://huggingbay.xyz/browser-extension/store-kit.json","hfOverlayStoreZip":"https://huggingbay.xyz/browser-extension/hugging-bay-overlay-store.zip","hfOverlayZip":"https://huggingbay.xyz/browser-extension/hugging-bay-overlay.zip","crawlerPolicy":"https://huggingbay.xyz/.well-known/ai-crawler-policy.json","agentCatalog":"https://huggingbay.xyz/.well-known/ai-catalog.json","mcpManifest":"https://huggingbay.xyz/.well-known/mcp.json","remoteMcp":"https://huggingbay.xyz/api/mcp","remoteMcpAlias":"https://huggingbay.xyz/mcp","pythonSdk":"https://huggingbay.xyz/sdk/hugging_bay.py","pythonSdkRelease":"https://huggingbay.xyz/sdk/python/release.json","pythonSdkReadme":"https://huggingbay.xyz/sdk/python/README.md","openapi":"https://huggingbay.xyz/openapi.json","indexingStatus":"https://huggingbay.xyz/api/indexing/status","urlBatches":"https://huggingbay.xyz/api/indexing/urls?kind=agent-doc&limit=100"},"wafGuidance":{"allowPublicPaths":["/","/answers","/ai-search","/ai-citations","/ai-visibility","/artifact/","/rankings","/topics","/sources","/publishers","/publishers/launch-kit","/publishers/release-template","/downloads","/stacks","/radar","/trending","/community","/python-sdk","/sdk/","/feeds/","/data/","/robots.txt","/llms.txt","/llms-full.txt","/agent-discovery.json","/.well-known/","/openapi.json","/mcp","/browser-extension","/browser-extension/release.json","/browser-extension/install.json","/browser-extension/store-kit.json","/browser-extension/hugging-bay-overlay-store.zip","/browser-extension/hugging-bay-overlay.zip","/api/agent-discovery","/api/ai-search","/api/citations","/api/ai-visibility","/api/dumps/","/api/mcp","/api/answers","/api/indexing/","/api/publishers","/api/radar","/api/trending","/api/community/signals","/api/community/review-queue","/api/artifacts","/api/search/semantic","/api/badges/artifact/","/api/badges/publisher/","/api/badges/lab/","/badges/artifact/","/badges/publisher/","/badges/lab/"],"protectPrivatePaths":[],"recommendedVerification":["Allow by user-agent plus provider-published IP ranges where the provider publishes ranges.","Do not treat spoofed user-agent strings as trusted for authenticated endpoints.","Use robots.txt and this allowlist for crawl policy; use authentication for non-public data.","Crawler policy changes may take time to propagate through third-party systems."],"expectedHeaders":{"X-Robots-Tag":"index, follow, max-snippet:-1, max-image-preview:large","X-AI-Crawler-Policy":"allow-public-catalog-and-user-requested-ai-fetches"}},"providers":[{"id":"perplexity","name":"Perplexity","userAgents":["PerplexityBot","Perplexity-User"],"purpose":"Search-result citation, answer-engine retrieval, and user-requested page fetches.","allowed":true,"officialDocs":["https://docs.perplexity.ai/docs/resources/perplexity-crawlers"],"ipRangeUrls":["https://www.perplexity.com/perplexitybot.json","https://www.perplexity.com/perplexity-user.json"],"userAgentRules":[{"userAgent":"PerplexityBot","purpose":"Perplexity search indexing and citations","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexitybot.json"},{"userAgent":"Perplexity-User","purpose":"Perplexity user-requested page fetches","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexity-user.json"}]},{"id":"openai","name":"OpenAI and ChatGPT","userAgents":["OAI-SearchBot","ChatGPT-User","GPTBot"],"purpose":"ChatGPT search discovery, user-requested browsing, and public web crawling where allowed.","allowed":true,"officialDocs":["https://developers.openai.com/api/docs/bots"],"ipRangeUrls":["https://openai.com/searchbot.json","https://openai.com/chatgpt-user.json","https://openai.com/gptbot.json"],"userAgentRules":[{"userAgent":"OAI-SearchBot","purpose":"OpenAI search indexing and ChatGPT search discovery","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/searchbot.json"},{"userAgent":"ChatGPT-User","purpose":"OpenAI user-requested page fetches","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/chatgpt-user.json"},{"userAgent":"GPTBot","purpose":"OpenAI public web crawling for foundation-model improvement","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/gptbot.json"}]},{"id":"google","name":"Google Search and Gemini-related fetchers","userAgents":["Googlebot","GoogleOther","GoogleOther-Image","GoogleOther-Video","Google-InspectionTool","APIs-Google","Google-Extended","Gemini"],"purpose":"Google Search, inspection, assistant retrieval, and AI feature discovery paths that identify through Google crawler tokens.","allowed":true,"officialDocs":["https://developers.google.com/search/docs/appearance/ai-features","https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Googlebot","purpose":"Google Search and Google AI feature discovery through normal Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features"},{"userAgent":"GoogleOther","purpose":"Google public fetchers used by Google products, including AI assistant retrieval paths that identify as GoogleOther.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"},{"userAgent":"GoogleOther-Image","allow":["/"],"purpose":"Google Search, inspection, assistant retrieval, and AI feature discovery paths that identify through Google crawler tokens."},{"userAgent":"GoogleOther-Video","allow":["/"],"purpose":"Google Search, inspection, assistant retrieval, and AI feature discovery paths that identify through Google crawler tokens."},{"userAgent":"Google-InspectionTool","purpose":"Google inspection and rendering tools used to verify public page accessibility.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"},{"userAgent":"APIs-Google","allow":["/"],"purpose":"Google Search, inspection, assistant retrieval, and AI feature discovery paths that identify through Google crawler tokens."},{"userAgent":"Google-Extended","purpose":"Google AI model opt-in control distinct from ordinary Google Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features"},{"userAgent":"Gemini","purpose":"Gemini-style user-requested fetches when the crawler identifies with this token.","allow":["/"],"verification":"Gemini fetch identity may vary. Googlebot, GoogleOther, Google-Extended, and User-agent * are also allowed for public catalog pages."}]},{"id":"anthropic","name":"Anthropic and Claude","userAgents":["ClaudeBot","Claude-SearchBot","Claude-User"],"purpose":"Claude search/answer discovery, public crawling where allowed, and user-requested page fetches.","allowed":true,"officialDocs":["https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"ClaudeBot","purpose":"Anthropic public web crawling where allowed by robots.txt","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"},{"userAgent":"Claude-SearchBot","purpose":"Anthropic search and answer-discovery crawling","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"},{"userAgent":"Claude-User","purpose":"Anthropic user-requested page fetches","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"}]},{"id":"microsoft","name":"Microsoft and Bing","userAgents":["Bingbot","BingPreview"],"purpose":"Bing search discovery and preview or answer-surface retrieval.","allowed":true,"officialDocs":[],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"Bingbot","purpose":"Microsoft Bing search discovery","allow":["/"]},{"userAgent":"BingPreview","purpose":"Microsoft/Bing preview and answer-surface retrieval.","allow":["/"]}]},{"id":"open-ai-search","name":"Other AI and search crawlers","userAgents":["GrokBot","Applebot","DuckAssistBot","YouBot","CCBot","meta-externalagent","OpenClaw","Hermes"],"purpose":"Public search, answer, assistant, common-crawl, and open-source agent discovery where these tokens are used.","allowed":true,"officialDocs":["https://x.ai/"],"ipRangeUrls":[],"userAgentRules":[{"userAgent":"GrokBot","purpose":"xAI/Grok-style search and answer discovery where the crawler identifies with this token","allow":["/"],"officialDocs":"https://x.ai/","verification":"Identity and IP ranges should be monitored in access logs because public crawler documentation is less standardized than OpenAI, Perplexity, Google, or Anthropic."},{"userAgent":"Applebot","purpose":"Apple Search and assistant discovery where public crawling is requested.","allow":["/"]},{"userAgent":"DuckAssistBot","purpose":"DuckDuckGo answer assistant discovery.","allow":["/"]},{"userAgent":"YouBot","purpose":"You.com answer/search discovery.","allow":["/"]},{"userAgent":"CCBot","purpose":"Common Crawl public web crawl used by many open-source search and model ecosystems.","allow":["/"]},{"userAgent":"meta-externalagent","purpose":"Meta public web/AI retrieval where the crawler identifies with this token.","allow":["/"]},{"userAgent":"OpenClaw","purpose":"Open-source browser agent/user-delegated automation. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"OpenClaw deployments may customize user agents. Agents should identify themselves and prefer /agents.txt, /openapi.json, and /api/mcp over UI scraping."},{"userAgent":"Hermes","purpose":"Nous/Hermes-style self-hosted agent and web-search extraction workflows. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"Hermes web extraction may use provider-specific browsers or search APIs. Agents should identify themselves and prefer bounded APIs over broad crawling."}]}],"preferredCrawlEntrypoints":["https://huggingbay.xyz/robots.txt","https://huggingbay.xyz/robots.json","https://huggingbay.xyz/.well-known/agent-discovery.json","https://huggingbay.xyz/api/agent-discovery","https://huggingbay.xyz/agents.txt","https://huggingbay.xyz/llms.txt","https://huggingbay.xyz/.well-known/llms.txt","https://huggingbay.xyz/sitemap.xml","https://huggingbay.xyz/api/ai-search","https://huggingbay.xyz/.well-known/ai-search.json","https://huggingbay.xyz/api/citations","https://huggingbay.xyz/.well-known/ai-citations.json","https://huggingbay.xyz/api/ai-visibility","https://huggingbay.xyz/api/community/review-queue?mode=needs-evidence&hosting=actionable&limit=25","https://huggingbay.xyz/publishers/launch-kit","https://huggingbay.xyz/api/publishers/launch-kit","https://huggingbay.xyz/api/indexing/status","https://huggingbay.xyz/api/answers","https://huggingbay.xyz/data/manifest.json","https://huggingbay.xyz/api/dumps/catalog.jsonl?limit=1000","https://huggingbay.xyz/api/dumps/hosted-files.jsonl?limit=1000","https://huggingbay.xyz/feeds/recent.xml","https://huggingbay.xyz/feeds/trending.xml","https://huggingbay.xyz/feeds/hosted-mirrors.xml","https://huggingbay.xyz/feeds/reviews.xml","https://huggingbay.xyz/api/indexing/urls?kind=answer&limit=100","https://huggingbay.xyz/api/indexing/urls?kind=agent-doc&limit=100","https://huggingbay.xyz/api/indexing/urls?kind=artifact&limit=1000&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=artifact-section&limit=1000&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=artifact-evidence&limit=1000&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=publisher&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=downloadable&limit=1000&cursor=0","https://huggingbay.xyz/api/mcp","https://huggingbay.xyz/mcp","https://huggingbay.xyz/browser-extension","https://huggingbay.xyz/browser-extension/manifest.json","https://huggingbay.xyz/browser-extension/store-kit.json","https://huggingbay.xyz/browser-extension/hugging-bay-overlay-store.zip","https://huggingbay.xyz/browser-extension/hugging-bay-overlay.zip","https://huggingbay.xyz/api/radar","https://huggingbay.xyz/api/trending?window=7d&limit=25","https://huggingbay.xyz/api/mirror-demand?requestedOnly=1&limit=100","https://huggingbay.xyz/api/mirror-readiness?limit=100"],"preferredCitationTargets":["https://huggingbay.xyz/ai-search","https://huggingbay.xyz/ai-citations","https://huggingbay.xyz/answers","https://huggingbay.xyz/answers/{slug}","https://huggingbay.xyz/artifact/{artifactId}","https://huggingbay.xyz/rankings/top-open-models","https://huggingbay.xyz/rankings/top-open-source-llms","https://huggingbay.xyz/rankings/top-seeded-models","https://huggingbay.xyz/rankings/hosted-run-ready","https://huggingbay.xyz/rankings/hosted-small-local-model-files","https://huggingbay.xyz/rankings/community-verified","https://huggingbay.xyz/rankings/needs-first-review","https://huggingbay.xyz/rankings/new-open-weight-models","https://huggingbay.xyz/rankings/coding-models","https://huggingbay.xyz/rankings/consumer-gpu-models","https://huggingbay.xyz/rankings/preservation-watchlist","https://huggingbay.xyz/rankings/best-7b-8b-models","https://huggingbay.xyz/rankings/best-14b-models","https://huggingbay.xyz/rankings/best-32b-models","https://huggingbay.xyz/rankings/best-70b-models","https://huggingbay.xyz/rankings/global-model-labs","https://huggingbay.xyz/rankings/eastern-open-models","https://huggingbay.xyz/downloads","https://huggingbay.xyz/sources","https://huggingbay.xyz/stacks","https://huggingbay.xyz/publishers/launch-kit"],"noBypassPolicy":{"noGatedPrivateBypass":true,"hostedDownloads":"Only files listed by hosted manifests, /api/downloadable-files, or /downloads pages are downloadable from Hugging Bay.","externalRows":"External metadata rows stay upstream until a reviewable public artifact is approved for mirroring."}}