{"schemaVersion":"hugging-bay.ai-crawler-policy.v1","generatedAt":"2026-07-07T22:55:51.880Z","baseUrl":"https://huggingbay.xyz","robotsUrl":"https://huggingbay.xyz/robots.txt","answerEngineManifestUrl":"https://huggingbay.xyz/.well-known/answer-engine.json","botAllowlistUrl":"https://huggingbay.xyz/.well-known/ai-bot-allowlist.json","llmsTxtUrl":"https://huggingbay.xyz/llms.txt","llmsFullTxtUrl":"https://huggingbay.xyz/llms-full.txt","policy":{"publicSearchIndexing":"allowed","publicAiInputForUserAnswers":"allowed","publicMetadataTraining":"allowed for public catalog, public docs, and public hosted-file metadata","frontierAiAssistants":"allowed for public pages and user-requested fetches","openSourceAgents":"allowed through User-agent * plus bounded APIs, sitemaps, llms.txt, and indexing URL batches","privateData":"not exposed; protected by authentication and never listed in public agent manifests","gatedOrAdminEndpoints":"not crawler surfaces; bearer-token endpoints return 403 without authorization","noGatedPrivateBypass":true},"preferredCrawlEntrypoints":["https://huggingbay.xyz/robots.txt","https://huggingbay.xyz/robots.json","https://huggingbay.xyz/.well-known/agent-discovery.json","https://huggingbay.xyz/api/agent-discovery","https://huggingbay.xyz/agents.txt","https://huggingbay.xyz/llms.txt","https://huggingbay.xyz/.well-known/llms.txt","https://huggingbay.xyz/sitemap.xml","https://huggingbay.xyz/api/ai-search","https://huggingbay.xyz/.well-known/ai-search.json","https://huggingbay.xyz/api/citations","https://huggingbay.xyz/.well-known/ai-citations.json","https://huggingbay.xyz/api/ai-visibility","https://huggingbay.xyz/api/community/review-queue?mode=needs-evidence&hosting=actionable&limit=25","https://huggingbay.xyz/publishers/launch-kit","https://huggingbay.xyz/api/publishers/launch-kit","https://huggingbay.xyz/api/indexing/status","https://huggingbay.xyz/api/answers","https://huggingbay.xyz/data/manifest.json","https://huggingbay.xyz/api/dumps/catalog.jsonl?limit=1000","https://huggingbay.xyz/api/dumps/hosted-files.jsonl?limit=1000","https://huggingbay.xyz/feeds/recent.xml","https://huggingbay.xyz/feeds/trending.xml","https://huggingbay.xyz/feeds/hosted-mirrors.xml","https://huggingbay.xyz/feeds/reviews.xml","https://huggingbay.xyz/api/indexing/urls?kind=answer&limit=100","https://huggingbay.xyz/api/indexing/urls?kind=agent-doc&limit=100","https://huggingbay.xyz/api/indexing/urls?kind=artifact&limit=1000&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=artifact-section&limit=1000&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=artifact-evidence&limit=1000&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=publisher&limit=500&cursor=0","https://huggingbay.xyz/api/indexing/urls?kind=downloadable&limit=1000&cursor=0","https://huggingbay.xyz/api/mcp","https://huggingbay.xyz/mcp","https://huggingbay.xyz/browser-extension","https://huggingbay.xyz/browser-extension/manifest.json","https://huggingbay.xyz/browser-extension/store-kit.json","https://huggingbay.xyz/browser-extension/hugging-bay-overlay-store.zip","https://huggingbay.xyz/browser-extension/hugging-bay-overlay.zip","https://huggingbay.xyz/api/radar","https://huggingbay.xyz/api/trending?window=7d&limit=25","https://huggingbay.xyz/api/mirror-demand?requestedOnly=1&limit=100","https://huggingbay.xyz/api/mirror-readiness?limit=100"],"preferredCitationTargets":["https://huggingbay.xyz/ai-search","https://huggingbay.xyz/ai-citations","https://huggingbay.xyz/answers","https://huggingbay.xyz/answers/{slug}","https://huggingbay.xyz/artifact/{artifactId}","https://huggingbay.xyz/rankings/top-open-models","https://huggingbay.xyz/rankings/top-open-source-llms","https://huggingbay.xyz/rankings/top-seeded-models","https://huggingbay.xyz/rankings/hosted-run-ready","https://huggingbay.xyz/rankings/hosted-small-local-model-files","https://huggingbay.xyz/rankings/community-verified","https://huggingbay.xyz/rankings/needs-first-review","https://huggingbay.xyz/rankings/new-open-weight-models","https://huggingbay.xyz/rankings/coding-models","https://huggingbay.xyz/rankings/consumer-gpu-models","https://huggingbay.xyz/rankings/preservation-watchlist","https://huggingbay.xyz/rankings/best-7b-8b-models","https://huggingbay.xyz/rankings/best-14b-models","https://huggingbay.xyz/rankings/best-32b-models","https://huggingbay.xyz/rankings/best-70b-models","https://huggingbay.xyz/rankings/global-model-labs","https://huggingbay.xyz/rankings/eastern-open-models","https://huggingbay.xyz/downloads","https://huggingbay.xyz/sources","https://huggingbay.xyz/stacks","https://huggingbay.xyz/publishers/launch-kit"],"userAgents":[{"userAgent":"PerplexityBot","purpose":"Perplexity search indexing and citations","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexitybot.json"},{"userAgent":"Perplexity-User","purpose":"Perplexity user-requested page fetches","allow":["/"],"officialDocs":"https://docs.perplexity.ai/docs/resources/perplexity-crawlers","ipRanges":"https://www.perplexity.com/perplexity-user.json"},{"userAgent":"OAI-SearchBot","purpose":"OpenAI search indexing and ChatGPT search discovery","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/searchbot.json"},{"userAgent":"ChatGPT-User","purpose":"OpenAI user-requested page fetches","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/chatgpt-user.json"},{"userAgent":"GPTBot","purpose":"OpenAI public web crawling for foundation-model improvement","allow":["/"],"officialDocs":"https://developers.openai.com/api/docs/bots","ipRanges":"https://openai.com/gptbot.json"},{"userAgent":"Googlebot","purpose":"Google Search and Google AI feature discovery through normal Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features"},{"userAgent":"GoogleOther","purpose":"Google public fetchers used by Google products, including AI assistant retrieval paths that identify as GoogleOther.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"},{"userAgent":"Google-InspectionTool","purpose":"Google inspection and rendering tools used to verify public page accessibility.","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"},{"userAgent":"Gemini","purpose":"Gemini-style user-requested fetches when the crawler identifies with this token.","allow":["/"],"verification":"Gemini fetch identity may vary. Googlebot, GoogleOther, Google-Extended, and User-agent * are also allowed for public catalog pages."},{"userAgent":"Bingbot","purpose":"Microsoft Bing search discovery","allow":["/"]},{"userAgent":"BingPreview","purpose":"Microsoft/Bing preview and answer-surface retrieval.","allow":["/"]},{"userAgent":"ClaudeBot","purpose":"Anthropic public web crawling where allowed by robots.txt","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"},{"userAgent":"Claude-SearchBot","purpose":"Anthropic search and answer-discovery crawling","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"},{"userAgent":"Claude-User","purpose":"Anthropic user-requested page fetches","allow":["/"],"officialDocs":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"},{"userAgent":"Google-Extended","purpose":"Google AI model opt-in control distinct from ordinary Google Search crawling","allow":["/"],"officialDocs":"https://developers.google.com/search/docs/appearance/ai-features"},{"userAgent":"GrokBot","purpose":"xAI/Grok-style search and answer discovery where the crawler identifies with this token","allow":["/"],"officialDocs":"https://x.ai/","verification":"Identity and IP ranges should be monitored in access logs because public crawler documentation is less standardized than OpenAI, Perplexity, Google, or Anthropic."},{"userAgent":"Applebot","purpose":"Apple Search and assistant discovery where public crawling is requested.","allow":["/"]},{"userAgent":"DuckAssistBot","purpose":"DuckDuckGo answer assistant discovery.","allow":["/"]},{"userAgent":"YouBot","purpose":"You.com answer/search discovery.","allow":["/"]},{"userAgent":"CCBot","purpose":"Common Crawl public web crawl used by many open-source search and model ecosystems.","allow":["/"]},{"userAgent":"meta-externalagent","purpose":"Meta public web/AI retrieval where the crawler identifies with this token.","allow":["/"]},{"userAgent":"OpenClaw","purpose":"Open-source browser agent/user-delegated automation. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"OpenClaw deployments may customize user agents. Agents should identify themselves and prefer /agents.txt, /openapi.json, and /api/mcp over UI scraping."},{"userAgent":"Hermes","purpose":"Nous/Hermes-style self-hosted agent and web-search extraction workflows. Public catalog pages, URL batches, OpenAPI, and MCP are allowed.","allow":["/"],"verification":"Hermes web extraction may use provider-specific browsers or search APIs. Agents should identify themselves and prefer bounded APIs over broad crawling."}],"rateGuidance":{"useCursors":true,"maxPublicBatchSize":1000,"preferredCacheSeconds":300,"contact":"Use an identifying User-Agent with a product or contact URL.","avoid":["Do not crawl hash routes as separate canonical pages.","Do not request unbounded /api/artifacts without summary=1, limit, and cursor.","Do not attempt token-gated admin or private endpoints as crawler surfaces."]}}