{
  "generatedAt": "2026-11-21T19:54:46.592Z",
  "count": 60,
  "bots": [
    {
      "slug": "oai-searchbot",
      "userAgent": "OAI-SearchBot",
      "operator": "OpenAI",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Indexes pages for ChatGPT Search citations and outbound links.",
      "impactLevel": "medium"
    },
    {
      "slug": "chatgpt-user",
      "userAgent": "ChatGPT-User",
      "operator": "OpenAI",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Real-time user browsing fetches initiated inside ChatGPT sessions.",
      "impactLevel": "medium"
    },
    {
      "slug": "perplexitybot",
      "userAgent": "PerplexityBot",
      "operator": "Perplexity AI",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Search index crawler powering Perplexity cited answers.",
      "impactLevel": "medium"
    },
    {
      "slug": "perplexity-user",
      "userAgent": "Perplexity-User",
      "operator": "Perplexity AI",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "On-demand user link fetcher for live Perplexity queries.",
      "impactLevel": "medium"
    },
    {
      "slug": "claude-searchbot",
      "userAgent": "Claude-SearchBot",
      "operator": "Anthropic",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Search indexing crawler for Claude answer citations.",
      "impactLevel": "medium"
    },
    {
      "slug": "duckassistbot",
      "userAgent": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Powers DuckDuckGo AI answer summaries with source links.",
      "impactLevel": "medium"
    },
    {
      "slug": "youbot",
      "userAgent": "YouBot",
      "operator": "You.com",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Indexes web docs for You.com research assistant citations.",
      "impactLevel": "medium"
    },
    {
      "slug": "gptbot",
      "userAgent": "GPTBot",
      "operator": "OpenAI",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Harvests full-site text for OpenAI foundation model pretraining.",
      "impactLevel": "high"
    },
    {
      "slug": "claudebot",
      "userAgent": "ClaudeBot",
      "operator": "Anthropic",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Bulk corpus scraping for Anthropic model training datasets.",
      "impactLevel": "high"
    },
    {
      "slug": "google-extended",
      "userAgent": "Google-Extended",
      "operator": "Google",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Opt-out token controlling Gemini training & grounding corpus usage (does not block Google Search).",
      "impactLevel": "high"
    },
    {
      "slug": "applebot-extended",
      "userAgent": "Applebot-Extended",
      "operator": "Apple",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Controls Apple Intelligence foundation model training ingestion.",
      "impactLevel": "medium"
    },
    {
      "slug": "meta-externalagent",
      "userAgent": "Meta-ExternalAgent",
      "operator": "Meta",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Scrapes web content for Meta Llama training & AI product grounding.",
      "impactLevel": "high"
    },
    {
      "slug": "bytespider",
      "userAgent": "Bytespider",
      "operator": "ByteDance",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Aggressive high-bandwidth crawler feeding ByteDance LLM datasets.",
      "impactLevel": "high"
    },
    {
      "slug": "ccbot",
      "userAgent": "CCBot",
      "operator": "Common Crawl",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Open web archive widely mirrored into public LLM training corpora.",
      "impactLevel": "high"
    },
    {
      "slug": "cohere-ai",
      "userAgent": "cohere-ai",
      "operator": "Cohere",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Scrapes text for Cohere model training and enterprise retrieval sets.",
      "impactLevel": "medium"
    },
    {
      "slug": "amazonbot",
      "userAgent": "Amazonbot",
      "operator": "Amazon",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Crawls sites to train Alexa/Nova models and enrich product answers.",
      "impactLevel": "high"
    },
    {
      "slug": "omgilibot",
      "userAgent": "omgilibot",
      "operator": "Webz.io",
      "category": "commercial-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Commercial data broker reselling scraped forums and blogs to AI labs.",
      "impactLevel": "high"
    },
    {
      "slug": "diffbot",
      "userAgent": "Diffbot",
      "operator": "Diffbot",
      "category": "commercial-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Extracts structured knowledge graphs from websites for commercial feeds.",
      "impactLevel": "medium"
    },
    {
      "slug": "petalbot",
      "userAgent": "PetalBot",
      "operator": "Huawei",
      "category": "commercial-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "High-volume crawler feeding Pangu AI and Petal search datasets.",
      "impactLevel": "high"
    },
    {
      "slug": "dataforseobot",
      "userAgent": "DataForSeoBot",
      "operator": "DataForSEO",
      "category": "commercial-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Commercial scraping API extracting SERP and page copy for third parties.",
      "impactLevel": "medium"
    },
    {
      "slug": "usage-ai-training",
      "userAgent": "__usage_ai-training",
      "operator": "Not applicable.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "`__usage_ai-training` is a policy token defined in an Internet Draft that Anthropic's Kevin Kelley submitted to the AI Preferences working group at the IETF to ",
      "impactLevel": "medium"
    },
    {
      "slug": "addsearchbot",
      "userAgent": "AddSearchBot",
      "operator": "Unclear at this time.",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "AddSearchBot is a web crawler that indexes website content for AddSearch's AI-powered site search solution, collecting data to provide fast and accurate search ",
      "impactLevel": "medium"
    },
    {
      "slug": "agentdatabot",
      "userAgent": "AgentDataBot",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "AgentDataBot visits public web pages to identify technology stacks, company details, and team members. The data builds AgentData's company-intelligence database",
      "impactLevel": "medium"
    },
    {
      "slug": "agenttimes",
      "userAgent": "AgentTimes",
      "operator": "[The Agent Times](https://theagenttimes.com/abou",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Scrapes data for AI news aggregation and republishing.",
      "impactLevel": "medium"
    },
    {
      "slug": "ai2bot",
      "userAgent": "AI2Bot",
      "operator": "[Ai2](https://allenai.org/crawler)",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Explores 'certain domains' to find web content.",
      "impactLevel": "medium"
    },
    {
      "slug": "ai2bot-deepresearcheval",
      "userAgent": "AI2Bot-DeepResearchEval",
      "operator": "Ai2, a non-profit AI research institute",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "AI2Bot-DeepResearchEval retrieves resources used to evaluate deep-research queries performed by Ai2's open-source models. More info can be found at https://know",
      "impactLevel": "medium"
    },
    {
      "slug": "ai2bot-dolma",
      "userAgent": "Ai2Bot-Dolma",
      "operator": "[Ai2](https://allenai.org/crawler)",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Explores 'certain domains' to find web content.",
      "impactLevel": "medium"
    },
    {
      "slug": "aihitbot",
      "userAgent": "aiHitBot",
      "operator": "[aiHit](https://www.aihitdata.com/about)",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Scrapes data for AI systems.",
      "impactLevel": "medium"
    },
    {
      "slug": "aiwebindex",
      "userAgent": "AIWebIndex",
      "operator": "[Lyrenth](https://lyrenth.com)",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Builds an index of public pages and serves them to AI agents as extracted, readable text with attribution and a link back to the source. Does not train foundati",
      "impactLevel": "medium"
    },
    {
      "slug": "amazon-kendra",
      "userAgent": "amazon-kendra",
      "operator": "Amazon",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Amazon Kendra is a highly accurate intelligent search service that enables your users to search unstructured data using natural language. It returns specific an",
      "impactLevel": "medium"
    },
    {
      "slug": "amazon-qbusiness",
      "userAgent": "amazon-QBusiness",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "amazon-QBusiness is an Amazon Q Business web crawler that fetches and indexes web content for Amazon Q Business applications. More info can be found at https://",
      "impactLevel": "medium"
    },
    {
      "slug": "amazonbuyforme",
      "userAgent": "AmazonBuyForMe",
      "operator": "[Amazon](https://amazon.com)",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "AmazonBuyForMe is an Amazon bot that crawls websites as part of the Amazon Buy For Me service. This bot visits product pages and e-commerce websites to gather p",
      "impactLevel": "medium"
    },
    {
      "slug": "amzn-searchbot",
      "userAgent": "Amzn-SearchBot",
      "operator": "Unclear at this time.",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/amzn-searchbot",
      "impactLevel": "medium"
    },
    {
      "slug": "amzn-user",
      "userAgent": "Amzn-User",
      "operator": "Amazon, used for fetching web content to answer ",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Amzn-User is an AI assistant operated by Amazon, used for fetching web content to answer user queries through Alexa and other Amazon AI services. More info can ",
      "impactLevel": "medium"
    },
    {
      "slug": "andibot",
      "userAgent": "Andibot",
      "operator": "[Andi](https://andisearch.com/)",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Scrapes website and provides AI summary.",
      "impactLevel": "medium"
    },
    {
      "slug": "anomura",
      "userAgent": "Anomura",
      "operator": "[Direqt](https://direqt.ai)",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Anomura is Direqt's search crawler, it discovers and indexes pages their customers websites.",
      "impactLevel": "medium"
    },
    {
      "slug": "anthropic-ai",
      "userAgent": "anthropic-ai",
      "operator": "[Anthropic](https://www.anthropic.com)",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Scrapes data to train LLMs and AI products offered by Anthropic.",
      "impactLevel": "medium"
    },
    {
      "slug": "apifybot",
      "userAgent": "ApifyBot",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "ApifyBot is a web scraping and data extraction crawler by Apify that collects website content for use in AI, LLMs, RAG, and automation workflows. More info can ",
      "impactLevel": "medium"
    },
    {
      "slug": "apifywebsitecontentcrawler",
      "userAgent": "ApifyWebsiteContentCrawler",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "ApifyWebsiteContentCrawler is a web crawler by Apify that extracts and downloads full website content for use in AI, data analysis, and automation workflows. Mo",
      "impactLevel": "medium"
    },
    {
      "slug": "applebot",
      "userAgent": "Applebot",
      "operator": "Unclear at this time.",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Applebot is a web crawler used by Apple to index search results that allow the Siri AI Assistant to answer user questions. Siri's answers normally contain refer",
      "impactLevel": "medium"
    },
    {
      "slug": "aranet-searchbot",
      "userAgent": "Aranet-SearchBot",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/aranet-searchbot",
      "impactLevel": "medium"
    },
    {
      "slug": "atlassian-bot",
      "userAgent": "atlassian-bot",
      "operator": "[Atlassian](https://www.atlassian.com)",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "atlassian-bot is a web crawler used to index website content for its AI search, assistants and agents available in its Rovo GenAI product.",
      "impactLevel": "medium"
    },
    {
      "slug": "awario",
      "userAgent": "Awario",
      "operator": "Awario",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Awario is an AI data scraper operated by Awario. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can pro",
      "impactLevel": "medium"
    },
    {
      "slug": "azureai-searchbot",
      "userAgent": "AzureAI-SearchBot",
      "operator": "Unclear at this time.",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/azureai-searchbot",
      "impactLevel": "medium"
    },
    {
      "slug": "bedrockbot",
      "userAgent": "bedrockbot",
      "operator": "[Amazon](https://amazon.com)",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Connects to and crawls URLs that have been selected for use in a user's AWS bedrock application.",
      "impactLevel": "medium"
    },
    {
      "slug": "bigsur-ai",
      "userAgent": "bigsur.ai",
      "operator": "Big Sur AI that fetches website content to enabl",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "bigsur.ai is a web crawler operated by Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solution",
      "impactLevel": "medium"
    },
    {
      "slug": "bixelbot",
      "userAgent": "BixelBot",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "BixelBot collects public company and market signals for Bixel's structured company-data platform, which supplies verified business information to AI agents and ",
      "impactLevel": "medium"
    },
    {
      "slug": "bravebot",
      "userAgent": "Bravebot",
      "operator": "https://safe.search.brave.com/help/brave-search-",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Bravebot is a web crawler by Brave that indexes pages for Brave Search, providing search data and AI-optimized context to power chatbots, agents, and RAG pipeli",
      "impactLevel": "medium"
    },
    {
      "slug": "brightbot",
      "userAgent": "Brightbot",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Brightbot is a web data collection crawler by Bright Data that extracts and structures public website content at scale, providing AI-ready data for model traini",
      "impactLevel": "medium"
    },
    {
      "slug": "buddybot",
      "userAgent": "BuddyBot",
      "operator": "[BuddyBotLearning](https://www.buddybotlearning.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "BuddyBot is a voice-controlled AI learning companion targeted at childhooded STEM education.",
      "impactLevel": "medium"
    },
    {
      "slug": "channel3bot",
      "userAgent": "Channel3Bot",
      "operator": "Unclear at this time.",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/channel3bot",
      "impactLevel": "medium"
    },
    {
      "slug": "chatglm-spider",
      "userAgent": "ChatGLM-Spider",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Description unavailable from knownagents.com More info can be found at https://knownagents.com/agents/chatglm-spider",
      "impactLevel": "medium"
    },
    {
      "slug": "claude-agent",
      "userAgent": "Claude-Agent",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Claude-Agent is Anthropic's cloud browser agent for Claude Duet. It navigates websites and completes multi-step tasks on behalf of users. More info can be found",
      "impactLevel": "medium"
    },
    {
      "slug": "claude-code",
      "userAgent": "Claude-Code",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Claude Code is an AI coding agent by Anthropic that can build, debug, and ship code directly from the terminal, handling tasks like codebase onboarding, multi-f",
      "impactLevel": "medium"
    },
    {
      "slug": "claude-user",
      "userAgent": "Claude-User",
      "operator": "[Anthropic](https://www.anthropic.com)",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Claude-User is dispatched by Anthropic's Claude AI assistant in response to user prompts, when it needs to fetch content to include in its answers. More info ca",
      "impactLevel": "medium"
    },
    {
      "slug": "claude-web",
      "userAgent": "Claude-Web",
      "operator": "Anthropic",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "Claude-Web is an AI-related agent operated by Anthropic. It's currently unclear exactly what it's used for, since there's no official documentation. If you can ",
      "impactLevel": "medium"
    },
    {
      "slug": "cledarabot",
      "userAgent": "CledaraBot",
      "operator": "Cledara that visits SaaS providers to complete c",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "CledaraBot is an AI agent operated by Cledara that visits SaaS providers to complete customer-approved administrative tasks, including invoice collection and us",
      "impactLevel": "medium"
    },
    {
      "slug": "cloudflare-autorag",
      "userAgent": "Cloudflare-AutoRAG",
      "operator": "[Cloudflare](https://developers.cloudflare.com/a",
      "category": "ai-search",
      "givesCitationTraffic": true,
      "defaultPolicy": "allow",
      "purpose": "AutoRAG is an all-in-one AI search solution.",
      "impactLevel": "medium"
    },
    {
      "slug": "cloudflarebrowserrenderingcrawler",
      "userAgent": "CloudflareBrowserRenderingCrawler",
      "operator": "Cloudflare that returns rendered website content",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "CloudflareBrowserRenderingCrawler is a web crawler operated by Cloudflare that returns rendered website content for research, monitoring, and AI data workflows.",
      "impactLevel": "medium"
    },
    {
      "slug": "cloudvertexbot",
      "userAgent": "CloudVertexBot",
      "operator": "Unclear at this time.",
      "category": "training-scraper",
      "givesCitationTraffic": false,
      "defaultPolicy": "block",
      "purpose": "CloudVertexBot is a Google-operated crawler available to site owners to request targeted crawls of their own sites for AI training purposes on the Vertex AI pla",
      "impactLevel": "medium"
    }
  ]
}
