{ "$schema": "https://aigeoradar.com/schemas/acr/1.0/crawler-registry.schema.json", "specification": { "name": "AI Crawler Registry", "shortName": "ACR", "version": "1.0.0", "rfc": "ACR-001" }, "maintainer": { "name": "AigeoRadar", "url": "https://aigeoradar.com", "specification": "https://aigeoradar.com/specifications/acr", "github": "https://github.com/aagedik/aigeoradar-acr" }, "license": "CC-BY-4.0", "generatedAt": "2026-07-09T22:54:58+00:00", "crawlerCount": 241, "categories": [ { "key": "ai_llm", "label": "AI & LLM" }, { "key": "search_engine", "label": "Search Engine" }, { "key": "seo", "label": "SEO" }, { "key": "social_preview", "label": "Social & Link Preview" }, { "key": "rss", "label": "RSS & Feed" }, { "key": "ads", "label": "Advertising" }, { "key": "security", "label": "Security" }, { "key": "monitoring", "label": "Monitoring & Uptime" }, { "key": "archive", "label": "Archive & Data Collection" } ], "crawlers": [ { "id": "ai-search-external", "name": "AI Search External", "robotsAgent": "AI Search External", "userAgent": "AI Search External", "company": null, "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "ai_crawl", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": true, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "addsearchbot", "name": "AddSearchBot", "robotsAgent": "AddSearchBot", "userAgent": "AddSearchBot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "agencytoolspro", "name": "AgencyToolsPro", "robotsAgent": "AgencyToolsPro", "userAgent": "AgencyToolsPro", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "ahrefs-site-audit", "name": "Ahrefs Site Audit", "robotsAgent": "Ahrefs Site Audit", "userAgent": "Ahrefs Site Audit", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "ahrefsbot", "name": "AhrefsBot", "robotsAgent": "AhrefsBot", "userAgent": "AhrefsBot", "company": "Ahrefs", "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": "Ahrefs SEO site audit and backlink crawler.", "documentationUrl": "https://ahrefs.com/robot", "verified": true, "official": true, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "algolia", "name": "Algolia", "robotsAgent": "Algolia", "userAgent": "Algolia", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "alli-ai-bot", "name": "Alli AI Bot", "robotsAgent": "Alli AI Bot", "userAgent": "Alli AI Bot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "amazon-adbot", "name": "Amazon AdBot", "robotsAgent": "Amazon AdBot", "userAgent": "Amazon AdBot", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "amazonbot", "name": "Amazonbot", "robotsAgent": "Amazonbot", "userAgent": "Amazonbot", "company": "Amazon", "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "training", "description": "Amazon crawler for product and AI model improvement.", "documentationUrl": "https://developer.amazon.com/amazonbot", "verified": true, "official": true, "capabilities": { "ai": true, "training": true, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "amzn-searchbot", "name": "Amzn-SearchBot", "robotsAgent": "Amzn-SearchBot", "userAgent": "Amzn-SearchBot", "company": "Amazon", "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": "Amazon search indexing crawler (Alexa and Amazon search). Does not train generative models.", "documentationUrl": "https://developer.amazon.com/amazonbot", "verified": true, "official": true, "capabilities": { "ai": true, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "applebot", "name": "Applebot", "robotsAgent": "Applebot", "userAgent": "Applebot", "company": "Apple", "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": "Apple search and Siri-related web crawler.", "documentationUrl": "https://support.apple.com/en-us/102498", "verified": true, "official": true, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "archive-it", "name": "Archive-It", "robotsAgent": "Archive-It", "userAgent": "Archive-It", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "arena-bot", "name": "Arena Bot", "robotsAgent": "Arena Bot", "userAgent": "Arena Bot", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "arquivo-web-crawler", "name": "Arquivo Web Crawler", "robotsAgent": "Arquivo Web Crawler", "userAgent": "Arquivo Web Crawler", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "attracta", "name": "Attracta", "robotsAgent": "Attracta", "userAgent": "Attracta", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "baiduspider", "name": "Baiduspider", "robotsAgent": "Baiduspider", "userAgent": "Baiduspider", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "barkrowler", "name": "Barkrowler", "robotsAgent": "Barkrowler", "userAgent": "Barkrowler", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "bazqux", "name": "Bazqux", "robotsAgent": "Bazqux", "userAgent": "Bazqux", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "better-stack-bot", "name": "Better Stack Bot", "robotsAgent": "Better Stack Bot", "userAgent": "Better Stack Bot", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "better-uptime", "name": "Better Uptime", "robotsAgent": "Better Uptime", "userAgent": "Better Uptime", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "bibliotheque-nationale-de-france", "name": "Bibliotheque Nationale de France", "robotsAgent": "Bibliotheque Nationale de France", "userAgent": "Bibliotheque Nationale de France", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "binarycanary", "name": "BinaryCanary", "robotsAgent": "BinaryCanary", "userAgent": "BinaryCanary", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "bing-ads", "name": "Bing Ads", "robotsAgent": "Bing Ads", "userAgent": "Bing Ads", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "bingbot", "name": "BingBot", "robotsAgent": "Bingbot", "userAgent": "Bingbot", "company": "Microsoft", "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": "Bing and Microsoft Copilot search indexing crawler.", "documentationUrl": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0", "verified": true, "official": true, "capabilities": { "ai": true, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "blogtrottr", "name": "Blogtrottr", "robotsAgent": "Blogtrottr", "userAgent": "Blogtrottr", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "bluesky-link-preview", "name": "Bluesky Link Preview", "robotsAgent": "Bluesky Link Preview", "userAgent": "Bluesky Link Preview", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "botify", "name": "Botify", "robotsAgent": "Botify", "userAgent": "Botify", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "bravebot", "name": "Bravebot", "robotsAgent": "Bravebot", "userAgent": "Bravebot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "brightedge-bot", "name": "BrightEdge Bot", "robotsAgent": "BrightEdge Bot", "userAgent": "BrightEdge Bot", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "buffer-link-preview-bot", "name": "Buffer Link Preview Bot", "robotsAgent": "Buffer Link Preview Bot", "userAgent": "Buffer Link Preview Bot", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "bushbaby", "name": "Bushbaby", "robotsAgent": "Bushbaby", "userAgent": "Bushbaby", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "buttondown-rss-feed-parser", "name": "Buttondown RSS Feed Parser", "robotsAgent": "Buttondown RSS Feed Parser", "userAgent": "Buttondown RSS Feed Parser", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "caliberbot", "name": "CaliberBot", "robotsAgent": "CaliberBot", "userAgent": "CaliberBot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "channel3bot", "name": "Channel3Bot", "robotsAgent": "Channel3Bot", "userAgent": "Channel3Bot", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "chatgpt-user", "name": "ChatGPT-User", "robotsAgent": "ChatGPT-User", "userAgent": "ChatGPT-User", "company": "OpenAI", "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "user_fetch", "description": "Fetches pages when a ChatGPT user explicitly requests a URL.", "documentationUrl": "https://platform.openai.com/docs/gptbot", "verified": true, "official": true, "capabilities": { "ai": true, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "chathive-crawler", "name": "Chathive crawler", "robotsAgent": "Chathive crawler", "userAgent": "Chathive crawler", "company": null, "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "ai_crawl", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": true, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "checkly-bot", "name": "Checkly Bot", "robotsAgent": "Checkly Bot", "userAgent": "Checkly Bot", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "claude-searchbot", "name": "Claude-SearchBot", "robotsAgent": "Claude-SearchBot", "userAgent": "Claude-SearchBot", "company": "Anthropic", "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "search", "description": "Anthropic search indexing crawler.", "documentationUrl": "https://www.anthropic.com/claude-bot", "verified": true, "official": true, "capabilities": { "ai": true, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "claude-user", "name": "Claude-User", "robotsAgent": "Claude-User", "userAgent": "Claude-User", "company": "Anthropic", "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "user_fetch", "description": "Fetches pages when Claude users request live web content.", "documentationUrl": "https://www.anthropic.com/claude-bot", "verified": true, "official": true, "capabilities": { "ai": true, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "clearscopebot", "name": "ClearscopeBot", "robotsAgent": "ClearscopeBot", "userAgent": "ClearscopeBot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "cloudflare-customerslo-prober", "name": "Cloudflare CustomerSLO Prober", "robotsAgent": "Cloudflare CustomerSLO Prober", "userAgent": "Cloudflare CustomerSLO Prober", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "cloudflare-prefetch", "name": "Cloudflare Prefetch", "robotsAgent": "Cloudflare Prefetch", "userAgent": "Cloudflare Prefetch", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "cloudflare-radar-url-scanner", "name": "Cloudflare Radar URL Scanner", "robotsAgent": "Cloudflare Radar URL Scanner", "userAgent": "Cloudflare Radar URL Scanner", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "cloudflare-ssldetector", "name": "Cloudflare SSLDetector", "robotsAgent": "Cloudflare SSLDetector", "userAgent": "Cloudflare SSLDetector", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "cloudflare-stream-webhook", "name": "Cloudflare Stream Webhook", "robotsAgent": "Cloudflare Stream Webhook", "userAgent": "Cloudflare Stream Webhook", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "cloudflare-traffic-manager", "name": "Cloudflare Traffic Manager", "robotsAgent": "Cloudflare Traffic Manager", "userAgent": "Cloudflare Traffic Manager", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "cloudflare-web-scanner", "name": "Cloudflare Web Scanner", "robotsAgent": "Cloudflare Web Scanner", "userAgent": "Cloudflare Web Scanner", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "contextualbot", "name": "ContextualBot", "robotsAgent": "ContextualBot", "userAgent": "ContextualBot", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "convermaxbot", "name": "ConvermaxBot", "robotsAgent": "ConvermaxBot", "userAgent": "ConvermaxBot", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "cookie-hub", "name": "Cookie Hub", "robotsAgent": "Cookie Hub", "userAgent": "Cookie Hub", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "cookieyesbot", "name": "CookieYesBot", "robotsAgent": "CookieYesBot", "userAgent": "CookieYesBot", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "cookiebot", "name": "Cookiebot", "robotsAgent": "Cookiebot", "userAgent": "Cookiebot", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "coveo-bot", "name": "Coveo Bot", "robotsAgent": "Coveo Bot", "userAgent": "Coveo Bot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "criteobot", "name": "CriteoBot", "robotsAgent": "CriteoBot", "userAgent": "CriteoBot", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "cxense", "name": "Cxense", "robotsAgent": "Cxense", "userAgent": "Cxense", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "c-c-c-c", "name": "Cốc Cốc", "robotsAgent": "Cốc Cốc", "userAgent": "Cốc Cốc", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "dataforseo", "name": "DataForSEO", "robotsAgent": "DataForSEO", "userAgent": "DataForSEO", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "dataforseo-bot", "name": "DataForSEO Bot", "robotsAgent": "DataForSEO Bot", "userAgent": "DataForSEO Bot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "datadog-synthetic-bot", "name": "Datadog Synthetic Bot", "robotsAgent": "Datadog Synthetic Bot", "userAgent": "Datadog Synthetic Bot", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "dataprovider-com", "name": "Dataprovider.com", "robotsAgent": "Dataprovider.com", "userAgent": "Dataprovider.com", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "deepcrawl", "name": "Deepcrawl", "robotsAgent": "Deepcrawl", "userAgent": "Deepcrawl", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "detectify", "name": "Detectify", "robotsAgent": "Detectify", "userAgent": "Detectify", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "digicert-dcv", "name": "DigiCert DCV", "robotsAgent": "DigiCert DCV", "userAgent": "DigiCert DCV", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "direqt-anomura", "name": "Direqt Anomura", "robotsAgent": "Direqt Anomura", "userAgent": "Direqt Anomura", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "dotcom-monitor", "name": "Dotcom Monitor", "robotsAgent": "Dotcom Monitor", "userAgent": "Dotcom Monitor", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "drata-autopilot", "name": "Drata Autopilot", "robotsAgent": "Drata Autopilot", "userAgent": "Drata Autopilot", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "duckassistbot", "name": "DuckAssistbot", "robotsAgent": "DuckAssistBot", "userAgent": "DuckAssistBot", "company": "DuckDuckGo", "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "search", "description": "DuckDuckGo DuckAssist AI answer crawler.", "documentationUrl": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "verified": true, "official": true, "capabilities": { "ai": true, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "duckduckbot", "name": "DuckDuckBot", "robotsAgent": "DuckDuckBot", "userAgent": "DuckDuckBot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "embedly", "name": "Embedly", "robotsAgent": "Embedly", "userAgent": "Embedly", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "facebookexternalhit", "name": "FacebookExternalHit", "robotsAgent": "facebookexternalhit", "userAgent": "facebookexternalhit", "company": "Meta", "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": "Legacy Facebook external hit crawler for link previews.", "documentationUrl": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "verified": true, "official": true, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "factset-spyderbot", "name": "Factset SpyderBot", "robotsAgent": "Factset SpyderBot", "userAgent": "Factset SpyderBot", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "fastmail-bot", "name": "Fastmail Bot", "robotsAgent": "Fastmail Bot", "userAgent": "Fastmail Bot", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "feedwind-crawler", "name": "FeedWind Crawler", "robotsAgent": "FeedWind Crawler", "userAgent": "FeedWind Crawler", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "feedbin", "name": "Feedbin", "robotsAgent": "Feedbin", "userAgent": "Feedbin", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "feeder", "name": "Feeder", "robotsAgent": "Feeder", "userAgent": "Feeder", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "feedly", "name": "Feedly", "robotsAgent": "Feedly", "userAgent": "Feedly", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "flipboardproxy", "name": "FlipboardProxy", "robotsAgent": "FlipboardProxy", "userAgent": "FlipboardProxy", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "flipboardrss", "name": "FlipboardRSS", "robotsAgent": "FlipboardRSS", "userAgent": "FlipboardRSS", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "freespoke", "name": "Freespoke", "robotsAgent": "Freespoke", "userAgent": "Freespoke", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "freshping", "name": "Freshping", "robotsAgent": "Freshping", "userAgent": "Freshping", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "funnelback", "name": "Funnelback", "robotsAgent": "Funnelback", "userAgent": "Funnelback", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "gptbot", "name": "GPTBot", "robotsAgent": "GPTBot", "userAgent": "GPTBot", "company": "OpenAI", "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "training", "description": "Crawls web content to improve OpenAI generative AI models and ChatGPT.", "documentationUrl": "https://platform.openai.com/docs/gptbot", "verified": true, "official": true, "capabilities": { "ai": true, "training": true, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "gtmetrix", "name": "GTmetrix", "robotsAgent": "GTmetrix", "userAgent": "GTmetrix", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "google-api", "name": "Google API", "robotsAgent": "Google API", "userAgent": "Google API", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-adsbot", "name": "Google AdsBot", "robotsAgent": "Google AdsBot", "userAgent": "Google AdsBot", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-adsense-mediapartners", "name": "Google Adsense Mediapartners", "robotsAgent": "Google Adsense Mediapartners", "userAgent": "Google Adsense Mediapartners", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-businesslinkverification", "name": "Google BusinessLinkVerification", "robotsAgent": "Google BusinessLinkVerification", "userAgent": "Google BusinessLinkVerification", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "google-calendar-importer", "name": "Google Calendar Importer", "robotsAgent": "Google Calendar Importer", "userAgent": "Google Calendar Importer", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "google-cloudvertexbot", "name": "Google CloudVertexBot", "robotsAgent": "Google CloudVertexBot", "userAgent": "Google CloudVertexBot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-digital-asset-links", "name": "Google Digital Asset Links", "robotsAgent": "Google Digital Asset Links", "userAgent": "Google Digital Asset Links", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "google-display-ads-bot", "name": "Google Display Ads Bot", "robotsAgent": "Google Display Ads Bot", "userAgent": "Google Display Ads Bot", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-feed-fetcher", "name": "Google Feed Fetcher", "robotsAgent": "Google Feed Fetcher", "userAgent": "Google Feed Fetcher", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-image-proxy", "name": "Google Image Proxy", "robotsAgent": "Google Image Proxy", "userAgent": "Google Image Proxy", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-images", "name": "Google Images", "robotsAgent": "Google Images", "userAgent": "Google Images", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-inspection-tool", "name": "Google Inspection Tool", "robotsAgent": "Google Inspection Tool", "userAgent": "Google Inspection Tool", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-page-renderer", "name": "Google Page Renderer", "robotsAgent": "Google Page Renderer", "userAgent": "Google Page Renderer", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-publisher-center", "name": "Google Publisher Center", "robotsAgent": "Google Publisher Center", "userAgent": "Google Publisher Center", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "google-read-aloud", "name": "Google Read Aloud", "robotsAgent": "Google Read Aloud", "userAgent": "Google Read Aloud", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-schema-markup-testing-tool", "name": "Google Schema Markup Testing Tool", "robotsAgent": "Google Schema Markup Testing Tool", "userAgent": "Google Schema Markup Testing Tool", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "google-scholar", "name": "Google Scholar", "robotsAgent": "Google Scholar", "userAgent": "Google Scholar", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-site-verification", "name": "Google Site Verification", "robotsAgent": "Google Site Verification", "userAgent": "Google Site Verification", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "google-storebot", "name": "Google StoreBot", "robotsAgent": "Google StoreBot", "userAgent": "Google StoreBot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-travelads-live", "name": "Google TravelAds Live", "robotsAgent": "Google TravelAds Live", "userAgent": "Google TravelAds Live", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-trust-services", "name": "Google Trust Services", "robotsAgent": "Google Trust Services", "userAgent": "Google Trust Services", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "google-videos", "name": "Google Videos", "robotsAgent": "Google Videos", "userAgent": "Google Videos", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "google-adwords-express", "name": "Google-AdWords-Express", "robotsAgent": "Google-AdWords-Express", "userAgent": "Google-AdWords-Express", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "google-adwords-instant", "name": "Google-Adwords-Instant", "robotsAgent": "Google-Adwords-Instant", "userAgent": "Google-Adwords-Instant", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "google-flights-search-spackle", "name": "Google-Flights-Search-Spackle", "robotsAgent": "Google-Flights-Search-Spackle", "userAgent": "Google-Flights-Search-Spackle", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "googlebot", "name": "GoogleBot", "robotsAgent": "Googlebot", "userAgent": "Googlebot", "company": "Google", "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": "Google Search indexing crawler.", "documentationUrl": "https://developers.google.com/search/docs/crawling-indexing/googlebot", "verified": true, "official": true, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "googleother", "name": "GoogleOther", "robotsAgent": "GoogleOther", "userAgent": "GoogleOther", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "grafana-synthetic-monitoring", "name": "Grafana Synthetic Monitoring", "robotsAgent": "Grafana Synthetic Monitoring", "userAgent": "Grafana Synthetic Monitoring", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "hatena", "name": "Hatena", "robotsAgent": "Hatena", "userAgent": "Hatena", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "hetrixtools", "name": "HetrixTools", "robotsAgent": "HetrixTools", "userAgent": "HetrixTools", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "hosttracker", "name": "HostTracker", "robotsAgent": "HostTracker", "userAgent": "HostTracker", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "hubspot-feed-fetcher", "name": "HubSpot Feed Fetcher", "robotsAgent": "HubSpot Feed Fetcher", "userAgent": "HubSpot Feed Fetcher", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "hubspot-page-fetcher", "name": "HubSpot Page Fetcher", "robotsAgent": "HubSpot Page Fetcher", "userAgent": "HubSpot Page Fetcher", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "huckabuy-bot", "name": "Huckabuy Bot", "robotsAgent": "Huckabuy Bot", "userAgent": "Huckabuy Bot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "ias-crawler", "name": "IAS crawler", "robotsAgent": "IAS crawler", "userAgent": "IAS crawler", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "iboubot", "name": "IbouBot", "robotsAgent": "IbouBot", "userAgent": "IbouBot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "iframely", "name": "Iframely", "robotsAgent": "Iframely", "userAgent": "Iframely", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "internet-archive", "name": "Internet Archive", "robotsAgent": "Internet Archive", "userAgent": "Internet Archive", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "internetarchivebot", "name": "InternetArchiveBot", "robotsAgent": "InternetArchiveBot", "userAgent": "InternetArchiveBot", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "jobswithgpt-bot", "name": "JobsWithGPT Bot", "robotsAgent": "JobsWithGPT Bot", "userAgent": "JobsWithGPT Bot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "kb-dk-bot", "name": "KB.dk Bot", "robotsAgent": "KB.dk Bot", "userAgent": "KB.dk Bot", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "kakaotalk-scrap", "name": "KakaoTalk Scrap", "robotsAgent": "KakaoTalk Scrap", "userAgent": "KakaoTalk Scrap", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "klaviyoaibot", "name": "KlaviyoAIBot", "robotsAgent": "KlaviyoAIBot", "userAgent": "KlaviyoAIBot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "leikibot", "name": "LeikiBot", "robotsAgent": "LeikiBot", "userAgent": "LeikiBot", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "let-s-encrypt", "name": "Let's Encrypt", "robotsAgent": "Let's Encrypt", "userAgent": "Let's Encrypt", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "library-of-congress-web-archiving", "name": "Library Of Congress Web Archiving", "robotsAgent": "Library Of Congress Web Archiving", "userAgent": "Library Of Congress Web Archiving", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "linkchecker-bot", "name": "LinkChecker Bot", "robotsAgent": "LinkChecker Bot", "userAgent": "LinkChecker Bot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "linkedinbot", "name": "LinkedInBot", "robotsAgent": "LinkedInBot", "userAgent": "LinkedInBot", "company": "LinkedIn", "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": "LinkedIn link preview crawler.", "documentationUrl": null, "verified": true, "official": true, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "logicmonitor", "name": "LogicMonitor", "robotsAgent": "LogicMonitor", "userAgent": "LogicMonitor", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "manus-bot", "name": "Manus Bot", "robotsAgent": "Manus Bot", "userAgent": "Manus Bot", "company": null, "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "ai_crawl", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": true, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "marginalia-search", "name": "Marginalia Search", "robotsAgent": "Marginalia Search", "userAgent": "Marginalia Search", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "mars-finder", "name": "Mars Finder", "robotsAgent": "Mars Finder", "userAgent": "Mars Finder", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "mediamonitoringbot", "name": "MediaMonitoringBot", "robotsAgent": "MediaMonitoringBot", "userAgent": "MediaMonitoringBot", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "mediatoolkitbot", "name": "MediatoolkitBot", "robotsAgent": "MediatoolkitBot", "userAgent": "MediatoolkitBot", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "meta-externalads", "name": "Meta-ExternalAds", "robotsAgent": "Meta-ExternalAds", "userAgent": "Meta-ExternalAds", "company": null, "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "ai_crawl", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": true, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "meta-externalagent", "name": "Meta-ExternalAgent", "robotsAgent": "Meta-ExternalAgent", "userAgent": "Meta-ExternalAgent", "company": "Meta", "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "training", "description": "Meta crawler for AI model training.", "documentationUrl": "https://www.facebook.com/help/655915885954774", "verified": true, "official": true, "capabilities": { "ai": true, "training": true, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "meta-externalfetcher", "name": "Meta-ExternalFetcher", "robotsAgent": "Meta-ExternalFetcher", "userAgent": "Meta-ExternalFetcher", "company": "Meta", "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "user_fetch", "description": "Meta crawler for user-initiated link previews and fetches.", "documentationUrl": "https://www.facebook.com/help/655915885954774", "verified": true, "official": true, "capabilities": { "ai": true, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "midasnetworkbot", "name": "MidasNetworkBot", "robotsAgent": "MidasNetworkBot", "userAgent": "MidasNetworkBot", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "mirrorwebcrawler", "name": "MirrorWebCrawler", "robotsAgent": "MirrorWebCrawler", "userAgent": "MirrorWebCrawler", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "mistralai-user", "name": "MistralAI-User", "robotsAgent": "MistralAI-User", "userAgent": "MistralAI-User", "company": null, "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "ai_crawl", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": true, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "mojeek", "name": "Mojeek", "robotsAgent": "Mojeek", "userAgent": "Mojeek", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "monspark", "name": "MonSpark", "robotsAgent": "MonSpark", "userAgent": "MonSpark", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "montasticmonitor", "name": "MontasticMonitor", "robotsAgent": "MontasticMonitor", "userAgent": "MontasticMonitor", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "moz-dotbot", "name": "Moz dotbot", "robotsAgent": "Moz dotbot", "userAgent": "Moz dotbot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "moz-rogerbot", "name": "Moz rogerbot", "robotsAgent": "Moz rogerbot", "userAgent": "Moz rogerbot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "netvigie", "name": "NETVIGIE", "robotsAgent": "NETVIGIE", "userAgent": "NETVIGIE", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "navu", "name": "Navu", "robotsAgent": "Navu", "userAgent": "Navu", "company": null, "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "ai_crawl", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": true, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "netumo", "name": "Netumo", "robotsAgent": "Netumo", "userAgent": "Netumo", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "newsbank", "name": "NewsBank", "robotsAgent": "NewsBank", "userAgent": "NewsBank", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "newsblur", "name": "NewsBlur", "robotsAgent": "NewsBlur", "userAgent": "NewsBlur", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "nitrobot", "name": "NitroBot", "robotsAgent": "NitroBot", "userAgent": "NitroBot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "nodeping", "name": "NodePing", "robotsAgent": "NodePing", "userAgent": "NodePing", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "oai-adsbot", "name": "OAI-AdsBot", "robotsAgent": "OAI-AdsBot", "userAgent": "OAI-AdsBot", "company": null, "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "ai_crawl", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": true, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "oai-searchbot", "name": "OAI-SearchBot", "robotsAgent": "OAI-SearchBot", "userAgent": "OAI-SearchBot", "company": "OpenAI", "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "search", "description": "Indexes content for OpenAI search experiences.", "documentationUrl": "https://platform.openai.com/docs/bots", "verified": true, "official": true, "capabilities": { "ai": true, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "okx-dolphin-crawler", "name": "OKX-dolphin-crawler", "robotsAgent": "OKX-dolphin-crawler", "userAgent": "OKX-dolphin-crawler", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "oncrawl", "name": "OnCrawl", "robotsAgent": "OnCrawl", "userAgent": "OnCrawl", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "online-webceo-bot", "name": "Online Webceo Bot", "robotsAgent": "Online Webceo Bot", "userAgent": "Online Webceo Bot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "oseox", "name": "Oseox", "robotsAgent": "Oseox", "userAgent": "Oseox", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "outbrain", "name": "Outbrain", "robotsAgent": "Outbrain", "userAgent": "Outbrain", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "overcast", "name": "Overcast", "robotsAgent": "Overcast", "userAgent": "Overcast", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "ping-admin", "name": "Ping-Admin", "robotsAgent": "Ping-Admin", "userAgent": "Ping-Admin", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "pingdom", "name": "Pingdom", "robotsAgent": "Pingdom", "userAgent": "Pingdom", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "pinterestbot", "name": "PinterestBot", "robotsAgent": "PinterestBot", "userAgent": "PinterestBot", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "pocket-casts-feed-parser", "name": "Pocket Casts Feed Parser", "robotsAgent": "Pocket Casts Feed Parser", "userAgent": "Pocket Casts Feed Parser", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "pressengine-bot", "name": "PressEngine Bot", "robotsAgent": "PressEngine Bot", "userAgent": "PressEngine Bot", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "probely", "name": "Probely", "robotsAgent": "Probely", "userAgent": "Probely", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "productsup-crawler", "name": "Productsup Crawler", "robotsAgent": "Productsup Crawler", "userAgent": "Productsup Crawler", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "projectshield-url-check", "name": "ProjectShield Url Check", "robotsAgent": "ProjectShield Url Check", "userAgent": "ProjectShield Url Check", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "protopage", "name": "Protopage", "robotsAgent": "Protopage", "userAgent": "Protopage", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "proximic", "name": "Proximic", "robotsAgent": "Proximic", "userAgent": "Proximic", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "pulsetic", "name": "Pulsetic", "robotsAgent": "Pulsetic", "userAgent": "Pulsetic", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "qualifiedbot", "name": "QualifiedBot", "robotsAgent": "QualifiedBot", "userAgent": "QualifiedBot", "company": null, "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "ai_crawl", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": true, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "qualys", "name": "Qualys", "robotsAgent": "Qualys", "userAgent": "Qualys", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "qwantbot", "name": "Qwantbot", "robotsAgent": "Qwantbot", "userAgent": "Qwantbot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "rss-api", "name": "RSS API", "robotsAgent": "RSS API", "userAgent": "RSS API", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "rackspace", "name": "Rackspace", "robotsAgent": "Rackspace", "userAgent": "Rackspace", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "retool", "name": "Retool", "robotsAgent": "Retool", "userAgent": "Retool", "company": null, "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "ai_crawl", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": true, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "se-ranking-bot", "name": "SE Ranking Bot", "robotsAgent": "SE Ranking Bot", "userAgent": "SE Ranking Bot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "seo-audit-check-bot", "name": "SEO Audit Check Bot", "robotsAgent": "SEO Audit Check Bot", "userAgent": "SEO Audit Check Bot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "sansec-security-monitor", "name": "Sansec Security Monitor", "robotsAgent": "Sansec Security Monitor", "userAgent": "Sansec Security Monitor", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "screpy", "name": "Screpy", "robotsAgent": "Screpy", "userAgent": "Screpy", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "searchatlas-bot", "name": "SearchAtlas Bot", "robotsAgent": "SearchAtlas Bot", "userAgent": "SearchAtlas Bot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "securityheaders", "name": "SecurityHeaders", "robotsAgent": "SecurityHeaders", "userAgent": "SecurityHeaders", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "seekport", "name": "Seekport", "robotsAgent": "Seekport", "userAgent": "Seekport", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "selectika-ai", "name": "Selectika AI", "robotsAgent": "Selectika AI", "userAgent": "Selectika AI", "company": null, "category": "ai_llm", "categoryLabel": "AI & LLM", "purpose": "ai_crawl", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": true, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "semrushbot", "name": "SemrushBot", "robotsAgent": "SemrushBot", "userAgent": "SemrushBot", "company": "Semrush", "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": "Semrush SEO marketing crawler.", "documentationUrl": "https://www.semrush.com/bot/", "verified": true, "official": true, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "semrushbot-ocob", "name": "SemrushBot-OCOB", "robotsAgent": "SemrushBot-OCOB", "userAgent": "SemrushBot-OCOB", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "semrushbotbacklinks", "name": "SemrushBotBacklinks", "robotsAgent": "SemrushBotBacklinks", "userAgent": "SemrushBotBacklinks", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "sentinelbot", "name": "SentinelBot", "robotsAgent": "SentinelBot", "userAgent": "SentinelBot", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "sentry", "name": "Sentry", "robotsAgent": "Sentry", "userAgent": "Sentry", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "sentry-uptime-monitoring", "name": "Sentry Uptime Monitoring", "robotsAgent": "Sentry Uptime Monitoring", "userAgent": "Sentry Uptime Monitoring", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "seobilitybot", "name": "SeobilityBot", "robotsAgent": "SeobilityBot", "userAgent": "SeobilityBot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "serpstatbot", "name": "SerpstatBot", "robotsAgent": "SerpstatBot", "userAgent": "SerpstatBot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "seznam", "name": "Seznam", "robotsAgent": "Seznam", "userAgent": "Seznam", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "shapbot", "name": "ShapBot", "robotsAgent": "ShapBot", "userAgent": "ShapBot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "sirdatabot", "name": "SirdataBot", "robotsAgent": "SirdataBot", "userAgent": "SirdataBot", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "site24x7", "name": "Site24x7", "robotsAgent": "Site24x7", "userAgent": "Site24x7", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "siteauditbot", "name": "SiteAuditBot", "robotsAgent": "SiteAuditBot", "userAgent": "SiteAuditBot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "sitelock", "name": "SiteLock", "robotsAgent": "SiteLock", "userAgent": "SiteLock", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "sitesearch360", "name": "SiteSearch360", "robotsAgent": "SiteSearch360", "userAgent": "SiteSearch360", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "skype", "name": "Skype", "robotsAgent": "Skype", "userAgent": "Skype", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "slack-image-proxy", "name": "Slack Image Proxy", "robotsAgent": "Slack Image Proxy", "userAgent": "Slack Image Proxy", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "slickstream", "name": "Slickstream", "robotsAgent": "Slickstream", "userAgent": "Slickstream", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "smarshbot", "name": "SmarshBot", "robotsAgent": "SmarshBot", "userAgent": "SmarshBot", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "smartologybot", "name": "SmartologyBot", "robotsAgent": "SmartologyBot", "userAgent": "SmartologyBot", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "solarwinds-observability", "name": "SolarWinds Observability", "robotsAgent": "SolarWinds Observability", "userAgent": "SolarWinds Observability", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "statuscake", "name": "StatusCake", "robotsAgent": "StatusCake", "userAgent": "StatusCake", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "sucuri", "name": "Sucuri", "robotsAgent": "Sucuri", "userAgent": "Sucuri", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "ttd-content", "name": "TTD Content", "robotsAgent": "TTD Content", "userAgent": "TTD Content", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "taboola", "name": "Taboola", "robotsAgent": "Taboola", "userAgent": "Taboola", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "telegram-bot", "name": "Telegram Bot", "robotsAgent": "Telegram Bot", "userAgent": "Telegram Bot", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "termly", "name": "Termly", "robotsAgent": "Termly", "userAgent": "Termly", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "trustedsite", "name": "TrustedSite", "robotsAgent": "TrustedSite", "userAgent": "TrustedSite", "company": null, "category": "archive", "categoryLabel": "Archive & Data Collection", "purpose": "archive", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "tumblr", "name": "Tumblr", "robotsAgent": "Tumblr", "userAgent": "Tumblr", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "updownbot", "name": "UpDownBot", "robotsAgent": "UpDownBot", "userAgent": "UpDownBot", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "uptimerobot", "name": "UptimeRobot", "robotsAgent": "UptimeRobot", "userAgent": "UptimeRobot", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "uptimia", "name": "Uptimia", "robotsAgent": "Uptimia", "userAgent": "Uptimia", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "w3-validator-services", "name": "W3 Validator Services", "robotsAgent": "W3 Validator Services", "userAgent": "W3 Validator Services", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "wmf-zotero-translation-server", "name": "WMF Zotero Translation Server", "robotsAgent": "WMF Zotero Translation Server", "userAgent": "WMF Zotero Translation Server", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "watchbot", "name": "Watchbot", "robotsAgent": "Watchbot", "userAgent": "Watchbot", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "watchful", "name": "Watchful", "robotsAgent": "Watchful", "userAgent": "Watchful", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "webpagetest", "name": "WebPageTest", "robotsAgent": "WebPageTest", "userAgent": "WebPageTest", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "webtotembot", "name": "WebTotemBot", "robotsAgent": "WebTotemBot", "userAgent": "WebTotemBot", "company": null, "category": "security", "categoryLabel": "Security", "purpose": "security_scan", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": true, "seo": false } }, { "id": "websitepulsebot", "name": "WebsitePulseBot", "robotsAgent": "WebsitePulseBot", "userAgent": "WebsitePulseBot", "company": null, "category": "monitoring", "categoryLabel": "Monitoring & Uptime", "purpose": "uptime", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": true, "rss": false, "security": false, "seo": false } }, { "id": "yahoo-ad-monitoring", "name": "Yahoo Ad Monitoring", "robotsAgent": "Yahoo Ad Monitoring", "userAgent": "Yahoo Ad Monitoring", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "yahoo-japan-advertising-bot", "name": "Yahoo Japan Advertising Bot", "robotsAgent": "Yahoo Japan Advertising Bot", "userAgent": "Yahoo Japan Advertising Bot", "company": null, "category": "ads", "categoryLabel": "Advertising", "purpose": "advertising", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": true, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "yahoo-link-preview", "name": "Yahoo Link Preview", "robotsAgent": "Yahoo Link Preview", "userAgent": "Yahoo Link Preview", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "yahoo-slurp", "name": "Yahoo Slurp", "robotsAgent": "Yahoo Slurp", "userAgent": "Yahoo Slurp", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "yahoo-japan", "name": "Yahoo! JAPAN", "robotsAgent": "Yahoo! JAPAN", "userAgent": "Yahoo! JAPAN", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "yahoomailproxy", "name": "YahooMailProxy", "robotsAgent": "YahooMailProxy", "userAgent": "YahooMailProxy", "company": null, "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "yandexbot", "name": "YandexBot", "robotsAgent": "YandexBot", "userAgent": "YandexBot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "yeti", "name": "Yeti", "robotsAgent": "Yeti", "userAgent": "Yeti", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "youbot", "name": "YouBot", "robotsAgent": "YouBot", "userAgent": "YouBot", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "zoominfo", "name": "ZoomInfo", "robotsAgent": "ZoomInfo", "userAgent": "ZoomInfo", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "zoombot", "name": "Zoombot", "robotsAgent": "Zoombot", "userAgent": "Zoombot", "company": null, "category": "seo", "categoryLabel": "SEO", "purpose": "seo_audit", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": true } }, { "id": "dlvr-it-feed-fetcher", "name": "dlvr.it Feed Fetcher", "robotsAgent": "dlvr.it Feed Fetcher", "userAgent": "dlvr.it Feed Fetcher", "company": null, "category": "rss", "categoryLabel": "RSS & Feed", "purpose": "feed", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": false, "preview": false, "ads": false, "monitoring": false, "rss": true, "security": false, "seo": false } }, { "id": "i-search-crawler", "name": "i-search-crawler", "robotsAgent": "i-search-crawler", "userAgent": "i-search-crawler", "company": null, "category": "search_engine", "categoryLabel": "Search Engine", "purpose": "search", "description": null, "documentationUrl": null, "verified": false, "official": false, "capabilities": { "ai": false, "training": false, "search": true, "preview": false, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } }, { "id": "twitterbot", "name": "twitterbot", "robotsAgent": "Twitterbot", "userAgent": "Twitterbot", "company": "X (Twitter)", "category": "social_preview", "categoryLabel": "Social & Link Preview", "purpose": "preview", "description": "X/Twitter card and link preview crawler.", "documentationUrl": null, "verified": true, "official": true, "capabilities": { "ai": false, "training": false, "search": false, "preview": true, "ads": false, "monitoring": false, "rss": false, "security": false, "seo": false } } ] }