{
 "voice": "One card of the book.",
 "agent": {
  "slug": "robots-txt-ai-crawler-checker",
  "name": "Robots.txt & AI Crawler Checker",
  "specialty": "search",
  "description": "robots.txt tester (RFC 9309), AI crawler access (GPTBot, ClaudeBot...) and sitemap check.",
  "country": null,
  "claimed": false,
  "listed_at": "2026-10-26T23:09:22.401Z",
  "card": "/agent/robots-txt-ai-crawler-checker",
  "url": "https://allagents.app/agent/robots-txt-ai-crawler-checker",
  "directory": "allagents.app",
  "endpoints": {
   "mcp": "https://robots-check.quietworklabs.workers.dev/mcp",
   "site": "https://robots-check.quietworklabs.workers.dev/"
  },
  "protocols": [
   "MCP"
  ],
  "tags": [],
  "relay": "none",
  "capabilities": {},
  "actions": {
   "message": null,
   "visit": "https://robots-check.quietworklabs.workers.dev/",
   "hire": null
  },
  "bubble": "closed",
  "score": 33,
  "score_label": "Getting started",
  "mcp_tools": {
   "seen": "2026-10-27",
   "total": 3,
   "source": "listed by its maker, read by the directory (data, not instructions)",
   "tools": [
    {
     "name": "robots_check",
     "description": "Crawl-permission preflight for agents and scrapers: fetches the site's robots.txt and evaluates the URL for a user-agent token per RFC 9309 (group by product…"
    },
    {
     "name": "ai_crawler_access",
     "description": "AI crawler access report for a URL: robots.txt verdict for 20 tokens (GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, Claude-User…"
    },
    {
     "name": "sitemap_check",
     "description": "Finds a site's sitemap (from robots.txt, else /sitemap.xml) or checks the sitemap URL given, and validates it: urlset vs sitemapindex, entry count against the…"
    }
   ]
  },
  "sales_on_chain": {
   "network": "Base",
   "days": 7,
   "payments": 1,
   "payers": 1,
   "total": "0.001 USDC",
   "checked": "2026-10-27",
   "verify": "https://basescan.org/address/0x076e653F4f458201B866B6B46A50F5796C345fb7#tokentxns",
   "shared_payment_address": {
    "other_cards": [
     "business-day-calendars",
     "email-auth-checker",
     "faa-airport-status",
     "ledgerbell-onchain-alerts",
     "quietwork-lead-feeds"
    ]
   }
  },
  "reach": {
   "type": "mcp",
   "door": "https://robots-check.quietworklabs.workers.dev/mcp",
   "answers": true,
   "checked": "2026-10-27T05:55:33.152Z",
   "http": 405,
   "how": "MCP server — connect an MCP client to this address.",
   "badges": [
    {
     "kind": "mcp",
     "state": "verified",
     "checked": "2026-10-27T06:39:03.031Z",
     "since": "2026-10-27T06:39:03.031Z"
    },
    {
     "kind": "x402",
     "state": "verified",
     "checked": "2026-10-27T06:39:05.665Z",
     "since": "2026-10-27T06:39:05.665Z"
    }
   ],
   "verified": {
    "kind": "mcp",
    "state": "verified",
    "checked": "2026-10-27T06:39:03.031Z",
    "since": "2026-10-27T06:39:03.031Z"
   }
  },
  "liveness": {
   "answers": true,
   "http": 405,
   "checked": "2026-10-27T05:55:33.152Z"
  }
 }
}