nahbro.dev/src/generators/robots.ts at main · unsigned-gg/nahbro.dev · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
const AI_BOTS = [
  // OpenAI
  "GPTBot",
  "ChatGPT-User",
  "OAI-SearchBot",
  // Anthropic
  "ClaudeBot",
  "Claude-Web",
  "anthropic-ai",
  // Google AI
  "Google-Extended",
  // Meta
  "meta-externalagent",
  "FacebookBot",
  // Apple
  "Applebot-Extended",
  // Amazon
  "Amazonbot",
  // Common Crawl / training data
  "CCBot",
  "DataForSeoBot",
  // Bytedance
  "Bytespider",
  // Perplexity
  "PerplexityBot",
  // Cohere
  "cohere-ai",
  // Diffbot
  "Diffbot",
  "ai2-unsafe-agent",
  // Scrapers commonly used for AI training
  "ImagesiftBot",
  "omgili",
  "omgilibot",
  "Meltwater",
  // Misc AI agents
  "YouBot",
  "Timpibot",
  "PetalBot",
  "Kangaroo Bot",
  "SemrushBot-OCOB",
];

export function generateRobotsTxt(): string {
  const lines: string[] = [
    "# nahbro.dev — nah bro as a service",
    "# https://nahbro.dev",
    "",
    "# nah.",
    "",
    ...AI_BOTS.flatMap((bot) => [`User-agent: ${bot}`, "Disallow: /", ""]),
    "User-agent: *",
    "Allow: /",
  ];
  return lines.join("\n");
}