AI 搜索爬虫技术规范
与适配手册

# AI爬虫UA验证脚本 - 通过反向DNS和IP段双重验证
import socket
import re

# 已知的AI爬虫IP段白名单
AI_CRAWLER_IP_RANGES = {
    "GPTBot": [
        "44.233.0.0/16", "52.16.0.0/15",
        "34.208.0.0/12", "54.148.0.0/15"
    ],
    "Google-Extended": [
        "66.249.64.0/19"
    ],
    "PerplexityBot": [
        "44.224.0.0/14", "52.44.0.0/14"
    ],
    "ClaudeBot": [
        "52.40.0.0/14", "54.148.0.0/15"
    ],
}

def verify_crawler_ip(ip_address):
    """通过反向DNS验证爬虫IP真实性"""
    try:
        hostname = socket.gethostbyaddr(ip_address)[0]
        # 验证反向解析的主机名
        verified_ip = socket.gethostbyname(hostname)
        return verified_ip == ip_address, hostname
    except socket.herror:
        return False, None

def identify_ai_crawler(user_agent, ip_address):
    """综合UA和IP识别AI爬虫"""
    ua_patterns = {
        "GPTBot": r"GPTBot",
        "ChatGPT-User": r"ChatGPT-User",
        "Google-Extended": r"Google-Extended",
        "PerplexityBot": r"PerplexityBot",
        "ClaudeBot": r"ClaudeBot",
        "Bytespider": r"Bytespider",
        "CCBot": r"CCBot",
    }

    detected = None
    for name, pattern in ua_patterns.items():
        if re.search(pattern, user_agent, re.IGNORECASE):
            detected = name
            break

    if detected:
        is_verified, hostname = verify_crawler_ip(ip_address)
        return {
            "crawler": detected,
            "ip": ip_address,
            "verified": is_verified,
            "hostname": hostname
        }
    return {"crawler": None, "verified": False}

# ==========================================
# AI 搜索爬虫适配 - 全面开放策略
# 适用于：希望最大化 AI 搜索曝光的网站
# ==========================================

# --- 通用规则（所有爬虫）---
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /checkout
Disallow: /account
Disallow: /search?
Disallow: /*.json$
Disallow: /*.xml$
Crawl-delay: 5
Sitemap: https://example.com/sitemap.xml

# --- OpenAI GPTBot（AI训练语料采集）---
User-agent: GPTBot
Allow: /
Allow: /blog/
Allow: /docs/
Allow: /knowledge/
Disallow: /premium/
Disallow: /internal/
Crawl-delay: 10

# --- OpenAI ChatGPT-User（实时搜索请求）---
User-agent: ChatGPT-User
Allow: /
Disallow: /premium/
Crawl-delay: 5

# --- OpenAI OAI-SearchBot（搜索索引）---
User-agent: OAI-SearchBot
Allow: /
Disallow: /premium/
Crawl-delay: 5

# --- Google-Extended（AI训练数据Opt-out）---
# 注意：Google-Extended 仅控制AI训练用途
# 不影响 Google 搜索收录（Googlebot 另行配置）
User-agent: Google-Extended
Disallow: /premium/
Disallow: /internal/

# --- PerplexityBot ---
User-agent: PerplexityBot
Allow: /
Allow: /blog/
Allow: /docs/
Disallow: /premium/
Crawl-delay: 10

# --- ClaudeBot（Anthropic）---
User-agent: ClaudeBot
Allow: /
Disallow: /premium/
Crawl-delay: 10

# --- Applebot-Extended ---
User-agent: Applebot-Extended
Disallow: /premium/
Disallow: /internal/

# --- Bytespider / ByteSparrowBot ---
# 字节跳动爬虫活跃度极高，建议更严格的速率限制
User-agent: Bytespider
Allow: /blog/
Allow: /docs/
Disallow: /
Crawl-delay: 20

User-agent: ByteSparrowBot
Allow: /blog/
Disallow: /
Crawl-delay: 20

# --- CCBot（Common Crawl）---
# Common Crawl 数据被大量AI模型使用
# 开放高价值内容有助于模型训练中包含你的信息
User-agent: CCBot
Allow: /blog/
Allow: /docs/
Disallow: /premium/
Crawl-delay: 20

# --- KimiBot（月之暗面）---
User-agent: KimiBot
Allow: /
Disallow: /premium/
Crawl-delay: 10

# --- FacebookBot（Meta AI）---
User-agent: FacebookBot
Allow: /
Disallow: /premium/
Crawl-delay: 10

# ==========================================
# 策略：允许AI搜索索引，但禁止训练数据使用
# 适用于：希望被AI引用但不想被用于模型训练
# ==========================================

# 通用规则：允许所有爬虫访问公开内容
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Crawl-delay: 5
Sitemap: https://example.com/sitemap.xml

# 明确禁止AI训练用途的爬虫
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ByteSparrowBot
Disallow: /

# 但允许AI搜索用途的爬虫（索引但不训练）
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 5

User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 5

User-agent: PerplexityBot
Allow: /
Crawl-delay: 10

User-agent: ClaudeBot
Allow: /
Crawl-delay: 10

# ==========================================
# 策略：付费内容网站 - 仅开放摘要页
# 适用于：付费墙/会员制内容网站
# ==========================================

User-agent: *
Allow: /
Allow: /article/summary/
Disallow: /article/full/
Disallow: /premium/
Disallow: /members/
Disallow: /admin/
Crawl-delay: 5
Sitemap: https://example.com/sitemap.xml

# AI爬虫：允许摘要页，禁止全文页
User-agent: GPTBot
Allow: /article/summary/
Allow: /blog/
Allow: /landing/
Disallow: /
Crawl-delay: 10

User-agent: ChatGPT-User
Allow: /article/summary/
Allow: /blog/
Disallow: /
Crawl-delay: 5

User-agent: PerplexityBot
Allow: /article/summary/
Disallow: /
Crawl-delay: 10

User-agent: Google-Extended
Allow: /article/summary/
Disallow: /

User-agent: ClaudeBot
Allow: /article/summary/
Disallow: /
Crawl-delay: 10

# 激进爬虫严格限制
User-agent: Bytespider
Disallow: /

User-agent: ByteSparrowBot
Disallow: /

#!/bin/bash
# 从Nginx访问日志中提取AI爬虫访问记录

LOG_FILE="/var/log/nginx/access.log"
OUTPUT_DIR="./crawler-analysis"
mkdir -p "$OUTPUT_DIR"

# 定义AI爬虫UA关键词
AI_UAS="GPTBot|ChatGPT-User|Google-Extended|PerplexityBot|\
Bytespider|ByteSparrowBot|CCBot|KimiBot|ClaudeBot|\
Applebot-Extended|OAI-SearchBot|DuckAssistBot|FacebookBot"

# 提取AI爬虫访问记录
echo "[1/4] 提取AI爬虫访问记录..."
grep -E "$AI_UAS" "$LOG_FILE" > "$OUTPUT_DIR/ai_crawler_raw.log"

# 按爬虫类型分类统计
echo "[2/4] 按爬虫类型分类统计..."
for ua in GPTBot ChatGPT-User Google-Extended PerplexityBot \
    Bytespider ByteSparrowBot CCBot KimiBot ClaudeBot; do
    count=$(grep -c "$ua" "$OUTPUT_DIR/ai_crawler_raw.log" 2>/dev/null || echo 0)
    echo "$ua: $count requests"
done | tee "$OUTPUT_DIR/crawler_summary.txt"

# 提取AI爬虫访问的URL列表
echo "[3/4] 提取AI爬虫访问的URL..."
grep -E "$AI_UAS" "$LOG_FILE" | \
    awk '{print $7}' | sort | uniq -c | sort -rn | \
    head -50 > "$OUTPUT_DIR/top_ai_urls.txt"

# 提取AI爬虫的IP地址
echo "[4/4] 提取AI爬虫IP..."
grep -E "$AI_UAS" "$LOG_FILE" | \
    awk '{print $1}' | sort | uniq -c | sort -rn | \
    head -30 > "$OUTPUT_DIR/ai_crawler_ips.txt"

echo "分析完成！结果保存在 $OUTPUT_DIR/"

# Nginx配置：根据UA将AI爬虫请求代理到预渲染服务

server {
    listen 80;
    server_name example.com;

    # 预渲染服务地址
    set $prerender "http://127.0.0.1:3000";

    # AI爬虫UA列表
    map $http_user_agent $is_ai_crawler {
        default 0;
        "~*GPTBot" 1;
        "~*ChatGPT-User" 1;
        "~*PerplexityBot" 1;
        "~*ClaudeBot" 1;
        "~*CCBot" 1;
        "~*KimiBot" 1;
        "~*Google-Extended" 1;
        "~*Applebot-Extended" 1;
        "~*OAI-SearchBot" 1;
    }

    location / {
        # 如果是AI爬虫，代理到预渲染服务
        if ($is_ai_crawler) {
            proxy_pass $prerender;
            proxy_set_header Host $host;
            proxy_set_header X-Prerender-URL $scheme://$host$request_uri;
            proxy_cache prerender_cache;
            proxy_cache_valid 200 1d;
            proxy_cache_key $request_uri;
        }

        # 普通用户请求正常处理
        try_files $uri $uri/ /index.html;
    }
}

# 预渲染缓存配置
proxy_cache_path /var/cache/prerender levels=1:2
    keys_zone=prerender_cache:100m max_size=1g inactive=7d;

// Next.js SSR页面 - 确保AI爬虫可见性
import { GetServerSideProps } from 'next';
import Head from 'next/head';

interface ArticleProps {
  article: {
    title: string;
    description: string;
    content: string;
    author: string;
    datePublished: string;
    dateModified: string;
  };
}

export default function ArticlePage({ article }: ArticleProps) {
  const jsonLd = {
    "@context": "https://schema.org",
    "@type": "Article",
    "headline": article.title,
    "description": article.description,
    "author": { "@type": "Person", "name": article.author },
    "datePublished": article.datePublished,
    "dateModified": article.dateModified,
  };

  return (
    <>
      <Head>
        {/* 核心元信息 - AI爬虫首先解析 */}
        <title>{article.title} | GEO技能网</title>
        <meta name="description" content={article.description} />
        <link rel="canonical" href={`https://example.com/article/${article.slug}`} />

        {/* JSON-LD结构化数据 - AI搜索引擎的关键信号 */}
        <script
          type="application/ld+json"
          dangerouslySetInnerHTML={{ __html: JSON.stringify(jsonLd) }}
        />
      </Head>

      {/* 核心内容直接在HTML中 - 不依赖JS渲染 */}
      <article>
        <h1>{article.title}</h1>
        <div dangerouslySetInnerHTML={{ __html: article.content }} />
      </article>
    </>
  );
}

// 服务端数据获取 - AI爬虫获取完整渲染的HTML
export const getServerSideProps: GetServerSideProps = async (context) => {
  const { slug } = context.params!;
  const article = await fetchArticle(slug as string);

  if (!article) {
    return { notFound: true };
  }

  return {
    props: { article },
    // ISR：每60秒重新验证 - 平衡性能与内容新鲜度
    revalidate: 60,
  };
};

<!-- Nuxt.js SSR页面 - AI爬虫友好的内容展示 -->
<template>
  <article>
    <h1>{{ article.title }}</h1>
    <div v-html="article.content"></div>
  </article>
</template>

<script setup>
const route = useRoute()
const { data: article } = await useFetch(
  `/api/articles/${route.params.slug}`,
  {
    // SSR: 服务端获取数据并渲染完整HTML
    server: true,
    // 客户端首次加载时使用服务端数据
    lazy: false,
  }
)

// 动态元信息 - 在服务端注入到HTML head中
useHead({
  title: `${article.value.title} | GEO技能网`,
  meta: [
    { name: 'description', content: article.value.description },
    { property: 'og:title', content: article.value.title },
    { property: 'og:description', content: article.value.description },
    { property: 'og:type', content: 'article' },
  ],
  link: [
    { rel: 'canonical', href: `https://example.com/article/${route.params.slug}` }
  ],
})

// JSON-LD结构化数据注入
useHead({
  script: [
    {
      type: 'application/ld+json',
      children: JSON.stringify({
        "@context": "https://schema.org",
        "@type": "Article",
        "headline": article.value.title,
        "description": article.value.description,
        "datePublished": article.value.datePublished,
        "dateModified": article.value.dateModified,
      })
    }
  ]
})
</script>

# Nginx速率限制配置 - 针对AI爬虫的精细化控制

# 定义不同级别的速率限制区域
limit_req_zone $binary_remote_addr zone=ai_crawler_limit:10m rate=5r/m;
limit_req_zone $binary_remote_addr zone=aggressive_crawler:10m rate=2r/m;
limit_req_zone $binary_remote_addr zone=general_limit:10m rate=30r/m;

# UA到限速级别的映射
map $http_user_agent $crawler_rate_class {
    default                    "general";
    "~*GPTBot"                 "ai_crawler";
    "~*ChatGPT-User"          "ai_crawler";
    "~*PerplexityBot"         "ai_crawler";
    "~*ClaudeBot"             "ai_crawler";
    "~*OAI-SearchBot"         "ai_crawler";
    "~*CCBot"                 "ai_crawler";
    "~*Google-Extended"       "ai_crawler";
    "~*Applebot-Extended"     "ai_crawler";
    # 激进爬虫 - 更严格的限制
    "~*Bytespider"            "aggressive";
    "~*ByteSparrowBot"        "aggressive";
    "~*KimiBot"               "aggressive";
}

server {
    listen 443 ssl http2;
    server_name example.com;

    # 根据爬虫类型应用不同速率限制
    location / {
        if ($crawler_rate_class = "ai_crawler") {
            limit_req zone=ai_crawler_limit burst=10 nodelay;
        }
        if ($crawler_rate_class = "aggressive") {
            limit_req zone=aggressive_crawler burst=5 nodelay;
        }
        if ($crawler_rate_class = "general") {
            limit_req zone=general_limit burst=50 nodelay;
        }

        # 自定义429错误页面
        limit_req_status 429;
        error_page 429 /429.html;

        try_files $uri $uri/ /index.html;
    }

    # 为AI爬虫提供精简版页面（可选）
    location /api/ {
        if ($crawler_rate_class != "general") {
            # AI爬虫不允许访问API端点
            return 403;
        }
        proxy_pass http://backend;
    }
}

# GPTBot 访问记录
44.233.128.45 - - [18/Jun/2026:03:24:17 +0800] "GET /blog/ai-search-optimization HTTP/2.0" 200 45231 "-" "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2"

# PerplexityBot 访问记录
52.44.212.89 - - [18/Jun/2026:04:12:33 +0800] "GET /docs/getting-started HTTP/2.0" 200 23456 "-" "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; PerplexityBot/1.0"

# Bytespider 高频访问记录
110.249.201.33 - - [18/Jun/2026:05:01:02 +0800] "GET /knowledge/graph-basics HTTP/2.0" 200 31245 "-" "Mozilla/5.0 (compatible; Bytespider)"

# ClaudeBot 访问记录
52.40.178.22 - - [18/Jun/2026:06:45:11 +0800] "GET /blog/semantic-vector-optimization HTTP/2.0" 200 38901 "-" "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ClaudeBot/1.0"

# 429 响应（被限速的AI爬虫）
110.249.201.34 - - [18/Jun/2026:05:01:05 +0800] "GET /blog/seo-tips HTTP/2.0" 429 0 "-" "Mozilla/5.0 (compatible; Bytespider)"

#!/usr/bin/env python3
"""Nginx日志AI爬虫分析脚本 - 全功能版"""

import re
import sys
from collections import defaultdict, Counter
from datetime import datetime

# AI爬虫UA匹配模式
AI_CRAWLER_PATTERNS = {
    "GPTBot":          r"GPTBot",
    "ChatGPT-User":    r"ChatGPT-User",
    "OAI-SearchBot":   r"OAI-SearchBot",
    "Google-Extended": r"Google-Extended",
    "PerplexityBot":   r"PerplexityBot",
    "ClaudeBot":        r"ClaudeBot",
    "Bytespider":      r"Bytespider",
    "ByteSparrowBot":  r"ByteSparrowBot",
    "CCBot":            r"CCBot",
    "KimiBot":          r"KimiBot",
    "Applebot-Extended": r"Applebot-Extended",
}

# Nginx日志解析正则
LOG_PATTERN = r'(\S+) - - \[(.+?)\] "(\S+) (\S+) \S+" (\d{3}) (\d+) "[^"]*" "([^"]*)"'

def parse_log(log_file):
    """解析Nginx日志并提取AI爬虫记录"""
    crawler_data = defaultdict(lambda: {
        "requests": [],
        "ips": set(),
        "status_codes": Counter(),
        "urls": Counter(),
        "hourly": defaultdict(int),
    })

    with open(log_file, 'r') as f:
        for line in f:
            match = re.search(LOG_PATTERN, line)
            if not match:
                continue

            ip, timestamp, method, url, status, size, ua = match.groups()

            # 匹配AI爬虫
            for crawler_name, pattern in AI_CRAWLER_PATTERNS.items():
                if re.search(pattern, ua, re.IGNORECASE):
                    data = crawler_data[crawler_name]
                    data["requests"].append({
                        "ip": ip, "time": timestamp,
                        "url": url, "status": int(status),
                        "size": int(size)
                    })
                    data["ips"].add(ip)
                    data["status_codes"[status] += 1
                    data["urls"[url] += 1
                    # 按小时统计
                    hour = timestamp.split(":")[1]
                    data["hourly"[hour] += 1
                    break

    return crawler_data

def generate_report(crawler_data):
    """生成分析报告"""
    print("=" * 60
    print("AI 爬虫日志分析报告")
    print("=" * 60)

    for name, data in sorted(crawler_data.items()):
        total = len(data["requests"])
        success_rate = data["status_codes"].get("200", 0) / max(total, 1) * 100
        avg_size = sum(r["size"] for r in data["requests"]) / max(total, 1)

        print(f"\n--- {name} ---")
        print(f"  总请求: {total}")
        print(f"  独立IP: {len(data['ips'])}")
        print(f"  成功率: {success_rate:.1f}%")
        print(f"  平均响应大小: {avg_size/1024:.1f}KB")
        print(f"  Top 5 URL:")
        for url, count in data["urls"].most_common(5):
            print(f"    {url}: {count}次")
        print(f"  状态码分布: {dict(data['status_codes'])}")

if __name__ == "__main__":
    log_file = sys.argv[1] if len(sys.argv) > 1 else "/var/log/nginx/access.log"
    data = parse_log(log_file)
    generate_report(data)

<!-- 隐私友好的AI爬虫元信息配置 -->

<!-- 场景1：允许AI搜索索引，但禁止训练数据使用 -->
<meta name="robots" content="noai, noimageai">

<!-- 场景2：完全禁止AI爬虫访问（含敏感用户信息） -->
<meta name="robots" content="noindex, noai, noimageai, noarchive">

<!-- 场景3：允许AI搜索索引和训练，但禁止缓存 -->
<meta name="robots" content="noarchive">

<!-- 场景4：内容级别的AI训练排除 -->
<article>
  <h1>公开文章标题</h1>
  <p>公开的文章正文，可被AI爬虫抓取和训练...</p>

  <!-- 用户评论区域：排除AI训练 -->
  <div data-noai="true">
    <div class="comment">
      <span class="author">用户A</span>
      <p>这是用户评论，不应被用于AI训练</p>
    </div>
  </div>
</article>

<!-- Google-Extended特定控制 -->
<meta name="googlebot-news" content="noai">