AI爬虫优化
从robots.txt到渲染策略，确保AI搜索引擎爬虫能高效抓取和理解你的内容

# robots.txt AI爬虫配置

# 允许Google AI（Googlebot）
User-agent: Googlebot
Allow: /

# 允许Bing AI（Bingbot）
User-agent: Bingbot
Allow: /

# 允许百度AI（Baiduspider）
User-agent: Baiduspider
Allow: /

# 允许OpenAI GPTBot
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /private/

# 允许Anthropic ClaudeBot
User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /private/

# 允许Common Crawl（AI训练数据源）
User-agent: CCBot
Allow: /

# 禁止未知爬虫访问敏感区域
User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /api/internal/

# 站点地图位置
Sitemap: https://example.com/sitemap.xml

# AI爬虫日志分析框架
import re
from collections import Counter, defaultdict
from datetime import datetime

# AI爬虫User-Agent模式
AI_BOTS = {
    'Googlebot': re.compile(r'Googlebot'),
    'Bingbot': re.compile(r'bingbot'),
    'Baiduspider': re.compile(r'Baiduspider'),
    'GPTBot': re.compile(r'GPTBot'),
    'ClaudeBot': re.compile(r'ClaudeBot'),
    'CCBot': re.compile(r'CCBot')
}

def parse_log_line(line):
    """解析日志行"""
    # 假设nginx日志格式
    pattern = r'(\S+) \S+ \S+ \[(.+?)\] "(\S+) (\S+) \S+" (\d+) (\d+) "[^"]*" "([^"]+)"'
    match = re.match(pattern, line)
    if match:
        return {
            'ip': match.group(1),
            'time': match.group(2),
            'method': match.group(3),
            'url': match.group(4),
            'status': int(match.group(5)),
            'size': int(match.group(6)),
            'user_agent': match.group(7)
        }
    return None

def identify_bot(user_agent):
    """识别AI爬虫类型"""
    for bot_name, pattern in AI_BOTS.items():
        if pattern.search(user_agent):
            return bot_name
    return None

def analyze_crawler_stats(log_lines):
    """分析爬虫统计数据"""
    stats = defaultdict(lambda: {
        'requests': 0, 'success': 0, 'errors': 0,
        'urls_crawled': set(), 'avg_response_size': []
    })
    
    for line in log_lines:
        parsed = parse_log_line(line)
        if not parsed:
            continue
        bot = identify_bot(parsed['user_agent'])
        if bot:
            stats[bot]['requests'] += 1
            if 200 <= parsed['status'] < 300:
                stats[bot]['success'] += 1
            else:
                stats[bot]['errors'] += 1
            stats[bot]['urls_crawled'].add(parsed['url'])
            stats[bot]['avg_response_size'].append(parsed['size'])
    
    # 计算汇总
    for bot, data in stats.items():
        data['crawl_rate'] = data['success'] / data['requests'] if data['requests'] > 0 else 0
        data['unique_urls'] = len(data['urls_crawled'])
        data['avg_size'] = sum(data['avg_response_size']) / len(data['avg_response_size']) if data['avg_response_size'] else 0
    
    return dict(stats)