日志分析与诊断
从服务器日志洞察AI爬虫真实行为

# Python: AI爬虫日志分析脚本
import re
from collections import defaultdict, Counter
from datetime import datetime

AI_BOTS = {
    'GPTBot': r'GPTBot',
    'PerplexityBot': r'PerplexityBot',
    'Google-Extended': r'Google-Extended',
    'Bytespider': r'Bytespider',
    'AppleBot-Extended': r'AppleBot-Extended',
    'CCBot': r'CCBot'
}

# Nginx日志正则
LOG_PATTERN = re.compile(
    r'(\S+) \S+ \S+ \[(.+?)\] "(\S+) (\S+) \S+" '
    r'(\d+) (\d+) ".+?" "(.+?)" (\d+\.\d+)'
)

def parse_log(log_file):
    bot_stats = defaultdict(lambda: {
        'total_requests': 0,
        'status_codes': Counter(),
        'urls': Counter(),
        'response_times': [],
        'crawl_dates': Counter()
    })

    with open(log_file, 'r', encoding='utf-8') as f:
        for line in f:
            match = LOG_PATTERN.match(line)
            if not match:
                continue

            ip, timestamp, method, url, status, size, ua, resp_time = match.groups()

            # 识别AI爬虫
            for bot_name, bot_pattern in AI_BOTS.items():
                if re.search(bot_pattern, ua, re.IGNORECASE):
                    stats = bot_stats[bot_name]
                    stats['total_requests'] += 1
                    stats['status_codes'][status] += 1
                    stats['urls'][url] += 1
                    stats['response_times'].append(float(resp_time))
                    date = timestamp.split(':')[0]
                    stats['crawl_dates'][date] += 1
                    break

    return bot_stats

def report(bot_stats):
    for bot, stats in bot_stats.items():
        print(f'\n=== {bot} ===')
        print(f'总请求数: {stats["total_requests"]}')
        print(f'状态码分布: {dict(stats["status_codes"])}')
        print(f'抓取URL数: {len(stats["urls"])}')
        avg_time = sum(stats['response_times']) / len(stats['response_times'])
        print(f'平均响应时间: {avg_time:.0f}ms')
        print(f'抓取Top10 URL:')
        for url, count in stats['urls'].most_common(10):
            print(f'  {count:4d} | {url}')

# 使用: report(parse_log('/var/log/nginx/access.log'))

# 自动化告警脚本示例
import smtplib
from email.mime.text import MIMEText

def check_ai_crawler_health(bot_stats, thresholds):
    alerts = []
    for bot, stats in bot_stats.items():
        # 抓取量检查
        if stats['total_requests'] < thresholds.get('min_requests', 10):
            alerts.append(f'{bot}: 抓取量异常低 ({stats["total_requests"]}次)')

        # 错误率检查
        error_count = sum(v for k, v in stats['status_codes'].items()
                         if k.startswith('4') or k.startswith('5'))
        error_rate = error_count / max(stats['total_requests'], 1)
        if error_rate > thresholds.get('max_error_rate', 0.05):
            alerts.append(f'{bot}: 错误率过高 ({error_rate:.1%})')

        # 响应时间检查
        if stats['response_times']:
            avg_time = sum(stats['response_times']) / len(stats['response_times'])
            if avg_time > thresholds.get('max_response_time', 1000):
                alerts.append(f'{bot}: 响应时间过长 ({avg_time:.0f}ms)')

    return alerts

def send_alert(alerts, email_config):
    if not alerts:
        return
    body = '\n'.join(alerts)
    msg = MIMEText(f'AI爬虫行为异常告警:\n\n{body}')
    msg['Subject'] = '[GEO告警] AI爬虫行为异常'
    msg['From'] = email_config['from']
    msg['To'] = email_config['to']
    with smtplib.SMTP(email_config['smtp_server']) as s:
        s.send_message(msg)
    print(f'已发送{len(alerts)}条告警')