AI爬虫监控告警
爬虫活动实时监控系统搭建

# AI爬虫User-Agent识别规则
import re

AI_CRAWLER_PATTERNS = {
    'GPTBot': r'GPTBot/\d+',
    'PerplexityBot': r'PerplexityBot/\d+',
    'Google-Extended': r'Google-Extended',
    'Bytespider': r'Bytespider',
    'AppleBot-Extended': r'Applebot-Extended/\d+',
    'CCBot': r'CCBot/\d+',
    'ClaudeBot': r'ClaudeBot/\d+',
    'Amazonbot': r'Amazonbot/\d+'
}

def identify_ai_crawler(user_agent):
    '''识别AI爬虫类型'''
    for name, pattern in AI_CRAWLER_PATTERNS.items():
        if re.search(pattern, user_agent, re.IGNORECASE):
            return name
    return None

# Nginx日志解析示例
def parse_nginx_log(log_line):
    '''解析Nginx access log，提取爬虫信息'''
    # Nginx默认日志格式: IP - - [time] "method path protocol" status size "referer" "user_agent"
    pattern = r'(\S+) \S+ \S+ \[([^\]]+)\] "(\S+) (\S+) \S+" (\d+) (\d+) "[^"]*" "([^"]*)"'
    match = re.match(pattern, log_line)
    if match:
        ip, timestamp, method, path, status, size, ua = match.groups()
        crawler = identify_ai_crawler(ua)
        if crawler:
            return {
                'ip': ip,
                'timestamp': timestamp,
                'method': method,
                'path': path,
                'status': int(status),
                'crawler': crawler,
                'user_agent': ua
            }
    return None

# Prometheus告警规则配置

# alert_rules.yml
alert_rules = '''
groups:
  - name: ai_crawler_alerts
    rules:
      # Critical: AI爬虫完全停止抓取
      - alert: AICrawlerStopped
        expr: rate(ai_crawler_requests_total[6h]) == 0
        for: 6h
        labels:
          severity: critical
        annotations:
          summary: 'AI爬虫停止抓取超过6小时'
          description: '{{ $labels.crawler }} 已6小时无抓取记录，请检查robots.txt和服务器状态'

      # Warning: 抓取量下降50%
      - alert: AICrawlerDrop50
        expr: rate(ai_crawler_requests_total[24h]) < rate(ai_crawler_requests_total[24h] offset 1d) * 0.5
        for: 24h
        labels:
          severity: warning
        annotations:
          summary: 'AI爬虫抓取量下降超过50%'
          description: '{{ $labels.crawler }} 抓取量较昨日下降超过50%'

      # Warning: 抓取成功率低
      - alert: AICrawlerLowSuccess
        expr: ai_crawler_success_rate < 0.9
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: 'AI爬虫抓取成功率低于90%'
          description: '{{ $labels.crawler }} 成功率仅 {{ $value }}%，检查服务器响应'
'''

print(alert_rules)

# AI爬虫异常自动诊断脚本
import subprocess
import requests
from datetime import datetime, timedelta

def diagnose_crawler_anomaly(crawler_name, anomaly_type):
    '''AI爬虫异常自动诊断'''
    diagnosis = []

    # 1. 检查robots.txt
    robots = requests.get('https://your-domain.com/robots.txt').text
    if 'Disallow: /' in robots and 'User-agent: *' in robots:
        diagnosis.append({
            'check': 'robots.txt',
            'status': 'CRITICAL',
            'finding': 'robots.txt屏蔽了所有爬虫',
            'fix': '修改robots.txt允许AI爬虫访问'
        })

    # 2. 检查服务器响应
    try:
        resp = requests.get('https://your-domain.com', timeout=10)
        if resp.status_code != 200:
            diagnosis.append({
                'check': 'server_response',
                'status': 'CRITICAL',
                'finding': f'服务器返回 {resp.status_code}',
                'fix': '检查服务器状态和Web应用配置'
            })
        elif resp.elapsed.total_seconds() > 3:
            diagnosis.append({
                'check': 'server_response',
                'status': 'WARNING',
                'finding': f'服务器响应时间 {resp.elapsed.total_seconds():.1f}s',
                'fix': '优化服务器性能，AI爬虫可能因超时放弃抓取'
            })
    except Exception as e:
        diagnosis.append({
            'check': 'server_response',
            'status': 'CRITICAL',
            'finding': f'服务器无法访问: {e}',
            'fix': '检查服务器和网络状态'
        })

    # 3. 检查sitemap可访问性
    try:
        sm = requests.get('https://your-domain.com/sitemap.xml', timeout=10)
        if sm.status_code != 200:
            diagnosis.append({
                'check': 'sitemap',
                'status': 'WARNING',
                'finding': f'sitemap.xml返回 {sm.status_code}',
                'fix': '修复sitemap.xml可访问性'
            })
    except:
        diagnosis.append({
            'check': 'sitemap',
            'status': 'WARNING',
            'finding': 'sitemap.xml无法访问',
            'fix': '检查sitemap.xml是否存在且可访问'
        })

    # 输出诊断报告
    print(f'\n=== AI爬虫异常诊断报告 ===')
    print(f'爬虫: {crawler_name} | 异常类型: {anomaly_type}')
    print(f'诊断时间: {datetime.now()}')
    for d in diagnosis:
        print(f'  [{d["status"]}] {d["check"]}: {d["finding"]}')
        print(f'         修复建议: {d["fix"]}')

    return diagnosis