src/index.ts

import express, { Request, Response, NextFunction } from 'express';
import { requestLogger, errorHandler, AppError, MissingMaterialError } from './middlewares';
import { callLLM } from './llm';
import {
  CaseContext,
  PeerRegSummary,
  GapAnalysis,
  ComplianceCheck,
  LegalReviewPack,
  MissingMaterialItem,
  LogAnalysisResult,
  CodeSecurityResult,
  RegulatoryPenaltySummary
} from './models';
import crawlerRouter from './crawler/api';
import { initDB } from './crawler/db';
import { initScheduler } from './crawler/scheduler';

const app = express();
app.use(express.json());

// Initialize crawler database
initDB();

// Initialize crawler scheduler
initScheduler();

// 挂载日志中间件
app.use(requestLogger);

const PORT = process.env.PORT || 3000;

// 挂载抓取子系统路由
app.use('/api/crawler', crawlerRouter);

// API-01 同业/监管变化摘要
app.post('/api/agents/peer-reg-summary', async (req: Request, res: Response, next: NextFunction) => {
  try {
    const { peer_updates, regulatory_updates, business_line, app_name } = req.body;

    const missingItems: MissingMaterialItem[] = [];
    if (!peer_updates) missingItems.push({ material_type: 'peer_updates', reason: '缺少同业动态', requirement_level: 'required' });
    if (!regulatory_updates) missingItems.push({ material_type: 'regulatory_updates', reason: '缺少监管动态', requirement_level: 'required' });

    if (missingItems.length > 0) {
      throw new MissingMaterialError(missingItems);
    }

    const startLLM = Date.now();
    const response = await callLLM<PeerRegSummary>(JSON.stringify(req.body), 'PeerRegSummary');
    res.locals.llm_latency_ms = Date.now() - startLLM;
    
    res.json({
      ...response,
      schema_version: 'v1.0'
    });
  } catch (error) {
    next(error);
  }
});

// API-02 协议重构
app.post('/api/agents/policy-rewrite', async (req: Request, res: Response, next: NextFunction) => {
  try {
    const { case_context, peer_reg_summary } = req.body;

    const missingItems: MissingMaterialItem[] = [];
    if (!case_context || !case_context.materials) {
      missingItems.push({ material_type: 'case_context.materials', reason: '缺少用例材料上下文', requirement_level: 'required' });
    } else {
      const { current_policy_text, prd_text, permission_items, sdk_items } = case_context.materials;
      if (!current_policy_text) missingItems.push({ material_type: 'current_policy_text', reason: '缺少当前协议文本', requirement_level: 'required' });
      if (!prd_text) missingItems.push({ material_type: 'prd_text', reason: '缺少PRD文本', requirement_level: 'required' });
      if (!permission_items) missingItems.push({ material_type: 'permission_items', reason: '缺少权限清单', requirement_level: 'required' });
      if (!sdk_items) missingItems.push({ material_type: 'sdk_items', reason: '缺少SDK清单', requirement_level: 'required' });
    }

    if (missingItems.length > 0) {
      throw new MissingMaterialError(missingItems);
    }

    if (!peer_reg_summary) {
      throw new Error('MISSING_CONTEXT');
    }

    const startLLM = Date.now();
    const response = await callLLM<GapAnalysis>(JSON.stringify(req.body), 'GapAnalysis');
    res.locals.llm_latency_ms = Date.now() - startLLM;

    res.json({
      ...response,
      schema_version: 'v1.0'
    });
  } catch (error) {
    next(error);
  }
});

// API-03 合规校验
app.post('/api/agents/compliance-check', async (req: Request, res: Response, next: NextFunction) => {
  try {
    const { case_context, gap_analysis } = req.body;

    const missingItems: MissingMaterialItem[] = [];
    if (!case_context || !case_context.materials) {
      missingItems.push({ material_type: 'case_context.materials', reason: '缺少用例材料上下文', requirement_level: 'required' });
    }

    if (missingItems.length > 0) {
      throw new MissingMaterialError(missingItems);
    }

    if (!gap_analysis) {
      throw new Error('MISSING_CONTEXT');
    }

    const startLLM = Date.now();
    const response = await callLLM<ComplianceCheck>(JSON.stringify(req.body), 'ComplianceCheck');
    res.locals.llm_latency_ms = Date.now() - startLLM;

    res.json({
      ...response,
      schema_version: 'v1.0'
    });
  } catch (error) {
    next(error);
  }
});

// API-04 法务审核包生成
app.post('/api/agents/legal-pack', async (req: Request, res: Response, next: NextFunction) => {
  try {
    const { case_context, peer_reg_summary, gap_analysis, compliance_check } = req.body;

    const missingItems: MissingMaterialItem[] = [];
    if (!case_context) missingItems.push({ material_type: 'case_context', reason: '缺少用例材料上下文', requirement_level: 'required' });
    if (!peer_reg_summary) missingItems.push({ material_type: 'peer_reg_summary', reason: '缺少同业/监管变化摘要', requirement_level: 'required' });
    if (!gap_analysis) missingItems.push({ material_type: 'gap_analysis', reason: '缺少协议重构结果', requirement_level: 'required' });
    if (!compliance_check) missingItems.push({ material_type: 'compliance_check', reason: '缺少合规校验结果', requirement_level: 'required' });

    if (missingItems.length > 0) {
      throw new MissingMaterialError(missingItems);
    }

    const startLLM = Date.now();
    const response = await callLLM<LegalReviewPack>(JSON.stringify(req.body), 'LegalReviewPack');
    res.locals.llm_latency_ms = Date.now() - startLLM;

    res.json({
      ...response,
      schema_version: 'v1.0'
    });
  } catch (error) {
    next(error);
  }
});

function normString(value: unknown): string {
  if (value === null || value === undefined) return '';
  if (Array.isArray(value)) return value.map(v => String(v)).join('\n').trim();
  if (typeof value !== 'string') return String(value).trim();
  return value.trim();
}

function maskSecrets(text: string): string {
  let s = text;
  s = s.replace(/sk-[A-Za-z0-9_-]{8,}/g, 'sk-***');
  s = s.replace(/(?<![A-Za-z0-9])[A-Fa-f0-9]{32,}(?![A-Za-z0-9])/g, '***');
  s = s.replace(/(?<![A-Za-z0-9])[A-Za-z0-9_\\-]{24,}(?![A-Za-z0-9])/g, '***');
  return s;
}

function parseLogPatterns(logText: string) {
  const lines = logText.split(/\r?\n/).map(l => l.trim()).filter(Boolean);
  const map = new Map<string, { count: number; first?: string; last?: string; samples: string[]; level: 'high' | 'medium' | 'low' }>();

  const timeRe = /(\d{2}:\d{2}:\d{2}|\d{2}:\d{2})/;
  for (const line of lines) {
    const timeMatch = line.match(timeRe);
    const time = timeMatch ? timeMatch[1] : '';
    const upper = line.toUpperCase();
    const level: 'high' | 'medium' | 'low' =
      upper.includes('ERROR') || upper.includes('EXCEPTION') || upper.includes('FATAL') || upper.includes('TIMEOUT') ? 'high' :
      upper.includes('WARN') ? 'medium' : 'low';

    const normalized = line
      .replace(/^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?\s*/g, '')
      .replace(/^\[?\d{2}:\d{2}:\d{2}\]?\s*/g, '')
      .replace(/\b0x[0-9a-fA-F]+\b/g, '0x#')
      .replace(/\b\d+\b/g, '#')
      .replace(/\s+/g, ' ')
      .slice(0, 220);

    const key = normalized || line.slice(0, 220);
    const existing = map.get(key);
    if (!existing) {
      const record: { count: number; first?: string; last?: string; samples: string[]; level: 'high' | 'medium' | 'low' } = {
        count: 1,
        samples: [maskSecrets(line).slice(0, 320)],
        level
      };
      if (time) {
        record.first = time;
        record.last = time;
      }
      map.set(key, record);
    } else {
      existing.count += 1;
      if (time) {
        if (!existing.first) existing.first = time;
        existing.last = time;
      }
      if (existing.samples.length < 3) existing.samples.push(maskSecrets(line).slice(0, 320));
      if (level === 'high' || (level === 'medium' && existing.level === 'low')) existing.level = level;
    }
  }

  const patterns = Array.from(map.entries())
    .sort((a, b) => b[1].count - a[1].count)
    .slice(0, 12)
    .map(([pattern, v]) => ({
      pattern,
      count: v.count,
      first_seen: v.first || '',
      last_seen: v.last || '',
      level: v.level,
      sample_logs: v.samples
    }));

  const timeline = patterns
    .filter(p => p.first_seen)
    .slice(0, 8)
    .map(p => ({ time: p.first_seen, event: `首次出现：${p.pattern}` }));

  return { patterns, timeline, line_count: lines.length };
}

function parseScanFindings(scanText: string) {
  const s = scanText.trim();
  if (!s) return [];
  let obj: any;
  try {
    obj = JSON.parse(s);
  } catch {
    return [];
  }

  const findings: any[] = [];

  const semgrepResults = Array.isArray(obj?.results) ? obj.results : [];
  for (const r of semgrepResults) {
    const file = r?.path || r?.extra?.path || '';
    const line = Number(r?.start?.line || r?.extra?.line || 0);
    const ruleId = r?.check_id || r?.rule_id || r?.extra?.metadata?.id || '';
    const severityRaw = String(r?.extra?.severity || r?.extra?.metadata?.severity || r?.severity || 'medium').toLowerCase();
    const severity =
      severityRaw.includes('critical') ? 'critical' :
      severityRaw.includes('high') || severityRaw.includes('error') ? 'high' :
      severityRaw.includes('low') ? 'low' :
      severityRaw.includes('info') ? 'info' :
      'medium';
    const message = r?.extra?.message || r?.extra?.metadata?.message || r?.message || '';
    const evidence = r?.extra?.lines || r?.extra?.metavars ? JSON.stringify({ lines: r?.extra?.lines, metavars: r?.extra?.metavars }) : '';
    findings.push({
      tool: 'semgrep',
      type: 'static_analysis',
      severity,
      status: 'confirmed',
      file: String(file),
      line,
      rule_id: String(ruleId),
      description: String(message),
      evidence: maskSecrets(String(evidence || message || ''))
    });
  }

  const gitleaksResults = Array.isArray(obj) ? obj : Array.isArray(obj?.leaks) ? obj.leaks : [];
  for (const r of gitleaksResults) {
    const ruleId = r?.RuleID || r?.rule_id || '';
    const file = r?.File || r?.file || '';
    const line = Number(r?.StartLine || r?.line || 0);
    const desc = r?.Description || r?.description || r?.Message || r?.message || 'Possible secret';
    const secret = r?.Secret || r?.secret || '';
    const evidence = secret ? maskSecrets(String(secret)) : maskSecrets(JSON.stringify(r).slice(0, 240));
    if (ruleId || file) {
      findings.push({
        tool: 'gitleaks',
        type: 'secret',
        severity: 'high',
        status: 'confirmed',
        file: String(file),
        line,
        rule_id: String(ruleId || 'gitleaks'),
        description: String(desc),
        evidence
      });
    }
  }

  return findings.slice(0, 80);
}

const AGENT_VERSION = 'v1.0.0';
const PROMPT_VERSION = 'v1.0.0';

const handleLogAnalysis = async (req: Request, res: Response, next: NextFunction) => {
  try {
    const system_name = normString(req.body?.system_name);
    const environment = normString(req.body?.environment);
    const time_range = normString(req.body?.time_range);
    const log_text = normString(req.body?.log_text || req.body?.log_content || req.body?.logs);
    const known_changes = req.body?.known_changes ?? req.body?.change_notes ?? '';
    const business_impact = normString(req.body?.business_impact);
    const log_format = normString(req.body?.log_format);
    const knowledge_context = normString(req.body?.knowledge_context || req.body?.kb || req.body?.knowledge_base);

    const missingItems: MissingMaterialItem[] = [];
    if (!system_name) missingItems.push({ material_type: 'system_name', reason: '缺少系统名称', requirement_level: 'required' });
    if (!environment) missingItems.push({ material_type: 'environment', reason: '缺少环境信息', requirement_level: 'required' });
    if (!time_range) missingItems.push({ material_type: 'time_range', reason: '缺少时间范围', requirement_level: 'required' });
    if (!log_text) missingItems.push({ material_type: 'log_text', reason: '缺少日志文本', requirement_level: 'required' });

    if (missingItems.length > 0) {
      throw new MissingMaterialError(missingItems);
    }

    const parsed = parseLogPatterns(log_text);
    const payload = {
      system_name,
      environment,
      time_range,
      business_impact,
      log_format,
      parsed,
      known_changes,
      knowledge_context
    };

    const startLLM = Date.now();
    const response = await callLLM<LogAnalysisResult>(JSON.stringify(payload), 'LogAnalysisResult');
    res.locals.llm_latency_ms = Date.now() - startLLM;

    res.json({
      ...response,
      schema_version: 'v1.0',
      agent_version: AGENT_VERSION,
      prompt_version: PROMPT_VERSION,
      model_version: process.env.LLM_MODEL_NAME || 'deepseek-chat'
    });
  } catch (error) {
    next(error);
  }
};

const handleCodeSecurity = async (req: Request, res: Response, next: NextFunction) => {
  try {
    const repo_name = normString(req.body?.repo_name);
    const language = normString(req.body?.language);
    const branch = normString(req.body?.branch);
    const security_standard = normString(req.body?.security_standard);
    const review_mode = normString(req.body?.review_mode);
    const scan_result_text = normString(req.body?.scan_result_text || req.body?.scan_results || req.body?.scan_result);
    const code_text = normString(req.body?.code_text || req.body?.code_snippet || req.body?.code);
    const knowledge_context = normString(req.body?.knowledge_context || req.body?.kb || req.body?.knowledge_base);

    const missingItems: MissingMaterialItem[] = [];
    if (!repo_name) missingItems.push({ material_type: 'repo_name', reason: '缺少仓库名称', requirement_level: 'required' });
    if (!language) missingItems.push({ material_type: 'language', reason: '缺少代码语言', requirement_level: 'required' });
    if (!security_standard) missingItems.push({ material_type: 'security_standard', reason: '缺少安全标准', requirement_level: 'required' });
    if (!review_mode) missingItems.push({ material_type: 'review_mode', reason: '缺少审查模式', requirement_level: 'required' });

    if (missingItems.length > 0) {
      throw new MissingMaterialError(missingItems);
    }

    const findings = parseScanFindings(scan_result_text);
    const hasScan = findings.length > 0;

    if (!hasScan && !code_text) {
      throw new MissingMaterialError([{ material_type: 'scan_result_text/code_text', reason: '缺少扫描结果或代码内容', requirement_level: 'required' }]);
    }

    const payload = {
      repo_name,
      language,
      branch,
      security_standard,
      review_mode,
      findings,
      code_text: maskSecrets(code_text).slice(0, 8000),
      knowledge_context,
      has_scan_results: hasScan
    };

    const startLLM = Date.now();
    const response = await callLLM<CodeSecurityResult>(JSON.stringify(payload), 'CodeSecurityResult');
    res.locals.llm_latency_ms = Date.now() - startLLM;

    res.json({
      ...response,
      schema_version: 'v1.0',
      agent_version: AGENT_VERSION,
      prompt_version: PROMPT_VERSION,
      model_version: process.env.LLM_MODEL_NAME || 'deepseek-chat'
    });
  } catch (error) {
    next(error);
  }
};

const handleRegPenaltySummary = async (req: Request, res: Response, next: NextFunction) => {
  try {
    const topic = normString(req.body?.topic);
    const industry = normString(req.body?.industry);
    const time_range = normString(req.body?.time_range);
    const regulators = req.body?.regulators;
    const keywords = req.body?.keywords;
    const report_type = normString(req.body?.report_type);
    const cases = Array.isArray(req.body?.cases) ? req.body.cases : [];
    const knowledge_context = normString(req.body?.knowledge_context || req.body?.kb || req.body?.knowledge_base);

    const missingItems: MissingMaterialItem[] = [];
    if (!topic) missingItems.push({ material_type: 'topic', reason: '缺少主题', requirement_level: 'required' });
    if (!industry) missingItems.push({ material_type: 'industry', reason: '缺少行业', requirement_level: 'required' });
    if (!time_range) missingItems.push({ material_type: 'time_range', reason: '缺少时间范围', requirement_level: 'required' });
    if (!regulators || (Array.isArray(regulators) && regulators.length === 0)) {
      missingItems.push({ material_type: 'regulators', reason: '缺少监管机构', requirement_level: 'required' });
    }
    if (!report_type) missingItems.push({ material_type: 'report_type', reason: '缺少报告类型', requirement_level: 'required' });

    if (missingItems.length > 0) {
      throw new MissingMaterialError(missingItems);
    }

    const normalizedCases = cases
      .map((c: any) => ({
        case_name: normString(c?.case_name || c?.title || ''),
        regulator: normString(c?.regulator || ''),
        date: normString(c?.date || c?.publish_date || ''),
        source_url: normString(c?.source_url || c?.url || ''),
        excerpt: normString(c?.excerpt || c?.summary || '')
      }))
      .filter((c: any) => c.case_name && c.regulator && c.date && c.source_url && c.excerpt);

    const payload = {
      topic,
      industry,
      time_range,
      regulators,
      keywords,
      report_type,
      cases: normalizedCases.slice(0, 80),
      knowledge_context
    };

    const startLLM = Date.now();
    const response = await callLLM<RegulatoryPenaltySummary>(JSON.stringify(payload), 'RegulatoryPenaltySummary');
    res.locals.llm_latency_ms = Date.now() - startLLM;

    res.json({
      ...response,
      schema_version: 'v1.0',
      agent_version: AGENT_VERSION,
      prompt_version: PROMPT_VERSION,
      model_version: process.env.LLM_MODEL_NAME || 'deepseek-chat'
    });
  } catch (error) {
    next(error);
  }
};

app.post('/api/agents/log-analysis', handleLogAnalysis);
app.post('/api/agents/log-analysis/run', handleLogAnalysis);

app.post('/api/agents/code-security', handleCodeSecurity);
app.post('/api/agents/code-security/run', handleCodeSecurity);

app.post('/api/agents/regulatory-penalty-summary', handleRegPenaltySummary);
app.post('/api/agents/regulatory-penalty-summary/run', handleRegPenaltySummary);

// 挂载全局错误处理中间件
app.use(errorHandler);

if (require.main === module) {
  app.listen(PORT, () => {
    console.log(`Agent Services API is running on http://localhost:${PORT}`);
  });
}

export default app;