第 3 課：多模型閘道 — LLM 路由器、成本最佳化與回退

企業人工智慧聊天機器人平台架構－從原型到生產

第 1 部分：基礎與平台概述

亞洲開發網

1. 為什麼需要多模型網關？

不存在「一種適合所有人的模型」。每個模型都有自己的權衡：

型號	實力	輸入 $/1M 代幣	輸出 $/1M 代幣	延遲 (TTFT)
GPT-4o	多功能、多模式、工具使用	2.50 美元	10.00 美元	〜300毫秒
GPT-4o-迷你	快，便宜，夠好	0.15 美元	0.60 美元	〜150毫秒
克勞德 3.5 十四行詩	推理，長上下文 (200K)	$3.00	$15.00	〜400毫秒
克勞德 3.5 俳句	快速、廉價的替代方案	0.25 美元	1.25 美元	〜200毫秒
雙子座2.0快閃記憶體	1M上下文，速度快	0.10 美元	0.40 美元	〜200毫秒
Llama 3.3 70B（自架）	無API成本，資料隱私	0 美元（GPU 成本）	0 美元（GPU 成本）	〜500毫秒
米斯特拉爾大號	歐洲，GDPR 友善型	$2.00	$6.00	〜350毫秒

多模型網關允許：

• 成本最佳化 — 將簡單查詢路由到廉價模型 (4o-mini)，將複雜查詢路由到強大模型 (GPT-4o)
• 可用性 — 當提供者中斷時的回退
• 合規性 — 將敏感資料路由至自架站模型
• A/B 測試 — 在不影響生產的情況下測試新型號

2. 網關架構

┌─────────────────────────────────────────────────────────────────┐
│                    MULTI-MODEL GATEWAY                           │
├─────────────────────────────────────────────────────────────────┤
│                                                                  │
│  Request ──▶ ┌──────────┐    ┌──────────┐    ┌──────────┐       │
│              │  Router   │───▶│ Provider │───▶│ Response │       │
│              │  Engine   │    │ Adapter  │    │ Normalizer│      │
│              └────┬─────┘    └──────────┘    └──────────┘       │
│                   │                                              │
│           ┌───────┼───────┐                                      │
│           │       │       │                                      │
│     ┌─────▼──┐ ┌─▼─────┐ ┌▼────────┐                           │
│     │ Rate   │ │Token  │ │Fallback │                            │
│     │ Limiter│ │Budget │ │Chain    │                            │
│     └────────┘ └───────┘ └─────────┘                            │
│                                                                  │
│  Provider Adapters:                                              │
│  ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐ ┌────────┐       │
│  │ OpenAI │ │Anthropic│ │ Google │ │ vLLM   │ │ Mistral│       │
│  │Adapter │ │ Adapter │ │Adapter │ │Adapter │ │Adapter │       │
│  └────────┘ └────────┘ └────────┘ └────────┘ └────────┘       │
│                                                                  │
└─────────────────────────────────────────────────────────────────┘

3. 提供者抽象層

// Unified interface cho tất cả LLM providers
interface LLMProvider {
  readonly name: string;
  readonly models: ModelInfo[];

  chat(request: ChatRequest): Promise<ChatResponse>;
  chatStream(request: ChatRequest): AsyncIterable<ChatChunk>;
  countTokens(messages: Message[]): Promise<number>;
}

interface ChatRequest {
  model: string;
  messages: Message[];
  temperature?: number;
  maxTokens?: number;
  tools?: ToolDefinition[];
  responseFormat?: 'text' | 'json';
  stream?: boolean;
}

interface ChatResponse {
  id: string;
  content: string;
  toolCalls?: ToolCall[];
  usage: { inputTokens: number; outputTokens: number };
  model: string;
  finishReason: 'stop' | 'tool_calls' | 'length' | 'content_filter';
  latencyMs: number;
}

interface ChatChunk {
  delta: string;
  toolCallDelta?: Partial<ToolCall>;
  finishReason?: string;
}

interface ModelInfo {
  id: string;
  provider: string;
  maxContextTokens: number;
  inputPricePerMToken: number;
  outputPricePerMToken: number;
  supportsTools: boolean;
  supportsVision: boolean;
  supportsStreaming: boolean;
}

4.OpenAI 提供者適配器

import OpenAI from 'openai';

class OpenAIProvider implements LLMProvider {
  readonly name = 'openai';
  readonly models: ModelInfo[] = [
    {
      id: 'gpt-4o', provider: 'openai',
      maxContextTokens: 128_000,
      inputPricePerMToken: 2.5, outputPricePerMToken: 10,
      supportsTools: true, supportsVision: true, supportsStreaming: true,
    },
    {
      id: 'gpt-4o-mini', provider: 'openai',
      maxContextTokens: 128_000,
      inputPricePerMToken: 0.15, outputPricePerMToken: 0.6,
      supportsTools: true, supportsVision: true, supportsStreaming: true,
    },
  ];

  private client: OpenAI;

  constructor(apiKey: string) {
    this.client = new OpenAI({ apiKey });
  }

  async chat(request: ChatRequest): Promise<ChatResponse> {
    const startTime = performance.now();

    const response = await this.client.chat.completions.create({
      model: request.model,
      messages: this.mapMessages(request.messages),
      temperature: request.temperature ?? 0.7,
      max_tokens: request.maxTokens,
      tools: request.tools?.map(this.mapTool),
      response_format: request.responseFormat === 'json'
        ? { type: 'json_object' } : undefined,
    });

    const choice = response.choices[0];
    return {
      id: response.id,
      content: choice.message.content ?? '',
      toolCalls: choice.message.tool_calls?.map(tc => ({
        id: tc.id,
        name: tc.function.name,
        arguments: JSON.parse(tc.function.arguments),
      })),
      usage: {
        inputTokens: response.usage?.prompt_tokens ?? 0,
        outputTokens: response.usage?.completion_tokens ?? 0,
      },
      model: response.model,
      finishReason: this.mapFinishReason(choice.finish_reason),
      latencyMs: Math.round(performance.now() - startTime),
    };
  }

  async *chatStream(request: ChatRequest): AsyncIterable<ChatChunk> {
    const stream = await this.client.chat.completions.create({
      model: request.model,
      messages: this.mapMessages(request.messages),
      temperature: request.temperature ?? 0.7,
      max_tokens: request.maxTokens,
      tools: request.tools?.map(this.mapTool),
      stream: true,
    });

    for await (const chunk of stream) {
      const delta = chunk.choices[0]?.delta;
      yield {
        delta: delta?.content ?? '',
        toolCallDelta: delta?.tool_calls?.[0] ? {
          id: delta.tool_calls[0].id,
          name: delta.tool_calls[0].function?.name,
          arguments: delta.tool_calls[0].function?.arguments
            ? JSON.parse(delta.tool_calls[0].function.arguments) : undefined,
        } : undefined,
        finishReason: chunk.choices[0]?.finish_reason ?? undefined,
      };
    }
  }

  private mapMessages(messages: Message[]): OpenAI.ChatCompletionMessageParam[] {
    return messages.map(m => ({ role: m.role, content: m.content }));
  }

  private mapTool(tool: ToolDefinition): OpenAI.ChatCompletionTool {
    return {
      type: 'function',
      function: {
        name: tool.name,
        description: tool.description,
        parameters: tool.parameters,
      },
    };
  }

  private mapFinishReason(reason: string | null): ChatResponse['finishReason'] {
    const map: Record<string, ChatResponse['finishReason']> = {
      stop: 'stop', tool_calls: 'tool_calls',
      length: 'length', content_filter: 'content_filter',
    };
    return map[reason ?? 'stop'] ?? 'stop';
  }
}

5. 路由器引擎－模型選擇策略

interface RoutingConfig {
  strategy: 'cost-optimized' | 'quality-first' | 'latency-first' | 'custom';
  rules: RoutingRule[];
  fallbackChain: string[];   // Ordered list of model IDs
  costBudget?: {
    maxCostPerRequest: number;   // USD
    dailyBudget: number;
    monthlyBudget: number;
  };
}

interface RoutingRule {
  condition: RoutingCondition;
  model: string;
  priority: number;
}

type RoutingCondition =
  | { type: 'complexity'; threshold: 'simple' | 'medium' | 'complex' }
  | { type: 'topic'; topics: string[] }
  | { type: 'language'; languages: string[] }
  | { type: 'tokenCount'; maxTokens: number }
  | { type: 'requiresTool'; value: boolean }
  | { type: 'requiresVision'; value: boolean }
  | { type: 'sensitive'; value: boolean };

class ModelRouter {
  constructor(
    private providers: Map<string, LLMProvider>,
    private config: RoutingConfig,
    private complexityClassifier: ComplexityClassifier,
    private costTracker: CostTracker,
  ) {}

  async selectModel(request: ChatRequest, context: RoutingContext): Promise<string> {
    // 1. Evaluate explicit rules
    for (const rule of this.config.rules.sort((a, b) => a.priority - b.priority)) {
      if (await this.evaluateCondition(rule.condition, request, context)) {
        if (this.isModelAvailable(rule.model)) {
          return rule.model;
        }
      }
    }

    // 2. Strategy-based selection
    switch (this.config.strategy) {
      case 'cost-optimized':
        return this.selectCheapestCapable(request, context);
      case 'quality-first':
        return this.selectStrongest(request, context);
      case 'latency-first':
        return this.selectFastest(request, context);
      default:
        return this.config.fallbackChain[0];
    }
  }

  private async selectCheapestCapable(
    request: ChatRequest,
    context: RoutingContext,
  ): Promise<string> {
    const complexity = await this.complexityClassifier.classify(request);

    // Simple queries → cheapest model
    if (complexity === 'simple') {
      return 'gpt-4o-mini'; // $0.15/1M input
    }

    // Complex reasoning → best model within budget
    if (this.config.costBudget) {
      const budget = await this.costTracker.getRemainingBudget(context.tenantId);
      if (budget.daily < 1.0) { // Low budget — force cheap model
        return 'gpt-4o-mini';
      }
    }

    return 'gpt-4o';
  }

  private isModelAvailable(modelId: string): boolean {
    const provider = this.getProviderForModel(modelId);
    return provider !== undefined && !this.isProviderDown(provider.name);
  }
}

6.查詢複雜度分類器

// Classify query complexity to route to appropriate model
class ComplexityClassifier {
  private readonly SIMPLE_PATTERNS = [
    /^(hi|hello|hey|xin chào|chào)/i,
    /^(thanks|thank you|cảm ơn)/i,
    /^(yes|no|ok|được|không)/i,
  ];

  async classify(request: ChatRequest): Promise<'simple' | 'medium' | 'complex'> {
    const lastMessage = request.messages[request.messages.length - 1];
    const content = lastMessage.content;

    // Rule-based fast path
    if (this.SIMPLE_PATTERNS.some(p => p.test(content))) return 'simple';
    if (content.length < 20) return 'simple';

    // Feature-based classification
    const features = {
      messageLength: content.length,
      wordCount: content.split(/\s+/).length,
      hasCode: /```|`[^`]+`/.test(content),
      hasNumbers: /\d{3,}/.test(content),
      questionMarks: (content.match(/\?/g) || []).length,
      requiresTool: request.tools && request.tools.length > 0,
      conversationLength: request.messages.length,
      hasMultipleSteps: /bước|step|đầu tiên|thứ hai|sau đó|tiếp theo/i.test(content),
    };

    // Scoring
    let score = 0;
    if (features.messageLength > 200) score += 2;
    if (features.wordCount > 50) score += 2;
    if (features.hasCode) score += 3;
    if (features.questionMarks > 2) score += 2;
    if (features.requiresTool) score += 3;
    if (features.conversationLength > 10) score += 1;
    if (features.hasMultipleSteps) score += 2;

    if (score <= 2) return 'simple';
    if (score <= 6) return 'medium';
    return 'complex';
  }
}

7. 後備鍊和斷路器

class FallbackChain {
  private circuitBreakers: Map<string, CircuitBreaker> = new Map();

  constructor(private providers: Map<string, LLMProvider>) {
    for (const [name] of providers) {
      this.circuitBreakers.set(name, new CircuitBreaker({
        failureThreshold: 5,
        resetTimeoutMs: 60_000, // 1 minute
        halfOpenRequests: 2,
      }));
    }
  }

  async executeWithFallback(
    request: ChatRequest,
    modelChain: string[],
  ): Promise<ChatResponse> {
    const errors: Array<{ model: string; error: Error }> = [];

    for (const modelId of modelChain) {
      const provider = this.getProvider(modelId);
      const breaker = this.circuitBreakers.get(provider.name)!;

      if (breaker.state === 'open') {
        errors.push({ model: modelId, error: new Error('Circuit breaker open') });
        continue;
      }

      try {
        const response = await breaker.execute(() =>
          provider.chat({ ...request, model: modelId }),
        );
        return response;
      } catch (error) {
        errors.push({ model: modelId, error: error as Error });
        // Continue to next model in chain
      }
    }

    throw new AllProvidersFailedError(errors);
  }
}

class CircuitBreaker {
  state: 'closed' | 'open' | 'half-open' = 'closed';
  private failureCount = 0;
  private lastFailureTime = 0;
  private halfOpenSuccesses = 0;

  constructor(private config: {
    failureThreshold: number;
    resetTimeoutMs: number;
    halfOpenRequests: number;
  }) {}

  async execute<T>(fn: () => Promise<T>): Promise<T> {
    if (this.state === 'open') {
      if (Date.now() - this.lastFailureTime > this.config.resetTimeoutMs) {
        this.state = 'half-open';
        this.halfOpenSuccesses = 0;
      } else {
        throw new Error('Circuit breaker is open');
      }
    }

    try {
      const result = await fn();
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      throw error;
    }
  }

  private onSuccess(): void {
    if (this.state === 'half-open') {
      this.halfOpenSuccesses++;
      if (this.halfOpenSuccesses >= this.config.halfOpenRequests) {
        this.state = 'closed';
        this.failureCount = 0;
      }
    } else {
      this.failureCount = 0;
    }
  }

  private onFailure(): void {
    this.failureCount++;
    this.lastFailureTime = Date.now();
    if (this.failureCount >= this.config.failureThreshold) {
      this.state = 'open';
    }
  }
}

8. 代幣預算管理

interface TokenBudget {
  tenantId: string;
  dailyLimit: number;
  monthlyLimit: number;
  dailyUsed: number;
  monthlyUsed: number;
}

class CostTracker {
  constructor(private redis: Redis, private db: Database) {}

  async trackUsage(
    tenantId: string,
    model: string,
    usage: { inputTokens: number; outputTokens: number },
  ): Promise<void> {
    const modelInfo = this.getModelInfo(model);
    const cost =
      (usage.inputTokens / 1_000_000) * modelInfo.inputPricePerMToken +
      (usage.outputTokens / 1_000_000) * modelInfo.outputPricePerMToken;

    const today = new Date().toISOString().split('T')[0];
    const month = today.substring(0, 7);

    // Atomic increment in Redis for real-time tracking
    await this.redis
      .multi()
      .incrbyfloat(`budget:${tenantId}:daily:${today}`, cost)
      .incrbyfloat(`budget:${tenantId}:monthly:${month}`, cost)
      .incrby(`tokens:${tenantId}:daily:${today}`, usage.inputTokens + usage.outputTokens)
      .expire(`budget:${tenantId}:daily:${today}`, 86400 * 2)
      .expire(`budget:${tenantId}:monthly:${month}`, 86400 * 35)
      .exec();

    // Async write to DB for persistence
    await this.db.usageLog.create({
      tenantId, model, cost,
      inputTokens: usage.inputTokens,
      outputTokens: usage.outputTokens,
      timestamp: new Date(),
    });
  }

  async getRemainingBudget(tenantId: string): Promise<{ daily: number; monthly: number }> {
    const tenant = await this.db.tenant.findById(tenantId);
    const today = new Date().toISOString().split('T')[0];
    const month = today.substring(0, 7);

    const dailyUsed = parseFloat(
      await this.redis.get(`budget:${tenantId}:daily:${today}`) ?? '0'
    );
    const monthlyUsed = parseFloat(
      await this.redis.get(`budget:${tenantId}:monthly:${month}`) ?? '0'
    );

    return {
      daily: (tenant.config.dailyBudget ?? Infinity) - dailyUsed,
      monthly: (tenant.config.monthlyBudget ?? Infinity) - monthlyUsed,
    };
  }

  async checkBudget(tenantId: string): Promise<{ allowed: boolean; reason?: string }> {
    const remaining = await this.getRemainingBudget(tenantId);
    if (remaining.daily <= 0) return { allowed: false, reason: 'Daily budget exceeded' };
    if (remaining.monthly <= 0) return { allowed: false, reason: 'Monthly budget exceeded' };
    return { allowed: true };
  }
}

9. 每個提供者的速率限制

// Sliding window rate limiter per provider
class ProviderRateLimiter {
  private limits: Map<string, { rpm: number; tpm: number }> = new Map([
    ['openai', { rpm: 5000, tpm: 800_000 }],
    ['anthropic', { rpm: 4000, tpm: 400_000 }],
    ['google', { rpm: 6000, tpm: 1_000_000 }],
  ]);

  constructor(private redis: Redis) {}

  async checkAndConsume(
    provider: string,
    estimatedTokens: number,
  ): Promise<{ allowed: boolean; retryAfterMs?: number }> {
    const limits = this.limits.get(provider);
    if (!limits) return { allowed: true };

    const now = Date.now();
    const windowKey = `ratelimit:${provider}`;

    // Sliding window counter with Redis
    const [requestCount, tokenCount] = await this.redis
      .multi()
      .zcount(`${windowKey}:rpm`, now - 60_000, now)
      .get(`${windowKey}:tpm:${Math.floor(now / 60_000)}`)
      .exec() as [number, string | null];

    if (requestCount >= limits.rpm) {
      return { allowed: false, retryAfterMs: 60_000 };
    }

    const currentTokens = parseInt(tokenCount ?? '0', 10);
    if (currentTokens + estimatedTokens > limits.tpm) {
      return { allowed: false, retryAfterMs: 60_000 - (now % 60_000) };
    }

    // Consume
    await this.redis
      .multi()
      .zadd(`${windowKey}:rpm`, now, `${now}-${Math.random()}`)
      .zremrangebyscore(`${windowKey}:rpm`, 0, now - 60_000)
      .incrby(`${windowKey}:tpm:${Math.floor(now / 60_000)}`, estimatedTokens)
      .expire(`${windowKey}:tpm:${Math.floor(now / 60_000)}`, 120)
      .exec();

    return { allowed: true };
  }
}

10. 綜合起來－網關服務

class MultiModelGateway {
  constructor(
    private router: ModelRouter,
    private fallbackChain: FallbackChain,
    private rateLimiter: ProviderRateLimiter,
    private costTracker: CostTracker,
    private eventBus: EventBus,
  ) {}

  async chat(request: ChatRequest, context: RoutingContext): Promise<ChatResponse> {
    // 1. Check budget
    const budget = await this.costTracker.checkBudget(context.tenantId);
    if (!budget.allowed) {
      throw new BudgetExceededError(budget.reason!);
    }

    // 2. Select model
    const primaryModel = await this.router.selectModel(request, context);

    // 3. Build fallback chain
    const chain = [primaryModel, ...this.router.getFallbacks(primaryModel)];

    // 4. Execute with fallback
    const response = await this.fallbackChain.executeWithFallback(request, chain);

    // 5. Track usage & cost
    await this.costTracker.trackUsage(context.tenantId, response.model, response.usage);

    // 6. Emit events
    await this.eventBus.publish('ai-engine', {
      type: 'response.generated',
      payload: {
        sessionId: context.sessionId,
        tokensUsed: response.usage.inputTokens + response.usage.outputTokens,
        latencyMs: response.latencyMs,
        model: response.model,
        tenantId: context.tenantId,
      },
    });

    return response;
  }
}

第 3 課總結

組件	責任
提供者適配器	標準化 OpenAI/Claude/Gemini/vLLM 之間的 API 差異
路由器引擎	根據複雜性、主題、成本、功能選擇模型
複雜度分類器	將查詢分類（簡單/中型/複雜）以優化成本
斷路器	偵測提供者故障，自動切換到後備
代幣預算	追蹤並執行每個租戶的每日/每月支出限制
速率限制器	遵守提供者速率限制 (RPM/TPM) 以避免 429 錯誤

下一篇： 對話管理－會話生命週期、情境視窗最佳化（滑動視窗、摘要、壓縮）、短期與長期記憶架構。