AI-Native Frontend Architecture Explained

┌─────────────────────────────────────────────────────────────────────────┐
│                         AI-Native Frontend                               │
├─────────────────────────────────────────────────────────────────────────┤
│  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐    │
│  │   Chat UI   │  │  Copilot    │  │  AI Forms   │  │ Generative  │    │
│  │  Component  │  │  Overlay    │  │  Assistant  │  │     UI      │    │
│  └──────┬──────┘  └──────┬──────┘  └──────┬──────┘  └──────┬──────┘    │
│         │                │                │                │            │
│  ┌──────┴────────────────┴────────────────┴────────────────┴──────┐    │
│  │                    AI Interaction Layer                         │    │
│  │  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐             │    │
│  │  │  Streaming  │  │   Context   │  │   Action    │             │    │
│  │  │   Manager   │  │   Manager   │  │   Parser    │             │    │
│  │  └─────────────┘  └─────────────┘  └─────────────┘             │    │
│  └────────────────────────────┬───────────────────────────────────┘    │
│                               │                                         │
│  ┌────────────────────────────┴───────────────────────────────────┐    │
│  │                     State Management Layer                      │    │
│  │  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐             │    │
│  │  │ Conversation│  │   Message   │  │  Optimistic │             │    │
│  │  │    Store    │  │    Queue    │  │   Updates   │             │    │
│  │  └─────────────┘  └─────────────┘  └─────────────┘             │    │
│  └────────────────────────────┬───────────────────────────────────┘    │
│                               │                                         │
│  ┌────────────────────────────┴───────────────────────────────────┐    │
│  │                    Transport Layer                              │    │
│  │  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐             │    │
│  │  │     SSE     │  │  WebSocket  │  │    HTTP     │             │    │
│  │  │   Client    │  │   Client    │  │   Client    │             │    │
│  │  └─────────────┘  └─────────────┘  └─────────────┘             │    │
│  └────────────────────────────────────────────────────────────────┘    │
└─────────────────────────────────────────────────────────────────────────┘
                                    │
                    ┌───────────────┼───────────────┐
                    ▼               ▼               ▼
            ┌───────────┐   ┌───────────┐   ┌───────────┐
            │    BFF    │   │    API    │   │   Edge    │
            │  (AI API) │   │  Gateway  │   │  Cache    │
            └─────┬─────┘   └─────┬─────┘   └───────────┘
                  │               │
                  ▼               ▼
            ┌───────────────────────────────────────────┐
            │           AI Service Layer                │
            │  ┌─────────┐  ┌─────────┐  ┌─────────┐   │
            │  │  LLM    │  │  RAG    │  │ Vector  │   │
            │  │ Gateway │  │ Service │  │   DB    │   │
            │  └─────────┘  └─────────┘  └─────────┘   │
            └───────────────────────────────────────────┘

User Input → Optimistic UI Update → Context Assembly →
Stream Initiation → Token Processing → Incremental Render →
Action Parsing → Side Effect Execution → Final State Commit

// Core AI client abstraction
interface AIClient {
  stream(
    messages: Message[],
    options: StreamOptions
  ): AsyncIterable<StreamChunk>;

  complete(
    messages: Message[],
    options: CompleteOptions
  ): Promise<CompletionResult>;

  abort(requestId: string): void;
}

interface StreamChunk {
  type: 'token' | 'tool_call' | 'tool_result' | 'done' | 'error';
  content?: string;
  toolCall?: ToolCall;
  usage?: TokenUsage;
  finishReason?: FinishReason;
}

// Streaming manager with backpressure handling
class StreamingManager {
  private activeStreams = new Map<string, StreamController>();
  private tokenBuffer = new Map<string, string[]>();
  private flushInterval = 16; // ~60fps render rate

  async processStream(
    streamId: string,
    stream: AsyncIterable<StreamChunk>,
    handlers: StreamHandlers
  ): Promise<void> {
    const controller = new StreamController();
    this.activeStreams.set(streamId, controller);

    // Buffer tokens for batch rendering
    this.tokenBuffer.set(streamId, []);

    // Flush buffer at consistent intervals for smooth rendering
    const flushTimer = setInterval(() => {
      this.flushTokenBuffer(streamId, handlers);
    }, this.flushInterval);

    try {
      for await (const chunk of stream) {
        if (controller.aborted) break;

        switch (chunk.type) {
          case 'token':
            this.tokenBuffer.get(streamId)!.push(chunk.content!);
            break;

          case 'tool_call':
            // Flush pending tokens before tool call
            this.flushTokenBuffer(streamId, handlers);
            await handlers.onToolCall(chunk.toolCall!);
            break;

          case 'done':
            this.flushTokenBuffer(streamId, handlers);
            handlers.onComplete(chunk.usage!);
            break;

          case 'error':
            handlers.onError(chunk);
            break;
        }
      }
    } finally {
      clearInterval(flushTimer);
      this.flushTokenBuffer(streamId, handlers);
      this.activeStreams.delete(streamId);
      this.tokenBuffer.delete(streamId);
    }
  }

  private flushTokenBuffer(streamId: string, handlers: StreamHandlers): void {
    const buffer = this.tokenBuffer.get(streamId);
    if (buffer && buffer.length > 0) {
      const content = buffer.join('');
      buffer.length = 0;
      handlers.onTokens(content);
    }
  }

  abort(streamId: string): void {
    this.activeStreams.get(streamId)?.abort();
  }
}

interface ContextManager {
  // Build context for current request
  buildContext(sessionId: string): Promise<ContextPayload>;

  // Add new message to context
  addMessage(sessionId: string, message: Message): Promise<void>;

  // Compress context when approaching limits
  compressContext(sessionId: string): Promise<void>;

  // Get current token usage
  getTokenUsage(sessionId: string): TokenUsage;
}

class SlidingWindowContextManager implements ContextManager {
  private tokenizer: Tokenizer;
  private maxContextTokens: number;
  private systemPromptTokens: number;
  private reservedResponseTokens: number;

  constructor(config: ContextConfig) {
    this.tokenizer = new Tiktoken(config.model);
    this.maxContextTokens = config.maxContextTokens;
    this.systemPromptTokens = this.tokenizer.count(config.systemPrompt);
    this.reservedResponseTokens = config.reservedResponseTokens;
  }

  async buildContext(sessionId: string): Promise<ContextPayload> {
    const session = await this.loadSession(sessionId);
    const availableTokens = this.maxContextTokens
      - this.systemPromptTokens
      - this.reservedResponseTokens;

    // Strategy: Keep recent messages, summarize older ones
    const messages = session.messages;
    const recentMessages: Message[] = [];
    let usedTokens = 0;

    // Walk backwards from most recent
    for (let i = messages.length - 1; i >= 0; i--) {
      const message = messages[i];
      const messageTokens = this.tokenizer.count(
        this.formatMessage(message)
      );

      if (usedTokens + messageTokens > availableTokens * 0.7) {
        // 70% for recent, 30% for summary
        break;
      }

      recentMessages.unshift(message);
      usedTokens += messageTokens;
    }

    // Summarize older messages if any
    const olderMessages = messages.slice(
      0,
      messages.length - recentMessages.length
    );

    let summary: Message | null = null;
    if (olderMessages.length > 0) {
      summary = await this.generateSummary(
        olderMessages,
        availableTokens - usedTokens
      );
    }

    return {
      systemPrompt: session.systemPrompt,
      summary,
      messages: recentMessages,
      tokenUsage: {
        used: usedTokens + (summary ? this.tokenizer.count(summary.content) : 0),
        available: availableTokens,
        total: this.maxContextTokens
      }
    };
  }

  private async generateSummary(
    messages: Message[],
    maxTokens: number
  ): Promise<Message> {
    // Use a smaller model for summarization
    const summaryPrompt = `Summarize this conversation concisely,
      preserving key facts, decisions, and context needed for
      continuation. Max ${maxTokens} tokens.`;

    const summary = await this.summaryClient.complete({
      messages: [
        { role: 'system', content: summaryPrompt },
        ...messages
      ],
      maxTokens
    });

    return {
      role: 'system',
      content: `[Previous conversation summary]: ${summary.content}`,
      metadata: { type: 'summary', originalMessageCount: messages.length }
    };
  }
}

// Zustand store for AI conversation state
interface AIConversationState {
  sessions: Map<string, ConversationSession>;
  activeSessionId: string | null;

  // Streaming state
  streamingMessages: Map<string, StreamingMessage>;

  // Actions
  createSession: (config: SessionConfig) => string;
  sendMessage: (sessionId: string, content: string) => Promise<void>;
  cancelStream: (messageId: string) => void;
  retryMessage: (messageId: string) => Promise<void>;

  // Optimistic updates
  pendingMessages: Map<string, PendingMessage>;
}

interface StreamingMessage {
  id: string;
  sessionId: string;
  content: string;
  tokens: number;
  startTime: number;
  status: 'streaming' | 'parsing' | 'executing' | 'complete' | 'error';
  toolCalls: ToolCall[];
}

const useAIConversationStore = create<AIConversationState>((set, get) => ({
  sessions: new Map(),
  activeSessionId: null,
  streamingMessages: new Map(),
  pendingMessages: new Map(),

  sendMessage: async (sessionId, content) => {
    const messageId = generateId();
    const userMessage: Message = {
      id: messageId,
      role: 'user',
      content,
      timestamp: Date.now()
    };

    // Optimistic update - show user message immediately
    set(state => {
      const session = state.sessions.get(sessionId)!;
      session.messages.push(userMessage);

      // Create placeholder for assistant response
      const assistantId = generateId();
      state.streamingMessages.set(assistantId, {
        id: assistantId,
        sessionId,
        content: '',
        tokens: 0,
        startTime: Date.now(),
        status: 'streaming',
        toolCalls: []
      });

      return { ...state };
    });

    // Build context and stream response
    const context = await contextManager.buildContext(sessionId);
    const stream = aiClient.stream([...context.messages, userMessage], {
      sessionId,
      maxTokens: context.tokenUsage.available
    });

    await streamingManager.processStream(
      messageId,
      stream,
      {
        onTokens: (tokens) => {
          set(state => {
            const streaming = state.streamingMessages.get(messageId);
            if (streaming) {
              streaming.content += tokens;
              streaming.tokens += tokens.length; // Approximate
            }
            return { ...state };
          });
        },
        onToolCall: async (toolCall) => {
          set(state => {
            const streaming = state.streamingMessages.get(messageId);
            if (streaming) {
              streaming.status = 'executing';
              streaming.toolCalls.push(toolCall);
            }
            return { ...state };
          });

          // Execute tool and inject result
          const result = await executeToolCall(toolCall);
          return result;
        },
        onComplete: (usage) => {
          set(state => {
            const streaming = state.streamingMessages.get(messageId);
            if (streaming) {
              // Convert streaming message to permanent message
              const session = state.sessions.get(sessionId)!;
              session.messages.push({
                id: messageId,
                role: 'assistant',
                content: streaming.content,
                toolCalls: streaming.toolCalls,
                usage,
                timestamp: Date.now()
              });
              state.streamingMessages.delete(messageId);
            }
            return { ...state };
          });
        },
        onError: (error) => {
          set(state => {
            const streaming = state.streamingMessages.get(messageId);
            if (streaming) {
              streaming.status = 'error';
            }
            return { ...state };
          });
        }
      }
    );
  }
}));

// High-performance token renderer
class TokenRenderer {
  private pendingTokens: string[] = [];
  private rafId: number | null = null;
  private container: HTMLElement;
  private lastRenderTime = 0;
  private minRenderInterval = 16; // 60fps cap

  constructor(container: HTMLElement) {
    this.container = container;
  }

  appendTokens(tokens: string): void {
    this.pendingTokens.push(tokens);
    this.scheduleRender();
  }

  private scheduleRender(): void {
    if (this.rafId !== null) return;

    this.rafId = requestAnimationFrame((timestamp) => {
      this.rafId = null;

      // Throttle renders to prevent jank
      if (timestamp - this.lastRenderTime < this.minRenderInterval) {
        this.scheduleRender();
        return;
      }

      this.render();
      this.lastRenderTime = timestamp;
    });
  }

  private render(): void {
    if (this.pendingTokens.length === 0) return;

    // Batch all pending tokens into single DOM update
    const content = this.pendingTokens.join('');
    this.pendingTokens = [];

    // Use insertAdjacentText for minimal reflow
    const textNode = this.container.lastChild;
    if (textNode && textNode.nodeType === Node.TEXT_NODE) {
      textNode.textContent += content;
    } else {
      this.container.insertAdjacentText('beforeend', content);
    }

    // Scroll to bottom with smooth behavior
    this.container.scrollTop = this.container.scrollHeight;
  }
}

// React hook for streaming content
function useStreamingContent(streamId: string | null) {
  const containerRef = useRef<HTMLDivElement>(null);
  const rendererRef = useRef<TokenRenderer | null>(null);
  const content = useAIConversationStore(
    state => streamId ? state.streamingMessages.get(streamId)?.content : null
  );
  const prevContentRef = useRef('');

  useEffect(() => {
    if (!containerRef.current) return;
    rendererRef.current = new TokenRenderer(containerRef.current);
  }, []);

  useEffect(() => {
    if (!content || !rendererRef.current) return;

    // Only append new tokens
    const newTokens = content.slice(prevContentRef.current.length);
    if (newTokens) {
      rendererRef.current.appendTokens(newTokens);
      prevContentRef.current = content;
    }
  }, [content]);

  return containerRef;
}

class AIMemoryManager {
  private maxCachedSessions = 10;
  private maxMessagesPerSession = 1000;
  private sessionCache = new LRUCache<string, ConversationSession>({
    max: this.maxCachedSessions,
    dispose: (session) => this.persistSession(session)
  });

  // Prune old messages when approaching limits
  pruneSession(sessionId: string): void {
    const session = this.sessionCache.get(sessionId);
    if (!session) return;

    if (session.messages.length > this.maxMessagesPerSession) {
      // Keep system messages and recent messages
      const systemMessages = session.messages.filter(
        m => m.metadata?.type === 'system'
      );
      const recentMessages = session.messages.slice(-this.maxMessagesPerSession / 2);

      // Generate summary of pruned messages
      const prunedMessages = session.messages.slice(
        systemMessages.length,
        -this.maxMessagesPerSession / 2
      );

      if (prunedMessages.length > 0) {
        this.queueSummaryGeneration(sessionId, prunedMessages);
      }

      session.messages = [...systemMessages, ...recentMessages];
    }
  }

  // Monitor memory pressure
  startMemoryMonitoring(): void {
    if ('memory' in performance) {
      setInterval(() => {
        const memory = (performance as any).memory;
        const usedRatio = memory.usedJSHeapSize / memory.jsHeapSizeLimit;

        if (usedRatio > 0.8) {
          this.aggressivePrune();
        }
      }, 30000);
    }
  }

  private aggressivePrune(): void {
    // Clear oldest sessions from cache
    const sessions = Array.from(this.sessionCache.entries());
    sessions
      .sort((a, b) => a[1].lastAccessTime - b[1].lastAccessTime)
      .slice(0, Math.floor(sessions.length / 2))
      .forEach(([key]) => this.sessionCache.delete(key));
  }
}

// Interaction to Next Paint (INP) optimization
class INPOptimizer {
  // Break up long tasks during streaming
  async processStreamWithYielding(
    stream: AsyncIterable<StreamChunk>,
    handler: (chunk: StreamChunk) => void
  ): Promise<void> {
    let chunkCount = 0;

    for await (const chunk of stream) {
      handler(chunk);
      chunkCount++;

      // Yield to main thread every 10 chunks
      if (chunkCount % 10 === 0) {
        await this.yieldToMainThread();
      }
    }
  }

  private yieldToMainThread(): Promise<void> {
    return new Promise(resolve => {
      if ('scheduler' in globalThis && 'yield' in (globalThis as any).scheduler) {
        (globalThis as any).scheduler.yield().then(resolve);
      } else {
        setTimeout(resolve, 0);
      }
    });
  }
}

// Largest Contentful Paint (LCP) - prioritize initial UI
function AIChat() {
  const [initialized, setInitialized] = useState(false);

  useEffect(() => {
    // Defer heavy initialization
    requestIdleCallback(() => {
      initializeAIClient();
      loadConversationHistory();
      setInitialized(true);
    });
  }, []);

  return (
    <div>
      {/* LCP element - render immediately */}
      <ChatHeader />
      <MessageList skeleton={!initialized} />

      {/* Defer input until ready */}
      {initialized && <ChatInput />}
    </div>
  );
}

┌─────────────────────────────────────────────────────────────────┐
│                   Transport Selection Matrix                     │
├─────────────────┬───────────┬───────────┬───────────┬──────────┤
│ Requirement     │    SSE    │ WebSocket │  HTTP/2   │  gRPC    │
├─────────────────┼───────────┼───────────┼───────────┼──────────┤
│ Token streaming │    ★★★    │    ★★★    │    ★★     │   ★★★    │
│ Bidirectional   │     ★     │    ★★★    │    ★★     │   ★★★    │
│ Browser support │    ★★★    │    ★★★    │    ★★     │    ★     │
│ Proxy compat    │    ★★★    │    ★★     │    ★★★    │    ★     │
│ Reconnection    │    ★★★    │    ★★     │    ★★★    │   ★★     │
│ Memory overhead │    ★★★    │    ★★     │    ★★★    │   ★★     │
└─────────────────┴───────────┴───────────┴───────────┴──────────┘

class AIStreamClient {
  private eventSource: EventSource | null = null;
  private reconnectAttempts = 0;
  private maxReconnectAttempts = 5;

  async stream(
    endpoint: string,
    payload: StreamRequest,
    handlers: StreamHandlers
  ): Promise<void> {
    return new Promise((resolve, reject) => {
      // POST request to initiate stream, get stream ID
      const streamId = await this.initiateStream(endpoint, payload);

      // Connect to SSE endpoint with stream ID
      const url = `${endpoint}/stream/${streamId}`;
      this.eventSource = new EventSource(url);

      this.eventSource.onmessage = (event) => {
        const chunk = JSON.parse(event.data) as StreamChunk;

        switch (chunk.type) {
          case 'token':
            handlers.onTokens(chunk.content!);
            break;
          case 'tool_call':
            handlers.onToolCall(chunk.toolCall!);
            break;
          case 'done':
            handlers.onComplete(chunk.usage!);
            this.eventSource?.close();
            resolve();
            break;
          case 'error':
            handlers.onError(chunk);
            this.eventSource?.close();
            reject(new Error(chunk.content));
            break;
        }
      };

      this.eventSource.onerror = (error) => {
        if (this.reconnectAttempts < this.maxReconnectAttempts) {
          this.reconnectAttempts++;
          this.handleReconnection(streamId, handlers);
        } else {
          handlers.onError({ type: 'error', content: 'Connection failed' });
          reject(error);
        }
      };
    });
  }

  private async handleReconnection(
    streamId: string,
    handlers: StreamHandlers
  ): Promise<void> {
    // Resume from last received position
    const lastPosition = await this.getLastPosition(streamId);
    const url = `${this.endpoint}/stream/${streamId}?from=${lastPosition}`;

    // Exponential backoff
    const delay = Math.min(1000 * Math.pow(2, this.reconnectAttempts), 30000);
    await new Promise(r => setTimeout(r, delay));

    this.eventSource = new EventSource(url);
    // ... reconnection logic
  }
}

class ConcurrentStreamManager {
  private activeStreams = new Map<string, ActiveStream>();
  private maxConcurrentStreams = 3;
  private pendingQueue: QueuedStream[] = [];

  async requestStream(
    sessionId: string,
    request: StreamRequest
  ): Promise<StreamHandle> {
    // Check if we can start immediately
    if (this.activeStreams.size < this.maxConcurrentStreams) {
      return this.startStream(sessionId, request);
    }

    // Queue the request
    return new Promise((resolve, reject) => {
      this.pendingQueue.push({
        sessionId,
        request,
        resolve,
        reject,
        queuedAt: Date.now()
      });
    });
  }

  private async startStream(
    sessionId: string,
    request: StreamRequest
  ): Promise<StreamHandle> {
    const stream: ActiveStream = {
      sessionId,
      request,
      startedAt: Date.now(),
      controller: new AbortController()
    };

    this.activeStreams.set(sessionId, stream);

    // Start the actual stream
    const handle = await this.aiClient.stream(request, {
      signal: stream.controller.signal,
      onComplete: () => this.onStreamComplete(sessionId)
    });

    return handle;
  }

  private onStreamComplete(sessionId: string): void {
    this.activeStreams.delete(sessionId);

    // Process next in queue
    if (this.pendingQueue.length > 0) {
      const next = this.pendingQueue.shift()!;
      this.startStream(next.sessionId, next.request)
        .then(next.resolve)
        .catch(next.reject);
    }
  }

  // Cancel stream to make room for higher priority
  async prioritize(sessionId: string): Promise<void> {
    if (this.activeStreams.size >= this.maxConcurrentStreams) {
      // Find lowest priority active stream
      const streams = Array.from(this.activeStreams.values());
      const oldest = streams.sort(
        (a, b) => a.startedAt - b.startedAt
      )[0];

      // Cancel it
      oldest.controller.abort();
      this.activeStreams.delete(oldest.sessionId);
    }

    // Move requested stream to front of queue or start immediately
    const queueIndex = this.pendingQueue.findIndex(
      q => q.sessionId === sessionId
    );

    if (queueIndex > -1) {
      const [queued] = this.pendingQueue.splice(queueIndex, 1);
      const handle = await this.startStream(queued.sessionId, queued.request);
      queued.resolve(handle);
    }
  }
}

// BFF routes for AI operations
const aiRouter = express.Router();

// Stream completion with context management
aiRouter.post('/chat/:sessionId/stream', async (req, res) => {
  const { sessionId } = req.params;
  const { message, context } = req.body;

  // Set up SSE
  res.setHeader('Content-Type', 'text/event-stream');
  res.setHeader('Cache-Control', 'no-cache');
  res.setHeader('Connection', 'keep-alive');

  try {
    // Load and build context server-side
    const session = await sessionService.getSession(sessionId);
    const fullContext = await contextBuilder.build(session, message, context);

    // Validate token limits
    const tokenCount = await tokenizer.count(fullContext);
    if (tokenCount > session.maxContextTokens) {
      await contextBuilder.compress(fullContext, session.maxContextTokens);
    }

    // Stream from LLM
    const stream = await llmClient.stream({
      messages: fullContext.messages,
      systemPrompt: fullContext.systemPrompt,
      tools: session.enabledTools,
      maxTokens: session.maxResponseTokens
    });

    for await (const chunk of stream) {
      // Process tool calls server-side for security
      if (chunk.type === 'tool_call') {
        const result = await toolExecutor.execute(
          chunk.toolCall,
          session.permissions
        );

        res.write(`data: ${JSON.stringify({
          type: 'tool_result',
          toolCall: chunk.toolCall,
          result
        })}\n\n`);
        continue;
      }

      res.write(`data: ${JSON.stringify(chunk)}\n\n`);
    }

    res.write('data: {"type":"done"}\n\n');
    res.end();

  } catch (error) {
    res.write(`data: ${JSON.stringify({
      type: 'error',
      message: error.message
    })}\n\n`);
    res.end();
  }
});

// Non-streaming operations
aiRouter.post('/chat/:sessionId/complete', async (req, res) => {
  // For operations that don't need streaming
  // (e.g., summarization, classification)
});

aiRouter.get('/chat/:sessionId/context', async (req, res) => {
  // Return current context state for debugging/display
  const session = await sessionService.getSession(req.params.sessionId);
  const context = await contextBuilder.getState(session);

  res.json({
    messageCount: context.messages.length,
    tokenUsage: context.tokenUsage,
    summary: context.summary,
    tools: context.enabledTools
  });
});

// Semantic caching for AI responses
class SemanticCache {
  private vectorStore: VectorStore;
  private responseCache: Map<string, CachedResponse>;
  private similarityThreshold = 0.95;

  async get(
    query: string,
    context: CacheContext
  ): Promise<CachedResponse | null> {
    // Generate embedding for query
    const embedding = await this.embedder.embed(query);

    // Search for similar cached queries
    const results = await this.vectorStore.search(embedding, {
      filter: {
        sessionId: context.sessionId,
        timestamp: { $gt: Date.now() - context.maxAge }
      },
      limit: 5
    });

    // Find exact or near-exact match
    for (const result of results) {
      if (result.score >= this.similarityThreshold) {
        const cached = this.responseCache.get(result.id);
        if (cached && this.validateContext(cached.context, context)) {
          return cached;
        }
      }
    }

    return null;
  }

  async set(
    query: string,
    response: string,
    context: CacheContext
  ): Promise<void> {
    const embedding = await this.embedder.embed(query);
    const id = generateId();

    await this.vectorStore.insert({
      id,
      embedding,
      metadata: {
        sessionId: context.sessionId,
        timestamp: Date.now()
      }
    });

    this.responseCache.set(id, {
      query,
      response,
      context,
      createdAt: Date.now()
    });
  }

  private validateContext(
    cached: CacheContext,
    current: CacheContext
  ): boolean {
    // Ensure context hasn't changed significantly
    return cached.messageCount === current.messageCount &&
           cached.systemPromptHash === current.systemPromptHash;
  }
}

class PromptSecurityLayer {
  private patterns: RegExp[];
  private maxInputLength = 10000;

  constructor() {
    this.patterns = [
      /ignore\s+(previous|all|above)\s+instructions/i,
      /system\s*:\s*/i,
      /\[INST\]/i,
      /<\|im_start\|>/i,
      /```system/i,
      /\bDAN\b/i,
      /jailbreak/i
    ];
  }

  sanitize(input: string): SanitizationResult {
    const warnings: SecurityWarning[] = [];

    // Length check
    if (input.length > this.maxInputLength) {
      return {
        safe: false,
        sanitized: null,
        warnings: [{ type: 'length', message: 'Input exceeds maximum length' }]
      };
    }

    // Pattern matching
    for (const pattern of this.patterns) {
      if (pattern.test(input)) {
        warnings.push({
          type: 'prompt_injection',
          message: `Suspicious pattern detected: ${pattern.source}`,
          severity: 'high'
        });
      }
    }

    // Output filtering markers
    const sanitized = input
      .replace(/\[SYSTEM\]/gi, '[FILTERED]')
      .replace(/\[ASSISTANT\]/gi, '[FILTERED]')
      .replace(/<\|.*?\|>/g, '');

    return {
      safe: warnings.filter(w => w.severity === 'high').length === 0,
      sanitized,
      warnings
    };
  }

  // Validate AI output before rendering
  validateOutput(output: string): ValidationResult {
    const issues: OutputIssue[] = [];

    // Check for leaked system prompts
    if (output.includes('[SYSTEM]') || output.includes('system prompt')) {
      issues.push({
        type: 'system_leak',
        message: 'Potential system prompt leakage detected'
      });
    }

    // Check for code execution attempts
    const codePatterns = [
      /javascript:/i,
      /data:text\/html/i,
      /<script/i,
      /on\w+\s*=/i
    ];

    for (const pattern of codePatterns) {
      if (pattern.test(output)) {
        issues.push({
          type: 'xss_attempt',
          message: 'Potential XSS in AI output'
        });
      }
    }

    return {
      safe: issues.length === 0,
      issues,
      sanitizedOutput: this.sanitizeOutput(output)
    };
  }

  private sanitizeOutput(output: string): string {
    // Use DOMPurify or similar for HTML content
    return DOMPurify.sanitize(output, {
      ALLOWED_TAGS: ['p', 'br', 'strong', 'em', 'code', 'pre', 'ul', 'ol', 'li'],
      ALLOWED_ATTR: ['class']
    });
  }
}

class AIRateLimiter {
  private tokenBudgets = new Map<string, TokenBudget>();
  private requestCounts = new Map<string, RequestWindow>();

  async checkLimits(
    userId: string,
    estimatedTokens: number
  ): Promise<LimitCheckResult> {
    const budget = await this.getTokenBudget(userId);
    const requests = this.getRequestWindow(userId);

    // Check token budget
    if (budget.used + estimatedTokens > budget.limit) {
      return {
        allowed: false,
        reason: 'token_budget_exceeded',
        resetAt: budget.resetAt,
        remaining: budget.limit - budget.used
      };
    }

    // Check request rate
    if (requests.count >= requests.limit) {
      return {
        allowed: false,
        reason: 'rate_limit_exceeded',
        resetAt: requests.resetAt,
        remaining: 0
      };
    }

    return { allowed: true, remaining: budget.limit - budget.used };
  }

  async recordUsage(
    userId: string,
    usage: TokenUsage
  ): Promise<void> {
    const budget = await this.getTokenBudget(userId);
    budget.used += usage.totalTokens;

    const requests = this.getRequestWindow(userId);
    requests.count++;

    // Persist for billing
    await this.usageStore.record({
      userId,
      timestamp: Date.now(),
      promptTokens: usage.promptTokens,
      completionTokens: usage.completionTokens,
      model: usage.model
    });
  }
}

// Problem: 100K+ token contexts become common
// Solution: Hierarchical context management

class HierarchicalContextManager {
  private levels = {
    immediate: 4000,    // Most recent messages
    session: 16000,     // Current session context
    persistent: 32000,  // Cross-session memory
    archive: Infinity   // Compressed long-term storage
  };

  async buildContext(userId: string, sessionId: string): Promise<Context> {
    // Layer 1: Immediate context (always included)
    const immediate = await this.getImmediateContext(sessionId);

    // Layer 2: Session context (summarized if needed)
    const session = await this.getSessionContext(sessionId, {
      maxTokens: this.levels.session - immediate.tokens
    });

    // Layer 3: Persistent memory (key facts, preferences)
    const persistent = await this.getPersistentMemory(userId, {
      maxTokens: this.levels.persistent - session.tokens - immediate.tokens
    });

    return this.merge(immediate, session, persistent);
  }

  // Automatic archival of old contexts
  async archiveSession(sessionId: string): Promise<void> {
    const session = await this.loadFullSession(sessionId);

    // Extract key information
    const summary = await this.generateSummary(session);
    const facts = await this.extractFacts(session);
    const preferences = await this.extractPreferences(session);

    // Store in persistent memory
    await this.persistentStore.merge({
      summary,
      facts,
      preferences,
      sessionId,
      archivedAt: Date.now()
    });

    // Delete detailed session data
    await this.sessionStore.archive(sessionId);
  }
}

// Problem: 500K concurrent streams overwhelm traditional architectures
// Solution: Edge-based stream fanout

class EdgeStreamFanout {
  private edgeConnections = new Map<string, EdgeConnection>();

  async initializeStream(
    sessionId: string,
    request: StreamRequest
  ): Promise<StreamHandle> {
    // Create stream on origin
    const originStream = await this.originClient.createStream(request);

    // Register with edge for fanout
    const edgeConfig = await this.edgeRouter.getOptimalEdge(sessionId);

    await edgeConfig.edge.registerStream({
      streamId: originStream.id,
      origin: originStream.endpoint,
      sessionId,
      // Edge handles reconnection, buffering
      bufferSize: 1000,
      reconnectPolicy: 'aggressive'
    });

    return {
      id: originStream.id,
      endpoint: edgeConfig.endpoint, // Client connects to edge
      connectionId: edgeConfig.connectionId
    };
  }
}

// Edge worker (Cloudflare Workers / Vercel Edge)
export default {
  async fetch(request: Request, env: Env): Promise<Response> {
    const url = new URL(request.url);

    if (url.pathname.startsWith('/stream/')) {
      const streamId = url.pathname.split('/')[2];

      // Create edge-to-client SSE connection
      const { readable, writable } = new TransformStream();
      const writer = writable.getWriter();

      // Connect to origin stream
      const originStream = await connectToOrigin(streamId, env);

      // Forward with buffering
      (async () => {
        for await (const chunk of originStream) {
          await writer.write(
            new TextEncoder().encode(`data: ${JSON.stringify(chunk)}\n\n`)
          );
        }
        writer.close();
      })();

      return new Response(readable, {
        headers: {
          'Content-Type': 'text/event-stream',
          'Cache-Control': 'no-cache'
        }
      });
    }
  }
};

// Problem: Accurate token counting requires expensive tokenization
// Solution: Tiered counting strategy

class TieredTokenCounter {
  private exactTokenizer: Tokenizer;
  private approximateRatio = 0.75; // chars to tokens

  // Use approximation for UI display
  approximateCount(text: string): number {
    return Math.ceil(text.length * this.approximateRatio);
  }

  // Use exact count for API calls (cached)
  async exactCount(text: string): Promise<number> {
    const cacheKey = hashString(text);

    const cached = await this.cache.get(cacheKey);
    if (cached !== null) return cached;

    const count = this.exactTokenizer.encode(text).length;
    await this.cache.set(cacheKey, count, { ttl: 3600 });

    return count;
  }

  // Incremental counting for streaming
  createIncrementalCounter(): IncrementalCounter {
    return new IncrementalCounter(this.exactTokenizer);
  }
}

class IncrementalCounter {
  private buffer = '';
  private confirmedTokens = 0;

  addChunk(chunk: string): number {
    this.buffer += chunk;

    // Only tokenize complete words (space-delimited)
    const lastSpace = this.buffer.lastIndexOf(' ');
    if (lastSpace > 0) {
      const toTokenize = this.buffer.slice(0, lastSpace);
      this.buffer = this.buffer.slice(lastSpace + 1);
      this.confirmedTokens += this.tokenizer.encode(toTokenize).length;
    }

    // Return estimate including buffer
    return this.confirmedTokens + Math.ceil(this.buffer.length * 0.75);
  }
}

interface AIMetrics {
  // Latency metrics
  timeToFirstToken: Histogram;
  tokensPerSecond: Histogram;
  totalResponseTime: Histogram;

  // Quality metrics
  streamCompletionRate: Counter;
  errorRate: Counter;
  retryRate: Counter;

  // Usage metrics
  tokensConsumed: Counter;
  activeStreams: Gauge;
  queueDepth: Gauge;

  // Context metrics
  contextUtilization: Histogram;
  contextCompressionRate: Counter;
}

class AIObservability {
  private metrics: AIMetrics;
  private tracer: Tracer;

  async traceStream(
    sessionId: string,
    stream: AsyncIterable<StreamChunk>
  ): AsyncIterable<StreamChunk> {
    const span = this.tracer.startSpan('ai.stream', {
      attributes: { sessionId }
    });

    const startTime = performance.now();
    let firstTokenTime: number | null = null;
    let tokenCount = 0;

    try {
      for await (const chunk of stream) {
        if (chunk.type === 'token' && !firstTokenTime) {
          firstTokenTime = performance.now();
          this.metrics.timeToFirstToken.record(firstTokenTime - startTime);
          span.addEvent('first_token');
        }

        if (chunk.type === 'token') {
          tokenCount++;
        }

        yield chunk;
      }

      const totalTime = performance.now() - startTime;
      this.metrics.totalResponseTime.record(totalTime);
      this.metrics.tokensPerSecond.record(tokenCount / (totalTime / 1000));
      this.metrics.streamCompletionRate.add(1, { status: 'success' });

      span.setStatus({ code: SpanStatusCode.OK });

    } catch (error) {
      this.metrics.streamCompletionRate.add(1, { status: 'error' });
      this.metrics.errorRate.add(1);

      span.setStatus({
        code: SpanStatusCode.ERROR,
        message: error.message
      });
      span.recordException(error);

      throw error;
    } finally {
      span.end();
    }
  }

  // Session replay for debugging AI interactions
  async recordSession(sessionId: string): SessionRecorder {
    return new SessionRecorder({
      sessionId,
      captureContext: true,
      captureTokens: true,
      captureToolCalls: true,
      storage: this.replayStorage
    });
  }
}

// Key AI metrics to monitor
const aiDashboard = {
  panels: [
    {
      title: 'Stream Health',
      metrics: [
        'ai_time_to_first_token_p50',
        'ai_time_to_first_token_p99',
        'ai_stream_completion_rate',
        'ai_error_rate'
      ]
    },
    {
      title: 'Throughput',
      metrics: [
        'ai_tokens_per_second_avg',
        'ai_active_streams',
        'ai_queue_depth',
        'ai_requests_per_second'
      ]
    },
    {
      title: 'Token Economics',
      metrics: [
        'ai_tokens_consumed_total',
        'ai_context_utilization_avg',
        'ai_compression_rate',
        'ai_cache_hit_rate'
      ]
    },
    {
      title: 'User Experience',
      metrics: [
        'ai_user_wait_time_p50',
        'ai_abandonment_rate',
        'ai_retry_rate',
        'ai_satisfaction_score'
      ]
    }
  ],

  alerts: [
    {
      name: 'High TTFT',
      condition: 'ai_time_to_first_token_p99 > 3000',
      severity: 'warning'
    },
    {
      name: 'Stream Failures',
      condition: 'ai_stream_completion_rate < 0.95',
      severity: 'critical'
    },
    {
      name: 'Token Budget Alert',
      condition: 'ai_tokens_consumed_rate > budget_threshold',
      severity: 'warning'
    }
  ]
};

// Before: Every token triggers re-render
function MessageContent({ streamId }) {
  const content = useStore(state => state.streams.get(streamId)?.content);
  return <div>{content}</div>; // Re-renders on every token
}

// After: Batched updates with RAF
function MessageContent({ streamId }) {
  const containerRef = useRef<HTMLDivElement>(null);
  const contentRef = useRef('');

  useEffect(() => {
    const unsubscribe = useStore.subscribe(
      state => state.streams.get(streamId)?.content,
      (content) => {
        if (!content || !containerRef.current) return;

        // Direct DOM manipulation, bypassing React
        const newContent = content.slice(contentRef.current.length);
        if (newContent) {
          containerRef.current.insertAdjacentText('beforeend', newContent);
          contentRef.current = content;
        }
      }
    );
    return unsubscribe;
  }, [streamId]);

  return <div ref={containerRef} />;
}

// Before: Naive truncation
const context = messages.slice(-50); // Arbitrary limit

// After: Token-aware truncation
async function buildSafeContext(messages: Message[], maxTokens: number) {
  const tokenizer = await getTokenizer();
  let totalTokens = 0;
  const safeMessages: Message[] = [];

  // Always include system message
  const systemMessage = messages.find(m => m.role === 'system');
  if (systemMessage) {
    totalTokens += tokenizer.count(systemMessage.content);
    safeMessages.push(systemMessage);
  }

  // Add messages from most recent, respecting boundaries
  for (let i = messages.length - 1; i >= 0; i--) {
    const msg = messages[i];
    if (msg.role === 'system') continue;

    const msgTokens = tokenizer.count(formatMessage(msg));

    if (totalTokens + msgTokens > maxTokens * 0.8) { // 20% buffer
      break;
    }

    safeMessages.unshift(msg);
    totalTokens += msgTokens;
  }

  // Ensure we don't cut off mid-exchange
  if (safeMessages[0]?.role === 'assistant') {
    safeMessages.shift(); // Remove orphaned assistant message
  }

  return safeMessages;
}

// Before: One SSE connection per chat
const eventSource = new EventSource(`/api/chat/${sessionId}/stream`);

// After: Multiplexed connection with message routing
class MultiplexedStreamClient {
  private connection: EventSource | null = null;
  private streams = new Map<string, StreamHandler>();

  private ensureConnection(): void {
    if (this.connection) return;

    // Single connection for all streams
    this.connection = new EventSource('/api/streams/multiplex');

    this.connection.onmessage = (event) => {
      const { streamId, chunk } = JSON.parse(event.data);
      const handler = this.streams.get(streamId);
      if (handler) {
        handler.onChunk(chunk);
      }
    };
  }

  async subscribe(streamId: string, handler: StreamHandler): void {
    this.ensureConnection();
    this.streams.set(streamId, handler);

    // Tell server to add this stream to our connection
    await fetch(`/api/streams/${streamId}/subscribe`, {
      method: 'POST',
      body: JSON.stringify({ connectionId: this.connectionId })
    });
  }

  unsubscribe(streamId: string): void {
    this.streams.delete(streamId);
    fetch(`/api/streams/${streamId}/unsubscribe`, { method: 'POST' });
  }
}

// Optimistic: Show user message immediately
// Optimistic: Show typing indicator
// NOT optimistic: AI response content

async function sendMessage(content: string) {
  // Optimistic
  addUserMessage(content);
  setAssistantTyping(true);

  try {
    // Stream is inherently progressive, not optimistic
    await streamResponse();
  } catch (error) {
    // Rollback: mark user message as failed
    markMessageFailed(messageId);
  } finally {
    setAssistantTyping(false);
  }
}

interface EdgeAIConfig {
  // Route queries based on complexity
  router: {
    simple: 'edge',      // Classification, extraction
    moderate: 'regional', // Summarization, Q&A
    complex: 'origin'    // Multi-step reasoning, code generation
  };

  edgeModels: {
    'phi-3-mini': { maxTokens: 4096, latencyP50: 50 },
    'llama-3-8b': { maxTokens: 8192, latencyP50: 100 }
  };
}

class SpeculativeStreamer {
  async predictAndCache(context: Context): Promise<void> {
    // Analyze conversation for likely next queries
    const predictions = await this.predictor.predict(context);

    for (const prediction of predictions) {
      if (prediction.confidence > 0.8) {
        // Pre-generate response in background
        const response = await this.aiClient.complete({
          messages: [...context.messages, prediction.query]
        });

        await this.cache.set(prediction.cacheKey, response, {
          ttl: 300 // 5 minute speculative cache
        });
      }
    }
  }
}

interface MultiModalStream {
  type: 'text' | 'image' | 'audio' | 'video';

  // Text streams token-by-token
  // Images stream progressive JPEG
  // Audio streams PCM chunks
  // Video streams I-frames then P-frames
}

class MultiModalRenderer {
  async render(stream: AsyncIterable<MultiModalStream>): void {
    for await (const chunk of stream) {
      switch (chunk.type) {
        case 'text':
          this.textRenderer.append(chunk.content);
          break;
        case 'image':
          this.imageRenderer.appendBytes(chunk.bytes);
          break;
        case 'audio':
          this.audioRenderer.queueChunk(chunk.pcmData);
          break;
      }
    }
  }
}