Semantic Cache Architecture for Next.js App Router

import { NextRequest, NextResponse } from 'next/server';
import { Redis } from '@upstash/redis';
import OpenAI from 'openai';

const redis = Redis.fromEnv();
const openai = new OpenAI();

const CACHE_CONFIG = {
  NAMESPACE: 'llm_cache:support',
  SIMILARITY_THRESHOLD: 0.92,
  TTL_SECONDS: 43200,
  EMBEDDING_MODEL: 'text-embedding-3-small',
  EXACT_BYPASS: true
};

/**
 * Next.js Semantic Cache Route Handler
 * Intercepts LLM queries and returns cached completions when cosine similarity >= threshold.
 */
export async function POST(req: NextRequest) {
  try {
    const { prompt } = await req.json();
    if (!prompt) {
      return NextResponse.json({ error: 'Missing prompt' }, { status: 400 });
    }

    // 1. Exact Match Fast-Path Bypass
    if (CACHE_CONFIG.EXACT_BYPASS) {
      const exactCacheKey = `${CACHE_CONFIG.NAMESPACE}:exact:${Buffer.from(prompt).toString('base64url')}`;
      const exactHit = await redis.get<string>(exactCacheKey);
      if (exactHit) {
        return NextResponse.json({ result: exactHit, cached: true, cacheType: 'exact' });
      }
    }

    // 2. Compute Dense Vector Embedding for Incoming Prompt
    const embeddingResponse = await openai.embeddings.create({
      model: CACHE_CONFIG.EMBEDDING_MODEL,
      input: prompt,
    });
    const promptVector = embeddingResponse.data[0].embedding;

    // 3. Query Vector Index for Nearest Neighbor
    const vectorKey = `${CACHE_CONFIG.NAMESPACE}:vectors`;
    // Simulated Upstash Vector / Redis Vector Search Call
    // const results = await redis.vectorSearch(vectorKey, promptVector, { topK: 1 });

    // 4. Evaluate Cosine Similarity Threshold
    const nearestMatchScore = 0.94; // Example score returned from vector index
    if (nearestMatchScore >= CACHE_CONFIG.SIMILARITY_THRESHOLD) {
      return NextResponse.json({
        result: 'Cached LLM Completion Payload',
        cached: true,
        cacheType: 'semantic',
        similarity: nearestMatchScore
      });
    }

    // 5. Cache Miss: Execute Frontier LLM Completion
    const completion = await openai.chat.completions.create({
      model: 'gpt-4o',
      messages: [{ role: 'user', content: prompt }]
    });

    const completionText = completion.choices[0].message.content;

    // 6. Write-Through to Vector Cache with TTL
    // await redis.set(`${CACHE_CONFIG.NAMESPACE}:entry:${id}`, completionText, { ex: CACHE_CONFIG.TTL_SECONDS });

    return NextResponse.json({
      result: completionText,
      cached: false,
      similarity: nearestMatchScore
    });

  } catch (error) {
    console.error('Semantic Cache Handler Error:', error);
    return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 });
  }
}