Semantic Cache Architecture for Next.js App Router

Vector query route handler checking semantic similarity before calling the upstream model.

import { NextRequest, NextResponse } from 'next/server';
import { Index } from '@upstash/vector';

const vectorIndex = new Index({
  url: process.env.UPSTASH_VECTOR_REST_URL!,
  token: process.env.UPSTASH_VECTOR_REST_TOKEN!,
});

export async function POST(req: NextRequest) {
  const { prompt } = await req.json();

  // 1. Query vector index for semantically similar cached responses
  const results = await vectorIndex.query({
    data: prompt,
    topK: 1,
    includeMetadata: true,
  });

  const bestMatch = results[0];
  // Cosine distance threshold: score > 0.92 indicates near-identical intent
  if (bestMatch && bestMatch.score > 0.92 && bestMatch.metadata?.response) {
    return NextResponse.json({
      cached: true,
      score: bestMatch.score,
      response: bestMatch.metadata.response,
    });
  }

  // 2. Generate new response from upstream LLM if cache missed
  const upstreamResponse = "Generated response from LLM...";

  // 3. Upsert response to vector cache
  await vectorIndex.upsert({
    id: 'cache_' + Date.now(),
    data: prompt,
    metadata: { response: upstreamResponse, timestamp: Date.now() },
  });

  return NextResponse.json({ cached: false, response: upstreamResponse });
}