Answer cache (Cache API, chunk-set keyed)

Source: platform/apps/app/src/lib/rag/answer-cache.ts · rendered from main on every deploy — edit in the repo, not here
/**
 * Answer cache — Cloudflare Cache API in front of the LLM (FinOps).
 *
 * The biggest single per-request cost on /api/answer is the Haiku call
 * (~$0.0015 amortized). For FAQ-style traffic — which is the vast majority
 * of widget queries — the same question routes to the same chunks and
 * deserves the same answer. Caching the final response by a key that
 * captures (tenant, locale, query, retrieved chunk set) lets us:
 *
 *   1. Skip the LLM call entirely on a hit.
 *   2. Auto-invalidate when document changes shift retrieval — the new
 *      top chunks produce a new key, so stale answers can never resurface.
 *   3. Stay edge-local: the Worker's `caches.default` is colocated and
 *      sub-ms to read; KV would add a global hop.
 *
 * Why Cache API and not KV?
 *   - Cache API is purpose-built for HTTP responses; we cache the JSON
 *     payload as a Response so headers + body round-trip cleanly.
 *   - Cache API entries don't count against the KV write budget.
 *   - Cache API is region-local — perfect for the colo where the request
 *     landed; no cross-region replication tax.
 *
 * The trade-off: cache is per-colo, so a tenant whose traffic spreads
 * across colos won't share entries between them. That's fine for the
 * FinOps target — even per-colo, a single repeated FAQ saves the LLM
 * call dozens of times an hour.
 */

import type { TenantContext } from '@puccha/types';

/** TTL keeps stale answers bounded even when retrieval doesn't shift.
 * 15 minutes is the calibrated point between FinOps win (long enough
 * to catch repeats during a session) and freshness (short enough that
 * a doc edit's effect is visible before anyone notices). */
export const ANSWER_CACHE_TTL_SECONDS = 15 * 60;

interface AnswerCacheKeyOpts {
  tenant: TenantContext;
  workspaceId: string | null;
  query: string;
  locale: string;
  /** Sorted, deterministic — the chunk-set fingerprint. */
  chunkIds: string[];
  /**
   * Tenant cache generation (see $lib/rag/cache-gen). Bumped by the
   * Clear-cache admin action; every cached entry becomes unreachable on
   * the next lookup without scanning Cache API entries. Defaults to 0 so
   * tenants without a tracked generation share the initial namespace.
   */
  cacheGeneration?: number;
  /** ADR-0201 D2 — content-addressed persona identity; undefined when the
   * tenant has no default persona (keys unchanged from before the ADR). */
  personaFingerprint?: string;
}

/**
 * Build a stable cache URL for an answer. Returned as a plain string —
 * Cache API accepts URL strings on both DOM and CF Workers without
 * forcing the caller to pick the right Request constructor for the
 * runtime. Hashed with SHA-256 so the key length is bounded and the
 * query text never appears in the URL (PDPA — query content is
 * sensitive).
 */
export async function buildAnswerCacheKey(opts: AnswerCacheKeyOpts): Promise<string> {
  const canonicalQuery = opts.query.trim().toLowerCase();
  const canonicalChunks = [...opts.chunkIds].sort().join(',');
  const fingerprint = [
    opts.tenant.id,
    opts.workspaceId ?? '',
    opts.locale,
    canonicalQuery,
    canonicalChunks,
    `gen:${opts.cacheGeneration ?? 0}`,
    ...(opts.personaFingerprint ? [`persona:${opts.personaFingerprint}`] : []),
  ].join('|');
  const buf = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(fingerprint));
  const hex = Array.from(new Uint8Array(buf))
    .map((b) => b.toString(16).padStart(2, '0'))
    .join('');
  // Use a tenant-scoped synthetic origin so a debug `caches.delete()` by
  // tenant slug (future cli) is straightforward. The path includes the
  // hash but no raw query — safe to log.
  return `https://answer-cache.puccha.internal/${opts.tenant.slug}/${hex}`;
}

export interface CachedAnswerBody {
  id: string;
  answer: string;
  sources: Array<{ id: number; title: string; chunkId: string; snippet: string }>;
  confidence: number;
  idk: boolean;
  latencyMs: number;
}

/** Minimal duck-typed view of the Cache API. The DOM `Cache` and the
 * Cloudflare Workers `Cache` agree at the call surface (string URL +
 * standard Response/Request) but diverge in their TypeScript types
 * (`Headers.getSetCookie`, `Request<unknown, CfProperties>` flavors).
 * Typing the response side as `unknown` lets us accept both runtimes
 * without a per-environment shim — we only need `.json()` on the hit. */
export interface AnswerCacheLike {
  match(url: string): Promise<{ json(): Promise<unknown> } | null | undefined>;
  put(url: string, response: Response): Promise<void>;
}

/** Look up an answer; returns null on miss or any error (cache must
 * never break the request path). */
export async function readAnswerCache(
  cache: AnswerCacheLike,
  key: string,
): Promise<CachedAnswerBody | null> {
  try {
    const hit = await cache.match(key);
    if (!hit) return null;
    return (await hit.json()) as CachedAnswerBody;
  } catch {
    return null;
  }
}

/** Schedule a cache write without blocking the response. Use waitUntil
 * so the work outlives the request. */
export function writeAnswerCache(
  cache: AnswerCacheLike,
  key: string,
  body: CachedAnswerBody,
  waitUntil: (p: Promise<unknown>) => void,
): void {
  const res = new Response(JSON.stringify(body), {
    headers: {
      'Content-Type': 'application/json',
      'Cache-Control': `public, max-age=${ANSWER_CACHE_TTL_SECONDS}`,
    },
  });
  try {
    waitUntil(cache.put(key, res));
  } catch {
    // Cache write failed — non-fatal, the request already returned.
  }
}