Scope gate — canned intents + refusal threshold

Source: platform/apps/app/src/lib/rag/scope-gate.ts · rendered from main on every deploy — edit in the repo, not here

platform/apps/app/src/lib/rag/scope-gate.ts

/**
 * Scope gate — refuse out-of-KB queries deterministically (FinOps + safety).
 *
 * Two problems this solves:
 *
 *   1. **Hallucination.** When a tenant's KB has nothing relevant, the LLM
 *      will sometimes confidently answer from its world knowledge. For a
 *      compliance product that's a trust-killer ("Puccha told my customer
 *      our refund policy is 30 days, but we say 14"). Refusing politely
 *      and redirecting to human staff is the only safe default.
 *
 *   2. **Cost.** Every LLM call is ~$0.0015 amortized (Haiku 4.5 in/out).
 *      If we already know — from retrieval scores — that the answer
 *      can't be grounded, calling the model anyway is pure burn. Skipping
 *      it on out-of-scope queries cuts ~30-50% of widget chat spend on
 *      a typical small-KB tenant where greetings + off-topic dominate.
 *
 * The gate runs in two stages:
 *
 *   Stage A — pre-retrieval intent classifier. Cheap regex-based check
 *   for greetings / thanks / chit-chat. Returns a canned reply WITHOUT
 *   running embedding, search, rerank, or LLM. This is the biggest
 *   single FinOps win because greetings have zero retrieval value but
 *   the LLM-driven flow would still call the model.
 *
 *   Stage B — post-retrieval scope check. After hybridSearch + rerank,
 *   if the top result's score is below the confidence threshold (or
 *   there are no results), we refuse without calling the LLM. The
 *   threshold is calibrated for Cohere rerank scores (0..1, semantic
 *   relevance) — when the noop reranker is in use we fall back to
 *   "any result at all" since RRF/fusion scores aren't comparable.
 */

/** Cohere rerank threshold below which we treat the top hit as off-topic.
 * 0.30 was picked from the eval corpus: scores below this consistently
 * paired with citation_recall = 0 in the Promptfoo runs. Tune by tenant
 * later if needed. */
export const SCOPE_RERANK_MIN_SCORE = 0.3;

export type QueryIntent = 'greeting' | 'gratitude' | 'scope';

/** Thai polite particles that commonly suffix a greeting or thanks
 * (สวัสดี**ครับ**, ขอบคุณ**ค่ะ**). Without these the anchored regex below
 * rejected the two MOST common Thai greetings — `สวัสดีครับ` / `สวัสดีค่ะ` —
 * as `scope`, so a bare "hello" got an out-of-scope refusal in prod. */
const TH_POLITE = '(?:ครับ|คร้าบ|ครับผม|ค่ะ|คะ|ค่า|จ้า|จ้ะ|จ๋า|นะ|น่ะ|นะคะ|นะครับ)';

/** Anchored alternation — match if the *whole trimmed* query is a greeting/
 * thanks token plus optional polite particles and trailing punctuation. The
 * anchor avoids matching "hello, what's your refund policy?" as a greeting. */
const GREETING_RE = new RegExp(
  `^(?:สวัสดี|หวัดดี|ดีครับ|ดีค่ะ|hi|hello|hey|good\\s+(?:morning|afternoon|evening))(?:\\s*${TH_POLITE})*[\\s.!?]*$`,
  'i',
);
const GRATITUDE_RE = new RegExp(
  `^(?:ขอบคุณ|ขอบใจ|thanks|thank\\s*you|ty|thx)(?:\\s*(?:มาก|มากๆ|${TH_POLITE}))*[\\s.!?]*$`,
  'i',
);

export function classifyQueryIntent(query: string): QueryIntent {
  const trimmed = query.trim();
  if (GREETING_RE.test(trimmed)) return 'greeting';
  if (GRATITUDE_RE.test(trimmed)) return 'gratitude';
  return 'scope';
}

/** Commercial / lead intent: the visitor is expressing buying interest, a need,
 * or asking whether we offer something ("สนใจ software CRM", "interested in X",
 * "do you build …") rather than asking a precise factual question. A QA reranker
 * scores these LOW — they aren't questions the KB *answers* — so they fall below
 * the scope floor and get refused, even when they're squarely on-topic. But a
 * refused lead is the worst outcome for a sales assistant. When this matches AND
 * we retrieved at least some context, the caller skips the scope-gate refusal
 * and lets the grounded LLM engage + offer a handoff. False positives are
 * harmless: the query was already below the floor, so the alternative was a flat
 * refusal; a grounded reply (which still says "I don't have that — contact our
 * team" when context is thin) is strictly better UX. Refines ADR-0112's gate. */
const COMMERCIAL_INTENT_RE =
  /สนใจ|อยาก|ต้องการ|มองหา|ปรึกษา|ว่าจ้าง|จ้าง|ขอราคา|ใบเสนอราคา|พัฒนา|สร้าง|จอง|ซื้อ|สั่งซื้อ|สมัคร|interested|looking\s+for|need|want|do\s+you|can\s+you|could\s+you|quote|hire|consult|build|develop|help\s+me|book|reserve|booking|buy|purchase|sign\s*up|subscribe/i;

export function hasCommercialIntent(query: string): boolean {
  return COMMERCIAL_INTENT_RE.test(query);
}

// Canned greeting / gratitude / out-of-scope wording lives in chat/canned.ts
// (ADR-0205) so it follows the persona's voice.

interface ScopedSearchResult {
  score: number;
}

interface ScopeCheckOpts {
  /** True when the active reranker returns calibrated semantic scores
   * (Cohere v3.5). When false (LLM reranker, noop), the score scale is
   * not comparable across tenants, so we fall back to "any hit at all". */
  rerankerCalibrated: boolean;
  /** Override the default threshold per-tenant (e.g. for noisy KBs). */
  minScore?: number;
}

export function isWithinScope(results: ScopedSearchResult[], opts: ScopeCheckOpts): boolean {
  if (results.length === 0) return false;
  if (!opts.rerankerCalibrated) return true; // best-effort when uncalibrated
  const top = results[0]?.score ?? 0;
  return top >= (opts.minScore ?? SCOPE_RERANK_MIN_SCORE);
}