Promptfoo config — CI eval gate

Source: platform/packages/eval/promptfooconfig.yaml · rendered from main on every deploy — edit in the repo, not here
# Puccha RAG eval — golden Q&A gate (ADR-0006 thresholds: faithfulness ≥ 0.85).
#
# Runs against the LIVE `perftest` tenant (real Refund/Shipping doc, embedded in
# Vectorize on dev/uat → retrieval actually works). The original certogo set was
# orphaned by the legacy→new-account migration and is archived under
# golden-sets/_archive-certogo/ pending a corpus re-seed (see that README).
#
# Usage:
#   pnpm eval                                  # -> dev perftest (default below)
#   PUCCHA_API_URL=https://app-uat.puccha.ai/c/perftest pnpm eval
#   Admin: triggered from dashboard "Run eval" button
#
# NOTE: the default filter uses nunjucks `or` (NOT the Liquid `| default:` syntax,
# which promptfoo 0.121's strict nunjucks rejects with a "Column N" parse error).

description: 'Puccha RAG evaluation — perftest Refund/Shipping (live gate)'

providers:
  - id: http
    config:
      url: '{{ PUCCHA_API_URL or "https://app-dev.puccha.ai/c/perftest" }}/api/answer'
      method: POST
      headers:
        Content-Type: application/json
      body:
        query: '{{query}}'
        locale: '{{locale}}'
        mode: answer
      responseParser: 'json.answer'

prompts:
  - '{{query}}'

tests: './golden-sets/perftest/*.yaml'

# Grader for the per-test llm-rubric asserts. Faithfulness/citation asserts live
# per-test (in the golden sets) so refusal cases aren't wrongly failed for not
# citing — do NOT add global content asserts here.
#
# Grades via Anthropic direct (ANTHROPIC_API_KEY), NOT OpenRouter: chat moved to
# the AI-Gateway→Anthropic native path (ADR-0173), leaving the OpenRouter key a
# fallback that has hit its spend cap (403 "Key limit exceeded") — which would
# 403 every llm-rubric assert. CI already exports ANTHROPIC_API_KEY as a secret.
defaultTest:
  options:
    provider:
      id: anthropic:messages:claude-haiku-4-5-20251001