Promptfoo config — Certogo

Source: platform/packages/eval/promptfooconfig.certogo.yaml · rendered from main on every deploy — edit in the repo, not here
# EXTENDED RAG eval — certogo domain corpus (44 cases: FAQ 36 + operations 8).
#
# Richer coverage than the default perftest gate (promptfooconfig.yaml), against
# the `certogo-eval` tenant on dev — the Certogo training-FAQ + 3-tier operations
# corpus (content/certogo/*.md), seeded + embedded on the new account after the
# migration orphaned the original tenant. See golden-sets/_archive-certogo/README.
#
# This is OPT-IN, not the default CI gate, because it needs two things the anon
# perftest gate doesn't:
#   1. CERTOGO_EVAL_KEY — a `puccha_live_` API key for certogo-eval. The 44 cases
#      exceed the 15/hour anonymous /api/answer limit; an API key gets 60/hour
#      (lib/rate-limit.ts). Seed one per the recipe in the archive README.
#   2. A browser User-Agent — CF WAF 403s default library UAs (Python-urllib etc).
#
# Run:  CERTOGO_EVAL_KEY=puccha_live_... pnpm exec promptfoo eval -c promptfooconfig.certogo.yaml
# Proven 44/44 on the deterministic contains-any asserts (2026-07-07).

description: 'Puccha RAG eval (extended) — certogo FAQ + operations'

providers:
  - id: http
    config:
      url: 'https://app-dev.puccha.ai/c/certogo-eval/api/answer'
      method: POST
      headers:
        Content-Type: application/json
        Authorization: 'Bearer {{ CERTOGO_EVAL_KEY }}'
        # CF WAF 403s default library User-Agents — send a browser one.
        User-Agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36'
      body:
        query: '{{query}}'
        locale: '{{locale}}'
        mode: answer
      responseParser: 'json.answer'

prompts:
  - '{{query}}'

tests: './golden-sets/certogo/*.yaml'

defaultTest:
  options:
    provider:
      id: anthropic:messages:claude-haiku-4-5-20251001