Compliance pack — example golden set

Source: platform/content/packs/_example/golden.yaml · rendered from main on every deploy — edit in the repo, not here
# Example golden eval set (ADR-0101 §10).
#
# Format: minimal Promptfoo-compatible. Each test case has a `vars`
# (input variables — typically `{ query }`) and an `assert` (expected
# citation / faithfulness checks). `pnpm pack:eval <slug>` validates
# this file's shape and counts test cases; full Promptfoo invocation
# (against a pack-only retrieval target) is the next slice once the
# eval target lands.
#
# Required: at least 20 test cases for production packs per CLAUDE.md
# quality gate. This example carries 3 cases (smoke only — `_example`
# is non-shipping).

prompts:
  - "{{query}}"

providers:
  # Placeholder — real provider will be the pack-only retrieval endpoint
  # added in a follow-up slice. Until then `pack:eval` validates shape
  # only and refuses to invoke Promptfoo.
  - id: pack-rag-stub
    label: "pack-only RAG (stub)"

tests:
  - description: "Article 1 — overview question"
    vars:
      query: "What is the purpose of the Example Regulation?"
    assert:
      - type: contains-any
        value:
          - "framework"
          - "demonstration"
      - type: cite-pack-source
        value: "Example Regulation Art. 1"

  - description: "Article 2 — definitions lookup"
    vars:
      query: "How is a 'Document' defined in the regulation?"
    assert:
      - type: contains-any
        value:
          - "markdown"
          - "YAML frontmatter"
      - type: cite-pack-source
        value: "Example Regulation Art. 2"

  - description: "Out-of-scope refusal"
    vars:
      query: "What is the current weather in Bangkok?"
    assert:
      - type: not-contains
        value: "weather"
      - type: refusal
        value: true

# Publish gate (ADR-0101 §10). `pack:publish` will check this — for now
# the gate ships as configuration; runner wires up in a follow-up.
publish_gate:
  faithfulness_threshold: 0.85
  min_test_cases: 20
  # `_example` is allowed to skip the min_test_cases gate because it's
  # not a shipping pack. Real packs MUST satisfy both thresholds.
  skip_min_test_cases_for_unshipped: true