security-decision-labs/tools/llm-classification-validator/eval_config.yaml at main · security-decision-science/security-decision-labs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# LLM Classification Validator — Configuration
#
# Source of truth for defaults: src/llm_classification_validator/eval_defaults.json
# This YAML template mirrors those defaults. Copy and override what you need.
# Load with: EvalConfig.from_yaml("eval_config.yaml")

# ── Coherence (Dimension 1) ─────────────────────────────────
# Inter-rater agreement via Cohen's/Fleiss' kappa.
coherence:
  thresholds:
    - metric: mean_kappa
      target: 0.65
      minimum: 0.50
  bootstrap:
    iterations: 1000
    confidence: 0.90
    seed: 42

# ── Consistency (Dimension 2) ────────────────────────────────
# Deterministic rule-based validation.
consistency:
  fail_on_error: true
  fail_on_warning: false
  pass_rate_target: 0.95
  pass_rate_minimum: 0.80

# ── Convergent validity (Dimension 3) ────────────────────────
# Agreement with external references (crosswalks, ground truth).
convergent:
  thresholds:
    - metric: jaccard
      target: 0.60
      minimum: 0.45
    - metric: kappa
      target: 0.65
      minimum: 0.50
  bootstrap:
    iterations: 1000
    confidence: 0.90
    seed: 42

# ── Adversarial discrimination (Dimension 4) ─────────────────
# Minimal pair and ambiguity testing.
adversarial:
  discrimination_target: 0.85
  discrimination_minimum: 0.70
  ambiguity_target: 0.75
  ambiguity_minimum: 0.60
  combined_target: 0.80
  combined_minimum: 0.65
  discrimination_weight: 0.6
  ambiguity_weight: 0.4

# ── Stability and sensitivity (Dimension 5) ──────────────────
# Paraphrase invariance and perturbation sensitivity.
stability:
  thresholds:
    - metric: stability
      target: 0.85
      minimum: 0.70
    - metric: change_detection
      target: 0.70
      minimum: 0.55
    - metric: direction_accuracy
      target: 0.80
      minimum: 0.65
  false_change_max: 0.15
  bootstrap:
    iterations: 1000
    confidence: 0.90
    seed: 42

# ── Sampling ─────────────────────────────────────────────────
# Stratified expert review sampling parameters.
sampling:
  min_per_stratum: 3
  min_total: 20
  confidence: 0.90
  seed: 42

# ── Runner ───────────────────────────────────────────────────
# Evaluation orchestration.
runner:
  max_workers: 2
  parallel_advanced: true