| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354 |
- from __future__ import annotations
- from pathlib import Path
- import pytest
- from app.core.llm.runtime_governance import (
- DatasetLeakageError,
- evaluate_governed_dataset,
- load_engineering_fixture,
- )
- def test_governed_eval_reports_k6_metrics_and_rejects_train_evaluation_overlap():
- result = evaluate_governed_dataset(
- train_case_ids={"train-1"},
- evaluation_cases=[
- {
- "case_id": "eval-1",
- "expected_evidence_id": "evidence-1",
- "retrieved_evidence_ids": ["evidence-1", "evidence-2"],
- "citation_evidence_ids": ["evidence-1"],
- "authorized": True,
- "fresh": True,
- },
- {
- "case_id": "eval-2",
- "expected_evidence_id": "evidence-2",
- "retrieved_evidence_ids": ["evidence-2"],
- "citation_evidence_ids": ["evidence-2"],
- "authorized": False,
- "fresh": False,
- },
- ],
- )
- assert result == {
- "recall": 1.0,
- "mrr": 1.0,
- "citation_correctness": 1.0,
- "privilege_violation_rate": 0.5,
- "freshness": 0.5,
- }
- with pytest.raises(DatasetLeakageError, match="dataset_leakage_detected"):
- evaluate_governed_dataset(
- train_case_ids={"eval-1"}, evaluation_cases=[{"case_id": "eval-1"}]
- )
- def test_fixture_manifest_has_real_digests_and_is_stably_read_once_per_file():
- manifest = Path(__file__).resolve().parents[2] / "docs/phase3/p3-wp09-agent-governance-fixture/manifest.json"
- first = load_engineering_fixture(manifest)
- second = load_engineering_fixture(manifest)
- assert first == second
- assert first["dataset.json"]["dataset_version"] == "engineering-fixture-v1"
|
