DataOps-platform/tests/knowledge/test_evaluation.py

from __future__ import annotations


def test_evaluation_gate_accepts_grounded_authorized_fresh_results():
    from app.core.knowledge.evaluation import (
        EvaluationCase,
        EvaluationObservation,
        evaluate,
    )

    case = EvaluationCase(
        case_id="exact-1",
        case_type="exact",
        expected_source_uids=frozenset({"source-1"}),
        allowed_business_domains=frozenset({"domain-a"}),
    )
    observation = EvaluationObservation(
        retrieved_source_uids=("source-1",),
        retrieved_business_domains=("domain-a",),
        cited_source_uids=("source-1",),
        cited_freshness=("fresh",),
        answer_status="grounded",
        latency_ms=100,
    )

    summary = evaluate([(case, observation)])

    assert summary.passed is True
    assert summary.source_recall == 1.0
    assert summary.citation_precision == 1.0


def test_evaluation_gate_fails_closed_on_permission_leak_or_stale_citation():
    from app.core.knowledge.evaluation import (
        EvaluationCase,
        EvaluationObservation,
        evaluate,
    )

    case = EvaluationCase(
        case_id="negative-1",
        case_type="permission",
        expected_source_uids=frozenset(),
        allowed_business_domains=frozenset({"domain-a"}),
        must_refuse=True,
    )
    observation = EvaluationObservation(
        retrieved_source_uids=("secret-source",),
        retrieved_business_domains=("domain-b",),
        cited_source_uids=("secret-source",),
        cited_freshness=("stale",),
        answer_status="grounded",
        latency_ms=100,
    )

    summary = evaluate([(case, observation)])

    assert summary.passed is False
    assert summary.permission_leak_count == 1
    assert summary.stale_citation_count == 1
    assert summary.refusal_accuracy == 0.0