DataOps-platform/docs/validation/P3_WP05_FAILURE_INJECTION.json

{
  "work_package": "P3-WP05",
  "engineering_status": "ENGINEERING_COMPLETE_ENTERPRISE_OBSERVABILITY_UAT_BLOCKED",
  "generated_at": "2026-08-11",
  "scope_boundary": "Synthetic local contract verification only. No enterprise monitoring, SMTP, collaboration, ITSM, on-call, or pilot system was contacted.",
  "cases": [
    {
      "case_id": "WP05-FI-001",
      "scope": "telemetry correlation and sensitive-input rejection",
      "preconditions": ["Local Python test process", "Synthetic IDs and resource references"],
      "steps": ["Normalize API, task, database, graph, and object-storage events", "Inject secret-named and raw-SQL attributes"],
      "safety_boundary": "No raw rows, SQL, credentials, URLs, tokens, or provider endpoints",
      "expected": "All valid events retain one correlation ID; invalid evidence is rejected before adaptation",
      "evidence": ["tests/core/orchestration/test_production_observability.py"],
      "rollback": "No persistent state or network operation",
      "status": "VERIFIED_LOCAL"
    },
    {
      "case_id": "WP05-FI-002",
      "scope": "alert delivery lease, retry, dead letter, and compensation",
      "preconditions": ["Local PostgreSQL at migration 20260811_480", "Synthetic incident and alert UUIDs"],
      "steps": ["Replay exact and conflicting delivery requests", "Race two database workers", "Use an expired fencing token", "Fail three attempts", "Record controlled human compensation"],
      "safety_boundary": "No SMTP, collaboration, paging, or external queue",
      "expected": "Exact replay is stable; conflict and stale worker fail closed; retry is bounded; dead letter and compensation are separately auditable",
      "evidence": ["tests/core/orchestration/test_production_operations_service.py", "tests/integration/test_production_operations_postgres.py"],
      "rollback": "Delete only WP05-TEST-* and wp05-test-* synthetic rows through the controlled test cleanup",
      "status": "VERIFIED_LOCAL"
    },
    {
      "case_id": "WP05-FI-003",
      "scope": "capacity and recovery",
      "preconditions": ["Synthetic 100-500 user / 50-500 source snapshots"],
      "steps": ["Raise utilization and queue watermarks", "Attempt queue overflow", "Apply backpressure recommendation"],
      "safety_boundary": "No production load or source access",
      "expected": "Throttle/backpressure precedes hard-limit refusal; no accepted item is silently discarded",
      "evidence": ["tests/core/orchestration/test_production_observability.py"],
      "rollback": "Discard process-local state",
      "status": "VERIFIED_LOCAL"
    },
    {
      "case_id": "WP05-FI-004",
      "scope": "API, provider-output, and append-only audit boundary",
      "preconditions": ["Local Flask test client", "Local PostgreSQL runtime and evidence roles"],
      "steps": ["Submit oversized and sensitive delivery bodies", "Return a malicious provider envelope", "Attempt runtime INSERT, UPDATE, DELETE, and TRUNCATE against the audit ledger", "Roll back a caller transaction after recording an independent rejection"],
      "safety_boundary": "Only stable reason codes, references, and correlation IDs may enter the audit ledger",
      "expected": "Requests fail with no-store and no sensitive echo; provider output is rejected; direct ledger mutation fails; controlled and independent audit receipts remain",
      "evidence": ["tests/core/orchestration/test_production_operations_api.py", "tests/core/orchestration/test_production_operations_openapi_contract.py", "tests/integration/test_production_operations_postgres.py"],
      "rollback": "Remove only the namespaced synthetic rows using the evidence-owner test cleanup and restore triggers immediately",
      "status": "VERIFIED_LOCAL"
    },
    {
      "case_id": "WP05-FI-101",
      "scope": "enterprise monitoring, messaging, on-call, and ITSM",
      "preconditions": ["monitoring", "smtp", "collaboration", "itsm", "pilot_goal", "Approved change window"],
      "steps": ["Emit approved test alert", "Verify actual deduplication, silence, delivery, receipt, escalation, recovery, ITSM forward/reverse reference, and postmortem"],
      "safety_boundary": "Enterprise approval, test routing, and named operators required",
      "expected": "Enterprise interfaces demonstrate confirmed delivery and recovery without secret/raw-data persistence",
      "evidence": [],
      "rollback": "Disable test routing and close the approved synthetic incident",
      "status": "TBD_EXTERNAL"
    }
  ]
}