rules.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847
  1. """Load immutable rule plans and dispatch them through allowlisted adapters."""
  2. from __future__ import annotations
  3. import hashlib
  4. import json
  5. import logging
  6. import re
  7. import threading
  8. import time
  9. from asyncio import CancelledError
  10. from collections.abc import Mapping
  11. from typing import Any
  12. from sqlalchemy import text
  13. from app.core.common.identifiers import ensure_governance_uid
  14. from app.core.data_rules.compilers.polars import (
  15. COMPILER_VERSION as POLARS_COMPILER_VERSION,
  16. )
  17. from app.core.data_rules.compilers.polars import (
  18. validate_bound_polars_plan,
  19. )
  20. from app.core.data_rules.compilers.sql import (
  21. COMPILER_VERSION as SQL_COMPILER_VERSION,
  22. )
  23. from app.core.data_rules.compilers.sql import (
  24. bound_sql_plan_relations,
  25. validate_bound_sql_plan,
  26. )
  27. from app.runner.nodes import NodeExecutionError
  28. from app.runner.rule_evidence import validate_public_rule_result
  29. CONFIG_KEYS = {
  30. "component_binding_id",
  31. "rule_version_id",
  32. "execution_plan_hash",
  33. "provenance",
  34. }
  35. IDEMPOTENCY_STRATEGIES = {
  36. "partition_replace",
  37. "upsert",
  38. "deduplication_key",
  39. }
  40. LOGGER = logging.getLogger(__name__)
  41. def _canonical_hash(value: Any) -> str:
  42. try:
  43. canonical = json.dumps(
  44. value,
  45. sort_keys=True,
  46. separators=(",", ":"),
  47. ensure_ascii=False,
  48. )
  49. except (TypeError, ValueError) as exc:
  50. raise NodeExecutionError(
  51. "published rule plan is not JSON serializable"
  52. ) from exc
  53. return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
  54. def _uid(value: Any, label: str) -> str:
  55. try:
  56. return ensure_governance_uid({"uid": str(value)})
  57. except ValueError as exc:
  58. raise NodeExecutionError(f"{label} is invalid") from exc
  59. class PostgresRulePlanRepository:
  60. """Read one plan through all immutable binding and version constraints."""
  61. def __init__(self, engine):
  62. self.engine = engine
  63. def load(
  64. self,
  65. *,
  66. component_binding_id: str,
  67. rule_version_id: str,
  68. plan_hash: str,
  69. ) -> dict[str, Any] | None:
  70. statement = text(
  71. """
  72. SELECT
  73. p.component_binding_id::text AS component_binding_id,
  74. b.rule_version_id::text AS rule_version_id,
  75. p.backend,
  76. p.compiler_version,
  77. p.plan,
  78. p.plan_hash,
  79. p.schema_hashes,
  80. p.status AS plan_status,
  81. r.status AS rule_status,
  82. r.spec_hash AS canonical_rule_spec_hash,
  83. ins.id::text AS canonical_input_schema_snapshot_id,
  84. ins.schema_hash AS canonical_input_schema_hash,
  85. ib.binding_hash AS canonical_input_binding_hash,
  86. ib.data_source_uid::text AS canonical_input_data_source_uid,
  87. ib.object_kind AS canonical_input_object_kind,
  88. ib.object_ref AS canonical_input_object_ref,
  89. ib.dialect AS canonical_input_dialect,
  90. outs.id::text AS canonical_output_schema_snapshot_id,
  91. outs.schema_hash AS canonical_output_schema_hash,
  92. ob.binding_hash AS canonical_output_binding_hash,
  93. ob.data_source_uid::text AS canonical_output_data_source_uid,
  94. ob.object_kind AS canonical_output_object_kind,
  95. ob.object_ref AS canonical_output_object_ref,
  96. ob.dialect AS canonical_output_dialect,
  97. b.component_kind,
  98. b.idempotency AS binding_idempotency,
  99. EXISTS (
  100. SELECT 1 FROM public.rule_publication_audits pa
  101. WHERE pa.rule_execution_plan_id = p.id
  102. AND pa.rule_version_id = r.id
  103. AND pa.action = 'published'
  104. AND pa.to_status = 'published'
  105. AND pa.evidence_hash = p.plan_hash
  106. ) AS publication_audit_trusted,
  107. EXISTS (
  108. SELECT 1
  109. FROM public.rule_logical_plans lp
  110. JOIN public.rule_logical_compile_evidence lce
  111. ON lce.logical_plan_id = lp.id
  112. JOIN public.rule_logical_test_evidence lte
  113. ON lte.logical_plan_id = lp.id
  114. WHERE lp.rule_version_id = r.id
  115. AND lp.status = 'published'
  116. AND lce.status = 'success'
  117. AND lce.compiler_version = lp.compiler_version
  118. AND lce.plan_hash = lp.plan_hash
  119. AND lce.schema_hashes = lp.schema_hashes
  120. AND lce.capabilities = lp.capabilities
  121. AND lte.status = 'success'
  122. AND lte.plan_hash = lp.plan_hash
  123. AND lte.schema_hashes = lp.schema_hashes
  124. ) AS logical_evidence_trusted,
  125. EXISTS (
  126. SELECT 1
  127. FROM public.rule_compile_evidence pce
  128. JOIN public.rule_test_evidence pte
  129. ON pte.rule_execution_plan_id =
  130. pce.rule_execution_plan_id
  131. WHERE pce.rule_execution_plan_id = p.id
  132. AND pce.status = 'success'
  133. AND pce.legacy_untrusted = FALSE
  134. AND pce.compiler_version = p.compiler_version
  135. AND pce.plan_hash = p.plan_hash
  136. AND pce.schema_hashes = p.schema_hashes
  137. AND pce.binding_hashes = jsonb_build_object(
  138. 'input', ib.binding_hash,
  139. 'output', ob.binding_hash
  140. )
  141. AND pce.capabilities = CASE
  142. WHEN p.backend = 'polars_batch' THEN
  143. jsonb_build_object(
  144. 'resource_limits',
  145. p.plan->'resource_limits'
  146. )
  147. ELSE p.plan->'capabilities'
  148. END
  149. AND pte.status = 'success'
  150. AND pte.legacy_untrusted = FALSE
  151. AND pte.plan_hash = p.plan_hash
  152. AND pte.schema_hashes = p.schema_hashes
  153. AND pte.binding_hashes = jsonb_build_object(
  154. 'input', ib.binding_hash,
  155. 'output', ob.binding_hash
  156. )
  157. ) AS physical_evidence_trusted
  158. FROM public.rule_execution_plans p
  159. JOIN public.dataflow_component_bindings b
  160. ON b.id = p.component_binding_id
  161. JOIN public.data_rule_versions r
  162. ON r.id = b.rule_version_id
  163. LEFT JOIN public.dataflow_dataset_bindings ib
  164. ON ib.id = CAST(p.plan->>'input_binding_id' AS uuid)
  165. LEFT JOIN public.data_schema_snapshots ins
  166. ON ins.id = ib.schema_snapshot_id
  167. LEFT JOIN public.dataflow_dataset_bindings ob
  168. ON ob.id = CAST(p.plan->>'output_binding_id' AS uuid)
  169. LEFT JOIN public.data_schema_snapshots outs
  170. ON outs.id = ob.schema_snapshot_id
  171. WHERE p.component_binding_id = CAST(:component_binding_id AS uuid)
  172. AND b.rule_version_id = CAST(:rule_version_id AS uuid)
  173. AND p.plan_hash = :plan_hash
  174. """
  175. )
  176. with self.engine.connect() as connection:
  177. row = connection.execute(
  178. statement,
  179. {
  180. "component_binding_id": component_binding_id,
  181. "rule_version_id": rule_version_id,
  182. "plan_hash": plan_hash,
  183. },
  184. ).mappings().one_or_none()
  185. return dict(row) if row is not None else None
  186. class RulePlanExecutor:
  187. """Fail closed unless the exact published plan is still executable."""
  188. def __init__(
  189. self,
  190. repository,
  191. *,
  192. adapters: Mapping[str, Any],
  193. evidence_writer=None,
  194. ):
  195. self.repository = repository
  196. self.adapters = dict(adapters)
  197. self.evidence_writer = evidence_writer
  198. @staticmethod
  199. def _redacted_sample(value: Any) -> list[dict[str, Any]]:
  200. if not isinstance(value, list):
  201. return []
  202. sample = []
  203. for row in value[:100]:
  204. if not isinstance(row, dict):
  205. continue
  206. sample.append(
  207. {
  208. str(key): (
  209. None if item is None else "[REDACTED]"
  210. )
  211. for key, item in sorted(row.items())
  212. }
  213. )
  214. return sample
  215. def _start_evidence(
  216. self,
  217. *,
  218. component_binding_id: str,
  219. rule_version_id: str,
  220. plan_hash: str,
  221. execution_context: Mapping[str, Any],
  222. ):
  223. if self.evidence_writer is None:
  224. return None, None
  225. correlation_id = execution_context.get("correlation_id")
  226. dataflow_uid = execution_context.get("dataflow_uid")
  227. deployment_id = execution_context.get("deployment_id")
  228. environment = execution_context.get("environment")
  229. workflow_version = execution_context.get("workflow_version")
  230. node_id = execution_context.get("node_id")
  231. task_jti = execution_context.get("task_jti")
  232. if (
  233. not correlation_id
  234. or not dataflow_uid
  235. or not deployment_id
  236. or environment
  237. not in {"development", "test", "production"}
  238. or isinstance(workflow_version, bool)
  239. or not isinstance(workflow_version, int)
  240. or workflow_version < 1
  241. or not isinstance(node_id, str)
  242. or not node_id
  243. ):
  244. raise NodeExecutionError(
  245. "trusted rule evidence context is incomplete"
  246. )
  247. try:
  248. rule_run_id = self.evidence_writer.start(
  249. component_binding_id=component_binding_id,
  250. rule_version_id=rule_version_id,
  251. plan_hash=plan_hash,
  252. correlation_id=correlation_id,
  253. dataflow_uid=dataflow_uid,
  254. deployment_id=deployment_id,
  255. environment=environment,
  256. workflow_version=workflow_version,
  257. node_id=node_id,
  258. lease_owner=task_jti,
  259. )
  260. replay = self.evidence_writer.replay(rule_run_id)
  261. except Exception as exc:
  262. LOGGER.exception("governed rule evidence start failed")
  263. raise NodeExecutionError(
  264. "rule execution evidence is unavailable"
  265. ) from exc
  266. return rule_run_id, replay
  267. def _finish_evidence(self, rule_run_id, result):
  268. if rule_run_id is None:
  269. return
  270. try:
  271. self.evidence_writer.finish(rule_run_id, result)
  272. except Exception as exc:
  273. raise NodeExecutionError(
  274. "rule execution evidence commit outcome is unknown",
  275. commit_outcome="unknown",
  276. ) from exc
  277. def replay_task(
  278. self,
  279. *,
  280. node,
  281. correlation_id,
  282. deployment_id,
  283. task_jti,
  284. ):
  285. if self.evidence_writer is None:
  286. return None
  287. config = node.get("config")
  288. if not isinstance(config, dict):
  289. return None
  290. try:
  291. replay = self.evidence_writer.reconcile_expired_lease(
  292. lease_owner=task_jti,
  293. deployment_id=deployment_id,
  294. correlation_id=correlation_id,
  295. component_binding_id=_uid(
  296. config.get("component_binding_id"),
  297. "component_binding_id",
  298. ),
  299. rule_version_id=_uid(
  300. config.get("rule_version_id"),
  301. "rule_version_id",
  302. ),
  303. plan_hash=str(
  304. config.get("execution_plan_hash") or ""
  305. ),
  306. )
  307. except Exception as exc:
  308. raise NodeExecutionError(
  309. "rule execution evidence is unavailable"
  310. ) from exc
  311. return replay
  312. def execute(
  313. self,
  314. node,
  315. parameters,
  316. *,
  317. write_authorized=False,
  318. **execution_context,
  319. ):
  320. if node.get("type") not in {"rule.apply", "quality.check"}:
  321. raise NodeExecutionError("unsupported governed rule node")
  322. config = node.get("config")
  323. if not isinstance(config, dict) or set(config) - CONFIG_KEYS:
  324. raise NodeExecutionError(
  325. "governed rule node contains inline or unsupported fields"
  326. )
  327. component_binding_id = _uid(
  328. config.get("component_binding_id"),
  329. "component_binding_id",
  330. )
  331. rule_version_id = _uid(
  332. config.get("rule_version_id"),
  333. "rule_version_id",
  334. )
  335. plan_hash = str(config.get("execution_plan_hash") or "")
  336. if not re.fullmatch(r"[0-9a-f]{64}", plan_hash):
  337. raise NodeExecutionError("execution plan hash is invalid")
  338. if node.get("type") == "rule.apply":
  339. idempotency = node.get("idempotency")
  340. if (
  341. node.get("purpose") != "write"
  342. or not write_authorized
  343. or not isinstance(idempotency, dict)
  344. or idempotency.get("strategy")
  345. not in IDEMPOTENCY_STRATEGIES
  346. or not str(idempotency.get("key") or "").strip()
  347. ):
  348. raise NodeExecutionError(
  349. "governed write authorization and idempotency are required"
  350. )
  351. elif node.get("purpose") != "read":
  352. raise NodeExecutionError("quality check must be read only")
  353. record = self.repository.load(
  354. component_binding_id=component_binding_id,
  355. rule_version_id=rule_version_id,
  356. plan_hash=plan_hash,
  357. )
  358. if not isinstance(record, dict):
  359. raise NodeExecutionError("published rule plan was not found")
  360. if (
  361. record.get("component_binding_id") != component_binding_id
  362. or record.get("rule_version_id") != rule_version_id
  363. or record.get("plan_hash") != plan_hash
  364. or record.get("plan_status") != "published"
  365. or record.get("rule_status") != "published"
  366. or record.get("publication_audit_trusted") is not True
  367. or record.get("logical_evidence_trusted") is not True
  368. or record.get("physical_evidence_trusted") is not True
  369. or _canonical_hash(record.get("plan")) != plan_hash
  370. ):
  371. raise NodeExecutionError("published rule plan is not executable")
  372. if node.get("type") == "rule.apply" and (
  373. record.get("component_kind") != "rule.apply"
  374. or record.get("binding_idempotency") != node.get("idempotency")
  375. ):
  376. raise NodeExecutionError(
  377. "governed rule idempotency does not match its binding"
  378. )
  379. backend = record.get("backend")
  380. if backend == "sql_pushdown":
  381. try:
  382. plan = validate_bound_sql_plan(record.get("plan"))
  383. relations = bound_sql_plan_relations(plan)
  384. except ValueError as exc:
  385. raise NodeExecutionError(
  386. "published rule plan is not executable"
  387. ) from exc
  388. expected_schema_hashes = {
  389. "rule_spec_hash": plan["rule_spec_hash"],
  390. "input_schema_snapshot_id": plan[
  391. "input_schema_snapshot_id"
  392. ],
  393. "input_schema_hash": plan["input_schema_hash"],
  394. "output_schema_snapshot_id": plan[
  395. "output_schema_snapshot_id"
  396. ],
  397. "output_schema_hash": plan["output_schema_hash"],
  398. }
  399. if (
  400. record.get("compiler_version") != SQL_COMPILER_VERSION
  401. or plan["compiler_version"] != SQL_COMPILER_VERSION
  402. or record.get("schema_hashes") != expected_schema_hashes
  403. or record.get("canonical_rule_spec_hash")
  404. != plan["rule_spec_hash"]
  405. or record.get("canonical_input_schema_snapshot_id")
  406. != plan["input_schema_snapshot_id"]
  407. or record.get("canonical_input_schema_hash")
  408. != plan["input_schema_hash"]
  409. or record.get("canonical_output_schema_snapshot_id")
  410. != plan["output_schema_snapshot_id"]
  411. or record.get("canonical_output_schema_hash")
  412. != plan["output_schema_hash"]
  413. or record.get("canonical_input_data_source_uid")
  414. != plan["data_source_uid"]
  415. or record.get("canonical_output_data_source_uid")
  416. != plan["data_source_uid"]
  417. or record.get("canonical_input_object_ref")
  418. != relations["input_object_ref"]
  419. or record.get("canonical_output_object_ref")
  420. != relations["output_object_ref"]
  421. or (
  422. "postgresql"
  423. if record.get("canonical_input_dialect") == "postgres"
  424. else record.get("canonical_input_dialect")
  425. )
  426. != plan["dialect"]
  427. or (
  428. "postgresql"
  429. if record.get("canonical_output_dialect") == "postgres"
  430. else record.get("canonical_output_dialect")
  431. )
  432. != plan["dialect"]
  433. ):
  434. raise NodeExecutionError(
  435. "published rule plan canonical attestation does not match"
  436. )
  437. elif backend == "polars_batch":
  438. try:
  439. plan = validate_bound_polars_plan(record.get("plan"))
  440. except ValueError as exc:
  441. raise NodeExecutionError(
  442. "published rule plan is not executable"
  443. ) from exc
  444. expected_schema_hashes = {
  445. "rule_spec_hash": plan["rule_spec_hash"],
  446. "input_schema_snapshot_id": plan[
  447. "input_schema_snapshot_id"
  448. ],
  449. "input_schema_hash": plan["input_schema_hash"],
  450. "output_schema_snapshot_id": plan[
  451. "output_schema_snapshot_id"
  452. ],
  453. "output_schema_hash": plan["output_schema_hash"],
  454. }
  455. if (
  456. record.get("compiler_version") != POLARS_COMPILER_VERSION
  457. or plan["compiler_version"] != POLARS_COMPILER_VERSION
  458. or record.get("schema_hashes") != expected_schema_hashes
  459. or record.get("canonical_rule_spec_hash")
  460. != plan["rule_spec_hash"]
  461. or record.get("canonical_input_schema_snapshot_id")
  462. != plan["input_schema_snapshot_id"]
  463. or record.get("canonical_input_schema_hash")
  464. != plan["input_schema_hash"]
  465. or record.get("canonical_output_schema_snapshot_id")
  466. != plan["output_schema_snapshot_id"]
  467. or record.get("canonical_output_schema_hash")
  468. != plan["output_schema_hash"]
  469. or record.get("canonical_input_object_kind")
  470. != "parquet_artifact"
  471. or record.get("canonical_output_object_kind")
  472. != "parquet_artifact"
  473. or record.get("canonical_input_binding_hash")
  474. != plan["input_binding_hash"]
  475. or record.get("canonical_output_binding_hash")
  476. != plan["output_binding_hash"]
  477. ):
  478. raise NodeExecutionError(
  479. "published rule plan canonical attestation does not match"
  480. )
  481. adapter = self.adapters.get(backend)
  482. if adapter is None or not callable(getattr(adapter, "execute", None)):
  483. raise NodeExecutionError("rule plan backend is not registered")
  484. adapter_parameters = parameters
  485. staging_input = None
  486. if backend == "sql_pushdown" and parameters not in ({}, None):
  487. if (
  488. not isinstance(parameters, dict)
  489. or set(parameters) != {"input_artifact"}
  490. or not isinstance(parameters["input_artifact"], str)
  491. ):
  492. raise NodeExecutionError(
  493. "SQL rule handoff must contain one opaque receipt"
  494. )
  495. staging_input = parameters["input_artifact"]
  496. adapter_parameters = {}
  497. rule_run_id, replay = self._start_evidence(
  498. component_binding_id=component_binding_id,
  499. rule_version_id=rule_version_id,
  500. plan_hash=plan_hash,
  501. execution_context=execution_context,
  502. )
  503. if isinstance(replay, dict):
  504. status = replay.get("status")
  505. if status == "success":
  506. return {
  507. key: value
  508. for key, value in replay.items()
  509. if key != "status"
  510. }
  511. if status in {"failed", "unknown", "cancelled"}:
  512. raise NodeExecutionError(
  513. "rule execution was already finalized",
  514. commit_outcome=str(
  515. replay.get("commit_outcome")
  516. or "not_applicable"
  517. ),
  518. )
  519. raise NodeExecutionError("rule execution is already in progress")
  520. if self.evidence_writer is not None:
  521. try:
  522. self.evidence_writer.heartbeat(
  523. rule_run_id,
  524. execution_context.get("task_jti"),
  525. )
  526. except Exception as exc:
  527. raise NodeExecutionError(
  528. "rule execution lease is unavailable"
  529. ) from exc
  530. if staging_input is not None:
  531. if self.evidence_writer is None:
  532. raise NodeExecutionError(
  533. "SQL staging receipt resolver is unavailable"
  534. )
  535. try:
  536. self.evidence_writer.resolve_sql_staging(
  537. staging_input,
  538. deployment_id=execution_context.get(
  539. "deployment_id"
  540. ),
  541. correlation_id=execution_context.get(
  542. "correlation_id"
  543. ),
  544. input_binding_id=plan["input_binding_id"],
  545. )
  546. except Exception as exc:
  547. self._finish_evidence(
  548. rule_run_id,
  549. {
  550. "status": "failed",
  551. "commit_outcome": "not_committed",
  552. "timings": {"duration_ms": 0},
  553. },
  554. )
  555. raise NodeExecutionError(
  556. "SQL staging receipt is not executable"
  557. ) from exc
  558. adapter_context = {}
  559. if backend == "polars_batch":
  560. adapter_context["correlation_id"] = execution_context.get(
  561. "correlation_id"
  562. )
  563. started = time.monotonic()
  564. heartbeat_stop = threading.Event()
  565. heartbeat_failures = []
  566. heartbeat_thread = None
  567. if self.evidence_writer is not None:
  568. interval = float(
  569. getattr(
  570. self.evidence_writer,
  571. "heartbeat_interval_seconds",
  572. 10.0,
  573. )
  574. )
  575. def renew_lease():
  576. while not heartbeat_stop.wait(interval):
  577. try:
  578. self.evidence_writer.heartbeat(
  579. rule_run_id,
  580. execution_context.get("task_jti"),
  581. )
  582. except Exception as exc:
  583. heartbeat_failures.append(exc)
  584. heartbeat_stop.set()
  585. heartbeat_thread = threading.Thread(
  586. target=renew_lease,
  587. name=f"rule-heartbeat-{rule_run_id}",
  588. daemon=True,
  589. )
  590. heartbeat_thread.start()
  591. try:
  592. result = adapter.execute(
  593. plan=record["plan"],
  594. node=node,
  595. parameters=adapter_parameters,
  596. write_authorized=write_authorized,
  597. **adapter_context,
  598. )
  599. except CancelledError:
  600. self._finish_evidence(
  601. rule_run_id,
  602. {
  603. "status": "cancelled",
  604. "commit_outcome": "not_applicable",
  605. "timings": {
  606. "duration_ms": int(
  607. (time.monotonic() - started) * 1000
  608. )
  609. },
  610. },
  611. )
  612. raise
  613. except NodeExecutionError as exc:
  614. self._finish_evidence(
  615. rule_run_id,
  616. {
  617. "status": (
  618. "unknown"
  619. if exc.commit_outcome == "unknown"
  620. else "failed"
  621. ),
  622. "commit_outcome": exc.commit_outcome,
  623. "timings": {
  624. "duration_ms": int(
  625. (time.monotonic() - started) * 1000
  626. )
  627. },
  628. },
  629. )
  630. raise
  631. except Exception as exc:
  632. self._finish_evidence(
  633. rule_run_id,
  634. {
  635. "status": "failed",
  636. "commit_outcome": "not_committed",
  637. "timings": {
  638. "duration_ms": int(
  639. (time.monotonic() - started) * 1000
  640. )
  641. },
  642. },
  643. )
  644. raise NodeExecutionError(
  645. "rule plan execution failed",
  646. commit_outcome="not_committed",
  647. ) from exc
  648. finally:
  649. heartbeat_stop.set()
  650. if heartbeat_thread is not None:
  651. heartbeat_thread.join(timeout=5)
  652. if heartbeat_failures:
  653. result_outcome = (
  654. result.get("commit_outcome")
  655. if isinstance(result, dict)
  656. else None
  657. )
  658. evidence_outcome = (
  659. "committed"
  660. if result_outcome == "committed"
  661. else "unknown"
  662. )
  663. self._finish_evidence(
  664. rule_run_id,
  665. {
  666. "status": "unknown",
  667. "commit_outcome": evidence_outcome,
  668. "timings": {
  669. "duration_ms": int(
  670. (time.monotonic() - started) * 1000
  671. )
  672. },
  673. },
  674. )
  675. raise NodeExecutionError(
  676. "rule execution lease outcome is unknown",
  677. commit_outcome="unknown",
  678. )
  679. if not isinstance(result, dict):
  680. error = NodeExecutionError(
  681. "rule plan result must be an object"
  682. )
  683. self._finish_evidence(
  684. rule_run_id,
  685. {
  686. "status": "failed",
  687. "commit_outcome": error.commit_outcome,
  688. "timings": {
  689. "duration_ms": int(
  690. (time.monotonic() - started) * 1000
  691. )
  692. },
  693. },
  694. )
  695. raise error
  696. adapter_commit_outcome = str(
  697. result.get("commit_outcome", "not_applicable")
  698. )
  699. staging_output = None
  700. if backend == "sql_pushdown" and self.evidence_writer is not None:
  701. try:
  702. staging_output = self.evidence_writer.stage_sql_output(
  703. rule_run_id,
  704. output_binding_id=plan["output_binding_id"],
  705. )
  706. except Exception as exc:
  707. self._finish_evidence(
  708. rule_run_id,
  709. {
  710. "status": "unknown",
  711. "commit_outcome": (
  712. "committed"
  713. if adapter_commit_outcome == "committed"
  714. else "unknown"
  715. ),
  716. "timings": {
  717. "duration_ms": int(
  718. (time.monotonic() - started) * 1000
  719. )
  720. },
  721. },
  722. )
  723. raise NodeExecutionError(
  724. "SQL staging receipt outcome is unknown",
  725. commit_outcome="unknown",
  726. ) from exc
  727. try:
  728. sample = self._redacted_sample(
  729. result.pop("_violation_sample", None)
  730. )
  731. public_result = {
  732. **result,
  733. "component_binding_id": component_binding_id,
  734. "rule_version_id": rule_version_id,
  735. "execution_plan_hash": plan_hash,
  736. }
  737. for metric in (
  738. "rows_in",
  739. "rows_out",
  740. "rows_rejected",
  741. "rows_quarantined",
  742. ):
  743. value = public_result.get(metric, 0)
  744. if isinstance(value, bool):
  745. raise ValueError("boolean metric")
  746. public_result[metric] = max(0, int(value))
  747. artifact_ref = public_result.get("artifact_ref")
  748. if isinstance(artifact_ref, str):
  749. public_result["output_artifact"] = artifact_ref
  750. elif (
  751. backend == "sql_pushdown"
  752. and staging_output is not None
  753. ):
  754. public_result["output_artifact"] = staging_output
  755. public_result = validate_public_rule_result(
  756. public_result,
  757. backend=backend,
  758. )
  759. except (TypeError, ValueError) as exc:
  760. post_commit_unknown = adapter_commit_outcome in {
  761. "committed",
  762. "unknown",
  763. }
  764. self._finish_evidence(
  765. rule_run_id,
  766. {
  767. "status": (
  768. "unknown"
  769. if post_commit_unknown
  770. else "failed"
  771. ),
  772. "commit_outcome": (
  773. adapter_commit_outcome
  774. if post_commit_unknown
  775. else "not_committed"
  776. ),
  777. "timings": {
  778. "duration_ms": int(
  779. (time.monotonic() - started) * 1000
  780. )
  781. },
  782. },
  783. )
  784. raise NodeExecutionError(
  785. "rule plan result is invalid",
  786. commit_outcome=(
  787. "unknown"
  788. if post_commit_unknown
  789. else "not_committed"
  790. ),
  791. ) from exc
  792. evidence_result = {
  793. "status": "success",
  794. "rows_in": max(0, int(public_result.get("rows_in", 0))),
  795. "rows_out": max(0, int(public_result.get("rows_out", 0))),
  796. "rows_rejected": max(
  797. 0, int(public_result.get("rows_rejected", 0))
  798. ),
  799. "rows_quarantined": max(
  800. 0, int(public_result.get("rows_quarantined", 0))
  801. ),
  802. "commit_outcome": str(
  803. public_result.get("commit_outcome", "not_applicable")
  804. ),
  805. "timings": {
  806. "duration_ms": int(
  807. (time.monotonic() - started) * 1000
  808. )
  809. },
  810. "public_result": public_result,
  811. }
  812. if sample:
  813. evidence_result.update(
  814. {
  815. "violation_sample": sample,
  816. "sample_count": len(sample),
  817. "redaction_policy": (
  818. "rule-violation-default-v1"
  819. ),
  820. }
  821. )
  822. self._finish_evidence(rule_run_id, evidence_result)
  823. return public_result