|
|
@@ -865,7 +865,7 @@ class DataRuleRepository:
|
|
|
"plan_hash, schema_hashes, status) "
|
|
|
"VALUES (CAST(:id AS uuid), CAST(:binding_id AS uuid), "
|
|
|
":backend, :compiler_version, CAST(:plan AS jsonb), "
|
|
|
- ":plan_hash, CAST(:schema_hashes AS jsonb), 'published')"
|
|
|
+ ":plan_hash, CAST(:schema_hashes AS jsonb), 'compiled')"
|
|
|
),
|
|
|
{
|
|
|
"id": new_governance_uid(),
|
|
|
@@ -878,6 +878,157 @@ class DataRuleRepository:
|
|
|
},
|
|
|
)
|
|
|
|
|
|
+ def load_bound_compile_context(
|
|
|
+ self,
|
|
|
+ *,
|
|
|
+ component_binding_id: str,
|
|
|
+ rule_version_id: str,
|
|
|
+ input_schema_snapshot_id: str,
|
|
|
+ output_schema_snapshot_id: str,
|
|
|
+ input_binding_id: str,
|
|
|
+ output_binding_id: str,
|
|
|
+ ) -> dict[str, Any] | None:
|
|
|
+ """Load all physical compilation inputs through one canonical join."""
|
|
|
+
|
|
|
+ ids = {
|
|
|
+ "component_binding_id": _uid(
|
|
|
+ component_binding_id, "component_binding_id"
|
|
|
+ ),
|
|
|
+ "rule_version_id": _uid(rule_version_id, "rule_version_id"),
|
|
|
+ "input_schema_snapshot_id": _uid(
|
|
|
+ input_schema_snapshot_id, "input_schema_snapshot_id"
|
|
|
+ ),
|
|
|
+ "output_schema_snapshot_id": _uid(
|
|
|
+ output_schema_snapshot_id, "output_schema_snapshot_id"
|
|
|
+ ),
|
|
|
+ "input_binding_id": _uid(input_binding_id, "input_binding_id"),
|
|
|
+ "output_binding_id": _uid(output_binding_id, "output_binding_id"),
|
|
|
+ }
|
|
|
+ row = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT cb.id::text AS component_binding_id, "
|
|
|
+ "cb.rule_version_id::text AS component_rule_version_id, "
|
|
|
+ "cb.component_kind, cb.idempotency, "
|
|
|
+ "rv.id::text AS rule_version_id, rv.rule_spec, "
|
|
|
+ "rv.spec_hash, rv.status AS rule_status, "
|
|
|
+ "ins.id::text AS input_schema_snapshot_id, "
|
|
|
+ "ins.schema_ref AS input_schema_ref, "
|
|
|
+ "ins.schema_hash AS input_schema_hash, "
|
|
|
+ "ins.fields AS input_schema_fields, "
|
|
|
+ "ins.source_revision AS input_source_revision, "
|
|
|
+ "outs.id::text AS output_schema_snapshot_id, "
|
|
|
+ "outs.schema_ref AS output_schema_ref, "
|
|
|
+ "outs.schema_hash AS output_schema_hash, "
|
|
|
+ "outs.fields AS output_schema_fields, "
|
|
|
+ "outs.source_revision AS output_source_revision, "
|
|
|
+ "ib.id::text AS input_binding_id, "
|
|
|
+ "ib.data_source_uid::text AS input_data_source_uid, "
|
|
|
+ "ib.object_kind AS input_object_kind, "
|
|
|
+ "ib.object_ref AS input_object_ref, "
|
|
|
+ "ib.access_mode AS input_access_mode, "
|
|
|
+ "ib.dialect AS input_dialect, "
|
|
|
+ "ib.write_mode AS input_write_mode, "
|
|
|
+ "ob.id::text AS output_binding_id, "
|
|
|
+ "ob.data_source_uid::text AS output_data_source_uid, "
|
|
|
+ "ob.object_kind AS output_object_kind, "
|
|
|
+ "ob.object_ref AS output_object_ref, "
|
|
|
+ "ob.access_mode AS output_access_mode, "
|
|
|
+ "ob.dialect AS output_dialect, "
|
|
|
+ "ob.write_mode AS output_write_mode "
|
|
|
+ "FROM public.dataflow_component_bindings cb "
|
|
|
+ "JOIN public.data_rule_versions rv "
|
|
|
+ "ON rv.id = cb.rule_version_id "
|
|
|
+ "JOIN public.dataflow_deployments d "
|
|
|
+ "ON d.dataflow_version_id = cb.dataflow_version_id "
|
|
|
+ "JOIN public.dataflow_dataset_bindings ib "
|
|
|
+ "ON ib.dataflow_deployment_id = d.id "
|
|
|
+ "JOIN public.data_schema_snapshots ins "
|
|
|
+ "ON ins.id = ib.schema_snapshot_id "
|
|
|
+ "JOIN public.dataflow_dataset_bindings ob "
|
|
|
+ "ON ob.dataflow_deployment_id = d.id "
|
|
|
+ "JOIN public.data_schema_snapshots outs "
|
|
|
+ "ON outs.id = ob.schema_snapshot_id "
|
|
|
+ "WHERE cb.id = CAST(:component_binding_id AS uuid) "
|
|
|
+ "AND rv.id = CAST(:rule_version_id AS uuid) "
|
|
|
+ "AND ins.id = CAST(:input_schema_snapshot_id AS uuid) "
|
|
|
+ "AND outs.id = CAST(:output_schema_snapshot_id AS uuid) "
|
|
|
+ "AND ib.id = CAST(:input_binding_id AS uuid) "
|
|
|
+ "AND ob.id = CAST(:output_binding_id AS uuid) "
|
|
|
+ "/* canonical_bound_compile_context */"
|
|
|
+ ),
|
|
|
+ ids,
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one_or_none()
|
|
|
+ )
|
|
|
+ if row is None:
|
|
|
+ return None
|
|
|
+
|
|
|
+ def binding(prefix: str, snapshot_id: str) -> dict[str, Any]:
|
|
|
+ return {
|
|
|
+ "id": str(row[f"{prefix}_binding_id"]),
|
|
|
+ "data_source_uid": str(row[f"{prefix}_data_source_uid"]),
|
|
|
+ "object_kind": str(row[f"{prefix}_object_kind"]),
|
|
|
+ "object_ref": str(row[f"{prefix}_object_ref"]),
|
|
|
+ "schema_snapshot_id": snapshot_id,
|
|
|
+ "access_mode": str(row[f"{prefix}_access_mode"]),
|
|
|
+ "dialect": str(row[f"{prefix}_dialect"]),
|
|
|
+ "write_mode": row[f"{prefix}_write_mode"],
|
|
|
+ }
|
|
|
+
|
|
|
+ input_snapshot_id = str(row["input_schema_snapshot_id"])
|
|
|
+ output_snapshot_id = str(row["output_schema_snapshot_id"])
|
|
|
+ rule_spec = _object(row["rule_spec"], "rule_spec")
|
|
|
+ dialect = str(row["input_dialect"]).strip().lower()
|
|
|
+ dialect = "postgresql" if dialect == "postgres" else dialect
|
|
|
+ if dialect not in {"postgresql", "mysql"}:
|
|
|
+ raise ValueError("canonical SQL binding dialect is unsupported")
|
|
|
+ backend = {
|
|
|
+ "dialect": dialect,
|
|
|
+ "timezone": str(rule_spec.get("timezone") or ""),
|
|
|
+ "collation": (
|
|
|
+ "C" if dialect == "postgresql" else "utf8mb4_0900_bin"
|
|
|
+ ),
|
|
|
+ "rounding_mode": "half_away_from_zero",
|
|
|
+ "regex_engine": "posix" if dialect == "postgresql" else "icu",
|
|
|
+ }
|
|
|
+ return {
|
|
|
+ "component_binding": {
|
|
|
+ "id": str(row["component_binding_id"]),
|
|
|
+ "rule_version_id": str(row["component_rule_version_id"]),
|
|
|
+ "component_kind": str(row["component_kind"]),
|
|
|
+ "idempotency": row["idempotency"],
|
|
|
+ },
|
|
|
+ "rule_version": {
|
|
|
+ "id": str(row["rule_version_id"]),
|
|
|
+ "rule_spec": rule_spec,
|
|
|
+ "spec_hash": str(row["spec_hash"]),
|
|
|
+ "status": str(row["rule_status"]),
|
|
|
+ },
|
|
|
+ "input_schema": {
|
|
|
+ "id": input_snapshot_id,
|
|
|
+ "schema_ref": str(row["input_schema_ref"]),
|
|
|
+ "schema_hash": str(row["input_schema_hash"]),
|
|
|
+ "fields": _array(
|
|
|
+ row["input_schema_fields"], "input schema fields"
|
|
|
+ ),
|
|
|
+ "source_revision": str(row["input_source_revision"]),
|
|
|
+ },
|
|
|
+ "output_schema": {
|
|
|
+ "id": output_snapshot_id,
|
|
|
+ "schema_ref": str(row["output_schema_ref"]),
|
|
|
+ "schema_hash": str(row["output_schema_hash"]),
|
|
|
+ "fields": _array(
|
|
|
+ row["output_schema_fields"], "output schema fields"
|
|
|
+ ),
|
|
|
+ "source_revision": str(row["output_source_revision"]),
|
|
|
+ },
|
|
|
+ "input_binding": binding("input", input_snapshot_id),
|
|
|
+ "output_binding": binding("output", output_snapshot_id),
|
|
|
+ "backend": backend,
|
|
|
+ }
|
|
|
+
|
|
|
def persist_bound_component_plan(
|
|
|
self,
|
|
|
*,
|
|
|
@@ -930,6 +1081,7 @@ class DataRuleRepository:
|
|
|
plan["rule_version_id"] != rule_id
|
|
|
or plan["input_binding_id"] != input_id
|
|
|
or plan["output_binding_id"] != output_id
|
|
|
+ or plan["compiler_version"] != compiler_version
|
|
|
):
|
|
|
raise ValueError("compiled bound SQL plan identifiers do not match")
|
|
|
|
|
|
@@ -937,7 +1089,10 @@ class DataRuleRepository:
|
|
|
self.session.execute(
|
|
|
text(
|
|
|
"SELECT cb.id::text AS component_binding_id, "
|
|
|
+ "rv.spec_hash AS rule_spec_hash, "
|
|
|
+ "ins.id::text AS input_schema_snapshot_id, "
|
|
|
"ins.schema_hash AS input_schema_hash, "
|
|
|
+ "outs.id::text AS output_schema_snapshot_id, "
|
|
|
"outs.schema_hash AS output_schema_hash, "
|
|
|
"ib.object_ref AS input_object_ref, "
|
|
|
"ob.object_ref AS output_object_ref, "
|
|
|
@@ -945,6 +1100,8 @@ class DataRuleRepository:
|
|
|
"ib.dialect AS input_dialect, "
|
|
|
"ob.dialect AS output_dialect "
|
|
|
"FROM public.dataflow_component_bindings cb "
|
|
|
+ "JOIN public.data_rule_versions rv "
|
|
|
+ "ON rv.id = cb.rule_version_id "
|
|
|
"JOIN public.dataflow_deployments d "
|
|
|
"ON d.dataflow_version_id = cb.dataflow_version_id "
|
|
|
"JOIN public.dataflow_dataset_bindings ib "
|
|
|
@@ -982,6 +1139,13 @@ class DataRuleRepository:
|
|
|
or str(linkage["output_object_ref"]) != relations["output_object_ref"]
|
|
|
or str(linkage["input_dialect"]) != plan["dialect"]
|
|
|
or str(linkage["output_dialect"]) != plan["dialect"]
|
|
|
+ or str(linkage["rule_spec_hash"]) != plan["rule_spec_hash"]
|
|
|
+ or str(linkage["input_schema_snapshot_id"])
|
|
|
+ != plan["input_schema_snapshot_id"]
|
|
|
+ or str(linkage["input_schema_hash"]) != plan["input_schema_hash"]
|
|
|
+ or str(linkage["output_schema_snapshot_id"])
|
|
|
+ != plan["output_schema_snapshot_id"]
|
|
|
+ or str(linkage["output_schema_hash"]) != plan["output_schema_hash"]
|
|
|
):
|
|
|
raise ValueError(
|
|
|
"bound SQL plan does not match its physical dataset bindings"
|
|
|
@@ -1005,14 +1169,15 @@ class DataRuleRepository:
|
|
|
"plan_hash": plan_hash,
|
|
|
"schema_hashes": _json(
|
|
|
{
|
|
|
- "input": _digest(
|
|
|
- linkage["input_schema_hash"],
|
|
|
- "input_schema_hash",
|
|
|
- ),
|
|
|
- "output": _digest(
|
|
|
- linkage["output_schema_hash"],
|
|
|
- "output_schema_hash",
|
|
|
- ),
|
|
|
+ "rule_spec_hash": plan["rule_spec_hash"],
|
|
|
+ "input_schema_snapshot_id": plan[
|
|
|
+ "input_schema_snapshot_id"
|
|
|
+ ],
|
|
|
+ "input_schema_hash": plan["input_schema_hash"],
|
|
|
+ "output_schema_snapshot_id": plan[
|
|
|
+ "output_schema_snapshot_id"
|
|
|
+ ],
|
|
|
+ "output_schema_hash": plan["output_schema_hash"],
|
|
|
}
|
|
|
),
|
|
|
"status": status,
|
|
|
@@ -1028,6 +1193,112 @@ class DataRuleRepository:
|
|
|
"plan_hash": plan_hash,
|
|
|
}
|
|
|
|
|
|
+ def record_bound_plan_test(
|
|
|
+ self,
|
|
|
+ *,
|
|
|
+ plan_id: str,
|
|
|
+ evidence: dict[str, Any],
|
|
|
+ test_kind: str = "integration_preflight",
|
|
|
+ ) -> dict[str, Any]:
|
|
|
+ """Attest integration evidence and transition compiled to tested."""
|
|
|
+
|
|
|
+ execution_plan_id = _uid(plan_id, "plan_id")
|
|
|
+ kind = _text(test_kind, "test_kind", 40)
|
|
|
+ if not isinstance(evidence, dict):
|
|
|
+ raise ValueError("test evidence must be an object")
|
|
|
+ rows_in = evidence.get("rows_in")
|
|
|
+ rows_out = evidence.get("rows_out")
|
|
|
+ rows_rejected = evidence.get("rows_rejected")
|
|
|
+ counts = (rows_in, rows_out, rows_rejected)
|
|
|
+ if (
|
|
|
+ evidence.get("commit_outcome") != "committed"
|
|
|
+ or any(
|
|
|
+ isinstance(value, bool)
|
|
|
+ or not isinstance(value, int)
|
|
|
+ or value < 0
|
|
|
+ for value in counts
|
|
|
+ )
|
|
|
+ or rows_out > rows_in
|
|
|
+ or rows_rejected != rows_in - rows_out
|
|
|
+ ):
|
|
|
+ raise ValueError("successful integration preflight evidence is required")
|
|
|
+ evidence_hash = _canonical_hash(evidence)
|
|
|
+ row = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "WITH eligible AS ("
|
|
|
+ "SELECT id FROM public.rule_execution_plans "
|
|
|
+ "WHERE id = CAST(:plan_id AS uuid) "
|
|
|
+ "AND backend = 'sql_pushdown' AND status = 'compiled' "
|
|
|
+ "FOR UPDATE"
|
|
|
+ "), inserted AS ("
|
|
|
+ "INSERT INTO public.rule_test_evidence "
|
|
|
+ "(id, rule_execution_plan_id, test_kind, evidence_hash, "
|
|
|
+ "status, evidence) "
|
|
|
+ "SELECT CAST(:evidence_id AS uuid), id, :test_kind, "
|
|
|
+ ":evidence_hash, 'success', CAST(:evidence AS jsonb) "
|
|
|
+ "FROM eligible RETURNING rule_execution_plan_id"
|
|
|
+ ") "
|
|
|
+ "UPDATE public.rule_execution_plans p SET status = 'tested' "
|
|
|
+ "FROM inserted i WHERE p.id = i.rule_execution_plan_id "
|
|
|
+ "RETURNING p.id::text AS id, p.status"
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "plan_id": execution_plan_id,
|
|
|
+ "evidence_id": new_governance_uid(),
|
|
|
+ "test_kind": kind,
|
|
|
+ "evidence_hash": evidence_hash,
|
|
|
+ "evidence": _json(evidence),
|
|
|
+ },
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one_or_none()
|
|
|
+ )
|
|
|
+ if row is None:
|
|
|
+ raise ValueError("only a compiled SQL plan may be tested")
|
|
|
+ return {
|
|
|
+ "id": str(row["id"]),
|
|
|
+ "status": str(row["status"]),
|
|
|
+ "evidence_hash": evidence_hash,
|
|
|
+ }
|
|
|
+
|
|
|
+ def publish_tested_bound_plan(self, *, plan_id: str) -> dict[str, Any]:
|
|
|
+ """Publish only a tested plan with canonical successful evidence."""
|
|
|
+
|
|
|
+ execution_plan_id = _uid(plan_id, "plan_id")
|
|
|
+ row = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "UPDATE public.rule_execution_plans p "
|
|
|
+ "SET status = 'published' "
|
|
|
+ "FROM public.dataflow_component_bindings cb, "
|
|
|
+ "public.data_rule_versions rv "
|
|
|
+ "WHERE p.id = CAST(:plan_id AS uuid) "
|
|
|
+ "AND p.status = 'tested' "
|
|
|
+ "AND cb.id = p.component_binding_id "
|
|
|
+ "AND rv.id = cb.rule_version_id "
|
|
|
+ "AND rv.status = 'published' "
|
|
|
+ "AND EXISTS (SELECT 1 FROM public.rule_test_evidence e "
|
|
|
+ "WHERE e.rule_execution_plan_id = p.id "
|
|
|
+ "AND e.test_kind = 'integration_preflight' "
|
|
|
+ "AND e.status = 'success') "
|
|
|
+ "RETURNING p.id::text AS id, p.status, p.plan_hash"
|
|
|
+ ),
|
|
|
+ {"plan_id": execution_plan_id},
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one_or_none()
|
|
|
+ )
|
|
|
+ if row is None:
|
|
|
+ raise ValueError(
|
|
|
+ "only a successfully tested canonical plan may be published"
|
|
|
+ )
|
|
|
+ return {
|
|
|
+ "id": str(row["id"]),
|
|
|
+ "status": str(row["status"]),
|
|
|
+ "plan_hash": str(row["plan_hash"]),
|
|
|
+ }
|
|
|
+
|
|
|
def complete_dataflow_release(
|
|
|
self,
|
|
|
*,
|