|
|
@@ -2,8 +2,8 @@
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
-import hashlib
|
|
|
import json
|
|
|
+import hashlib
|
|
|
import re
|
|
|
from typing import Any
|
|
|
|
|
|
@@ -28,6 +28,7 @@ from app.core.data_rules.execution_contracts import (
|
|
|
validate_schema_snapshot,
|
|
|
)
|
|
|
|
|
|
+
|
|
|
RULE_CATEGORIES = {
|
|
|
"general",
|
|
|
"reusable",
|
|
|
@@ -178,11 +179,15 @@ class DataRuleRepository:
|
|
|
"id": generation_id,
|
|
|
"rule_version_id": version_id,
|
|
|
"authoring_surface": surface,
|
|
|
- "source_text_hash": hashlib.sha256(source.encode("utf-8")).hexdigest(),
|
|
|
+ "source_text_hash": hashlib.sha256(
|
|
|
+ source.encode("utf-8")
|
|
|
+ ).hexdigest(),
|
|
|
"model_provider": _text(
|
|
|
evidence.get("model_provider"), "model_provider", 80
|
|
|
),
|
|
|
- "model_name": _text(evidence.get("model_name"), "model_name", 120),
|
|
|
+ "model_name": _text(
|
|
|
+ evidence.get("model_name"), "model_name", 120
|
|
|
+ ),
|
|
|
"prompt_version": _text(
|
|
|
evidence.get("prompt_version"), "prompt_version", 80
|
|
|
),
|
|
|
@@ -196,7 +201,9 @@ class DataRuleRepository:
|
|
|
"ambiguities": _json(candidate["ambiguities"]),
|
|
|
"repair_attempts": repair_attempts,
|
|
|
"decision": decision,
|
|
|
- "decision_detail": _json({"explanation": candidate["explanation"]}),
|
|
|
+ "decision_detail": _json(
|
|
|
+ {"explanation": candidate["explanation"]}
|
|
|
+ ),
|
|
|
"correlation_id": correlation_id,
|
|
|
},
|
|
|
)
|
|
|
@@ -234,20 +241,16 @@ class DataRuleRepository:
|
|
|
rule_uid = spec["rule_uid"]
|
|
|
digest = rule_spec_hash(spec)
|
|
|
self._lock(rule_uid)
|
|
|
- existing = (
|
|
|
- self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT id::text AS id, version_no, status "
|
|
|
- "FROM public.data_rule_versions "
|
|
|
- "WHERE rule_uid = CAST(:rule_uid AS uuid) "
|
|
|
- "AND spec_hash = :spec_hash "
|
|
|
- "/* existing_version */"
|
|
|
- ),
|
|
|
- {"rule_uid": rule_uid, "spec_hash": digest},
|
|
|
- )
|
|
|
- .mappings()
|
|
|
- .one_or_none()
|
|
|
- )
|
|
|
+ existing = self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT id::text AS id, version_no, status "
|
|
|
+ "FROM public.data_rule_versions "
|
|
|
+ "WHERE rule_uid = CAST(:rule_uid AS uuid) "
|
|
|
+ "AND spec_hash = :spec_hash "
|
|
|
+ "/* existing_version */"
|
|
|
+ ),
|
|
|
+ {"rule_uid": rule_uid, "spec_hash": digest},
|
|
|
+ ).mappings().one_or_none()
|
|
|
if existing is not None:
|
|
|
return {
|
|
|
**dict(existing),
|
|
|
@@ -321,21 +324,17 @@ class DataRuleRepository:
|
|
|
) -> dict[str, Any]:
|
|
|
version = _uid(version_id, "version_id")
|
|
|
_uid(published_by, "published_by")
|
|
|
- row = (
|
|
|
- self.session.execute(
|
|
|
- text(
|
|
|
- "UPDATE public.data_rule_versions "
|
|
|
- "SET status = 'published', published_at = CURRENT_TIMESTAMP "
|
|
|
- "WHERE id = CAST(:version_id AS uuid) "
|
|
|
- "AND status = 'validated' "
|
|
|
- "RETURNING id::text AS id, rule_uid::text AS rule_uid, "
|
|
|
- "version_no, status, spec_hash"
|
|
|
- ),
|
|
|
- {"version_id": version},
|
|
|
- )
|
|
|
- .mappings()
|
|
|
- .one_or_none()
|
|
|
- )
|
|
|
+ row = self.session.execute(
|
|
|
+ text(
|
|
|
+ "UPDATE public.data_rule_versions "
|
|
|
+ "SET status = 'published', published_at = CURRENT_TIMESTAMP "
|
|
|
+ "WHERE id = CAST(:version_id AS uuid) "
|
|
|
+ "AND status = 'validated' "
|
|
|
+ "RETURNING id::text AS id, rule_uid::text AS rule_uid, "
|
|
|
+ "version_no, status, spec_hash"
|
|
|
+ ),
|
|
|
+ {"version_id": version},
|
|
|
+ ).mappings().one_or_none()
|
|
|
if row is not None:
|
|
|
return dict(row)
|
|
|
status = self.session.execute(
|
|
|
@@ -368,39 +367,33 @@ class DataRuleRepository:
|
|
|
{clause["rule_version_id"] for clause in spec["clauses"]}
|
|
|
)
|
|
|
|
|
|
- published_rows = (
|
|
|
- self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT id::text AS id "
|
|
|
- "FROM public.data_rule_versions "
|
|
|
- "WHERE id = ANY(CAST(:rule_version_ids AS uuid[])) "
|
|
|
- "AND status = 'published' "
|
|
|
- "/* published_rule_reference */"
|
|
|
- ),
|
|
|
- {"rule_version_ids": rule_version_ids},
|
|
|
- )
|
|
|
- .mappings()
|
|
|
- .all()
|
|
|
- )
|
|
|
+ published_rows = self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT id::text AS id "
|
|
|
+ "FROM public.data_rule_versions "
|
|
|
+ "WHERE id = ANY(CAST(:rule_version_ids AS uuid[])) "
|
|
|
+ "AND status = 'published' "
|
|
|
+ "/* published_rule_reference */"
|
|
|
+ ),
|
|
|
+ {"rule_version_ids": rule_version_ids},
|
|
|
+ ).mappings().all()
|
|
|
published_ids = {str(row["id"]) for row in published_rows}
|
|
|
if published_ids != set(rule_version_ids):
|
|
|
- raise ValueError("standard clauses require published rule versions")
|
|
|
+ raise ValueError(
|
|
|
+ "standard clauses require published rule versions"
|
|
|
+ )
|
|
|
|
|
|
self._lock(standard_uid)
|
|
|
- existing = (
|
|
|
- self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT id::text AS id, version_no, status "
|
|
|
- "FROM public.data_standard_versions "
|
|
|
- "WHERE standard_uid = CAST(:standard_uid AS uuid) "
|
|
|
- "AND spec_hash = :spec_hash "
|
|
|
- "/* existing_version */"
|
|
|
- ),
|
|
|
- {"standard_uid": standard_uid, "spec_hash": digest},
|
|
|
- )
|
|
|
- .mappings()
|
|
|
- .one_or_none()
|
|
|
- )
|
|
|
+ existing = self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT id::text AS id, version_no, status "
|
|
|
+ "FROM public.data_standard_versions "
|
|
|
+ "WHERE standard_uid = CAST(:standard_uid AS uuid) "
|
|
|
+ "AND spec_hash = :spec_hash "
|
|
|
+ "/* existing_version */"
|
|
|
+ ),
|
|
|
+ {"standard_uid": standard_uid, "spec_hash": digest},
|
|
|
+ ).mappings().one_or_none()
|
|
|
if existing is not None:
|
|
|
return {
|
|
|
**dict(existing),
|
|
|
@@ -477,7 +470,9 @@ class DataRuleRepository:
|
|
|
},
|
|
|
)
|
|
|
except IntegrityError as exc:
|
|
|
- raise ValueError("standard version conflicts with existing data") from exc
|
|
|
+ raise ValueError(
|
|
|
+ "standard version conflicts with existing data"
|
|
|
+ ) from exc
|
|
|
return {
|
|
|
"id": version_id,
|
|
|
"standard_uid": standard_uid,
|
|
|
@@ -492,21 +487,17 @@ class DataRuleRepository:
|
|
|
) -> dict[str, Any]:
|
|
|
version = _uid(version_id, "version_id")
|
|
|
_uid(published_by, "published_by")
|
|
|
- row = (
|
|
|
- self.session.execute(
|
|
|
- text(
|
|
|
- "UPDATE public.data_standard_versions "
|
|
|
- "SET status = 'published', published_at = CURRENT_TIMESTAMP "
|
|
|
- "WHERE id = CAST(:version_id AS uuid) "
|
|
|
- "AND status = 'validated' "
|
|
|
- "RETURNING id::text AS id, standard_uid::text AS standard_uid, "
|
|
|
- "version_no, status, spec_hash"
|
|
|
- ),
|
|
|
- {"version_id": version},
|
|
|
- )
|
|
|
- .mappings()
|
|
|
- .one_or_none()
|
|
|
- )
|
|
|
+ row = self.session.execute(
|
|
|
+ text(
|
|
|
+ "UPDATE public.data_standard_versions "
|
|
|
+ "SET status = 'published', published_at = CURRENT_TIMESTAMP "
|
|
|
+ "WHERE id = CAST(:version_id AS uuid) "
|
|
|
+ "AND status = 'validated' "
|
|
|
+ "RETURNING id::text AS id, standard_uid::text AS standard_uid, "
|
|
|
+ "version_no, status, spec_hash"
|
|
|
+ ),
|
|
|
+ {"version_id": version},
|
|
|
+ ).mappings().one_or_none()
|
|
|
if row is not None:
|
|
|
return dict(row)
|
|
|
status = self.session.execute(
|
|
|
@@ -518,7 +509,9 @@ class DataRuleRepository:
|
|
|
{"version_id": version},
|
|
|
).scalar_one_or_none()
|
|
|
if status == "published":
|
|
|
- raise ValueError("standard version is already published and immutable")
|
|
|
+ raise ValueError(
|
|
|
+ "standard version is already published and immutable"
|
|
|
+ )
|
|
|
if status is None:
|
|
|
raise ValueError("standard version was not found")
|
|
|
raise ValueError("only validated standard versions may be published")
|
|
|
@@ -538,29 +531,25 @@ class DataRuleRepository:
|
|
|
)
|
|
|
standard_rows = []
|
|
|
if standard_ids:
|
|
|
- standard_rows = (
|
|
|
- self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT sv.id::text AS id, sv.status, "
|
|
|
- "jsonb_agg(jsonb_build_object("
|
|
|
- "'clause_id', b.clause_id, "
|
|
|
- "'rule_version_id', b.rule_version_id::text, "
|
|
|
- "'severity', b.severity, "
|
|
|
- "'exception_policy', b.exception_policy) "
|
|
|
- "ORDER BY b.clause_id) AS clauses "
|
|
|
- "FROM public.data_standard_versions sv "
|
|
|
- "JOIN public.standard_rule_bindings b "
|
|
|
- "ON b.standard_version_id = sv.id "
|
|
|
- "WHERE sv.id = ANY(CAST(:standard_ids AS uuid[])) "
|
|
|
- "AND sv.status = 'published' "
|
|
|
- "/* published_standard_assets */ "
|
|
|
- "GROUP BY sv.id, sv.status"
|
|
|
- ),
|
|
|
- {"standard_ids": standard_ids},
|
|
|
- )
|
|
|
- .mappings()
|
|
|
- .all()
|
|
|
- )
|
|
|
+ standard_rows = self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT sv.id::text AS id, sv.status, "
|
|
|
+ "jsonb_agg(jsonb_build_object("
|
|
|
+ "'clause_id', b.clause_id, "
|
|
|
+ "'rule_version_id', b.rule_version_id::text, "
|
|
|
+ "'severity', b.severity, "
|
|
|
+ "'exception_policy', b.exception_policy) "
|
|
|
+ "ORDER BY b.clause_id) AS clauses "
|
|
|
+ "FROM public.data_standard_versions sv "
|
|
|
+ "JOIN public.standard_rule_bindings b "
|
|
|
+ "ON b.standard_version_id = sv.id "
|
|
|
+ "WHERE sv.id = ANY(CAST(:standard_ids AS uuid[])) "
|
|
|
+ "AND sv.status = 'published' "
|
|
|
+ "/* published_standard_assets */ "
|
|
|
+ "GROUP BY sv.id, sv.status"
|
|
|
+ ),
|
|
|
+ {"standard_ids": standard_ids},
|
|
|
+ ).mappings().all()
|
|
|
standards = {
|
|
|
str(row["id"]): {
|
|
|
"id": str(row["id"]),
|
|
|
@@ -581,25 +570,22 @@ class DataRuleRepository:
|
|
|
}
|
|
|
for standard in standards.values():
|
|
|
rule_ids.update(
|
|
|
- str(clause["rule_version_id"]) for clause in standard["clauses"]
|
|
|
+ str(clause["rule_version_id"])
|
|
|
+ for clause in standard["clauses"]
|
|
|
)
|
|
|
rule_rows = []
|
|
|
if rule_ids:
|
|
|
- rule_rows = (
|
|
|
- self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT rv.id::text AS id, rv.status, rv.rule_spec, "
|
|
|
- "rv.spec_hash "
|
|
|
- "FROM public.data_rule_versions rv "
|
|
|
- "WHERE rv.id = ANY(CAST(:rule_ids AS uuid[])) "
|
|
|
- "AND rv.status = 'published' "
|
|
|
- "/* published_rule_assets */"
|
|
|
- ),
|
|
|
- {"rule_ids": sorted(rule_ids)},
|
|
|
- )
|
|
|
- .mappings()
|
|
|
- .all()
|
|
|
- )
|
|
|
+ rule_rows = self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT rv.id::text AS id, rv.status, rv.rule_spec, "
|
|
|
+ "rv.spec_hash "
|
|
|
+ "FROM public.data_rule_versions rv "
|
|
|
+ "WHERE rv.id = ANY(CAST(:rule_ids AS uuid[])) "
|
|
|
+ "AND rv.status = 'published' "
|
|
|
+ "/* published_rule_assets */"
|
|
|
+ ),
|
|
|
+ {"rule_ids": sorted(rule_ids)},
|
|
|
+ ).mappings().all()
|
|
|
rules = {
|
|
|
str(row["id"]): {
|
|
|
"id": str(row["id"]),
|
|
|
@@ -610,7 +596,9 @@ class DataRuleRepository:
|
|
|
for row in rule_rows
|
|
|
}
|
|
|
if set(rules) != rule_ids:
|
|
|
- raise ValueError("dataflow references missing or unpublished rule versions")
|
|
|
+ raise ValueError(
|
|
|
+ "dataflow references missing or unpublished rule versions"
|
|
|
+ )
|
|
|
return standards, rules
|
|
|
|
|
|
def begin_dataflow_release(
|
|
|
@@ -832,7 +820,9 @@ class DataRuleRepository:
|
|
|
raise ValueError("compiled plan contains unsupported fields")
|
|
|
if plan["backend"] not in PLAN_BACKENDS:
|
|
|
raise ValueError("unsupported plan backend")
|
|
|
- compiler_version = _text(plan["compiler_version"], "compiler_version", 80)
|
|
|
+ compiler_version = _text(
|
|
|
+ plan["compiler_version"], "compiler_version", 80
|
|
|
+ )
|
|
|
plan_body = _object(plan["plan"], "compiled plan body")
|
|
|
plan_hash = _digest(plan["plan_hash"], "plan_hash")
|
|
|
if _canonical_hash(plan_body) != plan_hash:
|
|
|
@@ -863,7 +853,9 @@ class DataRuleRepository:
|
|
|
"rule_version_id": rule_id,
|
|
|
"stage": _text(stage, "stage", 30),
|
|
|
"order_no": order_no,
|
|
|
- "idempotency": _json(idempotency) if idempotency is not None else None,
|
|
|
+ "idempotency": _json(idempotency)
|
|
|
+ if idempotency is not None
|
|
|
+ else None,
|
|
|
"provenance": _json(provenance),
|
|
|
},
|
|
|
)
|
|
|
@@ -897,26 +889,22 @@ class DataRuleRepository:
|
|
|
if not isinstance(package, dict):
|
|
|
raise ValueError("production-line package must be an object")
|
|
|
package_hash = _digest(package.get("package_hash"), "package_hash")
|
|
|
- row = (
|
|
|
- self.session.execute(
|
|
|
- text(
|
|
|
- "UPDATE public.dataflow_versions "
|
|
|
- "SET package = CAST(:package AS jsonb), "
|
|
|
- "package_hash = :package_hash, status = 'released', "
|
|
|
- "released_at = CURRENT_TIMESTAMP "
|
|
|
- "WHERE id = CAST(:version_id AS uuid) "
|
|
|
- "AND status = 'validated' "
|
|
|
- "RETURNING id::text AS id, version_no, status, package_hash"
|
|
|
- ),
|
|
|
- {
|
|
|
- "version_id": version_id,
|
|
|
- "package": _json(package),
|
|
|
- "package_hash": package_hash,
|
|
|
- },
|
|
|
- )
|
|
|
- .mappings()
|
|
|
- .one_or_none()
|
|
|
- )
|
|
|
+ row = self.session.execute(
|
|
|
+ text(
|
|
|
+ "UPDATE public.dataflow_versions "
|
|
|
+ "SET package = CAST(:package AS jsonb), "
|
|
|
+ "package_hash = :package_hash, status = 'released', "
|
|
|
+ "released_at = CURRENT_TIMESTAMP "
|
|
|
+ "WHERE id = CAST(:version_id AS uuid) "
|
|
|
+ "AND status = 'validated' "
|
|
|
+ "RETURNING id::text AS id, version_no, status, package_hash"
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "version_id": version_id,
|
|
|
+ "package": _json(package),
|
|
|
+ "package_hash": package_hash,
|
|
|
+ },
|
|
|
+ ).mappings().one_or_none()
|
|
|
if row is None:
|
|
|
raise ValueError("dataflow release is not in validated state")
|
|
|
return {**dict(row), "package": package}
|
|
|
@@ -940,29 +928,25 @@ class DataRuleRepository:
|
|
|
|
|
|
standard_rows = []
|
|
|
if standard_ids:
|
|
|
- standard_rows = (
|
|
|
- self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT sv.id::text AS id, sv.status, "
|
|
|
- "jsonb_agg(jsonb_build_object("
|
|
|
- "'clause_id', b.clause_id, "
|
|
|
- "'rule_version_id', b.rule_version_id::text, "
|
|
|
- "'severity', b.severity, "
|
|
|
- "'exception_policy', b.exception_policy) "
|
|
|
- "ORDER BY b.clause_id) AS clauses "
|
|
|
- "FROM public.data_standard_versions sv "
|
|
|
- "JOIN public.standard_rule_bindings b "
|
|
|
- "ON b.standard_version_id = sv.id "
|
|
|
- "WHERE sv.id = ANY(CAST(:standard_ids AS uuid[])) "
|
|
|
- "AND sv.status = 'published' "
|
|
|
- "/* catalog_standard_versions */ "
|
|
|
- "GROUP BY sv.id, sv.status"
|
|
|
- ),
|
|
|
- {"standard_ids": standard_ids},
|
|
|
- )
|
|
|
- .mappings()
|
|
|
- .all()
|
|
|
- )
|
|
|
+ standard_rows = self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT sv.id::text AS id, sv.status, "
|
|
|
+ "jsonb_agg(jsonb_build_object("
|
|
|
+ "'clause_id', b.clause_id, "
|
|
|
+ "'rule_version_id', b.rule_version_id::text, "
|
|
|
+ "'severity', b.severity, "
|
|
|
+ "'exception_policy', b.exception_policy) "
|
|
|
+ "ORDER BY b.clause_id) AS clauses "
|
|
|
+ "FROM public.data_standard_versions sv "
|
|
|
+ "JOIN public.standard_rule_bindings b "
|
|
|
+ "ON b.standard_version_id = sv.id "
|
|
|
+ "WHERE sv.id = ANY(CAST(:standard_ids AS uuid[])) "
|
|
|
+ "AND sv.status = 'published' "
|
|
|
+ "/* catalog_standard_versions */ "
|
|
|
+ "GROUP BY sv.id, sv.status"
|
|
|
+ ),
|
|
|
+ {"standard_ids": standard_ids},
|
|
|
+ ).mappings().all()
|
|
|
standards = {
|
|
|
str(row["id"]): {
|
|
|
"id": str(row["id"]),
|
|
|
@@ -979,32 +963,29 @@ class DataRuleRepository:
|
|
|
rule_ids = set(direct_rule_ids)
|
|
|
for standard in standards.values():
|
|
|
rule_ids.update(
|
|
|
- str(clause["rule_version_id"]) for clause in standard["clauses"]
|
|
|
+ str(clause["rule_version_id"])
|
|
|
+ for clause in standard["clauses"]
|
|
|
)
|
|
|
rule_rows = []
|
|
|
if rule_ids:
|
|
|
- rule_rows = (
|
|
|
- self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT DISTINCT ON (rv.id) "
|
|
|
- "rv.id::text AS id, rv.status, rv.rule_spec, rv.spec_hash, "
|
|
|
- "p.backend, p.plan_hash "
|
|
|
- "FROM public.data_rule_versions rv "
|
|
|
- "JOIN public.dataflow_component_bindings cb "
|
|
|
- "ON cb.rule_version_id = rv.id "
|
|
|
- "JOIN public.rule_execution_plans p "
|
|
|
- "ON p.component_binding_id = cb.id "
|
|
|
- "WHERE rv.id = ANY(CAST(:rule_ids AS uuid[])) "
|
|
|
- "AND rv.status = 'published' "
|
|
|
- "AND p.status = 'published' "
|
|
|
- "/* catalog_rule_versions */ "
|
|
|
- "ORDER BY rv.id, p.created_at DESC"
|
|
|
- ),
|
|
|
- {"rule_ids": sorted(rule_ids)},
|
|
|
- )
|
|
|
- .mappings()
|
|
|
- .all()
|
|
|
- )
|
|
|
+ rule_rows = self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT DISTINCT ON (rv.id) "
|
|
|
+ "rv.id::text AS id, rv.status, rv.rule_spec, rv.spec_hash, "
|
|
|
+ "p.backend, p.plan_hash "
|
|
|
+ "FROM public.data_rule_versions rv "
|
|
|
+ "JOIN public.dataflow_component_bindings cb "
|
|
|
+ "ON cb.rule_version_id = rv.id "
|
|
|
+ "JOIN public.rule_execution_plans p "
|
|
|
+ "ON p.component_binding_id = cb.id "
|
|
|
+ "WHERE rv.id = ANY(CAST(:rule_ids AS uuid[])) "
|
|
|
+ "AND rv.status = 'published' "
|
|
|
+ "AND p.status = 'published' "
|
|
|
+ "/* catalog_rule_versions */ "
|
|
|
+ "ORDER BY rv.id, p.created_at DESC"
|
|
|
+ ),
|
|
|
+ {"rule_ids": sorted(rule_ids)},
|
|
|
+ ).mappings().all()
|
|
|
rules = {
|
|
|
str(row["id"]): {
|
|
|
"id": str(row["id"]),
|