|
|
@@ -2,8 +2,8 @@
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
-import json
|
|
|
import hashlib
|
|
|
+import json
|
|
|
import re
|
|
|
from typing import Any
|
|
|
|
|
|
@@ -23,7 +23,10 @@ from app.core.data_rules.contracts import (
|
|
|
validate_rule_spec,
|
|
|
validate_standard_spec,
|
|
|
)
|
|
|
-
|
|
|
+from app.core.data_rules.execution_contracts import (
|
|
|
+ validate_dataset_binding,
|
|
|
+ validate_schema_snapshot,
|
|
|
+)
|
|
|
|
|
|
RULE_CATEGORIES = {
|
|
|
"general",
|
|
|
@@ -175,15 +178,11 @@ class DataRuleRepository:
|
|
|
"id": generation_id,
|
|
|
"rule_version_id": version_id,
|
|
|
"authoring_surface": surface,
|
|
|
- "source_text_hash": hashlib.sha256(
|
|
|
- source.encode("utf-8")
|
|
|
- ).hexdigest(),
|
|
|
+ "source_text_hash": hashlib.sha256(source.encode("utf-8")).hexdigest(),
|
|
|
"model_provider": _text(
|
|
|
evidence.get("model_provider"), "model_provider", 80
|
|
|
),
|
|
|
- "model_name": _text(
|
|
|
- evidence.get("model_name"), "model_name", 120
|
|
|
- ),
|
|
|
+ "model_name": _text(evidence.get("model_name"), "model_name", 120),
|
|
|
"prompt_version": _text(
|
|
|
evidence.get("prompt_version"), "prompt_version", 80
|
|
|
),
|
|
|
@@ -197,9 +196,7 @@ class DataRuleRepository:
|
|
|
"ambiguities": _json(candidate["ambiguities"]),
|
|
|
"repair_attempts": repair_attempts,
|
|
|
"decision": decision,
|
|
|
- "decision_detail": _json(
|
|
|
- {"explanation": candidate["explanation"]}
|
|
|
- ),
|
|
|
+ "decision_detail": _json({"explanation": candidate["explanation"]}),
|
|
|
"correlation_id": correlation_id,
|
|
|
},
|
|
|
)
|
|
|
@@ -237,16 +234,20 @@ class DataRuleRepository:
|
|
|
rule_uid = spec["rule_uid"]
|
|
|
digest = rule_spec_hash(spec)
|
|
|
self._lock(rule_uid)
|
|
|
- existing = self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT id::text AS id, version_no, status "
|
|
|
- "FROM public.data_rule_versions "
|
|
|
- "WHERE rule_uid = CAST(:rule_uid AS uuid) "
|
|
|
- "AND spec_hash = :spec_hash "
|
|
|
- "/* existing_version */"
|
|
|
- ),
|
|
|
- {"rule_uid": rule_uid, "spec_hash": digest},
|
|
|
- ).mappings().one_or_none()
|
|
|
+ existing = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT id::text AS id, version_no, status "
|
|
|
+ "FROM public.data_rule_versions "
|
|
|
+ "WHERE rule_uid = CAST(:rule_uid AS uuid) "
|
|
|
+ "AND spec_hash = :spec_hash "
|
|
|
+ "/* existing_version */"
|
|
|
+ ),
|
|
|
+ {"rule_uid": rule_uid, "spec_hash": digest},
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one_or_none()
|
|
|
+ )
|
|
|
if existing is not None:
|
|
|
return {
|
|
|
**dict(existing),
|
|
|
@@ -320,17 +321,21 @@ class DataRuleRepository:
|
|
|
) -> dict[str, Any]:
|
|
|
version = _uid(version_id, "version_id")
|
|
|
_uid(published_by, "published_by")
|
|
|
- row = self.session.execute(
|
|
|
- text(
|
|
|
- "UPDATE public.data_rule_versions "
|
|
|
- "SET status = 'published', published_at = CURRENT_TIMESTAMP "
|
|
|
- "WHERE id = CAST(:version_id AS uuid) "
|
|
|
- "AND status = 'validated' "
|
|
|
- "RETURNING id::text AS id, rule_uid::text AS rule_uid, "
|
|
|
- "version_no, status, spec_hash"
|
|
|
- ),
|
|
|
- {"version_id": version},
|
|
|
- ).mappings().one_or_none()
|
|
|
+ row = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "UPDATE public.data_rule_versions "
|
|
|
+ "SET status = 'published', published_at = CURRENT_TIMESTAMP "
|
|
|
+ "WHERE id = CAST(:version_id AS uuid) "
|
|
|
+ "AND status = 'validated' "
|
|
|
+ "RETURNING id::text AS id, rule_uid::text AS rule_uid, "
|
|
|
+ "version_no, status, spec_hash"
|
|
|
+ ),
|
|
|
+ {"version_id": version},
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one_or_none()
|
|
|
+ )
|
|
|
if row is not None:
|
|
|
return dict(row)
|
|
|
status = self.session.execute(
|
|
|
@@ -363,33 +368,39 @@ class DataRuleRepository:
|
|
|
{clause["rule_version_id"] for clause in spec["clauses"]}
|
|
|
)
|
|
|
|
|
|
- published_rows = self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT id::text AS id "
|
|
|
- "FROM public.data_rule_versions "
|
|
|
- "WHERE id = ANY(CAST(:rule_version_ids AS uuid[])) "
|
|
|
- "AND status = 'published' "
|
|
|
- "/* published_rule_reference */"
|
|
|
- ),
|
|
|
- {"rule_version_ids": rule_version_ids},
|
|
|
- ).mappings().all()
|
|
|
+ published_rows = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT id::text AS id "
|
|
|
+ "FROM public.data_rule_versions "
|
|
|
+ "WHERE id = ANY(CAST(:rule_version_ids AS uuid[])) "
|
|
|
+ "AND status = 'published' "
|
|
|
+ "/* published_rule_reference */"
|
|
|
+ ),
|
|
|
+ {"rule_version_ids": rule_version_ids},
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .all()
|
|
|
+ )
|
|
|
published_ids = {str(row["id"]) for row in published_rows}
|
|
|
if published_ids != set(rule_version_ids):
|
|
|
- raise ValueError(
|
|
|
- "standard clauses require published rule versions"
|
|
|
- )
|
|
|
+ raise ValueError("standard clauses require published rule versions")
|
|
|
|
|
|
self._lock(standard_uid)
|
|
|
- existing = self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT id::text AS id, version_no, status "
|
|
|
- "FROM public.data_standard_versions "
|
|
|
- "WHERE standard_uid = CAST(:standard_uid AS uuid) "
|
|
|
- "AND spec_hash = :spec_hash "
|
|
|
- "/* existing_version */"
|
|
|
- ),
|
|
|
- {"standard_uid": standard_uid, "spec_hash": digest},
|
|
|
- ).mappings().one_or_none()
|
|
|
+ existing = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT id::text AS id, version_no, status "
|
|
|
+ "FROM public.data_standard_versions "
|
|
|
+ "WHERE standard_uid = CAST(:standard_uid AS uuid) "
|
|
|
+ "AND spec_hash = :spec_hash "
|
|
|
+ "/* existing_version */"
|
|
|
+ ),
|
|
|
+ {"standard_uid": standard_uid, "spec_hash": digest},
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one_or_none()
|
|
|
+ )
|
|
|
if existing is not None:
|
|
|
return {
|
|
|
**dict(existing),
|
|
|
@@ -466,9 +477,7 @@ class DataRuleRepository:
|
|
|
},
|
|
|
)
|
|
|
except IntegrityError as exc:
|
|
|
- raise ValueError(
|
|
|
- "standard version conflicts with existing data"
|
|
|
- ) from exc
|
|
|
+ raise ValueError("standard version conflicts with existing data") from exc
|
|
|
return {
|
|
|
"id": version_id,
|
|
|
"standard_uid": standard_uid,
|
|
|
@@ -483,17 +492,21 @@ class DataRuleRepository:
|
|
|
) -> dict[str, Any]:
|
|
|
version = _uid(version_id, "version_id")
|
|
|
_uid(published_by, "published_by")
|
|
|
- row = self.session.execute(
|
|
|
- text(
|
|
|
- "UPDATE public.data_standard_versions "
|
|
|
- "SET status = 'published', published_at = CURRENT_TIMESTAMP "
|
|
|
- "WHERE id = CAST(:version_id AS uuid) "
|
|
|
- "AND status = 'validated' "
|
|
|
- "RETURNING id::text AS id, standard_uid::text AS standard_uid, "
|
|
|
- "version_no, status, spec_hash"
|
|
|
- ),
|
|
|
- {"version_id": version},
|
|
|
- ).mappings().one_or_none()
|
|
|
+ row = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "UPDATE public.data_standard_versions "
|
|
|
+ "SET status = 'published', published_at = CURRENT_TIMESTAMP "
|
|
|
+ "WHERE id = CAST(:version_id AS uuid) "
|
|
|
+ "AND status = 'validated' "
|
|
|
+ "RETURNING id::text AS id, standard_uid::text AS standard_uid, "
|
|
|
+ "version_no, status, spec_hash"
|
|
|
+ ),
|
|
|
+ {"version_id": version},
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one_or_none()
|
|
|
+ )
|
|
|
if row is not None:
|
|
|
return dict(row)
|
|
|
status = self.session.execute(
|
|
|
@@ -505,9 +518,7 @@ class DataRuleRepository:
|
|
|
{"version_id": version},
|
|
|
).scalar_one_or_none()
|
|
|
if status == "published":
|
|
|
- raise ValueError(
|
|
|
- "standard version is already published and immutable"
|
|
|
- )
|
|
|
+ raise ValueError("standard version is already published and immutable")
|
|
|
if status is None:
|
|
|
raise ValueError("standard version was not found")
|
|
|
raise ValueError("only validated standard versions may be published")
|
|
|
@@ -527,25 +538,29 @@ class DataRuleRepository:
|
|
|
)
|
|
|
standard_rows = []
|
|
|
if standard_ids:
|
|
|
- standard_rows = self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT sv.id::text AS id, sv.status, "
|
|
|
- "jsonb_agg(jsonb_build_object("
|
|
|
- "'clause_id', b.clause_id, "
|
|
|
- "'rule_version_id', b.rule_version_id::text, "
|
|
|
- "'severity', b.severity, "
|
|
|
- "'exception_policy', b.exception_policy) "
|
|
|
- "ORDER BY b.clause_id) AS clauses "
|
|
|
- "FROM public.data_standard_versions sv "
|
|
|
- "JOIN public.standard_rule_bindings b "
|
|
|
- "ON b.standard_version_id = sv.id "
|
|
|
- "WHERE sv.id = ANY(CAST(:standard_ids AS uuid[])) "
|
|
|
- "AND sv.status = 'published' "
|
|
|
- "/* published_standard_assets */ "
|
|
|
- "GROUP BY sv.id, sv.status"
|
|
|
- ),
|
|
|
- {"standard_ids": standard_ids},
|
|
|
- ).mappings().all()
|
|
|
+ standard_rows = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT sv.id::text AS id, sv.status, "
|
|
|
+ "jsonb_agg(jsonb_build_object("
|
|
|
+ "'clause_id', b.clause_id, "
|
|
|
+ "'rule_version_id', b.rule_version_id::text, "
|
|
|
+ "'severity', b.severity, "
|
|
|
+ "'exception_policy', b.exception_policy) "
|
|
|
+ "ORDER BY b.clause_id) AS clauses "
|
|
|
+ "FROM public.data_standard_versions sv "
|
|
|
+ "JOIN public.standard_rule_bindings b "
|
|
|
+ "ON b.standard_version_id = sv.id "
|
|
|
+ "WHERE sv.id = ANY(CAST(:standard_ids AS uuid[])) "
|
|
|
+ "AND sv.status = 'published' "
|
|
|
+ "/* published_standard_assets */ "
|
|
|
+ "GROUP BY sv.id, sv.status"
|
|
|
+ ),
|
|
|
+ {"standard_ids": standard_ids},
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .all()
|
|
|
+ )
|
|
|
standards = {
|
|
|
str(row["id"]): {
|
|
|
"id": str(row["id"]),
|
|
|
@@ -566,22 +581,25 @@ class DataRuleRepository:
|
|
|
}
|
|
|
for standard in standards.values():
|
|
|
rule_ids.update(
|
|
|
- str(clause["rule_version_id"])
|
|
|
- for clause in standard["clauses"]
|
|
|
+ str(clause["rule_version_id"]) for clause in standard["clauses"]
|
|
|
)
|
|
|
rule_rows = []
|
|
|
if rule_ids:
|
|
|
- rule_rows = self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT rv.id::text AS id, rv.status, rv.rule_spec, "
|
|
|
- "rv.spec_hash "
|
|
|
- "FROM public.data_rule_versions rv "
|
|
|
- "WHERE rv.id = ANY(CAST(:rule_ids AS uuid[])) "
|
|
|
- "AND rv.status = 'published' "
|
|
|
- "/* published_rule_assets */"
|
|
|
- ),
|
|
|
- {"rule_ids": sorted(rule_ids)},
|
|
|
- ).mappings().all()
|
|
|
+ rule_rows = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT rv.id::text AS id, rv.status, rv.rule_spec, "
|
|
|
+ "rv.spec_hash "
|
|
|
+ "FROM public.data_rule_versions rv "
|
|
|
+ "WHERE rv.id = ANY(CAST(:rule_ids AS uuid[])) "
|
|
|
+ "AND rv.status = 'published' "
|
|
|
+ "/* published_rule_assets */"
|
|
|
+ ),
|
|
|
+ {"rule_ids": sorted(rule_ids)},
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .all()
|
|
|
+ )
|
|
|
rules = {
|
|
|
str(row["id"]): {
|
|
|
"id": str(row["id"]),
|
|
|
@@ -592,9 +610,7 @@ class DataRuleRepository:
|
|
|
for row in rule_rows
|
|
|
}
|
|
|
if set(rules) != rule_ids:
|
|
|
- raise ValueError(
|
|
|
- "dataflow references missing or unpublished rule versions"
|
|
|
- )
|
|
|
+ raise ValueError("dataflow references missing or unpublished rule versions")
|
|
|
return standards, rules
|
|
|
|
|
|
def begin_dataflow_release(
|
|
|
@@ -661,6 +677,125 @@ class DataRuleRepository:
|
|
|
"dataflow_spec_hash": dataflow_spec_hash(flow),
|
|
|
}
|
|
|
|
|
|
+ def find_schema_snapshot(
|
|
|
+ self, *, schema_ref: str, schema_hash: str
|
|
|
+ ) -> dict[str, Any] | None:
|
|
|
+ row = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT id::text AS id, schema_ref, schema_hash, fields, "
|
|
|
+ "source_revision FROM public.data_schema_snapshots "
|
|
|
+ "WHERE schema_ref = :schema_ref AND schema_hash = :schema_hash"
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "schema_ref": _text(schema_ref, "schema_ref", 500),
|
|
|
+ "schema_hash": _digest(schema_hash, "schema_hash"),
|
|
|
+ },
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one_or_none()
|
|
|
+ )
|
|
|
+ if row is None:
|
|
|
+ return None
|
|
|
+ snapshot = validate_schema_snapshot(
|
|
|
+ {
|
|
|
+ "schema_ref": str(row["schema_ref"]),
|
|
|
+ "schema_hash": str(row["schema_hash"]),
|
|
|
+ "fields": _array(row["fields"], "schema snapshot fields"),
|
|
|
+ "source_revision": str(row["source_revision"]),
|
|
|
+ }
|
|
|
+ )
|
|
|
+ return {"id": str(row["id"]), **snapshot}
|
|
|
+
|
|
|
+ def persist_schema_snapshot(self, *, snapshot: dict[str, Any]) -> dict[str, Any]:
|
|
|
+ normalized = validate_schema_snapshot(snapshot)
|
|
|
+ existing = self.find_schema_snapshot(
|
|
|
+ schema_ref=normalized["schema_ref"], schema_hash=normalized["schema_hash"]
|
|
|
+ )
|
|
|
+ if existing is not None:
|
|
|
+ return existing
|
|
|
+ snapshot_id = new_governance_uid()
|
|
|
+ inserted = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "INSERT INTO public.data_schema_snapshots "
|
|
|
+ "(id, schema_ref, schema_hash, fields, source_revision) "
|
|
|
+ "VALUES (CAST(:id AS uuid), :schema_ref, :schema_hash, "
|
|
|
+ "CAST(:fields AS jsonb), :source_revision) "
|
|
|
+ "ON CONFLICT (schema_ref, schema_hash) DO NOTHING "
|
|
|
+ "RETURNING id::text AS id"
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "id": snapshot_id,
|
|
|
+ "schema_ref": normalized["schema_ref"],
|
|
|
+ "schema_hash": normalized["schema_hash"],
|
|
|
+ "fields": _json(normalized["fields"]),
|
|
|
+ "source_revision": normalized["source_revision"],
|
|
|
+ },
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one_or_none()
|
|
|
+ )
|
|
|
+ if inserted is None:
|
|
|
+ existing = self.find_schema_snapshot(
|
|
|
+ schema_ref=normalized["schema_ref"],
|
|
|
+ schema_hash=normalized["schema_hash"],
|
|
|
+ )
|
|
|
+ if existing is not None:
|
|
|
+ return existing
|
|
|
+ raise ValueError("schema snapshot could not be persisted")
|
|
|
+ return {"id": str(inserted["id"]), **normalized}
|
|
|
+
|
|
|
+ def persist_dataset_binding(
|
|
|
+ self,
|
|
|
+ *,
|
|
|
+ deployment_id: str,
|
|
|
+ logical_ref: str,
|
|
|
+ binding: dict[str, Any],
|
|
|
+ actor_uid: str,
|
|
|
+ ) -> dict[str, Any]:
|
|
|
+ deployment = _uid(deployment_id, "deployment_id")
|
|
|
+ _uid(actor_uid, "actor_uid")
|
|
|
+ logical = _text(logical_ref, "logical_ref", 500)
|
|
|
+ normalized = validate_dataset_binding(binding)
|
|
|
+ binding_hash = _canonical_hash({"logical_ref": logical, **normalized})
|
|
|
+ binding_id = new_governance_uid()
|
|
|
+ try:
|
|
|
+ row = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "INSERT INTO public.dataflow_dataset_bindings "
|
|
|
+ "(id, dataflow_deployment_id, logical_ref, data_source_uid, "
|
|
|
+ "object_kind, object_ref, schema_snapshot_id, dialect, "
|
|
|
+ "access_mode, write_mode, binding_hash) "
|
|
|
+ "VALUES (CAST(:id AS uuid), CAST(:deployment_id AS uuid), "
|
|
|
+ ":logical_ref, CAST(:data_source_uid AS uuid), :object_kind, "
|
|
|
+ ":object_ref, CAST(:schema_snapshot_id AS uuid), :dialect, "
|
|
|
+ ":access_mode, :write_mode, :binding_hash) "
|
|
|
+ "RETURNING id::text AS id"
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "id": binding_id,
|
|
|
+ "deployment_id": deployment,
|
|
|
+ "logical_ref": logical,
|
|
|
+ **normalized,
|
|
|
+ "binding_hash": binding_hash,
|
|
|
+ },
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one()
|
|
|
+ )
|
|
|
+ except IntegrityError as exc:
|
|
|
+ raise ValueError(
|
|
|
+ "dataset binding already exists for deployment logical_ref"
|
|
|
+ ) from exc
|
|
|
+ return {
|
|
|
+ "id": str(row["id"]),
|
|
|
+ "logical_ref": logical,
|
|
|
+ **normalized,
|
|
|
+ "binding_hash": binding_hash,
|
|
|
+ }
|
|
|
+
|
|
|
def persist_component_plan(
|
|
|
self,
|
|
|
*,
|
|
|
@@ -697,9 +832,7 @@ class DataRuleRepository:
|
|
|
raise ValueError("compiled plan contains unsupported fields")
|
|
|
if plan["backend"] not in PLAN_BACKENDS:
|
|
|
raise ValueError("unsupported plan backend")
|
|
|
- compiler_version = _text(
|
|
|
- plan["compiler_version"], "compiler_version", 80
|
|
|
- )
|
|
|
+ compiler_version = _text(plan["compiler_version"], "compiler_version", 80)
|
|
|
plan_body = _object(plan["plan"], "compiled plan body")
|
|
|
plan_hash = _digest(plan["plan_hash"], "plan_hash")
|
|
|
if _canonical_hash(plan_body) != plan_hash:
|
|
|
@@ -730,9 +863,7 @@ class DataRuleRepository:
|
|
|
"rule_version_id": rule_id,
|
|
|
"stage": _text(stage, "stage", 30),
|
|
|
"order_no": order_no,
|
|
|
- "idempotency": _json(idempotency)
|
|
|
- if idempotency is not None
|
|
|
- else None,
|
|
|
+ "idempotency": _json(idempotency) if idempotency is not None else None,
|
|
|
"provenance": _json(provenance),
|
|
|
},
|
|
|
)
|
|
|
@@ -766,22 +897,26 @@ class DataRuleRepository:
|
|
|
if not isinstance(package, dict):
|
|
|
raise ValueError("production-line package must be an object")
|
|
|
package_hash = _digest(package.get("package_hash"), "package_hash")
|
|
|
- row = self.session.execute(
|
|
|
- text(
|
|
|
- "UPDATE public.dataflow_versions "
|
|
|
- "SET package = CAST(:package AS jsonb), "
|
|
|
- "package_hash = :package_hash, status = 'released', "
|
|
|
- "released_at = CURRENT_TIMESTAMP "
|
|
|
- "WHERE id = CAST(:version_id AS uuid) "
|
|
|
- "AND status = 'validated' "
|
|
|
- "RETURNING id::text AS id, version_no, status, package_hash"
|
|
|
- ),
|
|
|
- {
|
|
|
- "version_id": version_id,
|
|
|
- "package": _json(package),
|
|
|
- "package_hash": package_hash,
|
|
|
- },
|
|
|
- ).mappings().one_or_none()
|
|
|
+ row = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "UPDATE public.dataflow_versions "
|
|
|
+ "SET package = CAST(:package AS jsonb), "
|
|
|
+ "package_hash = :package_hash, status = 'released', "
|
|
|
+ "released_at = CURRENT_TIMESTAMP "
|
|
|
+ "WHERE id = CAST(:version_id AS uuid) "
|
|
|
+ "AND status = 'validated' "
|
|
|
+ "RETURNING id::text AS id, version_no, status, package_hash"
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "version_id": version_id,
|
|
|
+ "package": _json(package),
|
|
|
+ "package_hash": package_hash,
|
|
|
+ },
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .one_or_none()
|
|
|
+ )
|
|
|
if row is None:
|
|
|
raise ValueError("dataflow release is not in validated state")
|
|
|
return {**dict(row), "package": package}
|
|
|
@@ -805,25 +940,29 @@ class DataRuleRepository:
|
|
|
|
|
|
standard_rows = []
|
|
|
if standard_ids:
|
|
|
- standard_rows = self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT sv.id::text AS id, sv.status, "
|
|
|
- "jsonb_agg(jsonb_build_object("
|
|
|
- "'clause_id', b.clause_id, "
|
|
|
- "'rule_version_id', b.rule_version_id::text, "
|
|
|
- "'severity', b.severity, "
|
|
|
- "'exception_policy', b.exception_policy) "
|
|
|
- "ORDER BY b.clause_id) AS clauses "
|
|
|
- "FROM public.data_standard_versions sv "
|
|
|
- "JOIN public.standard_rule_bindings b "
|
|
|
- "ON b.standard_version_id = sv.id "
|
|
|
- "WHERE sv.id = ANY(CAST(:standard_ids AS uuid[])) "
|
|
|
- "AND sv.status = 'published' "
|
|
|
- "/* catalog_standard_versions */ "
|
|
|
- "GROUP BY sv.id, sv.status"
|
|
|
- ),
|
|
|
- {"standard_ids": standard_ids},
|
|
|
- ).mappings().all()
|
|
|
+ standard_rows = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT sv.id::text AS id, sv.status, "
|
|
|
+ "jsonb_agg(jsonb_build_object("
|
|
|
+ "'clause_id', b.clause_id, "
|
|
|
+ "'rule_version_id', b.rule_version_id::text, "
|
|
|
+ "'severity', b.severity, "
|
|
|
+ "'exception_policy', b.exception_policy) "
|
|
|
+ "ORDER BY b.clause_id) AS clauses "
|
|
|
+ "FROM public.data_standard_versions sv "
|
|
|
+ "JOIN public.standard_rule_bindings b "
|
|
|
+ "ON b.standard_version_id = sv.id "
|
|
|
+ "WHERE sv.id = ANY(CAST(:standard_ids AS uuid[])) "
|
|
|
+ "AND sv.status = 'published' "
|
|
|
+ "/* catalog_standard_versions */ "
|
|
|
+ "GROUP BY sv.id, sv.status"
|
|
|
+ ),
|
|
|
+ {"standard_ids": standard_ids},
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .all()
|
|
|
+ )
|
|
|
standards = {
|
|
|
str(row["id"]): {
|
|
|
"id": str(row["id"]),
|
|
|
@@ -840,29 +979,32 @@ class DataRuleRepository:
|
|
|
rule_ids = set(direct_rule_ids)
|
|
|
for standard in standards.values():
|
|
|
rule_ids.update(
|
|
|
- str(clause["rule_version_id"])
|
|
|
- for clause in standard["clauses"]
|
|
|
+ str(clause["rule_version_id"]) for clause in standard["clauses"]
|
|
|
)
|
|
|
rule_rows = []
|
|
|
if rule_ids:
|
|
|
- rule_rows = self.session.execute(
|
|
|
- text(
|
|
|
- "SELECT DISTINCT ON (rv.id) "
|
|
|
- "rv.id::text AS id, rv.status, rv.rule_spec, rv.spec_hash, "
|
|
|
- "p.backend, p.plan_hash "
|
|
|
- "FROM public.data_rule_versions rv "
|
|
|
- "JOIN public.dataflow_component_bindings cb "
|
|
|
- "ON cb.rule_version_id = rv.id "
|
|
|
- "JOIN public.rule_execution_plans p "
|
|
|
- "ON p.component_binding_id = cb.id "
|
|
|
- "WHERE rv.id = ANY(CAST(:rule_ids AS uuid[])) "
|
|
|
- "AND rv.status = 'published' "
|
|
|
- "AND p.status = 'published' "
|
|
|
- "/* catalog_rule_versions */ "
|
|
|
- "ORDER BY rv.id, p.created_at DESC"
|
|
|
- ),
|
|
|
- {"rule_ids": sorted(rule_ids)},
|
|
|
- ).mappings().all()
|
|
|
+ rule_rows = (
|
|
|
+ self.session.execute(
|
|
|
+ text(
|
|
|
+ "SELECT DISTINCT ON (rv.id) "
|
|
|
+ "rv.id::text AS id, rv.status, rv.rule_spec, rv.spec_hash, "
|
|
|
+ "p.backend, p.plan_hash "
|
|
|
+ "FROM public.data_rule_versions rv "
|
|
|
+ "JOIN public.dataflow_component_bindings cb "
|
|
|
+ "ON cb.rule_version_id = rv.id "
|
|
|
+ "JOIN public.rule_execution_plans p "
|
|
|
+ "ON p.component_binding_id = cb.id "
|
|
|
+ "WHERE rv.id = ANY(CAST(:rule_ids AS uuid[])) "
|
|
|
+ "AND rv.status = 'published' "
|
|
|
+ "AND p.status = 'published' "
|
|
|
+ "/* catalog_rule_versions */ "
|
|
|
+ "ORDER BY rv.id, p.created_at DESC"
|
|
|
+ ),
|
|
|
+ {"rule_ids": sorted(rule_ids)},
|
|
|
+ )
|
|
|
+ .mappings()
|
|
|
+ .all()
|
|
|
+ )
|
|
|
rules = {
|
|
|
str(row["id"]): {
|
|
|
"id": str(row["id"]),
|