|
|
@@ -110,17 +110,24 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
max_ttl_seconds=3600,
|
|
|
)
|
|
|
correlation_id = new_governance_uid()
|
|
|
+ failure_correlation_id = new_governance_uid()
|
|
|
+ unknown_correlation_id = new_governance_uid()
|
|
|
prefix = f"rules/{correlation_id}/"
|
|
|
customer_table = "task5_polars_customers"
|
|
|
segment_table = "task5_polars_segments"
|
|
|
rule_uid = new_governance_uid()
|
|
|
rule_id = new_governance_uid()
|
|
|
+ downstream_rule_uid = new_governance_uid()
|
|
|
+ downstream_rule_id = new_governance_uid()
|
|
|
dataflow_uid = new_governance_uid()
|
|
|
dataflow_version_id = new_governance_uid()
|
|
|
deployment_id = new_governance_uid()
|
|
|
component_binding_id = new_governance_uid()
|
|
|
plan_id = new_governance_uid()
|
|
|
+ downstream_component_binding_id = new_governance_uid()
|
|
|
+ downstream_plan_id = new_governance_uid()
|
|
|
ledger_jti = None
|
|
|
+ retry_ledger_jti = None
|
|
|
|
|
|
try:
|
|
|
with postgres.begin() as connection:
|
|
|
@@ -221,9 +228,15 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
output_binding = _binding(
|
|
|
output_schema,
|
|
|
source_uid=new_governance_uid(),
|
|
|
- access_mode="write",
|
|
|
+ access_mode="read_write",
|
|
|
object_ref="polars-output-artifact",
|
|
|
)
|
|
|
+ downstream_output_binding = _binding(
|
|
|
+ output_schema,
|
|
|
+ source_uid=new_governance_uid(),
|
|
|
+ access_mode="write",
|
|
|
+ object_ref="polars-downstream-output-artifact",
|
|
|
+ )
|
|
|
spec = validate_rule_spec(
|
|
|
{
|
|
|
"schema_version": "2.0",
|
|
|
@@ -295,6 +308,45 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
},
|
|
|
},
|
|
|
)
|
|
|
+ downstream_spec = validate_rule_spec(
|
|
|
+ {
|
|
|
+ "schema_version": "2.0",
|
|
|
+ "rule_uid": new_governance_uid(),
|
|
|
+ "name": "task6_real_artifact_handoff",
|
|
|
+ "input_schema_ref": output_schema["schema_ref"],
|
|
|
+ "output_schema_ref": output_schema["schema_ref"],
|
|
|
+ "steps": [
|
|
|
+ {
|
|
|
+ "id": "normalize_downstream_name",
|
|
|
+ "op": "normalize_text",
|
|
|
+ "column": "name",
|
|
|
+ "trim": True,
|
|
|
+ }
|
|
|
+ ],
|
|
|
+ "null_policy": "explicit",
|
|
|
+ "timezone": "Asia/Shanghai",
|
|
|
+ }
|
|
|
+ )
|
|
|
+ downstream_rule = {
|
|
|
+ "id": downstream_rule_id,
|
|
|
+ "status": "published",
|
|
|
+ "rule_spec": downstream_spec,
|
|
|
+ "spec_hash": rule_spec_hash(downstream_spec),
|
|
|
+ }
|
|
|
+ downstream_compiled = PolarsRuleCompiler().compile(
|
|
|
+ rule_version=downstream_rule,
|
|
|
+ input_schema=output_schema,
|
|
|
+ output_schema=output_schema,
|
|
|
+ input_binding=output_binding,
|
|
|
+ output_binding=downstream_output_binding,
|
|
|
+ backend={
|
|
|
+ "max_rows": 1_000,
|
|
|
+ "max_artifact_bytes": 4 * 1024 * 1024,
|
|
|
+ "memory_limit_bytes": 256 * 1024 * 1024,
|
|
|
+ "masking_policies": {},
|
|
|
+ "lookup_bindings": {},
|
|
|
+ },
|
|
|
+ )
|
|
|
lookup_operation = compiled["plan"]["operations"][1]
|
|
|
schema_hashes = {
|
|
|
"rule_spec_hash": compiled["plan"]["rule_spec_hash"],
|
|
|
@@ -331,6 +383,21 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
"name": spec["name"],
|
|
|
},
|
|
|
)
|
|
|
+ connection.execute(
|
|
|
+ text(
|
|
|
+ """
|
|
|
+ INSERT INTO public.data_rules
|
|
|
+ (id, rule_uid, name, category, status)
|
|
|
+ VALUES (CAST(:id AS uuid), CAST(:rule_uid AS uuid),
|
|
|
+ :name, 'general', 'active')
|
|
|
+ """
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "id": new_governance_uid(),
|
|
|
+ "rule_uid": downstream_rule_uid,
|
|
|
+ "name": downstream_spec["name"],
|
|
|
+ },
|
|
|
+ )
|
|
|
connection.execute(
|
|
|
text(
|
|
|
"""
|
|
|
@@ -351,6 +418,31 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
"spec_hash": rule["spec_hash"],
|
|
|
},
|
|
|
)
|
|
|
+ connection.execute(
|
|
|
+ text(
|
|
|
+ """
|
|
|
+ INSERT INTO public.data_rule_versions
|
|
|
+ (id, rule_uid, version_no, source_text,
|
|
|
+ source_language, rule_spec, spec_hash,
|
|
|
+ generated_kind, status, published_at)
|
|
|
+ VALUES (
|
|
|
+ CAST(:id AS uuid), CAST(:rule_uid AS uuid), 1,
|
|
|
+ :source_text, 'en', CAST(:rule_spec AS jsonb),
|
|
|
+ :spec_hash, 'polars', 'published',
|
|
|
+ CURRENT_TIMESTAMP
|
|
|
+ )
|
|
|
+ """
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "id": downstream_rule_id,
|
|
|
+ "rule_uid": downstream_rule_uid,
|
|
|
+ "source_text": (
|
|
|
+ "Task 6 real two-node artifact handoff"
|
|
|
+ ),
|
|
|
+ "rule_spec": json.dumps(downstream_spec),
|
|
|
+ "spec_hash": downstream_rule["spec_hash"],
|
|
|
+ },
|
|
|
+ )
|
|
|
connection.execute(
|
|
|
text(
|
|
|
"""
|
|
|
@@ -409,6 +501,13 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
output_binding,
|
|
|
compiled["plan"]["output_binding_hash"],
|
|
|
),
|
|
|
+ (
|
|
|
+ "downstream",
|
|
|
+ downstream_output_binding,
|
|
|
+ downstream_compiled["plan"][
|
|
|
+ "output_binding_hash"
|
|
|
+ ],
|
|
|
+ ),
|
|
|
):
|
|
|
connection.execute(
|
|
|
text(
|
|
|
@@ -461,6 +560,34 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
),
|
|
|
},
|
|
|
)
|
|
|
+ connection.execute(
|
|
|
+ text(
|
|
|
+ """
|
|
|
+ INSERT INTO public.dataflow_component_bindings
|
|
|
+ (id, dataflow_version_id, component_id,
|
|
|
+ component_kind, rule_version_id, stage, order_no,
|
|
|
+ idempotency, provenance)
|
|
|
+ VALUES (
|
|
|
+ CAST(:id AS uuid),
|
|
|
+ CAST(:dataflow_version_id AS uuid),
|
|
|
+ 'task6_real_handoff', 'rule.apply',
|
|
|
+ CAST(:rule_version_id AS uuid), 'transform', 1,
|
|
|
+ CAST(:idempotency AS jsonb), '{}'::jsonb
|
|
|
+ )
|
|
|
+ """
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "id": downstream_component_binding_id,
|
|
|
+ "dataflow_version_id": dataflow_version_id,
|
|
|
+ "rule_version_id": downstream_rule_id,
|
|
|
+ "idempotency": json.dumps(
|
|
|
+ {
|
|
|
+ "strategy": "deduplication_key",
|
|
|
+ "key": "customer_id",
|
|
|
+ }
|
|
|
+ ),
|
|
|
+ },
|
|
|
+ )
|
|
|
connection.execute(
|
|
|
text(
|
|
|
"""
|
|
|
@@ -483,6 +610,53 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
"schema_hashes": json.dumps(schema_hashes),
|
|
|
},
|
|
|
)
|
|
|
+ connection.execute(
|
|
|
+ text(
|
|
|
+ """
|
|
|
+ INSERT INTO public.rule_execution_plans
|
|
|
+ (id, component_binding_id, backend,
|
|
|
+ compiler_version, plan, plan_hash,
|
|
|
+ schema_hashes, status)
|
|
|
+ VALUES (
|
|
|
+ CAST(:id AS uuid),
|
|
|
+ CAST(:component_binding_id AS uuid),
|
|
|
+ 'polars_batch', :compiler_version,
|
|
|
+ CAST(:plan AS jsonb), :plan_hash,
|
|
|
+ CAST(:schema_hashes AS jsonb), 'published'
|
|
|
+ )
|
|
|
+ """
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "id": downstream_plan_id,
|
|
|
+ "component_binding_id": (
|
|
|
+ downstream_component_binding_id
|
|
|
+ ),
|
|
|
+ "compiler_version": downstream_compiled[
|
|
|
+ "compiler_version"
|
|
|
+ ],
|
|
|
+ "plan": json.dumps(downstream_compiled["plan"]),
|
|
|
+ "plan_hash": downstream_compiled["plan_hash"],
|
|
|
+ "schema_hashes": json.dumps(
|
|
|
+ {
|
|
|
+ "rule_spec_hash": downstream_compiled[
|
|
|
+ "plan"
|
|
|
+ ]["rule_spec_hash"],
|
|
|
+ "input_schema_snapshot_id": output_schema[
|
|
|
+ "id"
|
|
|
+ ],
|
|
|
+ "input_schema_hash": output_schema[
|
|
|
+ "schema_hash"
|
|
|
+ ],
|
|
|
+ "output_schema_snapshot_id": output_schema[
|
|
|
+ "id"
|
|
|
+ ],
|
|
|
+ "output_schema_hash": output_schema[
|
|
|
+ "schema_hash"
|
|
|
+ ],
|
|
|
+ }
|
|
|
+ ),
|
|
|
+ },
|
|
|
+ )
|
|
|
customer_path = tmp_path / "customers.parquet"
|
|
|
segment_path = tmp_path / "segments.parquet"
|
|
|
pl.DataFrame(customer_rows).write_parquet(customer_path)
|
|
|
@@ -594,6 +768,7 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
from app.runner.auth import TaskTokenIssuer, TaskTokenVerifier
|
|
|
from app.runner.ledger import PostgresTaskLedger
|
|
|
from app.runner.nodes import NodeRegistry
|
|
|
+ from app.runner.rule_evidence import PostgresRuleEvidenceWriter
|
|
|
|
|
|
task_secret = "task5-real-http-secret-value-32-bytes"
|
|
|
verifier = TaskTokenVerifier(task_secret)
|
|
|
@@ -606,10 +781,36 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
write_authorized=True,
|
|
|
)
|
|
|
ledger_jti = verifier.verify(task_token, node=node).jti
|
|
|
+ retry_token = TaskTokenIssuer(task_secret).issue(
|
|
|
+ task_uid=new_governance_uid(),
|
|
|
+ dataflow_uid=dataflow_uid,
|
|
|
+ workflow_version=1,
|
|
|
+ correlation_id=correlation_id,
|
|
|
+ node=node,
|
|
|
+ write_authorized=True,
|
|
|
+ )
|
|
|
+ retry_ledger_jti = verifier.verify(
|
|
|
+ retry_token, node=node
|
|
|
+ ).jti
|
|
|
+ evidenced_executor = RulePlanExecutor(
|
|
|
+ PostgresRulePlanRepository(platform),
|
|
|
+ adapters={
|
|
|
+ "polars_batch": PolarsRulePlanAdapter(
|
|
|
+ artifact_store=store,
|
|
|
+ artifact_resolver=resolver,
|
|
|
+ artifact_ttl_seconds=900,
|
|
|
+ )
|
|
|
+ },
|
|
|
+ evidence_writer=PostgresRuleEvidenceWriter(
|
|
|
+ platform,
|
|
|
+ store,
|
|
|
+ sample_ttl_seconds=900,
|
|
|
+ ),
|
|
|
+ )
|
|
|
runner_app = create_runner_app(
|
|
|
verifier=verifier,
|
|
|
ledger=PostgresTaskLedger(platform),
|
|
|
- registry=NodeRegistry({"rule.apply": executor}),
|
|
|
+ registry=NodeRegistry({"rule.apply": evidenced_executor}),
|
|
|
)
|
|
|
with runner_app.test_client() as client:
|
|
|
http_result = client.post(
|
|
|
@@ -628,15 +829,256 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
"parameters": {},
|
|
|
},
|
|
|
)
|
|
|
+ retried = client.post(
|
|
|
+ "/v1/tasks/execute",
|
|
|
+ json={
|
|
|
+ "task_token": retry_token,
|
|
|
+ "node": node,
|
|
|
+ "parameters": {},
|
|
|
+ },
|
|
|
+ )
|
|
|
assert http_result.status_code == 200
|
|
|
+ assert http_result.get_json()["output_artifact"] == result[
|
|
|
+ "artifact_ref"
|
|
|
+ ]
|
|
|
assert http_result.get_json()["result"]["artifact_ref"] == result[
|
|
|
"artifact_ref"
|
|
|
]
|
|
|
assert replay.status_code == 409
|
|
|
+ assert retried.status_code == 200
|
|
|
+ assert retried.get_json()["output_artifact"] == result[
|
|
|
+ "artifact_ref"
|
|
|
+ ]
|
|
|
ledger_record = PostgresTaskLedger(platform).get(ledger_jti)
|
|
|
assert ledger_record is not None
|
|
|
assert ledger_record.status == "success"
|
|
|
assert ledger_record.commit_outcome == "committed"
|
|
|
+ with platform.connect() as connection:
|
|
|
+ run_evidence = connection.execute(
|
|
|
+ text(
|
|
|
+ """
|
|
|
+ SELECT status, commit_outcome, rows_in, rows_out,
|
|
|
+ rows_rejected, rows_quarantined, public_result
|
|
|
+ FROM public.rule_runs
|
|
|
+ WHERE correlation_id = CAST(:correlation_id AS uuid)
|
|
|
+ AND component_binding_id =
|
|
|
+ CAST(:component_binding_id AS uuid)
|
|
|
+ """
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "correlation_id": correlation_id,
|
|
|
+ "component_binding_id": component_binding_id,
|
|
|
+ },
|
|
|
+ ).mappings().one()
|
|
|
+ sample_evidence = connection.execute(
|
|
|
+ text(
|
|
|
+ """
|
|
|
+ SELECT s.artifact_ref, s.artifact_digest,
|
|
|
+ s.sample_count, s.redaction_policy,
|
|
|
+ s.expires_at, s.handoff_status
|
|
|
+ FROM public.rule_violation_samples s
|
|
|
+ JOIN public.rule_runs r ON r.id = s.rule_run_id
|
|
|
+ WHERE r.correlation_id =
|
|
|
+ CAST(:correlation_id AS uuid)
|
|
|
+ """
|
|
|
+ ),
|
|
|
+ {"correlation_id": correlation_id},
|
|
|
+ ).mappings().one()
|
|
|
+ assert run_evidence["status"] == "success"
|
|
|
+ assert run_evidence["commit_outcome"] == "committed"
|
|
|
+ assert run_evidence["rows_in"] == 4
|
|
|
+ assert run_evidence["rows_out"] == 2
|
|
|
+ assert run_evidence["rows_rejected"] == 1
|
|
|
+ assert run_evidence["rows_quarantined"] == 0
|
|
|
+ assert run_evidence["public_result"]["output_artifact"] == result[
|
|
|
+ "artifact_ref"
|
|
|
+ ]
|
|
|
+ assert sample_evidence["artifact_digest"]
|
|
|
+ assert sample_evidence["sample_count"] == 1
|
|
|
+ assert (
|
|
|
+ sample_evidence["redaction_policy"]
|
|
|
+ == "rule-violation-default-v1"
|
|
|
+ )
|
|
|
+ assert sample_evidence["handoff_status"] == "ready"
|
|
|
+ assert store.read(
|
|
|
+ sample_evidence["artifact_ref"],
|
|
|
+ sample_evidence["artifact_digest"],
|
|
|
+ expected_schema_fields=[
|
|
|
+ {
|
|
|
+ "name": name,
|
|
|
+ "type": "string",
|
|
|
+ "nullable": True,
|
|
|
+ }
|
|
|
+ for name in (
|
|
|
+ "customer_id",
|
|
|
+ "mobile",
|
|
|
+ "name",
|
|
|
+ "segment_code",
|
|
|
+ "segment_name",
|
|
|
+ "version_no",
|
|
|
+ )
|
|
|
+ ],
|
|
|
+ ).collect().to_dicts() == [
|
|
|
+ {
|
|
|
+ "customer_id": "[REDACTED]",
|
|
|
+ "mobile": "[REDACTED]",
|
|
|
+ "name": "[REDACTED]",
|
|
|
+ "segment_code": "[REDACTED]",
|
|
|
+ "segment_name": "[REDACTED]",
|
|
|
+ "version_no": "[REDACTED]",
|
|
|
+ }
|
|
|
+ ]
|
|
|
+ downstream_node = {
|
|
|
+ "id": "task6_real_handoff",
|
|
|
+ "type": "rule.apply",
|
|
|
+ "purpose": "write",
|
|
|
+ "idempotency": {
|
|
|
+ "strategy": "deduplication_key",
|
|
|
+ "key": "customer_id",
|
|
|
+ },
|
|
|
+ "config": {
|
|
|
+ "component_binding_id": (
|
|
|
+ downstream_component_binding_id
|
|
|
+ ),
|
|
|
+ "rule_version_id": downstream_rule_id,
|
|
|
+ "execution_plan_hash": downstream_compiled[
|
|
|
+ "plan_hash"
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ }
|
|
|
+ downstream_result = evidenced_executor.execute(
|
|
|
+ downstream_node,
|
|
|
+ {"input_artifact": result["artifact_ref"]},
|
|
|
+ write_authorized=True,
|
|
|
+ correlation_id=correlation_id,
|
|
|
+ dataflow_uid=dataflow_uid,
|
|
|
+ workflow_version=1,
|
|
|
+ node_id="task6_real_handoff",
|
|
|
+ )
|
|
|
+ assert downstream_result["rows_in"] == 2
|
|
|
+ assert downstream_result["rows_out"] == 2
|
|
|
+ assert downstream_result["output_artifact"] != result[
|
|
|
+ "artifact_ref"
|
|
|
+ ]
|
|
|
+ assert store.read(
|
|
|
+ downstream_result["artifact_ref"],
|
|
|
+ downstream_result["digest"],
|
|
|
+ expected_schema_fields=output_schema["fields"],
|
|
|
+ limits=downstream_compiled["plan"]["resource_limits"],
|
|
|
+ ).collect().sort("customer_id").to_dicts() == (
|
|
|
+ output.sort("customer_id").to_dicts()
|
|
|
+ )
|
|
|
+ replayed_downstream = evidenced_executor.execute(
|
|
|
+ downstream_node,
|
|
|
+ {"input_artifact": result["artifact_ref"]},
|
|
|
+ write_authorized=True,
|
|
|
+ correlation_id=correlation_id,
|
|
|
+ dataflow_uid=dataflow_uid,
|
|
|
+ workflow_version=1,
|
|
|
+ node_id="task6_real_handoff",
|
|
|
+ )
|
|
|
+ assert replayed_downstream["artifact_ref"] == downstream_result[
|
|
|
+ "artifact_ref"
|
|
|
+ ]
|
|
|
+ with platform.connect() as connection:
|
|
|
+ assert connection.execute(
|
|
|
+ text(
|
|
|
+ """
|
|
|
+ SELECT COUNT(*)
|
|
|
+ FROM public.rule_runs
|
|
|
+ WHERE correlation_id =
|
|
|
+ CAST(:correlation_id AS uuid)
|
|
|
+ """
|
|
|
+ ),
|
|
|
+ {"correlation_id": correlation_id},
|
|
|
+ ).scalar_one() == 2
|
|
|
+ from app.runner.nodes import NodeExecutionError
|
|
|
+
|
|
|
+ class FailingAdapter:
|
|
|
+ def execute(self, **_kwargs):
|
|
|
+ raise NodeExecutionError(
|
|
|
+ "safe downstream failure",
|
|
|
+ commit_outcome="not_committed",
|
|
|
+ )
|
|
|
+
|
|
|
+ failed_executor = RulePlanExecutor(
|
|
|
+ PostgresRulePlanRepository(platform),
|
|
|
+ adapters={"polars_batch": FailingAdapter()},
|
|
|
+ evidence_writer=PostgresRuleEvidenceWriter(
|
|
|
+ platform,
|
|
|
+ store,
|
|
|
+ sample_ttl_seconds=900,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+ with pytest.raises(NodeExecutionError, match="safe downstream"):
|
|
|
+ failed_executor.execute(
|
|
|
+ node,
|
|
|
+ {},
|
|
|
+ write_authorized=True,
|
|
|
+ correlation_id=failure_correlation_id,
|
|
|
+ dataflow_uid=dataflow_uid,
|
|
|
+ workflow_version=1,
|
|
|
+ node_id="task5_real_polars",
|
|
|
+ )
|
|
|
+ with platform.connect() as connection:
|
|
|
+ failed_evidence = connection.execute(
|
|
|
+ text(
|
|
|
+ """
|
|
|
+ SELECT status, commit_outcome
|
|
|
+ FROM public.rule_runs
|
|
|
+ WHERE correlation_id =
|
|
|
+ CAST(:correlation_id AS uuid)
|
|
|
+ """
|
|
|
+ ),
|
|
|
+ {"correlation_id": failure_correlation_id},
|
|
|
+ ).mappings().one()
|
|
|
+ assert dict(failed_evidence) == {
|
|
|
+ "status": "failed",
|
|
|
+ "commit_outcome": "not_committed",
|
|
|
+ }
|
|
|
+
|
|
|
+ class UnknownAdapter:
|
|
|
+ def execute(self, **_kwargs):
|
|
|
+ raise NodeExecutionError(
|
|
|
+ "safe uncertain commit",
|
|
|
+ commit_outcome="unknown",
|
|
|
+ )
|
|
|
+
|
|
|
+ unknown_executor = RulePlanExecutor(
|
|
|
+ PostgresRulePlanRepository(platform),
|
|
|
+ adapters={"polars_batch": UnknownAdapter()},
|
|
|
+ evidence_writer=PostgresRuleEvidenceWriter(
|
|
|
+ platform,
|
|
|
+ store,
|
|
|
+ sample_ttl_seconds=900,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+ with pytest.raises(NodeExecutionError, match="uncertain commit"):
|
|
|
+ unknown_executor.execute(
|
|
|
+ node,
|
|
|
+ {},
|
|
|
+ write_authorized=True,
|
|
|
+ correlation_id=unknown_correlation_id,
|
|
|
+ dataflow_uid=dataflow_uid,
|
|
|
+ workflow_version=1,
|
|
|
+ node_id="task5_real_polars",
|
|
|
+ )
|
|
|
+ with platform.connect() as connection:
|
|
|
+ unknown_evidence = connection.execute(
|
|
|
+ text(
|
|
|
+ """
|
|
|
+ SELECT status, commit_outcome
|
|
|
+ FROM public.rule_runs
|
|
|
+ WHERE correlation_id =
|
|
|
+ CAST(:correlation_id AS uuid)
|
|
|
+ """
|
|
|
+ ),
|
|
|
+ {"correlation_id": unknown_correlation_id},
|
|
|
+ ).mappings().one()
|
|
|
+ assert dict(unknown_evidence) == {
|
|
|
+ "status": "unknown",
|
|
|
+ "commit_outcome": "unknown",
|
|
|
+ }
|
|
|
conflict_path = tmp_path / "conflict.parquet"
|
|
|
pl.DataFrame(
|
|
|
{
|
|
|
@@ -682,17 +1124,20 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
).mappings().all()
|
|
|
# The repeated deterministic output has the same digest and is
|
|
|
# idempotently retained as one stable catalog handoff.
|
|
|
- assert len(catalog_rows) == 3
|
|
|
+ assert len(catalog_rows) == 4
|
|
|
assert all(row["handoff_status"] == "ready" for row in catalog_rows)
|
|
|
assert all(len(row["binding_hash"]) == 64 for row in catalog_rows)
|
|
|
- assert next(
|
|
|
+ assert {
|
|
|
row["artifact_ref"]
|
|
|
for row in catalog_rows
|
|
|
if row["artifact_kind"] == "output"
|
|
|
- ) == result["artifact_ref"]
|
|
|
+ } == {
|
|
|
+ result["artifact_ref"],
|
|
|
+ downstream_result["artifact_ref"],
|
|
|
+ }
|
|
|
assert len(
|
|
|
list(minio.list_objects(bucket, prefix=prefix, recursive=True))
|
|
|
- ) == 3
|
|
|
+ ) == 5
|
|
|
finally:
|
|
|
for item in list(
|
|
|
minio.list_objects(bucket, prefix=prefix, recursive=True)
|
|
|
@@ -714,6 +1159,40 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
),
|
|
|
{"jti": ledger_jti},
|
|
|
)
|
|
|
+ if retry_ledger_jti is not None:
|
|
|
+ connection.execute(
|
|
|
+ text(
|
|
|
+ "DELETE FROM public.runner_task_executions "
|
|
|
+ "WHERE token_jti = CAST(:jti AS uuid)"
|
|
|
+ ),
|
|
|
+ {"jti": retry_ledger_jti},
|
|
|
+ )
|
|
|
+ connection.execute(
|
|
|
+ text(
|
|
|
+ """
|
|
|
+ DELETE FROM public.rule_violation_samples s
|
|
|
+ USING public.rule_runs r
|
|
|
+ WHERE s.rule_run_id = r.id
|
|
|
+ AND r.correlation_id =
|
|
|
+ CAST(:correlation_id AS uuid)
|
|
|
+ """
|
|
|
+ ),
|
|
|
+ {"correlation_id": correlation_id},
|
|
|
+ )
|
|
|
+ connection.execute(
|
|
|
+ text(
|
|
|
+ "DELETE FROM public.rule_runs "
|
|
|
+ "WHERE correlation_id IN ("
|
|
|
+ "CAST(:correlation_id AS uuid), "
|
|
|
+ "CAST(:failure_correlation_id AS uuid), "
|
|
|
+ "CAST(:unknown_correlation_id AS uuid))"
|
|
|
+ ),
|
|
|
+ {
|
|
|
+ "correlation_id": correlation_id,
|
|
|
+ "failure_correlation_id": failure_correlation_id,
|
|
|
+ "unknown_correlation_id": unknown_correlation_id,
|
|
|
+ },
|
|
|
+ )
|
|
|
connection.execute(
|
|
|
text(
|
|
|
"DELETE FROM public.rule_run_artifacts "
|
|
|
@@ -724,9 +1203,13 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
connection.execute(
|
|
|
text(
|
|
|
"DELETE FROM public.rule_execution_plans "
|
|
|
- "WHERE id = CAST(:id AS uuid)"
|
|
|
+ "WHERE id IN (CAST(:id AS uuid), "
|
|
|
+ "CAST(:downstream_id AS uuid))"
|
|
|
),
|
|
|
- {"id": plan_id},
|
|
|
+ {
|
|
|
+ "id": plan_id,
|
|
|
+ "downstream_id": downstream_plan_id,
|
|
|
+ },
|
|
|
)
|
|
|
connection.execute(
|
|
|
text(
|
|
|
@@ -738,9 +1221,13 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
connection.execute(
|
|
|
text(
|
|
|
"DELETE FROM public.dataflow_component_bindings "
|
|
|
- "WHERE id = CAST(:id AS uuid)"
|
|
|
+ "WHERE id IN (CAST(:id AS uuid), "
|
|
|
+ "CAST(:downstream_id AS uuid))"
|
|
|
),
|
|
|
- {"id": component_binding_id},
|
|
|
+ {
|
|
|
+ "id": component_binding_id,
|
|
|
+ "downstream_id": downstream_component_binding_id,
|
|
|
+ },
|
|
|
)
|
|
|
connection.execute(
|
|
|
text(
|
|
|
@@ -759,16 +1246,24 @@ def test_real_postgres_mysql_minio_polars_cross_source_execution(tmp_path):
|
|
|
connection.execute(
|
|
|
text(
|
|
|
"DELETE FROM public.data_rule_versions "
|
|
|
- "WHERE id = CAST(:id AS uuid)"
|
|
|
+ "WHERE id IN (CAST(:id AS uuid), "
|
|
|
+ "CAST(:downstream_id AS uuid))"
|
|
|
),
|
|
|
- {"id": rule_id},
|
|
|
+ {
|
|
|
+ "id": rule_id,
|
|
|
+ "downstream_id": downstream_rule_id,
|
|
|
+ },
|
|
|
)
|
|
|
connection.execute(
|
|
|
text(
|
|
|
"DELETE FROM public.data_rules "
|
|
|
- "WHERE rule_uid = CAST(:rule_uid AS uuid)"
|
|
|
+ "WHERE rule_uid IN (CAST(:rule_uid AS uuid), "
|
|
|
+ "CAST(:downstream_rule_uid AS uuid))"
|
|
|
),
|
|
|
- {"rule_uid": rule_uid},
|
|
|
+ {
|
|
|
+ "rule_uid": rule_uid,
|
|
|
+ "downstream_rule_uid": downstream_rule_uid,
|
|
|
+ },
|
|
|
)
|
|
|
for schema in (input_schema, lookup_schema, output_schema):
|
|
|
connection.execute(
|