|
|
@@ -255,6 +255,64 @@ def test_graph_failure_is_persisted_but_finalize_failure_leaves_reconcilable_lea
|
|
|
)
|
|
|
|
|
|
|
|
|
+def test_governed_tag_failure_never_finalizes_completed(monkeypatch):
|
|
|
+ from tests.core.data_rules.test_contracts import valid_dataflow_spec
|
|
|
+ from tests.test_legacy_governance_cutover import PublishedAssetRepository
|
|
|
+
|
|
|
+ flow = valid_dataflow_spec()
|
|
|
+ actor = new_governance_uid()
|
|
|
+ repository = PublishedAssetRepository()
|
|
|
+ receipt = repository.reserve_dataflow_draft(actor_uid=actor)
|
|
|
+ receipt["dataflow_uid"] = flow["dataflow_uid"]
|
|
|
+ failures = []
|
|
|
+ repository.commit_dataflow_create_failure = (
|
|
|
+ lambda **kwargs: failures.append(kwargs)
|
|
|
+ )
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService,
|
|
|
+ "_merge_governed_dataflow",
|
|
|
+ lambda node: (73, {"id": 73, **node}),
|
|
|
+ )
|
|
|
+
|
|
|
+ def fail_tags(_dataflow_id, _tags, *, strict):
|
|
|
+ assert strict is True
|
|
|
+ raise RuntimeError("neo4j tag merge failed")
|
|
|
+
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService, "_handle_tag_relationships", fail_tags
|
|
|
+ )
|
|
|
+ with pytest.raises(RuntimeError, match="neo4j tag merge failed"):
|
|
|
+ DataFlowService.create_dataflow(
|
|
|
+ {
|
|
|
+ "name_zh": "标签严格生产线",
|
|
|
+ "describe": "严格标签合并验收",
|
|
|
+ "script_type": "governed",
|
|
|
+ "script_requirement": {
|
|
|
+ "dataflow_spec": flow,
|
|
|
+ "dataset_edges": {
|
|
|
+ "source_table": flow["input_schema_refs"],
|
|
|
+ "target_table": flow["output_schema_ref"],
|
|
|
+ },
|
|
|
+ "migration_metadata": {
|
|
|
+ "status": "migrated",
|
|
|
+ "legacy_fields_present": False,
|
|
|
+ "preserved_for_read_only": True,
|
|
|
+ "governed_semantics": "dataflow_spec",
|
|
|
+ },
|
|
|
+ },
|
|
|
+ "tag": [{"id": 81}],
|
|
|
+ "draft_reservation": {
|
|
|
+ key: receipt[key]
|
|
|
+ for key in ("reservation_id", "dataflow_uid", "nonce")
|
|
|
+ },
|
|
|
+ },
|
|
|
+ repository=repository,
|
|
|
+ actor_uid=actor,
|
|
|
+ )
|
|
|
+ assert repository.completed == {}
|
|
|
+ assert failures[0]["error_code"] == "neo4j_tag_merge_failed"
|
|
|
+
|
|
|
+
|
|
|
def test_reconciler_dry_run_is_read_only_and_digest_failure_is_isolated(
|
|
|
monkeypatch,
|
|
|
):
|
|
|
@@ -278,18 +336,8 @@ def test_reconciler_dry_run_is_read_only_and_digest_failure_is_isolated(
|
|
|
self.claim_calls = 0
|
|
|
self.failures = []
|
|
|
self.completions = []
|
|
|
-
|
|
|
- def list_reconcilable_dataflow_creates(self, *, limit):
|
|
|
- assert limit == 2
|
|
|
- return [{"reservation_id": "preview", "state": "failed"}]
|
|
|
-
|
|
|
- def claim_reconcilable_dataflow_creates(
|
|
|
- self, *, limit, lease_seconds
|
|
|
- ):
|
|
|
- assert limit == 2
|
|
|
- assert lease_seconds == 60
|
|
|
- self.claim_calls += 1
|
|
|
- return [
|
|
|
+ self.renewals = []
|
|
|
+ self.claims = [
|
|
|
{
|
|
|
"reservation_id": new_governance_uid(),
|
|
|
"dataflow_uid": bad_uid,
|
|
|
@@ -310,6 +358,24 @@ def test_reconciler_dry_run_is_read_only_and_digest_failure_is_isolated(
|
|
|
},
|
|
|
]
|
|
|
|
|
|
+ def list_reconcilable_dataflow_creates(self, *, limit):
|
|
|
+ assert limit == 2
|
|
|
+ return [{"reservation_id": "preview", "state": "failed"}]
|
|
|
+
|
|
|
+ def claim_reconcilable_dataflow_creates(
|
|
|
+ self, *, limit, lease_seconds
|
|
|
+ ):
|
|
|
+ assert limit == 1
|
|
|
+ assert lease_seconds == 60
|
|
|
+ self.claim_calls += 1
|
|
|
+ return [self.claims.pop(0)] if self.claims else []
|
|
|
+
|
|
|
+ def commit_dataflow_create_claim(self):
|
|
|
+ self.session.commit()
|
|
|
+
|
|
|
+ def renew_dataflow_create_lease(self, **kwargs):
|
|
|
+ self.renewals.append(kwargs)
|
|
|
+
|
|
|
def commit_dataflow_create_failure(self, **kwargs):
|
|
|
self.failures.append(kwargs)
|
|
|
self.session.commit()
|
|
|
@@ -345,3 +411,322 @@ def test_reconciler_dry_run_is_read_only_and_digest_failure_is_isolated(
|
|
|
"create_intent_integrity_failed"
|
|
|
)
|
|
|
assert repository.completions[0]["dataflow_uid"] == good_uid
|
|
|
+ assert len(repository.renewals) == 1
|
|
|
+
|
|
|
+
|
|
|
+def test_reconciler_validation_is_terminal_and_lost_lease_is_reported(
|
|
|
+ monkeypatch,
|
|
|
+):
|
|
|
+ flow_uid = new_governance_uid()
|
|
|
+
|
|
|
+ class Session:
|
|
|
+ def commit(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ def rollback(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ class Repository:
|
|
|
+ session = Session()
|
|
|
+
|
|
|
+ def __init__(self):
|
|
|
+ self.claimed = False
|
|
|
+ self.failure_codes = []
|
|
|
+
|
|
|
+ def claim_reconcilable_dataflow_creates(self, **_kwargs):
|
|
|
+ if self.claimed:
|
|
|
+ return []
|
|
|
+ self.claimed = True
|
|
|
+ return [
|
|
|
+ {
|
|
|
+ "reservation_id": new_governance_uid(),
|
|
|
+ "dataflow_uid": flow_uid,
|
|
|
+ "lease_token": new_governance_uid(),
|
|
|
+ "request_digest": "c" * 64,
|
|
|
+ "create_intent": {"dataflow_uid": flow_uid},
|
|
|
+ "attempt": 1,
|
|
|
+ "integrity_valid": True,
|
|
|
+ }
|
|
|
+ ]
|
|
|
+
|
|
|
+ def commit_dataflow_create_claim(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ def commit_dataflow_create_failure(self, **kwargs):
|
|
|
+ self.failure_codes.append(kwargs["error_code"])
|
|
|
+
|
|
|
+ repository = Repository()
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService,
|
|
|
+ "validate_governed_create_intent",
|
|
|
+ lambda *_args, **_kwargs: (_ for _ in ()).throw(
|
|
|
+ ValueError("published asset is unavailable")
|
|
|
+ ),
|
|
|
+ )
|
|
|
+ report = DataFlowCreateReconciler(repository).run(
|
|
|
+ dry_run=False, limit=2
|
|
|
+ )
|
|
|
+ assert report["items"][0] == {
|
|
|
+ "reservation_id": report["items"][0]["reservation_id"],
|
|
|
+ "dataflow_uid": flow_uid,
|
|
|
+ "attempt": 1,
|
|
|
+ "request_digest": "c" * 64,
|
|
|
+ "status": "failed",
|
|
|
+ "error_code": "create_intent_validation_failed",
|
|
|
+ }
|
|
|
+ assert repository.failure_codes == ["create_intent_validation_failed"]
|
|
|
+
|
|
|
+ repository = Repository()
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService,
|
|
|
+ "validate_governed_create_intent",
|
|
|
+ lambda value, **_kwargs: {"node": value, "tags": []},
|
|
|
+ )
|
|
|
+ repository.renew_dataflow_create_lease = lambda **_kwargs: (_ for _ in ()).throw(
|
|
|
+ RuntimeError("dataflow create lease was lost")
|
|
|
+ )
|
|
|
+ repository.commit_dataflow_create_failure = lambda **_kwargs: (
|
|
|
+ _ for _ in ()
|
|
|
+ ).throw(RuntimeError("dataflow create lease was lost"))
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService,
|
|
|
+ "_merge_governed_dataflow",
|
|
|
+ lambda _node: pytest.fail("side effect ran after lease loss"),
|
|
|
+ )
|
|
|
+ report = DataFlowCreateReconciler(repository).run(
|
|
|
+ dry_run=False, limit=1
|
|
|
+ )
|
|
|
+ assert report["items"][0]["status"] == "lease_recovery_required"
|
|
|
+ assert report["items"][0]["error_code"] == (
|
|
|
+ "failure_state_persistence_failed"
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+def test_tag_relationship_uses_one_atomic_merge(monkeypatch):
|
|
|
+ calls = []
|
|
|
+
|
|
|
+ class Result:
|
|
|
+ def single(self):
|
|
|
+ return {"merged": 1}
|
|
|
+
|
|
|
+ class Session:
|
|
|
+ def run(self, query, **values):
|
|
|
+ calls.append((query, values))
|
|
|
+ return Result()
|
|
|
+
|
|
|
+ def __enter__(self):
|
|
|
+ return self
|
|
|
+
|
|
|
+ def __exit__(self, *_args):
|
|
|
+ return None
|
|
|
+
|
|
|
+ class Driver:
|
|
|
+ def session(self):
|
|
|
+ return Session()
|
|
|
+
|
|
|
+ def close(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ monkeypatch.setattr(
|
|
|
+ "app.core.data_flow.dataflows.connect_graph", lambda: Driver()
|
|
|
+ )
|
|
|
+ DataFlowService._handle_single_tag_relationship(73, 81)
|
|
|
+ DataFlowService._handle_single_tag_relationship(73, 81)
|
|
|
+
|
|
|
+ assert len(calls) == 2
|
|
|
+ assert all("MERGE (a)-[:LABEL]->(b)" in query for query, _ in calls)
|
|
|
+ assert all("CREATE (a)-[:LABEL]->(b)" not in query for query, _ in calls)
|
|
|
+
|
|
|
+
|
|
|
+def test_reconciler_tag_failure_does_not_complete_and_retry_succeeds(
|
|
|
+ monkeypatch,
|
|
|
+):
|
|
|
+ flow_uid = new_governance_uid()
|
|
|
+ reservation_id = new_governance_uid()
|
|
|
+ digest = "d" * 64
|
|
|
+
|
|
|
+ class Session:
|
|
|
+ def commit(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ def rollback(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ class Repository:
|
|
|
+ session = Session()
|
|
|
+
|
|
|
+ def __init__(self):
|
|
|
+ self.attempt = 0
|
|
|
+ self.available = True
|
|
|
+ self.failures = []
|
|
|
+ self.completions = []
|
|
|
+
|
|
|
+ def claim_reconcilable_dataflow_creates(self, **_kwargs):
|
|
|
+ if not self.available:
|
|
|
+ return []
|
|
|
+ self.available = False
|
|
|
+ self.attempt += 1
|
|
|
+ return [
|
|
|
+ {
|
|
|
+ "reservation_id": reservation_id,
|
|
|
+ "dataflow_uid": flow_uid,
|
|
|
+ "lease_token": new_governance_uid(),
|
|
|
+ "request_digest": digest,
|
|
|
+ "create_intent": {"dataflow_uid": flow_uid},
|
|
|
+ "attempt": self.attempt,
|
|
|
+ "integrity_valid": True,
|
|
|
+ }
|
|
|
+ ]
|
|
|
+
|
|
|
+ def commit_dataflow_create_claim(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ def renew_dataflow_create_lease(self, **_kwargs):
|
|
|
+ return None
|
|
|
+
|
|
|
+ def commit_dataflow_create_failure(self, **kwargs):
|
|
|
+ self.failures.append(kwargs)
|
|
|
+ self.available = True
|
|
|
+
|
|
|
+ def complete_dataflow_create(self, **kwargs):
|
|
|
+ self.completions.append(kwargs)
|
|
|
+ return {"id": 73, "uid": flow_uid}
|
|
|
+
|
|
|
+ repository = Repository()
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService,
|
|
|
+ "validate_governed_create_intent",
|
|
|
+ lambda _value, **_kwargs: {
|
|
|
+ "node": {"uid": flow_uid},
|
|
|
+ "tags": [{"id": 81}],
|
|
|
+ },
|
|
|
+ )
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService,
|
|
|
+ "_merge_governed_dataflow",
|
|
|
+ lambda node: (73, {"id": 73, **node}),
|
|
|
+ )
|
|
|
+ attempts = {"count": 0}
|
|
|
+
|
|
|
+ def merge_tags(_node_id, _tags, *, strict):
|
|
|
+ assert strict is True
|
|
|
+ attempts["count"] += 1
|
|
|
+ if attempts["count"] == 1:
|
|
|
+ raise RuntimeError("neo4j tag write failed")
|
|
|
+
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService, "_handle_tag_relationships", merge_tags
|
|
|
+ )
|
|
|
+ reconciler = DataFlowCreateReconciler(repository)
|
|
|
+ failed = reconciler.run(dry_run=False, limit=1)
|
|
|
+ assert failed["items"][0]["status"] == "failed"
|
|
|
+ assert repository.completions == []
|
|
|
+ assert repository.failures[0]["error_code"] == "reconciliation_failed"
|
|
|
+
|
|
|
+ completed = reconciler.run(dry_run=False, limit=1)
|
|
|
+ assert completed["items"][0]["status"] == "completed"
|
|
|
+ assert len(repository.completions) == 1
|
|
|
+ assert attempts["count"] == 2
|
|
|
+
|
|
|
+
|
|
|
+def test_slow_reconciler_does_not_preclaim_next_item_from_second_worker(
|
|
|
+ monkeypatch,
|
|
|
+):
|
|
|
+ first_uid = new_governance_uid()
|
|
|
+ second_uid = new_governance_uid()
|
|
|
+ shared = {
|
|
|
+ first_uid: {"state": "failed", "attempt": 0},
|
|
|
+ second_uid: {"state": "failed", "attempt": 0},
|
|
|
+ }
|
|
|
+ processed = []
|
|
|
+
|
|
|
+ class Session:
|
|
|
+ def commit(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ def rollback(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ class Repository:
|
|
|
+ session = Session()
|
|
|
+
|
|
|
+ def __init__(self, worker):
|
|
|
+ self.worker = worker
|
|
|
+
|
|
|
+ def claim_reconcilable_dataflow_creates(
|
|
|
+ self, *, limit, lease_seconds
|
|
|
+ ):
|
|
|
+ assert limit == 1
|
|
|
+ assert lease_seconds == 10
|
|
|
+ available = [
|
|
|
+ uid
|
|
|
+ for uid, record in shared.items()
|
|
|
+ if record["state"] == "failed"
|
|
|
+ ]
|
|
|
+ if not available:
|
|
|
+ return []
|
|
|
+ uid = available[0]
|
|
|
+ shared[uid]["state"] = f"creating:{self.worker}"
|
|
|
+ shared[uid]["attempt"] += 1
|
|
|
+ return [
|
|
|
+ {
|
|
|
+ "reservation_id": new_governance_uid(),
|
|
|
+ "dataflow_uid": uid,
|
|
|
+ "lease_token": new_governance_uid(),
|
|
|
+ "request_digest": ("a" if uid == first_uid else "b") * 64,
|
|
|
+ "create_intent": {"dataflow_uid": uid},
|
|
|
+ "attempt": shared[uid]["attempt"],
|
|
|
+ "integrity_valid": True,
|
|
|
+ }
|
|
|
+ ]
|
|
|
+
|
|
|
+ def commit_dataflow_create_claim(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ def renew_dataflow_create_lease(self, **_kwargs):
|
|
|
+ return None
|
|
|
+
|
|
|
+ def complete_dataflow_create(self, *, dataflow_uid, **_kwargs):
|
|
|
+ assert shared[dataflow_uid]["state"] == f"creating:{self.worker}"
|
|
|
+ shared[dataflow_uid]["state"] = "completed"
|
|
|
+ processed.append((self.worker, dataflow_uid))
|
|
|
+ return {"id": 73, "uid": dataflow_uid}
|
|
|
+
|
|
|
+ def commit_dataflow_create_failure(self, **_kwargs):
|
|
|
+ pytest.fail("unexpected reconciliation failure")
|
|
|
+
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService,
|
|
|
+ "validate_governed_create_intent",
|
|
|
+ lambda value, **_kwargs: {
|
|
|
+ "node": {"uid": value["dataflow_uid"]},
|
|
|
+ "tags": [],
|
|
|
+ },
|
|
|
+ )
|
|
|
+ worker_two = DataFlowCreateReconciler(Repository("worker-2"))
|
|
|
+ entered_slow_call = {"value": False}
|
|
|
+
|
|
|
+ def slow_merge(node):
|
|
|
+ if node["uid"] == first_uid and not entered_slow_call["value"]:
|
|
|
+ entered_slow_call["value"] = True
|
|
|
+ # This models worker 1 crossing its original lease boundary while
|
|
|
+ # processing item A. Item B was never preclaimed, so worker 2 can
|
|
|
+ # safely process B without ever sharing ownership of A.
|
|
|
+ second_report = worker_two.run(
|
|
|
+ dry_run=False, limit=1, lease_seconds=10
|
|
|
+ )
|
|
|
+ assert second_report["items"][0]["dataflow_uid"] == second_uid
|
|
|
+ return 73, {"id": 73, **node}
|
|
|
+
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService, "_merge_governed_dataflow", slow_merge
|
|
|
+ )
|
|
|
+ first_report = DataFlowCreateReconciler(Repository("worker-1")).run(
|
|
|
+ dry_run=False, limit=2, lease_seconds=10
|
|
|
+ )
|
|
|
+ assert first_report["reconciled_count"] == 1
|
|
|
+ assert processed == [
|
|
|
+ ("worker-2", second_uid),
|
|
|
+ ("worker-1", first_uid),
|
|
|
+ ]
|
|
|
+ assert all(record["attempt"] == 1 for record in shared.values())
|
马小龙