test_rule_physical_publication_lifecycle.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792
  1. from __future__ import annotations
  2. import json
  3. import time
  4. from concurrent.futures import ThreadPoolExecutor
  5. import pytest
  6. from sqlalchemy import create_engine, text
  7. from sqlalchemy.orm import Session
  8. from app.core.common.identifiers import new_governance_uid
  9. from app.core.data_rules.compilers.sql import SqlGlotRuleCompiler
  10. from app.core.data_rules.contracts import rule_spec_hash, validate_rule_spec
  11. from app.core.data_rules.publication import (
  12. PhysicalPlanPublicationService,
  13. ServerOwnedPhysicalPreflightRunner,
  14. )
  15. from app.core.data_rules.repository import DataRuleRepository
  16. from tests.integration.test_data_rule_polars_execution import _compose_value
  17. from tests.integration.test_data_rule_sql_execution import (
  18. CASES,
  19. ReadOnlyPreflightManager,
  20. _snapshot,
  21. )
  22. pytestmark = pytest.mark.integration
  23. def _platform_url() -> str:
  24. user = _compose_value(r"POSTGRES_USER:\s*([^\s]+)")
  25. password = _compose_value(r"POSTGRES_PASSWORD:\s*([^\s]+)")
  26. port = _compose_value(r'"(15432):5432"')
  27. return (
  28. f"postgresql+psycopg2://{user}:{password}"
  29. f"@127.0.0.1:{port}/dataops"
  30. )
  31. def _insert_logical_trust(
  32. connection,
  33. *,
  34. rule_id: str,
  35. actor_id: str,
  36. input_schema: dict,
  37. output_schema: dict,
  38. compiled: dict,
  39. ) -> None:
  40. profile_id = new_governance_uid()
  41. logical_id = new_governance_uid()
  42. logical_schema_hashes = {
  43. "input": input_schema["schema_hash"],
  44. "output": output_schema["schema_hash"],
  45. }
  46. capabilities = compiled["plan"]["capabilities"]
  47. connection.execute(
  48. text(
  49. """
  50. INSERT INTO public.rule_validation_profiles
  51. (id, rule_version_id, input_schema_snapshot_id,
  52. input_schema_hash, input_fields, output_schema_snapshot_id,
  53. output_schema_hash, output_fields,
  54. input_sample_artifact_ref, input_sample_artifact_digest,
  55. context_hash)
  56. VALUES
  57. (CAST(:id AS uuid), CAST(:rule_id AS uuid),
  58. CAST(:input_id AS uuid), :input_hash,
  59. CAST(:input_fields AS jsonb), CAST(:output_id AS uuid),
  60. :output_hash, CAST(:output_fields AS jsonb),
  61. 'test://physical-publication-input', :digest, :digest)
  62. """
  63. ),
  64. {
  65. "id": profile_id,
  66. "rule_id": rule_id,
  67. "input_id": input_schema["id"],
  68. "input_hash": input_schema["schema_hash"],
  69. "input_fields": json.dumps(input_schema["fields"]),
  70. "output_id": output_schema["id"],
  71. "output_hash": output_schema["schema_hash"],
  72. "output_fields": json.dumps(output_schema["fields"]),
  73. "digest": "a" * 64,
  74. },
  75. )
  76. connection.execute(
  77. text(
  78. """
  79. INSERT INTO public.rule_logical_plans
  80. (id, rule_version_id, validation_profile_id, compiler_version,
  81. backend, plan, plan_hash, schema_hashes, capabilities, status)
  82. VALUES
  83. (CAST(:id AS uuid), CAST(:rule_id AS uuid),
  84. CAST(:profile_id AS uuid), :compiler_version, 'sql_pushdown',
  85. CAST(:plan AS jsonb), :plan_hash, CAST(:schema_hashes AS jsonb),
  86. CAST(:capabilities AS jsonb), 'published')
  87. """
  88. ),
  89. {
  90. "id": logical_id,
  91. "rule_id": rule_id,
  92. "profile_id": profile_id,
  93. "compiler_version": compiled["compiler_version"],
  94. "plan": json.dumps(compiled["plan"]),
  95. "plan_hash": compiled["plan_hash"],
  96. "schema_hashes": json.dumps(logical_schema_hashes),
  97. "capabilities": json.dumps(capabilities),
  98. },
  99. )
  100. connection.execute(
  101. text(
  102. """
  103. INSERT INTO public.rule_logical_compile_evidence
  104. (id, logical_plan_id, compiler_version, compiler_digest,
  105. plan_hash, schema_hashes, capabilities, status, created_by)
  106. VALUES
  107. (CAST(:id AS uuid), CAST(:logical_id AS uuid),
  108. :compiler_version, :digest, :plan_hash,
  109. CAST(:schema_hashes AS jsonb), CAST(:capabilities AS jsonb),
  110. 'success', CAST(:actor_id AS uuid))
  111. """
  112. ),
  113. {
  114. "id": new_governance_uid(),
  115. "logical_id": logical_id,
  116. "compiler_version": compiled["compiler_version"],
  117. "digest": "b" * 64,
  118. "plan_hash": compiled["plan_hash"],
  119. "schema_hashes": json.dumps(logical_schema_hashes),
  120. "capabilities": json.dumps(capabilities),
  121. "actor_id": actor_id,
  122. },
  123. )
  124. connection.execute(
  125. text(
  126. """
  127. INSERT INTO public.rule_logical_test_evidence
  128. (id, logical_plan_id, test_kind, evidence_hash, run_id,
  129. plan_hash, schema_hashes, evidence, status, created_by)
  130. VALUES
  131. (CAST(:id AS uuid), CAST(:logical_id AS uuid), 'dry_run',
  132. :digest, CAST(:run_id AS uuid), :plan_hash,
  133. CAST(:schema_hashes AS jsonb), '{}'::jsonb, 'success',
  134. CAST(:actor_id AS uuid))
  135. """
  136. ),
  137. {
  138. "id": new_governance_uid(),
  139. "logical_id": logical_id,
  140. "digest": "c" * 64,
  141. "run_id": new_governance_uid(),
  142. "plan_hash": compiled["plan_hash"],
  143. "schema_hashes": json.dumps(logical_schema_hashes),
  144. "actor_id": actor_id,
  145. },
  146. )
  147. @pytest.mark.parametrize(
  148. ("dialect", "url", "schema_name", "collation", "regex_engine"), CASES
  149. )
  150. def test_physical_service_rejects_post_test_canonical_drift_and_replays(
  151. dialect, url, schema_name, collation, regex_engine
  152. ):
  153. source_engine = create_engine(url, pool_pre_ping=True)
  154. platform_engine = create_engine(_platform_url(), pool_pre_ping=True)
  155. suffix = dialect.replace("postgresql", "pg")
  156. source_table = f"task7_publish_source_{suffix}"
  157. target_table = f"task7_publish_target_{suffix}"
  158. source_ref = f"{schema_name}.{source_table}"
  159. target_ref = f"{schema_name}.{target_table}"
  160. input_schema = _snapshot(f"bd:task7:publish:{dialect}:input")
  161. output_schema = _snapshot(f"bd:task7:publish:{dialect}:output")
  162. data_source_uid = new_governance_uid()
  163. input_binding = {
  164. "id": new_governance_uid(),
  165. "data_source_uid": data_source_uid,
  166. "object_kind": "table",
  167. "object_ref": source_ref,
  168. "schema_snapshot_id": input_schema["id"],
  169. "access_mode": "read",
  170. "dialect": dialect,
  171. "write_mode": "append",
  172. }
  173. output_binding = {
  174. "id": new_governance_uid(),
  175. "data_source_uid": data_source_uid,
  176. "object_kind": "table",
  177. "object_ref": target_ref,
  178. "schema_snapshot_id": output_schema["id"],
  179. "access_mode": "write",
  180. "dialect": dialect,
  181. "write_mode": "append",
  182. }
  183. spec = validate_rule_spec(
  184. {
  185. "schema_version": "2.0",
  186. "rule_uid": new_governance_uid(),
  187. "name": f"task7_physical_publication_{dialect}",
  188. "input_schema_ref": input_schema["schema_ref"],
  189. "output_schema_ref": output_schema["schema_ref"],
  190. "steps": [
  191. {
  192. "id": "trim_name",
  193. "op": "normalize_text",
  194. "column": "name",
  195. "trim": True,
  196. }
  197. ],
  198. "null_policy": "explicit",
  199. "timezone": "Asia/Shanghai",
  200. }
  201. )
  202. rule_id = new_governance_uid()
  203. compiled = SqlGlotRuleCompiler(dialect).compile(
  204. rule_version={
  205. "id": rule_id,
  206. "status": "published",
  207. "rule_spec": spec,
  208. "spec_hash": rule_spec_hash(spec),
  209. },
  210. input_schema=input_schema,
  211. output_schema=output_schema,
  212. input_binding=input_binding,
  213. output_binding=output_binding,
  214. backend={
  215. "dialect": dialect,
  216. "timezone": "Asia/Shanghai",
  217. "collation": collation,
  218. "rounding_mode": "half_away_from_zero",
  219. "regex_engine": regex_engine,
  220. },
  221. )
  222. try:
  223. with source_engine.begin() as source:
  224. source.execute(text(f"DROP TABLE IF EXISTS {target_table}"))
  225. source.execute(text(f"DROP TABLE IF EXISTS {source_table}"))
  226. source.execute(
  227. text(
  228. f"CREATE TABLE {source_table} ("
  229. "customer_id BIGINT PRIMARY KEY, name VARCHAR(100), "
  230. "mobile VARCHAR(30))"
  231. )
  232. )
  233. source.execute(
  234. text(
  235. f"CREATE TABLE {target_table} ("
  236. "customer_id BIGINT PRIMARY KEY, name VARCHAR(100), "
  237. "mobile VARCHAR(30))"
  238. )
  239. )
  240. source.execute(
  241. text(
  242. f"INSERT INTO {source_table} "
  243. "(customer_id, name, mobile) VALUES "
  244. "(1, ' Alice ', '13800138000')"
  245. )
  246. )
  247. with platform_engine.connect() as connection:
  248. transaction = connection.begin()
  249. try:
  250. actor_id = new_governance_uid()
  251. rule_uid = spec["rule_uid"]
  252. dataflow_version_id = new_governance_uid()
  253. deployment_id = new_governance_uid()
  254. component_id = new_governance_uid()
  255. connection.execute(
  256. text(
  257. "INSERT INTO public.users "
  258. "(id, username, display_name, password_hash, status) "
  259. "VALUES (CAST(:id AS uuid), :username, 'Task7', "
  260. "'not-a-login-secret', 'active')"
  261. ),
  262. {
  263. "id": actor_id,
  264. "username": f"task7-{actor_id[:8]}",
  265. },
  266. )
  267. for snapshot in (input_schema, output_schema):
  268. connection.execute(
  269. text(
  270. "INSERT INTO public.data_schema_snapshots "
  271. "(id, schema_ref, schema_hash, fields, "
  272. "source_revision) VALUES "
  273. "(CAST(:id AS uuid), :schema_ref, :schema_hash, "
  274. "CAST(:fields AS jsonb), :source_revision)"
  275. ),
  276. {
  277. **snapshot,
  278. "fields": json.dumps(snapshot["fields"]),
  279. },
  280. )
  281. connection.execute(
  282. text(
  283. "INSERT INTO public.data_rules "
  284. "(id, rule_uid, name, category, status) VALUES "
  285. "(CAST(:id AS uuid), CAST(:rule_uid AS uuid), "
  286. ":name, 'general', 'active')"
  287. ),
  288. {
  289. "id": new_governance_uid(),
  290. "rule_uid": rule_uid,
  291. "name": spec["name"],
  292. },
  293. )
  294. connection.execute(
  295. text(
  296. "INSERT INTO public.data_rule_versions "
  297. "(id, rule_uid, version_no, source_text, "
  298. "source_language, rule_spec, spec_hash, "
  299. "generated_kind, status, published_at) VALUES "
  300. "(CAST(:id AS uuid), CAST(:rule_uid AS uuid), 1, "
  301. "'Task7 physical publication', 'en', "
  302. "CAST(:rule_spec AS jsonb), :spec_hash, 'sql', "
  303. "'published', CURRENT_TIMESTAMP)"
  304. ),
  305. {
  306. "id": rule_id,
  307. "rule_uid": rule_uid,
  308. "rule_spec": json.dumps(spec),
  309. "spec_hash": rule_spec_hash(spec),
  310. },
  311. )
  312. _insert_logical_trust(
  313. connection,
  314. rule_id=rule_id,
  315. actor_id=actor_id,
  316. input_schema=input_schema,
  317. output_schema=output_schema,
  318. compiled=compiled,
  319. )
  320. connection.execute(
  321. text(
  322. "INSERT INTO public.dataflow_versions "
  323. "(id, dataflow_uid, version_no, name, dataflow_spec, "
  324. "input_schema_hashes, output_schema_hash, status, "
  325. "released_at) VALUES "
  326. "(CAST(:id AS uuid), CAST(:uid AS uuid), 1, "
  327. "'Task7 physical publication', '{}'::jsonb, "
  328. "CAST(:inputs AS jsonb), :output, 'released', "
  329. "CURRENT_TIMESTAMP)"
  330. ),
  331. {
  332. "id": dataflow_version_id,
  333. "uid": new_governance_uid(),
  334. "inputs": json.dumps([input_schema["schema_hash"]]),
  335. "output": output_schema["schema_hash"],
  336. },
  337. )
  338. connection.execute(
  339. text(
  340. "INSERT INTO public.dataflow_deployments "
  341. "(id, dataflow_version_id, environment, "
  342. "deployment_config, status) VALUES "
  343. "(CAST(:id AS uuid), CAST(:version_id AS uuid), "
  344. "'test', '{}'::jsonb, 'disabled')"
  345. ),
  346. {
  347. "id": deployment_id,
  348. "version_id": dataflow_version_id,
  349. },
  350. )
  351. connection.execute(
  352. text(
  353. "INSERT INTO public.dataflow_component_bindings "
  354. "(id, dataflow_version_id, component_id, "
  355. "component_kind, rule_version_id, stage, order_no, "
  356. "idempotency, provenance) VALUES "
  357. "(CAST(:id AS uuid), CAST(:version_id AS uuid), "
  358. "'task7_publish', 'rule.apply', "
  359. "CAST(:rule_id AS uuid), 'transform', 0, "
  360. "'{\"strategy\":\"upsert\",\"key\":\"customer_id\"}'"
  361. "::jsonb, '{}'::jsonb)"
  362. ),
  363. {
  364. "id": component_id,
  365. "version_id": dataflow_version_id,
  366. "rule_id": rule_id,
  367. },
  368. )
  369. input_binding_hash = "d" * 64
  370. output_binding_hash = "e" * 64
  371. for logical_ref, binding, binding_hash in (
  372. (
  373. "input",
  374. input_binding,
  375. input_binding_hash,
  376. ),
  377. (
  378. "output",
  379. output_binding,
  380. output_binding_hash,
  381. ),
  382. ):
  383. connection.execute(
  384. text(
  385. "INSERT INTO public.dataflow_dataset_bindings "
  386. "(id, dataflow_deployment_id, logical_ref, "
  387. "data_source_uid, object_kind, object_ref, "
  388. "schema_snapshot_id, dialect, access_mode, "
  389. "write_mode, binding_hash) VALUES "
  390. "(CAST(:id AS uuid), CAST(:deployment_id AS uuid), "
  391. ":logical_ref, CAST(:source_uid AS uuid), 'table', "
  392. ":object_ref, CAST(:snapshot_id AS uuid), "
  393. ":dialect, :access_mode, 'append', :binding_hash)"
  394. ),
  395. {
  396. "id": binding["id"],
  397. "deployment_id": deployment_id,
  398. "logical_ref": logical_ref,
  399. "source_uid": data_source_uid,
  400. "object_ref": binding["object_ref"],
  401. "snapshot_id": binding["schema_snapshot_id"],
  402. "dialect": dialect,
  403. "access_mode": binding["access_mode"],
  404. "binding_hash": binding_hash,
  405. },
  406. )
  407. session = Session(bind=connection)
  408. repository = DataRuleRepository(session)
  409. persisted = repository.persist_bound_component_plan(
  410. component_binding_id=component_id,
  411. rule_version_id=rule_id,
  412. input_binding_id=input_binding["id"],
  413. output_binding_id=output_binding["id"],
  414. compiled=compiled,
  415. )
  416. service = PhysicalPlanPublicationService(
  417. repository,
  418. test_runner=ServerOwnedPhysicalPreflightRunner(
  419. artifact_store=None,
  420. datasource_manager=ReadOnlyPreflightManager(
  421. source_engine
  422. ),
  423. ),
  424. )
  425. validated = service.validate(persisted["id"], actor_id)
  426. assert service.validate(persisted["id"], actor_id) == validated
  427. tested = service.test(persisted["id"], actor_id)
  428. assert service.test(persisted["id"], actor_id) == tested
  429. drift_cases = (
  430. (
  431. "dataflow_dataset_bindings",
  432. input_binding["id"],
  433. "binding_hash",
  434. "f" * 64,
  435. input_binding_hash,
  436. ),
  437. (
  438. "dataflow_dataset_bindings",
  439. input_binding["id"],
  440. "data_source_uid",
  441. new_governance_uid(),
  442. data_source_uid,
  443. ),
  444. (
  445. "dataflow_dataset_bindings",
  446. input_binding["id"],
  447. "object_ref",
  448. f"{schema_name}.drifted_source",
  449. source_ref,
  450. ),
  451. (
  452. "dataflow_dataset_bindings",
  453. input_binding["id"],
  454. "dialect",
  455. "mysql" if dialect == "postgresql" else "postgresql",
  456. dialect,
  457. ),
  458. (
  459. "data_schema_snapshots",
  460. output_schema["id"],
  461. "schema_hash",
  462. "f" * 64,
  463. output_schema["schema_hash"],
  464. ),
  465. (
  466. "data_rule_versions",
  467. rule_id,
  468. "status",
  469. "revoked",
  470. "published",
  471. ),
  472. (
  473. "rule_execution_plans",
  474. persisted["id"],
  475. "compiler_version",
  476. "drifted-compiler",
  477. compiled["compiler_version"],
  478. ),
  479. )
  480. for table_name, row_id, column, drifted, canonical in drift_cases:
  481. connection.execute(
  482. text(
  483. f"UPDATE public.{table_name} SET {column} = :value "
  484. "WHERE id = CAST(:id AS uuid)"
  485. ),
  486. {"id": row_id, "value": drifted},
  487. )
  488. with pytest.raises(ValueError, match="drifted|not found"):
  489. service.publish(persisted["id"], actor_id)
  490. connection.execute(
  491. text(
  492. f"UPDATE public.{table_name} SET {column} = :value "
  493. "WHERE id = CAST(:id AS uuid)"
  494. ),
  495. {"id": row_id, "value": canonical},
  496. )
  497. connection.execute(
  498. text(
  499. "UPDATE public.rule_test_evidence "
  500. "SET schema_hashes = '{}'::jsonb "
  501. "WHERE rule_execution_plan_id = CAST(:id AS uuid)"
  502. ),
  503. {"id": persisted["id"]},
  504. )
  505. with pytest.raises(ValueError, match="drifted"):
  506. service.publish(persisted["id"], actor_id)
  507. connection.execute(
  508. text(
  509. "UPDATE public.rule_test_evidence te "
  510. "SET schema_hashes = p.schema_hashes "
  511. "FROM public.rule_execution_plans p "
  512. "WHERE te.rule_execution_plan_id = p.id "
  513. "AND p.id = CAST(:id AS uuid)"
  514. ),
  515. {"id": persisted["id"]},
  516. )
  517. if dialect == "postgresql":
  518. second_actor_id = new_governance_uid()
  519. second_component_id = new_governance_uid()
  520. connection.execute(
  521. text(
  522. "INSERT INTO public.users "
  523. "(id, username, display_name, password_hash, "
  524. "status) VALUES (CAST(:id AS uuid), :username, "
  525. "'Task7 Concurrent', 'not-a-login-secret', "
  526. "'active')"
  527. ),
  528. {
  529. "id": second_actor_id,
  530. "username": f"task7-{second_actor_id}",
  531. },
  532. )
  533. connection.execute(
  534. text(
  535. "INSERT INTO public.dataflow_component_bindings "
  536. "(id, dataflow_version_id, component_id, "
  537. "component_kind, rule_version_id, stage, order_no, "
  538. "idempotency, provenance) VALUES "
  539. "(CAST(:id AS uuid), CAST(:version_id AS uuid), "
  540. "'task7_publish_concurrent', 'rule.apply', "
  541. "CAST(:rule_id AS uuid), 'transform', 1, "
  542. "'{\"strategy\":\"upsert\","
  543. "\"key\":\"customer_id\"}'::jsonb, '{}'::jsonb)"
  544. ),
  545. {
  546. "id": second_component_id,
  547. "version_id": dataflow_version_id,
  548. "rule_id": rule_id,
  549. },
  550. )
  551. second_plan = repository.persist_bound_component_plan(
  552. component_binding_id=second_component_id,
  553. rule_version_id=rule_id,
  554. input_binding_id=input_binding["id"],
  555. output_binding_id=output_binding["id"],
  556. compiled=compiled,
  557. )
  558. service.validate(second_plan["id"], actor_id)
  559. service.test(second_plan["id"], actor_id)
  560. transaction.commit()
  561. def publish_in_session(
  562. plan_id: str,
  563. publishing_actor: str,
  564. delay: float = 0.0,
  565. ):
  566. if delay:
  567. time.sleep(delay)
  568. with platform_engine.begin() as worker_connection:
  569. worker_service = PhysicalPlanPublicationService(
  570. DataRuleRepository(
  571. Session(bind=worker_connection)
  572. ),
  573. test_runner=None,
  574. )
  575. try:
  576. return (
  577. "ok",
  578. worker_service.publish(
  579. plan_id, publishing_actor
  580. ),
  581. )
  582. except ValueError as exc:
  583. return ("rejected", str(exc))
  584. with ThreadPoolExecutor(max_workers=2) as executor:
  585. same_actor = [
  586. executor.submit(
  587. publish_in_session,
  588. persisted["id"],
  589. actor_id,
  590. )
  591. for _ in range(2)
  592. ]
  593. same_actor_results = [
  594. future.result() for future in same_actor
  595. ]
  596. assert [item[0] for item in same_actor_results] == [
  597. "ok",
  598. "ok",
  599. ]
  600. assert (
  601. same_actor_results[0][1]
  602. == same_actor_results[1][1]
  603. )
  604. with ThreadPoolExecutor(max_workers=2) as executor:
  605. owner_future = executor.submit(
  606. publish_in_session,
  607. second_plan["id"],
  608. actor_id,
  609. )
  610. other_future = executor.submit(
  611. publish_in_session,
  612. second_plan["id"],
  613. second_actor_id,
  614. 0.05,
  615. )
  616. assert owner_future.result()[0] == "ok"
  617. assert other_future.result()[0] == "rejected"
  618. with platform_engine.begin() as cleanup:
  619. plan_ids = [
  620. persisted["id"],
  621. second_plan["id"],
  622. ]
  623. cleanup.execute(
  624. text(
  625. "DELETE FROM public.rule_publication_audits "
  626. "WHERE rule_execution_plan_id = "
  627. "ANY(CAST(:ids AS uuid[]))"
  628. ),
  629. {"ids": plan_ids},
  630. )
  631. for evidence_table in (
  632. "rule_test_evidence",
  633. "rule_compile_evidence",
  634. ):
  635. cleanup.execute(
  636. text(
  637. f"DELETE FROM public.{evidence_table} "
  638. "WHERE rule_execution_plan_id = "
  639. "ANY(CAST(:ids AS uuid[]))"
  640. ),
  641. {"ids": plan_ids},
  642. )
  643. cleanup.execute(
  644. text(
  645. "DELETE FROM public.rule_execution_plans "
  646. "WHERE id = ANY(CAST(:ids AS uuid[]))"
  647. ),
  648. {"ids": plan_ids},
  649. )
  650. cleanup.execute(
  651. text(
  652. "DELETE FROM public.dataflow_dataset_bindings "
  653. "WHERE dataflow_deployment_id = "
  654. "CAST(:id AS uuid)"
  655. ),
  656. {"id": deployment_id},
  657. )
  658. cleanup.execute(
  659. text(
  660. "DELETE FROM "
  661. "public.dataflow_component_bindings "
  662. "WHERE id = ANY(CAST(:ids AS uuid[]))"
  663. ),
  664. {
  665. "ids": [
  666. component_id,
  667. second_component_id,
  668. ]
  669. },
  670. )
  671. cleanup.execute(
  672. text(
  673. "DELETE FROM public.dataflow_deployments "
  674. "WHERE id = CAST(:id AS uuid)"
  675. ),
  676. {"id": deployment_id},
  677. )
  678. cleanup.execute(
  679. text(
  680. "DELETE FROM public.dataflow_versions "
  681. "WHERE id = CAST(:id AS uuid)"
  682. ),
  683. {"id": dataflow_version_id},
  684. )
  685. cleanup.execute(
  686. text(
  687. "DELETE FROM "
  688. "public.rule_logical_test_evidence "
  689. "WHERE logical_plan_id IN (SELECT id FROM "
  690. "public.rule_logical_plans WHERE "
  691. "rule_version_id = CAST(:id AS uuid))"
  692. ),
  693. {"id": rule_id},
  694. )
  695. cleanup.execute(
  696. text(
  697. "DELETE FROM "
  698. "public.rule_logical_compile_evidence "
  699. "WHERE logical_plan_id IN (SELECT id FROM "
  700. "public.rule_logical_plans WHERE "
  701. "rule_version_id = CAST(:id AS uuid))"
  702. ),
  703. {"id": rule_id},
  704. )
  705. cleanup.execute(
  706. text(
  707. "DELETE FROM public.rule_logical_plans "
  708. "WHERE rule_version_id = CAST(:id AS uuid)"
  709. ),
  710. {"id": rule_id},
  711. )
  712. cleanup.execute(
  713. text(
  714. "DELETE FROM "
  715. "public.rule_validation_profiles "
  716. "WHERE rule_version_id = CAST(:id AS uuid)"
  717. ),
  718. {"id": rule_id},
  719. )
  720. cleanup.execute(
  721. text(
  722. "DELETE FROM public.data_rule_versions "
  723. "WHERE id = CAST(:id AS uuid)"
  724. ),
  725. {"id": rule_id},
  726. )
  727. cleanup.execute(
  728. text(
  729. "DELETE FROM public.data_rules "
  730. "WHERE rule_uid = CAST(:id AS uuid)"
  731. ),
  732. {"id": rule_uid},
  733. )
  734. cleanup.execute(
  735. text(
  736. "DELETE FROM public.data_schema_snapshots "
  737. "WHERE id = ANY(CAST(:ids AS uuid[]))"
  738. ),
  739. {
  740. "ids": [
  741. input_schema["id"],
  742. output_schema["id"],
  743. ]
  744. },
  745. )
  746. cleanup.execute(
  747. text(
  748. "DELETE FROM public.users "
  749. "WHERE id = ANY(CAST(:ids AS uuid[]))"
  750. ),
  751. {"ids": [actor_id, second_actor_id]},
  752. )
  753. else:
  754. published = service.publish(persisted["id"], actor_id)
  755. assert (
  756. service.publish(persisted["id"], actor_id)
  757. == published
  758. )
  759. audit_count = connection.execute(
  760. text(
  761. "SELECT COUNT(*) FROM "
  762. "public.rule_publication_audits "
  763. "WHERE rule_execution_plan_id = "
  764. "CAST(:id AS uuid) AND action = 'published'"
  765. ),
  766. {"id": persisted["id"]},
  767. ).scalar_one()
  768. assert audit_count == 1
  769. finally:
  770. if transaction.is_active:
  771. transaction.rollback()
  772. finally:
  773. with source_engine.begin() as source:
  774. source.execute(text(f"DROP TABLE IF EXISTS {target_table}"))
  775. source.execute(text(f"DROP TABLE IF EXISTS {source_table}"))
  776. source_engine.dispose()
  777. platform_engine.dispose()