|
|
@@ -0,0 +1,408 @@
|
|
|
+from __future__ import annotations
|
|
|
+
|
|
|
+import json
|
|
|
+import uuid
|
|
|
+from datetime import UTC, datetime, timedelta
|
|
|
+
|
|
|
+import pytest
|
|
|
+
|
|
|
+from app.core.common.identifiers import new_governance_uid
|
|
|
+from app.core.system.tokens import decode_access_token, issue_access_token
|
|
|
+from tests.core.data_rules.test_contracts import valid_dataflow_spec
|
|
|
+
|
|
|
+
|
|
|
+class PublishedAssetRepository:
|
|
|
+ def __init__(self, *, published=True):
|
|
|
+ self.published = published
|
|
|
+ self.rule_calls = []
|
|
|
+ self.dataflow_calls = []
|
|
|
+
|
|
|
+ def require_published_rule_version(self, rule_version_id):
|
|
|
+ self.rule_calls.append(rule_version_id)
|
|
|
+ if not self.published:
|
|
|
+ raise ValueError("rule version is not published")
|
|
|
+ return {"id": rule_version_id, "status": "published"}
|
|
|
+
|
|
|
+ def load_published_assets(self, dataflow_spec):
|
|
|
+ self.dataflow_calls.append(dataflow_spec)
|
|
|
+ if not self.published:
|
|
|
+ raise ValueError("dataflow references unpublished assets")
|
|
|
+ return {}, {}
|
|
|
+
|
|
|
+
|
|
|
+def _headers(app, role="editor"):
|
|
|
+ token = issue_access_token(
|
|
|
+ user_id=new_governance_uid(),
|
|
|
+ roles=[role],
|
|
|
+ secret=app.config["SECRET_KEY"],
|
|
|
+ now=datetime.now(UTC),
|
|
|
+ lifetime=timedelta(minutes=10),
|
|
|
+ )
|
|
|
+ return {"Authorization": f"Bearer {token}"}
|
|
|
+
|
|
|
+
|
|
|
+def _use_token_identity(monkeypatch):
|
|
|
+ def load(token, *, secret):
|
|
|
+ claims = decode_access_token(token, secret=secret)
|
|
|
+ return {
|
|
|
+ "id": claims["sub"],
|
|
|
+ "username": "cutover-test",
|
|
|
+ "display_name": "Cutover Test",
|
|
|
+ "roles": claims["roles"],
|
|
|
+ }
|
|
|
+
|
|
|
+ monkeypatch.setattr("app.core.system.auth.load_identity_from_token", load)
|
|
|
+
|
|
|
+
|
|
|
+def test_production_line_draft_identity_is_server_owned_closed_and_governed(
|
|
|
+ monkeypatch,
|
|
|
+):
|
|
|
+ from app import create_app
|
|
|
+
|
|
|
+ app = create_app()
|
|
|
+ app.config["TESTING"] = True
|
|
|
+ _use_token_identity(monkeypatch)
|
|
|
+ client = app.test_client()
|
|
|
+
|
|
|
+ response = client.post(
|
|
|
+ "/api/rules/production-lines/draft-identity",
|
|
|
+ json={},
|
|
|
+ headers=_headers(app),
|
|
|
+ )
|
|
|
+
|
|
|
+ assert response.status_code == 201
|
|
|
+ value = response.get_json()["data"]["dataflow_uid"]
|
|
|
+ parsed = uuid.UUID(value)
|
|
|
+ assert parsed.version == 7
|
|
|
+ assert parsed.variant == uuid.RFC_4122
|
|
|
+ assert (
|
|
|
+ client.post(
|
|
|
+ "/api/rules/production-lines/draft-identity",
|
|
|
+ json={"dataflow_uid": value},
|
|
|
+ headers=_headers(app),
|
|
|
+ ).status_code
|
|
|
+ == 400
|
|
|
+ )
|
|
|
+ assert (
|
|
|
+ client.post(
|
|
|
+ "/api/rules/production-lines/draft-identity",
|
|
|
+ json={},
|
|
|
+ headers=_headers(app, "viewer"),
|
|
|
+ ).status_code
|
|
|
+ == 403
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+def test_legacy_standard_code_generation_is_closed_and_code_cannot_be_written(
|
|
|
+ monkeypatch,
|
|
|
+):
|
|
|
+ from app import create_app
|
|
|
+
|
|
|
+ app = create_app()
|
|
|
+ app.config["TESTING"] = True
|
|
|
+ _use_token_identity(monkeypatch)
|
|
|
+ client = app.test_client()
|
|
|
+
|
|
|
+ monkeypatch.setattr(
|
|
|
+ "app.api.data_interface.routes.create_or_get_node",
|
|
|
+ lambda *_args, **_kwargs: pytest.fail("legacy code was persisted"),
|
|
|
+ )
|
|
|
+
|
|
|
+ generated = client.post(
|
|
|
+ "/api/interface/data/standard/code",
|
|
|
+ json={"input": [], "describe": "生成代码", "output": []},
|
|
|
+ headers=_headers(app),
|
|
|
+ )
|
|
|
+ assert generated.status_code == 410
|
|
|
+ assert generated.get_json()["data"]["semantics"] == "read_only_migration"
|
|
|
+
|
|
|
+ added = client.post(
|
|
|
+ "/api/interface/data/standard/add",
|
|
|
+ json={
|
|
|
+ "name_zh": "旧代码标准",
|
|
|
+ "tag": [],
|
|
|
+ "code": "print('must not persist')",
|
|
|
+ },
|
|
|
+ headers=_headers(app),
|
|
|
+ )
|
|
|
+ assert added.status_code == 400
|
|
|
+ updated = client.post(
|
|
|
+ "/api/interface/data/standard/update",
|
|
|
+ json={
|
|
|
+ "name_zh": "旧代码标准",
|
|
|
+ "tag": [],
|
|
|
+ "code": "print('must not persist')",
|
|
|
+ },
|
|
|
+ headers=_headers(app),
|
|
|
+ )
|
|
|
+ assert updated.status_code == 400
|
|
|
+
|
|
|
+
|
|
|
+def test_governed_legacy_standard_link_is_attested_server_side(monkeypatch):
|
|
|
+ from app import create_app
|
|
|
+
|
|
|
+ app = create_app()
|
|
|
+ app.config["TESTING"] = True
|
|
|
+ _use_token_identity(monkeypatch)
|
|
|
+ repository = PublishedAssetRepository()
|
|
|
+ app.extensions["data_rule_repository"] = repository
|
|
|
+ captured = {}
|
|
|
+
|
|
|
+ monkeypatch.setattr(
|
|
|
+ "app.api.data_interface.routes.translate_and_parse",
|
|
|
+ lambda _value: ["published_standard"],
|
|
|
+ )
|
|
|
+ monkeypatch.setattr(
|
|
|
+ "app.api.data_interface.routes.create_or_get_node",
|
|
|
+ lambda _label, **properties: captured.update(properties) or 17,
|
|
|
+ )
|
|
|
+ client = app.test_client()
|
|
|
+ rule_version_id = new_governance_uid()
|
|
|
+
|
|
|
+ response = client.post(
|
|
|
+ "/api/interface/data/standard/add",
|
|
|
+ json={
|
|
|
+ "name_zh": "已治理标准",
|
|
|
+ "tag": [],
|
|
|
+ "rule_version_id": rule_version_id,
|
|
|
+ "rule_version_status": "draft",
|
|
|
+ },
|
|
|
+ headers=_headers(app),
|
|
|
+ )
|
|
|
+
|
|
|
+ assert response.status_code == 200
|
|
|
+ assert repository.rule_calls == [rule_version_id]
|
|
|
+ assert captured["rule_version_id"] == rule_version_id
|
|
|
+ assert "rule_version_status" not in captured
|
|
|
+
|
|
|
+ repository.published = False
|
|
|
+ rejected = client.post(
|
|
|
+ "/api/interface/data/standard/add",
|
|
|
+ json={
|
|
|
+ "name_zh": "伪造发布状态",
|
|
|
+ "tag": [],
|
|
|
+ "rule_version_id": new_governance_uid(),
|
|
|
+ "rule_version_status": "published",
|
|
|
+ },
|
|
|
+ headers=_headers(app),
|
|
|
+ )
|
|
|
+ assert rejected.status_code == 400
|
|
|
+
|
|
|
+
|
|
|
+def test_governed_dataflow_envelope_is_closed_and_uses_published_assets():
|
|
|
+ from app.core.data_flow.dataflows import DataFlowService
|
|
|
+
|
|
|
+ repository = PublishedAssetRepository()
|
|
|
+ flow = valid_dataflow_spec()
|
|
|
+ envelope = {
|
|
|
+ "dataflow_spec": flow,
|
|
|
+ "dataset_edges": {
|
|
|
+ "source_table": list(flow["input_schema_refs"]),
|
|
|
+ "target_table": flow["output_schema_ref"],
|
|
|
+ },
|
|
|
+ "migration_metadata": {
|
|
|
+ "status": "migrated",
|
|
|
+ "legacy_fields_present": False,
|
|
|
+ "preserved_for_read_only": True,
|
|
|
+ "governed_semantics": "dataflow_spec",
|
|
|
+ },
|
|
|
+ }
|
|
|
+
|
|
|
+ normalized = DataFlowService.validate_governed_requirement(
|
|
|
+ envelope, repository=repository
|
|
|
+ )
|
|
|
+
|
|
|
+ assert normalized["dataflow_spec"]["dataflow_uid"] == flow["dataflow_uid"]
|
|
|
+ assert repository.dataflow_calls == [normalized["dataflow_spec"]]
|
|
|
+
|
|
|
+ invalid = dict(envelope)
|
|
|
+ invalid["task_list"] = []
|
|
|
+ with pytest.raises(ValueError, match="unsupported fields"):
|
|
|
+ DataFlowService.validate_governed_requirement(
|
|
|
+ invalid, repository=repository
|
|
|
+ )
|
|
|
+
|
|
|
+ mismatched = json.loads(json.dumps(envelope))
|
|
|
+ mismatched["dataset_edges"]["target_table"] = "bd:other:v1"
|
|
|
+ with pytest.raises(ValueError, match="dataset edges"):
|
|
|
+ DataFlowService.validate_governed_requirement(
|
|
|
+ mismatched, repository=repository
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+def test_governed_dataflow_creation_never_generates_legacy_task_or_workflow(
|
|
|
+ monkeypatch,
|
|
|
+):
|
|
|
+ from app.core.data_flow.dataflows import DataFlowService
|
|
|
+
|
|
|
+ repository = PublishedAssetRepository()
|
|
|
+ flow = valid_dataflow_spec()
|
|
|
+ data = {
|
|
|
+ "name_zh": "客户治理生产线",
|
|
|
+ "describe": "固定发布版本的数据生产线",
|
|
|
+ "script_type": "python",
|
|
|
+ "script_requirement": {
|
|
|
+ "dataflow_spec": flow,
|
|
|
+ "dataset_edges": {
|
|
|
+ "source_table": list(flow["input_schema_refs"]),
|
|
|
+ "target_table": flow["output_schema_ref"],
|
|
|
+ },
|
|
|
+ "migration_metadata": {
|
|
|
+ "status": "migrated",
|
|
|
+ "legacy_fields_present": False,
|
|
|
+ "preserved_for_read_only": True,
|
|
|
+ "governed_semantics": "dataflow_spec",
|
|
|
+ },
|
|
|
+ },
|
|
|
+ }
|
|
|
+ created = {}
|
|
|
+
|
|
|
+ class Result:
|
|
|
+ def single(self):
|
|
|
+ return None
|
|
|
+
|
|
|
+ class Session:
|
|
|
+ def run(self, *_args, **_kwargs):
|
|
|
+ return Result()
|
|
|
+
|
|
|
+ def __enter__(self):
|
|
|
+ return self
|
|
|
+
|
|
|
+ def __exit__(self, *_args):
|
|
|
+ return None
|
|
|
+
|
|
|
+ class Driver:
|
|
|
+ def session(self):
|
|
|
+ return Session()
|
|
|
+
|
|
|
+ monkeypatch.setattr(
|
|
|
+ "app.core.data_flow.dataflows.translate_and_parse",
|
|
|
+ lambda _name: ["customer_governed_line"],
|
|
|
+ )
|
|
|
+ monkeypatch.setattr(
|
|
|
+ "app.core.data_flow.dataflows.get_node", lambda *_args, **_kwargs: None
|
|
|
+ )
|
|
|
+ monkeypatch.setattr(
|
|
|
+ "app.core.data_flow.dataflows.create_or_get_node",
|
|
|
+ lambda _label, **properties: created.update(properties) or 31,
|
|
|
+ )
|
|
|
+ monkeypatch.setattr(
|
|
|
+ "app.core.data_flow.dataflows.connect_graph", lambda: Driver()
|
|
|
+ )
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService,
|
|
|
+ "_save_to_pg_database",
|
|
|
+ lambda *_args, **_kwargs: pytest.fail("task_list write was invoked"),
|
|
|
+ )
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService,
|
|
|
+ "_handle_script_relationships",
|
|
|
+ lambda *_args, **_kwargs: pytest.fail("legacy script path was invoked"),
|
|
|
+ )
|
|
|
+ monkeypatch.setattr(
|
|
|
+ DataFlowService,
|
|
|
+ "_register_data_product",
|
|
|
+ lambda *_args, **_kwargs: None,
|
|
|
+ )
|
|
|
+
|
|
|
+ result = DataFlowService.create_dataflow(data, repository=repository)
|
|
|
+
|
|
|
+ assert result["id"] == 31
|
|
|
+ assert created["uid"] == flow["dataflow_uid"]
|
|
|
+ assert json.loads(created["script_requirement"])["dataflow_spec"] == flow
|
|
|
+
|
|
|
+
|
|
|
+def test_governed_dataflow_update_preserves_identity_and_closed_envelope(
|
|
|
+ monkeypatch,
|
|
|
+):
|
|
|
+ from app.core.data_flow.dataflows import DataFlowService
|
|
|
+
|
|
|
+ repository = PublishedAssetRepository()
|
|
|
+ flow = valid_dataflow_spec()
|
|
|
+ envelope = {
|
|
|
+ "dataflow_spec": flow,
|
|
|
+ "dataset_edges": {
|
|
|
+ "source_table": list(flow["input_schema_refs"]),
|
|
|
+ "target_table": flow["output_schema_ref"],
|
|
|
+ },
|
|
|
+ "migration_metadata": {
|
|
|
+ "status": "migrated",
|
|
|
+ "legacy_fields_present": False,
|
|
|
+ "preserved_for_read_only": True,
|
|
|
+ "governed_semantics": "dataflow_spec",
|
|
|
+ },
|
|
|
+ }
|
|
|
+ updated = {}
|
|
|
+
|
|
|
+ class Result:
|
|
|
+ def __init__(self, data=None, single=None):
|
|
|
+ self._data = data or []
|
|
|
+ self._single = single
|
|
|
+
|
|
|
+ def data(self):
|
|
|
+ return self._data
|
|
|
+
|
|
|
+ def single(self):
|
|
|
+ return self._single
|
|
|
+
|
|
|
+ class Session:
|
|
|
+ def run(self, query, params=None, **kwargs):
|
|
|
+ values = params or kwargs
|
|
|
+ if "RETURN n" in query and "SET " not in query:
|
|
|
+ return Result(data=[{"n": {"uid": flow["dataflow_uid"]}}])
|
|
|
+ if "SET " in query:
|
|
|
+ updated.update(values)
|
|
|
+ return Result(
|
|
|
+ data=[
|
|
|
+ {
|
|
|
+ "n": {
|
|
|
+ "uid": values["uid"],
|
|
|
+ "script_type": values["script_type"],
|
|
|
+ "script_requirement": values[
|
|
|
+ "script_requirement"
|
|
|
+ ],
|
|
|
+ },
|
|
|
+ "node_id": 44,
|
|
|
+ }
|
|
|
+ ]
|
|
|
+ )
|
|
|
+ return Result(single={"tags": []})
|
|
|
+
|
|
|
+ def __enter__(self):
|
|
|
+ return self
|
|
|
+
|
|
|
+ def __exit__(self, *_args):
|
|
|
+ return None
|
|
|
+
|
|
|
+ class Driver:
|
|
|
+ def session(self):
|
|
|
+ return Session()
|
|
|
+
|
|
|
+ monkeypatch.setattr(
|
|
|
+ "app.core.data_flow.dataflows.connect_graph", lambda: Driver()
|
|
|
+ )
|
|
|
+
|
|
|
+ result = DataFlowService.update_dataflow(
|
|
|
+ 44,
|
|
|
+ {
|
|
|
+ "script_type": "python",
|
|
|
+ "script_path": "/tmp/forged.py",
|
|
|
+ "script_requirement": envelope,
|
|
|
+ },
|
|
|
+ repository=repository,
|
|
|
+ )
|
|
|
+
|
|
|
+ assert result["id"] == 44
|
|
|
+ assert updated["uid"] == flow["dataflow_uid"]
|
|
|
+ assert updated["script_type"] == "governed"
|
|
|
+ assert updated["script_path"] == ""
|
|
|
+ assert json.loads(updated["script_requirement"]) == envelope
|
|
|
+
|
|
|
+ mismatched = json.loads(json.dumps(envelope))
|
|
|
+ mismatched["dataflow_spec"]["dataflow_uid"] = new_governance_uid()
|
|
|
+ with pytest.raises(ValueError, match="cannot replace"):
|
|
|
+ DataFlowService.update_dataflow(
|
|
|
+ 44,
|
|
|
+ {"script_requirement": mismatched},
|
|
|
+ repository=repository,
|
|
|
+ )
|