merge: integrate ontology workbench for phase one

app/__init__.py

@@ -38,6 +38,7 @@ def create_app():
     from app.api.data_rules import bp as data_rules_bp
     from app.api.data_service import bp as data_service_bp
     from app.api.data_source import bp as data_source_bp
+    from app.api.data_development import bp as data_development_bp
     from app.api.graph import bp as graph_bp
     from app.api.knowledge_base import bp as knowledge_base_bp
     from app.api.meta_data import bp as meta_bp
@@ -49,6 +50,7 @@ def create_app():
     app.register_blueprint(graph_bp, url_prefix="/api/graph")
     app.register_blueprint(system_bp, url_prefix="/api/system")
     app.register_blueprint(data_source_bp, url_prefix="/api/datasource")
+    app.register_blueprint(data_development_bp, url_prefix="/api/development/v1")
     app.register_blueprint(data_flow_bp, url_prefix="/api/dataflow")
     app.register_blueprint(business_domain_bp, url_prefix="/api/bd")
     app.register_blueprint(data_factory_bp, url_prefix="/api/datafactory")

app/api/data_development/__init__.py

@@ -0,0 +1,6 @@
+from flask import Blueprint
+
+
+bp = Blueprint("data_development", __name__)
+
+from app.api.data_development import routes  # noqa: E402, F401

app/api/data_development/routes.py

@@ -0,0 +1,691 @@
+"""HTTP orchestration boundary for data-research ingestion jobs."""
+
+from __future__ import annotations
+
+import io
+import logging
+
+from flask import current_app, g, jsonify, request, send_file
+
+from app import db
+from app.api.data_development import bp
+from app.core.data_research.errors import DataResearchError
+from app.models.result import failed, success
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_ingestion_service():
+    from app.core.data_research.ingestion import IngestionService
+    from app.core.data_research.repository import SqlAlchemyIngestionJobRepository
+
+    return IngestionService(
+        SqlAlchemyIngestionJobRepository(db.session),
+        commit=db.session.commit,
+        rollback=db.session.rollback,
+    )
+
+
+def get_data_element_service():
+    from app.core.data_research.data_elements import DataElementService
+    from app.core.data_research.repository import SqlAlchemyDataElementRepository
+    from app.core.events.outbox import enqueue_outbox
+
+    return DataElementService(
+        SqlAlchemyDataElementRepository(db.session),
+        outbox_enqueue=lambda **event: enqueue_outbox(db.session, **event),
+    )
+
+
+def get_candidate_decision_service():
+    from app.core.data_research.candidate_decisions import CandidateDecisionService
+    from app.core.data_research.data_elements import DataElementService
+    from app.core.data_research.repository import (
+        SqlAlchemyCandidateDecisionRepository,
+        SqlAlchemyDataElementRepository,
+    )
+
+    return CandidateDecisionService(
+        SqlAlchemyCandidateDecisionRepository(db.session),
+        data_elements=DataElementService(SqlAlchemyDataElementRepository(db.session)),
+    )
+
+
+def _artifact_storage():
+    from minio import Minio
+
+    from app.core.data_research.artifacts import MinioArtifactStorage
+
+    client = Minio(
+        current_app.config["MINIO_HOST"],
+        access_key=current_app.config["MINIO_USER"],
+        secret_key=current_app.config["MINIO_PASSWORD"],
+        secure=bool(current_app.config.get("MINIO_SECURE")),
+    )
+    return MinioArtifactStorage(client, current_app.config["MINIO_BUCKET"])
+
+
+def get_artifact_service():
+    from app.core.common.identifiers import new_governance_uid
+    from app.core.data_research.artifacts import (
+        ArtifactService,
+        SqlAlchemyArtifactRepository,
+    )
+    from app.core.data_research.file_policy import FilePolicy
+
+    return ArtifactService(
+        SqlAlchemyArtifactRepository(db.session),
+        _artifact_storage(),
+        FilePolicy(
+            max_bytes=int(current_app.config.get("DATA_RESEARCH_FILE_MAX_BYTES", 25 * 1024 * 1024)),
+            max_pages=int(current_app.config.get("DATA_RESEARCH_FILE_MAX_PAGES", 500)),
+        ),
+        uid_factory=new_governance_uid,
+    )
+
+
+def get_evidence_service():
+    from app.core.data_research.artifacts import EvidenceService
+
+    return EvidenceService(db.session, _artifact_storage())
+
+
+def get_ontology_service():
+    from app.core.data_research.ontology.publication import (
+        OntologyApplicationService,
+        OntologyPublicationService,
+    )
+    from app.core.data_research.ontology.repository import SqlAlchemyOntologyRepository
+    from app.core.events.outbox import enqueue_outbox
+
+    repository = SqlAlchemyOntologyRepository(db.session)
+    publication = OntologyPublicationService(
+        repository,
+        outbox_enqueue=lambda **event: enqueue_outbox(db.session, **event),
+        commit=db.session.commit,
+        rollback=db.session.rollback,
+    )
+    return OntologyApplicationService(
+        repository,
+        publication=publication,
+        commit=db.session.commit,
+        rollback=db.session.rollback,
+    )
+
+
+def get_ontology_dynamic_service():
+    from app.core.data_research.ontology.change_sets import (
+        SqlAlchemyDynamicOntologyService,
+    )
+
+    return SqlAlchemyDynamicOntologyService(db.session)
+
+
+def get_ontology_exchange_service():
+    from app.core.data_research.ontology.exchange import OntologyExchangeService
+    from app.core.data_research.ontology.repository import SqlAlchemyOntologyRepository
+
+    return OntologyExchangeService(
+        SqlAlchemyOntologyRepository(db.session),
+        commit=db.session.commit,
+        rollback=db.session.rollback,
+    )
+
+
+def get_semantic_query_service():
+    from app.core.data_research.ontology.query import (
+        Neo4jSemanticRepository,
+        SemanticQueryService,
+    )
+    from app.services.neo4j_driver import neo4j_driver
+
+    return SemanticQueryService(Neo4jSemanticRepository(neo4j_driver))
+
+
+def _identity():
+    return getattr(g, "current_user", {}) or {}
+
+
+def _record(record):
+    return {
+        "uid": str(record.uid),
+        "source_uid": str(record.source_uid),
+        "artifact_uid": str(record.artifact_uid) if record.artifact_uid else None,
+        "job_type": record.job_type,
+        "parser_version": record.parser_version,
+        "status": record.status,
+        "parameters": dict(record.parameters or {}),
+        "statistics": dict(record.statistics or {}),
+        "last_error": record.last_error,
+        "actor_uid": record.actor_uid,
+        "created_at": record.created_at.isoformat() if record.created_at else None,
+        "updated_at": record.updated_at.isoformat() if record.updated_at else None,
+        "started_at": record.started_at.isoformat() if record.started_at else None,
+        "finished_at": record.finished_at.isoformat() if record.finished_at else None,
+    }
+
+
+def _element(record):
+    return {
+        "uid": str(record.uid),
+        "code": record.code,
+        "status": record.status,
+        "current_version": int(record.current_version),
+        "snapshot": dict(record.snapshot or {}),
+        "created_by": record.created_by,
+        "updated_by": record.updated_by,
+    }
+
+
+def _decision(record):
+    return {
+        "uid": str(record.uid),
+        "candidate_uid": str(record.candidate_uid),
+        "action": record.action,
+        "data_element_uid": (
+            str(record.data_element_uid) if record.data_element_uid else None
+        ),
+        "evidence_uids": list(record.evidence_uids),
+        "actor_uid": record.actor_uid,
+        "reason": record.reason,
+    }
+
+
+def _artifact(record):
+    return {
+        "uid": str(record.uid),
+        "source_uid": str(record.source_uid),
+        "filename": record.filename,
+        "media_type": record.media_type,
+        "size_bytes": int(record.size_bytes),
+        "content_hash": record.content_hash,
+        "parser_version": record.parser_version,
+    }
+
+
+def _ontology(record):
+    return {
+        "uid": str(record.uid),
+        "code": record.code,
+        "name": record.name,
+        "owner_uid": record.owner_uid,
+        "status": record.status,
+        "draft_revision": int(record.draft_revision),
+        "active_version_uid": record.active_version_uid,
+        "domain_links": [
+            {"domain_uid": link.domain_uid, "role": link.role}
+            for link in record.domain_links
+        ],
+    }
+
+
+def _ontology_version(record):
+    return {
+        "uid": str(record.uid),
+        "ontology_uid": str(record.ontology_uid),
+        "version": int(record.version),
+        "parent_version_uid": record.parent_version_uid,
+        "status": record.status,
+        "content_hash": record.content_hash,
+        "graph_document": record.graph_document.to_dict(),
+        "created_by": record.created_by,
+    }
+
+
+def _error(error):
+    if isinstance(error, DataResearchError):
+        return (
+            jsonify(
+                failed(
+                    str(error),
+                    code=error.http_status,
+                    error={"code": error.code},
+                )
+            ),
+            error.http_status,
+        )
+    logger.exception("data-research ingestion request failed")
+    return (
+        jsonify(
+            failed(
+                "数据采集任务处理失败",
+                code=500,
+                error={"code": "DATA_RESEARCH_ERROR"},
+            )
+        ),
+        500,
+    )
+
+
+@bp.route("/ingestion-jobs", methods=["POST"])
+def create_ingestion_job():
+    payload = request.get_json(silent=True) or {}
+    try:
+        record, created = get_ingestion_service().create_job(
+            payload,
+            actor_uid=_identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(_record(record))), 201 if created else 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ingestion-jobs", methods=["GET"])
+def list_ingestion_jobs():
+    filters = {
+        name: request.args.get(name)
+        for name in ("status", "source_uid")
+        if request.args.get(name)
+    }
+    try:
+        records = get_ingestion_service().list_jobs(filters)
+        return jsonify(
+            success({"records": [_record(item) for item in records], "total": len(records)})
+        ), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ingestion-jobs/<job_uid>", methods=["GET"])
+def get_ingestion_job(job_uid):
+    try:
+        return jsonify(success(_record(get_ingestion_service().get_job(job_uid)))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ingestion-jobs/<job_uid>/retry", methods=["POST"])
+def retry_ingestion_job(job_uid):
+    try:
+        return jsonify(success(_record(get_ingestion_service().retry(job_uid)))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ingestion-jobs/<job_uid>/cancel", methods=["POST"])
+def cancel_ingestion_job(job_uid):
+    try:
+        service = get_ingestion_service()
+        record = service.get_job(job_uid)
+        identity = _identity()
+        permissions = set(identity.get("permissions") or [])
+        actor_uid = identity.get("id") or identity.get("sub")
+        if record.actor_uid != actor_uid and "ingestion:admin" not in permissions:
+            return jsonify(failed("权限不足", code=403)), 403
+        return jsonify(success(_record(service.cancel(job_uid)))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/data-elements", methods=["GET"])
+def list_data_elements():
+    try:
+        records = get_data_element_service().list(
+            status=request.args.get("status"),
+            business_domain_uid=request.args.get("business_domain_uid"),
+        )
+        return jsonify(
+            success(
+                {
+                    "records": [_element(item) for item in records],
+                    "total": len(records),
+                }
+            )
+        ), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/data-elements", methods=["POST"])
+def create_data_element():
+    try:
+        record = get_data_element_service().create_draft(
+            request.get_json(silent=True) or {},
+            actor_uid=_identity().get("id") or _identity().get("sub"),
+        )
+        db.session.commit()
+        return jsonify(success(_element(record))), 201
+    except Exception as error:
+        db.session.rollback()
+        return _error(error)
+
+
+@bp.route("/data-elements/<element_uid>/transitions", methods=["POST"])
+def transition_data_element(element_uid):
+    payload = request.get_json(silent=True) or {}
+    identity = _identity()
+    target_status = str(payload.get("target_status") or "")
+    if (
+        target_status in {"published", "deprecated", "retired"}
+        and "data-elements:publish" not in set(identity.get("permissions") or [])
+    ):
+        return jsonify(failed("权限不足", code=403)), 403
+    try:
+        record = get_data_element_service().transition(
+            element_uid,
+            target_status,
+            expected_version=int(payload.get("expected_version")),
+            actor_uid=identity.get("id") or identity.get("sub"),
+        )
+        db.session.commit()
+        return jsonify(success(_element(record))), 200
+    except Exception as error:
+        db.session.rollback()
+        return _error(error)
+
+
+@bp.route("/candidate-decisions", methods=["POST"])
+def decide_candidates():
+    payload = request.get_json(silent=True) or {}
+    try:
+        records = get_candidate_decision_service().decide(
+            payload.get("decisions") or [],
+            actor_uid=_identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success([_decision(record) for record in records])), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/sources/files", methods=["POST"])
+def upload_source_file():
+    uploaded = request.files.get("file")
+    if uploaded is None:
+        return jsonify(failed("缺少上传文件", code=400)), 400
+    try:
+        record, created = get_artifact_service().store(
+            request.form.get("source_uid"),
+            uploaded.filename,
+            uploaded.mimetype,
+            uploaded.read(),
+            request.form.get("parser_version") or "auto-v1",
+        )
+        db.session.commit()
+        return jsonify(success(_artifact(record))), 201 if created else 200
+    except Exception as error:
+        db.session.rollback()
+        return _error(error)
+
+
+@bp.route("/evidence/<evidence_uid>", methods=["GET"])
+def get_evidence(evidence_uid):
+    identity = _identity()
+    try:
+        service = get_evidence_service()
+        if request.args.get("download") in {"1", "true", "yes"}:
+            if "evidence:download" not in set(identity.get("permissions") or []):
+                return jsonify(failed("权限不足", code=403)), 403
+            content, filename, media_type = service.download(evidence_uid)
+            return send_file(
+                io.BytesIO(content),
+                mimetype=media_type,
+                as_attachment=True,
+                download_name=filename,
+            )
+        preview = service.preview(evidence_uid)
+        from app.core.data_research.artifacts import redact_excerpt
+
+        preview["excerpt"] = redact_excerpt(preview.get("excerpt"))
+        return jsonify(success(preview)), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies", methods=["GET"])
+def list_ontologies():
+    try:
+        return jsonify(success([_ontology(item) for item in get_ontology_service().list()])), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies", methods=["POST"])
+def create_ontology():
+    try:
+        record = get_ontology_service().create(
+            request.get_json(silent=True) or {},
+            _identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(_ontology(record))), 201
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>", methods=["GET"])
+def get_ontology(ontology_uid):
+    try:
+        return jsonify(success(_ontology(get_ontology_service().get(ontology_uid)))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/versions", methods=["GET"])
+def list_ontology_versions(ontology_uid):
+    try:
+        return jsonify(
+            success(
+                [
+                    _ontology_version(item)
+                    for item in get_ontology_service().list_versions(ontology_uid)
+                ]
+            )
+        ), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/graph", methods=["GET"])
+def get_ontology_graph(ontology_uid):
+    try:
+        service = get_ontology_service()
+        ontology = service.get(ontology_uid)
+        version = service.latest_version(ontology_uid)
+        data = (
+            _ontology_version(version)
+            if version is not None
+            else {
+                "uid": None,
+                "ontology_uid": str(ontology_uid),
+                "version": 0,
+                "parent_version_uid": None,
+                "status": "draft",
+                "content_hash": None,
+                "graph_document": {
+                    name: []
+                    for name in (
+                        "classes",
+                        "properties",
+                        "relations",
+                        "constraints",
+                        "domain_links",
+                        "element_mappings",
+                    )
+                },
+                "created_by": None,
+            }
+        )
+        response = jsonify(success(data))
+        response.headers["ETag"] = f'"{int(ontology.draft_revision)}"'
+        return response, 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/graph", methods=["PATCH"])
+def save_ontology_graph(ontology_uid):
+    etag = str(request.headers.get("If-Match") or "").strip().strip('"')
+    if not etag.isdigit():
+        return jsonify(failed("缺少有效 If-Match 修订号", code=428)), 428
+    try:
+        version = get_ontology_service().save_draft(
+            ontology_uid,
+            request.get_json(silent=True) or {},
+            int(etag),
+            _identity().get("id") or _identity().get("sub"),
+        )
+        response = jsonify(success(_ontology_version(version)))
+        response.headers["ETag"] = f'"{int(etag) + 1}"'
+        return response, 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/validate", methods=["POST"])
+def validate_ontology(ontology_uid):
+    try:
+        issues = get_ontology_service().validate(ontology_uid)
+        data = [
+            {"code": item.code, "message": item.message, "path": item.path}
+            for item in issues
+        ]
+        return jsonify(success(data)), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/publish", methods=["POST"])
+def publish_ontology(ontology_uid):
+    try:
+        version = get_ontology_service().publish(
+            ontology_uid,
+            request.headers.get("Idempotency-Key") or f"{ontology_uid}:publish",
+            _identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(_ontology_version(version))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/diff", methods=["GET"])
+def diff_ontology(ontology_uid):
+    try:
+        return jsonify(success(get_ontology_service().diff(
+            ontology_uid, request.args.get("left"), request.args.get("right")
+        ))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/rollback", methods=["POST"])
+def rollback_ontology(ontology_uid):
+    payload = request.get_json(silent=True) or {}
+    try:
+        version = get_ontology_service().rollback(
+            ontology_uid,
+            payload.get("target_version_uid"),
+            int(payload.get("expected_revision")),
+            _identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(_ontology_version(version))), 201
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/suggestions", methods=["POST"])
+def generate_ontology_suggestions(ontology_uid):
+    try:
+        result = get_ontology_dynamic_service().generate(
+            ontology_uid,
+            request.get_json(silent=True) or {},
+            _identity().get("id") or _identity().get("sub"),
+        )
+        records = (
+            result.get("suggestions") or []
+            if isinstance(result, dict)
+            else result
+        )
+        suggestions = [
+            item
+            if isinstance(item, dict)
+            else {
+                "uid": item.uid,
+                "kind": item.kind,
+                "payload": item.payload,
+                "evidence_uids": list(item.evidence_uids),
+                "confidence": item.confidence,
+                "source": item.source,
+                "model_version": item.model_version,
+                "prompt_version": item.prompt_version,
+            }
+            for item in records
+        ]
+        data = {
+            "change_set_uid": (
+                result.get("change_set_uid")
+                if isinstance(result, dict)
+                else None
+            ),
+            "suggestions": suggestions,
+        }
+        return jsonify(success(data)), 201
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route(
+    "/ontologies/<ontology_uid>/change-sets/<change_set_uid>/decisions",
+    methods=["POST"],
+)
+def decide_ontology_change_set(ontology_uid, change_set_uid):
+    payload = request.get_json(silent=True) or {}
+    try:
+        result = get_ontology_dynamic_service().decide(
+            ontology_uid,
+            change_set_uid,
+            payload.get("decisions") or [],
+            _identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(result)), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/export", methods=["GET"])
+def export_ontology(ontology_uid):
+    try:
+        content, media_type, filename = get_ontology_exchange_service().export(
+            ontology_uid, request.args.get("format") or "json"
+        )
+        return send_file(
+            io.BytesIO(content),
+            mimetype=media_type,
+            as_attachment=True,
+            download_name=filename,
+        )
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/import", methods=["POST"])
+def import_ontology():
+    try:
+        uploaded = request.files.get("file")
+        content = uploaded.read() if uploaded is not None else request.get_data(cache=False)
+        result = get_ontology_exchange_service().import_document(
+            content,
+            request.args.get("format") or "json",
+            _identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(result)), 201
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/semantic/properties/<property_uid>", methods=["GET"])
+def query_semantic_property(property_uid):
+    domain = str(request.args.get("business_domain_uid") or "").strip()
+    identity = _identity()
+    scoped = set(identity.get("business_domains") or [domain])
+    if "*" not in scoped and domain not in scoped:
+        return jsonify(failed("权限不足", code=403)), 403
+    try:
+        result = get_semantic_query_service().trace_property(
+            property_uid,
+            allowed_domains={domain},
+            limit=int(request.args.get("limit") or 20),
+            after_uid=request.args.get("after_uid"),
+        )
+        return jsonify(success(result)), 200
+    except Exception as error:
+        return _error(error)

app/commands/reconcile_data_research.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import argparse
+import json
+
+from app import create_app
+
+
+def run(reconciler, *, repair=False):
+    return reconciler.reconcile(repair=repair)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Reconcile data-research control and projection stores")
+    parser.add_argument("--repair", action="store_true", help="repair rebuildable projections")
+    args = parser.parse_args()
+    app = create_app()
+    with app.app_context():
+        factory = app.extensions.get("data_research_reconciler_factory")
+        if factory is None:
+            raise RuntimeError("data-research reconciliation adapters are not configured")
+        print(json.dumps(run(factory(), repair=args.repair), ensure_ascii=False, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

app/config/config.py

@@ -388,6 +388,15 @@ class BaseConfig:
         "docx",
         "doc",
     }
+    DATA_RESEARCH_FILE_MAX_BYTES = int(
+        os.environ.get("DATA_RESEARCH_FILE_MAX_BYTES", str(25 * 1024 * 1024))
+    )
+    DATA_RESEARCH_FILE_MAX_PAGES = int(
+        os.environ.get("DATA_RESEARCH_FILE_MAX_PAGES", "500")
+    )
+    DATA_RESEARCH_MAX_CONCURRENT_JOBS = int(
+        os.environ.get("DATA_RESEARCH_MAX_CONCURRENT_JOBS", "10")
+    )
 
     # 数据抽取配置
     DATA_EXTRACT_BATCH_SIZE = 1000  # 每批处理的记录数

app/core/data_research/__init__.py

@@ -0,0 +1,5 @@
+"""Data-research ingestion, governance, and ontology services."""
+
+from app.core.data_research.ingestion import IngestionService
+
+__all__ = ["IngestionService"]

app/core/data_research/artifacts.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import hashlib
+import io
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ArtifactRecord:
+    uid: str
+    source_uid: str
+    filename: str
+    media_type: str
+    size_bytes: int
+    content_hash: str
+    storage_ref: str
+    parser_version: str
+
+
+class ArtifactService:
+    def __init__(self, repository, storage, file_policy, *, uid_factory):
+        self.repository = repository
+        self.storage = storage
+        self.file_policy = file_policy
+        self.uid_factory = uid_factory
+
+    def store(self, source_uid, filename, media_type, content, parser_version):
+        validated = self.file_policy.validate(filename, media_type, content)
+        digest = hashlib.sha256(content).hexdigest()
+        existing = self.repository.find(source_uid, digest, parser_version)
+        if existing is not None:
+            return existing, False
+        object_key = f"data-research/{source_uid}/{digest}"
+        storage_ref = self.storage.put(object_key, content, validated.media_type)
+        record = ArtifactRecord(
+            uid=self.uid_factory(),
+            source_uid=str(source_uid),
+            filename=validated.filename,
+            media_type=validated.media_type,
+            size_bytes=validated.size_bytes,
+            content_hash=digest,
+            storage_ref=storage_ref,
+            parser_version=str(parser_version),
+        )
+        return self.repository.save(record), True
+
+
+def redact_excerpt(value):
+    value = re.sub(r"(?<!\d)(1\d{2})\d{4}(\d{4})(?!\d)", r"\1****\2", str(value or ""))
+    value = re.sub(
+        r"(?i)\b(token|password|secret|api[_-]?key)\s*[=:]\s*[^\s,;]+",
+        lambda match: f"{match.group(1)}=[redacted]",
+        value,
+    )
+    return value
+
+
+class EvidenceService:
+    def __init__(self, session, storage):
+        self.session = session
+        self.storage = storage
+
+    def preview(self, uid):
+        from app.models.data_research import EvidenceFragment
+
+        model = self.session.get(EvidenceFragment, str(uid))
+        if model is None:
+            raise LookupError("evidence was not found")
+        data = model.to_dict()
+        data["excerpt"] = redact_excerpt(data.get("excerpt"))
+        return data
+
+    def download(self, uid):
+        from app.models.data_research import EvidenceFragment, SourceArtifact
+
+        evidence = self.session.get(EvidenceFragment, str(uid))
+        artifact = (
+            self.session.get(SourceArtifact, str(evidence.artifact_uid))
+            if evidence is not None and evidence.artifact_uid
+            else None
+        )
+        if artifact is None:
+            raise LookupError("evidence artifact was not found")
+        return self.storage.get(artifact.storage_ref), artifact.filename, artifact.media_type
+
+
+class SqlAlchemyArtifactRepository:
+    def __init__(self, session):
+        self.session = session
+
+    @staticmethod
+    def _record(model):
+        return ArtifactRecord(
+            uid=str(model.uid),
+            source_uid=str(model.source_uid),
+            filename=model.filename,
+            media_type=model.media_type,
+            size_bytes=int(model.size_bytes),
+            content_hash=model.content_hash,
+            storage_ref=model.storage_ref,
+            parser_version=model.parser_version,
+        )
+
+    def find(self, source_uid, content_hash, parser_version):
+        from app.models.data_research import SourceArtifact
+
+        model = self.session.query(SourceArtifact).filter_by(
+            source_uid=str(source_uid),
+            content_hash=content_hash,
+            parser_version=str(parser_version),
+        ).first()
+        return self._record(model) if model is not None else None
+
+    def save(self, record):
+        from app.models.data_research import SourceArtifact
+
+        model = SourceArtifact(**record.__dict__)
+        self.session.add(model)
+        self.session.flush()
+        return self._record(model)
+
+
+class MinioArtifactStorage:
+    def __init__(self, client, bucket):
+        self.client = client
+        self.bucket = bucket
+
+    def put(self, object_key, content, media_type):
+        self.client.put_object(
+            self.bucket,
+            object_key,
+            io.BytesIO(content),
+            len(content),
+            content_type=media_type,
+        )
+        return f"minio://{self.bucket}/{object_key}"
+
+    def get(self, storage_ref):
+        prefix = f"minio://{self.bucket}/"
+        if not str(storage_ref).startswith(prefix):
+            raise ValueError("invalid storage reference")
+        response = self.client.get_object(self.bucket, str(storage_ref)[len(prefix):])
+        try:
+            return response.read()
+        finally:
+            response.close()
+            response.release_conn()

app/core/data_research/candidate_decisions.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from app.core.common.identifiers import new_governance_uid
+from app.core.data_research.errors import CandidateDecisionInvalid
+
+
+ALLOWED_ACTIONS = frozenset({"reuse", "create", "map", "ignore"})
+
+
+@dataclass(frozen=True)
+class CandidateDecision:
+    uid: str
+    candidate_uid: str
+    action: str
+    data_element_uid: str | None
+    evidence_uids: tuple[str, ...]
+    actor_uid: str | None
+    reason: str | None = None
+
+
+class CandidateDecisionService:
+    def __init__(
+        self,
+        repository,
+        *,
+        data_elements,
+        uid_factory: Callable[[], str] = new_governance_uid,
+    ):
+        self.repository = repository
+        self.data_elements = data_elements
+        self.uid_factory = uid_factory
+
+    def _validate(self, item):
+        if not isinstance(item, dict):
+            raise CandidateDecisionInvalid("decision must be an object")
+        action = str(item.get("action") or "").strip()
+        if action not in ALLOWED_ACTIONS:
+            raise CandidateDecisionInvalid(f"unsupported action: {action}")
+        candidate_uid = str(item.get("candidate_uid") or "").strip()
+        candidate = self.repository.get(candidate_uid)
+        if candidate is None:
+            raise CandidateDecisionInvalid(f"candidate {candidate_uid} was not found")
+        data_element_uid = str(item.get("data_element_uid") or "").strip() or None
+        if action in {"reuse", "map"} and not data_element_uid:
+            raise CandidateDecisionInvalid("data_element_uid is required")
+        return item, candidate, action, data_element_uid
+
+    def decide(self, items, *, actor_uid):
+        try:
+            validated = [self._validate(item) for item in items]
+            decisions = []
+            for item, candidate, action, data_element_uid in validated:
+                evidence_uids = tuple(candidate.get("evidence_uids") or ())
+                if action == "create":
+                    payload = dict(candidate.get("normalized_data") or {})
+                    payload["evidence_uids"] = list(evidence_uids)
+                    element = self.data_elements.create_draft(payload, actor_uid)
+                    data_element_uid = str(element.uid)
+                decisions.append(
+                    CandidateDecision(
+                        uid=self.uid_factory(),
+                        candidate_uid=str(candidate["uid"]),
+                        action=action,
+                        data_element_uid=data_element_uid,
+                        evidence_uids=evidence_uids,
+                        actor_uid=actor_uid,
+                        reason=str(item.get("reason") or "").strip() or None,
+                    )
+                )
+            self.repository.save_decisions(decisions)
+            self.repository.commit()
+            return decisions
+        except Exception:
+            self.repository.rollback()
+            raise

app/core/data_research/catalog/__init__.py

@@ -0,0 +1,6 @@
+from app.core.data_research.catalog.service import (
+    CatalogCollectionService,
+    diff_snapshots,
+)
+
+__all__ = ["CatalogCollectionService", "diff_snapshots"]

app/core/data_research/catalog/base.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from collections import OrderedDict
+
+from app.core.data_research.catalog.models import (
+    CatalogAsset,
+    CatalogField,
+    CatalogSnapshot,
+)
+
+
+class BaseCatalogCollector:
+    database_type = ""
+
+    @staticmethod
+    def _asset_type(value: str) -> str:
+        return "view" if "VIEW" in str(value).upper() else "table"
+
+    def _snapshot(self, rows, data_source_uid: str):
+        grouped = OrderedDict()
+        for row in rows:
+            schema = str(row["schema_name"])
+            asset_name = str(row["asset_name"])
+            asset_key = f"{data_source_uid}:{schema}.{asset_name}"
+            entry = grouped.setdefault(
+                asset_key,
+                {
+                    "schema": schema,
+                    "name": asset_name,
+                    "asset_type": self._asset_type(row["asset_type"]),
+                    "comment": row.get("asset_comment"),
+                    "fields": [],
+                },
+            )
+            field_name = str(row["column_name"])
+            entry["fields"].append(
+                CatalogField(
+                    key=f"{asset_key}.{field_name}",
+                    schema=schema,
+                    asset=asset_name,
+                    name=field_name,
+                    ordinal_position=int(row["ordinal_position"]),
+                    data_type=str(row["data_type"]),
+                    nullable=str(row["is_nullable"]).upper() == "YES",
+                    default=row.get("column_default"),
+                    comment=row.get("column_comment"),
+                )
+            )
+        assets = tuple(
+            CatalogAsset(
+                key=key,
+                schema=value["schema"],
+                name=value["name"],
+                asset_type=value["asset_type"],
+                comment=value["comment"],
+                fields=tuple(
+                    sorted(value["fields"], key=lambda item: item.ordinal_position)
+                ),
+            )
+            for key, value in grouped.items()
+        )
+        return CatalogSnapshot(
+            data_source_uid=str(data_source_uid),
+            database_type=self.database_type,
+            assets=assets,
+        )

app/core/data_research/catalog/models.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class CatalogScope:
+    include_schemas: tuple[str, ...] = ()
+    exclude_schemas: tuple[str, ...] = ()
+    include_tables: tuple[str, ...] = ()
+    exclude_tables: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class CatalogField:
+    key: str
+    schema: str
+    asset: str
+    name: str
+    ordinal_position: int
+    data_type: str
+    nullable: bool
+    default: str | None = None
+    comment: str | None = None
+
+
+@dataclass(frozen=True)
+class CatalogAsset:
+    key: str
+    schema: str
+    name: str
+    asset_type: str
+    fields: tuple[CatalogField, ...]
+    comment: str | None = None
+
+
+@dataclass(frozen=True)
+class CatalogSnapshot:
+    data_source_uid: str
+    database_type: str
+    assets: tuple[CatalogAsset, ...]
+
+    @property
+    def content_hash(self) -> str:
+        payload = json.dumps(
+            asdict(self),
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        )
+        return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+@dataclass(frozen=True)
+class CatalogFieldChange:
+    before: CatalogField
+    after: CatalogField
+
+
+@dataclass(frozen=True)
+class CatalogDiff:
+    added: tuple[CatalogField, ...] = ()
+    removed: tuple[CatalogField, ...] = ()
+    changed: tuple[CatalogFieldChange, ...] = ()
+    renamed: tuple[CatalogFieldChange, ...] = ()

app/core/data_research/catalog/mysql.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from sqlalchemy import bindparam, text
+
+from app.core.data_research.catalog.base import BaseCatalogCollector
+
+
+MYSQL_CATALOG_SQL = """
+SELECT
+    c.TABLE_SCHEMA AS schema_name,
+    c.TABLE_NAME AS asset_name,
+    t.TABLE_TYPE AS asset_type,
+    t.TABLE_COMMENT AS asset_comment,
+    c.COLUMN_NAME AS column_name,
+    c.ORDINAL_POSITION AS ordinal_position,
+    c.COLUMN_TYPE AS data_type,
+    c.IS_NULLABLE AS is_nullable,
+    c.COLUMN_DEFAULT AS column_default,
+    c.COLUMN_COMMENT AS column_comment
+FROM information_schema.COLUMNS c
+JOIN information_schema.TABLES t
+  ON t.TABLE_SCHEMA = c.TABLE_SCHEMA AND t.TABLE_NAME = c.TABLE_NAME
+WHERE c.TABLE_SCHEMA = :database_name
+  AND (:include_table_count = 0 OR c.TABLE_NAME IN :include_tables)
+  AND (:exclude_table_count = 0 OR c.TABLE_NAME NOT IN :exclude_tables)
+ORDER BY c.TABLE_NAME, c.ORDINAL_POSITION
+"""
+
+
+class MySqlCatalogCollector(BaseCatalogCollector):
+    database_type = "mysql"
+
+    def collect(self, connection, *, data_source_uid, database_name, scope):
+        statement = text(MYSQL_CATALOG_SQL).bindparams(
+            bindparam("include_tables", expanding=True),
+            bindparam("exclude_tables", expanding=True),
+        )
+        parameters = {
+            "database_name": str(database_name),
+            "include_tables": list(scope.include_tables),
+            "exclude_tables": list(scope.exclude_tables),
+            "include_table_count": len(scope.include_tables),
+            "exclude_table_count": len(scope.exclude_tables),
+        }
+        rows = connection.execute(statement, parameters).mappings().all()
+        return self._snapshot(rows, str(data_source_uid))

app/core/data_research/catalog/postgresql.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from sqlalchemy import bindparam, text
+
+from app.core.data_research.catalog.base import BaseCatalogCollector
+
+
+POSTGRESQL_CATALOG_SQL = """
+SELECT
+    c.table_schema AS schema_name,
+    c.table_name AS asset_name,
+    t.table_type AS asset_type,
+    obj_description(pc.oid) AS asset_comment,
+    c.column_name,
+    c.ordinal_position,
+    c.data_type,
+    c.is_nullable,
+    c.column_default,
+    col_description(pc.oid, c.ordinal_position) AS column_comment
+FROM information_schema.columns c
+JOIN information_schema.tables t
+  ON t.table_schema = c.table_schema AND t.table_name = c.table_name
+LEFT JOIN pg_catalog.pg_namespace pn ON pn.nspname = c.table_schema
+LEFT JOIN pg_catalog.pg_class pc
+  ON pc.relnamespace = pn.oid AND pc.relname = c.table_name
+WHERE c.table_schema NOT IN ('pg_catalog', 'information_schema')
+  AND (:include_schema_count = 0 OR c.table_schema IN :include_schemas)
+  AND (:exclude_schema_count = 0 OR c.table_schema NOT IN :exclude_schemas)
+  AND (:include_table_count = 0 OR c.table_name IN :include_tables)
+  AND (:exclude_table_count = 0 OR c.table_name NOT IN :exclude_tables)
+ORDER BY c.table_schema, c.table_name, c.ordinal_position
+"""
+
+
+class PostgreSqlCatalogCollector(BaseCatalogCollector):
+    database_type = "postgresql"
+
+    def collect(self, connection, *, data_source_uid, scope):
+        statement = text(POSTGRESQL_CATALOG_SQL).bindparams(
+            bindparam("include_schemas", expanding=True),
+            bindparam("exclude_schemas", expanding=True),
+            bindparam("include_tables", expanding=True),
+            bindparam("exclude_tables", expanding=True),
+        )
+        parameters = {
+            "include_schemas": list(scope.include_schemas),
+            "exclude_schemas": list(scope.exclude_schemas),
+            "include_tables": list(scope.include_tables),
+            "exclude_tables": list(scope.exclude_tables),
+            "include_schema_count": len(scope.include_schemas),
+            "exclude_schema_count": len(scope.exclude_schemas),
+            "include_table_count": len(scope.include_tables),
+            "exclude_table_count": len(scope.exclude_tables),
+        }
+        rows = connection.execute(statement, parameters).mappings().all()
+        return self._snapshot(rows, str(data_source_uid))

app/core/data_research/catalog/service.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from app.core.data_research.catalog.models import (
+    CatalogDiff,
+    CatalogFieldChange,
+)
+
+
+def _fields(snapshot):
+    return {
+        field.key: field
+        for asset in snapshot.assets
+        for field in asset.fields
+    }
+
+
+def diff_snapshots(previous, current):
+    before = _fields(previous)
+    after = _fields(current)
+    changed = []
+    for key in sorted(set(before) & set(after)):
+        left = before[key]
+        right = after[key]
+        if (
+            left.data_type,
+            left.nullable,
+            left.default,
+            left.comment,
+        ) != (
+            right.data_type,
+            right.nullable,
+            right.default,
+            right.comment,
+        ):
+            changed.append(CatalogFieldChange(left, right))
+
+    removed = [before[key] for key in sorted(set(before) - set(after))]
+    added = [after[key] for key in sorted(set(after) - set(before))]
+    renamed = []
+    used_added = set()
+    used_removed = set()
+    for old in removed:
+        for new in added:
+            if new.key in used_added:
+                continue
+            if (
+                old.schema,
+                old.asset,
+                old.ordinal_position,
+                old.data_type,
+            ) == (
+                new.schema,
+                new.asset,
+                new.ordinal_position,
+                new.data_type,
+            ):
+                renamed.append(CatalogFieldChange(old, new))
+                used_removed.add(old.key)
+                used_added.add(new.key)
+                break
+    return CatalogDiff(
+        added=tuple(item for item in added if item.key not in used_added),
+        removed=tuple(item for item in removed if item.key not in used_removed),
+        changed=tuple(changed),
+        renamed=tuple(renamed),
+    )
+
+
+class CatalogCollectionService:
+    def __init__(self, manager, *, definition_resolver, collector_resolver):
+        self.manager = manager
+        self.definition_resolver = definition_resolver
+        self.collector_resolver = collector_resolver
+
+    def collect(self, data_source_uid, scope):
+        definition = self.definition_resolver(str(data_source_uid))
+        database_type = (
+            definition.get("database_type")
+            if isinstance(definition, dict)
+            else definition.database_type
+        )
+        database_name = (
+            definition.get("database")
+            if isinstance(definition, dict)
+            else definition.database
+        )
+        collector = self.collector_resolver(database_type)
+        with self.manager.connect(
+            str(data_source_uid),
+            "metadata_collection",
+        ) as connection:
+            kwargs = {
+                "data_source_uid": str(data_source_uid),
+                "scope": scope,
+            }
+            if database_type == "mysql":
+                kwargs["database_name"] = database_name
+            return collector.collect(connection, **kwargs)

app/core/data_research/data_elements.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from typing import Any, Callable
+
+from app.core.common.identifiers import new_governance_uid
+from app.core.data_research.errors import (
+    DataElementInvalid,
+    DataElementNotFound,
+    DataElementTransitionInvalid,
+    DataElementVersionConflict,
+)
+
+
+ELEMENT_TRANSITIONS = {
+    "draft": frozenset({"in_review"}),
+    "in_review": frozenset({"draft", "published"}),
+    "published": frozenset({"deprecated"}),
+    "deprecated": frozenset({"published", "retired"}),
+    "retired": frozenset(),
+}
+
+
+@dataclass(frozen=True)
+class DataElementRecord:
+    uid: str
+    code: str
+    status: str
+    current_version: int
+    snapshot: dict[str, Any]
+    created_by: str | None = None
+    updated_by: str | None = None
+
+
+def _required(payload, name):
+    value = str(payload.get(name) or "").strip()
+    if not value:
+        raise DataElementInvalid(f"{name} is required")
+    return value
+
+
+class DataElementService:
+    def __init__(
+        self,
+        repository,
+        *,
+        uid_factory: Callable[[], str] = new_governance_uid,
+        outbox_enqueue: Callable[..., Any] = lambda **_event: None,
+    ):
+        self.repository = repository
+        self.uid_factory = uid_factory
+        self.outbox_enqueue = outbox_enqueue
+
+    def list(self, *, status=None, business_domain_uid=None):
+        return self.repository.list(
+            status=status, business_domain_uid=business_domain_uid
+        )
+
+    def create_draft(self, payload, actor_uid):
+        if not isinstance(payload, dict):
+            raise DataElementInvalid("payload must be an object")
+        snapshot = {
+            "code": _required(payload, "code"),
+            "name_zh": _required(payload, "name_zh"),
+            "name_en": _required(payload, "name_en"),
+            "definition": _required(payload, "definition"),
+            "logical_type": _required(payload, "logical_type"),
+            "evidence_uids": list(payload.get("evidence_uids") or []),
+            "business_domain_uids": list(payload.get("business_domain_uids") or []),
+        }
+        for name in (
+            "aliases",
+            "format",
+            "value_domain",
+            "unit",
+            "sensitivity",
+            "quality_rules",
+        ):
+            if name in payload:
+                snapshot[name] = payload[name]
+        if self.repository.get_by_code(snapshot["code"]) is not None:
+            raise DataElementInvalid("code already exists")
+        return self.repository.save(
+            DataElementRecord(
+                uid=self.uid_factory(),
+                code=snapshot["code"],
+                status="draft",
+                current_version=1,
+                snapshot=snapshot,
+                created_by=actor_uid,
+                updated_by=actor_uid,
+            )
+        )
+
+    def transition(self, uid, target_status, *, expected_version, actor_uid):
+        record = self.repository.get(str(uid))
+        if record is None:
+            raise DataElementNotFound(f"data element {uid} was not found")
+        if int(expected_version) != int(record.current_version):
+            raise DataElementVersionConflict(
+                f"expected version {expected_version}, current version {record.current_version}"
+            )
+        if target_status not in ELEMENT_TRANSITIONS.get(record.status, frozenset()):
+            raise DataElementTransitionInvalid(
+                f"{record.status} -> {target_status} is not allowed"
+            )
+        updated = self.repository.save(
+            replace(
+                record,
+                status=target_status,
+                current_version=record.current_version + 1,
+                updated_by=actor_uid,
+            )
+        )
+        if target_status == "published":
+            self.outbox_enqueue(
+                aggregate_type="data_element",
+                aggregate_id=updated.uid,
+                event_type="data_element.version_published",
+                payload={"uid": updated.uid, "version": updated.current_version},
+            )
+        return updated

app/core/data_research/errors.py

@@ -0,0 +1,45 @@
+class DataResearchError(ValueError):
+    code = "DATA_RESEARCH_ERROR"
+    http_status = 400
+
+
+class IngestionPayloadInvalid(DataResearchError):
+    code = "INGESTION_PAYLOAD_INVALID"
+
+
+class InvalidJobTransition(DataResearchError):
+    code = "INGESTION_TRANSITION_INVALID"
+    http_status = 409
+
+
+class IngestionJobNotFound(DataResearchError):
+    code = "INGESTION_JOB_NOT_FOUND"
+    http_status = 404
+
+
+class DataElementInvalid(DataResearchError):
+    code = "DATA_ELEMENT_INVALID"
+
+
+class DataElementNotFound(DataResearchError):
+    code = "DATA_ELEMENT_NOT_FOUND"
+    http_status = 404
+
+
+class DataElementVersionConflict(DataResearchError):
+    code = "DATA_ELEMENT_VERSION_CONFLICT"
+    http_status = 409
+
+
+class DataElementTransitionInvalid(DataResearchError):
+    code = "DATA_ELEMENT_TRANSITION_INVALID"
+    http_status = 409
+
+
+class CandidateDecisionInvalid(DataResearchError):
+    code = "CANDIDATE_DECISION_INVALID"
+
+
+class DataElementProjectionInvalid(DataResearchError):
+    code = "DATA_ELEMENT_PROJECTION_INVALID"
+    http_status = 422

app/core/data_research/extractors/__init__.py

@@ -0,0 +1,3 @@
+from app.core.data_research.extractors.registry import default_registry
+
+__all__ = ["default_registry"]

app/core/data_research/extractors/base.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class ExtractionContext:
+    filename: str
+    media_type: str
+    max_bytes: int = 25 * 1024 * 1024
+
+
+@dataclass(frozen=True)
+class ExtractionEvidence:
+    locator: dict[str, Any]
+    excerpt: str | None = None
+    confidence: float | None = None
+
+
+@dataclass(frozen=True)
+class ExtractedItem:
+    candidate_type: str
+    data: dict[str, Any]
+    evidence: tuple[ExtractionEvidence, ...] = ()
+    confidence: float = 1.0
+
+
+@dataclass(frozen=True)
+class ExtractionBatch:
+    parser_version: str
+    content_hash: str
+    items: tuple[ExtractedItem, ...]
+    warnings: tuple[str, ...] = ()
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class BaseExtractor:
+    parser_version = ""
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        raise NotImplementedError
+
+    def extract(self, content: bytes, context: ExtractionContext) -> ExtractionBatch:
+        raise NotImplementedError
+
+    def _batch(
+        self,
+        content: bytes,
+        items: list[ExtractedItem],
+        *,
+        warnings: list[str] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> ExtractionBatch:
+        return ExtractionBatch(
+            parser_version=self.parser_version,
+            content_hash=hashlib.sha256(content).hexdigest(),
+            items=tuple(items),
+            warnings=tuple(warnings or ()),
+            metadata=dict(metadata or {}),
+        )

app/core/data_research/extractors/csv.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import csv
+import io
+import re
+from pathlib import Path
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+_INTEGER = re.compile(r"^[+-]?\d+$")
+_DECIMAL = re.compile(r"^[+-]?(?:\d+\.\d*|\d*\.\d+)$")
+_BOOLEAN = frozenset({"true", "false", "yes", "no"})
+
+
+def infer_type(values) -> str:
+    present = [str(value).strip() for value in values if str(value).strip()]
+    if not present:
+        return "string"
+    lowered = [value.lower() for value in present]
+    if all(value in _BOOLEAN for value in lowered):
+        return "boolean"
+    if all(_INTEGER.fullmatch(value) for value in present):
+        return "integer"
+    if all(_INTEGER.fullmatch(value) or _DECIMAL.fullmatch(value) for value in present):
+        return "decimal"
+    return "string"
+
+
+class CsvExtractor(BaseExtractor):
+    parser_version = "csv-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        return filename.lower().endswith(".csv") or media_type in {
+            "text/csv",
+            "application/csv",
+        }
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        try:
+            text = content.decode("utf-8-sig")
+        except UnicodeDecodeError as exc:
+            raise ValueError("CSV file must be UTF-8 encoded") from exc
+        if not text.strip():
+            raise ValueError("CSV header is required")
+        try:
+            dialect = csv.Sniffer().sniff(text[:8192], delimiters=",;\t|")
+        except csv.Error:
+            dialect = csv.excel
+        rows = list(csv.reader(io.StringIO(text), dialect))
+        if not rows or not any(str(value).strip() for value in rows[0]):
+            raise ValueError("CSV header is required")
+        headers = [str(value).strip() for value in rows[0]]
+        data_rows = rows[1:]
+        columns = []
+        for index, name in enumerate(headers, start=1):
+            values = [row[index - 1] if len(row) >= index else "" for row in data_rows]
+            columns.append(
+                {
+                    "name_zh": "",
+                    "name_en": name,
+                    "data_type": infer_type(values),
+                    "nullable": any(not str(value).strip() for value in values),
+                    "evidence_locator": {
+                        "kind": "csv.header",
+                        "column": index,
+                        "name": name,
+                    },
+                }
+            )
+        item = ExtractedItem(
+            candidate_type="physical_asset",
+            data={
+                "table_info": {
+                    "name_zh": "",
+                    "name_en": Path(context.filename).stem,
+                },
+                "columns": columns,
+            },
+            evidence=(
+                ExtractionEvidence(
+                    locator={"kind": "csv.header", "row": 1},
+                    excerpt=",".join(headers),
+                ),
+            ),
+        )
+        return self._batch(
+            content,
+            [item],
+            metadata={"row_count": len(data_rows), "delimiter": dialect.delimiter},
+        )

app/core/data_research/extractors/docx.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import io
+
+from docx import Document
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+class DocxExtractor(BaseExtractor):
+    parser_version = "docx-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        return filename.lower().endswith(".docx") or media_type == (
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+        )
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        document = Document(io.BytesIO(content))
+        items = []
+        for index, paragraph in enumerate(document.paragraphs, start=1):
+            value = paragraph.text.strip()
+            if not value:
+                continue
+            evidence = ExtractionEvidence(
+                locator={"kind": "docx.paragraph", "paragraph": index},
+                excerpt=value,
+            )
+            items.append(
+                ExtractedItem(
+                    candidate_type="document_fragment",
+                    data={"kind": "paragraph", "text": value},
+                    evidence=(evidence,),
+                )
+            )
+        for table_index, table in enumerate(document.tables, start=1):
+            header = tuple(cell.text.strip() for cell in table.rows[0].cells) if table.rows else ()
+            for row_index, row in enumerate(table.rows, start=1):
+                values = tuple(cell.text.strip() for cell in row.cells)
+                if row_index > 1 and values == header:
+                    continue
+                for cell_index, value in enumerate(values, start=1):
+                    if not value:
+                        continue
+                    evidence = ExtractionEvidence(
+                        locator={
+                            "kind": "docx.table",
+                            "table": table_index,
+                            "row": row_index,
+                            "cell": cell_index,
+                        },
+                        excerpt=value,
+                    )
+                    items.append(
+                        ExtractedItem(
+                            candidate_type="document_fragment",
+                            data={"kind": "table_cell", "text": value},
+                            evidence=(evidence,),
+                        )
+                    )
+        warnings = [] if items else ["document contains no usable text"]
+        return self._batch(content, items, warnings=warnings)

app/core/data_research/extractors/excel.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import io
+
+import pandas as pd
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+def _column_letter(index: int) -> str:
+    result = ""
+    while index:
+        index, remainder = divmod(index - 1, 26)
+        result = chr(65 + remainder) + result
+    return result
+
+
+def _pandas_type(series) -> str:
+    if pd.api.types.is_bool_dtype(series.dtype):
+        return "boolean"
+    if pd.api.types.is_integer_dtype(series.dtype):
+        return "integer"
+    if pd.api.types.is_float_dtype(series.dtype):
+        return "decimal"
+    if pd.api.types.is_datetime64_any_dtype(series.dtype):
+        return "datetime"
+    return "string"
+
+
+class ExcelExtractor(BaseExtractor):
+    parser_version = "excel-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        lowered = filename.lower()
+        return lowered.endswith((".xlsx", ".xls")) or "spreadsheet" in media_type or "excel" in media_type
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        workbook = pd.ExcelFile(io.BytesIO(content))
+        items = []
+        for sheet_name in workbook.sheet_names:
+            frame = pd.read_excel(workbook, sheet_name=sheet_name)
+            columns = []
+            last_row = max(1, len(frame) + 1)
+            for index, name in enumerate(frame.columns, start=1):
+                letter = _column_letter(index)
+                columns.append(
+                    {
+                        "name_zh": "",
+                        "name_en": str(name),
+                        "data_type": _pandas_type(frame[name]),
+                        "nullable": bool(frame[name].isna().any()),
+                        "evidence_locator": {
+                            "kind": "excel.cell_range",
+                            "sheet": sheet_name,
+                            "range": f"{letter}1:{letter}{last_row}",
+                        },
+                    }
+                )
+            items.append(
+                ExtractedItem(
+                    candidate_type="physical_asset",
+                    data={
+                        "table_info": {"name_zh": "", "name_en": sheet_name},
+                        "columns": columns,
+                    },
+                    evidence=(
+                        ExtractionEvidence(
+                            locator={"kind": "excel.sheet", "sheet": sheet_name},
+                        ),
+                    ),
+                )
+            )
+        return self._batch(content, items, metadata={"sheet_count": len(items)})

app/core/data_research/extractors/image.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+class ImageExtractor(BaseExtractor):
+    parser_version = "image-ocr-v1"
+
+    def __init__(self, ocr_service):
+        self.ocr_service = ocr_service
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        suffix = filename.lower()
+        return media_type in {"image/png", "image/jpeg"} or suffix.endswith(
+            (".png", ".jpg", ".jpeg")
+        )
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        items = []
+        for result in self.ocr_service.extract_pages([content]):
+            evidence = ExtractionEvidence(
+                locator={
+                    "kind": "image.ocr",
+                    "page": result.page,
+                    "bbox": list(result.bbox),
+                },
+                excerpt=result.text,
+                confidence=result.confidence,
+            )
+            items.append(
+                ExtractedItem(
+                    candidate_type="document_fragment",
+                    data={
+                        "kind": "ocr_block",
+                        "text": result.text,
+                        "review_required": result.review_required,
+                    },
+                    evidence=(evidence,),
+                    confidence=result.confidence,
+                )
+            )
+        return self._batch(content, items)

app/core/data_research/extractors/pdf.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import io
+
+import pdfplumber
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+class PdfExtractor(BaseExtractor):
+    parser_version = "pdf-text-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        return filename.lower().endswith(".pdf") or media_type == "application/pdf"
+
+    @staticmethod
+    def _page_evidence(page_number, text, bbox):
+        x0, top, x1, bottom = [float(value) for value in bbox]
+        width = max(1.0, x1 - x0)
+        height = max(1.0, bottom - top)
+        return ExtractionEvidence(
+            locator={
+                "kind": "pdf.page",
+                "page": page_number,
+                "bbox": [0.0, 0.0, round(width / width, 6), round(height / height, 6)],
+            },
+            excerpt=text,
+        )
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        items = []
+        ocr_required = []
+        with pdfplumber.open(io.BytesIO(content)) as document:
+            for page_number, page in enumerate(document.pages, start=1):
+                text = str(page.extract_text() or "").strip()
+                if not text:
+                    ocr_required.append(page_number)
+                    continue
+                items.append(
+                    ExtractedItem(
+                        candidate_type="document_fragment",
+                        data={"kind": "page", "text": text},
+                        evidence=(self._page_evidence(page_number, text, page.bbox),),
+                    )
+                )
+                for table_index, table in enumerate(page.extract_tables() or (), start=1):
+                    for row_index, row in enumerate(table or (), start=1):
+                        for cell_index, cell in enumerate(row or (), start=1):
+                            value = str(cell or "").strip()
+                            if not value:
+                                continue
+                            items.append(
+                                ExtractedItem(
+                                    candidate_type="document_fragment",
+                                    data={"kind": "table_cell", "text": value},
+                                    evidence=(
+                                        ExtractionEvidence(
+                                            locator={
+                                                "kind": "pdf.table",
+                                                "page": page_number,
+                                                "table": table_index,
+                                                "row": row_index,
+                                                "cell": cell_index,
+                                            },
+                                            excerpt=value,
+                                        ),
+                                    ),
+                                )
+                            )
+        warnings = []
+        if ocr_required:
+            pages = ", ".join(str(value) for value in ocr_required)
+            warnings.append(f"OCR required for pages: {pages}")
+        return self._batch(
+            content,
+            items,
+            warnings=warnings,
+            metadata={"ocr_required_pages": ocr_required},
+        )

app/core/data_research/extractors/registry.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from app.core.data_research.extractors.base import ExtractionContext
+from app.core.data_research.extractors.csv import CsvExtractor
+from app.core.data_research.extractors.excel import ExcelExtractor
+from app.core.data_research.extractors.docx import DocxExtractor
+from app.core.data_research.extractors.pdf import PdfExtractor
+from app.core.data_research.extractors.sql import SqlExtractor
+
+
+class ExtractorRegistry:
+    def __init__(self, extractors):
+        self.extractors = tuple(extractors)
+
+    def resolve(self, media_type: str, filename: str):
+        for extractor in self.extractors:
+            if extractor.can_handle(media_type, filename):
+                return extractor
+        raise ValueError(f"unsupported source format: {filename}")
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        if len(content) > int(context.max_bytes):
+            raise ValueError("source exceeds configured size limit")
+        return self.resolve(context.media_type, context.filename).extract(
+            content,
+            context,
+        )
+
+
+def default_registry():
+    return ExtractorRegistry(
+        (SqlExtractor(), CsvExtractor(), ExcelExtractor(), DocxExtractor(), PdfExtractor())
+    )

app/core/data_research/extractors/sql.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+from app.core.llm.ddl_parser import DDLParser
+
+
+class SqlExtractor(BaseExtractor):
+    parser_version = "sql-ddl-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        return filename.lower().endswith(".sql") or media_type in {
+            "application/sql",
+            "text/sql",
+        }
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        try:
+            sql_text = content.decode("utf-8-sig")
+        except UnicodeDecodeError as exc:
+            raise ValueError("SQL file must be UTF-8 encoded") from exc
+        tables = DDLParser()._parse_sql_ddl_fallback(sql_text)
+        if not tables:
+            raise ValueError("no supported CREATE TABLE or CREATE VIEW statements found")
+        items = [
+            ExtractedItem(
+                candidate_type="physical_asset",
+                data=table,
+                evidence=(
+                    ExtractionEvidence(
+                        locator={"kind": "sql.statement", "statement": index},
+                    ),
+                ),
+            )
+            for index, table in enumerate(tables, start=1)
+        ]
+        return self._batch(content, items)

app/core/data_research/file_policy.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+class FilePolicyViolation(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class ValidatedFile:
+    filename: str
+    extension: str
+    media_type: str
+    size_bytes: int
+
+
+class FilePolicy:
+    MEDIA_TYPES = {
+        ".sql": {"text/plain", "application/sql", "text/sql"},
+        ".csv": {"text/csv", "application/csv", "text/plain"},
+        ".xls": {"application/vnd.ms-excel"},
+        ".xlsx": {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
+        ".docx": {"application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
+        ".pdf": {"application/pdf"},
+        ".png": {"image/png"},
+        ".jpg": {"image/jpeg"},
+        ".jpeg": {"image/jpeg"},
+    }
+
+    def __init__(self, *, max_bytes=25 * 1024 * 1024, max_pages=500):
+        self.max_bytes = int(max_bytes)
+        self.max_pages = int(max_pages)
+
+    @staticmethod
+    def _magic_matches(extension, content):
+        signatures = {
+            ".pdf": (b"%PDF-",),
+            ".png": (b"\x89PNG\r\n\x1a\n",),
+            ".jpg": (b"\xff\xd8\xff",),
+            ".jpeg": (b"\xff\xd8\xff",),
+            ".docx": (b"PK\x03\x04",),
+            ".xlsx": (b"PK\x03\x04",),
+            ".xls": (b"\xd0\xcf\x11\xe0",),
+        }
+        expected = signatures.get(extension)
+        if expected is None:
+            try:
+                content[:4096].decode("utf-8-sig")
+                return True
+            except UnicodeDecodeError:
+                return False
+        return any(content.startswith(signature) for signature in expected)
+
+    def validate(self, filename, media_type, content, *, page_count=None):
+        filename = str(filename or "")
+        if not filename or Path(filename).name != filename or "\x00" in filename:
+            raise FilePolicyViolation("unsafe filename")
+        extension = Path(filename).suffix.lower()
+        if extension == ".doc":
+            raise FilePolicyViolation("legacy DOC must be converted to DOCX")
+        if extension not in self.MEDIA_TYPES:
+            raise FilePolicyViolation("file extension is not allowed")
+        if str(media_type or "").lower() not in self.MEDIA_TYPES[extension]:
+            raise FilePolicyViolation("media type does not match file extension")
+        if len(content) > self.max_bytes:
+            raise FilePolicyViolation("file exceeds configured size limit")
+        if page_count is not None and int(page_count) > self.max_pages:
+            raise FilePolicyViolation("document exceeds configured page limit")
+        if not self._magic_matches(extension, content):
+            raise FilePolicyViolation("file signature does not match extension")
+        return ValidatedFile(filename, extension, str(media_type).lower(), len(content))

app/core/data_research/graph_projection.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from app.core.data_research.errors import DataElementProjectionInvalid
+
+
+class DataElementGraphProjector:
+    """Idempotent effective-graph projection for published data elements."""
+
+    STATEMENT = """
+    MERGE (element:DataElement {uid: $uid})
+    SET element.code = $code,
+        element.name_zh = $name_zh,
+        element.version = $version,
+        element.status = 'published'
+    """
+
+    def __init__(self, graph_session):
+        self.graph_session = graph_session
+
+    def project(self, payload):
+        if str(payload.get("status") or "") != "published":
+            raise DataElementProjectionInvalid(
+                "only a published data element version can be projected"
+            )
+        snapshot = dict(payload.get("snapshot") or {})
+        self.graph_session.run(
+            self.STATEMENT,
+            uid=str(payload["uid"]),
+            version=int(payload["version"]),
+            code=str(snapshot.get("code") or ""),
+            name_zh=str(snapshot.get("name_zh") or ""),
+        )

app/core/data_research/ingestion.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import replace
+from typing import Any, Callable
+
+from app.core.common.identifiers import new_governance_uid
+from app.core.common.timezone_utils import now_china_naive
+from app.core.data_research.errors import (
+    IngestionJobNotFound,
+    IngestionPayloadInvalid,
+    InvalidJobTransition,
+)
+from app.core.data_research.models import IngestionJobRecord, IngestionJobSpec
+
+
+ALLOWED_TRANSITIONS = {
+    "created": frozenset({"queued", "cancelled"}),
+    "queued": frozenset({"extracting", "failed", "cancelled"}),
+    "extracting": frozenset({"normalizing", "failed", "cancelled"}),
+    "normalizing": frozenset({"matching", "failed", "cancelled"}),
+    "matching": frozenset(
+        {"awaiting_review", "partial", "failed", "cancelled"}
+    ),
+    "awaiting_review": frozenset({"published", "partial", "cancelled"}),
+    "partial": frozenset({"queued", "cancelled"}),
+    "failed": frozenset({"queued", "cancelled"}),
+    "published": frozenset(),
+    "cancelled": frozenset(),
+}
+TERMINAL_STATUSES = frozenset({"published", "cancelled"})
+_SECRET_ASSIGNMENT = re.compile(
+    r"(?i)\b(password|token|secret|authorization)\s*[:=]\s*[^\s,;]+"
+)
+
+
+def _required_text(payload: dict[str, Any], name: str) -> str:
+    value = str(payload.get(name) or "").strip()
+    if not value:
+        raise IngestionPayloadInvalid(f"{name} is required")
+    return value
+
+
+def _sanitize_error(error: Any) -> str | None:
+    if error is None:
+        return None
+    value = _SECRET_ASSIGNMENT.sub(lambda match: f"{match.group(1)}=[redacted]", str(error))
+    return value[:1000]
+
+
+class IngestionService:
+    def __init__(
+        self,
+        repository,
+        *,
+        uid_factory: Callable[[], str] = new_governance_uid,
+        nonce_factory: Callable[[], str] = new_governance_uid,
+        clock: Callable[[], Any] = now_china_naive,
+        commit: Callable[[], Any] = lambda: None,
+        rollback: Callable[[], Any] = lambda: None,
+    ):
+        self.repository = repository
+        self.uid_factory = uid_factory
+        self.nonce_factory = nonce_factory
+        self.clock = clock
+        self.commit = commit
+        self.rollback = rollback
+
+    @staticmethod
+    def _spec(payload: dict[str, Any]) -> IngestionJobSpec:
+        if not isinstance(payload, dict):
+            raise IngestionPayloadInvalid("payload must be an object")
+        parameters = payload.get("parameters", {})
+        if parameters is None:
+            parameters = {}
+        if not isinstance(parameters, dict):
+            raise IngestionPayloadInvalid("parameters must be an object")
+        artifact_uid = str(payload.get("artifact_uid") or "").strip() or None
+        return IngestionJobSpec(
+            source_uid=_required_text(payload, "source_uid"),
+            artifact_uid=artifact_uid,
+            job_type=_required_text(payload, "job_type"),
+            parser_version=_required_text(payload, "parser_version"),
+            parameters=dict(parameters),
+            force_rerun=bool(payload.get("force_rerun", False)),
+        )
+
+    def _idempotency_key(self, spec: IngestionJobSpec) -> str:
+        value = {
+            "source_uid": spec.source_uid,
+            "artifact_uid": spec.artifact_uid,
+            "job_type": spec.job_type,
+            "parser_version": spec.parser_version,
+            "parameters": spec.parameters,
+        }
+        if spec.force_rerun:
+            value["rerun_nonce"] = self.nonce_factory()
+        canonical = json.dumps(
+            value,
+            sort_keys=True,
+            ensure_ascii=False,
+            separators=(",", ":"),
+        )
+        return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+    def create_job(self, payload: dict[str, Any], actor_uid: str | None):
+        spec = self._spec(payload)
+        idempotency_key = self._idempotency_key(spec)
+        if not spec.force_rerun:
+            existing = self.repository.get_by_idempotency_key(idempotency_key)
+            if existing is not None:
+                return existing, False
+        now = self.clock()
+        record = IngestionJobRecord(
+            uid=self.uid_factory(),
+            source_uid=spec.source_uid,
+            artifact_uid=spec.artifact_uid,
+            job_type=spec.job_type,
+            parser_version=spec.parser_version,
+            idempotency_key=idempotency_key,
+            actor_uid=actor_uid,
+            parameters=spec.parameters,
+            force_rerun=spec.force_rerun,
+            created_at=now,
+            updated_at=now,
+        )
+        try:
+            saved = self.repository.add(record)
+            self.commit()
+            return saved, True
+        except Exception:
+            self.rollback()
+            raise
+
+    def _get(self, uid: str) -> IngestionJobRecord:
+        record = self.repository.get(str(uid))
+        if record is None:
+            raise IngestionJobNotFound(f"ingestion job {uid} was not found")
+        return record
+
+    def transition(
+        self,
+        uid: str,
+        target_status: str,
+        statistics: dict[str, Any] | None = None,
+        error: Any = None,
+    ) -> IngestionJobRecord:
+        record = self._get(uid)
+        target_status = str(target_status or "").strip()
+        if target_status not in ALLOWED_TRANSITIONS.get(record.status, frozenset()):
+            raise InvalidJobTransition(f"{record.status} -> {target_status} is not allowed")
+        now = self.clock()
+        updated = replace(
+            record,
+            status=target_status,
+            statistics=dict(statistics or record.statistics),
+            last_error=_sanitize_error(error),
+            updated_at=now,
+            started_at=(now if target_status == "extracting" and record.started_at is None else record.started_at),
+            finished_at=(now if target_status in TERMINAL_STATUSES | {"failed", "partial"} else None),
+        )
+        try:
+            saved = self.repository.save(updated)
+            self.commit()
+            return saved
+        except Exception:
+            self.rollback()
+            raise
+
+    def get_job(self, uid: str) -> IngestionJobRecord:
+        return self._get(uid)
+
+    def list_jobs(self, filters: dict[str, Any] | None = None):
+        return self.repository.list(filters or {})
+
+    def retry(self, uid: str) -> IngestionJobRecord:
+        return self.transition(uid, "queued")
+
+    def cancel(self, uid: str) -> IngestionJobRecord:
+        return self.transition(uid, "cancelled")

app/core/data_research/models.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass(frozen=True)
+class IngestionJobSpec:
+    source_uid: str
+    job_type: str
+    parser_version: str
+    artifact_uid: str | None = None
+    parameters: dict[str, Any] = field(default_factory=dict)
+    force_rerun: bool = False
+
+
+@dataclass(frozen=True)
+class IngestionJobRecord:
+    uid: str
+    source_uid: str
+    job_type: str
+    parser_version: str
+    idempotency_key: str
+    actor_uid: str | None
+    artifact_uid: str | None = None
+    parameters: dict[str, Any] = field(default_factory=dict)
+    status: str = "created"
+    statistics: dict[str, Any] = field(default_factory=dict)
+    last_error: str | None = None
+    force_rerun: bool = False
+    created_at: datetime | None = None
+    updated_at: datetime | None = None
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+
+
+@dataclass(frozen=True)
+class JobTransition:
+    source_status: str
+    target_status: str

app/core/data_research/ocr/__init__.py

@@ -0,0 +1,4 @@
+from app.core.data_research.ocr.base import OcrBlock, OcrProvider
+from app.core.data_research.ocr.service import OcrResult, OcrService
+
+__all__ = ["OcrBlock", "OcrProvider", "OcrResult", "OcrService"]

app/core/data_research/ocr/base.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class OcrBlock:
+    text: str
+    bbox: tuple[float, float, float, float]
+    page: int
+    confidence: float
+
+
+class OcrProvider(Protocol):
+    def extract(self, image_bytes: bytes) -> list[OcrBlock]: ...

app/core/data_research/ocr/http_provider.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import requests
+
+from app.core.data_research.ocr.base import OcrBlock
+from app.core.data_research.ocr.service import OcrResponseInvalid
+
+
+class HttpOcrProvider:
+    def __init__(
+        self,
+        endpoint,
+        *,
+        token=None,
+        timeout_seconds=10,
+        verify_tls=True,
+        max_response_bytes=2 * 1024 * 1024,
+        post=requests.post,
+    ):
+        endpoint = str(endpoint or "").strip()
+        if not endpoint.startswith("https://"):
+            raise ValueError("OCR endpoint must use HTTPS")
+        self.endpoint = endpoint
+        self.token = token
+        self.timeout_seconds = float(timeout_seconds)
+        self.verify_tls = bool(verify_tls)
+        self.max_response_bytes = int(max_response_bytes)
+        self.post = post
+
+    def extract(self, image_bytes):
+        headers = {"Accept": "application/json"}
+        if self.token:
+            headers["Authorization"] = f"Bearer {self.token}"
+        try:
+            response = self.post(
+                self.endpoint,
+                files={"file": ("page.png", image_bytes, "image/png")},
+                headers=headers,
+                timeout=self.timeout_seconds,
+                verify=self.verify_tls,
+            )
+            response.raise_for_status()
+            if len(response.content) > self.max_response_bytes:
+                raise OcrResponseInvalid("OCR response size exceeds configured limit")
+            payload = response.json()
+            values = payload.get("blocks")
+            if not isinstance(values, list):
+                raise OcrResponseInvalid("OCR response blocks are malformed")
+            return [
+                OcrBlock(
+                    text=str(value["text"]),
+                    bbox=tuple(float(item) for item in value["bbox"]),
+                    page=int(value.get("page") or 1),
+                    confidence=float(value["confidence"]),
+                )
+                for value in values
+            ]
+        except OcrResponseInvalid:
+            raise
+        except Exception as exc:
+            raise OcrResponseInvalid("OCR provider request failed") from exc

app/core/data_research/ocr/service.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+
+
+class OcrError(ValueError):
+    pass
+
+
+class OcrDisabled(OcrError):
+    pass
+
+
+class OcrResponseInvalid(OcrError):
+    pass
+
+
+@dataclass(frozen=True)
+class OcrResult:
+    text: str
+    bbox: tuple[float, float, float, float]
+    page: int
+    confidence: float
+    review_required: bool
+
+
+class OcrService:
+    def __init__(self, provider, *, review_threshold=0.75):
+        self.provider = provider
+        self.review_threshold = float(review_threshold)
+
+    def extract_pages(self, pages):
+        if self.provider is None:
+            raise OcrDisabled("OCR provider is not configured")
+        results = []
+        for page_number, image_bytes in enumerate(pages, start=1):
+            blocks = self.provider.extract(image_bytes)
+            if not isinstance(blocks, (list, tuple)):
+                raise OcrResponseInvalid("OCR blocks must be a list")
+            for block in blocks:
+                try:
+                    bbox = tuple(float(value) for value in block.bbox)
+                    confidence = float(block.confidence)
+                    text = str(block.text).strip()
+                except (AttributeError, TypeError, ValueError) as exc:
+                    raise OcrResponseInvalid("malformed OCR block") from exc
+                if len(bbox) != 4 or any(value < 0 or value > 1 for value in bbox):
+                    raise OcrResponseInvalid("OCR bounding box must be normalized")
+                if bbox[0] > bbox[2] or bbox[1] > bbox[3]:
+                    raise OcrResponseInvalid("OCR bounding box is invalid")
+                if not 0 <= confidence <= 1 or not text:
+                    raise OcrResponseInvalid("OCR text or confidence is invalid")
+                results.append(
+                    OcrResult(
+                        text=text,
+                        bbox=bbox,
+                        page=page_number,
+                        confidence=confidence,
+                        review_required=confidence < self.review_threshold,
+                    )
+                )
+        return results

app/core/data_research/ontology/__init__.py

@@ -0,0 +1 @@
+"""Versioned ontology governance services."""

app/core/data_research/ontology/change_sets.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+
+from app.core.common.identifiers import new_governance_uid
+
+
+class ChangeSetConflict(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class ChangeDecision:
+    suggestion_uid: str
+    decision: str
+    actor_uid: str
+    reason: str
+    payload: dict | None = None
+
+
+@dataclass(frozen=True)
+class ChangeSet:
+    uid: str
+    ontology_uid: str
+    suggestions: tuple
+    decisions: tuple[ChangeDecision, ...]
+    created_by: str
+
+
+class ChangeSetService:
+    def __init__(self, *, uid_factory=new_governance_uid):
+        self.uid_factory = uid_factory
+
+    def create(self, ontology_uid, suggestions, *, actor_uid):
+        targets = {}
+        for suggestion in suggestions:
+            target = (suggestion.kind, str(suggestion.payload.get("uid")))
+            if target in targets and targets[target].payload != suggestion.payload:
+                raise ChangeSetConflict("multiple suggestions conflict on the same target")
+            targets[target] = suggestion
+        return ChangeSet(self.uid_factory(), str(ontology_uid), tuple(suggestions), (), str(actor_uid))
+
+    def decide(self, change_set, decisions, *, actor_uid):
+        known = {item.uid: item for item in change_set.suggestions}
+        audited = list(change_set.decisions)
+        decided_uids = {item.suggestion_uid for item in audited}
+        for item in decisions:
+            uid = str(item.get("suggestion_uid") or "")
+            decision = str(item.get("decision") or "")
+            if uid not in known or decision not in {"accept", "reject", "edit"}:
+                raise ValueError("invalid change-set decision")
+            if uid in decided_uids:
+                raise ValueError("change-set decision is immutable")
+            payload = dict(item.get("payload") or {}) if decision == "edit" else None
+            if decision == "edit" and not payload:
+                raise ValueError("edited suggestion payload is required")
+            audited.append(
+                ChangeDecision(
+                    uid,
+                    decision,
+                    str(actor_uid),
+                    str(item.get("reason") or "").strip(),
+                    payload,
+                )
+            )
+            decided_uids.add(uid)
+        return replace(change_set, decisions=tuple(audited))
+
+    def apply(self, graph, change_set):
+        result = {
+            "classes": list(graph.get("classes") or []),
+            "properties": list(graph.get("properties") or []),
+            "relations": list(graph.get("relations") or []),
+            "constraints": list(graph.get("constraints") or []),
+            "domain_links": list(graph.get("domain_links") or []),
+            "element_mappings": list(graph.get("element_mappings") or []),
+        }
+        suggestions = {item.uid: item for item in change_set.suggestions}
+        sections = {"class": "classes", "property": "properties", "relation": "relations", "constraint": "constraints"}
+        for decision in change_set.decisions:
+            if decision.decision == "reject":
+                continue
+            suggestion = suggestions[decision.suggestion_uid]
+            payload = dict(decision.payload or suggestion.payload)
+            section = sections.get(suggestion.kind)
+            if section:
+                result[section].append(payload)
+        return result
+
+
+class SqlAlchemyDynamicOntologyService:
+    def __init__(self, session, *, suggestion_engine=None, change_set_service=None):
+        from app.core.data_research.ontology.suggestions import OntologySuggestionEngine
+
+        self.session = session
+        self.suggestion_engine = suggestion_engine or OntologySuggestionEngine()
+        self.change_set_service = change_set_service or ChangeSetService()
+
+    @staticmethod
+    def _suggestion_dict(item):
+        return {
+            "uid": item.uid,
+            "kind": item.kind,
+            "payload": item.payload,
+            "evidence_uids": list(item.evidence_uids),
+            "confidence": item.confidence,
+            "source": item.source,
+            "model_version": item.model_version,
+            "prompt_version": item.prompt_version,
+        }
+
+    def generate(self, ontology_uid, payload, actor_uid):
+        from app.models.data_research import OntologyChangeSetModel
+
+        suggestions = self.suggestion_engine.generate(
+            domains=list(payload.get("domains") or []),
+            data_elements=list(payload.get("data_elements") or []),
+            foreign_keys=list(payload.get("foreign_keys") or []),
+            include_ai=bool(payload.get("include_ai", False)),
+        )
+        change_set = self.change_set_service.create(
+            ontology_uid, suggestions, actor_uid=actor_uid
+        )
+        self.session.add(
+            OntologyChangeSetModel(
+                uid=change_set.uid,
+                ontology_uid=str(ontology_uid),
+                changes=[self._suggestion_dict(item) for item in suggestions],
+                decisions=[],
+                created_by=actor_uid,
+            )
+        )
+        self.session.commit()
+        return {
+            "change_set_uid": change_set.uid,
+            "suggestions": suggestions,
+        }
+
+    def decide(self, ontology_uid, change_set_uid, decisions, actor_uid):
+        from app.core.data_research.ontology.suggestions import Suggestion
+        from app.models.data_research import OntologyChangeSetModel
+
+        model = self.session.get(OntologyChangeSetModel, str(change_set_uid))
+        if model is None or str(model.ontology_uid) != str(ontology_uid):
+            raise LookupError("ontology change set was not found")
+        suggestions = tuple(
+            Suggestion(
+                item["uid"],
+                item["kind"],
+                dict(item["payload"]),
+                tuple(item.get("evidence_uids") or []),
+                float(item["confidence"]),
+                item["source"],
+                item.get("model_version"),
+                item.get("prompt_version"),
+            )
+            for item in model.changes
+        )
+        change_set = ChangeSet(
+            str(model.uid), str(model.ontology_uid), suggestions, (), model.created_by
+        )
+        decided = self.change_set_service.decide(
+            change_set, decisions, actor_uid=actor_uid
+        )
+        model.decisions = [item.__dict__ for item in decided.decisions]
+        model.status = "reviewed"
+        self.session.commit()
+        return {
+            "uid": str(model.uid),
+            "ontology_uid": str(model.ontology_uid),
+            "decisions": list(model.decisions),
+            "actor_uid": actor_uid,
+        }

app/core/data_research/ontology/exchange.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import json
+import xml.etree.ElementTree as ET
+
+from app.core.data_research.ontology.models import GRAPH_SECTIONS, GraphDocument
+
+
+class ExchangeInvalid(ValueError):
+    pass
+
+
+SECRET_KEYS = {"password", "secret", "token", "api_key", "authorization", "credentials"}
+OWL_NS = "http://www.w3.org/2002/07/owl#"
+RDF_NS = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+DATAOPS_NS = "https://dataops.local/ontology#"
+
+
+def _safe(value, key=""):
+    if key.casefold() in SECRET_KEYS:
+        return "[redacted]"
+    if isinstance(value, dict):
+        return {name: _safe(item, name) for name, item in sorted(value.items())}
+    if isinstance(value, list):
+        return [_safe(item) for item in value]
+    return value
+
+
+class OntologyExchange:
+    def __init__(self, *, max_import_bytes=5 * 1024 * 1024):
+        self.max_import_bytes = int(max_import_bytes)
+
+    def _bounded(self, content):
+        if len(content) > self.max_import_bytes:
+            raise ExchangeInvalid("ontology import exceeds size limit")
+
+    def export_json(self, ontology_uid, version, graph):
+        payload = {
+            "format": "dataops-ontology-json-v1",
+            "ontology_uid": str(ontology_uid),
+            "version": int(version),
+            "graph_document": _safe(GraphDocument.from_dict(graph).to_dict()),
+        }
+        return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
+
+    def import_json(self, content):
+        self._bounded(content)
+        try:
+            payload = json.loads(content.decode("utf-8"))
+            return {
+                "ontology_uid": str(payload["ontology_uid"]),
+                "version": int(payload["version"]),
+                "graph_document": GraphDocument.from_dict(payload["graph_document"]).to_dict(),
+            }
+        except (KeyError, TypeError, ValueError, UnicodeDecodeError) as exc:
+            raise ExchangeInvalid("invalid ontology JSON") from exc
+
+    def export_owl(self, ontology_uid, version, graph):
+        ET.register_namespace("owl", OWL_NS)
+        ET.register_namespace("rdf", RDF_NS)
+        ET.register_namespace("dataops", DATAOPS_NS)
+        root = ET.Element(f"{{{RDF_NS}}}RDF")
+        ontology = ET.SubElement(root, f"{{{OWL_NS}}}Ontology")
+        ontology.set(f"{{{RDF_NS}}}about", f"urn:dataops:ontology:{ontology_uid}")
+        ontology.set(f"{{{DATAOPS_NS}}}uid", str(ontology_uid))
+        ontology.set(f"{{{DATAOPS_NS}}}version", str(int(version)))
+        safe_graph = _safe(GraphDocument.from_dict(graph).to_dict())
+        for section in GRAPH_SECTIONS:
+            node = ET.SubElement(ontology, f"{{{DATAOPS_NS}}}{section}")
+            node.text = json.dumps(safe_graph[section], ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+        return ET.tostring(root, encoding="utf-8", xml_declaration=True)
+
+    def import_owl(self, content):
+        self._bounded(content)
+        upper = content.upper()
+        if b"<!DOCTYPE" in upper or b"<!ENTITY" in upper:
+            raise ExchangeInvalid("unsafe XML declarations are forbidden")
+        try:
+            root = ET.fromstring(content)
+            ontology = root.find(f"{{{OWL_NS}}}Ontology")
+            if ontology is None:
+                raise ValueError("missing ontology")
+            graph = {}
+            for section in GRAPH_SECTIONS:
+                node = ontology.find(f"{{{DATAOPS_NS}}}{section}")
+                graph[section] = json.loads(node.text or "[]") if node is not None else []
+            return {
+                "ontology_uid": str(ontology.attrib[f"{{{DATAOPS_NS}}}uid"]),
+                "version": int(ontology.attrib[f"{{{DATAOPS_NS}}}version"]),
+                "graph_document": GraphDocument.from_dict(graph).to_dict(),
+            }
+        except (ET.ParseError, KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
+            raise ExchangeInvalid("invalid ontology RDF/OWL XML") from exc
+
+
+class OntologyExchangeService:
+    def __init__(self, repository, *, exchange=None, commit=lambda: None, rollback=lambda: None):
+        self.repository = repository
+        self.exchange = exchange or OntologyExchange()
+        self.commit = commit
+        self.rollback = rollback
+
+    def export(self, ontology_uid, format_name):
+        version = self.repository.latest_version(ontology_uid)
+        if version is None or version.status not in {"published", "superseded"}:
+            raise LookupError("published ontology version was not found")
+        graph = version.graph_document.to_dict()
+        if format_name == "json":
+            content = self.exchange.export_json(ontology_uid, version.version, graph)
+            return content, "application/json", f"{ontology_uid}-v{version.version}.json"
+        if format_name in {"rdf", "owl"}:
+            content = self.exchange.export_owl(ontology_uid, version.version, graph)
+            return content, "application/rdf+xml", f"{ontology_uid}-v{version.version}.owl"
+        raise ExchangeInvalid("unsupported ontology exchange format")
+
+    def import_document(self, content, format_name, actor_uid):
+        try:
+            imported = (
+                self.exchange.import_json(content)
+                if format_name == "json"
+                else self.exchange.import_owl(content)
+            )
+            code = "IMPORTED_" + imported["ontology_uid"].replace("-", "_").upper()
+            ontology = self.repository.create(
+                code=code,
+                name=code,
+                owner_uid=actor_uid,
+                domain_links=imported["graph_document"].get("domain_links") or [],
+                created_by=actor_uid,
+            )
+            version = self.repository.save_draft(
+                ontology.uid,
+                imported["graph_document"],
+                expected_revision=0,
+                actor_uid=actor_uid,
+            )
+            self.commit()
+            return {
+                "ontology_uid": ontology.uid,
+                "version_uid": version.uid,
+                "version": version.version,
+                "format": format_name,
+                "actor_uid": actor_uid,
+            }
+        except Exception:
+            self.rollback()
+            raise

app/core/data_research/ontology/knowledge_sync.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from app.core.knowledge.document_builder import build_document, chunk_document
+
+
+class OntologyKnowledgeSync:
+    def __init__(self, repository, *, max_chunk_chars=1200):
+        self.repository = repository
+        self.max_chunk_chars = int(max_chunk_chars)
+
+    def sync(self, published_version):
+        if published_version.get("status") != "published":
+            raise ValueError("only a published ontology version can be synchronized")
+        source = {
+            "uid": str(published_version["uid"]),
+            "name": published_version.get("name") or published_version["uid"],
+            "version": int(published_version["version"]),
+            "status": "published",
+            "graph_document": published_version.get("graph_document") or {},
+            "token": published_version.get("token"),
+        }
+        document = build_document("ontology", source)
+        changed = self.repository.upsert_document(document)
+        key = (document["object_uid"], document["object_version"])
+        if changed or key not in getattr(self.repository, "chunks", {}):
+            self.repository.replace_chunks(
+                key,
+                chunk_document(document["content"], self.max_chunk_chars),
+            )
+        return {"changed": changed, "document": document}

app/core/data_research/ontology/models.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+GRAPH_SECTIONS = (
+    "classes",
+    "properties",
+    "relations",
+    "constraints",
+    "domain_links",
+    "element_mappings",
+)
+
+
+@dataclass(frozen=True)
+class GraphDocument:
+    classes: tuple[dict[str, Any], ...] = ()
+    properties: tuple[dict[str, Any], ...] = ()
+    relations: tuple[dict[str, Any], ...] = ()
+    constraints: tuple[dict[str, Any], ...] = ()
+    domain_links: tuple[dict[str, Any], ...] = ()
+    element_mappings: tuple[dict[str, Any], ...] = ()
+
+    @classmethod
+    def from_dict(cls, payload):
+        payload = dict(payload or {})
+        return cls(**{name: tuple(dict(item) for item in payload.get(name, ())) for name in GRAPH_SECTIONS})
+
+    def to_dict(self):
+        return {name: [dict(item) for item in getattr(self, name)] for name in GRAPH_SECTIONS}
+
+
+@dataclass(frozen=True)
+class DomainLink:
+    domain_uid: str
+    role: str
+
+
+@dataclass(frozen=True)
+class Ontology:
+    uid: str
+    code: str
+    name: str
+    owner_uid: str
+    domain_links: tuple[DomainLink, ...]
+    status: str = "draft"
+    draft_revision: int = 0
+    active_version_uid: str | None = None
+
+
+@dataclass(frozen=True)
+class OntologyVersion:
+    uid: str
+    ontology_uid: str
+    version: int
+    graph_document: GraphDocument
+    content_hash: str
+    status: str = "draft"
+    parent_version_uid: str | None = None
+    created_by: str | None = None
+
+
+@dataclass(frozen=True)
+class OntologyChangeSet:
+    uid: str
+    ontology_uid: str
+    changes: tuple[dict[str, Any], ...]
+    decisions: tuple[dict[str, Any], ...] = ()
+    status: str = "draft"
+
+
+@dataclass(frozen=True)
+class OntologyPublishRun:
+    uid: str
+    ontology_uid: str
+    version_uid: str
+    idempotency_key: str
+    status: str
+    validation_result: dict[str, Any]

app/core/data_research/ontology/projection.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+
+class OntologyGraphProjector:
+    STATEMENT = """
+    MERGE (ontology:Ontology {uid: $ontology_uid})
+    SET ontology.active_version_uid = $version_uid,
+        ontology.active_version = $version
+    WITH ontology
+    MERGE (version:OntologyVersion {uid: $version_uid})
+    SET version.version = $version,
+        version.graph_document = $graph_document
+    MERGE (ontology)-[:ACTIVE_VERSION]->(version)
+    """
+
+    def __init__(self, graph_session):
+        self.graph_session = graph_session
+
+    def project(self, payload):
+        self.graph_session.run(
+            self.STATEMENT,
+            ontology_uid=str(payload["ontology_uid"]),
+            version_uid=str(payload["version_uid"]),
+            version=int(payload["version"]),
+            graph_document=payload["graph_document"],
+        )

app/core/data_research/ontology/publication.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+from app.core.data_research.ontology.models import GRAPH_SECTIONS, GraphDocument
+from app.core.data_research.ontology.validation import validate_graph
+
+
+class OntologyValidationFailed(ValueError):
+    def __init__(self, issues):
+        super().__init__("ontology validation failed")
+        self.issues = tuple(issues)
+
+
+def diff_graphs(before, after):
+    old = GraphDocument.from_dict(before).to_dict()
+    new = GraphDocument.from_dict(after).to_dict()
+    result = {}
+    for section in GRAPH_SECTIONS:
+        old_by_uid = {str(item.get("uid") or item.get("property_uid") or item.get("domain_uid")): item for item in old[section]}
+        new_by_uid = {str(item.get("uid") or item.get("property_uid") or item.get("domain_uid")): item for item in new[section]}
+        result[section] = {
+            "added": sorted(new_by_uid.keys() - old_by_uid.keys()),
+            "removed": sorted(old_by_uid.keys() - new_by_uid.keys()),
+            "changed": sorted(uid for uid in old_by_uid.keys() & new_by_uid.keys() if old_by_uid[uid] != new_by_uid[uid]),
+        }
+    return result
+
+
+class OntologyPublicationService:
+    def __init__(
+        self,
+        repository,
+        *,
+        outbox_enqueue=lambda **_event: None,
+        commit=lambda: None,
+        rollback=lambda: None,
+    ):
+        self.repository = repository
+        self.outbox_enqueue = outbox_enqueue
+        self.commit = commit
+        self.rollback = rollback
+        self._published_by_key = {}
+
+    def publish(self, version_uid, *, idempotency_key, actor_uid):
+        if idempotency_key in self._published_by_key:
+            return self._published_by_key[idempotency_key]
+        try:
+            version = self.repository.get_version(version_uid)
+            if version is None:
+                raise LookupError("ontology version was not found")
+            issues = validate_graph(version.graph_document.to_dict())
+            if issues:
+                raise OntologyValidationFailed(issues)
+            published = self.repository.mark_published(version_uid)
+            self.outbox_enqueue(
+                aggregate_type="ontology",
+                aggregate_id=published.ontology_uid,
+                event_type="ontology.version_published",
+                payload={
+                    "ontology_uid": published.ontology_uid,
+                    "version_uid": published.uid,
+                    "version": published.version,
+                    "status": "published",
+                    "graph_document": published.graph_document.to_dict(),
+                    "actor_uid": actor_uid,
+                },
+            )
+            self.commit()
+            self._published_by_key[idempotency_key] = published
+            return published
+        except Exception:
+            self.rollback()
+            raise
+
+    def create_rollback(
+        self,
+        ontology_uid,
+        *,
+        target_version_uid,
+        expected_revision,
+        actor_uid,
+    ):
+        target = self.repository.get_version(target_version_uid)
+        if target is None or target.ontology_uid != str(ontology_uid):
+            raise LookupError("rollback target was not found")
+        return self.repository.save_draft(
+            ontology_uid,
+            target.graph_document.to_dict(),
+            expected_revision=expected_revision,
+            actor_uid=actor_uid,
+        )
+
+
+class OntologyApplicationService:
+    def __init__(self, repository, *, publication=None, commit=lambda: None, rollback=lambda: None):
+        self.repository = repository
+        self.publication = publication or OntologyPublicationService(repository, commit=commit, rollback=rollback)
+        self.commit = commit
+        self.rollback_transaction = rollback
+
+    def list(self):
+        return self.repository.list()
+
+    def get(self, uid):
+        return self.repository.get(uid)
+
+    def list_versions(self, uid):
+        return self.repository.list_versions(uid)
+
+    def latest_version(self, uid):
+        return self.repository.latest_version(uid)
+
+    def create(self, payload, actor_uid):
+        try:
+            ontology = self.repository.create(
+                code=str(payload.get("code") or "").strip(),
+                name=str(payload.get("name") or "").strip(),
+                owner_uid=str(payload.get("owner_uid") or actor_uid),
+                domain_links=list(payload.get("domain_links") or []),
+                created_by=actor_uid,
+            )
+            self.commit()
+            return ontology
+        except Exception:
+            self.rollback_transaction()
+            raise
+
+    def save_draft(self, uid, graph, expected_revision, actor_uid):
+        try:
+            version = self.repository.save_draft(
+                uid, graph, expected_revision=expected_revision, actor_uid=actor_uid
+            )
+            self.commit()
+            return version
+        except Exception:
+            self.rollback_transaction()
+            raise
+
+    def validate(self, uid):
+        version = self.repository.latest_version(uid)
+        if version is None:
+            return validate_graph({})
+        return validate_graph(version.graph_document.to_dict())
+
+    def publish(self, uid, idempotency_key, actor_uid):
+        version = self.repository.latest_version(uid)
+        if version is None:
+            raise LookupError("ontology draft was not found")
+        return self.publication.publish(
+            version.uid, idempotency_key=idempotency_key, actor_uid=actor_uid
+        )
+
+    def diff(self, uid, left, right):
+        left_version = self.repository.get_version(left)
+        right_version = self.repository.get_version(right)
+        if not left_version or not right_version:
+            raise LookupError("ontology diff version was not found")
+        if left_version.ontology_uid != str(uid) or right_version.ontology_uid != str(uid):
+            raise LookupError("ontology diff version mismatch")
+        return diff_graphs(
+            left_version.graph_document.to_dict(), right_version.graph_document.to_dict()
+        )
+
+    def rollback(self, uid, target_version_uid, expected_revision, actor_uid):
+        try:
+            version = self.publication.create_rollback(
+                uid,
+                target_version_uid=target_version_uid,
+                expected_revision=expected_revision,
+                actor_uid=actor_uid,
+            )
+            self.commit()
+            return version
+        except Exception:
+            self.rollback_transaction()
+            raise

app/core/data_research/ontology/query.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+
+SAFE_FIELDS = (
+    "path_uid",
+    "ontology_uid",
+    "property_uid",
+    "data_element_uid",
+    "field_uid",
+    "evidence_uid",
+    "business_domain_uid",
+)
+
+
+class SemanticQueryService:
+    def __init__(self, repository):
+        self.repository = repository
+
+    def trace_property(self, property_uid, *, allowed_domains, limit=20, after_uid=None):
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1 or limit > 50:
+            raise ValueError("limit must be between 1 and 50")
+        rows = self.repository.find_property_paths(
+            str(property_uid), depth=3, limit=limit + 1, after_uid=after_uid
+        )
+        safe = []
+        for row in sorted(rows, key=lambda item: str(item.get("path_uid"))):
+            if row.get("ontology_status") != "published":
+                continue
+            domain = str(row.get("business_domain_uid") or "")
+            if "*" not in allowed_domains and domain not in allowed_domains:
+                continue
+            safe.append({key: row.get(key) for key in SAFE_FIELDS if row.get(key) is not None})
+        truncated = len(safe) > limit
+        items = safe[:limit]
+        return {
+            "items": items,
+            "count": len(items),
+            "truncated": truncated,
+            "next_cursor": items[-1]["path_uid"] if truncated and items else None,
+        }
+
+
+class Neo4jSemanticRepository:
+    STATEMENT = """
+    MATCH (ontology:Ontology)-[:ACTIVE_VERSION]->(:OntologyVersion)
+    MATCH (ontology)-[:HAS_PROPERTY]->(property:OntologyProperty {uid: $property_uid})
+    OPTIONAL MATCH (property)-[:MAPS_TO]->(element:DataElement)
+    OPTIONAL MATCH (element)-[:IMPLEMENTED_BY]->(field:PhysicalField)
+    OPTIONAL MATCH (field)-[:HAS_EVIDENCE]->(evidence:Evidence)
+    OPTIONAL MATCH (ontology)-[:SERVES_DOMAIN]->(domain:BusinessDomain)
+    WHERE $after_uid IS NULL OR property.uid > $after_uid
+    RETURN property.uid AS path_uid, ontology.uid AS ontology_uid,
+           'published' AS ontology_status, property.uid AS property_uid,
+           element.uid AS data_element_uid, field.uid AS field_uid,
+           evidence.uid AS evidence_uid, domain.uid AS business_domain_uid
+    ORDER BY path_uid LIMIT $limit
+    """
+
+    def __init__(self, driver):
+        self.driver = driver
+
+    def find_property_paths(self, property_uid, *, depth, limit, after_uid):
+        if int(depth) != 3:
+            raise ValueError("semantic traversal depth is fixed")
+        with self.driver.get_session() as session:
+            result = session.run(
+                self.STATEMENT,
+                property_uid=str(property_uid),
+                after_uid=after_uid,
+                limit=int(limit),
+            )
+            return [dict(record) for record in result]

app/core/data_research/ontology/repository.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import replace
+
+from app.core.common.identifiers import new_governance_uid
+from app.core.data_research.ontology.models import (
+    DomainLink,
+    GraphDocument,
+    Ontology,
+    OntologyVersion,
+)
+
+
+class OntologyRepositoryError(ValueError):
+    pass
+
+
+class OntologyConflict(OntologyRepositoryError):
+    pass
+
+
+class OntologyImmutable(OntologyRepositoryError):
+    pass
+
+
+def canonical_graph_hash(graph):
+    document = GraphDocument.from_dict(graph).to_dict()
+    encoded = json.dumps(document, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+class MemoryOntologyRepository:
+    def __init__(self, *, uid_factory=new_governance_uid):
+        self.uid_factory = uid_factory
+        self.ontologies = {}
+        self.versions = {}
+
+    def create(self, *, code, name, owner_uid, domain_links, created_by=None):
+        if any(item.code == code for item in self.ontologies.values()):
+            raise OntologyConflict("ontology code already exists")
+        links = tuple(DomainLink(str(item["domain_uid"]), str(item["role"])) for item in domain_links)
+        ontology = Ontology(
+            uid=self.uid_factory(),
+            code=str(code),
+            name=str(name),
+            owner_uid=str(owner_uid),
+            domain_links=links,
+        )
+        self.ontologies[ontology.uid] = ontology
+        return ontology
+
+    def get(self, uid):
+        return self.ontologies.get(str(uid))
+
+    def get_version(self, uid):
+        return self.versions.get(str(uid))
+
+    def list_versions(self, ontology_uid):
+        return sorted(
+            (
+                item
+                for item in self.versions.values()
+                if item.ontology_uid == str(ontology_uid)
+            ),
+            key=lambda item: item.version,
+            reverse=True,
+        )
+
+    def latest_version(self, ontology_uid):
+        versions = self.list_versions(ontology_uid)
+        return versions[0] if versions else None
+
+    def save_draft(self, ontology_uid, graph, *, expected_revision, actor_uid):
+        ontology = self.get(ontology_uid)
+        if ontology is None:
+            raise LookupError("ontology was not found")
+        if ontology.draft_revision != int(expected_revision):
+            raise OntologyConflict("ontology draft revision conflict")
+        version_number = 1 + max(
+            (item.version for item in self.versions.values() if item.ontology_uid == ontology.uid),
+            default=0,
+        )
+        version = OntologyVersion(
+            uid=self.uid_factory(),
+            ontology_uid=ontology.uid,
+            version=version_number,
+            parent_version_uid=ontology.active_version_uid,
+            graph_document=GraphDocument.from_dict(graph),
+            content_hash=canonical_graph_hash(graph),
+            created_by=actor_uid,
+        )
+        self.versions[version.uid] = version
+        self.ontologies[ontology.uid] = replace(ontology, draft_revision=ontology.draft_revision + 1)
+        return version
+
+    def mark_published(self, version_uid):
+        version = self.get_version(version_uid)
+        if version is None:
+            raise LookupError("ontology version was not found")
+        published = replace(version, status="published")
+        self.versions[published.uid] = published
+        ontology = self.get(published.ontology_uid)
+        self.ontologies[ontology.uid] = replace(
+            ontology, status="published", active_version_uid=published.uid
+        )
+        return published
+
+    def replace_version(self, version_uid, graph):
+        version = self.get_version(version_uid)
+        if version.status == "published":
+            raise OntologyImmutable("published ontology version is immutable")
+        updated = replace(
+            version,
+            graph_document=GraphDocument.from_dict(graph),
+            content_hash=canonical_graph_hash(graph),
+        )
+        self.versions[updated.uid] = updated
+        return updated
+
+
+class SqlAlchemyOntologyRepository:
+    def __init__(self, session, *, uid_factory=new_governance_uid):
+        self.session = session
+        self.uid_factory = uid_factory
+
+    @staticmethod
+    def _version(model):
+        return OntologyVersion(
+            uid=str(model.uid),
+            ontology_uid=str(model.ontology_uid),
+            version=int(model.version),
+            parent_version_uid=str(model.parent_version_uid) if model.parent_version_uid else None,
+            status=model.status,
+            graph_document=GraphDocument.from_dict(model.graph_document),
+            content_hash=model.content_hash,
+            created_by=model.created_by,
+        )
+
+    def _ontology(self, model):
+        from app.models.data_research import OntologyDomainLinkModel
+
+        links = self.session.query(OntologyDomainLinkModel).filter_by(
+            ontology_uid=str(model.uid)
+        ).all()
+        return Ontology(
+            uid=str(model.uid),
+            code=model.code,
+            name=model.name,
+            owner_uid=model.owner_uid,
+            status=model.status,
+            draft_revision=int(model.draft_revision),
+            active_version_uid=str(model.active_version_uid) if model.active_version_uid else None,
+            domain_links=tuple(DomainLink(str(item.domain_uid), item.role) for item in links),
+        )
+
+    def list(self):
+        from app.models.data_research import OntologyModel
+
+        return [self._ontology(model) for model in self.session.query(OntologyModel).order_by(OntologyModel.code).all()]
+
+    def create(self, *, code, name, owner_uid, domain_links, created_by=None):
+        from app.models.data_research import OntologyDomainLinkModel, OntologyModel
+
+        if self.session.query(OntologyModel).filter_by(code=str(code)).first():
+            raise OntologyConflict("ontology code already exists")
+        uid = self.uid_factory()
+        model = OntologyModel(
+            uid=uid,
+            code=str(code),
+            name=str(name),
+            owner_uid=str(owner_uid),
+            created_by=created_by,
+        )
+        self.session.add(model)
+        # The links are written with explicit foreign-key values rather than an
+        # ORM relationship, so persist the parent before flushing link rows.
+        self.session.flush()
+        for link in domain_links:
+            self.session.add(
+                OntologyDomainLinkModel(
+                    ontology_uid=uid,
+                    domain_uid=str(link["domain_uid"]),
+                    role=str(link["role"]),
+                )
+            )
+        self.session.flush()
+        return self._ontology(model)
+
+    def get(self, uid):
+        from app.models.data_research import OntologyModel
+
+        model = self.session.get(OntologyModel, str(uid))
+        return self._ontology(model) if model is not None else None
+
+    def get_version(self, uid):
+        from app.models.data_research import OntologyVersionModel
+
+        model = self.session.get(OntologyVersionModel, str(uid))
+        return self._version(model) if model is not None else None
+
+    def latest_version(self, ontology_uid):
+        from app.models.data_research import OntologyVersionModel
+
+        model = self.session.query(OntologyVersionModel).filter_by(
+            ontology_uid=str(ontology_uid)
+        ).order_by(OntologyVersionModel.version.desc()).first()
+        return self._version(model) if model is not None else None
+
+    def list_versions(self, ontology_uid):
+        from app.models.data_research import OntologyVersionModel
+
+        models = (
+            self.session.query(OntologyVersionModel)
+            .filter_by(ontology_uid=str(ontology_uid))
+            .order_by(OntologyVersionModel.version.desc())
+            .all()
+        )
+        return [self._version(model) for model in models]
+
+    def save_draft(self, ontology_uid, graph, *, expected_revision, actor_uid):
+        from app.models.data_research import OntologyModel, OntologyVersionModel
+
+        model = self.session.get(OntologyModel, str(ontology_uid))
+        if model is None:
+            raise LookupError("ontology was not found")
+        if int(model.draft_revision) != int(expected_revision):
+            raise OntologyConflict("ontology draft revision conflict")
+        latest = self.latest_version(ontology_uid)
+        version = OntologyVersionModel(
+            uid=self.uid_factory(),
+            ontology_uid=str(ontology_uid),
+            version=(latest.version + 1) if latest else 1,
+            parent_version_uid=model.active_version_uid,
+            status="draft",
+            graph_document=GraphDocument.from_dict(graph).to_dict(),
+            content_hash=canonical_graph_hash(graph),
+            created_by=actor_uid,
+        )
+        model.draft_revision = int(model.draft_revision) + 1
+        self.session.add(version)
+        self.session.flush()
+        return self._version(version)
+
+    def mark_published(self, version_uid):
+        from app.models.data_research import OntologyModel, OntologyVersionModel
+
+        model = self.session.get(OntologyVersionModel, str(version_uid))
+        if model is None:
+            raise LookupError("ontology version was not found")
+        ontology = self.session.get(OntologyModel, str(model.ontology_uid))
+        if ontology.active_version_uid:
+            active = self.session.get(OntologyVersionModel, str(ontology.active_version_uid))
+            if active is not None:
+                active.status = "superseded"
+        model.status = "published"
+        ontology.status = "published"
+        ontology.active_version_uid = model.uid
+        self.session.flush()
+        return self._version(model)
+
+    def replace_version(self, version_uid, graph):
+        from app.models.data_research import OntologyVersionModel
+
+        model = self.session.get(OntologyVersionModel, str(version_uid))
+        if model.status == "published":
+            raise OntologyImmutable("published ontology version is immutable")
+        model.graph_document = GraphDocument.from_dict(graph).to_dict()
+        model.content_hash = canonical_graph_hash(graph)
+        self.session.flush()
+        return self._version(model)

app/core/data_research/ontology/suggestions.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+
+
+class SuggestionInvalid(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class Suggestion:
+    uid: str
+    kind: str
+    payload: dict
+    evidence_uids: tuple[str, ...]
+    confidence: float
+    source: str
+    model_version: str | None = None
+    prompt_version: str | None = None
+
+
+def _uid(kind, payload):
+    value = json.dumps([kind, payload], ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return "suggestion-" + hashlib.sha256(value.encode()).hexdigest()[:16]
+
+
+class OntologySuggestionEngine:
+    def __init__(self, *, ai_port=None, model_version=None, prompt_version=None):
+        self.ai_port = ai_port
+        self.model_version = model_version
+        self.prompt_version = prompt_version
+
+    def generate(self, *, domains, data_elements, foreign_keys, include_ai=False):
+        suggestions = []
+        for domain in sorted(domains, key=lambda item: str(item.get("uid"))):
+            payload = {"uid": f"class-{domain['uid']}", "name": domain.get("name") or domain["uid"]}
+            suggestions.append(Suggestion(_uid("class", payload), "class", payload, (str(domain["uid"]),), 0.98, "rule"))
+        for element in sorted(data_elements, key=lambda item: str(item.get("uid"))):
+            if element.get("status") != "published":
+                continue
+            payload = {
+                "uid": f"property-{element['uid']}",
+                "name": element.get("name") or element["uid"],
+                "data_element_uid": str(element["uid"]),
+                "aliases": list(element.get("aliases") or []),
+            }
+            suggestions.append(Suggestion(_uid("property", payload), "property", payload, (str(element["uid"]),), 0.92, "rule"))
+        for relation in sorted(foreign_keys, key=lambda item: (str(item.get("from_uid")), str(item.get("to_uid")))):
+            payload = {
+                "uid": f"relation-{relation['from_uid']}-{relation['to_uid']}",
+                "from_class_uid": str(relation["from_uid"]),
+                "to_class_uid": str(relation["to_uid"]),
+            }
+            suggestions.append(Suggestion(_uid("relation", payload), "relation", payload, (str(relation["evidence_uid"]),), 0.85, "rule"))
+        if include_ai:
+            if self.ai_port is None:
+                raise SuggestionInvalid("AI suggestion provider is not configured")
+            for item in self.ai_port({"domains": domains, "data_elements": data_elements, "foreign_keys": foreign_keys}):
+                evidence = tuple(str(value) for value in item.get("evidence_uids") or ())
+                if not evidence:
+                    raise SuggestionInvalid("AI suggestion evidence is required")
+                payload = dict(item.get("payload") or {})
+                suggestions.append(
+                    Suggestion(
+                        _uid(str(item["kind"]), payload),
+                        str(item["kind"]),
+                        payload,
+                        evidence,
+                        float(item.get("confidence") or 0),
+                        "ai",
+                        self.model_version,
+                        self.prompt_version,
+                    )
+                )
+        return sorted(suggestions, key=lambda item: (-item.confidence, item.uid))

app/core/data_research/ontology/validation.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from app.core.data_research.ontology.models import GraphDocument
+
+
+@dataclass(frozen=True)
+class ValidationIssue:
+    code: str
+    message: str
+    path: str | None = None
+
+
+def validate_graph(payload):
+    graph = GraphDocument.from_dict(payload).to_dict()
+    issues = []
+    classes = graph["classes"]
+    class_uids = {str(item.get("uid")) for item in classes}
+    names = set()
+    if not any(item.get("role") == "owner" for item in graph["domain_links"]):
+        issues.append(ValidationIssue("ONTOLOGY_OWNER_REQUIRED", "an owner domain is required", "domain_links"))
+    for index, item in enumerate(classes):
+        name = str(item.get("name") or "").casefold()
+        if name in names:
+            issues.append(ValidationIssue("ONTOLOGY_DUPLICATE_NAME", "class names must be unique", f"classes.{index}.name"))
+        names.add(name)
+        parent = item.get("parent_uid")
+        if parent and str(parent) not in class_uids:
+            issues.append(ValidationIssue("ONTOLOGY_DANGLING_EDGE", "parent class does not exist", f"classes.{index}.parent_uid"))
+    for index, relation in enumerate(graph["relations"]):
+        if str(relation.get("from_class_uid")) not in class_uids or str(relation.get("to_class_uid")) not in class_uids:
+            issues.append(ValidationIssue("ONTOLOGY_DANGLING_EDGE", "relation endpoint does not exist", f"relations.{index}"))
+    parents = {str(item.get("uid")): str(item.get("parent_uid")) for item in classes if item.get("parent_uid")}
+    for uid in parents:
+        seen = set()
+        current = uid
+        while current in parents:
+            if current in seen:
+                issues.append(ValidationIssue("ONTOLOGY_INHERITANCE_CYCLE", "class inheritance contains a cycle", "classes"))
+                parents = {}
+                break
+            seen.add(current)
+            current = parents[current]
+        if not parents:
+            break
+    mapped = {str(item.get("property_uid")) for item in graph["element_mappings"]}
+    for index, prop in enumerate(graph["properties"]):
+        if str(prop.get("class_uid")) not in class_uids:
+            issues.append(ValidationIssue("ONTOLOGY_DANGLING_EDGE", "property owner class does not exist", f"properties.{index}.class_uid"))
+        if str(prop.get("cardinality") or "1") not in {"0..1", "1", "0..*", "1..*"}:
+            issues.append(ValidationIssue("ONTOLOGY_INVALID_CARDINALITY", "invalid property cardinality", f"properties.{index}.cardinality"))
+        if bool(prop.get("required")) and str(prop.get("uid")) not in mapped and not prop.get("data_element_uid"):
+            issues.append(ValidationIssue("ONTOLOGY_REQUIRED_PROPERTY_UNMAPPED", "required property must map to a published data element", f"properties.{index}"))
+    unique = {}
+    for issue in issues:
+        unique.setdefault(issue.code, issue)
+    return list(unique.values())

app/core/data_research/operations.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+from datetime import timedelta
+
+
+METRIC_FIELDS = (
+    "job_states",
+    "duration_seconds_p95",
+    "failure_stage",
+    "candidate_acceptance_ratio",
+    "review_time_seconds_p95",
+    "ontology_coverage_ratio",
+    "projection_lag_seconds",
+)
+
+
+class DataResearchOperations:
+    def __init__(self, repository, *, max_concurrent_jobs=10):
+        if int(max_concurrent_jobs) < 1:
+            raise ValueError("max_concurrent_jobs must be positive")
+        self.repository = repository
+        self.max_concurrent_jobs = int(max_concurrent_jobs)
+
+    def can_start_job(self):
+        return int(self.repository.running_job_count()) < self.max_concurrent_jobs
+
+    def recover_stale_jobs(self, *, now, stale_after_seconds=900):
+        before = now - timedelta(seconds=int(stale_after_seconds))
+        recovered = []
+        recoverable = {"extracting", "normalizing", "matching"}
+        for job in self.repository.list_stale_jobs(before):
+            if job.get("status") not in recoverable:
+                continue
+            self.repository.recover_job(
+                str(job["uid"]),
+                "queued",
+                f"recovered stale {job['status']} job",
+            )
+            recovered.append(str(job["uid"]))
+        return recovered
+
+    def health(self):
+        raw = dict(self.repository.aggregate_metrics() or {})
+        return {
+            "can_start_job": self.can_start_job(),
+            "running_job_count": int(self.repository.running_job_count()),
+            "max_concurrent_jobs": self.max_concurrent_jobs,
+            "dead_letter_count": int(self.repository.dead_letter_count()),
+            "metrics": {key: raw.get(key) for key in METRIC_FIELDS},
+        }
+
+
+class DataResearchReconciler:
+    def __init__(self, control_plane, graph, storage, knowledge):
+        self.control_plane = control_plane
+        self.graph = graph
+        self.storage = storage
+        self.knowledge = knowledge
+
+    def reconcile(self, *, repair=False):
+        issues = []
+        active = self.graph.active_versions()
+        versions = self.control_plane.published_versions()
+        for version in versions:
+            if active.get(version["ontology_uid"]) != version["version_uid"]:
+                issues.append({"type": "projection_mismatch", **version})
+        for artifact in self.control_plane.artifacts():
+            if not self.storage.exists(artifact["storage_ref"]):
+                issues.append({"type": "artifact_missing", **artifact})
+        documents = {
+            (item["object_uid"], int(item["object_version"])): item
+            for item in self.control_plane.governance_documents()
+        }
+        for version in versions:
+            key = (version["ontology_uid"], int(version.get("version", 2)))
+            document = documents.get(key)
+            expected = self.knowledge.expected_hash(
+                version["ontology_uid"], version["version_uid"]
+            )
+            if document is None or document.get("content_hash") != expected:
+                issues.append(
+                    {
+                        "type": "knowledge_mismatch",
+                        **version,
+                        "expected_hash": expected,
+                    }
+                )
+        repair_plan = [
+            issue
+            for issue in issues
+            if issue["type"] in {"projection_mismatch", "knowledge_mismatch"}
+        ]
+        repaired = 0
+        if repair:
+            for issue in repair_plan:
+                if issue["type"] == "projection_mismatch":
+                    self.graph.repair_projection(issue)
+                else:
+                    self.knowledge.repair_document(issue)
+                repaired += 1
+        return {
+            "repair_mode": bool(repair),
+            "issues": issues,
+            "repair_plan": repair_plan,
+            "repaired": repaired,
+        }
+
+
+class SqlAlchemyOperationsRepository:
+    def __init__(self, session):
+        self.session = session
+
+    def running_job_count(self):
+        from app.models.data_research import IngestionJob
+
+        return self.session.query(IngestionJob).filter(
+            IngestionJob.status.in_(("queued", "extracting", "normalizing", "matching"))
+        ).count()
+
+    def list_stale_jobs(self, before):
+        from app.models.data_research import IngestionJob
+
+        return [model.to_dict() for model in self.session.query(IngestionJob).filter(
+            IngestionJob.updated_at < before,
+            IngestionJob.status.in_(("extracting", "normalizing", "matching")),
+        ).all()]
+
+    def recover_job(self, uid, status, safe_error):
+        from app.models.data_research import IngestionJob
+
+        model = self.session.get(IngestionJob, str(uid))
+        model.status = status
+        model.last_error = safe_error
+        self.session.flush()
+
+    def aggregate_metrics(self):
+        from app.models.data_research import IngestionJob
+
+        states = {}
+        for model in self.session.query(IngestionJob.status).all():
+            states[model[0]] = states.get(model[0], 0) + 1
+        return {
+            "job_states": states,
+            "duration_seconds_p95": None,
+            "failure_stage": {},
+            "candidate_acceptance_ratio": None,
+            "review_time_seconds_p95": None,
+            "ontology_coverage_ratio": None,
+            "projection_lag_seconds": None,
+        }
+
+    def dead_letter_count(self):
+        from sqlalchemy import text
+
+        return self.session.execute(
+            text("SELECT COUNT(*) FROM public.outbox_events WHERE status = 'dead_letter'")
+        ).scalar() or 0

app/core/data_research/repository.py

@@ -0,0 +1,206 @@
+from __future__ import annotations
+
+from dataclasses import replace
+
+from app.models.data_research import (
+    CandidateDecisionRecord,
+    DataElement,
+    DataElementVersion,
+    ExtractionCandidate,
+    IngestionJob,
+)
+
+
+class SqlAlchemyIngestionJobRepository:
+    def __init__(self, session):
+        self.session = session
+
+    @staticmethod
+    def _record(model):
+        from app.core.data_research.models import IngestionJobRecord
+
+        return IngestionJobRecord(
+            uid=str(model.uid),
+            source_uid=str(model.source_uid),
+            artifact_uid=str(model.artifact_uid) if model.artifact_uid else None,
+            job_type=model.job_type,
+            parser_version=model.parser_version,
+            idempotency_key=model.idempotency_key,
+            actor_uid=model.actor_uid,
+            parameters=dict(model.parameters or {}),
+            status=model.status,
+            statistics=dict(model.statistics or {}),
+            last_error=model.last_error,
+            force_rerun=bool(model.force_rerun),
+            created_at=model.created_at,
+            updated_at=model.updated_at,
+            started_at=model.started_at,
+            finished_at=model.finished_at,
+        )
+
+    def get(self, uid):
+        model = self.session.get(IngestionJob, str(uid))
+        return self._record(model) if model is not None else None
+
+    def get_by_idempotency_key(self, key):
+        model = self.session.query(IngestionJob).filter_by(
+            idempotency_key=str(key)
+        ).first()
+        return self._record(model) if model is not None else None
+
+    def list(self, filters=None):
+        filters = filters or {}
+        query = self.session.query(IngestionJob)
+        if filters.get("status"):
+            query = query.filter(IngestionJob.status == str(filters["status"]))
+        if filters.get("source_uid"):
+            query = query.filter(
+                IngestionJob.source_uid == str(filters["source_uid"])
+            )
+        return [
+            self._record(model)
+            for model in query.order_by(IngestionJob.created_at.desc()).all()
+        ]
+
+    def add(self, record):
+        model = IngestionJob(
+            uid=record.uid,
+            source_uid=record.source_uid,
+            artifact_uid=record.artifact_uid,
+            job_type=record.job_type,
+            status=record.status,
+            idempotency_key=record.idempotency_key,
+            parser_version=record.parser_version,
+            parameters=record.parameters,
+            statistics=record.statistics,
+            last_error=record.last_error,
+            actor_uid=record.actor_uid,
+            force_rerun=record.force_rerun,
+            started_at=record.started_at,
+            finished_at=record.finished_at,
+        )
+        self.session.add(model)
+        self.session.flush()
+        return self._record(model)
+
+    def save(self, record):
+        model = self.session.get(IngestionJob, str(record.uid))
+        if model is None:
+            return self.add(record)
+        for name in (
+            "status",
+            "statistics",
+            "last_error",
+            "started_at",
+            "finished_at",
+            "updated_at",
+        ):
+            setattr(model, name, getattr(record, name))
+        self.session.flush()
+        return self._record(model)
+
+
+class SqlAlchemyDataElementRepository:
+    def __init__(self, session):
+        self.session = session
+
+    @staticmethod
+    def _record(model):
+        from app.core.data_research.data_elements import DataElementRecord
+
+        version = max(model.versions, key=lambda item: item.version)
+        return DataElementRecord(
+            uid=str(model.uid),
+            code=model.code,
+            status=model.status,
+            current_version=int(model.current_version),
+            snapshot=dict(version.snapshot or {}),
+            created_by=model.created_by,
+            updated_by=version.created_by,
+        )
+
+    def get(self, uid):
+        model = self.session.get(DataElement, str(uid))
+        return self._record(model) if model is not None else None
+
+    def get_by_code(self, code):
+        model = self.session.query(DataElement).filter_by(code=str(code)).first()
+        return self._record(model) if model is not None else None
+
+    def list(self, *, status=None, business_domain_uid=None):
+        query = self.session.query(DataElement)
+        if status:
+            query = query.filter(DataElement.status == str(status))
+        if business_domain_uid:
+            query = query.filter(
+                DataElement.business_domain_uids.contains(
+                    [str(business_domain_uid)]
+                )
+            )
+        return [
+            self._record(model)
+            for model in query.order_by(DataElement.code.asc()).all()
+        ]
+
+    def save(self, record):
+        model = self.session.get(DataElement, str(record.uid))
+        if model is None:
+            model = DataElement(
+                uid=record.uid,
+                code=record.code,
+                current_version=record.current_version,
+                status=record.status,
+                business_domain_uids=list(
+                    record.snapshot.get("business_domain_uids") or []
+                ),
+                created_by=record.created_by,
+            )
+            self.session.add(model)
+        else:
+            model.current_version = record.current_version
+            model.status = record.status
+            model.business_domain_uids = list(
+                record.snapshot.get("business_domain_uids") or []
+            )
+        self.session.add(
+            DataElementVersion(
+                data_element_uid=record.uid,
+                version=record.current_version,
+                status=record.status,
+                snapshot=record.snapshot,
+                evidence_uids=list(record.snapshot.get("evidence_uids") or []),
+                created_by=record.updated_by or record.created_by,
+            )
+        )
+        self.session.flush()
+        return record
+
+
+class SqlAlchemyCandidateDecisionRepository:
+    def __init__(self, session):
+        self.session = session
+
+    def get(self, uid):
+        model = self.session.get(ExtractionCandidate, str(uid))
+        return model.to_dict() if model is not None else None
+
+    def save_decisions(self, decisions):
+        for decision in decisions:
+            self.session.add(
+                CandidateDecisionRecord(
+                    uid=decision.uid,
+                    candidate_uid=decision.candidate_uid,
+                    action=decision.action,
+                    data_element_uid=decision.data_element_uid,
+                    evidence_uids=list(decision.evidence_uids),
+                    actor_uid=decision.actor_uid,
+                    reason=decision.reason,
+                )
+            )
+        self.session.flush()
+
+    def commit(self):
+        self.session.commit()
+
+    def rollback(self):
+        self.session.rollback()

app/core/mcp/context.py

@@ -285,3 +285,32 @@ class ContextService:
                 _untrusted(value) for value in list(row.get("differences") or [])[:50]
             ],
         }
+
+    def get_semantic_property_context(
+        self,
+        identity,
+        property_uid,
+        *,
+        business_domain,
+        limit=20,
+    ):
+        identity.require_domain(business_domain)
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1 or limit > 50:
+            raise ValueError("limit must be between 1 and 50")
+        fields = (
+            "path_uid",
+            "property_uid",
+            "ontology_uid",
+            "business_domain_uid",
+            "data_element_uid",
+            "field_uid",
+            "evidence_uid",
+        )
+        safe = []
+        for row in self.repository.list_semantic_property_context(str(property_uid)):
+            if row.get("ontology_status") != "published":
+                continue
+            if row.get("business_domain_uid") != business_domain:
+                continue
+            safe.append({key: row.get(key) for key in fields if row.get(key) is not None})
+        return _page(safe, limit)

app/core/system/health.py

@@ -81,6 +81,21 @@ def check_system_health():
     except Exception as exc:
         logger.warning("Knowledge health unavailable: %s", exc)
         health_status["knowledge"] = {"status": "unavailable"}
+    try:
+        from app.core.data_research.operations import (
+            DataResearchOperations,
+            SqlAlchemyOperationsRepository,
+        )
+
+        health_status["data_research"] = DataResearchOperations(
+            SqlAlchemyOperationsRepository(db.session),
+            max_concurrent_jobs=int(
+                current_app.config.get("DATA_RESEARCH_MAX_CONCURRENT_JOBS", 10)
+            ),
+        ).health()
+    except Exception as exc:
+        logger.warning("Data-research health unavailable: %s", exc)
+        health_status["data_research"] = {"status": "unavailable"}
 
     # 检查所有组件是否都正常
     all_healthy = all([health_status["database"], health_status["neo4j"]])

app/core/system/permissions.py

@@ -24,6 +24,13 @@ DATAFLOW_CANARY = "dataflows:canary"
 DATAFLOW_ACTIVATE = "dataflows:activate"
 DATAFLOW_ROLLBACK = "dataflows:rollback"
 RULES_EXECUTE = "rules:execute"
+INGESTION_RUN = "ingestion:run"
+INGESTION_ADMIN = "ingestion:admin"
+EVIDENCE_DOWNLOAD = "evidence:download"
+DATA_ELEMENTS_EDIT = "data-elements:edit"
+DATA_ELEMENTS_PUBLISH = "data-elements:publish"
+ONTOLOGIES_EDIT = "ontologies:edit"
+ONTOLOGIES_PUBLISH = "ontologies:publish"
 
 ROLE_PERMISSIONS = {
     "viewer": frozenset({READ_GOVERNANCE, RULES_READ}),
@@ -36,6 +43,9 @@ ROLE_PERMISSIONS = {
             RULES_READ,
             RULES_EDIT,
             RULES_EXECUTE,
+            INGESTION_RUN,
+            DATA_ELEMENTS_EDIT,
+            ONTOLOGIES_EDIT,
         }
     ),
     "admin": frozenset(
@@ -57,6 +67,13 @@ ROLE_PERMISSIONS = {
             DATAFLOW_ACTIVATE,
             DATAFLOW_ROLLBACK,
             RULES_EXECUTE,
+            INGESTION_RUN,
+            INGESTION_ADMIN,
+            EVIDENCE_DOWNLOAD,
+            DATA_ELEMENTS_EDIT,
+            DATA_ELEMENTS_PUBLISH,
+            ONTOLOGIES_EDIT,
+            ONTOLOGIES_PUBLISH,
         }
     ),
 }
@@ -113,6 +130,40 @@ def permission_for_request(path: str, method: str) -> tuple[str, ...]:
         return (KNOWLEDGE_MANAGE,)
     if path.startswith("/api/system/users"):
         return (MANAGE_USERS,)
+    if path.startswith("/api/development/v1/ingestion-jobs"):
+        if method == "GET":
+            return (READ_GOVERNANCE,)
+        if path.endswith("/retry"):
+            return (INGESTION_ADMIN,)
+        if path.endswith("/cancel"):
+            return (INGESTION_RUN,)
+        if method == "POST":
+            return (INGESTION_RUN,)
+    if path.startswith("/api/development/v1/data-elements"):
+        if method == "GET":
+            return (READ_GOVERNANCE,)
+        return (DATA_ELEMENTS_EDIT,)
+    if path.startswith("/api/development/v1/candidate-decisions"):
+        return (DATA_ELEMENTS_EDIT,)
+    if path.startswith("/api/development/v1/sources/files"):
+        return (INGESTION_RUN,)
+    if path.startswith("/api/development/v1/evidence"):
+        return (READ_GOVERNANCE,)
+    if path.startswith("/api/development/v1/ontologies"):
+        if method == "GET":
+            return (READ_GOVERNANCE,)
+        if path.endswith("/publish") or path.endswith("/rollback"):
+            return (ONTOLOGIES_PUBLISH,)
+        return (ONTOLOGIES_EDIT,)
+    if path.startswith("/api/bd/"):
+        if path in {
+            "/api/bd/list",
+            "/api/bd/detail",
+            "/api/bd/graphall",
+            "/api/bd/search",
+        }:
+            return (READ_GOVERNANCE,)
+        return (EDIT_GOVERNANCE,)
     if path.startswith("/api/system/workbench"):
         return (READ_GOVERNANCE,)
     if path == "/api/datasource/pools" or (

app/models/__init__.py

@@ -2,10 +2,38 @@
 
 from app.models.data_product import DataOrder, DataProduct
 from app.models.metadata_review import MetadataReviewRecord, MetadataVersionHistory
+from app.models.data_research import (
+    EvidenceFragment,
+    ExtractionCandidate,
+    IngestionJob,
+    IngestionSource,
+    SourceArtifact,
+    DataElement,
+    DataElementVersion,
+    CandidateDecisionRecord,
+    OntologyModel,
+    OntologyVersionModel,
+    OntologyDomainLinkModel,
+    OntologyChangeSetModel,
+    OntologyPublishRunModel,
+)
 
 __all__ = [
     "DataOrder",
     "DataProduct",
     "MetadataReviewRecord",
     "MetadataVersionHistory",
+    "IngestionSource",
+    "IngestionJob",
+    "SourceArtifact",
+    "EvidenceFragment",
+    "ExtractionCandidate",
+    "DataElement",
+    "DataElementVersion",
+    "CandidateDecisionRecord",
+    "OntologyModel",
+    "OntologyVersionModel",
+    "OntologyDomainLinkModel",
+    "OntologyChangeSetModel",
+    "OntologyPublishRunModel",
 ]

app/models/data_research.py

@@ -0,0 +1,405 @@
+from __future__ import annotations
+
+from typing import Any
+
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+
+from app import db
+from app.core.common.identifiers import new_governance_uid
+from app.core.common.timezone_utils import now_china_naive
+
+
+JOB_STATUSES = (
+    "created",
+    "queued",
+    "extracting",
+    "normalizing",
+    "matching",
+    "awaiting_review",
+    "published",
+    "partial",
+    "failed",
+    "cancelled",
+)
+
+
+def _iso(value: Any) -> str | None:
+    return value.isoformat() if value is not None else None
+
+
+class IngestionSource(db.Model):
+    __tablename__ = "ingestion_sources"
+    __table_args__ = (
+        db.CheckConstraint(
+            "source_type IN ('database','file','ddl')",
+            name="ck_ingestion_source_type",
+        ),
+        db.CheckConstraint(
+            "status IN ('active','disabled')",
+            name="ck_ingestion_source_status",
+        ),
+        {"schema": "public"},
+    )
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    source_type = db.Column(db.String(20), nullable=False)
+    name = db.Column(db.String(300), nullable=False)
+    config = db.Column(JSONB, nullable=False, default=dict)
+    permission_scope = db.Column(JSONB, nullable=False, default=dict)
+    status = db.Column(db.String(20), nullable=False, default="active")
+    created_by = db.Column(db.String(100))
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+    updated_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "uid": str(self.uid),
+            "source_type": self.source_type,
+            "name": self.name,
+            "permission_scope": dict(self.permission_scope or {}),
+            "status": self.status,
+            "created_by": self.created_by,
+            "created_at": _iso(self.created_at),
+            "updated_at": _iso(self.updated_at),
+        }
+
+
+class SourceArtifact(db.Model):
+    __tablename__ = "source_artifacts"
+    __table_args__ = (
+        db.CheckConstraint("size_bytes >= 0", name="ck_source_artifact_size"),
+        db.UniqueConstraint(
+            "source_uid",
+            "content_hash",
+            "parser_version",
+            name="uq_source_artifact_parser_content",
+        ),
+        {"schema": "public"},
+    )
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    source_uid = db.Column(
+        UUID(as_uuid=False),
+        db.ForeignKey("public.ingestion_sources.uid"),
+        nullable=False,
+    )
+    filename = db.Column(db.String(500), nullable=False)
+    media_type = db.Column(db.String(200), nullable=False)
+    size_bytes = db.Column(db.BigInteger, nullable=False)
+    content_hash = db.Column(db.String(64), nullable=False)
+    storage_ref = db.Column(db.Text, nullable=False)
+    parser_version = db.Column(db.String(100), nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "uid": str(self.uid),
+            "source_uid": str(self.source_uid),
+            "filename": self.filename,
+            "media_type": self.media_type,
+            "size_bytes": int(self.size_bytes),
+            "content_hash": self.content_hash,
+            "parser_version": self.parser_version,
+            "created_at": _iso(self.created_at),
+        }
+
+
+class IngestionJob(db.Model):
+    __tablename__ = "ingestion_jobs"
+    __table_args__ = (
+        db.CheckConstraint(
+            "status IN (" + ",".join(f"'{value}'" for value in JOB_STATUSES) + ")",
+            name="ck_ingestion_job_status",
+        ),
+        db.UniqueConstraint(
+            "idempotency_key",
+            name="uq_ingestion_job_idempotency_key",
+        ),
+        {"schema": "public"},
+    )
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    source_uid = db.Column(
+        UUID(as_uuid=False),
+        db.ForeignKey("public.ingestion_sources.uid"),
+        nullable=False,
+    )
+    artifact_uid = db.Column(
+        UUID(as_uuid=False),
+        db.ForeignKey("public.source_artifacts.uid"),
+    )
+    job_type = db.Column(db.String(30), nullable=False)
+    status = db.Column(db.String(30), nullable=False, default="created")
+    idempotency_key = db.Column(db.String(64), nullable=False, unique=True)
+    parser_version = db.Column(db.String(100), nullable=False)
+    parameters = db.Column(JSONB, nullable=False, default=dict)
+    statistics = db.Column(JSONB, nullable=False, default=dict)
+    last_error = db.Column(db.String(1000))
+    actor_uid = db.Column(db.String(100))
+    force_rerun = db.Column(db.Boolean, nullable=False, default=False)
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+    updated_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+    started_at = db.Column(db.DateTime)
+    finished_at = db.Column(db.DateTime)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "uid": str(self.uid),
+            "source_uid": str(self.source_uid),
+            "artifact_uid": str(self.artifact_uid) if self.artifact_uid else None,
+            "job_type": self.job_type,
+            "status": self.status,
+            "parser_version": self.parser_version,
+            "parameters": dict(self.parameters or {}),
+            "statistics": dict(self.statistics or {}),
+            "last_error": self.last_error,
+            "actor_uid": self.actor_uid,
+            "created_at": _iso(self.created_at),
+            "updated_at": _iso(self.updated_at),
+            "started_at": _iso(self.started_at),
+            "finished_at": _iso(self.finished_at),
+        }
+
+
+class EvidenceFragment(db.Model):
+    __tablename__ = "evidence_fragments"
+    __table_args__ = (
+        db.CheckConstraint(
+            "confidence IS NULL OR (confidence >= 0 AND confidence <= 1)",
+            name="ck_evidence_confidence",
+        ),
+        {"schema": "public"},
+    )
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    job_uid = db.Column(
+        UUID(as_uuid=False),
+        db.ForeignKey("public.ingestion_jobs.uid", ondelete="CASCADE"),
+        nullable=False,
+    )
+    artifact_uid = db.Column(
+        UUID(as_uuid=False),
+        db.ForeignKey("public.source_artifacts.uid"),
+    )
+    locator = db.Column(JSONB, nullable=False)
+    excerpt = db.Column(db.Text)
+    confidence = db.Column(db.Float)
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "uid": str(self.uid),
+            "job_uid": str(self.job_uid),
+            "artifact_uid": str(self.artifact_uid) if self.artifact_uid else None,
+            "locator": dict(self.locator or {}),
+            "excerpt": self.excerpt,
+            "confidence": self.confidence,
+            "created_at": _iso(self.created_at),
+        }
+
+
+class ExtractionCandidate(db.Model):
+    __tablename__ = "extraction_candidates"
+    __table_args__ = (
+        db.CheckConstraint(
+            "confidence >= 0 AND confidence <= 1",
+            name="ck_candidate_confidence",
+        ),
+        db.CheckConstraint(
+            "status IN ('candidate','accepted','rejected','ignored')",
+            name="ck_candidate_status",
+        ),
+        {"schema": "public"},
+    )
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    job_uid = db.Column(
+        UUID(as_uuid=False),
+        db.ForeignKey("public.ingestion_jobs.uid", ondelete="CASCADE"),
+        nullable=False,
+    )
+    candidate_type = db.Column(db.String(40), nullable=False)
+    normalized_data = db.Column(JSONB, nullable=False)
+    evidence_uids = db.Column(JSONB, nullable=False, default=list)
+    confidence = db.Column(db.Float, nullable=False)
+    status = db.Column(db.String(20), nullable=False, default="candidate")
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+    updated_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "uid": str(self.uid),
+            "job_uid": str(self.job_uid),
+            "candidate_type": self.candidate_type,
+            "normalized_data": dict(self.normalized_data or {}),
+            "evidence_uids": list(self.evidence_uids or []),
+            "confidence": float(self.confidence),
+            "status": self.status,
+            "created_at": _iso(self.created_at),
+            "updated_at": _iso(self.updated_at),
+        }
+
+
+class DataElement(db.Model):
+    __tablename__ = "data_elements"
+    __table_args__ = (
+        db.CheckConstraint(
+            "status IN ('draft','in_review','published','deprecated','retired')",
+            name="ck_data_element_status",
+        ),
+        {"schema": "public"},
+    )
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    code = db.Column(db.String(120), nullable=False, unique=True)
+    current_version = db.Column(db.Integer, nullable=False, default=1)
+    status = db.Column(db.String(20), nullable=False, default="draft")
+    business_domain_uids = db.Column(JSONB, nullable=False, default=list)
+    created_by = db.Column(db.String(100))
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+    updated_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+    versions = db.relationship(
+        "DataElementVersion",
+        lazy="selectin",
+        order_by="DataElementVersion.version",
+        cascade="all, delete-orphan",
+    )
+
+
+class DataElementVersion(db.Model):
+    __tablename__ = "data_element_versions"
+    __table_args__ = (
+        db.UniqueConstraint(
+            "data_element_uid",
+            "version",
+            name="uq_data_element_version",
+        ),
+        {"schema": "public"},
+    )
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    data_element_uid = db.Column(
+        UUID(as_uuid=False),
+        db.ForeignKey("public.data_elements.uid"),
+        nullable=False,
+    )
+    version = db.Column(db.Integer, nullable=False)
+    status = db.Column(db.String(20), nullable=False)
+    snapshot = db.Column(JSONB, nullable=False)
+    evidence_uids = db.Column(JSONB, nullable=False, default=list)
+    created_by = db.Column(db.String(100))
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+
+
+class CandidateDecisionRecord(db.Model):
+    __tablename__ = "candidate_decisions"
+    __table_args__ = (
+        db.CheckConstraint(
+            "action IN ('reuse','create','map','ignore')",
+            name="ck_candidate_decision_action",
+        ),
+        db.UniqueConstraint("candidate_uid", name="uq_candidate_decision"),
+        {"schema": "public"},
+    )
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    candidate_uid = db.Column(
+        UUID(as_uuid=False),
+        db.ForeignKey("public.extraction_candidates.uid"),
+        nullable=False,
+    )
+    action = db.Column(db.String(20), nullable=False)
+    data_element_uid = db.Column(
+        UUID(as_uuid=False),
+        db.ForeignKey("public.data_elements.uid"),
+    )
+    evidence_uids = db.Column(JSONB, nullable=False, default=list)
+    payload = db.Column(JSONB, nullable=False, default=dict)
+    actor_uid = db.Column(db.String(100))
+    reason = db.Column(db.String(1000))
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+
+
+class OntologyModel(db.Model):
+    __tablename__ = "ontologies"
+    __table_args__ = ({"schema": "public"},)
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    code = db.Column(db.String(120), nullable=False, unique=True)
+    name = db.Column(db.String(300), nullable=False)
+    owner_uid = db.Column(db.String(100), nullable=False)
+    status = db.Column(db.String(20), nullable=False, default="draft")
+    draft_revision = db.Column(db.Integer, nullable=False, default=0)
+    active_version_uid = db.Column(UUID(as_uuid=False))
+    created_by = db.Column(db.String(100))
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+    updated_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+
+
+class OntologyVersionModel(db.Model):
+    __tablename__ = "ontology_versions"
+    __table_args__ = (
+        db.UniqueConstraint("ontology_uid", "version", name="uq_ontology_version"),
+        {"schema": "public"},
+    )
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    ontology_uid = db.Column(
+        UUID(as_uuid=False), db.ForeignKey("public.ontologies.uid"), nullable=False
+    )
+    version = db.Column(db.Integer, nullable=False)
+    parent_version_uid = db.Column(
+        UUID(as_uuid=False), db.ForeignKey("public.ontology_versions.uid")
+    )
+    status = db.Column(db.String(20), nullable=False, default="draft")
+    graph_document = db.Column(JSONB, nullable=False)
+    content_hash = db.Column(db.String(64), nullable=False)
+    created_by = db.Column(db.String(100))
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+    published_at = db.Column(db.DateTime)
+
+
+class OntologyDomainLinkModel(db.Model):
+    __tablename__ = "ontology_domain_links"
+    __table_args__ = ({"schema": "public"},)
+
+    ontology_uid = db.Column(
+        UUID(as_uuid=False), db.ForeignKey("public.ontologies.uid"), primary_key=True
+    )
+    domain_uid = db.Column(UUID(as_uuid=False), primary_key=True)
+    role = db.Column(db.String(20), nullable=False)
+
+
+class OntologyChangeSetModel(db.Model):
+    __tablename__ = "ontology_change_sets"
+    __table_args__ = ({"schema": "public"},)
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    ontology_uid = db.Column(
+        UUID(as_uuid=False), db.ForeignKey("public.ontologies.uid"), nullable=False
+    )
+    base_version_uid = db.Column(UUID(as_uuid=False))
+    status = db.Column(db.String(20), nullable=False, default="draft")
+    changes = db.Column(JSONB, nullable=False, default=list)
+    decisions = db.Column(JSONB, nullable=False, default=list)
+    created_by = db.Column(db.String(100))
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+
+
+class OntologyPublishRunModel(db.Model):
+    __tablename__ = "ontology_publish_runs"
+    __table_args__ = ({"schema": "public"},)
+
+    uid = db.Column(UUID(as_uuid=False), primary_key=True, default=new_governance_uid)
+    ontology_uid = db.Column(
+        UUID(as_uuid=False), db.ForeignKey("public.ontologies.uid"), nullable=False
+    )
+    version_uid = db.Column(
+        UUID(as_uuid=False), db.ForeignKey("public.ontology_versions.uid"), nullable=False
+    )
+    idempotency_key = db.Column(db.String(128), nullable=False, unique=True)
+    status = db.Column(db.String(20), nullable=False)
+    validation_result = db.Column(JSONB, nullable=False, default=dict)
+    actor_uid = db.Column(db.String(100))
+    created_at = db.Column(db.DateTime, nullable=False, default=now_china_naive)
+    finished_at = db.Column(db.DateTime)

app/runner/auth.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import hashlib
+import base64
 import json
 import time
 import uuid
@@ -126,6 +127,19 @@ class TaskTokenVerifier:
             raise ValueError("task replay window is invalid")
 
     def _verify(self, token, *, node, replay_only):
+        try:
+            segments = str(token).split(".")
+            if len(segments) != 3:
+                raise ValueError("segment count")
+            for segment in segments:
+                decoded = base64.urlsafe_b64decode(
+                    segment + "=" * (-len(segment) % 4)
+                )
+                canonical = base64.urlsafe_b64encode(decoded).rstrip(b"=").decode("ascii")
+                if canonical != segment:
+                    raise ValueError("non-canonical base64url")
+        except (TypeError, ValueError, UnicodeError) as exc:
+            raise TaskTokenInvalid("task token is invalid") from exc
         try:
             payload = jwt.decode(
                 token,

deploy/docker/nginx.conf

@@ -5,8 +5,11 @@ server {
     root /usr/share/nginx/html;
     index index.html;
 
+    resolver 127.0.0.11 valid=10s ipv6=off;
+    set $backend_upstream http://backend:5500;
+
     location /api/ {
-        proxy_pass http://backend:5500/api/;
+        proxy_pass $backend_upstream;
         proxy_http_version 1.1;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;

deployment/app/__init__.py

@@ -37,6 +37,7 @@ def create_app():
     from app.api.data_interface import bp as data_interface_bp
     from app.api.data_service import bp as data_service_bp
     from app.api.data_source import bp as data_source_bp
+    from app.api.data_development import bp as data_development_bp
     from app.api.graph import bp as graph_bp
     from app.api.meta_data import bp as meta_bp
     from app.api.system import bp as system_bp
@@ -46,6 +47,7 @@ def create_app():
     app.register_blueprint(graph_bp, url_prefix="/api/graph")
     app.register_blueprint(system_bp, url_prefix="/api/system")
     app.register_blueprint(data_source_bp, url_prefix="/api/datasource")
+    app.register_blueprint(data_development_bp, url_prefix="/api/development/v1")
     app.register_blueprint(data_flow_bp, url_prefix="/api/dataflow")
     app.register_blueprint(business_domain_bp, url_prefix="/api/bd")
     app.register_blueprint(data_factory_bp, url_prefix="/api/datafactory")

deployment/app/api/data_development/__init__.py

@@ -0,0 +1,6 @@
+from flask import Blueprint
+
+
+bp = Blueprint("data_development", __name__)
+
+from app.api.data_development import routes  # noqa: E402, F401

deployment/app/api/data_development/routes.py

@@ -0,0 +1,691 @@
+"""HTTP orchestration boundary for data-research ingestion jobs."""
+
+from __future__ import annotations
+
+import io
+import logging
+
+from flask import current_app, g, jsonify, request, send_file
+
+from app import db
+from app.api.data_development import bp
+from app.core.data_research.errors import DataResearchError
+from app.models.result import failed, success
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_ingestion_service():
+    from app.core.data_research.ingestion import IngestionService
+    from app.core.data_research.repository import SqlAlchemyIngestionJobRepository
+
+    return IngestionService(
+        SqlAlchemyIngestionJobRepository(db.session),
+        commit=db.session.commit,
+        rollback=db.session.rollback,
+    )
+
+
+def get_data_element_service():
+    from app.core.data_research.data_elements import DataElementService
+    from app.core.data_research.repository import SqlAlchemyDataElementRepository
+    from app.core.events.outbox import enqueue_outbox
+
+    return DataElementService(
+        SqlAlchemyDataElementRepository(db.session),
+        outbox_enqueue=lambda **event: enqueue_outbox(db.session, **event),
+    )
+
+
+def get_candidate_decision_service():
+    from app.core.data_research.candidate_decisions import CandidateDecisionService
+    from app.core.data_research.data_elements import DataElementService
+    from app.core.data_research.repository import (
+        SqlAlchemyCandidateDecisionRepository,
+        SqlAlchemyDataElementRepository,
+    )
+
+    return CandidateDecisionService(
+        SqlAlchemyCandidateDecisionRepository(db.session),
+        data_elements=DataElementService(SqlAlchemyDataElementRepository(db.session)),
+    )
+
+
+def _artifact_storage():
+    from minio import Minio
+
+    from app.core.data_research.artifacts import MinioArtifactStorage
+
+    client = Minio(
+        current_app.config["MINIO_HOST"],
+        access_key=current_app.config["MINIO_USER"],
+        secret_key=current_app.config["MINIO_PASSWORD"],
+        secure=bool(current_app.config.get("MINIO_SECURE")),
+    )
+    return MinioArtifactStorage(client, current_app.config["MINIO_BUCKET"])
+
+
+def get_artifact_service():
+    from app.core.common.identifiers import new_governance_uid
+    from app.core.data_research.artifacts import (
+        ArtifactService,
+        SqlAlchemyArtifactRepository,
+    )
+    from app.core.data_research.file_policy import FilePolicy
+
+    return ArtifactService(
+        SqlAlchemyArtifactRepository(db.session),
+        _artifact_storage(),
+        FilePolicy(
+            max_bytes=int(current_app.config.get("DATA_RESEARCH_FILE_MAX_BYTES", 25 * 1024 * 1024)),
+            max_pages=int(current_app.config.get("DATA_RESEARCH_FILE_MAX_PAGES", 500)),
+        ),
+        uid_factory=new_governance_uid,
+    )
+
+
+def get_evidence_service():
+    from app.core.data_research.artifacts import EvidenceService
+
+    return EvidenceService(db.session, _artifact_storage())
+
+
+def get_ontology_service():
+    from app.core.data_research.ontology.publication import (
+        OntologyApplicationService,
+        OntologyPublicationService,
+    )
+    from app.core.data_research.ontology.repository import SqlAlchemyOntologyRepository
+    from app.core.events.outbox import enqueue_outbox
+
+    repository = SqlAlchemyOntologyRepository(db.session)
+    publication = OntologyPublicationService(
+        repository,
+        outbox_enqueue=lambda **event: enqueue_outbox(db.session, **event),
+        commit=db.session.commit,
+        rollback=db.session.rollback,
+    )
+    return OntologyApplicationService(
+        repository,
+        publication=publication,
+        commit=db.session.commit,
+        rollback=db.session.rollback,
+    )
+
+
+def get_ontology_dynamic_service():
+    from app.core.data_research.ontology.change_sets import (
+        SqlAlchemyDynamicOntologyService,
+    )
+
+    return SqlAlchemyDynamicOntologyService(db.session)
+
+
+def get_ontology_exchange_service():
+    from app.core.data_research.ontology.exchange import OntologyExchangeService
+    from app.core.data_research.ontology.repository import SqlAlchemyOntologyRepository
+
+    return OntologyExchangeService(
+        SqlAlchemyOntologyRepository(db.session),
+        commit=db.session.commit,
+        rollback=db.session.rollback,
+    )
+
+
+def get_semantic_query_service():
+    from app.core.data_research.ontology.query import (
+        Neo4jSemanticRepository,
+        SemanticQueryService,
+    )
+    from app.services.neo4j_driver import neo4j_driver
+
+    return SemanticQueryService(Neo4jSemanticRepository(neo4j_driver))
+
+
+def _identity():
+    return getattr(g, "current_user", {}) or {}
+
+
+def _record(record):
+    return {
+        "uid": str(record.uid),
+        "source_uid": str(record.source_uid),
+        "artifact_uid": str(record.artifact_uid) if record.artifact_uid else None,
+        "job_type": record.job_type,
+        "parser_version": record.parser_version,
+        "status": record.status,
+        "parameters": dict(record.parameters or {}),
+        "statistics": dict(record.statistics or {}),
+        "last_error": record.last_error,
+        "actor_uid": record.actor_uid,
+        "created_at": record.created_at.isoformat() if record.created_at else None,
+        "updated_at": record.updated_at.isoformat() if record.updated_at else None,
+        "started_at": record.started_at.isoformat() if record.started_at else None,
+        "finished_at": record.finished_at.isoformat() if record.finished_at else None,
+    }
+
+
+def _element(record):
+    return {
+        "uid": str(record.uid),
+        "code": record.code,
+        "status": record.status,
+        "current_version": int(record.current_version),
+        "snapshot": dict(record.snapshot or {}),
+        "created_by": record.created_by,
+        "updated_by": record.updated_by,
+    }
+
+
+def _decision(record):
+    return {
+        "uid": str(record.uid),
+        "candidate_uid": str(record.candidate_uid),
+        "action": record.action,
+        "data_element_uid": (
+            str(record.data_element_uid) if record.data_element_uid else None
+        ),
+        "evidence_uids": list(record.evidence_uids),
+        "actor_uid": record.actor_uid,
+        "reason": record.reason,
+    }
+
+
+def _artifact(record):
+    return {
+        "uid": str(record.uid),
+        "source_uid": str(record.source_uid),
+        "filename": record.filename,
+        "media_type": record.media_type,
+        "size_bytes": int(record.size_bytes),
+        "content_hash": record.content_hash,
+        "parser_version": record.parser_version,
+    }
+
+
+def _ontology(record):
+    return {
+        "uid": str(record.uid),
+        "code": record.code,
+        "name": record.name,
+        "owner_uid": record.owner_uid,
+        "status": record.status,
+        "draft_revision": int(record.draft_revision),
+        "active_version_uid": record.active_version_uid,
+        "domain_links": [
+            {"domain_uid": link.domain_uid, "role": link.role}
+            for link in record.domain_links
+        ],
+    }
+
+
+def _ontology_version(record):
+    return {
+        "uid": str(record.uid),
+        "ontology_uid": str(record.ontology_uid),
+        "version": int(record.version),
+        "parent_version_uid": record.parent_version_uid,
+        "status": record.status,
+        "content_hash": record.content_hash,
+        "graph_document": record.graph_document.to_dict(),
+        "created_by": record.created_by,
+    }
+
+
+def _error(error):
+    if isinstance(error, DataResearchError):
+        return (
+            jsonify(
+                failed(
+                    str(error),
+                    code=error.http_status,
+                    error={"code": error.code},
+                )
+            ),
+            error.http_status,
+        )
+    logger.exception("data-research ingestion request failed")
+    return (
+        jsonify(
+            failed(
+                "数据采集任务处理失败",
+                code=500,
+                error={"code": "DATA_RESEARCH_ERROR"},
+            )
+        ),
+        500,
+    )
+
+
+@bp.route("/ingestion-jobs", methods=["POST"])
+def create_ingestion_job():
+    payload = request.get_json(silent=True) or {}
+    try:
+        record, created = get_ingestion_service().create_job(
+            payload,
+            actor_uid=_identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(_record(record))), 201 if created else 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ingestion-jobs", methods=["GET"])
+def list_ingestion_jobs():
+    filters = {
+        name: request.args.get(name)
+        for name in ("status", "source_uid")
+        if request.args.get(name)
+    }
+    try:
+        records = get_ingestion_service().list_jobs(filters)
+        return jsonify(
+            success({"records": [_record(item) for item in records], "total": len(records)})
+        ), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ingestion-jobs/<job_uid>", methods=["GET"])
+def get_ingestion_job(job_uid):
+    try:
+        return jsonify(success(_record(get_ingestion_service().get_job(job_uid)))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ingestion-jobs/<job_uid>/retry", methods=["POST"])
+def retry_ingestion_job(job_uid):
+    try:
+        return jsonify(success(_record(get_ingestion_service().retry(job_uid)))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ingestion-jobs/<job_uid>/cancel", methods=["POST"])
+def cancel_ingestion_job(job_uid):
+    try:
+        service = get_ingestion_service()
+        record = service.get_job(job_uid)
+        identity = _identity()
+        permissions = set(identity.get("permissions") or [])
+        actor_uid = identity.get("id") or identity.get("sub")
+        if record.actor_uid != actor_uid and "ingestion:admin" not in permissions:
+            return jsonify(failed("权限不足", code=403)), 403
+        return jsonify(success(_record(service.cancel(job_uid)))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/data-elements", methods=["GET"])
+def list_data_elements():
+    try:
+        records = get_data_element_service().list(
+            status=request.args.get("status"),
+            business_domain_uid=request.args.get("business_domain_uid"),
+        )
+        return jsonify(
+            success(
+                {
+                    "records": [_element(item) for item in records],
+                    "total": len(records),
+                }
+            )
+        ), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/data-elements", methods=["POST"])
+def create_data_element():
+    try:
+        record = get_data_element_service().create_draft(
+            request.get_json(silent=True) or {},
+            actor_uid=_identity().get("id") or _identity().get("sub"),
+        )
+        db.session.commit()
+        return jsonify(success(_element(record))), 201
+    except Exception as error:
+        db.session.rollback()
+        return _error(error)
+
+
+@bp.route("/data-elements/<element_uid>/transitions", methods=["POST"])
+def transition_data_element(element_uid):
+    payload = request.get_json(silent=True) or {}
+    identity = _identity()
+    target_status = str(payload.get("target_status") or "")
+    if (
+        target_status in {"published", "deprecated", "retired"}
+        and "data-elements:publish" not in set(identity.get("permissions") or [])
+    ):
+        return jsonify(failed("权限不足", code=403)), 403
+    try:
+        record = get_data_element_service().transition(
+            element_uid,
+            target_status,
+            expected_version=int(payload.get("expected_version")),
+            actor_uid=identity.get("id") or identity.get("sub"),
+        )
+        db.session.commit()
+        return jsonify(success(_element(record))), 200
+    except Exception as error:
+        db.session.rollback()
+        return _error(error)
+
+
+@bp.route("/candidate-decisions", methods=["POST"])
+def decide_candidates():
+    payload = request.get_json(silent=True) or {}
+    try:
+        records = get_candidate_decision_service().decide(
+            payload.get("decisions") or [],
+            actor_uid=_identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success([_decision(record) for record in records])), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/sources/files", methods=["POST"])
+def upload_source_file():
+    uploaded = request.files.get("file")
+    if uploaded is None:
+        return jsonify(failed("缺少上传文件", code=400)), 400
+    try:
+        record, created = get_artifact_service().store(
+            request.form.get("source_uid"),
+            uploaded.filename,
+            uploaded.mimetype,
+            uploaded.read(),
+            request.form.get("parser_version") or "auto-v1",
+        )
+        db.session.commit()
+        return jsonify(success(_artifact(record))), 201 if created else 200
+    except Exception as error:
+        db.session.rollback()
+        return _error(error)
+
+
+@bp.route("/evidence/<evidence_uid>", methods=["GET"])
+def get_evidence(evidence_uid):
+    identity = _identity()
+    try:
+        service = get_evidence_service()
+        if request.args.get("download") in {"1", "true", "yes"}:
+            if "evidence:download" not in set(identity.get("permissions") or []):
+                return jsonify(failed("权限不足", code=403)), 403
+            content, filename, media_type = service.download(evidence_uid)
+            return send_file(
+                io.BytesIO(content),
+                mimetype=media_type,
+                as_attachment=True,
+                download_name=filename,
+            )
+        preview = service.preview(evidence_uid)
+        from app.core.data_research.artifacts import redact_excerpt
+
+        preview["excerpt"] = redact_excerpt(preview.get("excerpt"))
+        return jsonify(success(preview)), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies", methods=["GET"])
+def list_ontologies():
+    try:
+        return jsonify(success([_ontology(item) for item in get_ontology_service().list()])), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies", methods=["POST"])
+def create_ontology():
+    try:
+        record = get_ontology_service().create(
+            request.get_json(silent=True) or {},
+            _identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(_ontology(record))), 201
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>", methods=["GET"])
+def get_ontology(ontology_uid):
+    try:
+        return jsonify(success(_ontology(get_ontology_service().get(ontology_uid)))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/versions", methods=["GET"])
+def list_ontology_versions(ontology_uid):
+    try:
+        return jsonify(
+            success(
+                [
+                    _ontology_version(item)
+                    for item in get_ontology_service().list_versions(ontology_uid)
+                ]
+            )
+        ), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/graph", methods=["GET"])
+def get_ontology_graph(ontology_uid):
+    try:
+        service = get_ontology_service()
+        ontology = service.get(ontology_uid)
+        version = service.latest_version(ontology_uid)
+        data = (
+            _ontology_version(version)
+            if version is not None
+            else {
+                "uid": None,
+                "ontology_uid": str(ontology_uid),
+                "version": 0,
+                "parent_version_uid": None,
+                "status": "draft",
+                "content_hash": None,
+                "graph_document": {
+                    name: []
+                    for name in (
+                        "classes",
+                        "properties",
+                        "relations",
+                        "constraints",
+                        "domain_links",
+                        "element_mappings",
+                    )
+                },
+                "created_by": None,
+            }
+        )
+        response = jsonify(success(data))
+        response.headers["ETag"] = f'"{int(ontology.draft_revision)}"'
+        return response, 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/graph", methods=["PATCH"])
+def save_ontology_graph(ontology_uid):
+    etag = str(request.headers.get("If-Match") or "").strip().strip('"')
+    if not etag.isdigit():
+        return jsonify(failed("缺少有效 If-Match 修订号", code=428)), 428
+    try:
+        version = get_ontology_service().save_draft(
+            ontology_uid,
+            request.get_json(silent=True) or {},
+            int(etag),
+            _identity().get("id") or _identity().get("sub"),
+        )
+        response = jsonify(success(_ontology_version(version)))
+        response.headers["ETag"] = f'"{int(etag) + 1}"'
+        return response, 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/validate", methods=["POST"])
+def validate_ontology(ontology_uid):
+    try:
+        issues = get_ontology_service().validate(ontology_uid)
+        data = [
+            {"code": item.code, "message": item.message, "path": item.path}
+            for item in issues
+        ]
+        return jsonify(success(data)), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/publish", methods=["POST"])
+def publish_ontology(ontology_uid):
+    try:
+        version = get_ontology_service().publish(
+            ontology_uid,
+            request.headers.get("Idempotency-Key") or f"{ontology_uid}:publish",
+            _identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(_ontology_version(version))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/diff", methods=["GET"])
+def diff_ontology(ontology_uid):
+    try:
+        return jsonify(success(get_ontology_service().diff(
+            ontology_uid, request.args.get("left"), request.args.get("right")
+        ))), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/rollback", methods=["POST"])
+def rollback_ontology(ontology_uid):
+    payload = request.get_json(silent=True) or {}
+    try:
+        version = get_ontology_service().rollback(
+            ontology_uid,
+            payload.get("target_version_uid"),
+            int(payload.get("expected_revision")),
+            _identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(_ontology_version(version))), 201
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/suggestions", methods=["POST"])
+def generate_ontology_suggestions(ontology_uid):
+    try:
+        result = get_ontology_dynamic_service().generate(
+            ontology_uid,
+            request.get_json(silent=True) or {},
+            _identity().get("id") or _identity().get("sub"),
+        )
+        records = (
+            result.get("suggestions") or []
+            if isinstance(result, dict)
+            else result
+        )
+        suggestions = [
+            item
+            if isinstance(item, dict)
+            else {
+                "uid": item.uid,
+                "kind": item.kind,
+                "payload": item.payload,
+                "evidence_uids": list(item.evidence_uids),
+                "confidence": item.confidence,
+                "source": item.source,
+                "model_version": item.model_version,
+                "prompt_version": item.prompt_version,
+            }
+            for item in records
+        ]
+        data = {
+            "change_set_uid": (
+                result.get("change_set_uid")
+                if isinstance(result, dict)
+                else None
+            ),
+            "suggestions": suggestions,
+        }
+        return jsonify(success(data)), 201
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route(
+    "/ontologies/<ontology_uid>/change-sets/<change_set_uid>/decisions",
+    methods=["POST"],
+)
+def decide_ontology_change_set(ontology_uid, change_set_uid):
+    payload = request.get_json(silent=True) or {}
+    try:
+        result = get_ontology_dynamic_service().decide(
+            ontology_uid,
+            change_set_uid,
+            payload.get("decisions") or [],
+            _identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(result)), 200
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/<ontology_uid>/export", methods=["GET"])
+def export_ontology(ontology_uid):
+    try:
+        content, media_type, filename = get_ontology_exchange_service().export(
+            ontology_uid, request.args.get("format") or "json"
+        )
+        return send_file(
+            io.BytesIO(content),
+            mimetype=media_type,
+            as_attachment=True,
+            download_name=filename,
+        )
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/ontologies/import", methods=["POST"])
+def import_ontology():
+    try:
+        uploaded = request.files.get("file")
+        content = uploaded.read() if uploaded is not None else request.get_data(cache=False)
+        result = get_ontology_exchange_service().import_document(
+            content,
+            request.args.get("format") or "json",
+            _identity().get("id") or _identity().get("sub"),
+        )
+        return jsonify(success(result)), 201
+    except Exception as error:
+        return _error(error)
+
+
+@bp.route("/semantic/properties/<property_uid>", methods=["GET"])
+def query_semantic_property(property_uid):
+    domain = str(request.args.get("business_domain_uid") or "").strip()
+    identity = _identity()
+    scoped = set(identity.get("business_domains") or [domain])
+    if "*" not in scoped and domain not in scoped:
+        return jsonify(failed("权限不足", code=403)), 403
+    try:
+        result = get_semantic_query_service().trace_property(
+            property_uid,
+            allowed_domains={domain},
+            limit=int(request.args.get("limit") or 20),
+            after_uid=request.args.get("after_uid"),
+        )
+        return jsonify(success(result)), 200
+    except Exception as error:
+        return _error(error)

deployment/app/commands/migrate_n8n_workflow.py

@@ -0,0 +1,72 @@
+"""Convert one exported n8n workflow into governed migration artifacts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from app.core.orchestration.migration.dual_run import (
+    MigrationBlocked,
+    N8nWorkflowConverter,
+)
+
+
+def convert_export(
+    workflow,
+    *,
+    dataflow_uid,
+    data_source_uids,
+):
+    try:
+        converted = N8nWorkflowConverter().convert(
+            workflow,
+            dataflow_uid=dataflow_uid,
+            data_source_uids=data_source_uids,
+        )
+    except MigrationBlocked as exc:
+        return exc.report
+    return {
+        "status": "converted",
+        "source_workflow_id": converted.source_workflow_id,
+        "definition_hash": converted.definition_hash,
+        "workflow_spec": converted.workflow_spec,
+        "schedule_plan": converted.schedule_plan,
+        "deployment_performed": False,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Convert one n8n export without deploying it or copying credentials."
+        )
+    )
+    parser.add_argument("--workflow-json", required=True)
+    parser.add_argument("--dataflow-uid", required=True)
+    parser.add_argument("--datasource-map-json", required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+
+    workflow = json.loads(
+        Path(args.workflow_json).read_text(encoding="utf-8")
+    )
+    mapping = json.loads(
+        Path(args.datasource_map_json).read_text(encoding="utf-8")
+    )
+    if not isinstance(mapping, dict):
+        raise ValueError("datasource map must be an object")
+    report = convert_export(
+        workflow,
+        dataflow_uid=args.dataflow_uid,
+        data_source_uids=mapping,
+    )
+    Path(args.output).write_text(
+        json.dumps(report, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    return 0 if report["status"] == "converted" else 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

deployment/app/commands/reconcile_data_research.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import argparse
+import json
+
+from app import create_app
+
+
+def run(reconciler, *, repair=False):
+    return reconciler.reconcile(repair=repair)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Reconcile data-research control and projection stores")
+    parser.add_argument("--repair", action="store_true", help="repair rebuildable projections")
+    args = parser.parse_args()
+    app = create_app()
+    with app.app_context():
+        factory = app.extensions.get("data_research_reconciler_factory")
+        if factory is None:
+            raise RuntimeError("data-research reconciliation adapters are not configured")
+        print(json.dumps(run(factory(), repair=args.repair), ensure_ascii=False, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()

deployment/app/commands/reconcile_dual_run.py

@@ -0,0 +1,60 @@
+"""Compare credential-safe n8n and Kestra execution snapshots."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from app.core.orchestration.migration.reconciliation import (
+    ExecutionSnapshot,
+    ReconciliationPolicy,
+    WorkflowReconciler,
+)
+
+
+def reconcile_exports(baseline, candidate, policy=None):
+    policy = policy or {}
+    if not isinstance(policy, dict):
+        raise ValueError("reconciliation policy must be an object")
+    return WorkflowReconciler().compare(
+        ExecutionSnapshot.from_dict(baseline),
+        ExecutionSnapshot.from_dict(candidate),
+        ReconciliationPolicy(
+            ignored_paths=policy.get("ignored_paths"),
+            numeric_tolerances=policy.get("numeric_tolerances"),
+        ),
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Reconcile bounded execution metrics without raw rows."
+    )
+    parser.add_argument("--n8n-snapshot", required=True)
+    parser.add_argument("--kestra-snapshot", required=True)
+    parser.add_argument("--policy-json")
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+
+    baseline = json.loads(
+        Path(args.n8n_snapshot).read_text(encoding="utf-8")
+    )
+    candidate = json.loads(
+        Path(args.kestra_snapshot).read_text(encoding="utf-8")
+    )
+    policy = (
+        json.loads(Path(args.policy_json).read_text(encoding="utf-8"))
+        if args.policy_json
+        else {}
+    )
+    report = reconcile_exports(baseline, candidate, policy)
+    Path(args.output).write_text(
+        json.dumps(report, ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+    return 0 if report["status"] == "passed" else 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

deployment/app/config/config.py

@@ -387,6 +387,15 @@ class BaseConfig:
         "docx",
         "doc",
     }
+    DATA_RESEARCH_FILE_MAX_BYTES = int(
+        os.environ.get("DATA_RESEARCH_FILE_MAX_BYTES", str(25 * 1024 * 1024))
+    )
+    DATA_RESEARCH_FILE_MAX_PAGES = int(
+        os.environ.get("DATA_RESEARCH_FILE_MAX_PAGES", "500")
+    )
+    DATA_RESEARCH_MAX_CONCURRENT_JOBS = int(
+        os.environ.get("DATA_RESEARCH_MAX_CONCURRENT_JOBS", "10")
+    )
 
     # 数据抽取配置
     DATA_EXTRACT_BATCH_SIZE = 1000  # 每批处理的记录数

deployment/app/core/__init__.py

@@ -1,12 +1,3 @@
-# app/core/__init__.py
-# 核心业务逻辑模块
-# 这里包含与数据库无关的纯业务逻辑
-
-# 导入核心功能模块
-from app.core import (
-    common,  # noqa: F401
-    llm,  # noqa: F401
-    meta_data,  # noqa: F401
-)
-
+# Import submodules explicitly at their use sites. Eager imports here pulled
+# graph and LLM dependencies into otherwise isolated Runner and MCP processes.
 __all__ = ["common", "llm", "meta_data"]

deployment/app/core/common/__init__.py

@@ -1,16 +1,3 @@
-"""
-通用工具函数模块
-提供项目中需要的各种通用功能
-"""
+"""Shared helpers; import concrete modules to avoid hidden dependencies."""
 
-from app.core.common.functions import (
-    delete_relationships,
-    update_or_create_node,
-    get_node_by_id_no_label
-)
-
-__all__ = [
-    'delete_relationships',
-    'update_or_create_node',
-    'get_node_by_id_no_label'
-]
+__all__ = []

deployment/app/core/data_flow/workflow_repository.py

@@ -82,15 +82,23 @@ def create_version(
             text(
                 "INSERT INTO public.dataflow_workflow_versions "
                 "(id, dataflow_uid, environment, version_no, n8n_workflow_id, "
-                "n8n_workflow_name, definition_hash, definition_snapshot, created_by) "
+                "n8n_workflow_name, definition_hash, definition_snapshot, created_by, "
+                "engine_type, engine_definition_id, engine_revision, "
+                "deployment_metadata) "
                 "VALUES (CAST(:id AS uuid), CAST(:uid AS uuid), :environment, :version_no, "
-                ":n8n_id, :name, :digest, CAST(:snapshot AS jsonb), CAST(:created_by AS uuid))"
+                ":n8n_id, :name, :digest, CAST(:snapshot AS jsonb), "
+                "CAST(:created_by AS uuid), :engine_type, :engine_definition_id, "
+                ":engine_revision, CAST(:deployment_metadata AS jsonb))"
             ),
             {
                 "id": version_id, "uid": dataflow_uid, "environment": environment,
                 "version_no": version_no, "n8n_id": n8n_id, "name": workflow.get("name"),
                 "digest": digest, "snapshot": json.dumps(snapshot, ensure_ascii=False),
                 "created_by": created_by,
+                "engine_type": "n8n",
+                "engine_definition_id": n8n_id,
+                "engine_revision": workflow.get("versionId"),
+                "deployment_metadata": "{}",
             },
         )
     except IntegrityError as exc:
@@ -103,11 +111,18 @@ def list_versions(session, dataflow_uid: str) -> list[dict[str, Any]]:
     rows = session.execute(
         text(
             "SELECT id::text, dataflow_uid::text, environment, version_no, n8n_workflow_id, "
-            "n8n_workflow_name, definition_hash, status, activation_error, created_at, activated_at "
+            "n8n_workflow_name, definition_hash, status, activation_error, created_at, "
+            "activated_at, engine_type, engine_definition_id, engine_revision, "
+            "deployment_metadata "
             "FROM public.dataflow_workflow_versions WHERE dataflow_uid = CAST(:uid AS uuid) "
             "ORDER BY environment, version_no DESC"
         ),
         {"uid": uid},
     )
-    keys = ("id", "dataflow_uid", "environment", "version_no", "n8n_workflow_id", "n8n_workflow_name", "definition_hash", "status", "activation_error", "created_at", "activated_at")
+    keys = (
+        "id", "dataflow_uid", "environment", "version_no", "n8n_workflow_id",
+        "n8n_workflow_name", "definition_hash", "status", "activation_error",
+        "created_at", "activated_at", "engine_type", "engine_definition_id",
+        "engine_revision", "deployment_metadata",
+    )
     return [{key: (value.isoformat() if hasattr(value, "isoformat") else value) for key, value in zip(keys, row)} for row in rows]

deployment/app/core/data_research/__init__.py

@@ -0,0 +1,5 @@
+"""Data-research ingestion, governance, and ontology services."""
+
+from app.core.data_research.ingestion import IngestionService
+
+__all__ = ["IngestionService"]

deployment/app/core/data_research/artifacts.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import hashlib
+import io
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ArtifactRecord:
+    uid: str
+    source_uid: str
+    filename: str
+    media_type: str
+    size_bytes: int
+    content_hash: str
+    storage_ref: str
+    parser_version: str
+
+
+class ArtifactService:
+    def __init__(self, repository, storage, file_policy, *, uid_factory):
+        self.repository = repository
+        self.storage = storage
+        self.file_policy = file_policy
+        self.uid_factory = uid_factory
+
+    def store(self, source_uid, filename, media_type, content, parser_version):
+        validated = self.file_policy.validate(filename, media_type, content)
+        digest = hashlib.sha256(content).hexdigest()
+        existing = self.repository.find(source_uid, digest, parser_version)
+        if existing is not None:
+            return existing, False
+        object_key = f"data-research/{source_uid}/{digest}"
+        storage_ref = self.storage.put(object_key, content, validated.media_type)
+        record = ArtifactRecord(
+            uid=self.uid_factory(),
+            source_uid=str(source_uid),
+            filename=validated.filename,
+            media_type=validated.media_type,
+            size_bytes=validated.size_bytes,
+            content_hash=digest,
+            storage_ref=storage_ref,
+            parser_version=str(parser_version),
+        )
+        return self.repository.save(record), True
+
+
+def redact_excerpt(value):
+    value = re.sub(r"(?<!\d)(1\d{2})\d{4}(\d{4})(?!\d)", r"\1****\2", str(value or ""))
+    value = re.sub(
+        r"(?i)\b(token|password|secret|api[_-]?key)\s*[=:]\s*[^\s,;]+",
+        lambda match: f"{match.group(1)}=[redacted]",
+        value,
+    )
+    return value
+
+
+class EvidenceService:
+    def __init__(self, session, storage):
+        self.session = session
+        self.storage = storage
+
+    def preview(self, uid):
+        from app.models.data_research import EvidenceFragment
+
+        model = self.session.get(EvidenceFragment, str(uid))
+        if model is None:
+            raise LookupError("evidence was not found")
+        data = model.to_dict()
+        data["excerpt"] = redact_excerpt(data.get("excerpt"))
+        return data
+
+    def download(self, uid):
+        from app.models.data_research import EvidenceFragment, SourceArtifact
+
+        evidence = self.session.get(EvidenceFragment, str(uid))
+        artifact = (
+            self.session.get(SourceArtifact, str(evidence.artifact_uid))
+            if evidence is not None and evidence.artifact_uid
+            else None
+        )
+        if artifact is None:
+            raise LookupError("evidence artifact was not found")
+        return self.storage.get(artifact.storage_ref), artifact.filename, artifact.media_type
+
+
+class SqlAlchemyArtifactRepository:
+    def __init__(self, session):
+        self.session = session
+
+    @staticmethod
+    def _record(model):
+        return ArtifactRecord(
+            uid=str(model.uid),
+            source_uid=str(model.source_uid),
+            filename=model.filename,
+            media_type=model.media_type,
+            size_bytes=int(model.size_bytes),
+            content_hash=model.content_hash,
+            storage_ref=model.storage_ref,
+            parser_version=model.parser_version,
+        )
+
+    def find(self, source_uid, content_hash, parser_version):
+        from app.models.data_research import SourceArtifact
+
+        model = self.session.query(SourceArtifact).filter_by(
+            source_uid=str(source_uid),
+            content_hash=content_hash,
+            parser_version=str(parser_version),
+        ).first()
+        return self._record(model) if model is not None else None
+
+    def save(self, record):
+        from app.models.data_research import SourceArtifact
+
+        model = SourceArtifact(**record.__dict__)
+        self.session.add(model)
+        self.session.flush()
+        return self._record(model)
+
+
+class MinioArtifactStorage:
+    def __init__(self, client, bucket):
+        self.client = client
+        self.bucket = bucket
+
+    def put(self, object_key, content, media_type):
+        self.client.put_object(
+            self.bucket,
+            object_key,
+            io.BytesIO(content),
+            len(content),
+            content_type=media_type,
+        )
+        return f"minio://{self.bucket}/{object_key}"
+
+    def get(self, storage_ref):
+        prefix = f"minio://{self.bucket}/"
+        if not str(storage_ref).startswith(prefix):
+            raise ValueError("invalid storage reference")
+        response = self.client.get_object(self.bucket, str(storage_ref)[len(prefix):])
+        try:
+            return response.read()
+        finally:
+            response.close()
+            response.release_conn()

deployment/app/core/data_research/candidate_decisions.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from app.core.common.identifiers import new_governance_uid
+from app.core.data_research.errors import CandidateDecisionInvalid
+
+
+ALLOWED_ACTIONS = frozenset({"reuse", "create", "map", "ignore"})
+
+
+@dataclass(frozen=True)
+class CandidateDecision:
+    uid: str
+    candidate_uid: str
+    action: str
+    data_element_uid: str | None
+    evidence_uids: tuple[str, ...]
+    actor_uid: str | None
+    reason: str | None = None
+
+
+class CandidateDecisionService:
+    def __init__(
+        self,
+        repository,
+        *,
+        data_elements,
+        uid_factory: Callable[[], str] = new_governance_uid,
+    ):
+        self.repository = repository
+        self.data_elements = data_elements
+        self.uid_factory = uid_factory
+
+    def _validate(self, item):
+        if not isinstance(item, dict):
+            raise CandidateDecisionInvalid("decision must be an object")
+        action = str(item.get("action") or "").strip()
+        if action not in ALLOWED_ACTIONS:
+            raise CandidateDecisionInvalid(f"unsupported action: {action}")
+        candidate_uid = str(item.get("candidate_uid") or "").strip()
+        candidate = self.repository.get(candidate_uid)
+        if candidate is None:
+            raise CandidateDecisionInvalid(f"candidate {candidate_uid} was not found")
+        data_element_uid = str(item.get("data_element_uid") or "").strip() or None
+        if action in {"reuse", "map"} and not data_element_uid:
+            raise CandidateDecisionInvalid("data_element_uid is required")
+        return item, candidate, action, data_element_uid
+
+    def decide(self, items, *, actor_uid):
+        try:
+            validated = [self._validate(item) for item in items]
+            decisions = []
+            for item, candidate, action, data_element_uid in validated:
+                evidence_uids = tuple(candidate.get("evidence_uids") or ())
+                if action == "create":
+                    payload = dict(candidate.get("normalized_data") or {})
+                    payload["evidence_uids"] = list(evidence_uids)
+                    element = self.data_elements.create_draft(payload, actor_uid)
+                    data_element_uid = str(element.uid)
+                decisions.append(
+                    CandidateDecision(
+                        uid=self.uid_factory(),
+                        candidate_uid=str(candidate["uid"]),
+                        action=action,
+                        data_element_uid=data_element_uid,
+                        evidence_uids=evidence_uids,
+                        actor_uid=actor_uid,
+                        reason=str(item.get("reason") or "").strip() or None,
+                    )
+                )
+            self.repository.save_decisions(decisions)
+            self.repository.commit()
+            return decisions
+        except Exception:
+            self.repository.rollback()
+            raise

deployment/app/core/data_research/catalog/__init__.py

@@ -0,0 +1,6 @@
+from app.core.data_research.catalog.service import (
+    CatalogCollectionService,
+    diff_snapshots,
+)
+
+__all__ = ["CatalogCollectionService", "diff_snapshots"]

deployment/app/core/data_research/catalog/base.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from collections import OrderedDict
+
+from app.core.data_research.catalog.models import (
+    CatalogAsset,
+    CatalogField,
+    CatalogSnapshot,
+)
+
+
+class BaseCatalogCollector:
+    database_type = ""
+
+    @staticmethod
+    def _asset_type(value: str) -> str:
+        return "view" if "VIEW" in str(value).upper() else "table"
+
+    def _snapshot(self, rows, data_source_uid: str):
+        grouped = OrderedDict()
+        for row in rows:
+            schema = str(row["schema_name"])
+            asset_name = str(row["asset_name"])
+            asset_key = f"{data_source_uid}:{schema}.{asset_name}"
+            entry = grouped.setdefault(
+                asset_key,
+                {
+                    "schema": schema,
+                    "name": asset_name,
+                    "asset_type": self._asset_type(row["asset_type"]),
+                    "comment": row.get("asset_comment"),
+                    "fields": [],
+                },
+            )
+            field_name = str(row["column_name"])
+            entry["fields"].append(
+                CatalogField(
+                    key=f"{asset_key}.{field_name}",
+                    schema=schema,
+                    asset=asset_name,
+                    name=field_name,
+                    ordinal_position=int(row["ordinal_position"]),
+                    data_type=str(row["data_type"]),
+                    nullable=str(row["is_nullable"]).upper() == "YES",
+                    default=row.get("column_default"),
+                    comment=row.get("column_comment"),
+                )
+            )
+        assets = tuple(
+            CatalogAsset(
+                key=key,
+                schema=value["schema"],
+                name=value["name"],
+                asset_type=value["asset_type"],
+                comment=value["comment"],
+                fields=tuple(
+                    sorted(value["fields"], key=lambda item: item.ordinal_position)
+                ),
+            )
+            for key, value in grouped.items()
+        )
+        return CatalogSnapshot(
+            data_source_uid=str(data_source_uid),
+            database_type=self.database_type,
+            assets=assets,
+        )

deployment/app/core/data_research/catalog/models.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class CatalogScope:
+    include_schemas: tuple[str, ...] = ()
+    exclude_schemas: tuple[str, ...] = ()
+    include_tables: tuple[str, ...] = ()
+    exclude_tables: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class CatalogField:
+    key: str
+    schema: str
+    asset: str
+    name: str
+    ordinal_position: int
+    data_type: str
+    nullable: bool
+    default: str | None = None
+    comment: str | None = None
+
+
+@dataclass(frozen=True)
+class CatalogAsset:
+    key: str
+    schema: str
+    name: str
+    asset_type: str
+    fields: tuple[CatalogField, ...]
+    comment: str | None = None
+
+
+@dataclass(frozen=True)
+class CatalogSnapshot:
+    data_source_uid: str
+    database_type: str
+    assets: tuple[CatalogAsset, ...]
+
+    @property
+    def content_hash(self) -> str:
+        payload = json.dumps(
+            asdict(self),
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        )
+        return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+@dataclass(frozen=True)
+class CatalogFieldChange:
+    before: CatalogField
+    after: CatalogField
+
+
+@dataclass(frozen=True)
+class CatalogDiff:
+    added: tuple[CatalogField, ...] = ()
+    removed: tuple[CatalogField, ...] = ()
+    changed: tuple[CatalogFieldChange, ...] = ()
+    renamed: tuple[CatalogFieldChange, ...] = ()

deployment/app/core/data_research/catalog/mysql.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from sqlalchemy import bindparam, text
+
+from app.core.data_research.catalog.base import BaseCatalogCollector
+
+
+MYSQL_CATALOG_SQL = """
+SELECT
+    c.TABLE_SCHEMA AS schema_name,
+    c.TABLE_NAME AS asset_name,
+    t.TABLE_TYPE AS asset_type,
+    t.TABLE_COMMENT AS asset_comment,
+    c.COLUMN_NAME AS column_name,
+    c.ORDINAL_POSITION AS ordinal_position,
+    c.COLUMN_TYPE AS data_type,
+    c.IS_NULLABLE AS is_nullable,
+    c.COLUMN_DEFAULT AS column_default,
+    c.COLUMN_COMMENT AS column_comment
+FROM information_schema.COLUMNS c
+JOIN information_schema.TABLES t
+  ON t.TABLE_SCHEMA = c.TABLE_SCHEMA AND t.TABLE_NAME = c.TABLE_NAME
+WHERE c.TABLE_SCHEMA = :database_name
+  AND (:include_table_count = 0 OR c.TABLE_NAME IN :include_tables)
+  AND (:exclude_table_count = 0 OR c.TABLE_NAME NOT IN :exclude_tables)
+ORDER BY c.TABLE_NAME, c.ORDINAL_POSITION
+"""
+
+
+class MySqlCatalogCollector(BaseCatalogCollector):
+    database_type = "mysql"
+
+    def collect(self, connection, *, data_source_uid, database_name, scope):
+        statement = text(MYSQL_CATALOG_SQL).bindparams(
+            bindparam("include_tables", expanding=True),
+            bindparam("exclude_tables", expanding=True),
+        )
+        parameters = {
+            "database_name": str(database_name),
+            "include_tables": list(scope.include_tables),
+            "exclude_tables": list(scope.exclude_tables),
+            "include_table_count": len(scope.include_tables),
+            "exclude_table_count": len(scope.exclude_tables),
+        }
+        rows = connection.execute(statement, parameters).mappings().all()
+        return self._snapshot(rows, str(data_source_uid))

deployment/app/core/data_research/catalog/postgresql.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from sqlalchemy import bindparam, text
+
+from app.core.data_research.catalog.base import BaseCatalogCollector
+
+
+POSTGRESQL_CATALOG_SQL = """
+SELECT
+    c.table_schema AS schema_name,
+    c.table_name AS asset_name,
+    t.table_type AS asset_type,
+    obj_description(pc.oid) AS asset_comment,
+    c.column_name,
+    c.ordinal_position,
+    c.data_type,
+    c.is_nullable,
+    c.column_default,
+    col_description(pc.oid, c.ordinal_position) AS column_comment
+FROM information_schema.columns c
+JOIN information_schema.tables t
+  ON t.table_schema = c.table_schema AND t.table_name = c.table_name
+LEFT JOIN pg_catalog.pg_namespace pn ON pn.nspname = c.table_schema
+LEFT JOIN pg_catalog.pg_class pc
+  ON pc.relnamespace = pn.oid AND pc.relname = c.table_name
+WHERE c.table_schema NOT IN ('pg_catalog', 'information_schema')
+  AND (:include_schema_count = 0 OR c.table_schema IN :include_schemas)
+  AND (:exclude_schema_count = 0 OR c.table_schema NOT IN :exclude_schemas)
+  AND (:include_table_count = 0 OR c.table_name IN :include_tables)
+  AND (:exclude_table_count = 0 OR c.table_name NOT IN :exclude_tables)
+ORDER BY c.table_schema, c.table_name, c.ordinal_position
+"""
+
+
+class PostgreSqlCatalogCollector(BaseCatalogCollector):
+    database_type = "postgresql"
+
+    def collect(self, connection, *, data_source_uid, scope):
+        statement = text(POSTGRESQL_CATALOG_SQL).bindparams(
+            bindparam("include_schemas", expanding=True),
+            bindparam("exclude_schemas", expanding=True),
+            bindparam("include_tables", expanding=True),
+            bindparam("exclude_tables", expanding=True),
+        )
+        parameters = {
+            "include_schemas": list(scope.include_schemas),
+            "exclude_schemas": list(scope.exclude_schemas),
+            "include_tables": list(scope.include_tables),
+            "exclude_tables": list(scope.exclude_tables),
+            "include_schema_count": len(scope.include_schemas),
+            "exclude_schema_count": len(scope.exclude_schemas),
+            "include_table_count": len(scope.include_tables),
+            "exclude_table_count": len(scope.exclude_tables),
+        }
+        rows = connection.execute(statement, parameters).mappings().all()
+        return self._snapshot(rows, str(data_source_uid))

deployment/app/core/data_research/catalog/service.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from app.core.data_research.catalog.models import (
+    CatalogDiff,
+    CatalogFieldChange,
+)
+
+
+def _fields(snapshot):
+    return {
+        field.key: field
+        for asset in snapshot.assets
+        for field in asset.fields
+    }
+
+
+def diff_snapshots(previous, current):
+    before = _fields(previous)
+    after = _fields(current)
+    changed = []
+    for key in sorted(set(before) & set(after)):
+        left = before[key]
+        right = after[key]
+        if (
+            left.data_type,
+            left.nullable,
+            left.default,
+            left.comment,
+        ) != (
+            right.data_type,
+            right.nullable,
+            right.default,
+            right.comment,
+        ):
+            changed.append(CatalogFieldChange(left, right))
+
+    removed = [before[key] for key in sorted(set(before) - set(after))]
+    added = [after[key] for key in sorted(set(after) - set(before))]
+    renamed = []
+    used_added = set()
+    used_removed = set()
+    for old in removed:
+        for new in added:
+            if new.key in used_added:
+                continue
+            if (
+                old.schema,
+                old.asset,
+                old.ordinal_position,
+                old.data_type,
+            ) == (
+                new.schema,
+                new.asset,
+                new.ordinal_position,
+                new.data_type,
+            ):
+                renamed.append(CatalogFieldChange(old, new))
+                used_removed.add(old.key)
+                used_added.add(new.key)
+                break
+    return CatalogDiff(
+        added=tuple(item for item in added if item.key not in used_added),
+        removed=tuple(item for item in removed if item.key not in used_removed),
+        changed=tuple(changed),
+        renamed=tuple(renamed),
+    )
+
+
+class CatalogCollectionService:
+    def __init__(self, manager, *, definition_resolver, collector_resolver):
+        self.manager = manager
+        self.definition_resolver = definition_resolver
+        self.collector_resolver = collector_resolver
+
+    def collect(self, data_source_uid, scope):
+        definition = self.definition_resolver(str(data_source_uid))
+        database_type = (
+            definition.get("database_type")
+            if isinstance(definition, dict)
+            else definition.database_type
+        )
+        database_name = (
+            definition.get("database")
+            if isinstance(definition, dict)
+            else definition.database
+        )
+        collector = self.collector_resolver(database_type)
+        with self.manager.connect(
+            str(data_source_uid),
+            "metadata_collection",
+        ) as connection:
+            kwargs = {
+                "data_source_uid": str(data_source_uid),
+                "scope": scope,
+            }
+            if database_type == "mysql":
+                kwargs["database_name"] = database_name
+            return collector.collect(connection, **kwargs)

deployment/app/core/data_research/data_elements.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from typing import Any, Callable
+
+from app.core.common.identifiers import new_governance_uid
+from app.core.data_research.errors import (
+    DataElementInvalid,
+    DataElementNotFound,
+    DataElementTransitionInvalid,
+    DataElementVersionConflict,
+)
+
+
+ELEMENT_TRANSITIONS = {
+    "draft": frozenset({"in_review"}),
+    "in_review": frozenset({"draft", "published"}),
+    "published": frozenset({"deprecated"}),
+    "deprecated": frozenset({"published", "retired"}),
+    "retired": frozenset(),
+}
+
+
+@dataclass(frozen=True)
+class DataElementRecord:
+    uid: str
+    code: str
+    status: str
+    current_version: int
+    snapshot: dict[str, Any]
+    created_by: str | None = None
+    updated_by: str | None = None
+
+
+def _required(payload, name):
+    value = str(payload.get(name) or "").strip()
+    if not value:
+        raise DataElementInvalid(f"{name} is required")
+    return value
+
+
+class DataElementService:
+    def __init__(
+        self,
+        repository,
+        *,
+        uid_factory: Callable[[], str] = new_governance_uid,
+        outbox_enqueue: Callable[..., Any] = lambda **_event: None,
+    ):
+        self.repository = repository
+        self.uid_factory = uid_factory
+        self.outbox_enqueue = outbox_enqueue
+
+    def list(self, *, status=None, business_domain_uid=None):
+        return self.repository.list(
+            status=status, business_domain_uid=business_domain_uid
+        )
+
+    def create_draft(self, payload, actor_uid):
+        if not isinstance(payload, dict):
+            raise DataElementInvalid("payload must be an object")
+        snapshot = {
+            "code": _required(payload, "code"),
+            "name_zh": _required(payload, "name_zh"),
+            "name_en": _required(payload, "name_en"),
+            "definition": _required(payload, "definition"),
+            "logical_type": _required(payload, "logical_type"),
+            "evidence_uids": list(payload.get("evidence_uids") or []),
+            "business_domain_uids": list(payload.get("business_domain_uids") or []),
+        }
+        for name in (
+            "aliases",
+            "format",
+            "value_domain",
+            "unit",
+            "sensitivity",
+            "quality_rules",
+        ):
+            if name in payload:
+                snapshot[name] = payload[name]
+        if self.repository.get_by_code(snapshot["code"]) is not None:
+            raise DataElementInvalid("code already exists")
+        return self.repository.save(
+            DataElementRecord(
+                uid=self.uid_factory(),
+                code=snapshot["code"],
+                status="draft",
+                current_version=1,
+                snapshot=snapshot,
+                created_by=actor_uid,
+                updated_by=actor_uid,
+            )
+        )
+
+    def transition(self, uid, target_status, *, expected_version, actor_uid):
+        record = self.repository.get(str(uid))
+        if record is None:
+            raise DataElementNotFound(f"data element {uid} was not found")
+        if int(expected_version) != int(record.current_version):
+            raise DataElementVersionConflict(
+                f"expected version {expected_version}, current version {record.current_version}"
+            )
+        if target_status not in ELEMENT_TRANSITIONS.get(record.status, frozenset()):
+            raise DataElementTransitionInvalid(
+                f"{record.status} -> {target_status} is not allowed"
+            )
+        updated = self.repository.save(
+            replace(
+                record,
+                status=target_status,
+                current_version=record.current_version + 1,
+                updated_by=actor_uid,
+            )
+        )
+        if target_status == "published":
+            self.outbox_enqueue(
+                aggregate_type="data_element",
+                aggregate_id=updated.uid,
+                event_type="data_element.version_published",
+                payload={"uid": updated.uid, "version": updated.current_version},
+            )
+        return updated

deployment/app/core/data_research/errors.py

@@ -0,0 +1,45 @@
+class DataResearchError(ValueError):
+    code = "DATA_RESEARCH_ERROR"
+    http_status = 400
+
+
+class IngestionPayloadInvalid(DataResearchError):
+    code = "INGESTION_PAYLOAD_INVALID"
+
+
+class InvalidJobTransition(DataResearchError):
+    code = "INGESTION_TRANSITION_INVALID"
+    http_status = 409
+
+
+class IngestionJobNotFound(DataResearchError):
+    code = "INGESTION_JOB_NOT_FOUND"
+    http_status = 404
+
+
+class DataElementInvalid(DataResearchError):
+    code = "DATA_ELEMENT_INVALID"
+
+
+class DataElementNotFound(DataResearchError):
+    code = "DATA_ELEMENT_NOT_FOUND"
+    http_status = 404
+
+
+class DataElementVersionConflict(DataResearchError):
+    code = "DATA_ELEMENT_VERSION_CONFLICT"
+    http_status = 409
+
+
+class DataElementTransitionInvalid(DataResearchError):
+    code = "DATA_ELEMENT_TRANSITION_INVALID"
+    http_status = 409
+
+
+class CandidateDecisionInvalid(DataResearchError):
+    code = "CANDIDATE_DECISION_INVALID"
+
+
+class DataElementProjectionInvalid(DataResearchError):
+    code = "DATA_ELEMENT_PROJECTION_INVALID"
+    http_status = 422

deployment/app/core/data_research/extractors/__init__.py

@@ -0,0 +1,3 @@
+from app.core.data_research.extractors.registry import default_registry
+
+__all__ = ["default_registry"]

deployment/app/core/data_research/extractors/base.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class ExtractionContext:
+    filename: str
+    media_type: str
+    max_bytes: int = 25 * 1024 * 1024
+
+
+@dataclass(frozen=True)
+class ExtractionEvidence:
+    locator: dict[str, Any]
+    excerpt: str | None = None
+    confidence: float | None = None
+
+
+@dataclass(frozen=True)
+class ExtractedItem:
+    candidate_type: str
+    data: dict[str, Any]
+    evidence: tuple[ExtractionEvidence, ...] = ()
+    confidence: float = 1.0
+
+
+@dataclass(frozen=True)
+class ExtractionBatch:
+    parser_version: str
+    content_hash: str
+    items: tuple[ExtractedItem, ...]
+    warnings: tuple[str, ...] = ()
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class BaseExtractor:
+    parser_version = ""
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        raise NotImplementedError
+
+    def extract(self, content: bytes, context: ExtractionContext) -> ExtractionBatch:
+        raise NotImplementedError
+
+    def _batch(
+        self,
+        content: bytes,
+        items: list[ExtractedItem],
+        *,
+        warnings: list[str] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> ExtractionBatch:
+        return ExtractionBatch(
+            parser_version=self.parser_version,
+            content_hash=hashlib.sha256(content).hexdigest(),
+            items=tuple(items),
+            warnings=tuple(warnings or ()),
+            metadata=dict(metadata or {}),
+        )

deployment/app/core/data_research/extractors/csv.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import csv
+import io
+import re
+from pathlib import Path
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+_INTEGER = re.compile(r"^[+-]?\d+$")
+_DECIMAL = re.compile(r"^[+-]?(?:\d+\.\d*|\d*\.\d+)$")
+_BOOLEAN = frozenset({"true", "false", "yes", "no"})
+
+
+def infer_type(values) -> str:
+    present = [str(value).strip() for value in values if str(value).strip()]
+    if not present:
+        return "string"
+    lowered = [value.lower() for value in present]
+    if all(value in _BOOLEAN for value in lowered):
+        return "boolean"
+    if all(_INTEGER.fullmatch(value) for value in present):
+        return "integer"
+    if all(_INTEGER.fullmatch(value) or _DECIMAL.fullmatch(value) for value in present):
+        return "decimal"
+    return "string"
+
+
+class CsvExtractor(BaseExtractor):
+    parser_version = "csv-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        return filename.lower().endswith(".csv") or media_type in {
+            "text/csv",
+            "application/csv",
+        }
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        try:
+            text = content.decode("utf-8-sig")
+        except UnicodeDecodeError as exc:
+            raise ValueError("CSV file must be UTF-8 encoded") from exc
+        if not text.strip():
+            raise ValueError("CSV header is required")
+        try:
+            dialect = csv.Sniffer().sniff(text[:8192], delimiters=",;\t|")
+        except csv.Error:
+            dialect = csv.excel
+        rows = list(csv.reader(io.StringIO(text), dialect))
+        if not rows or not any(str(value).strip() for value in rows[0]):
+            raise ValueError("CSV header is required")
+        headers = [str(value).strip() for value in rows[0]]
+        data_rows = rows[1:]
+        columns = []
+        for index, name in enumerate(headers, start=1):
+            values = [row[index - 1] if len(row) >= index else "" for row in data_rows]
+            columns.append(
+                {
+                    "name_zh": "",
+                    "name_en": name,
+                    "data_type": infer_type(values),
+                    "nullable": any(not str(value).strip() for value in values),
+                    "evidence_locator": {
+                        "kind": "csv.header",
+                        "column": index,
+                        "name": name,
+                    },
+                }
+            )
+        item = ExtractedItem(
+            candidate_type="physical_asset",
+            data={
+                "table_info": {
+                    "name_zh": "",
+                    "name_en": Path(context.filename).stem,
+                },
+                "columns": columns,
+            },
+            evidence=(
+                ExtractionEvidence(
+                    locator={"kind": "csv.header", "row": 1},
+                    excerpt=",".join(headers),
+                ),
+            ),
+        )
+        return self._batch(
+            content,
+            [item],
+            metadata={"row_count": len(data_rows), "delimiter": dialect.delimiter},
+        )

deployment/app/core/data_research/extractors/docx.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import io
+
+from docx import Document
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+class DocxExtractor(BaseExtractor):
+    parser_version = "docx-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        return filename.lower().endswith(".docx") or media_type == (
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+        )
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        document = Document(io.BytesIO(content))
+        items = []
+        for index, paragraph in enumerate(document.paragraphs, start=1):
+            value = paragraph.text.strip()
+            if not value:
+                continue
+            evidence = ExtractionEvidence(
+                locator={"kind": "docx.paragraph", "paragraph": index},
+                excerpt=value,
+            )
+            items.append(
+                ExtractedItem(
+                    candidate_type="document_fragment",
+                    data={"kind": "paragraph", "text": value},
+                    evidence=(evidence,),
+                )
+            )
+        for table_index, table in enumerate(document.tables, start=1):
+            header = tuple(cell.text.strip() for cell in table.rows[0].cells) if table.rows else ()
+            for row_index, row in enumerate(table.rows, start=1):
+                values = tuple(cell.text.strip() for cell in row.cells)
+                if row_index > 1 and values == header:
+                    continue
+                for cell_index, value in enumerate(values, start=1):
+                    if not value:
+                        continue
+                    evidence = ExtractionEvidence(
+                        locator={
+                            "kind": "docx.table",
+                            "table": table_index,
+                            "row": row_index,
+                            "cell": cell_index,
+                        },
+                        excerpt=value,
+                    )
+                    items.append(
+                        ExtractedItem(
+                            candidate_type="document_fragment",
+                            data={"kind": "table_cell", "text": value},
+                            evidence=(evidence,),
+                        )
+                    )
+        warnings = [] if items else ["document contains no usable text"]
+        return self._batch(content, items, warnings=warnings)

deployment/app/core/data_research/extractors/excel.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import io
+
+import pandas as pd
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+def _column_letter(index: int) -> str:
+    result = ""
+    while index:
+        index, remainder = divmod(index - 1, 26)
+        result = chr(65 + remainder) + result
+    return result
+
+
+def _pandas_type(series) -> str:
+    if pd.api.types.is_bool_dtype(series.dtype):
+        return "boolean"
+    if pd.api.types.is_integer_dtype(series.dtype):
+        return "integer"
+    if pd.api.types.is_float_dtype(series.dtype):
+        return "decimal"
+    if pd.api.types.is_datetime64_any_dtype(series.dtype):
+        return "datetime"
+    return "string"
+
+
+class ExcelExtractor(BaseExtractor):
+    parser_version = "excel-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        lowered = filename.lower()
+        return lowered.endswith((".xlsx", ".xls")) or "spreadsheet" in media_type or "excel" in media_type
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        workbook = pd.ExcelFile(io.BytesIO(content))
+        items = []
+        for sheet_name in workbook.sheet_names:
+            frame = pd.read_excel(workbook, sheet_name=sheet_name)
+            columns = []
+            last_row = max(1, len(frame) + 1)
+            for index, name in enumerate(frame.columns, start=1):
+                letter = _column_letter(index)
+                columns.append(
+                    {
+                        "name_zh": "",
+                        "name_en": str(name),
+                        "data_type": _pandas_type(frame[name]),
+                        "nullable": bool(frame[name].isna().any()),
+                        "evidence_locator": {
+                            "kind": "excel.cell_range",
+                            "sheet": sheet_name,
+                            "range": f"{letter}1:{letter}{last_row}",
+                        },
+                    }
+                )
+            items.append(
+                ExtractedItem(
+                    candidate_type="physical_asset",
+                    data={
+                        "table_info": {"name_zh": "", "name_en": sheet_name},
+                        "columns": columns,
+                    },
+                    evidence=(
+                        ExtractionEvidence(
+                            locator={"kind": "excel.sheet", "sheet": sheet_name},
+                        ),
+                    ),
+                )
+            )
+        return self._batch(content, items, metadata={"sheet_count": len(items)})

deployment/app/core/data_research/extractors/image.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+class ImageExtractor(BaseExtractor):
+    parser_version = "image-ocr-v1"
+
+    def __init__(self, ocr_service):
+        self.ocr_service = ocr_service
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        suffix = filename.lower()
+        return media_type in {"image/png", "image/jpeg"} or suffix.endswith(
+            (".png", ".jpg", ".jpeg")
+        )
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        items = []
+        for result in self.ocr_service.extract_pages([content]):
+            evidence = ExtractionEvidence(
+                locator={
+                    "kind": "image.ocr",
+                    "page": result.page,
+                    "bbox": list(result.bbox),
+                },
+                excerpt=result.text,
+                confidence=result.confidence,
+            )
+            items.append(
+                ExtractedItem(
+                    candidate_type="document_fragment",
+                    data={
+                        "kind": "ocr_block",
+                        "text": result.text,
+                        "review_required": result.review_required,
+                    },
+                    evidence=(evidence,),
+                    confidence=result.confidence,
+                )
+            )
+        return self._batch(content, items)

deployment/app/core/data_research/extractors/pdf.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import io
+
+import pdfplumber
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+class PdfExtractor(BaseExtractor):
+    parser_version = "pdf-text-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        return filename.lower().endswith(".pdf") or media_type == "application/pdf"
+
+    @staticmethod
+    def _page_evidence(page_number, text, bbox):
+        x0, top, x1, bottom = [float(value) for value in bbox]
+        width = max(1.0, x1 - x0)
+        height = max(1.0, bottom - top)
+        return ExtractionEvidence(
+            locator={
+                "kind": "pdf.page",
+                "page": page_number,
+                "bbox": [0.0, 0.0, round(width / width, 6), round(height / height, 6)],
+            },
+            excerpt=text,
+        )
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        items = []
+        ocr_required = []
+        with pdfplumber.open(io.BytesIO(content)) as document:
+            for page_number, page in enumerate(document.pages, start=1):
+                text = str(page.extract_text() or "").strip()
+                if not text:
+                    ocr_required.append(page_number)
+                    continue
+                items.append(
+                    ExtractedItem(
+                        candidate_type="document_fragment",
+                        data={"kind": "page", "text": text},
+                        evidence=(self._page_evidence(page_number, text, page.bbox),),
+                    )
+                )
+                for table_index, table in enumerate(page.extract_tables() or (), start=1):
+                    for row_index, row in enumerate(table or (), start=1):
+                        for cell_index, cell in enumerate(row or (), start=1):
+                            value = str(cell or "").strip()
+                            if not value:
+                                continue
+                            items.append(
+                                ExtractedItem(
+                                    candidate_type="document_fragment",
+                                    data={"kind": "table_cell", "text": value},
+                                    evidence=(
+                                        ExtractionEvidence(
+                                            locator={
+                                                "kind": "pdf.table",
+                                                "page": page_number,
+                                                "table": table_index,
+                                                "row": row_index,
+                                                "cell": cell_index,
+                                            },
+                                            excerpt=value,
+                                        ),
+                                    ),
+                                )
+                            )
+        warnings = []
+        if ocr_required:
+            pages = ", ".join(str(value) for value in ocr_required)
+            warnings.append(f"OCR required for pages: {pages}")
+        return self._batch(
+            content,
+            items,
+            warnings=warnings,
+            metadata={"ocr_required_pages": ocr_required},
+        )

deployment/app/core/data_research/extractors/registry.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from app.core.data_research.extractors.base import ExtractionContext
+from app.core.data_research.extractors.csv import CsvExtractor
+from app.core.data_research.extractors.excel import ExcelExtractor
+from app.core.data_research.extractors.docx import DocxExtractor
+from app.core.data_research.extractors.pdf import PdfExtractor
+from app.core.data_research.extractors.sql import SqlExtractor
+
+
+class ExtractorRegistry:
+    def __init__(self, extractors):
+        self.extractors = tuple(extractors)
+
+    def resolve(self, media_type: str, filename: str):
+        for extractor in self.extractors:
+            if extractor.can_handle(media_type, filename):
+                return extractor
+        raise ValueError(f"unsupported source format: {filename}")
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        if len(content) > int(context.max_bytes):
+            raise ValueError("source exceeds configured size limit")
+        return self.resolve(context.media_type, context.filename).extract(
+            content,
+            context,
+        )
+
+
+def default_registry():
+    return ExtractorRegistry(
+        (SqlExtractor(), CsvExtractor(), ExcelExtractor(), DocxExtractor(), PdfExtractor())
+    )

deployment/app/core/data_research/extractors/sql.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+from app.core.llm.ddl_parser import DDLParser
+
+
+class SqlExtractor(BaseExtractor):
+    parser_version = "sql-ddl-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        return filename.lower().endswith(".sql") or media_type in {
+            "application/sql",
+            "text/sql",
+        }
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        try:
+            sql_text = content.decode("utf-8-sig")
+        except UnicodeDecodeError as exc:
+            raise ValueError("SQL file must be UTF-8 encoded") from exc
+        tables = DDLParser()._parse_sql_ddl_fallback(sql_text)
+        if not tables:
+            raise ValueError("no supported CREATE TABLE or CREATE VIEW statements found")
+        items = [
+            ExtractedItem(
+                candidate_type="physical_asset",
+                data=table,
+                evidence=(
+                    ExtractionEvidence(
+                        locator={"kind": "sql.statement", "statement": index},
+                    ),
+                ),
+            )
+            for index, table in enumerate(tables, start=1)
+        ]
+        return self._batch(content, items)

deployment/app/core/data_research/file_policy.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+class FilePolicyViolation(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class ValidatedFile:
+    filename: str
+    extension: str
+    media_type: str
+    size_bytes: int
+
+
+class FilePolicy:
+    MEDIA_TYPES = {
+        ".sql": {"text/plain", "application/sql", "text/sql"},
+        ".csv": {"text/csv", "application/csv", "text/plain"},
+        ".xls": {"application/vnd.ms-excel"},
+        ".xlsx": {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
+        ".docx": {"application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
+        ".pdf": {"application/pdf"},
+        ".png": {"image/png"},
+        ".jpg": {"image/jpeg"},
+        ".jpeg": {"image/jpeg"},
+    }
+
+    def __init__(self, *, max_bytes=25 * 1024 * 1024, max_pages=500):
+        self.max_bytes = int(max_bytes)
+        self.max_pages = int(max_pages)
+
+    @staticmethod
+    def _magic_matches(extension, content):
+        signatures = {
+            ".pdf": (b"%PDF-",),
+            ".png": (b"\x89PNG\r\n\x1a\n",),
+            ".jpg": (b"\xff\xd8\xff",),
+            ".jpeg": (b"\xff\xd8\xff",),
+            ".docx": (b"PK\x03\x04",),
+            ".xlsx": (b"PK\x03\x04",),
+            ".xls": (b"\xd0\xcf\x11\xe0",),
+        }
+        expected = signatures.get(extension)
+        if expected is None:
+            try:
+                content[:4096].decode("utf-8-sig")
+                return True
+            except UnicodeDecodeError:
+                return False
+        return any(content.startswith(signature) for signature in expected)
+
+    def validate(self, filename, media_type, content, *, page_count=None):
+        filename = str(filename or "")
+        if not filename or Path(filename).name != filename or "\x00" in filename:
+            raise FilePolicyViolation("unsafe filename")
+        extension = Path(filename).suffix.lower()
+        if extension == ".doc":
+            raise FilePolicyViolation("legacy DOC must be converted to DOCX")
+        if extension not in self.MEDIA_TYPES:
+            raise FilePolicyViolation("file extension is not allowed")
+        if str(media_type or "").lower() not in self.MEDIA_TYPES[extension]:
+            raise FilePolicyViolation("media type does not match file extension")
+        if len(content) > self.max_bytes:
+            raise FilePolicyViolation("file exceeds configured size limit")
+        if page_count is not None and int(page_count) > self.max_pages:
+            raise FilePolicyViolation("document exceeds configured page limit")
+        if not self._magic_matches(extension, content):
+            raise FilePolicyViolation("file signature does not match extension")
+        return ValidatedFile(filename, extension, str(media_type).lower(), len(content))

deployment/app/core/data_research/graph_projection.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from app.core.data_research.errors import DataElementProjectionInvalid
+
+
+class DataElementGraphProjector:
+    """Idempotent effective-graph projection for published data elements."""
+
+    STATEMENT = """
+    MERGE (element:DataElement {uid: $uid})
+    SET element.code = $code,
+        element.name_zh = $name_zh,
+        element.version = $version,
+        element.status = 'published'
+    """
+
+    def __init__(self, graph_session):
+        self.graph_session = graph_session
+
+    def project(self, payload):
+        if str(payload.get("status") or "") != "published":
+            raise DataElementProjectionInvalid(
+                "only a published data element version can be projected"
+            )
+        snapshot = dict(payload.get("snapshot") or {})
+        self.graph_session.run(
+            self.STATEMENT,
+            uid=str(payload["uid"]),
+            version=int(payload["version"]),
+            code=str(snapshot.get("code") or ""),
+            name_zh=str(snapshot.get("name_zh") or ""),
+        )

deployment/app/core/data_research/ingestion.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import replace
+from typing import Any, Callable
+
+from app.core.common.identifiers import new_governance_uid
+from app.core.common.timezone_utils import now_china_naive
+from app.core.data_research.errors import (
+    IngestionJobNotFound,
+    IngestionPayloadInvalid,
+    InvalidJobTransition,
+)
+from app.core.data_research.models import IngestionJobRecord, IngestionJobSpec
+
+
+ALLOWED_TRANSITIONS = {
+    "created": frozenset({"queued", "cancelled"}),
+    "queued": frozenset({"extracting", "failed", "cancelled"}),
+    "extracting": frozenset({"normalizing", "failed", "cancelled"}),
+    "normalizing": frozenset({"matching", "failed", "cancelled"}),
+    "matching": frozenset(
+        {"awaiting_review", "partial", "failed", "cancelled"}
+    ),
+    "awaiting_review": frozenset({"published", "partial", "cancelled"}),
+    "partial": frozenset({"queued", "cancelled"}),
+    "failed": frozenset({"queued", "cancelled"}),
+    "published": frozenset(),
+    "cancelled": frozenset(),
+}
+TERMINAL_STATUSES = frozenset({"published", "cancelled"})
+_SECRET_ASSIGNMENT = re.compile(
+    r"(?i)\b(password|token|secret|authorization)\s*[:=]\s*[^\s,;]+"
+)
+
+
+def _required_text(payload: dict[str, Any], name: str) -> str:
+    value = str(payload.get(name) or "").strip()
+    if not value:
+        raise IngestionPayloadInvalid(f"{name} is required")
+    return value
+
+
+def _sanitize_error(error: Any) -> str | None:
+    if error is None:
+        return None
+    value = _SECRET_ASSIGNMENT.sub(lambda match: f"{match.group(1)}=[redacted]", str(error))
+    return value[:1000]
+
+
+class IngestionService:
+    def __init__(
+        self,
+        repository,
+        *,
+        uid_factory: Callable[[], str] = new_governance_uid,
+        nonce_factory: Callable[[], str] = new_governance_uid,
+        clock: Callable[[], Any] = now_china_naive,
+        commit: Callable[[], Any] = lambda: None,
+        rollback: Callable[[], Any] = lambda: None,
+    ):
+        self.repository = repository
+        self.uid_factory = uid_factory
+        self.nonce_factory = nonce_factory
+        self.clock = clock
+        self.commit = commit
+        self.rollback = rollback
+
+    @staticmethod
+    def _spec(payload: dict[str, Any]) -> IngestionJobSpec:
+        if not isinstance(payload, dict):
+            raise IngestionPayloadInvalid("payload must be an object")
+        parameters = payload.get("parameters", {})
+        if parameters is None:
+            parameters = {}
+        if not isinstance(parameters, dict):
+            raise IngestionPayloadInvalid("parameters must be an object")
+        artifact_uid = str(payload.get("artifact_uid") or "").strip() or None
+        return IngestionJobSpec(
+            source_uid=_required_text(payload, "source_uid"),
+            artifact_uid=artifact_uid,
+            job_type=_required_text(payload, "job_type"),
+            parser_version=_required_text(payload, "parser_version"),
+            parameters=dict(parameters),
+            force_rerun=bool(payload.get("force_rerun", False)),
+        )
+
+    def _idempotency_key(self, spec: IngestionJobSpec) -> str:
+        value = {
+            "source_uid": spec.source_uid,
+            "artifact_uid": spec.artifact_uid,
+            "job_type": spec.job_type,
+            "parser_version": spec.parser_version,
+            "parameters": spec.parameters,
+        }
+        if spec.force_rerun:
+            value["rerun_nonce"] = self.nonce_factory()
+        canonical = json.dumps(
+            value,
+            sort_keys=True,
+            ensure_ascii=False,
+            separators=(",", ":"),
+        )
+        return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+    def create_job(self, payload: dict[str, Any], actor_uid: str | None):
+        spec = self._spec(payload)
+        idempotency_key = self._idempotency_key(spec)
+        if not spec.force_rerun:
+            existing = self.repository.get_by_idempotency_key(idempotency_key)
+            if existing is not None:
+                return existing, False
+        now = self.clock()
+        record = IngestionJobRecord(
+            uid=self.uid_factory(),
+            source_uid=spec.source_uid,
+            artifact_uid=spec.artifact_uid,
+            job_type=spec.job_type,
+            parser_version=spec.parser_version,
+            idempotency_key=idempotency_key,
+            actor_uid=actor_uid,
+            parameters=spec.parameters,
+            force_rerun=spec.force_rerun,
+            created_at=now,
+            updated_at=now,
+        )
+        try:
+            saved = self.repository.add(record)
+            self.commit()
+            return saved, True
+        except Exception:
+            self.rollback()
+            raise
+
+    def _get(self, uid: str) -> IngestionJobRecord:
+        record = self.repository.get(str(uid))
+        if record is None:
+            raise IngestionJobNotFound(f"ingestion job {uid} was not found")
+        return record
+
+    def transition(
+        self,
+        uid: str,
+        target_status: str,
+        statistics: dict[str, Any] | None = None,
+        error: Any = None,
+    ) -> IngestionJobRecord:
+        record = self._get(uid)
+        target_status = str(target_status or "").strip()
+        if target_status not in ALLOWED_TRANSITIONS.get(record.status, frozenset()):
+            raise InvalidJobTransition(f"{record.status} -> {target_status} is not allowed")
+        now = self.clock()
+        updated = replace(
+            record,
+            status=target_status,
+            statistics=dict(statistics or record.statistics),
+            last_error=_sanitize_error(error),
+            updated_at=now,
+            started_at=(now if target_status == "extracting" and record.started_at is None else record.started_at),
+            finished_at=(now if target_status in TERMINAL_STATUSES | {"failed", "partial"} else None),
+        )
+        try:
+            saved = self.repository.save(updated)
+            self.commit()
+            return saved
+        except Exception:
+            self.rollback()
+            raise
+
+    def get_job(self, uid: str) -> IngestionJobRecord:
+        return self._get(uid)
+
+    def list_jobs(self, filters: dict[str, Any] | None = None):
+        return self.repository.list(filters or {})
+
+    def retry(self, uid: str) -> IngestionJobRecord:
+        return self.transition(uid, "queued")
+
+    def cancel(self, uid: str) -> IngestionJobRecord:
+        return self.transition(uid, "cancelled")

deployment/app/core/data_research/models.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass(frozen=True)
+class IngestionJobSpec:
+    source_uid: str
+    job_type: str
+    parser_version: str
+    artifact_uid: str | None = None
+    parameters: dict[str, Any] = field(default_factory=dict)
+    force_rerun: bool = False
+
+
+@dataclass(frozen=True)
+class IngestionJobRecord:
+    uid: str
+    source_uid: str
+    job_type: str
+    parser_version: str
+    idempotency_key: str
+    actor_uid: str | None
+    artifact_uid: str | None = None
+    parameters: dict[str, Any] = field(default_factory=dict)
+    status: str = "created"
+    statistics: dict[str, Any] = field(default_factory=dict)
+    last_error: str | None = None
+    force_rerun: bool = False
+    created_at: datetime | None = None
+    updated_at: datetime | None = None
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+
+
+@dataclass(frozen=True)
+class JobTransition:
+    source_status: str
+    target_status: str

deployment/app/core/data_research/ocr/__init__.py

@@ -0,0 +1,4 @@
+from app.core.data_research.ocr.base import OcrBlock, OcrProvider
+from app.core.data_research.ocr.service import OcrResult, OcrService
+
+__all__ = ["OcrBlock", "OcrProvider", "OcrResult", "OcrService"]

deployment/app/core/data_research/ocr/base.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class OcrBlock:
+    text: str
+    bbox: tuple[float, float, float, float]
+    page: int
+    confidence: float
+
+
+class OcrProvider(Protocol):
+    def extract(self, image_bytes: bytes) -> list[OcrBlock]: ...

deployment/app/core/data_research/ocr/http_provider.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import requests
+
+from app.core.data_research.ocr.base import OcrBlock
+from app.core.data_research.ocr.service import OcrResponseInvalid
+
+
+class HttpOcrProvider:
+    def __init__(
+        self,
+        endpoint,
+        *,
+        token=None,
+        timeout_seconds=10,
+        verify_tls=True,
+        max_response_bytes=2 * 1024 * 1024,
+        post=requests.post,
+    ):
+        endpoint = str(endpoint or "").strip()
+        if not endpoint.startswith("https://"):
+            raise ValueError("OCR endpoint must use HTTPS")
+        self.endpoint = endpoint
+        self.token = token
+        self.timeout_seconds = float(timeout_seconds)
+        self.verify_tls = bool(verify_tls)
+        self.max_response_bytes = int(max_response_bytes)
+        self.post = post
+
+    def extract(self, image_bytes):
+        headers = {"Accept": "application/json"}
+        if self.token:
+            headers["Authorization"] = f"Bearer {self.token}"
+        try:
+            response = self.post(
+                self.endpoint,
+                files={"file": ("page.png", image_bytes, "image/png")},
+                headers=headers,
+                timeout=self.timeout_seconds,
+                verify=self.verify_tls,
+            )
+            response.raise_for_status()
+            if len(response.content) > self.max_response_bytes:
+                raise OcrResponseInvalid("OCR response size exceeds configured limit")
+            payload = response.json()
+            values = payload.get("blocks")
+            if not isinstance(values, list):
+                raise OcrResponseInvalid("OCR response blocks are malformed")
+            return [
+                OcrBlock(
+                    text=str(value["text"]),
+                    bbox=tuple(float(item) for item in value["bbox"]),
+                    page=int(value.get("page") or 1),
+                    confidence=float(value["confidence"]),
+                )
+                for value in values
+            ]
+        except OcrResponseInvalid:
+            raise
+        except Exception as exc:
+            raise OcrResponseInvalid("OCR provider request failed") from exc

deployment/app/core/data_research/ocr/service.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+
+
+class OcrError(ValueError):
+    pass
+
+
+class OcrDisabled(OcrError):
+    pass
+
+
+class OcrResponseInvalid(OcrError):
+    pass
+
+
+@dataclass(frozen=True)
+class OcrResult:
+    text: str
+    bbox: tuple[float, float, float, float]
+    page: int
+    confidence: float
+    review_required: bool
+
+
+class OcrService:
+    def __init__(self, provider, *, review_threshold=0.75):
+        self.provider = provider
+        self.review_threshold = float(review_threshold)
+
+    def extract_pages(self, pages):
+        if self.provider is None:
+            raise OcrDisabled("OCR provider is not configured")
+        results = []
+        for page_number, image_bytes in enumerate(pages, start=1):
+            blocks = self.provider.extract(image_bytes)
+            if not isinstance(blocks, (list, tuple)):
+                raise OcrResponseInvalid("OCR blocks must be a list")
+            for block in blocks:
+                try:
+                    bbox = tuple(float(value) for value in block.bbox)
+                    confidence = float(block.confidence)
+                    text = str(block.text).strip()
+                except (AttributeError, TypeError, ValueError) as exc:
+                    raise OcrResponseInvalid("malformed OCR block") from exc
+                if len(bbox) != 4 or any(value < 0 or value > 1 for value in bbox):
+                    raise OcrResponseInvalid("OCR bounding box must be normalized")
+                if bbox[0] > bbox[2] or bbox[1] > bbox[3]:
+                    raise OcrResponseInvalid("OCR bounding box is invalid")
+                if not 0 <= confidence <= 1 or not text:
+                    raise OcrResponseInvalid("OCR text or confidence is invalid")
+                results.append(
+                    OcrResult(
+                        text=text,
+                        bbox=bbox,
+                        page=page_number,
+                        confidence=confidence,
+                        review_required=confidence < self.review_threshold,
+                    )
+                )
+        return results

deployment/app/core/data_research/ontology/__init__.py

@@ -0,0 +1 @@
+"""Versioned ontology governance services."""

deployment/app/core/data_research/ontology/change_sets.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+
+from app.core.common.identifiers import new_governance_uid
+
+
+class ChangeSetConflict(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class ChangeDecision:
+    suggestion_uid: str
+    decision: str
+    actor_uid: str
+    reason: str
+    payload: dict | None = None
+
+
+@dataclass(frozen=True)
+class ChangeSet:
+    uid: str
+    ontology_uid: str
+    suggestions: tuple
+    decisions: tuple[ChangeDecision, ...]
+    created_by: str
+
+
+class ChangeSetService:
+    def __init__(self, *, uid_factory=new_governance_uid):
+        self.uid_factory = uid_factory
+
+    def create(self, ontology_uid, suggestions, *, actor_uid):
+        targets = {}
+        for suggestion in suggestions:
+            target = (suggestion.kind, str(suggestion.payload.get("uid")))
+            if target in targets and targets[target].payload != suggestion.payload:
+                raise ChangeSetConflict("multiple suggestions conflict on the same target")
+            targets[target] = suggestion
+        return ChangeSet(self.uid_factory(), str(ontology_uid), tuple(suggestions), (), str(actor_uid))
+
+    def decide(self, change_set, decisions, *, actor_uid):
+        known = {item.uid: item for item in change_set.suggestions}
+        audited = list(change_set.decisions)
+        decided_uids = {item.suggestion_uid for item in audited}
+        for item in decisions:
+            uid = str(item.get("suggestion_uid") or "")
+            decision = str(item.get("decision") or "")
+            if uid not in known or decision not in {"accept", "reject", "edit"}:
+                raise ValueError("invalid change-set decision")
+            if uid in decided_uids:
+                raise ValueError("change-set decision is immutable")
+            payload = dict(item.get("payload") or {}) if decision == "edit" else None
+            if decision == "edit" and not payload:
+                raise ValueError("edited suggestion payload is required")
+            audited.append(
+                ChangeDecision(
+                    uid,
+                    decision,
+                    str(actor_uid),
+                    str(item.get("reason") or "").strip(),
+                    payload,
+                )
+            )
+            decided_uids.add(uid)
+        return replace(change_set, decisions=tuple(audited))
+
+    def apply(self, graph, change_set):
+        result = {
+            "classes": list(graph.get("classes") or []),
+            "properties": list(graph.get("properties") or []),
+            "relations": list(graph.get("relations") or []),
+            "constraints": list(graph.get("constraints") or []),
+            "domain_links": list(graph.get("domain_links") or []),
+            "element_mappings": list(graph.get("element_mappings") or []),
+        }
+        suggestions = {item.uid: item for item in change_set.suggestions}
+        sections = {"class": "classes", "property": "properties", "relation": "relations", "constraint": "constraints"}
+        for decision in change_set.decisions:
+            if decision.decision == "reject":
+                continue
+            suggestion = suggestions[decision.suggestion_uid]
+            payload = dict(decision.payload or suggestion.payload)
+            section = sections.get(suggestion.kind)
+            if section:
+                result[section].append(payload)
+        return result
+
+
+class SqlAlchemyDynamicOntologyService:
+    def __init__(self, session, *, suggestion_engine=None, change_set_service=None):
+        from app.core.data_research.ontology.suggestions import OntologySuggestionEngine
+
+        self.session = session
+        self.suggestion_engine = suggestion_engine or OntologySuggestionEngine()
+        self.change_set_service = change_set_service or ChangeSetService()
+
+    @staticmethod
+    def _suggestion_dict(item):
+        return {
+            "uid": item.uid,
+            "kind": item.kind,
+            "payload": item.payload,
+            "evidence_uids": list(item.evidence_uids),
+            "confidence": item.confidence,
+            "source": item.source,
+            "model_version": item.model_version,
+            "prompt_version": item.prompt_version,
+        }
+
+    def generate(self, ontology_uid, payload, actor_uid):
+        from app.models.data_research import OntologyChangeSetModel
+
+        suggestions = self.suggestion_engine.generate(
+            domains=list(payload.get("domains") or []),
+            data_elements=list(payload.get("data_elements") or []),
+            foreign_keys=list(payload.get("foreign_keys") or []),
+            include_ai=bool(payload.get("include_ai", False)),
+        )
+        change_set = self.change_set_service.create(
+            ontology_uid, suggestions, actor_uid=actor_uid
+        )
+        self.session.add(
+            OntologyChangeSetModel(
+                uid=change_set.uid,
+                ontology_uid=str(ontology_uid),
+                changes=[self._suggestion_dict(item) for item in suggestions],
+                decisions=[],
+                created_by=actor_uid,
+            )
+        )
+        self.session.commit()
+        return {
+            "change_set_uid": change_set.uid,
+            "suggestions": suggestions,
+        }
+
+    def decide(self, ontology_uid, change_set_uid, decisions, actor_uid):
+        from app.core.data_research.ontology.suggestions import Suggestion
+        from app.models.data_research import OntologyChangeSetModel
+
+        model = self.session.get(OntologyChangeSetModel, str(change_set_uid))
+        if model is None or str(model.ontology_uid) != str(ontology_uid):
+            raise LookupError("ontology change set was not found")
+        suggestions = tuple(
+            Suggestion(
+                item["uid"],
+                item["kind"],
+                dict(item["payload"]),
+                tuple(item.get("evidence_uids") or []),
+                float(item["confidence"]),
+                item["source"],
+                item.get("model_version"),
+                item.get("prompt_version"),
+            )
+            for item in model.changes
+        )
+        change_set = ChangeSet(
+            str(model.uid), str(model.ontology_uid), suggestions, (), model.created_by
+        )
+        decided = self.change_set_service.decide(
+            change_set, decisions, actor_uid=actor_uid
+        )
+        model.decisions = [item.__dict__ for item in decided.decisions]
+        model.status = "reviewed"
+        self.session.commit()
+        return {
+            "uid": str(model.uid),
+            "ontology_uid": str(model.ontology_uid),
+            "decisions": list(model.decisions),
+            "actor_uid": actor_uid,
+        }

deployment/app/core/data_research/ontology/exchange.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import json
+import xml.etree.ElementTree as ET
+
+from app.core.data_research.ontology.models import GRAPH_SECTIONS, GraphDocument
+
+
+class ExchangeInvalid(ValueError):
+    pass
+
+
+SECRET_KEYS = {"password", "secret", "token", "api_key", "authorization", "credentials"}
+OWL_NS = "http://www.w3.org/2002/07/owl#"
+RDF_NS = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+DATAOPS_NS = "https://dataops.local/ontology#"
+
+
+def _safe(value, key=""):
+    if key.casefold() in SECRET_KEYS:
+        return "[redacted]"
+    if isinstance(value, dict):
+        return {name: _safe(item, name) for name, item in sorted(value.items())}
+    if isinstance(value, list):
+        return [_safe(item) for item in value]
+    return value
+
+
+class OntologyExchange:
+    def __init__(self, *, max_import_bytes=5 * 1024 * 1024):
+        self.max_import_bytes = int(max_import_bytes)
+
+    def _bounded(self, content):
+        if len(content) > self.max_import_bytes:
+            raise ExchangeInvalid("ontology import exceeds size limit")
+
+    def export_json(self, ontology_uid, version, graph):
+        payload = {
+            "format": "dataops-ontology-json-v1",
+            "ontology_uid": str(ontology_uid),
+            "version": int(version),
+            "graph_document": _safe(GraphDocument.from_dict(graph).to_dict()),
+        }
+        return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
+
+    def import_json(self, content):
+        self._bounded(content)
+        try:
+            payload = json.loads(content.decode("utf-8"))
+            return {
+                "ontology_uid": str(payload["ontology_uid"]),
+                "version": int(payload["version"]),
+                "graph_document": GraphDocument.from_dict(payload["graph_document"]).to_dict(),
+            }
+        except (KeyError, TypeError, ValueError, UnicodeDecodeError) as exc:
+            raise ExchangeInvalid("invalid ontology JSON") from exc
+
+    def export_owl(self, ontology_uid, version, graph):
+        ET.register_namespace("owl", OWL_NS)
+        ET.register_namespace("rdf", RDF_NS)
+        ET.register_namespace("dataops", DATAOPS_NS)
+        root = ET.Element(f"{{{RDF_NS}}}RDF")
+        ontology = ET.SubElement(root, f"{{{OWL_NS}}}Ontology")
+        ontology.set(f"{{{RDF_NS}}}about", f"urn:dataops:ontology:{ontology_uid}")
+        ontology.set(f"{{{DATAOPS_NS}}}uid", str(ontology_uid))
+        ontology.set(f"{{{DATAOPS_NS}}}version", str(int(version)))
+        safe_graph = _safe(GraphDocument.from_dict(graph).to_dict())
+        for section in GRAPH_SECTIONS:
+            node = ET.SubElement(ontology, f"{{{DATAOPS_NS}}}{section}")
+            node.text = json.dumps(safe_graph[section], ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+        return ET.tostring(root, encoding="utf-8", xml_declaration=True)
+
+    def import_owl(self, content):
+        self._bounded(content)
+        upper = content.upper()
+        if b"<!DOCTYPE" in upper or b"<!ENTITY" in upper:
+            raise ExchangeInvalid("unsafe XML declarations are forbidden")
+        try:
+            root = ET.fromstring(content)
+            ontology = root.find(f"{{{OWL_NS}}}Ontology")
+            if ontology is None:
+                raise ValueError("missing ontology")
+            graph = {}
+            for section in GRAPH_SECTIONS:
+                node = ontology.find(f"{{{DATAOPS_NS}}}{section}")
+                graph[section] = json.loads(node.text or "[]") if node is not None else []
+            return {
+                "ontology_uid": str(ontology.attrib[f"{{{DATAOPS_NS}}}uid"]),
+                "version": int(ontology.attrib[f"{{{DATAOPS_NS}}}version"]),
+                "graph_document": GraphDocument.from_dict(graph).to_dict(),
+            }
+        except (ET.ParseError, KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
+            raise ExchangeInvalid("invalid ontology RDF/OWL XML") from exc
+
+
+class OntologyExchangeService:
+    def __init__(self, repository, *, exchange=None, commit=lambda: None, rollback=lambda: None):
+        self.repository = repository
+        self.exchange = exchange or OntologyExchange()
+        self.commit = commit
+        self.rollback = rollback
+
+    def export(self, ontology_uid, format_name):
+        version = self.repository.latest_version(ontology_uid)
+        if version is None or version.status not in {"published", "superseded"}:
+            raise LookupError("published ontology version was not found")
+        graph = version.graph_document.to_dict()
+        if format_name == "json":
+            content = self.exchange.export_json(ontology_uid, version.version, graph)
+            return content, "application/json", f"{ontology_uid}-v{version.version}.json"
+        if format_name in {"rdf", "owl"}:
+            content = self.exchange.export_owl(ontology_uid, version.version, graph)
+            return content, "application/rdf+xml", f"{ontology_uid}-v{version.version}.owl"
+        raise ExchangeInvalid("unsupported ontology exchange format")
+
+    def import_document(self, content, format_name, actor_uid):
+        try:
+            imported = (
+                self.exchange.import_json(content)
+                if format_name == "json"
+                else self.exchange.import_owl(content)
+            )
+            code = "IMPORTED_" + imported["ontology_uid"].replace("-", "_").upper()
+            ontology = self.repository.create(
+                code=code,
+                name=code,
+                owner_uid=actor_uid,
+                domain_links=imported["graph_document"].get("domain_links") or [],
+                created_by=actor_uid,
+            )
+            version = self.repository.save_draft(
+                ontology.uid,
+                imported["graph_document"],
+                expected_revision=0,
+                actor_uid=actor_uid,
+            )
+            self.commit()
+            return {
+                "ontology_uid": ontology.uid,
+                "version_uid": version.uid,
+                "version": version.version,
+                "format": format_name,
+                "actor_uid": actor_uid,
+            }
+        except Exception:
+            self.rollback()
+            raise

deployment/app/core/data_research/ontology/knowledge_sync.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from app.core.knowledge.document_builder import build_document, chunk_document
+
+
+class OntologyKnowledgeSync:
+    def __init__(self, repository, *, max_chunk_chars=1200):
+        self.repository = repository
+        self.max_chunk_chars = int(max_chunk_chars)
+
+    def sync(self, published_version):
+        if published_version.get("status") != "published":
+            raise ValueError("only a published ontology version can be synchronized")
+        source = {
+            "uid": str(published_version["uid"]),
+            "name": published_version.get("name") or published_version["uid"],
+            "version": int(published_version["version"]),
+            "status": "published",
+            "graph_document": published_version.get("graph_document") or {},
+            "token": published_version.get("token"),
+        }
+        document = build_document("ontology", source)
+        changed = self.repository.upsert_document(document)
+        key = (document["object_uid"], document["object_version"])
+        if changed or key not in getattr(self.repository, "chunks", {}):
+            self.repository.replace_chunks(
+                key,
+                chunk_document(document["content"], self.max_chunk_chars),
+            )
+        return {"changed": changed, "document": document}

deployment/app/core/data_research/ontology/models.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+GRAPH_SECTIONS = (
+    "classes",
+    "properties",
+    "relations",
+    "constraints",
+    "domain_links",
+    "element_mappings",
+)
+
+
+@dataclass(frozen=True)
+class GraphDocument:
+    classes: tuple[dict[str, Any], ...] = ()
+    properties: tuple[dict[str, Any], ...] = ()
+    relations: tuple[dict[str, Any], ...] = ()
+    constraints: tuple[dict[str, Any], ...] = ()
+    domain_links: tuple[dict[str, Any], ...] = ()
+    element_mappings: tuple[dict[str, Any], ...] = ()
+
+    @classmethod
+    def from_dict(cls, payload):
+        payload = dict(payload or {})
+        return cls(**{name: tuple(dict(item) for item in payload.get(name, ())) for name in GRAPH_SECTIONS})
+
+    def to_dict(self):
+        return {name: [dict(item) for item in getattr(self, name)] for name in GRAPH_SECTIONS}
+
+
+@dataclass(frozen=True)
+class DomainLink:
+    domain_uid: str
+    role: str
+
+
+@dataclass(frozen=True)
+class Ontology:
+    uid: str
+    code: str
+    name: str
+    owner_uid: str
+    domain_links: tuple[DomainLink, ...]
+    status: str = "draft"
+    draft_revision: int = 0
+    active_version_uid: str | None = None
+
+
+@dataclass(frozen=True)
+class OntologyVersion:
+    uid: str
+    ontology_uid: str
+    version: int
+    graph_document: GraphDocument
+    content_hash: str
+    status: str = "draft"
+    parent_version_uid: str | None = None
+    created_by: str | None = None
+
+
+@dataclass(frozen=True)
+class OntologyChangeSet:
+    uid: str
+    ontology_uid: str
+    changes: tuple[dict[str, Any], ...]
+    decisions: tuple[dict[str, Any], ...] = ()
+    status: str = "draft"
+
+
+@dataclass(frozen=True)
+class OntologyPublishRun:
+    uid: str
+    ontology_uid: str
+    version_uid: str
+    idempotency_key: str
+    status: str
+    validation_result: dict[str, Any]

deployment/app/core/data_research/ontology/projection.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+
+class OntologyGraphProjector:
+    STATEMENT = """
+    MERGE (ontology:Ontology {uid: $ontology_uid})
+    SET ontology.active_version_uid = $version_uid,
+        ontology.active_version = $version
+    WITH ontology
+    MERGE (version:OntologyVersion {uid: $version_uid})
+    SET version.version = $version,
+        version.graph_document = $graph_document
+    MERGE (ontology)-[:ACTIVE_VERSION]->(version)
+    """
+
+    def __init__(self, graph_session):
+        self.graph_session = graph_session
+
+    def project(self, payload):
+        self.graph_session.run(
+            self.STATEMENT,
+            ontology_uid=str(payload["ontology_uid"]),
+            version_uid=str(payload["version_uid"]),
+            version=int(payload["version"]),
+            graph_document=payload["graph_document"],
+        )

deployment/app/core/data_research/ontology/publication.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+from app.core.data_research.ontology.models import GRAPH_SECTIONS, GraphDocument
+from app.core.data_research.ontology.validation import validate_graph
+
+
+class OntologyValidationFailed(ValueError):
+    def __init__(self, issues):
+        super().__init__("ontology validation failed")
+        self.issues = tuple(issues)
+
+
+def diff_graphs(before, after):
+    old = GraphDocument.from_dict(before).to_dict()
+    new = GraphDocument.from_dict(after).to_dict()
+    result = {}
+    for section in GRAPH_SECTIONS:
+        old_by_uid = {str(item.get("uid") or item.get("property_uid") or item.get("domain_uid")): item for item in old[section]}
+        new_by_uid = {str(item.get("uid") or item.get("property_uid") or item.get("domain_uid")): item for item in new[section]}
+        result[section] = {
+            "added": sorted(new_by_uid.keys() - old_by_uid.keys()),
+            "removed": sorted(old_by_uid.keys() - new_by_uid.keys()),
+            "changed": sorted(uid for uid in old_by_uid.keys() & new_by_uid.keys() if old_by_uid[uid] != new_by_uid[uid]),
+        }
+    return result
+
+
+class OntologyPublicationService:
+    def __init__(
+        self,
+        repository,
+        *,
+        outbox_enqueue=lambda **_event: None,
+        commit=lambda: None,
+        rollback=lambda: None,
+    ):
+        self.repository = repository
+        self.outbox_enqueue = outbox_enqueue
+        self.commit = commit
+        self.rollback = rollback
+        self._published_by_key = {}
+
+    def publish(self, version_uid, *, idempotency_key, actor_uid):
+        if idempotency_key in self._published_by_key:
+            return self._published_by_key[idempotency_key]
+        try:
+            version = self.repository.get_version(version_uid)
+            if version is None:
+                raise LookupError("ontology version was not found")
+            issues = validate_graph(version.graph_document.to_dict())
+            if issues:
+                raise OntologyValidationFailed(issues)
+            published = self.repository.mark_published(version_uid)
+            self.outbox_enqueue(
+                aggregate_type="ontology",
+                aggregate_id=published.ontology_uid,
+                event_type="ontology.version_published",
+                payload={
+                    "ontology_uid": published.ontology_uid,
+                    "version_uid": published.uid,
+                    "version": published.version,
+                    "status": "published",
+                    "graph_document": published.graph_document.to_dict(),
+                    "actor_uid": actor_uid,
+                },
+            )
+            self.commit()
+            self._published_by_key[idempotency_key] = published
+            return published
+        except Exception:
+            self.rollback()
+            raise
+
+    def create_rollback(
+        self,
+        ontology_uid,
+        *,
+        target_version_uid,
+        expected_revision,
+        actor_uid,
+    ):
+        target = self.repository.get_version(target_version_uid)
+        if target is None or target.ontology_uid != str(ontology_uid):
+            raise LookupError("rollback target was not found")
+        return self.repository.save_draft(
+            ontology_uid,
+            target.graph_document.to_dict(),
+            expected_revision=expected_revision,
+            actor_uid=actor_uid,
+        )
+
+
+class OntologyApplicationService:
+    def __init__(self, repository, *, publication=None, commit=lambda: None, rollback=lambda: None):
+        self.repository = repository
+        self.publication = publication or OntologyPublicationService(repository, commit=commit, rollback=rollback)
+        self.commit = commit
+        self.rollback_transaction = rollback
+
+    def list(self):
+        return self.repository.list()
+
+    def get(self, uid):
+        return self.repository.get(uid)
+
+    def list_versions(self, uid):
+        return self.repository.list_versions(uid)
+
+    def latest_version(self, uid):
+        return self.repository.latest_version(uid)
+
+    def create(self, payload, actor_uid):
+        try:
+            ontology = self.repository.create(
+                code=str(payload.get("code") or "").strip(),
+                name=str(payload.get("name") or "").strip(),
+                owner_uid=str(payload.get("owner_uid") or actor_uid),
+                domain_links=list(payload.get("domain_links") or []),
+                created_by=actor_uid,
+            )
+            self.commit()
+            return ontology
+        except Exception:
+            self.rollback_transaction()
+            raise
+
+    def save_draft(self, uid, graph, expected_revision, actor_uid):
+        try:
+            version = self.repository.save_draft(
+                uid, graph, expected_revision=expected_revision, actor_uid=actor_uid
+            )
+            self.commit()
+            return version
+        except Exception:
+            self.rollback_transaction()
+            raise
+
+    def validate(self, uid):
+        version = self.repository.latest_version(uid)
+        if version is None:
+            return validate_graph({})
+        return validate_graph(version.graph_document.to_dict())
+
+    def publish(self, uid, idempotency_key, actor_uid):
+        version = self.repository.latest_version(uid)
+        if version is None:
+            raise LookupError("ontology draft was not found")
+        return self.publication.publish(
+            version.uid, idempotency_key=idempotency_key, actor_uid=actor_uid
+        )
+
+    def diff(self, uid, left, right):
+        left_version = self.repository.get_version(left)
+        right_version = self.repository.get_version(right)
+        if not left_version or not right_version:
+            raise LookupError("ontology diff version was not found")
+        if left_version.ontology_uid != str(uid) or right_version.ontology_uid != str(uid):
+            raise LookupError("ontology diff version mismatch")
+        return diff_graphs(
+            left_version.graph_document.to_dict(), right_version.graph_document.to_dict()
+        )
+
+    def rollback(self, uid, target_version_uid, expected_revision, actor_uid):
+        try:
+            version = self.publication.create_rollback(
+                uid,
+                target_version_uid=target_version_uid,
+                expected_revision=expected_revision,
+                actor_uid=actor_uid,
+            )
+            self.commit()
+            return version
+        except Exception:
+            self.rollback_transaction()
+            raise

deployment/app/core/data_research/ontology/query.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+
+SAFE_FIELDS = (
+    "path_uid",
+    "ontology_uid",
+    "property_uid",
+    "data_element_uid",
+    "field_uid",
+    "evidence_uid",
+    "business_domain_uid",
+)
+
+
+class SemanticQueryService:
+    def __init__(self, repository):
+        self.repository = repository
+
+    def trace_property(self, property_uid, *, allowed_domains, limit=20, after_uid=None):
+        if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1 or limit > 50:
+            raise ValueError("limit must be between 1 and 50")
+        rows = self.repository.find_property_paths(
+            str(property_uid), depth=3, limit=limit + 1, after_uid=after_uid
+        )
+        safe = []
+        for row in sorted(rows, key=lambda item: str(item.get("path_uid"))):
+            if row.get("ontology_status") != "published":
+                continue
+            domain = str(row.get("business_domain_uid") or "")
+            if "*" not in allowed_domains and domain not in allowed_domains:
+                continue
+            safe.append({key: row.get(key) for key in SAFE_FIELDS if row.get(key) is not None})
+        truncated = len(safe) > limit
+        items = safe[:limit]
+        return {
+            "items": items,
+            "count": len(items),
+            "truncated": truncated,
+            "next_cursor": items[-1]["path_uid"] if truncated and items else None,
+        }
+
+
+class Neo4jSemanticRepository:
+    STATEMENT = """
+    MATCH (ontology:Ontology)-[:ACTIVE_VERSION]->(:OntologyVersion)
+    MATCH (ontology)-[:HAS_PROPERTY]->(property:OntologyProperty {uid: $property_uid})
+    OPTIONAL MATCH (property)-[:MAPS_TO]->(element:DataElement)
+    OPTIONAL MATCH (element)-[:IMPLEMENTED_BY]->(field:PhysicalField)
+    OPTIONAL MATCH (field)-[:HAS_EVIDENCE]->(evidence:Evidence)
+    OPTIONAL MATCH (ontology)-[:SERVES_DOMAIN]->(domain:BusinessDomain)
+    WHERE $after_uid IS NULL OR property.uid > $after_uid
+    RETURN property.uid AS path_uid, ontology.uid AS ontology_uid,
+           'published' AS ontology_status, property.uid AS property_uid,
+           element.uid AS data_element_uid, field.uid AS field_uid,
+           evidence.uid AS evidence_uid, domain.uid AS business_domain_uid
+    ORDER BY path_uid LIMIT $limit
+    """
+
+    def __init__(self, driver):
+        self.driver = driver
+
+    def find_property_paths(self, property_uid, *, depth, limit, after_uid):
+        if int(depth) != 3:
+            raise ValueError("semantic traversal depth is fixed")
+        with self.driver.get_session() as session:
+            result = session.run(
+                self.STATEMENT,
+                property_uid=str(property_uid),
+                after_uid=after_uid,
+                limit=int(limit),
+            )
+            return [dict(record) for record in result]

deployment/app/core/data_research/ontology/repository.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import replace
+
+from app.core.common.identifiers import new_governance_uid
+from app.core.data_research.ontology.models import (
+    DomainLink,
+    GraphDocument,
+    Ontology,
+    OntologyVersion,
+)
+
+
+class OntologyRepositoryError(ValueError):
+    pass
+
+
+class OntologyConflict(OntologyRepositoryError):
+    pass
+
+
+class OntologyImmutable(OntologyRepositoryError):
+    pass
+
+
+def canonical_graph_hash(graph):
+    document = GraphDocument.from_dict(graph).to_dict()
+    encoded = json.dumps(document, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+class MemoryOntologyRepository:
+    def __init__(self, *, uid_factory=new_governance_uid):
+        self.uid_factory = uid_factory
+        self.ontologies = {}
+        self.versions = {}
+
+    def create(self, *, code, name, owner_uid, domain_links, created_by=None):
+        if any(item.code == code for item in self.ontologies.values()):
+            raise OntologyConflict("ontology code already exists")
+        links = tuple(DomainLink(str(item["domain_uid"]), str(item["role"])) for item in domain_links)
+        ontology = Ontology(
+            uid=self.uid_factory(),
+            code=str(code),
+            name=str(name),
+            owner_uid=str(owner_uid),
+            domain_links=links,
+        )
+        self.ontologies[ontology.uid] = ontology
+        return ontology
+
+    def get(self, uid):
+        return self.ontologies.get(str(uid))
+
+    def get_version(self, uid):
+        return self.versions.get(str(uid))
+
+    def list_versions(self, ontology_uid):
+        return sorted(
+            (
+                item
+                for item in self.versions.values()
+                if item.ontology_uid == str(ontology_uid)
+            ),
+            key=lambda item: item.version,
+            reverse=True,
+        )
+
+    def latest_version(self, ontology_uid):
+        versions = self.list_versions(ontology_uid)
+        return versions[0] if versions else None
+
+    def save_draft(self, ontology_uid, graph, *, expected_revision, actor_uid):
+        ontology = self.get(ontology_uid)
+        if ontology is None:
+            raise LookupError("ontology was not found")
+        if ontology.draft_revision != int(expected_revision):
+            raise OntologyConflict("ontology draft revision conflict")
+        version_number = 1 + max(
+            (item.version for item in self.versions.values() if item.ontology_uid == ontology.uid),
+            default=0,
+        )
+        version = OntologyVersion(
+            uid=self.uid_factory(),
+            ontology_uid=ontology.uid,
+            version=version_number,
+            parent_version_uid=ontology.active_version_uid,
+            graph_document=GraphDocument.from_dict(graph),
+            content_hash=canonical_graph_hash(graph),
+            created_by=actor_uid,
+        )
+        self.versions[version.uid] = version
+        self.ontologies[ontology.uid] = replace(ontology, draft_revision=ontology.draft_revision + 1)
+        return version
+
+    def mark_published(self, version_uid):
+        version = self.get_version(version_uid)
+        if version is None:
+            raise LookupError("ontology version was not found")
+        published = replace(version, status="published")
+        self.versions[published.uid] = published
+        ontology = self.get(published.ontology_uid)
+        self.ontologies[ontology.uid] = replace(
+            ontology, status="published", active_version_uid=published.uid
+        )
+        return published
+
+    def replace_version(self, version_uid, graph):
+        version = self.get_version(version_uid)
+        if version.status == "published":
+            raise OntologyImmutable("published ontology version is immutable")
+        updated = replace(
+            version,
+            graph_document=GraphDocument.from_dict(graph),
+            content_hash=canonical_graph_hash(graph),
+        )
+        self.versions[updated.uid] = updated
+        return updated
+
+
+class SqlAlchemyOntologyRepository:
+    def __init__(self, session, *, uid_factory=new_governance_uid):
+        self.session = session
+        self.uid_factory = uid_factory
+
+    @staticmethod
+    def _version(model):
+        return OntologyVersion(
+            uid=str(model.uid),
+            ontology_uid=str(model.ontology_uid),
+            version=int(model.version),
+            parent_version_uid=str(model.parent_version_uid) if model.parent_version_uid else None,
+            status=model.status,
+            graph_document=GraphDocument.from_dict(model.graph_document),
+            content_hash=model.content_hash,
+            created_by=model.created_by,
+        )
+
+    def _ontology(self, model):
+        from app.models.data_research import OntologyDomainLinkModel
+
+        links = self.session.query(OntologyDomainLinkModel).filter_by(
+            ontology_uid=str(model.uid)
+        ).all()
+        return Ontology(
+            uid=str(model.uid),
+            code=model.code,
+            name=model.name,
+            owner_uid=model.owner_uid,
+            status=model.status,
+            draft_revision=int(model.draft_revision),
+            active_version_uid=str(model.active_version_uid) if model.active_version_uid else None,
+            domain_links=tuple(DomainLink(str(item.domain_uid), item.role) for item in links),
+        )
+
+    def list(self):
+        from app.models.data_research import OntologyModel
+
+        return [self._ontology(model) for model in self.session.query(OntologyModel).order_by(OntologyModel.code).all()]
+
+    def create(self, *, code, name, owner_uid, domain_links, created_by=None):
+        from app.models.data_research import OntologyDomainLinkModel, OntologyModel
+
+        if self.session.query(OntologyModel).filter_by(code=str(code)).first():
+            raise OntologyConflict("ontology code already exists")
+        uid = self.uid_factory()
+        model = OntologyModel(
+            uid=uid,
+            code=str(code),
+            name=str(name),
+            owner_uid=str(owner_uid),
+            created_by=created_by,
+        )
+        self.session.add(model)
+        # The links are written with explicit foreign-key values rather than an
+        # ORM relationship, so persist the parent before flushing link rows.
+        self.session.flush()
+        for link in domain_links:
+            self.session.add(
+                OntologyDomainLinkModel(
+                    ontology_uid=uid,
+                    domain_uid=str(link["domain_uid"]),
+                    role=str(link["role"]),
+                )
+            )
+        self.session.flush()
+        return self._ontology(model)
+
+    def get(self, uid):
+        from app.models.data_research import OntologyModel
+
+        model = self.session.get(OntologyModel, str(uid))
+        return self._ontology(model) if model is not None else None
+
+    def get_version(self, uid):
+        from app.models.data_research import OntologyVersionModel
+
+        model = self.session.get(OntologyVersionModel, str(uid))
+        return self._version(model) if model is not None else None
+
+    def latest_version(self, ontology_uid):
+        from app.models.data_research import OntologyVersionModel
+
+        model = self.session.query(OntologyVersionModel).filter_by(
+            ontology_uid=str(ontology_uid)
+        ).order_by(OntologyVersionModel.version.desc()).first()
+        return self._version(model) if model is not None else None
+
+    def list_versions(self, ontology_uid):
+        from app.models.data_research import OntologyVersionModel
+
+        models = (
+            self.session.query(OntologyVersionModel)
+            .filter_by(ontology_uid=str(ontology_uid))
+            .order_by(OntologyVersionModel.version.desc())
+            .all()
+        )
+        return [self._version(model) for model in models]
+
+    def save_draft(self, ontology_uid, graph, *, expected_revision, actor_uid):
+        from app.models.data_research import OntologyModel, OntologyVersionModel
+
+        model = self.session.get(OntologyModel, str(ontology_uid))
+        if model is None:
+            raise LookupError("ontology was not found")
+        if int(model.draft_revision) != int(expected_revision):
+            raise OntologyConflict("ontology draft revision conflict")
+        latest = self.latest_version(ontology_uid)
+        version = OntologyVersionModel(
+            uid=self.uid_factory(),
+            ontology_uid=str(ontology_uid),
+            version=(latest.version + 1) if latest else 1,
+            parent_version_uid=model.active_version_uid,
+            status="draft",
+            graph_document=GraphDocument.from_dict(graph).to_dict(),
+            content_hash=canonical_graph_hash(graph),
+            created_by=actor_uid,
+        )
+        model.draft_revision = int(model.draft_revision) + 1
+        self.session.add(version)
+        self.session.flush()
+        return self._version(version)
+
+    def mark_published(self, version_uid):
+        from app.models.data_research import OntologyModel, OntologyVersionModel
+
+        model = self.session.get(OntologyVersionModel, str(version_uid))
+        if model is None:
+            raise LookupError("ontology version was not found")
+        ontology = self.session.get(OntologyModel, str(model.ontology_uid))
+        if ontology.active_version_uid:
+            active = self.session.get(OntologyVersionModel, str(ontology.active_version_uid))
+            if active is not None:
+                active.status = "superseded"
+        model.status = "published"
+        ontology.status = "published"
+        ontology.active_version_uid = model.uid
+        self.session.flush()
+        return self._version(model)
+
+    def replace_version(self, version_uid, graph):
+        from app.models.data_research import OntologyVersionModel
+
+        model = self.session.get(OntologyVersionModel, str(version_uid))
+        if model.status == "published":
+            raise OntologyImmutable("published ontology version is immutable")
+        model.graph_document = GraphDocument.from_dict(graph).to_dict()
+        model.content_hash = canonical_graph_hash(graph)
+        self.session.flush()
+        return self._version(model)