feat: deliver V62 document and OCR governance

app/api/data_development/routes.py

@@ -2,9 +2,10 @@
 
 from __future__ import annotations
 
+import io
 import logging
 
-from flask import g, jsonify, request
+from flask import current_app, g, jsonify, request, send_file
 
 from app import db
 from app.api.data_development import bp
@@ -51,6 +52,45 @@ def get_candidate_decision_service():
     )
 
 
+def _artifact_storage():
+    from minio import Minio
+
+    from app.core.data_research.artifacts import MinioArtifactStorage
+
+    client = Minio(
+        current_app.config["MINIO_HOST"],
+        access_key=current_app.config["MINIO_USER"],
+        secret_key=current_app.config["MINIO_PASSWORD"],
+        secure=bool(current_app.config.get("MINIO_SECURE")),
+    )
+    return MinioArtifactStorage(client, current_app.config["MINIO_BUCKET"])
+
+
+def get_artifact_service():
+    from app.core.common.identifiers import new_governance_uid
+    from app.core.data_research.artifacts import (
+        ArtifactService,
+        SqlAlchemyArtifactRepository,
+    )
+    from app.core.data_research.file_policy import FilePolicy
+
+    return ArtifactService(
+        SqlAlchemyArtifactRepository(db.session),
+        _artifact_storage(),
+        FilePolicy(
+            max_bytes=int(current_app.config.get("DATA_RESEARCH_FILE_MAX_BYTES", 25 * 1024 * 1024)),
+            max_pages=int(current_app.config.get("DATA_RESEARCH_FILE_MAX_PAGES", 500)),
+        ),
+        uid_factory=new_governance_uid,
+    )
+
+
+def get_evidence_service():
+    from app.core.data_research.artifacts import EvidenceService
+
+    return EvidenceService(db.session, _artifact_storage())
+
+
 def _identity():
     return getattr(g, "current_user", {}) or {}
 
@@ -100,6 +140,18 @@ def _decision(record):
     }
 
 
+def _artifact(record):
+    return {
+        "uid": str(record.uid),
+        "source_uid": str(record.source_uid),
+        "filename": record.filename,
+        "media_type": record.media_type,
+        "size_bytes": int(record.size_bytes),
+        "content_hash": record.content_hash,
+        "parser_version": record.parser_version,
+    }
+
+
 def _error(error):
     if isinstance(error, DataResearchError):
         return (
@@ -234,3 +286,47 @@ def decide_candidates():
         return jsonify(success([_decision(record) for record in records])), 200
     except Exception as error:
         return _error(error)
+
+
+@bp.route("/sources/files", methods=["POST"])
+def upload_source_file():
+    uploaded = request.files.get("file")
+    if uploaded is None:
+        return jsonify(failed("缺少上传文件", code=400)), 400
+    try:
+        record, created = get_artifact_service().store(
+            request.form.get("source_uid"),
+            uploaded.filename,
+            uploaded.mimetype,
+            uploaded.read(),
+            request.form.get("parser_version") or "auto-v1",
+        )
+        db.session.commit()
+        return jsonify(success(_artifact(record))), 201 if created else 200
+    except Exception as error:
+        db.session.rollback()
+        return _error(error)
+
+
+@bp.route("/evidence/<evidence_uid>", methods=["GET"])
+def get_evidence(evidence_uid):
+    identity = _identity()
+    try:
+        service = get_evidence_service()
+        if request.args.get("download") in {"1", "true", "yes"}:
+            if "evidence:download" not in set(identity.get("permissions") or []):
+                return jsonify(failed("权限不足", code=403)), 403
+            content, filename, media_type = service.download(evidence_uid)
+            return send_file(
+                io.BytesIO(content),
+                mimetype=media_type,
+                as_attachment=True,
+                download_name=filename,
+            )
+        preview = service.preview(evidence_uid)
+        from app.core.data_research.artifacts import redact_excerpt
+
+        preview["excerpt"] = redact_excerpt(preview.get("excerpt"))
+        return jsonify(success(preview)), 200
+    except Exception as error:
+        return _error(error)

app/config/config.py

@@ -387,6 +387,12 @@ class BaseConfig:
         "docx",
         "doc",
     }
+    DATA_RESEARCH_FILE_MAX_BYTES = int(
+        os.environ.get("DATA_RESEARCH_FILE_MAX_BYTES", str(25 * 1024 * 1024))
+    )
+    DATA_RESEARCH_FILE_MAX_PAGES = int(
+        os.environ.get("DATA_RESEARCH_FILE_MAX_PAGES", "500")
+    )
 
     # 数据抽取配置
     DATA_EXTRACT_BATCH_SIZE = 1000  # 每批处理的记录数

app/core/data_research/artifacts.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import hashlib
+import io
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ArtifactRecord:
+    uid: str
+    source_uid: str
+    filename: str
+    media_type: str
+    size_bytes: int
+    content_hash: str
+    storage_ref: str
+    parser_version: str
+
+
+class ArtifactService:
+    def __init__(self, repository, storage, file_policy, *, uid_factory):
+        self.repository = repository
+        self.storage = storage
+        self.file_policy = file_policy
+        self.uid_factory = uid_factory
+
+    def store(self, source_uid, filename, media_type, content, parser_version):
+        validated = self.file_policy.validate(filename, media_type, content)
+        digest = hashlib.sha256(content).hexdigest()
+        existing = self.repository.find(source_uid, digest, parser_version)
+        if existing is not None:
+            return existing, False
+        object_key = f"data-research/{source_uid}/{digest}"
+        storage_ref = self.storage.put(object_key, content, validated.media_type)
+        record = ArtifactRecord(
+            uid=self.uid_factory(),
+            source_uid=str(source_uid),
+            filename=validated.filename,
+            media_type=validated.media_type,
+            size_bytes=validated.size_bytes,
+            content_hash=digest,
+            storage_ref=storage_ref,
+            parser_version=str(parser_version),
+        )
+        return self.repository.save(record), True
+
+
+def redact_excerpt(value):
+    value = re.sub(r"(?<!\d)(1\d{2})\d{4}(\d{4})(?!\d)", r"\1****\2", str(value or ""))
+    value = re.sub(
+        r"(?i)\b(token|password|secret|api[_-]?key)\s*[=:]\s*[^\s,;]+",
+        lambda match: f"{match.group(1)}=[redacted]",
+        value,
+    )
+    return value
+
+
+class EvidenceService:
+    def __init__(self, session, storage):
+        self.session = session
+        self.storage = storage
+
+    def preview(self, uid):
+        from app.models.data_research import EvidenceFragment
+
+        model = self.session.get(EvidenceFragment, str(uid))
+        if model is None:
+            raise LookupError("evidence was not found")
+        data = model.to_dict()
+        data["excerpt"] = redact_excerpt(data.get("excerpt"))
+        return data
+
+    def download(self, uid):
+        from app.models.data_research import EvidenceFragment, SourceArtifact
+
+        evidence = self.session.get(EvidenceFragment, str(uid))
+        artifact = (
+            self.session.get(SourceArtifact, str(evidence.artifact_uid))
+            if evidence is not None and evidence.artifact_uid
+            else None
+        )
+        if artifact is None:
+            raise LookupError("evidence artifact was not found")
+        return self.storage.get(artifact.storage_ref), artifact.filename, artifact.media_type
+
+
+class SqlAlchemyArtifactRepository:
+    def __init__(self, session):
+        self.session = session
+
+    @staticmethod
+    def _record(model):
+        return ArtifactRecord(
+            uid=str(model.uid),
+            source_uid=str(model.source_uid),
+            filename=model.filename,
+            media_type=model.media_type,
+            size_bytes=int(model.size_bytes),
+            content_hash=model.content_hash,
+            storage_ref=model.storage_ref,
+            parser_version=model.parser_version,
+        )
+
+    def find(self, source_uid, content_hash, parser_version):
+        from app.models.data_research import SourceArtifact
+
+        model = self.session.query(SourceArtifact).filter_by(
+            source_uid=str(source_uid),
+            content_hash=content_hash,
+            parser_version=str(parser_version),
+        ).first()
+        return self._record(model) if model is not None else None
+
+    def save(self, record):
+        from app.models.data_research import SourceArtifact
+
+        model = SourceArtifact(**record.__dict__)
+        self.session.add(model)
+        self.session.flush()
+        return self._record(model)
+
+
+class MinioArtifactStorage:
+    def __init__(self, client, bucket):
+        self.client = client
+        self.bucket = bucket
+
+    def put(self, object_key, content, media_type):
+        self.client.put_object(
+            self.bucket,
+            object_key,
+            io.BytesIO(content),
+            len(content),
+            content_type=media_type,
+        )
+        return f"minio://{self.bucket}/{object_key}"
+
+    def get(self, storage_ref):
+        prefix = f"minio://{self.bucket}/"
+        if not str(storage_ref).startswith(prefix):
+            raise ValueError("invalid storage reference")
+        response = self.client.get_object(self.bucket, str(storage_ref)[len(prefix):])
+        try:
+            return response.read()
+        finally:
+            response.close()
+            response.release_conn()

app/core/data_research/extractors/docx.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import io
+
+from docx import Document
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+class DocxExtractor(BaseExtractor):
+    parser_version = "docx-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        return filename.lower().endswith(".docx") or media_type == (
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+        )
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        document = Document(io.BytesIO(content))
+        items = []
+        for index, paragraph in enumerate(document.paragraphs, start=1):
+            value = paragraph.text.strip()
+            if not value:
+                continue
+            evidence = ExtractionEvidence(
+                locator={"kind": "docx.paragraph", "paragraph": index},
+                excerpt=value,
+            )
+            items.append(
+                ExtractedItem(
+                    candidate_type="document_fragment",
+                    data={"kind": "paragraph", "text": value},
+                    evidence=(evidence,),
+                )
+            )
+        for table_index, table in enumerate(document.tables, start=1):
+            header = tuple(cell.text.strip() for cell in table.rows[0].cells) if table.rows else ()
+            for row_index, row in enumerate(table.rows, start=1):
+                values = tuple(cell.text.strip() for cell in row.cells)
+                if row_index > 1 and values == header:
+                    continue
+                for cell_index, value in enumerate(values, start=1):
+                    if not value:
+                        continue
+                    evidence = ExtractionEvidence(
+                        locator={
+                            "kind": "docx.table",
+                            "table": table_index,
+                            "row": row_index,
+                            "cell": cell_index,
+                        },
+                        excerpt=value,
+                    )
+                    items.append(
+                        ExtractedItem(
+                            candidate_type="document_fragment",
+                            data={"kind": "table_cell", "text": value},
+                            evidence=(evidence,),
+                        )
+                    )
+        warnings = [] if items else ["document contains no usable text"]
+        return self._batch(content, items, warnings=warnings)
+

app/core/data_research/extractors/image.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+class ImageExtractor(BaseExtractor):
+    parser_version = "image-ocr-v1"
+
+    def __init__(self, ocr_service):
+        self.ocr_service = ocr_service
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        suffix = filename.lower()
+        return media_type in {"image/png", "image/jpeg"} or suffix.endswith(
+            (".png", ".jpg", ".jpeg")
+        )
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        items = []
+        for result in self.ocr_service.extract_pages([content]):
+            evidence = ExtractionEvidence(
+                locator={
+                    "kind": "image.ocr",
+                    "page": result.page,
+                    "bbox": list(result.bbox),
+                },
+                excerpt=result.text,
+                confidence=result.confidence,
+            )
+            items.append(
+                ExtractedItem(
+                    candidate_type="document_fragment",
+                    data={
+                        "kind": "ocr_block",
+                        "text": result.text,
+                        "review_required": result.review_required,
+                    },
+                    evidence=(evidence,),
+                    confidence=result.confidence,
+                )
+            )
+        return self._batch(content, items)

app/core/data_research/extractors/pdf.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import io
+
+import pdfplumber
+
+from app.core.data_research.extractors.base import (
+    BaseExtractor,
+    ExtractedItem,
+    ExtractionContext,
+    ExtractionEvidence,
+)
+
+
+class PdfExtractor(BaseExtractor):
+    parser_version = "pdf-text-v1"
+
+    def can_handle(self, media_type: str, filename: str) -> bool:
+        return filename.lower().endswith(".pdf") or media_type == "application/pdf"
+
+    @staticmethod
+    def _page_evidence(page_number, text, bbox):
+        x0, top, x1, bottom = [float(value) for value in bbox]
+        width = max(1.0, x1 - x0)
+        height = max(1.0, bottom - top)
+        return ExtractionEvidence(
+            locator={
+                "kind": "pdf.page",
+                "page": page_number,
+                "bbox": [0.0, 0.0, round(width / width, 6), round(height / height, 6)],
+            },
+            excerpt=text,
+        )
+
+    def extract(self, content: bytes, context: ExtractionContext):
+        items = []
+        ocr_required = []
+        with pdfplumber.open(io.BytesIO(content)) as document:
+            for page_number, page in enumerate(document.pages, start=1):
+                text = str(page.extract_text() or "").strip()
+                if not text:
+                    ocr_required.append(page_number)
+                    continue
+                items.append(
+                    ExtractedItem(
+                        candidate_type="document_fragment",
+                        data={"kind": "page", "text": text},
+                        evidence=(self._page_evidence(page_number, text, page.bbox),),
+                    )
+                )
+                for table_index, table in enumerate(page.extract_tables() or (), start=1):
+                    for row_index, row in enumerate(table or (), start=1):
+                        for cell_index, cell in enumerate(row or (), start=1):
+                            value = str(cell or "").strip()
+                            if not value:
+                                continue
+                            items.append(
+                                ExtractedItem(
+                                    candidate_type="document_fragment",
+                                    data={"kind": "table_cell", "text": value},
+                                    evidence=(
+                                        ExtractionEvidence(
+                                            locator={
+                                                "kind": "pdf.table",
+                                                "page": page_number,
+                                                "table": table_index,
+                                                "row": row_index,
+                                                "cell": cell_index,
+                                            },
+                                            excerpt=value,
+                                        ),
+                                    ),
+                                )
+                            )
+        warnings = []
+        if ocr_required:
+            pages = ", ".join(str(value) for value in ocr_required)
+            warnings.append(f"OCR required for pages: {pages}")
+        return self._batch(
+            content,
+            items,
+            warnings=warnings,
+            metadata={"ocr_required_pages": ocr_required},
+        )
+

app/core/data_research/extractors/registry.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 from app.core.data_research.extractors.base import ExtractionContext
 from app.core.data_research.extractors.csv import CsvExtractor
 from app.core.data_research.extractors.excel import ExcelExtractor
+from app.core.data_research.extractors.docx import DocxExtractor
+from app.core.data_research.extractors.pdf import PdfExtractor
 from app.core.data_research.extractors.sql import SqlExtractor
 
 
@@ -26,5 +28,6 @@ class ExtractorRegistry:
 
 
 def default_registry():
-    return ExtractorRegistry((SqlExtractor(), CsvExtractor(), ExcelExtractor()))
-
+    return ExtractorRegistry(
+        (SqlExtractor(), CsvExtractor(), ExcelExtractor(), DocxExtractor(), PdfExtractor())
+    )

app/core/data_research/file_policy.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+class FilePolicyViolation(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class ValidatedFile:
+    filename: str
+    extension: str
+    media_type: str
+    size_bytes: int
+
+
+class FilePolicy:
+    MEDIA_TYPES = {
+        ".sql": {"text/plain", "application/sql", "text/sql"},
+        ".csv": {"text/csv", "application/csv", "text/plain"},
+        ".xls": {"application/vnd.ms-excel"},
+        ".xlsx": {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
+        ".docx": {"application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
+        ".pdf": {"application/pdf"},
+        ".png": {"image/png"},
+        ".jpg": {"image/jpeg"},
+        ".jpeg": {"image/jpeg"},
+    }
+
+    def __init__(self, *, max_bytes=25 * 1024 * 1024, max_pages=500):
+        self.max_bytes = int(max_bytes)
+        self.max_pages = int(max_pages)
+
+    @staticmethod
+    def _magic_matches(extension, content):
+        signatures = {
+            ".pdf": (b"%PDF-",),
+            ".png": (b"\x89PNG\r\n\x1a\n",),
+            ".jpg": (b"\xff\xd8\xff",),
+            ".jpeg": (b"\xff\xd8\xff",),
+            ".docx": (b"PK\x03\x04",),
+            ".xlsx": (b"PK\x03\x04",),
+            ".xls": (b"\xd0\xcf\x11\xe0",),
+        }
+        expected = signatures.get(extension)
+        if expected is None:
+            try:
+                content[:4096].decode("utf-8-sig")
+                return True
+            except UnicodeDecodeError:
+                return False
+        return any(content.startswith(signature) for signature in expected)
+
+    def validate(self, filename, media_type, content, *, page_count=None):
+        filename = str(filename or "")
+        if not filename or Path(filename).name != filename or "\x00" in filename:
+            raise FilePolicyViolation("unsafe filename")
+        extension = Path(filename).suffix.lower()
+        if extension == ".doc":
+            raise FilePolicyViolation("legacy DOC must be converted to DOCX")
+        if extension not in self.MEDIA_TYPES:
+            raise FilePolicyViolation("file extension is not allowed")
+        if str(media_type or "").lower() not in self.MEDIA_TYPES[extension]:
+            raise FilePolicyViolation("media type does not match file extension")
+        if len(content) > self.max_bytes:
+            raise FilePolicyViolation("file exceeds configured size limit")
+        if page_count is not None and int(page_count) > self.max_pages:
+            raise FilePolicyViolation("document exceeds configured page limit")
+        if not self._magic_matches(extension, content):
+            raise FilePolicyViolation("file signature does not match extension")
+        return ValidatedFile(filename, extension, str(media_type).lower(), len(content))
+

app/core/data_research/ocr/__init__.py

@@ -0,0 +1,5 @@
+from app.core.data_research.ocr.base import OcrBlock, OcrProvider
+from app.core.data_research.ocr.service import OcrResult, OcrService
+
+__all__ = ["OcrBlock", "OcrProvider", "OcrResult", "OcrService"]
+

app/core/data_research/ocr/base.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+
+@dataclass(frozen=True)
+class OcrBlock:
+    text: str
+    bbox: tuple[float, float, float, float]
+    page: int
+    confidence: float
+
+
+class OcrProvider(Protocol):
+    def extract(self, image_bytes: bytes) -> list[OcrBlock]: ...
+

app/core/data_research/ocr/http_provider.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import requests
+
+from app.core.data_research.ocr.base import OcrBlock
+from app.core.data_research.ocr.service import OcrResponseInvalid
+
+
+class HttpOcrProvider:
+    def __init__(
+        self,
+        endpoint,
+        *,
+        token=None,
+        timeout_seconds=10,
+        verify_tls=True,
+        max_response_bytes=2 * 1024 * 1024,
+        post=requests.post,
+    ):
+        endpoint = str(endpoint or "").strip()
+        if not endpoint.startswith("https://"):
+            raise ValueError("OCR endpoint must use HTTPS")
+        self.endpoint = endpoint
+        self.token = token
+        self.timeout_seconds = float(timeout_seconds)
+        self.verify_tls = bool(verify_tls)
+        self.max_response_bytes = int(max_response_bytes)
+        self.post = post
+
+    def extract(self, image_bytes):
+        headers = {"Accept": "application/json"}
+        if self.token:
+            headers["Authorization"] = f"Bearer {self.token}"
+        try:
+            response = self.post(
+                self.endpoint,
+                files={"file": ("page.png", image_bytes, "image/png")},
+                headers=headers,
+                timeout=self.timeout_seconds,
+                verify=self.verify_tls,
+            )
+            response.raise_for_status()
+            if len(response.content) > self.max_response_bytes:
+                raise OcrResponseInvalid("OCR response size exceeds configured limit")
+            payload = response.json()
+            values = payload.get("blocks")
+            if not isinstance(values, list):
+                raise OcrResponseInvalid("OCR response blocks are malformed")
+            return [
+                OcrBlock(
+                    text=str(value["text"]),
+                    bbox=tuple(float(item) for item in value["bbox"]),
+                    page=int(value.get("page") or 1),
+                    confidence=float(value["confidence"]),
+                )
+                for value in values
+            ]
+        except OcrResponseInvalid:
+            raise
+        except Exception as exc:
+            raise OcrResponseInvalid("OCR provider request failed") from exc
+

app/core/data_research/ocr/service.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+
+
+class OcrError(ValueError):
+    pass
+
+
+class OcrDisabled(OcrError):
+    pass
+
+
+class OcrResponseInvalid(OcrError):
+    pass
+
+
+@dataclass(frozen=True)
+class OcrResult:
+    text: str
+    bbox: tuple[float, float, float, float]
+    page: int
+    confidence: float
+    review_required: bool
+
+
+class OcrService:
+    def __init__(self, provider, *, review_threshold=0.75):
+        self.provider = provider
+        self.review_threshold = float(review_threshold)
+
+    def extract_pages(self, pages):
+        if self.provider is None:
+            raise OcrDisabled("OCR provider is not configured")
+        results = []
+        for page_number, image_bytes in enumerate(pages, start=1):
+            blocks = self.provider.extract(image_bytes)
+            if not isinstance(blocks, (list, tuple)):
+                raise OcrResponseInvalid("OCR blocks must be a list")
+            for block in blocks:
+                try:
+                    bbox = tuple(float(value) for value in block.bbox)
+                    confidence = float(block.confidence)
+                    text = str(block.text).strip()
+                except (AttributeError, TypeError, ValueError) as exc:
+                    raise OcrResponseInvalid("malformed OCR block") from exc
+                if len(bbox) != 4 or any(value < 0 or value > 1 for value in bbox):
+                    raise OcrResponseInvalid("OCR bounding box must be normalized")
+                if bbox[0] > bbox[2] or bbox[1] > bbox[3]:
+                    raise OcrResponseInvalid("OCR bounding box is invalid")
+                if not 0 <= confidence <= 1 or not text:
+                    raise OcrResponseInvalid("OCR text or confidence is invalid")
+                results.append(
+                    OcrResult(
+                        text=text,
+                        bbox=bbox,
+                        page=page_number,
+                        confidence=confidence,
+                        review_required=confidence < self.review_threshold,
+                    )
+                )
+        return results
+

app/core/system/permissions.py

@@ -81,6 +81,10 @@ def permission_for_request(path: str, method: str) -> tuple[str, ...]:
         return (DATA_ELEMENTS_EDIT,)
     if path.startswith("/api/development/v1/candidate-decisions"):
         return (DATA_ELEMENTS_EDIT,)
+    if path.startswith("/api/development/v1/sources/files"):
+        return (INGESTION_RUN,)
+    if path.startswith("/api/development/v1/evidence"):
+        return (READ_GOVERNANCE,)
     if path.startswith("/api/system/workbench"):
         return (READ_GOVERNANCE,)
     if (

docs/architecture/OPENAPI.yaml

@@ -3,7 +3,7 @@ info:
   title: "DataOps Platform API（当前代码基线）"
   version: "2026-07-16"
   description: "由 scripts/generate_openapi.py 从 app/api/*/routes.py 生成。请求体与响应细节仍以现有专项 API 文档和代码为准。"
-x-route-count: 124
+x-route-count: 126
 servers:
   - url: "http://localhost:15500"
     description: "全本地隔离测试后端"
@@ -1726,6 +1726,31 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ApiEnvelope'
+  "/api/development/v1/evidence/{evidence_uid}":
+    get:
+      tags: [data_development]
+      operationId: data_development_get_evidence_get
+      summary: "get evidence"
+      x-source: "app/api/data_development/routes.py"
+      parameters:
+        - name: evidence_uid
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: "请求已由当前实现处理"
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiEnvelope'
+        default:
+          description: "错误响应"
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiEnvelope'
   "/api/development/v1/ingestion-jobs":
     get:
       tags: [data_development]
@@ -1859,6 +1884,32 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/ApiEnvelope'
+  "/api/development/v1/sources/files":
+    post:
+      tags: [data_development]
+      operationId: data_development_upload_source_file_post
+      summary: "upload source file"
+      x-source: "app/api/data_development/routes.py"
+      requestBody:
+        required: false
+        content:
+          application/json:
+            schema:
+              type: object
+              additionalProperties: true
+      responses:
+        "200":
+          description: "请求已由当前实现处理"
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiEnvelope'
+        default:
+          description: "错误响应"
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiEnvelope'
   "/api/graph/node/create":
     post:
       tags: [graph]

docs/superpowers/plans/2026-07-22-data-research-v60-v65-delivery-plan.md

@@ -166,10 +166,10 @@
 **Interfaces:**
 - Produces evidence locators `docx.paragraph/table/row/cell` and `pdf.page/table/bbox`.
 
-- [ ] Write in-memory fixture tests for paragraphs, tables, repeated headers, empty documents, page ordering, and exact evidence locations.
-- [ ] Run focused tests and verify missing extractors.
-- [ ] Implement structured extraction using python-docx and pdfplumber; return OCR-required classification when a PDF page has no usable text.
-- [ ] Rerun focused tests and existing DDL document parser tests.
+- [x] Write in-memory fixture tests for paragraphs, tables, repeated headers, empty documents, page ordering, and exact evidence locations.
+- [x] Run focused tests and verify missing extractors.
+- [x] Implement structured extraction using python-docx and pdfplumber; return OCR-required classification when a PDF page has no usable text.
+- [x] Rerun focused tests and existing DDL document parser tests.
 
 ### Task 8: Add pluggable OCR and scanned-document processing
 
@@ -185,10 +185,10 @@
 - Produces: `OcrProvider.extract(image_bytes) -> list[OcrBlock]` with text, normalized bounding box, page, and confidence.
 - Produces: a fail-closed HTTP provider with configured endpoint, timeout, TLS verification, response-size limit, and no implicit external default.
 
-- [ ] Write tests with a local fake provider for PNG/JPEG, multi-page scanned PDF page ordering, normalized boxes, low-confidence review flags, provider timeout, malformed response, and disabled-provider failure.
-- [ ] Run tests and verify missing OCR interfaces.
-- [ ] Implement provider abstraction, fail-closed configuration, image extractor, and scanned-PDF page handoff.
-- [ ] Rerun focused tests and verify no source bytes or tokens appear in errors/logs.
+- [x] Write tests with a local fake provider for PNG/JPEG, multi-page scanned PDF page ordering, normalized boxes, low-confidence review flags, provider timeout, malformed response, and disabled-provider failure.
+- [x] Run tests and verify missing OCR interfaces.
+- [x] Implement provider abstraction, fail-closed configuration, image extractor, and scanned-PDF page handoff.
+- [x] Rerun focused tests and verify no source bytes or tokens appear in errors/logs.
 
 ### Task 9: Unify file policy, artifact storage, and evidence preview
 
@@ -204,15 +204,15 @@
 - Produces: one allowlist for SQL, XLS/XLSX, CSV, DOCX, PDF, PNG, JPG/JPEG; legacy DOC returns a deterministic conversion-required error.
 - Produces: `POST /sources/files` and permission-filtered `GET /evidence/<uid>`; full download requires `evidence:download`.
 
-- [ ] Write tests for extension/MIME/magic agreement, size/page limits, legacy DOC rejection, hash deduplication, unsafe filenames, evidence redaction, and download permissions.
-- [ ] Run focused tests and verify failures against current split policies.
-- [ ] Implement the unified policy, MinIO gateway abstraction, artifact hash reuse, and redacted preview endpoint.
-- [ ] Rerun focused tests plus datasource/file security regression tests.
+- [x] Write tests for extension/MIME/magic agreement, size/page limits, legacy DOC rejection, hash deduplication, unsafe filenames, evidence redaction, and download permissions.
+- [x] Run focused tests and verify failures against current split policies.
+- [x] Implement the unified policy, MinIO gateway abstraction, artifact hash reuse, and redacted preview endpoint.
+- [x] Rerun focused tests plus datasource/file security regression tests.
 
 ### V62 gate
 
-- [ ] Run all V62-focused tests, the complete Python suite, and `git diff --check`.
-- [ ] Record evidence in `docs/validation/data-research-v62.md` and commit V62.
+- [x] Run all V62-focused tests, the complete Python suite, and `git diff --check`.
+- [x] Record evidence in `docs/validation/data-research-v62.md` and commit V62.
 
 ## V63 — Ontology MVP
 

docs/validation/data-research-v62.md

@@ -0,0 +1,22 @@
+# V62 文档、OCR、文件策略与证据验证
+
+日期：2026-07-22
+
+## 交付范围
+
+- DOCX 段落/表格单元格定位，PDF 页/表格定位及扫描页 OCR 分流。
+- 可插拔 OCR 服务、HTTPS 提供方、超时/TLS/响应大小边界、低置信度人工复核标记。
+- PNG/JPEG OCR 提取与标准化坐标证据。
+- SQL、CSV、XLS/XLSX、DOCX、PDF、PNG、JPG/JPEG 统一文件策略；旧 DOC 明确要求转换。
+- MinIO 网关、源级内容哈希去重、脱敏证据预览及管理员原件下载。
+
+## 测试证据
+
+- DOCX/PDF 与既有文档 DDL 回归：`8 passed`。
+- OCR 与图片解析：`5 passed`。
+- V62 专项、安全策略、权限与 OpenAPI：`29 passed`。
+- OpenAPI 共 126 个操作，生成器可重复性通过。
+- 全量后端回归：`334 passed, 23 skipped, 59 subtests passed`。
+- `git diff --check`：通过。
+
+跳过项为依赖外部服务的环境型测试；OCR 单元测试使用本地假提供方，确认未配置时失败关闭，异常中不泄漏令牌或原始字节。

tests/data_research/test_document_extractors.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import io
+
+from docx import Document
+
+from app.core.data_research.extractors.base import ExtractionContext
+
+
+def docx_bytes(*, empty=False):
+    document = Document()
+    if not empty:
+        document.add_paragraph("客户主数据定义")
+        table = document.add_table(rows=3, cols=2)
+        table.rows[0].cells[0].text = "字段"
+        table.rows[0].cells[1].text = "定义"
+        table.rows[1].cells[0].text = "customer_id"
+        table.rows[1].cells[1].text = "客户唯一编号"
+        table.rows[2].cells[0].text = "字段"
+        table.rows[2].cells[1].text = "定义"
+    stream = io.BytesIO()
+    document.save(stream)
+    return stream.getvalue()
+
+
+def test_docx_extracts_paragraphs_and_table_cells_with_exact_locations():
+    from app.core.data_research.extractors.docx import DocxExtractor
+
+    batch = DocxExtractor().extract(
+        docx_bytes(),
+        ExtractionContext(
+            filename="governance.docx",
+            media_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        ),
+    )
+
+    assert batch.parser_version == "docx-v1"
+    assert batch.items[0].data["text"] == "客户主数据定义"
+    assert batch.items[0].evidence[0].locator == {"kind": "docx.paragraph", "paragraph": 1}
+    table_cells = [item for item in batch.items if item.data["kind"] == "table_cell"]
+    assert table_cells[-1].data["text"] == "客户唯一编号"
+    assert table_cells[-1].evidence[0].locator == {
+        "kind": "docx.table",
+        "table": 1,
+        "row": 2,
+        "cell": 2,
+    }
+    assert len(table_cells) == 4  # repeated header row is removed
+
+
+def test_empty_docx_returns_a_deterministic_warning():
+    from app.core.data_research.extractors.docx import DocxExtractor
+
+    batch = DocxExtractor().extract(
+        docx_bytes(empty=True),
+        ExtractionContext(filename="empty.docx", media_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document"),
+    )
+    assert batch.items == ()
+    assert batch.warnings == ("document contains no usable text",)
+
+
+class FakePage:
+    def __init__(self, text, tables=(), bbox=(0, 0, 600, 800)):
+        self._text = text
+        self._tables = tables
+        self.bbox = bbox
+
+    def extract_text(self):
+        return self._text
+
+    def extract_tables(self):
+        return list(self._tables)
+
+
+class FakePdf:
+    def __init__(self, pages):
+        self.pages = pages
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        return False
+
+
+def test_pdf_preserves_page_order_tables_and_marks_scanned_pages(monkeypatch):
+    import pdfplumber
+
+    from app.core.data_research.extractors.pdf import PdfExtractor
+
+    pages = [
+        FakePage("第一页定义", tables=[[['字段', '定义'], ['id', '唯一编号']]]),
+        FakePage(None),
+        FakePage("第三页定义"),
+    ]
+    monkeypatch.setattr(pdfplumber, "open", lambda _stream: FakePdf(pages))
+
+    batch = PdfExtractor().extract(
+        b"%PDF-fixture",
+        ExtractionContext(filename="standard.pdf", media_type="application/pdf"),
+    )
+
+    assert [item.evidence[0].locator["page"] for item in batch.items] == [1, 1, 1, 1, 1, 3]
+    assert batch.items[0].evidence[0].locator == {
+        "kind": "pdf.page",
+        "page": 1,
+        "bbox": [0.0, 0.0, 1.0, 1.0],
+    }
+    assert batch.metadata["ocr_required_pages"] == [2]
+    assert batch.warnings == ("OCR required for pages: 2",)
+
+
+def test_registry_resolves_docx_and_pdf():
+    from app.core.data_research.extractors.docx import DocxExtractor
+    from app.core.data_research.extractors.pdf import PdfExtractor
+    from app.core.data_research.extractors.registry import default_registry
+
+    registry = default_registry()
+    assert isinstance(registry.resolve("application/pdf", "a.pdf"), PdfExtractor)
+    assert isinstance(
+        registry.resolve(
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            "a.docx",
+        ),
+        DocxExtractor,
+    )

tests/data_research/test_evidence_api.py

@@ -0,0 +1,112 @@
+from __future__ import annotations
+
+import io
+
+import pytest
+
+
+class EvidenceService:
+    def preview(self, uid):
+        assert uid == "evidence-1"
+        return {
+            "uid": uid,
+            "locator": {"kind": "pdf.page", "page": 2},
+            "excerpt": "客户编号 13800138000，token=secret-value",
+            "confidence": 0.88,
+        }
+
+    def download(self, uid):
+        assert uid == "evidence-1"
+        return b"original-secret-source", "evidence.pdf", "application/pdf"
+
+
+class ArtifactService:
+    def __init__(self):
+        self.calls = []
+
+    def store(self, source_uid, filename, media_type, content, parser_version):
+        self.calls.append((source_uid, filename, media_type, content, parser_version))
+        record = type(
+            "Artifact",
+            (),
+            {
+                "uid": "artifact-1",
+                "source_uid": source_uid,
+                "filename": filename,
+                "media_type": media_type,
+                "size_bytes": len(content),
+                "content_hash": "a" * 64,
+                "parser_version": parser_version,
+            },
+        )()
+        return record, True
+
+
+@pytest.fixture()
+def client(monkeypatch):
+    from flask import request
+
+    from app import create_app
+    from app.api.data_development import routes
+    from app.core.system import permissions
+
+    artifacts = ArtifactService()
+
+    def identity():
+        token = request.headers.get("Authorization", "")
+        if token == "Bearer viewer":
+            return {"id": "viewer-1", "roles": ["viewer"]}
+        if token == "Bearer editor":
+            return {"id": "editor-1", "roles": ["editor"]}
+        if token == "Bearer admin":
+            return {"id": "admin-1", "roles": ["admin"]}
+        return None
+
+    monkeypatch.setattr(permissions, "authenticate_request", identity)
+    monkeypatch.setattr(routes, "get_evidence_service", lambda: EvidenceService())
+    monkeypatch.setattr(routes, "get_artifact_service", lambda: artifacts)
+    app = create_app()
+    app.config.update(TESTING=True)
+    return app.test_client(), artifacts
+
+
+def test_preview_is_redacted_and_original_download_is_admin_only(client):
+    http, _artifacts = client
+    preview = http.get(
+        "/api/development/v1/evidence/evidence-1",
+        headers={"Authorization": "Bearer viewer"},
+    )
+    denied = http.get(
+        "/api/development/v1/evidence/evidence-1?download=1",
+        headers={"Authorization": "Bearer viewer"},
+    )
+    downloaded = http.get(
+        "/api/development/v1/evidence/evidence-1?download=1",
+        headers={"Authorization": "Bearer admin"},
+    )
+
+    assert preview.status_code == 200
+    text = preview.get_data(as_text=True)
+    assert "13800138000" not in text and "secret-value" not in text
+    assert "138****8000" in text and "[redacted]" in text
+    assert denied.status_code == 403
+    assert downloaded.data == b"original-secret-source"
+
+
+def test_editor_uploads_allowed_file_without_returning_storage_credentials(client):
+    http, artifacts = client
+    response = http.post(
+        "/api/development/v1/sources/files",
+        headers={"Authorization": "Bearer editor"},
+        data={
+            "source_uid": "source-1",
+            "parser_version": "csv-v1",
+            "file": (io.BytesIO(b"id,name\n1,A"), "governance.csv", "text/csv"),
+        },
+        content_type="multipart/form-data",
+    )
+
+    assert response.status_code == 201
+    assert response.get_json()["data"]["uid"] == "artifact-1"
+    assert "storage_ref" not in response.get_data(as_text=True)
+    assert artifacts.calls[0][3] == b"id,name\n1,A"

tests/data_research/test_file_policy.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import pytest
+
+
+def test_extension_mime_and_magic_must_agree():
+    from app.core.data_research.file_policy import FilePolicy, FilePolicyViolation
+
+    policy = FilePolicy(max_bytes=1024)
+    result = policy.validate("definition.pdf", "application/pdf", b"%PDF-1.7\nbody")
+    assert result.extension == ".pdf"
+    assert result.media_type == "application/pdf"
+
+    with pytest.raises(FilePolicyViolation, match="file signature"):
+        policy.validate("definition.pdf", "application/pdf", b"not-a-pdf")
+    with pytest.raises(FilePolicyViolation, match="media type"):
+        policy.validate("definition.csv", "image/png", b"id,name\n1,A")
+
+
+def test_legacy_doc_unsafe_names_and_size_are_rejected():
+    from app.core.data_research.file_policy import FilePolicy, FilePolicyViolation
+
+    policy = FilePolicy(max_bytes=8)
+    with pytest.raises(FilePolicyViolation, match="convert.*DOCX"):
+        policy.validate("legacy.doc", "application/msword", b"1234")
+    with pytest.raises(FilePolicyViolation, match="unsafe filename"):
+        policy.validate("../secret.csv", "text/csv", b"id\n1")
+    with pytest.raises(FilePolicyViolation, match="size limit"):
+        policy.validate("large.csv", "text/csv", b"id\n123456")
+
+
+class ArtifactRepository:
+    def __init__(self):
+        self.by_key = {}
+        self.saved = []
+
+    def find(self, source_uid, content_hash, parser_version):
+        return self.by_key.get((source_uid, content_hash, parser_version))
+
+    def save(self, artifact):
+        key = (artifact.source_uid, artifact.content_hash, artifact.parser_version)
+        self.by_key[key] = artifact
+        self.saved.append(artifact)
+        return artifact
+
+
+class Storage:
+    def __init__(self):
+        self.puts = []
+
+    def put(self, object_key, content, media_type):
+        self.puts.append((object_key, content, media_type))
+        return f"minio://data-research/{object_key}"
+
+
+def test_artifact_storage_reuses_source_hash_and_hides_unsafe_filename():
+    from app.core.data_research.artifacts import ArtifactService
+    from app.core.data_research.file_policy import FilePolicy
+
+    repository = ArtifactRepository()
+    storage = Storage()
+    service = ArtifactService(
+        repository,
+        storage,
+        FilePolicy(max_bytes=1024),
+        uid_factory=lambda: "artifact-1",
+    )
+
+    first, created = service.store(
+        "source-1", "客户定义.csv", "text/csv", b"id,name\n1,A", "csv-v1"
+    )
+    second, created_again = service.store(
+        "source-1", "客户定义.csv", "text/csv", b"id,name\n1,A", "csv-v1"
+    )
+
+    assert created is True and created_again is False
+    assert first == second
+    assert len(storage.puts) == 1
+    assert "客户定义.csv" not in first.storage_ref
+    assert first.content_hash in first.storage_ref
+

tests/data_research/test_image_extractor.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from app.core.data_research.extractors.base import ExtractionContext
+
+
+class FakeOcrService:
+    def extract_pages(self, pages):
+        from app.core.data_research.ocr.service import OcrResult
+
+        assert pages == [b"image"]
+        return [
+            OcrResult(
+                text="customer_id 客户编号",
+                bbox=(0.1, 0.2, 0.7, 0.3),
+                page=1,
+                confidence=0.65,
+                review_required=True,
+            )
+        ]
+
+
+def test_png_and_jpeg_are_supported_with_normalized_ocr_evidence():
+    from app.core.data_research.extractors.image import ImageExtractor
+
+    extractor = ImageExtractor(FakeOcrService())
+    assert extractor.can_handle("image/png", "scan.png")
+    assert extractor.can_handle("image/jpeg", "scan.jpg")
+
+    batch = extractor.extract(
+        b"image",
+        ExtractionContext(filename="scan.png", media_type="image/png"),
+    )
+    assert batch.parser_version == "image-ocr-v1"
+    assert batch.items[0].data["review_required"] is True
+    assert batch.items[0].evidence[0].locator == {
+        "kind": "image.ocr",
+        "page": 1,
+        "bbox": [0.1, 0.2, 0.7, 0.3],
+    }
+    assert batch.items[0].evidence[0].confidence == 0.65

tests/data_research/test_ocr_service.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import pytest
+
+
+class FakeProvider:
+    def __init__(self, responses):
+        self.responses = list(responses)
+        self.calls = []
+
+    def extract(self, image_bytes):
+        self.calls.append(image_bytes)
+        return self.responses.pop(0)
+
+
+def block(text, bbox, confidence=0.9):
+    from app.core.data_research.ocr.base import OcrBlock
+
+    return OcrBlock(text=text, bbox=bbox, page=99, confidence=confidence)
+
+
+def test_ocr_service_preserves_scanned_pdf_page_order_and_review_flags():
+    from app.core.data_research.ocr.service import OcrService
+
+    provider = FakeProvider(
+        [
+            [block("第一页", (0.1, 0.2, 0.8, 0.3), 0.95)],
+            [block("第二页低置信", (0.0, 0.0, 1.0, 1.0), 0.4)],
+        ]
+    )
+    results = OcrService(provider, review_threshold=0.75).extract_pages(
+        [b"page-one", b"page-two"]
+    )
+
+    assert [result.page for result in results] == [1, 2]
+    assert results[0].bbox == (0.1, 0.2, 0.8, 0.3)
+    assert results[0].review_required is False
+    assert results[1].review_required is True
+
+
+def test_ocr_service_rejects_malformed_provider_response():
+    from app.core.data_research.ocr.service import OcrResponseInvalid, OcrService
+
+    provider = FakeProvider([[block("bad", (-1.0, 0.0, 2.0, 1.0))]])
+    with pytest.raises(OcrResponseInvalid, match="bounding box"):
+        OcrService(provider).extract_pages([b"image"])
+
+
+def test_ocr_is_fail_closed_when_provider_is_disabled():
+    from app.core.data_research.ocr.service import OcrDisabled, OcrService
+
+    with pytest.raises(OcrDisabled, match="not configured"):
+        OcrService(None).extract_pages([b"image"])
+
+
+class FakeResponse:
+    def __init__(self, payload, *, content=b"{}", headers=None):
+        self._payload = payload
+        self.content = content
+        self.headers = headers or {}
+
+    def raise_for_status(self):
+        return None
+
+    def json(self):
+        return self._payload
+
+
+def test_http_provider_enforces_timeout_tls_and_response_limit():
+    from app.core.data_research.ocr.http_provider import HttpOcrProvider
+    from app.core.data_research.ocr.service import OcrResponseInvalid
+
+    calls = []
+
+    def post(*args, **kwargs):
+        calls.append((args, kwargs))
+        return FakeResponse(
+            {"blocks": [{"text": "编号", "bbox": [0, 0, 1, 1], "confidence": 0.9}]},
+            content=b"x" * 20,
+        )
+
+    provider = HttpOcrProvider(
+        "https://ocr.internal/v1/extract",
+        token="secret-token",
+        timeout_seconds=3,
+        verify_tls=True,
+        max_response_bytes=100,
+        post=post,
+    )
+    blocks = provider.extract(b"png")
+    assert blocks[0].text == "编号"
+    assert calls[0][1]["timeout"] == 3
+    assert calls[0][1]["verify"] is True
+
+    provider.max_response_bytes = 10
+    with pytest.raises(OcrResponseInvalid, match="response size") as error:
+        provider.extract(b"png-secret-source")
+    assert "secret-token" not in str(error.value)
+    assert "png-secret-source" not in str(error.value)
+