DataOps-platform/scripts/generate_openapi.py

#!/usr/bin/env python3
"""Generate the current Flask route inventory as an OpenAPI 3.1 contract."""

from __future__ import annotations

import argparse
import ast
import json
import re
from collections import defaultdict
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
OUTPUT = ROOT / "docs" / "architecture" / "OPENAPI.yaml"
PREFIXES = {
    "meta_data": "/api/meta",
    "data_interface": "/api/interface",
    "data_rules": "/api/rules",
    "graph": "/api/graph",
    "system": "/api/system",
    "data_source": "/api/datasource",
    "data_development": "/api/development/v1",
    "data_flow": "/api/dataflow",
    "business_domain": "/api/bd",
    "data_factory": "/api/datafactory",
    "data_service": "/api/dataservice",
    "knowledge_base": "/api/knowledge",
}

SHORTHAND_METHODS = {"get", "post", "put", "patch", "delete"}
RESPONSE_FIELDS = {
    ("data_development", "summary"): [
        "scope",
        "metrics",
    ],
    ("data_development", "details"): [
        "metric",
        "state",
        "records",
        "page",
        "page_size",
        "total",
    ],
    ("knowledge_base", "ask"): [
        "query_id",
        "mode",
        "answer",
        "answer_status",
        "degraded_components",
        "citations",
        "evidence",
        "freshness_status",
    ],
}
CONNECTOR_REQUEST_SCHEMAS = {
    "/api/datasource/connectors/config/validate": "ConnectorConfigValidationRequest",
    "/api/datasource/connectors/runs": "ConnectorRunRequest",
    "/api/datasource/connectors/machine/runs": "MachineConnectorRunRequest",
    "/api/datasource/connectors/source-bindings": "ConnectorSourceBindingRequest",
    "/api/datasource/connectors/principals": "ConnectorPrincipalRequest",
    "/api/datasource/connectors/{connector_id}/{version}/health": "ConnectorHealthRequest",
    "/api/datasource/connectors/principals/{principal_uid}/credentials": "ConnectorCredentialRequest",
    "/api/datasource/connectors/credentials/{credential_uid}/{action}": "ConnectorCredentialRequest",
    "/api/datasource/graph": "ConnectorGraphRequest",
}
CONNECTOR_RESPONSE_SCHEMAS = {
    ("/api/datasource/connectors/manifests", "get"): "ConnectorManifestEnvelope",
    ("/api/datasource/connectors/runs", "get"): "ConnectorRunListEnvelope",
    ("/api/datasource/connectors/runs", "post"): "ConnectorRunResultEnvelope",
    ("/api/datasource/connectors/machine/runs", "post"): "ConnectorRunResultEnvelope",
    (
        "/api/datasource/connectors/runs/{idempotency_key}/cancel",
        "post",
    ): "ConnectorRunRecordEnvelope",
    (
        "/api/datasource/connectors/runs/{idempotency_key}/resume",
        "post",
    ): "ConnectorRunResultEnvelope",
    (
        "/api/datasource/connectors/config/validate",
        "post",
    ): "ConnectorValidationEnvelope",
    (
        "/api/datasource/connectors/{connector_id}/{version}/health",
        "post",
    ): "ConnectorHealthEnvelope",
    (
        "/api/datasource/connectors/{connector_id}/{version}/compatibility",
        "get",
    ): "ConnectorCompatibilityEnvelope",
    ("/api/datasource/connectors/principals", "post"): "ConnectorPrincipalEnvelope",
    (
        "/api/datasource/connectors/source-bindings",
        "post",
    ): "ConnectorSourceBindingEnvelope",
    (
        "/api/datasource/connectors/principals/{principal_uid}/credentials",
        "post",
    ): "ConnectorCredentialEnvelope",
    (
        "/api/datasource/connectors/credentials/{credential_uid}/{action}",
        "post",
    ): "ConnectorCredentialEnvelope",
    ("/api/datasource/graph", "post"): "ConnectorGraphEnvelope",
}
NO_REQUEST_BODY_PATHS = {
    "/api/datasource/connectors/runs/{idempotency_key}/cancel",
    "/api/datasource/connectors/runs/{idempotency_key}/resume",
    "/api/datasource/connectors/machine/runs/{idempotency_key}/cancel",
    "/api/datasource/connectors/machine/runs/{idempotency_key}/resume",
    "/api/datasource/connectors/source-bindings/{binding_uid}/revoke",
}
OPTIONAL_REQUEST_BODY_PATHS = {
    "/api/datasource/connectors/principals/{principal_uid}/credentials",
    "/api/datasource/connectors/credentials/{credential_uid}/{action}",
}


def quoted(value: str) -> str:
    return json.dumps(value, ensure_ascii=False)


def route_path(raw_path: str) -> tuple[str, list[dict[str, str]]]:
    parameters: list[dict[str, str]] = []

    def replace(match: re.Match[str]) -> str:
        converter = match.group(1) or "string"
        name = match.group(2)
        schema_type = "integer" if converter in {"int", "float"} else "string"
        parameters.append({"name": name, "type": schema_type})
        return "{" + name + "}"

    return re.sub(
        r"<(?:([a-zA-Z_]+):)?([a-zA-Z_][a-zA-Z0-9_]*)>", replace, raw_path
    ), parameters


def extract_routes() -> list[dict[str, object]]:
    routes: list[dict[str, object]] = []
    for module, prefix in PREFIXES.items():
        for route_file in sorted((ROOT / "app" / "api" / module).glob("*.py")):
            if route_file.name == "__init__.py":
                continue
            tree = ast.parse(route_file.read_text(encoding="utf-8"))
            for node in tree.body:
                if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
                    continue
                for decorator in node.decorator_list:
                    if not isinstance(decorator, ast.Call):
                        continue
                    function = decorator.func
                    if not (
                        isinstance(function, ast.Attribute)
                        and isinstance(function.value, ast.Name)
                        and function.value.id == "bp"
                        and decorator.args
                    ):
                        continue
                    if function.attr == "route":
                        methods = ["GET"]
                        for keyword in decorator.keywords:
                            if keyword.arg == "methods":
                                methods = ast.literal_eval(keyword.value)
                    elif function.attr in SHORTHAND_METHODS:
                        methods = [function.attr.upper()]
                    else:
                        continue
                    raw_path = ast.literal_eval(decorator.args[0])
                    path, parameters = route_path(prefix + raw_path)
                    summary = (
                        ast.get_docstring(node) or node.name.replace("_", " ")
                    ).splitlines()[0]
                    for method in methods:
                        routes.append(
                            {
                                "path": path,
                                "method": method.lower(),
                                "tag": module,
                                "operation_id": f"{module}_{node.name}_{method.lower()}",
                                "summary": summary,
                                "parameters": parameters,
                                "source": str(route_file.relative_to(ROOT)),
                            }
                        )
    return sorted(routes, key=lambda item: (str(item["path"]), str(item["method"])))


def render(routes: list[dict[str, object]]) -> str:
    by_path: dict[str, list[dict[str, object]]] = defaultdict(list)
    for route in routes:
        by_path[str(route["path"])].append(route)

    lines = [
        "openapi: 3.1.0",
        "info:",
        '  title: "DataOps Platform API（当前代码基线）"',
        '  version: "2026-07-16"',
        '  description: "由 scripts/generate_openapi.py 从 app/api/*/routes.py 生成。请求体与响应细节仍以现有专项 API 文档和代码为准。"',
        f"x-route-count: {len(routes)}",
        "servers:",
        '  - url: "http://localhost:15500"',
        '    description: "全本地隔离测试后端"',
        "tags:",
    ]
    for tag in PREFIXES:
        lines.extend([f"  - name: {tag}", f"    description: {quoted(PREFIXES[tag])}"])
    lines.append("paths:")

    for path, operations in sorted(by_path.items()):
        lines.append(f"  {quoted(path)}:")
        for operation in operations:
            lines.extend(
                [
                    f"    {operation['method']}:",
                    f"      tags: [{operation['tag']}]",
                    f"      operationId: {operation['operation_id']}",
                    f"      summary: {quoted(str(operation['summary']))}",
                    f"      x-source: {quoted(str(operation['source']))}",
                ]
            )
            parameters = operation["parameters"]
            if path.startswith("/api/datasource/connectors/machine/runs"):
                parameters = [
                    *parameters,
                    {
                        "name": "X-Connector-Credential",
                        "type": "string",
                        "header": True,
                    },
                ]
            if parameters:
                lines.append("      parameters:")
                for parameter in parameters:
                    lines.extend(
                        [
                            f"        - name: {parameter['name']}",
                            f"          in: {'header' if parameter.get('header') else 'path'}",
                            "          required: true",
                            "          schema:",
                            f"            type: {parameter['type']}",
                        ]
                    )
                    if (
                        path
                        == "/api/datasource/connectors/credentials/{credential_uid}/{action}"
                        and parameter["name"] == "action"
                    ):
                        lines.append("            enum: [revoke, rotate]")
            if (
                operation["method"] in {"post", "put", "patch"}
                and path not in NO_REQUEST_BODY_PATHS
            ):
                connector_schema = CONNECTOR_REQUEST_SCHEMAS.get(path)
                lines.extend(
                    [
                        "      requestBody:",
                        "        required: "
                        + (
                            "false"
                            if path in OPTIONAL_REQUEST_BODY_PATHS
                            else "true" if connector_schema else "false"
                        ),
                        "        content:",
                        "          application/json:",
                        "            schema:",
                        f"              $ref: '#/components/schemas/{connector_schema}'"
                        if connector_schema
                        else "              type: object",
                    ]
                )
                if not connector_schema:
                    lines.append("              additionalProperties: true")
            response_fields = RESPONSE_FIELDS.get(
                (str(operation["tag"]), str(operation["operation_id"]).split("_")[-2])
            )
            if response_fields:
                lines.append(
                    "      x-response-fields: [" + ", ".join(response_fields) + "]"
                )
            connector_response = CONNECTOR_RESPONSE_SCHEMAS.get(
                (path, str(operation["method"]))
            )
            response_status = (
                "201"
                if operation["method"] == "post"
                and path
                in {
                    "/api/datasource/connectors/runs",
                    "/api/datasource/connectors/machine/runs",
                    "/api/datasource/connectors/principals",
                    "/api/datasource/connectors/principals/{principal_uid}/credentials",
                    "/api/datasource/connectors/source-bindings",
                }
                else "200"
            )
            if path.startswith("/api/datasource/connectors"):
                required_permission = (
                    "machine-credential"
                    if path.startswith("/api/datasource/connectors/machine/runs")
                    else (
                        "connectors:manage"
                        if "/source-bindings" in path
                        else (
                            "connectors:read"
                            if operation["method"] == "get"
                            else (
                                "connectors:manage"
                                if "/principals" in path or "/credentials" in path
                                else "connectors:operate"
                            )
                        )
                    )
                )
                lines.append(
                    f"      x-required-permission: {quoted(required_permission)}"
                )
            elif path == "/api/datasource/graph":
                lines.append('      x-required-permission: "connectors:read"')
            lines.extend(
                [
                    "      responses:",
                    f'        "{response_status}":',
                    '          description: "请求已由当前实现处理"',
                    "          content:",
                    "            application/json:",
                    "              schema:",
                    f"                $ref: '#/components/schemas/{connector_response or 'ApiEnvelope'}'",
                    "        default:",
                    '          description: "错误响应"',
                    "          content:",
                    "            application/json:",
                    "              schema:",
                    f"                $ref: '#/components/schemas/{'ConnectorErrorEnvelope' if path.startswith('/api/datasource/connectors') or path == '/api/datasource/graph' else 'ApiEnvelope'}'",
                ]
            )

    lines.extend(
        [
            "components:",
            "  schemas:",
            "    ApiEnvelope:",
            "      type: object",
            "      additionalProperties: true",
            "      required: [code, message, data]",
            "      properties:",
            "        code:",
            "          type: integer",
            "        message:",
            "          type: string",
            "        data: {}",
            "        error: {type: object}",
            "    ConnectorConfigValidationRequest:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [connector_id, version, config]",
            "      properties:",
            "        connector_id: {type: string, minLength: 3, maxLength: 64}",
            "        version: {type: string, pattern: '^[0-9]+\\.[0-9]+\\.[0-9]+'}",
            "        config: {type: object}",
            "    ConnectorRunRequest:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [connector_id, version, source_uid, operation, config, scope, dry_run]",
            "      properties:",
            "        connector_id: {type: string}",
            "        version: {type: string}",
            "        source_uid: {type: string, format: uuid}",
            "        operation: {type: string, enum: [discover, snapshot, incremental, lineage, profile, evidence]}",
            "        config: {type: object}",
            "        scope: {$ref: '#/components/schemas/ConnectorScope'}",
            "        cursor: {type: object}",
            "        checkpoint: {type: object}",
            "        idempotency_key: {type: string, pattern: '^[a-f0-9]{64}$'}",
            "        dry_run: {const: true}",
            "        process_key: {type: string, minLength: 1, maxLength: 300}",
            "    MachineConnectorRunRequest:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [connector_id, version, source_uid, business_domain_uid, environment, process_key, operation, scope]",
            "      properties:",
            "        connector_id: {type: string}",
            "        version: {type: string}",
            "        source_uid: {type: string, format: uuid}",
            "        business_domain_uid: {type: string, format: uuid}",
            "        environment: {type: string, enum: [development, staging, production]}",
            "        process_key: {type: string, minLength: 1, maxLength: 300}",
            "        operation: {type: string, enum: [discover, snapshot, incremental, lineage, profile, evidence]}",
            "        scope: {$ref: '#/components/schemas/ConnectorScope'}",
            "        cursor: {type: object}",
            "        checkpoint: {type: object}",
            "        idempotency_key: {type: string, pattern: '^[a-f0-9]{64}$'}",
            "        dry_run: {type: boolean, default: false}",
            "    ConnectorPrincipalRequest:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [connector_id, version, source_uid, business_domain_uid, environment, operations, scopes]",
            "      properties:",
            "        connector_id: {type: string}",
            "        version: {type: string}",
            "        source_uid: {type: string, format: uuid}",
            "        business_domain_uid: {type: string, format: uuid}",
            "        environment: {type: string, enum: [development, staging, production]}",
            "        operations: {type: array, minItems: 1, uniqueItems: true, items: {type: string}}",
            "        scopes: {$ref: '#/components/schemas/ConnectorScope'}",
            "        source_binding_uid: {type: string, format: uuid}",
            "        source_binding_version: {type: integer, minimum: 1}",
            "    ConnectorSourceBindingRequest:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [connector_id, version, source_uid, business_domain_uid, environment, approved_config]",
            "      properties:",
            "        binding_uid: {type: string, format: uuid}",
            "        connector_id: {type: string}",
            "        version: {type: string}",
            "        source_uid: {type: string, format: uuid}",
            "        business_domain_uid: {type: string, format: uuid}",
            "        environment: {type: string, enum: [development, staging, production]}",
            "        approved_config: {type: object}",
            "    ConnectorScope:",
            "      type: object",
            "      additionalProperties: false",
            "      properties:",
            "        include_schemas: {type: array, uniqueItems: true, items: {type: string, minLength: 1}}",
            "        exclude_schemas: {type: array, uniqueItems: true, items: {type: string, minLength: 1}}",
            "        include_tables: {type: array, uniqueItems: true, items: {type: string, minLength: 1}}",
            "        exclude_tables: {type: array, uniqueItems: true, items: {type: string, minLength: 1}}",
            "    ConnectorHealthRequest:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [config]",
            "      properties: {config: {type: object}}",
            "    ConnectorCredentialRequest:",
            "      type: object",
            "      additionalProperties: false",
            "      properties: {ttl_seconds: {type: integer, minimum: 60, maximum: 900}}",
            "    ConnectorGraphRequest:",
            "      type: object",
            "      additionalProperties: false",
            "      properties: {source_uid: {type: string, format: uuid}, business_domain_uid: {type: string, format: uuid}, process_key: {type: string}, run_uid: {type: string, format: uuid}, limit: {type: integer, minimum: 1, maximum: 1000}}",
            "    ConnectorManifestEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties:",
            "        code: {type: integer}",
            "        message: {type: string}",
            "        data: {type: object, additionalProperties: false, required: [manifests], properties: {manifests: {type: array, items: {$ref: '#/components/schemas/ConnectorManifest'}}}}",
            "    ConnectorManifest:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [connector_id, version, sdk_version, display_name, capabilities, config_schema]",
            "      properties: {connector_id: {type: string}, version: {type: string}, sdk_version: {type: string}, display_name: {type: string}, capabilities: {type: array, items: {type: string}}, config_schema: {type: object}}",
            "    ConnectorOperationResult:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [records, cursor, checkpoint, evidence, status]",
            "      properties: {records: {type: array, items: {type: object}}, cursor: {type: object}, checkpoint: {type: object}, evidence: {type: object}, status: {type: string, enum: [succeeded, dry_run]}}",
            "    ConnectorRunResultEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties: {code: {type: integer}, message: {type: string}, data: {$ref: '#/components/schemas/ConnectorOperationResult'}}",
            "    ConnectorRunRecord:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [uid, idempotency_key, connector_id, connector_version, source_uid, operation, status, attempt_count, checkpoint_summary, cursor_summary, dry_run]",
            "      properties: {uid: {type: string, format: uuid}, idempotency_key: {type: string}, connector_id: {type: string}, connector_version: {type: string}, source_uid: {type: string, format: uuid}, principal_uid: {type: [string, 'null'], format: uuid}, business_domain_uid: {type: [string, 'null'], format: uuid}, environment: {type: [string, 'null']}, process_key: {type: [string, 'null']}, operation: {type: string}, status: {type: string}, attempt_count: {type: integer}, checkpoint_summary: {type: object}, cursor_summary: {type: object}, scope: {type: object}, error_category: {type: [string, 'null']}, dry_run: {type: boolean}, created_at: {type: string}, updated_at: {type: string}}",
            "    ConnectorRunRecordEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties: {code: {type: integer}, message: {type: string}, data: {$ref: '#/components/schemas/ConnectorRunRecord'}}",
            "    ConnectorRunListEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties: {code: {type: integer}, message: {type: string}, data: {type: object, additionalProperties: false, required: [runs], properties: {runs: {type: array, items: {$ref: '#/components/schemas/ConnectorRunRecord'}}}}}",
            "    ConnectorValidationEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties: {code: {type: integer}, message: {type: string}, data: {type: object, required: [valid, config_keys], properties: {valid: {const: true}, config_keys: {type: array, items: {type: string}}}, additionalProperties: false}}",
            "    ConnectorErrorEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data, error]",
            "      properties:",
            "        code: {type: integer}",
            "        message: {type: string}",
            "        data: {type: 'null'}",
            "        error: {type: object, additionalProperties: false, required: [code, category, retryable], properties: {code: {const: CONNECTOR_ERROR}, category: {type: string, enum: [configuration, conflict, authentication, permission, rate_limit, timeout, upstream, contract, cancelled]}, retryable: {type: boolean}}}",
            "    ConnectorHealthEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties: {code: {type: integer}, message: {type: string}, data: {type: object, additionalProperties: false, required: [status, detail], properties: {status: {type: string}, detail: {type: string}}}}",
            "    ConnectorCompatibilityEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties: {code: {type: integer}, message: {type: string}, data: {type: object, additionalProperties: false, required: [compatible, connector_version, sdk_version, detail], properties: {compatible: {type: boolean}, connector_version: {type: string}, sdk_version: {type: string}, detail: {type: string}}}}",
            "    ConnectorPrincipalEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties: {code: {type: integer}, message: {type: string}, data: {type: object, additionalProperties: false, required: [principal_uid], properties: {principal_uid: {type: string, format: uuid}}}}",
            "    ConnectorSourceBindingEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties: {code: {type: integer}, message: {type: string}, data: {type: object, additionalProperties: false, required: [binding_uid, binding_version, status, rebound_principals], properties: {binding_uid: {type: string, format: uuid}, binding_version: {type: integer, minimum: 1}, status: {const: approved}, rebound_principals: {type: integer, minimum: 0}}}}",
            "    ConnectorCredentialEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties: {code: {type: integer}, message: {type: string}, data: {oneOf: [{type: object, additionalProperties: false, required: [credential_uid, credential, expires_in, returned_once], properties: {credential_uid: {type: string, format: uuid}, credential: {type: string, writeOnly: true}, expires_in: {type: integer}, returned_once: {const: true}}}, {type: object, additionalProperties: false, required: [revoked], properties: {revoked: {type: boolean}}}]}}",
            "    ConnectorGraphEnvelope:",
            "      type: object",
            "      additionalProperties: false",
            "      required: [code, message, data]",
            "      properties: {code: {type: integer}, message: {type: string}, data: {type: object, additionalProperties: false, required: [nodes, edges, summary], properties: {nodes: {type: array, items: {type: object, additionalProperties: false, required: [type, key], properties: {type: {type: string, enum: [source, asset, process, business_domain, run]}, key: {type: string}}}}, edges: {type: array, items: {type: object}}, summary: {type: object, additionalProperties: false, required: [node_count, edge_count], properties: {node_count: {type: integer}, edge_count: {type: integer}}}}}}",
            "  securitySchemes:",
            "    bearerAuth:",
            "      type: http",
            "      scheme: bearer",
            "      bearerFormat: JWT",
            '      description: "下一阶段统一认证方案；当前路由尚未全部接入。"',
        ]
    )
    return "\n".join(lines) + "\n"


def main(output: Path = OUTPUT) -> None:
    routes = extract_routes()
    if not routes:
        raise SystemExit("No Flask routes found")
    output.parent.mkdir(parents=True, exist_ok=True)
    output.write_text(render(routes), encoding="utf-8")
    try:
        label = output.relative_to(ROOT)
    except ValueError:
        label = output
    print(f"Generated {label} with {len(routes)} operations")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--output", type=Path, default=OUTPUT)
    args = parser.parse_args()
    main(args.output)