config.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724
  1. import os
  2. import platform
  3. from collections.abc import Mapping
  4. from app.config.database_urls import validate_postgresql_url
  5. def get_bool_env(name: str, default: bool = False) -> bool:
  6. """Return a boolean value from an environment variable."""
  7. value = os.environ.get(name)
  8. if value is None:
  9. return default
  10. return value.strip().lower() in {"1", "true", "yes", "on"}
  11. def _bounded_int(name: str, default: int, minimum: int, maximum: int) -> int:
  12. try:
  13. value = int(os.environ.get(name, str(default)))
  14. except (TypeError, ValueError) as exc:
  15. raise ValueError(f"{name} must be an integer") from exc
  16. if value < minimum or value > maximum:
  17. raise ValueError(f"{name} must be between {minimum} and {maximum}")
  18. return value
  19. def _apply_datasource_pool_overrides(
  20. settings: dict, overrides: Mapping[str, int]
  21. ) -> dict:
  22. bounds = {
  23. "pool_size": (1, 10),
  24. "max_overflow": (0, 10),
  25. }
  26. unsupported = sorted(set(overrides) - set(bounds))
  27. if unsupported:
  28. raise ValueError(
  29. f"Unsupported data source pool overrides: {', '.join(unsupported)}"
  30. )
  31. resolved = dict(settings)
  32. for name, value in overrides.items():
  33. minimum, maximum = bounds[name]
  34. if type(value) is not int or value < minimum or value > maximum:
  35. raise ValueError(f"{name} must be between {minimum} and {maximum}")
  36. resolved[name] = value
  37. return resolved
  38. def datasource_pool_settings(
  39. overrides: Mapping[str, int] = None,
  40. ) -> dict:
  41. """Return bounded settings for external business data-source pools."""
  42. settings = {
  43. "pool_size": _bounded_int("DATASOURCE_POOL_SIZE", 2, 1, 10),
  44. "max_overflow": _bounded_int("DATASOURCE_MAX_OVERFLOW", 3, 0, 10),
  45. "pool_timeout": _bounded_int("DATASOURCE_POOL_TIMEOUT", 10, 1, 60),
  46. "pool_recycle": _bounded_int(
  47. "DATASOURCE_POOL_RECYCLE", 1800, 60, 86400
  48. ),
  49. "idle_ttl": _bounded_int(
  50. "DATASOURCE_POOL_IDLE_TTL", 900, 60, 86400
  51. ),
  52. "max_idle_pools": _bounded_int(
  53. "DATASOURCE_MAX_IDLE_POOLS", 20, 1, 100
  54. ),
  55. "query_timeout": _bounded_int(
  56. "DATASOURCE_QUERY_TIMEOUT", 30, 1, 300
  57. ),
  58. }
  59. return _apply_datasource_pool_overrides(settings, overrides or {})
  60. def get_environment():
  61. """
  62. 获取当前运行环境
  63. 优先级:
  64. 1. 环境变量 FLASK_ENV
  65. 2. 根据操作系统自动判断(Windows -> development, Linux -> production)
  66. """
  67. # 首先检查环境变量
  68. env = os.environ.get("FLASK_ENV")
  69. if env:
  70. return env.lower()
  71. # 根据操作系统判断
  72. system = platform.system().lower()
  73. if system == "windows":
  74. return "development"
  75. elif system == "linux":
  76. return "production"
  77. else:
  78. return "development" # 其他系统默认使用开发环境
  79. def resolve_log_file(default_filename: str = "flask_production.log") -> str:
  80. """Resolve application log path to an absolute file under LOG_DIR."""
  81. raw = os.environ.get("LOG_FILE", default_filename)
  82. if os.path.isabs(raw):
  83. return raw
  84. app_dir = os.environ.get("APP_DIR", os.getcwd())
  85. log_dir = os.environ.get("LOG_DIR", os.path.join(app_dir, "logs"))
  86. return os.path.join(log_dir, os.path.basename(raw))
  87. def _clean_env(name: str, default: str = "") -> str:
  88. value = os.environ.get(name)
  89. if value is None:
  90. return default
  91. return value.strip().strip("\r\n\t")
  92. def load_production_env_file() -> None:
  93. """Load production env file into os.environ without overriding existing keys."""
  94. env_file = os.environ.get(
  95. "APP_ENV_FILE",
  96. os.environ.get("ENV_FILE", "/etc/dataops-platform/dataops.env"),
  97. )
  98. if not env_file or not os.path.isfile(env_file):
  99. return
  100. if not os.access(env_file, os.R_OK):
  101. return
  102. try:
  103. from dotenv import load_dotenv
  104. load_dotenv(env_file, override=False)
  105. except Exception:
  106. return
  107. def is_placeholder_env_value(value: str) -> bool:
  108. """Detect template placeholders that should not override production defaults."""
  109. if not value:
  110. return True
  111. lower = value.lower()
  112. if lower.startswith("replace-"):
  113. return True
  114. placeholders = (
  115. "replace-password",
  116. "replace-neo4j-password",
  117. "replace-minio",
  118. "replace-with-a-long-random-secret",
  119. "replace-n8n-api-key",
  120. "your-api-key",
  121. )
  122. if any(item in lower for item in placeholders):
  123. return True
  124. if "dataops_user@" in lower and "127.0.0.1" in lower:
  125. return True
  126. return lower in {"127.0.0.1:9000", "localhost:9000"}
  127. def _is_local_minio_host(host: str) -> bool:
  128. if not host:
  129. return False
  130. return host.split(":")[0].lower() in {"127.0.0.1", "localhost"}
  131. def _apply_minio_config(app, defaults: dict) -> None:
  132. """Apply MinIO settings as a whole to avoid host/user/password mismatch."""
  133. host = _clean_env("MINIO_HOST")
  134. user = _clean_env("MINIO_USER")
  135. password = _clean_env("MINIO_PASSWORD")
  136. bucket = _clean_env("MINIO_BUCKET")
  137. prefix = _clean_env("MINIO_PREFIX")
  138. is_production = str(app.config.get("FLASK_ENV", "")).lower() == "production"
  139. if is_production:
  140. app.config["MINIO_HOST"] = host
  141. app.config["MINIO_USER"] = user
  142. app.config["MINIO_PASSWORD"] = password
  143. app.config["MINIO_BUCKET"] = bucket
  144. app.config["PREFIX"] = prefix
  145. app.config["MINIO_SECURE"] = get_bool_env("MINIO_SECURE", False)
  146. return
  147. use_defaults = (
  148. is_placeholder_env_value(user)
  149. or is_placeholder_env_value(password)
  150. or is_placeholder_env_value(host)
  151. or (is_production and _is_local_minio_host(host))
  152. )
  153. if use_defaults:
  154. app.config["MINIO_HOST"] = defaults["MINIO_HOST"]
  155. app.config["MINIO_USER"] = defaults["MINIO_USER"]
  156. app.config["MINIO_PASSWORD"] = defaults["MINIO_PASSWORD"]
  157. app.config["MINIO_BUCKET"] = defaults["MINIO_BUCKET"]
  158. app.config["PREFIX"] = defaults["MINIO_PREFIX"]
  159. app.config["MINIO_SECURE"] = defaults["MINIO_SECURE"]
  160. return
  161. app.config["MINIO_HOST"] = host or defaults["MINIO_HOST"]
  162. app.config["MINIO_USER"] = user or defaults["MINIO_USER"]
  163. app.config["MINIO_PASSWORD"] = password or defaults["MINIO_PASSWORD"]
  164. app.config["MINIO_BUCKET"] = bucket or defaults["MINIO_BUCKET"]
  165. app.config["PREFIX"] = prefix if prefix else defaults["MINIO_PREFIX"]
  166. app.config["MINIO_SECURE"] = get_bool_env("MINIO_SECURE", defaults["MINIO_SECURE"])
  167. PRODUCTION_SERVICE_DEFAULTS = {
  168. "SQLALCHEMY_DATABASE_URI": "postgresql://postgres:postgres@127.0.0.1:5432/dataops",
  169. "NEO4J_URI": "bolt://127.0.0.1:7687",
  170. "NEO4J_HTTP_URI": "http://127.0.0.1:7474",
  171. "NEO4J_USER": "neo4j",
  172. "NEO4J_PASSWORD": "Passw0rd",
  173. "NEO4J_ENCRYPTED": False,
  174. "MINIO_HOST": "127.0.0.1:9000",
  175. "MINIO_USER": "citu-test",
  176. "MINIO_PASSWORD": "citu-test",
  177. "MINIO_SECURE": False,
  178. "MINIO_BUCKET": "dataops-bucket",
  179. "MINIO_PREFIX": "",
  180. }
  181. _PRODUCTION_REQUIRED_SERVICE_ENV = (
  182. "DATABASE_URL",
  183. "BI_AI_CATALOG_CONTROL_DATABASE_URL",
  184. "NEO4J_URI",
  185. "NEO4J_HTTP_URI",
  186. "NEO4J_USER",
  187. "NEO4J_PASSWORD",
  188. "MINIO_HOST",
  189. "MINIO_USER",
  190. "MINIO_PASSWORD",
  191. "MINIO_BUCKET",
  192. )
  193. def _validate_production_service_env() -> None:
  194. """Reject missing, template, or malformed production service settings."""
  195. values = {name: _clean_env(name) for name in _PRODUCTION_REQUIRED_SERVICE_ENV}
  196. runtime = validate_postgresql_url(values["DATABASE_URL"], "DATABASE_URL")
  197. control = validate_postgresql_url(values["BI_AI_CATALOG_CONTROL_DATABASE_URL"], "BI_AI_CATALOG_CONTROL_DATABASE_URL")
  198. if control.username != "dataops_bi_ai_catalog_control" or control.username == runtime.username:
  199. raise RuntimeError("production BI_AI_CATALOG_CONTROL_DATABASE_URL must use the dedicated control identity")
  200. for name, value in values.items():
  201. if name in {"DATABASE_URL", "BI_AI_CATALOG_CONTROL_DATABASE_URL"}:
  202. continue
  203. placeholder = is_placeholder_env_value(value)
  204. if name in {"NEO4J_URI", "NEO4J_HTTP_URI", "MINIO_HOST"}:
  205. placeholder = not value or value.lower().startswith("replace-")
  206. if placeholder:
  207. raise RuntimeError(f"production {name} is required and cannot be a placeholder")
  208. def _apply_config_from_env(app, config_key: str, env_name: str, default):
  209. value = _clean_env(env_name)
  210. if value and not is_placeholder_env_value(value):
  211. app.config[config_key] = value
  212. return value
  213. app.config[config_key] = default
  214. return default
  215. def apply_runtime_env_config(app) -> None:
  216. """Re-read env-backed settings when the worker starts (after dataops.env is loaded)."""
  217. load_production_env_file()
  218. defaults = PRODUCTION_SERVICE_DEFAULTS
  219. is_production = str(app.config.get("FLASK_ENV", "")).lower() == "production"
  220. if is_production:
  221. _validate_production_service_env()
  222. app.config["SQLALCHEMY_DATABASE_URI"] = _clean_env("DATABASE_URL")
  223. for config_key, env_name in (
  224. ("NEO4J_URI", "NEO4J_URI"),
  225. ("NEO4J_HTTP_URI", "NEO4J_HTTP_URI"),
  226. ("NEO4J_USER", "NEO4J_USER"),
  227. ("NEO4J_PASSWORD", "NEO4J_PASSWORD"),
  228. ):
  229. app.config[config_key] = _clean_env(env_name)
  230. app.config["NEO4J_ENCRYPTED"] = get_bool_env("NEO4J_ENCRYPTED", True)
  231. _apply_minio_config(app, defaults)
  232. else:
  233. _apply_config_from_env(
  234. app, "SQLALCHEMY_DATABASE_URI", "DATABASE_URL", defaults["SQLALCHEMY_DATABASE_URI"]
  235. )
  236. _apply_config_from_env(app, "NEO4J_URI", "NEO4J_URI", defaults["NEO4J_URI"])
  237. _apply_config_from_env(
  238. app, "NEO4J_HTTP_URI", "NEO4J_HTTP_URI", defaults["NEO4J_HTTP_URI"]
  239. )
  240. _apply_config_from_env(app, "NEO4J_USER", "NEO4J_USER", defaults["NEO4J_USER"])
  241. _apply_config_from_env(
  242. app, "NEO4J_PASSWORD", "NEO4J_PASSWORD", defaults["NEO4J_PASSWORD"]
  243. )
  244. app.config["NEO4J_ENCRYPTED"] = get_bool_env(
  245. "NEO4J_ENCRYPTED", defaults["NEO4J_ENCRYPTED"]
  246. )
  247. _apply_minio_config(app, defaults)
  248. secret_key = _clean_env("SECRET_KEY")
  249. if secret_key and not is_placeholder_env_value(secret_key):
  250. app.config["SECRET_KEY"] = secret_key
  251. deepseek_key = _clean_env("DEEPSEEK_API_KEY")
  252. llm_key = deepseek_key or _clean_env("LLM_API_KEY")
  253. if deepseek_key and not is_placeholder_env_value(deepseek_key):
  254. app.config["DEEPSEEK_API_KEY"] = deepseek_key
  255. app.config["LLM_API_KEY"] = deepseek_key
  256. elif llm_key and not is_placeholder_env_value(llm_key):
  257. app.config["LLM_API_KEY"] = llm_key
  258. _apply_config_from_env(
  259. app,
  260. "LLM_BASE_URL",
  261. "LLM_BASE_URL",
  262. "https://api.deepseek.com",
  263. )
  264. llm_model = _clean_env("LLM_MODEL_NAME")
  265. if llm_model:
  266. app.config["LLM_MODEL_NAME"] = llm_model
  267. llm_reasoning = _clean_env("LLM_REASONING_EFFORT")
  268. if llm_reasoning:
  269. app.config["LLM_REASONING_EFFORT"] = llm_reasoning
  270. pool_settings = datasource_pool_settings()
  271. app.config.update(
  272. {
  273. "DATASOURCE_POOL_SIZE": pool_settings["pool_size"],
  274. "DATASOURCE_MAX_OVERFLOW": pool_settings["max_overflow"],
  275. "DATASOURCE_POOL_TIMEOUT": pool_settings["pool_timeout"],
  276. "DATASOURCE_POOL_RECYCLE": pool_settings["pool_recycle"],
  277. "DATASOURCE_POOL_IDLE_TTL": pool_settings["idle_ttl"],
  278. "DATASOURCE_MAX_IDLE_POOLS": pool_settings["max_idle_pools"],
  279. "DATASOURCE_QUERY_TIMEOUT": pool_settings["query_timeout"],
  280. "DATASOURCE_CREDENTIAL_MASTER_KEY": _clean_env(
  281. "DATASOURCE_CREDENTIAL_MASTER_KEY"
  282. ),
  283. "DATASOURCE_CREDENTIAL_KEY_VERSION": _clean_env(
  284. "DATASOURCE_CREDENTIAL_KEY_VERSION", "v1"
  285. ),
  286. "DATASOURCE_CERT_DIR": _clean_env(
  287. "DATASOURCE_CERT_DIR",
  288. "/etc/dataops-platform/datasource-certs",
  289. ),
  290. "AUDIT_EVIDENCE_SECRET": _clean_env(
  291. "AUDIT_EVIDENCE_SECRET"
  292. ),
  293. "AUDIT_EVIDENCE_KEY_VERSION": _clean_env(
  294. "AUDIT_EVIDENCE_KEY_VERSION", "local-fallback-v1"
  295. ),
  296. "SECURITY_SIEM_HOST_ALLOWLIST": _clean_env(
  297. "SECURITY_SIEM_HOST_ALLOWLIST"
  298. ),
  299. "AGENT_CREDENTIAL_SECRET": _clean_env("AGENT_CREDENTIAL_SECRET"),
  300. }
  301. )
  302. def validate_production_database_identity(app, engine) -> None:
  303. """Verify the connected production login has runtime-only database powers."""
  304. if str(app.config.get("FLASK_ENV", "")).lower() != "production":
  305. return
  306. from sqlalchemy import text
  307. try:
  308. with engine.connect() as connection:
  309. identity = connection.execute(text("""
  310. SELECT current_user AS role_name,
  311. role.rolsuper,role.rolcreaterole,role.rolcreatedb,
  312. role.rolreplication,role.rolbypassrls,
  313. EXISTS (
  314. SELECT 1 FROM pg_class object
  315. JOIN pg_namespace namespace ON namespace.oid=object.relnamespace
  316. WHERE object.relowner=role.oid
  317. AND namespace.nspname NOT IN ('pg_catalog','information_schema')
  318. AND namespace.nspname NOT LIKE 'pg_toast%'
  319. ) OR EXISTS (
  320. SELECT 1 FROM pg_proc function
  321. JOIN pg_namespace namespace ON namespace.oid=function.pronamespace
  322. WHERE function.proowner=role.oid
  323. AND namespace.nspname NOT IN ('pg_catalog','information_schema')
  324. ) OR EXISTS (
  325. SELECT 1 FROM pg_namespace namespace
  326. WHERE namespace.nspowner=role.oid
  327. AND namespace.nspname NOT IN ('pg_catalog','information_schema')
  328. AND namespace.nspname NOT LIKE 'pg_%'
  329. ) AS owns_application_objects,
  330. EXISTS (
  331. SELECT 1 FROM pg_roles evidence_owner
  332. WHERE evidence_owner.rolname='dataops_edge_evidence_owner'
  333. AND pg_has_role(current_user,evidence_owner.oid,'MEMBER')
  334. ) AS evidence_owner_member,
  335. EXISTS (
  336. SELECT 1 FROM pg_roles trusted_owner
  337. WHERE trusted_owner.rolname IN (
  338. 'dataops_trusted_delivery_owner',
  339. 'dataops_trusted_delivery_writer'
  340. )
  341. AND pg_has_role(current_user,trusted_owner.oid,'MEMBER')
  342. ) AS trusted_delivery_owner_member
  343. FROM pg_roles role WHERE role.rolname=current_user
  344. """)).mappings().one()
  345. except Exception as exc:
  346. raise RuntimeError("production runtime database identity verification failed") from exc
  347. forbidden = (
  348. identity["rolsuper"] or identity["rolcreaterole"] or identity["rolcreatedb"]
  349. or identity["rolreplication"] or identity["rolbypassrls"]
  350. or identity["owns_application_objects"] or identity["evidence_owner_member"]
  351. or identity["trusted_delivery_owner_member"]
  352. )
  353. if forbidden:
  354. raise RuntimeError(
  355. "production DATABASE_URL must use a non-owner, least-privilege runtime identity"
  356. )
  357. def log_llm_env_status(app) -> None:
  358. """Log LLM env load result after logging is configured."""
  359. deepseek_key = _clean_env("DEEPSEEK_API_KEY") or _clean_env("LLM_API_KEY")
  360. if not deepseek_key:
  361. deepseek_key = str(
  362. app.config.get("DEEPSEEK_API_KEY") or app.config.get("LLM_API_KEY") or ""
  363. ).strip().strip("\r\n\t")
  364. env_file = os.environ.get("APP_ENV_FILE", "/etc/dataops-platform/dataops.env")
  365. if deepseek_key:
  366. app.logger.info(
  367. "DeepSeek API Key 已加载 (长度=%s, base_url=%s, model=%s)",
  368. len(deepseek_key),
  369. app.config.get("LLM_BASE_URL", "https://api.deepseek.com"),
  370. app.config.get("LLM_MODEL_NAME", "deepseek-chat"),
  371. )
  372. return
  373. if os.path.isfile(env_file) and not os.access(env_file, os.R_OK):
  374. app.logger.error(
  375. f"无法读取 {env_file},Supervisor 用户需有读权限。"
  376. f"请执行: sudo chown root:{os.environ.get('APP_USER', 'ubuntu')} "
  377. f"{env_file} && sudo chmod 640 {env_file}"
  378. )
  379. return
  380. app.logger.warning(
  381. f"DeepSeek API Key 未配置,请在 {env_file} 中设置 DEEPSEEK_API_KEY"
  382. )
  383. def log_service_env_status(app) -> None:
  384. """Log database/Neo4j endpoints after logging is configured."""
  385. db_uri = str(app.config.get("SQLALCHEMY_DATABASE_URI", ""))
  386. db_host = db_uri.split("@")[-1] if "@" in db_uri else db_uri
  387. app.logger.info(
  388. "服务连接配置: PostgreSQL=%s, Neo4j=%s, MinIO=%s (user=%s)",
  389. db_host,
  390. app.config.get("NEO4J_URI"),
  391. app.config.get("MINIO_HOST"),
  392. _mask_secret(str(app.config.get("MINIO_USER", ""))),
  393. )
  394. if str(app.config.get("FLASK_ENV", "")).lower() == "production":
  395. return
  396. if is_placeholder_env_value(_clean_env("DATABASE_URL")):
  397. app.logger.warning(
  398. "DATABASE_URL 仍为模板占位符,已回退到默认生产库 "
  399. f"{PRODUCTION_SERVICE_DEFAULTS['SQLALCHEMY_DATABASE_URI'].split('@')[-1]}"
  400. )
  401. if is_placeholder_env_value(_clean_env("NEO4J_PASSWORD")):
  402. app.logger.warning(
  403. "NEO4J_PASSWORD 仍为模板占位符,已回退到 config.py 中的默认生产配置"
  404. )
  405. minio_user_env = _clean_env("MINIO_USER")
  406. minio_host_env = _clean_env("MINIO_HOST")
  407. if (
  408. is_placeholder_env_value(minio_user_env)
  409. or is_placeholder_env_value(minio_host_env)
  410. or (
  411. os.environ.get("FLASK_ENV", "").lower() == "production"
  412. and _is_local_minio_host(minio_host_env)
  413. )
  414. ):
  415. app.logger.warning(
  416. "MinIO 配置仍为模板或 localhost,已回退到默认生产 MinIO "
  417. f"{PRODUCTION_SERVICE_DEFAULTS['MINIO_HOST']}"
  418. )
  419. def _mask_secret(value: str) -> str:
  420. if not value:
  421. return "(empty)"
  422. if len(value) <= 4:
  423. return "****"
  424. return f"{value[:4]}****"
  425. class BaseConfig:
  426. """基础配置类,包含所有环境共享的配置"""
  427. SECRET_KEY = os.environ.get("SECRET_KEY") or "you-will-never-guess"
  428. RULE_GENERATION_RECEIPT_SECRET = os.environ.get(
  429. "RULE_GENERATION_RECEIPT_SECRET"
  430. )
  431. JSON_AS_ASCII = False
  432. JSONIFY_PRETTYPRINT_REGULAR = True
  433. JSON_SORT_KEYS = False
  434. # 平台特定配置
  435. PLATFORM = platform.system().lower()
  436. # 文件上传配置
  437. ALLOWED_EXTENSIONS = {
  438. "txt",
  439. "pdf",
  440. "png",
  441. "jpg",
  442. "jpeg",
  443. "gif",
  444. "xlsx",
  445. "xls",
  446. "csv",
  447. "sql",
  448. "dll",
  449. "docx",
  450. "doc",
  451. }
  452. DATA_RESEARCH_FILE_MAX_BYTES = int(
  453. os.environ.get("DATA_RESEARCH_FILE_MAX_BYTES", str(25 * 1024 * 1024))
  454. )
  455. DATA_RESEARCH_FILE_MAX_PAGES = int(
  456. os.environ.get("DATA_RESEARCH_FILE_MAX_PAGES", "500")
  457. )
  458. DATA_RESEARCH_MAX_CONCURRENT_JOBS = int(
  459. os.environ.get("DATA_RESEARCH_MAX_CONCURRENT_JOBS", "10")
  460. )
  461. # 数据抽取配置
  462. DATA_EXTRACT_BATCH_SIZE = 1000 # 每批处理的记录数
  463. # PostgreSQL 基础配置
  464. SQLALCHEMY_ENGINE_OPTIONS = {
  465. "pool_pre_ping": True,
  466. "pool_recycle": 300,
  467. "pool_size": 10,
  468. "max_overflow": 20,
  469. }
  470. _DATASOURCE_POOL_SETTINGS = datasource_pool_settings()
  471. DATASOURCE_POOL_SIZE = _DATASOURCE_POOL_SETTINGS["pool_size"]
  472. DATASOURCE_MAX_OVERFLOW = _DATASOURCE_POOL_SETTINGS["max_overflow"]
  473. DATASOURCE_POOL_TIMEOUT = _DATASOURCE_POOL_SETTINGS["pool_timeout"]
  474. DATASOURCE_POOL_RECYCLE = _DATASOURCE_POOL_SETTINGS["pool_recycle"]
  475. DATASOURCE_POOL_IDLE_TTL = _DATASOURCE_POOL_SETTINGS["idle_ttl"]
  476. DATASOURCE_MAX_IDLE_POOLS = _DATASOURCE_POOL_SETTINGS["max_idle_pools"]
  477. DATASOURCE_QUERY_TIMEOUT = _DATASOURCE_POOL_SETTINGS["query_timeout"]
  478. DATASOURCE_CREDENTIAL_MASTER_KEY = os.environ.get(
  479. "DATASOURCE_CREDENTIAL_MASTER_KEY", ""
  480. )
  481. DATASOURCE_CREDENTIAL_KEY_VERSION = os.environ.get(
  482. "DATASOURCE_CREDENTIAL_KEY_VERSION", "v1"
  483. )
  484. AGENT_CREDENTIAL_SECRET = os.environ.get("AGENT_CREDENTIAL_SECRET", "")
  485. # Ed25519 raw-private-key hex or PEM. It is loaded only into process memory
  486. # and must never be persisted in a task, release, audit, or HTTP response.
  487. EDGE_GATEWAY_SIGNING_PRIVATE_KEY = os.environ.get(
  488. "EDGE_GATEWAY_SIGNING_PRIVATE_KEY", ""
  489. )
  490. EDGE_GATEWAY_SIGNING_PRIVATE_KEY_FILE = os.environ.get(
  491. "EDGE_GATEWAY_SIGNING_PRIVATE_KEY_FILE", ""
  492. )
  493. EDGE_GATEWAY_SIGNING_PUBLIC_KEY = os.environ.get(
  494. "EDGE_GATEWAY_SIGNING_PUBLIC_KEY", ""
  495. )
  496. EDGE_GATEWAY_SIGNING_KEY_ID = os.environ.get(
  497. "EDGE_GATEWAY_SIGNING_KEY_ID", ""
  498. )
  499. EDGE_MTLS_TRUSTED_PROXY_IPS = os.environ.get(
  500. "EDGE_MTLS_TRUSTED_PROXY_IPS", ""
  501. )
  502. DATASOURCE_CERT_DIR = os.environ.get(
  503. "DATASOURCE_CERT_DIR",
  504. "/etc/dataops-platform/datasource-certs",
  505. )
  506. # DeepSeek LLM(OpenAI 兼容接口)
  507. DEEPSEEK_API_KEY = os.environ.get("DEEPSEEK_API_KEY", "")
  508. LLM_BASE_URL = os.environ.get("LLM_BASE_URL", "https://api.deepseek.com")
  509. LLM_MODEL_NAME = os.environ.get("LLM_MODEL_NAME", "deepseek-chat")
  510. LLM_REASONING_EFFORT = os.environ.get("LLM_REASONING_EFFORT", "high")
  511. # 兼容旧环境变量名 LLM_API_KEY
  512. LLM_API_KEY = DEEPSEEK_API_KEY or os.environ.get("LLM_API_KEY", "")
  513. # 数据治理知识库
  514. KNOWLEDGE_ENABLED = os.environ.get("KNOWLEDGE_ENABLED", "true").lower() == "true"
  515. KNOWLEDGE_VECTOR_TOP_K = int(os.environ.get("KNOWLEDGE_VECTOR_TOP_K", "40"))
  516. KNOWLEDGE_RRF_K = int(os.environ.get("KNOWLEDGE_RRF_K", "60"))
  517. KNOWLEDGE_RERANK_TOP_K = int(os.environ.get("KNOWLEDGE_RERANK_TOP_K", "8"))
  518. KNOWLEDGE_EVIDENCE_TOKEN_BUDGET = int(
  519. os.environ.get("KNOWLEDGE_EVIDENCE_TOKEN_BUDGET", "8000")
  520. )
  521. KNOWLEDGE_IMPACT_MAX_HOPS = int(os.environ.get("KNOWLEDGE_IMPACT_MAX_HOPS", "3"))
  522. KNOWLEDGE_IMPACT_MAX_POINTS = int(
  523. os.environ.get("KNOWLEDGE_IMPACT_MAX_POINTS", "1000")
  524. )
  525. KNOWLEDGE_LIGHTRAG_ENABLED = (
  526. os.environ.get("KNOWLEDGE_LIGHTRAG_ENABLED", "false").lower() == "true"
  527. )
  528. KNOWLEDGE_LIGHTRAG_SHADOW_ONLY = (
  529. os.environ.get("KNOWLEDGE_LIGHTRAG_SHADOW_ONLY", "true").lower() == "true"
  530. )
  531. KNOWLEDGE_LIGHTRAG_BASE_URL = os.environ.get(
  532. "KNOWLEDGE_LIGHTRAG_BASE_URL", "http://lightrag:9621"
  533. )
  534. KNOWLEDGE_LIGHTRAG_API_KEY = os.environ.get("KNOWLEDGE_LIGHTRAG_API_KEY", "")
  535. QWEN_EMBEDDING_BASE_URL = os.environ.get("QWEN_EMBEDDING_BASE_URL", "")
  536. QWEN_EMBEDDING_API_KEY = os.environ.get("QWEN_EMBEDDING_API_KEY", "")
  537. QWEN_EMBEDDING_MODEL = os.environ.get("QWEN_EMBEDDING_MODEL", "text-embedding-v3")
  538. QWEN_EMBEDDING_DIMENSION = int(os.environ.get("QWEN_EMBEDDING_DIMENSION", "1024"))
  539. # 日志基础配置
  540. LOG_FORMAT = "%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(lineno)s - %(message)s"
  541. LOG_ENCODING = "UTF-8"
  542. LOG_ENABLED = True
  543. # DataFlow 配置
  544. DATAFLOW_SCHEMA = os.environ.get("DATAFLOW_SCHEMA", "dags")
  545. # n8n 工作流引擎配置
  546. N8N_API_URL = os.environ.get("N8N_API_URL", "")
  547. N8N_API_KEY = os.environ.get("N8N_API_KEY", "")
  548. N8N_API_TIMEOUT = int(os.environ.get("N8N_API_TIMEOUT", "30"))
  549. # Kestra-backed governed Data Factory. Activation remains fail-closed
  550. # unless the operator explicitly opens the post-acceptance gate.
  551. KESTRA_BASE_URL = os.environ.get(
  552. "KESTRA_BASE_URL", "http://127.0.0.1:18080/api/v1"
  553. )
  554. KESTRA_TENANT_ID = os.environ.get("KESTRA_TENANT_ID", "main")
  555. KESTRA_USERNAME = os.environ.get("KESTRA_USERNAME", "")
  556. KESTRA_PASSWORD = os.environ.get("KESTRA_PASSWORD", "")
  557. KESTRA_HTTP_TIMEOUT_SECONDS = int(
  558. os.environ.get("KESTRA_HTTP_TIMEOUT_SECONDS", "30")
  559. )
  560. RUNNER_TASK_TOKEN_SECRET = os.environ.get(
  561. "RUNNER_TASK_TOKEN_SECRET", ""
  562. )
  563. DATA_FACTORY_ACTIVATION_ENABLED = get_bool_env(
  564. "DATA_FACTORY_ACTIVATION_ENABLED", False
  565. )
  566. DEVICE_ENTITY_AUTO_MERGE_ENABLED = get_bool_env(
  567. "DEVICE_ENTITY_AUTO_MERGE_ENABLED", False
  568. )
  569. # DataOps 平台 API 基础 URL(用于 n8n 工作流回调等)
  570. API_BASE_URL = os.environ.get("API_BASE_URL", "http://127.0.0.1:5500/api")
  571. class DevelopmentConfig(BaseConfig):
  572. """Windows 开发环境配置"""
  573. FLASK_ENV = "development"
  574. DEBUG = True
  575. PORT = 5500
  576. # 开发环境 MinIO 配置
  577. MINIO_HOST = "localhost:9000"
  578. MINIO_USER = "citu-test"
  579. MINIO_PASSWORD = "citu-test"
  580. MINIO_SECURE = False
  581. MINIO_BUCKET = "dataops-bucket"
  582. PREFIX = ""
  583. # 开发环境 PostgreSQL 配置
  584. SQLALCHEMY_DATABASE_URI = "postgresql://postgres:postgres@localhost:5432/dataops"
  585. # 开发环境 Neo4j 配置
  586. NEO4J_URI = "bolt://localhost:7687"
  587. NEO4J_HTTP_URI = "http://localhost:7474"
  588. NEO4J_USER = "neo4j"
  589. NEO4J_PASSWORD = "Passw0rd"
  590. NEO4J_ENCRYPTED = False
  591. # 开发环境文件路径配置
  592. UPLOAD_BASE_PATH = "C:\\tmp\\upload"
  593. ARCHIVE_BASE_PATH = "C:\\tmp\\archive"
  594. # 开发环境日志配置
  595. LOG_LEVEL = "DEBUG"
  596. LOG_FILE = resolve_log_file("flask_development.log")
  597. LOG_TO_CONSOLE = True
  598. class ProductionConfig(BaseConfig):
  599. """Linux 生产环境配置"""
  600. FLASK_ENV = "production"
  601. DEBUG = False
  602. # 与 run_dataops.sh / dataops.env 中 LISTEN_PORT 保持一致(默认 5500,供 Nginx 反代)
  603. PORT = int(os.environ.get("LISTEN_PORT", os.environ.get("PORT", "5500")))
  604. # 生产环境 MinIO 配置
  605. MINIO_HOST = os.environ.get("MINIO_HOST", "")
  606. MINIO_USER = os.environ.get("MINIO_USER", "")
  607. MINIO_PASSWORD = os.environ.get("MINIO_PASSWORD", "")
  608. MINIO_SECURE = get_bool_env("MINIO_SECURE", False)
  609. MINIO_BUCKET = os.environ.get("MINIO_BUCKET", "dataops-bucket")
  610. PREFIX = os.environ.get("MINIO_PREFIX", "")
  611. # 生产环境 PostgreSQL 配置
  612. SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL", "")
  613. # 生产环境 Neo4j 配置
  614. NEO4J_URI = os.environ.get("NEO4J_URI", "bolt://127.0.0.1:7687")
  615. NEO4J_HTTP_URI = os.environ.get("NEO4J_HTTP_URI", "http://127.0.0.1:7474")
  616. NEO4J_USER = os.environ.get("NEO4J_USER", "")
  617. NEO4J_PASSWORD = os.environ.get("NEO4J_PASSWORD", "")
  618. NEO4J_ENCRYPTED = get_bool_env("NEO4J_ENCRYPTED", False)
  619. # 生产环境文件路径配置
  620. UPLOAD_BASE_PATH = os.environ.get("UPLOAD_BASE_PATH", "/data/upload")
  621. ARCHIVE_BASE_PATH = os.environ.get("ARCHIVE_BASE_PATH", "/data/archive")
  622. # 生产环境日志配置
  623. LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
  624. LOG_FILE = resolve_log_file("flask_production.log")
  625. LOG_TO_CONSOLE = get_bool_env("LOG_TO_CONSOLE", False)
  626. # 配置字典
  627. config = {
  628. "development": DevelopmentConfig,
  629. "production": ProductionConfig,
  630. "default": DevelopmentConfig,
  631. }
  632. # 获取当前环境
  633. current_env = get_environment()