routes.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808
  1. from io import BytesIO, StringIO
  2. import os
  3. import pandas as pd
  4. from flask import request, jsonify, send_file, current_app
  5. from app.api.data_resource import bp
  6. from app.models.result import success, failed
  7. import logging
  8. import json
  9. import re
  10. from minio import Minio
  11. from app.core.graph.graph_operations import MyEncoder
  12. from app.services.neo4j_driver import neo4j_driver
  13. from app.core.data_resource.resource import (
  14. resource_list,
  15. handle_node,
  16. resource_kinship_graph,
  17. resource_impact_all_graph,
  18. model_resource_list,
  19. select_create_ddl,
  20. data_resource_edit,
  21. handle_id_resource,
  22. id_data_search_list,
  23. table_sql,
  24. select_sql,
  25. id_resource_graph,
  26. status_query
  27. )
  28. from app.core.meta_data import (
  29. translate_and_parse,
  30. infer_column_type,
  31. text_resource_solve,
  32. get_file_content,
  33. get_formatted_time
  34. )
  35. import traceback
  36. from app.core.system.auth import require_auth
  37. from app.core.llm.ddl_parser import DDLParser
  38. logger = logging.getLogger("app")
  39. def get_minio_client():
  40. """获取 MinIO 客户端实例"""
  41. return Minio(
  42. current_app.config['MINIO_HOST'],
  43. access_key=current_app.config['MINIO_USER'],
  44. secret_key=current_app.config['MINIO_PASSWORD'],
  45. secure=current_app.config['MINIO_SECURE']
  46. )
  47. def get_minio_config():
  48. """获取 MinIO 配置"""
  49. return {
  50. 'bucket_name': current_app.config['BUCKET_NAME'],
  51. 'prefix': current_app.config['PREFIX'],
  52. 'allowed_extensions': current_app.config['ALLOWED_EXTENSIONS']
  53. }
  54. def is_english(text):
  55. """检查文本是否为英文"""
  56. return text.isascii() and bool(re.match(r'^[a-zA-Z0-9_\s.,;:!?()\'"-]+$', text))
  57. @bp.route('/translate', methods=['POST'])
  58. def data_resource_translate():
  59. # 获取表单数据
  60. data_resource = request.form.get('data_resource')
  61. meta_data = request.form.get('meta_data')
  62. file = request.files.get('file')
  63. if not data_resource or not file:
  64. return jsonify(failed("缺少必要参数:data_resource 或文件"))
  65. # 处理meta_data可能为None的情况
  66. if meta_data:
  67. try:
  68. # 修复JSON解析问题,处理可能包含特殊引号的情况
  69. # 替换可能存在的特殊引号字符
  70. meta_data = meta_data.replace('â', '"').replace('"', '"').replace('"', '"')
  71. meta_data_list = json.loads(meta_data)
  72. except json.JSONDecodeError as e:
  73. logger.error(f"解析meta_data失败: {meta_data}, 错误: {str(e)}")
  74. # 尝试进行基本的字符串解析,以处理简单的数组格式
  75. if meta_data.startswith('[') and meta_data.endswith(']'):
  76. try:
  77. # 使用ast.literal_eval作为备用解析方法
  78. import ast
  79. meta_data_list = ast.literal_eval(meta_data)
  80. except Exception:
  81. # 如果仍然失败,使用简单的字符串分割
  82. meta_data = meta_data.strip('[]')
  83. meta_data_list = [item.strip('"\'') for item in meta_data.split(',')]
  84. else:
  85. meta_data_list = []
  86. else:
  87. logger.warning("meta_data为空,将使用空列表")
  88. meta_data_list = []
  89. # 构建翻译后的内容组合
  90. translated_meta_data_list = []
  91. for meta_item in meta_data_list:
  92. if is_english(meta_item): # 检查是否为英文
  93. translated_meta_data_list.append(meta_item) # 如果是英文,则直接添加
  94. else:
  95. translated_meta_data_list.append(translate_and_parse(meta_item)[0]) # 否则翻译后添加
  96. # 对 data_resource 进行翻译
  97. translated_data_resource = translate_and_parse(data_resource)
  98. if translated_data_resource and len(translated_data_resource) > 0:
  99. translated_data_resource = translated_data_resource[0]
  100. else:
  101. translated_data_resource = data_resource # 翻译失败时使用原值
  102. try:
  103. # 构建最终的翻译结果
  104. resource = {"name_zh": data_resource, "name_en": translated_data_resource}
  105. parsed_data = []
  106. # 读取文件内容
  107. file_content = file.read()
  108. # 重置文件指针
  109. file.seek(0)
  110. try:
  111. df = pd.read_excel(BytesIO(file_content))
  112. except Exception as e:
  113. return jsonify(failed(f"文件格式错误: {str(e)}"))
  114. # 获取列名和对应的数据类型
  115. # 如果meta_data为空,使用DataFrame的列名
  116. if not meta_data_list and not df.empty:
  117. meta_data_list = df.columns.tolist()
  118. translated_meta_data_list = []
  119. for col in meta_data_list:
  120. if is_english(col):
  121. translated_meta_data_list.append(col)
  122. else:
  123. translated_meta_data_list.append(translate_and_parse(col)[0])
  124. columns_and_types = infer_column_type(df)
  125. for i in range(len(meta_data_list)):
  126. zh = meta_data_list[i]
  127. en = translated_meta_data_list[i]
  128. data_type = columns_and_types[i] if i < len(columns_and_types) else "varchar(255)"
  129. parsed_item = {"name_zh": zh, "name_en": en, "data_type": data_type}
  130. parsed_data.append(parsed_item)
  131. response_data = {
  132. "head_data": parsed_data,
  133. "data_resource": resource
  134. }
  135. return jsonify(success(response_data, "success"))
  136. except Exception as e:
  137. logger.error(f"翻译处理失败: {str(e)}", exc_info=True)
  138. return jsonify(failed(str(e)))
  139. @bp.route('/save', methods=['POST'])
  140. def data_resource_save():
  141. """保存数据资源"""
  142. try:
  143. # 获取表单数据
  144. receiver = request.get_json()
  145. # 检查receiver是否存在
  146. if not receiver:
  147. return jsonify(failed("参数不完整:缺少receiver"))
  148. # 检查url是否存在
  149. if 'url' not in receiver or not receiver['url']:
  150. logger.debug(f"url 为空")
  151. additional_info = receiver.get('additional_info')
  152. if not additional_info:
  153. return jsonify(failed("参数不完整: 缺少additional_info"))
  154. head_data = additional_info.get('head_data')
  155. file_extension = receiver['url'].split('.')[-1] if receiver.get('url') else ""
  156. if file_extension in ['xlsx', 'xls', 'csv']:
  157. # Excel/CSV文件必须有storage_location
  158. storage_location = receiver.get('storage_location', '')
  159. if not storage_location:
  160. return jsonify(failed("参数不完整:缺少storage_location或storage_location为空"))
  161. # 调用业务逻辑处理数据资源创建,设置resource_type为structure
  162. resource_id = handle_node(receiver, head_data, data_source=None, resource_type='structure')
  163. elif file_extension == 'sql' or file_extension == "" or not file_extension:
  164. data_source = additional_info.get('data_source', '')
  165. storage_location = receiver.get('storage_location', '')
  166. # 如果有storage_location,按结构化数据处理
  167. if storage_location:
  168. resource_id = handle_node(receiver, head_data, data_source=None, resource_type='structure')
  169. # 如果有data_source,按DDL处理
  170. elif data_source:
  171. # 检查data_source格式
  172. if not isinstance(data_source, dict) or not data_source.get("name_en"):
  173. return jsonify(failed("数据源信息不完整或无效"))
  174. resource_id = handle_node(receiver, head_data, data_source, resource_type='ddl')
  175. # 两者都没有
  176. else:
  177. return jsonify(failed("SQL文件处理需要提供storage_location或有效的data_source信息"))
  178. else:
  179. return jsonify(failed("文件格式错误"))
  180. return jsonify(success({"id": resource_id}))
  181. except Exception as e:
  182. logger.error(f"保存数据资源失败: {str(e)}")
  183. error_traceback = traceback.format_exc()
  184. logger.error(f"错误详情: {error_traceback}")
  185. return jsonify(failed(str(e)))
  186. @bp.route('/delete', methods=['POST'])
  187. def data_resource_delete():
  188. """删除数据资源"""
  189. try:
  190. # 获取资源ID
  191. if not request.json:
  192. return jsonify(failed("请求数据不能为空"))
  193. resource_id = request.json.get('id')
  194. if resource_id is None:
  195. return jsonify(failed("资源ID不能为空"))
  196. with neo4j_driver.get_session() as session:
  197. # 删除数据资源节点及其关系
  198. cypher = """
  199. MATCH (n:DataResource)
  200. WHERE id(n) = $resource_id
  201. DETACH DELETE n
  202. """
  203. session.run(cypher, resource_id=int(resource_id))
  204. return jsonify(success({"message": "数据资源删除成功"}))
  205. except Exception as e:
  206. logger.error(f"删除数据资源失败: {str(e)}")
  207. return jsonify(failed(str(e)))
  208. @bp.route('/update', methods=['POST'])
  209. def data_resource_update():
  210. """更新数据资源"""
  211. try:
  212. # 获取更新数据
  213. data = request.json
  214. if not data or "id" not in data:
  215. return jsonify(failed("参数不完整"))
  216. # 调用业务逻辑更新数据资源
  217. updated_data = data_resource_edit(data)
  218. return jsonify(success(updated_data))
  219. except Exception as e:
  220. logger.error(f"更新数据资源失败: {str(e)}")
  221. return jsonify(failed(str(e)))
  222. # 解析ddl,使用正则表达式匹配,但没有进行翻译,也没有对注释进行识别
  223. # 使用ddl创建数据资源时,调用该API
  224. @bp.route('/ddl', methods=['POST'])
  225. def id_data_ddl():
  226. """解析数据资源的DDL"""
  227. try:
  228. # 获取SQL内容
  229. if not request.json:
  230. return jsonify(failed("请求数据不能为空"))
  231. sql_content = request.json.get('sql', '')
  232. if not sql_content:
  233. return jsonify(failed("SQL内容不能为空"))
  234. # 记录原始SQL用于调试
  235. logger.debug(f"原始SQL: {sql_content}")
  236. # 提取创建表的DDL语句
  237. create_ddl_list = select_create_ddl(sql_content)
  238. if not create_ddl_list:
  239. return jsonify(failed("未找到有效的CREATE TABLE语句"))
  240. # 解析每个表定义
  241. tables_dict = {} # 最终返回的表字典
  242. for ddl in create_ddl_list:
  243. table_info = table_sql(ddl)
  244. if table_info:
  245. # table_info格式: {"table_name": {"exist": bool, "meta": [...], "table_comment": "..."}}
  246. # 合并到结果字典中
  247. tables_dict.update(table_info)
  248. if not tables_dict:
  249. return jsonify(failed("解析表结构失败"))
  250. # 记录结果
  251. logger.debug(f"解析结果: {json.dumps(tables_dict, ensure_ascii=False)}")
  252. # 直接返回解析结果
  253. return jsonify(success(tables_dict))
  254. except Exception as e:
  255. logger.error(f"解析DDL失败: {str(e)}")
  256. logger.error(traceback.format_exc()) # 添加详细错误堆栈
  257. return jsonify(failed(str(e)))
  258. @bp.route('/list', methods=['POST'])
  259. def data_resource_list():
  260. """获取数据资源列表"""
  261. try:
  262. # 获取分页和筛选参数
  263. if not request.json:
  264. return jsonify(failed('请求数据不能为空'))
  265. page = int(request.json.get('current', 1))
  266. page_size = int(request.json.get('size', 10))
  267. name_en_filter = request.json.get('name_en')
  268. name_zh_filter = request.json.get('name_zh')
  269. type_filter = request.json.get('type', 'all')
  270. category_filter = request.json.get('category')
  271. tag_filter = request.json.get('tag')
  272. # 调用业务逻辑查询数据资源列表
  273. resources, total_count = resource_list(
  274. page,
  275. page_size,
  276. name_en_filter,
  277. name_zh_filter,
  278. type_filter,
  279. category_filter,
  280. tag_filter
  281. )
  282. # 返回结果
  283. return jsonify(success({
  284. "records": resources,
  285. "total": total_count,
  286. "size": page_size,
  287. "current": page
  288. }))
  289. except Exception as e:
  290. logger.error(f"获取数据资源列表失败: {str(e)}")
  291. return jsonify(failed(str(e)))
  292. @bp.route('/search', methods=['POST'])
  293. def id_data_search():
  294. """数据资源关联元数据搜索"""
  295. try:
  296. # 获取分页和筛选参数
  297. if not request.json:
  298. return jsonify(failed('请求数据不能为空'))
  299. page = int(request.json.get('current', 1))
  300. page_size = int(request.json.get('size', 10))
  301. resource_id = request.json.get('id')
  302. name_en_filter = request.json.get('name_en')
  303. name_zh_filter = request.json.get('name_zh')
  304. category_filter = request.json.get('category')
  305. tag_filter = request.json.get('tag')
  306. if resource_id is None:
  307. return jsonify(failed("资源ID不能为空"))
  308. # 确保传入的ID为整数
  309. try:
  310. resource_id = int(resource_id)
  311. except (ValueError, TypeError):
  312. return jsonify(failed(f"资源ID必须为整数, 收到的是: {resource_id}"))
  313. # 记录请求信息
  314. logger.info(f"获取资源关联元数据请求,ID: {resource_id}")
  315. # 调用业务逻辑查询关联元数据
  316. metadata_list, total_count = id_data_search_list(
  317. resource_id,
  318. page,
  319. page_size,
  320. name_en_filter,
  321. name_zh_filter,
  322. category_filter,
  323. tag_filter
  324. )
  325. # 返回结果
  326. return jsonify(success({
  327. "records": metadata_list,
  328. "total": total_count,
  329. "size": page_size,
  330. "current": page
  331. }))
  332. except Exception as e:
  333. logger.error(f"数据资源关联元数据搜索失败: {str(e)}")
  334. return jsonify(failed(str(e)))
  335. def dynamic_type_conversion(value, target_type):
  336. """动态类型转换"""
  337. if value is None:
  338. return None
  339. if target_type == "int" or target_type == "INT":
  340. return int(value)
  341. elif target_type == "float" or target_type == "FLOAT" or target_type == "double" or target_type == "DOUBLE":
  342. return float(value)
  343. elif target_type == "bool" or target_type == "BOOL" or target_type == "boolean" or target_type == "BOOLEAN":
  344. if isinstance(value, str):
  345. return value.lower() in ('true', 'yes', '1', 't', 'y')
  346. return bool(value)
  347. else:
  348. return str(value)
  349. @bp.route('/graph/all', methods=['POST'])
  350. def data_resource_graph_all():
  351. """获取数据资源完整图谱"""
  352. try:
  353. # 获取参数
  354. if not request.json:
  355. return jsonify(failed('请求数据不能为空'))
  356. resource_id = request.json.get('id')
  357. meta = request.json.get('meta', True)
  358. if resource_id is None:
  359. return jsonify(failed("资源ID不能为空"))
  360. # 确保传入的ID为整数
  361. try:
  362. resource_id = int(resource_id)
  363. except (ValueError, TypeError):
  364. return jsonify(failed(f"资源ID必须为整数, 收到的是: {resource_id}"))
  365. # 调用业务逻辑获取完整图谱
  366. graph_data = resource_impact_all_graph(resource_id, meta)
  367. return jsonify(success(graph_data))
  368. except Exception as e:
  369. logger.error(f"获取数据资源完整图谱失败: {str(e)}")
  370. return jsonify(failed(str(e)))
  371. @bp.route('/graph', methods=['POST'])
  372. def data_resource_list_graph():
  373. """获取数据资源亲缘关系图谱"""
  374. try:
  375. # 获取参数
  376. if not request.json:
  377. return jsonify(failed('请求数据不能为空'))
  378. resource_id = request.json.get('id')
  379. meta = request.json.get('meta', True)
  380. if resource_id is None:
  381. return jsonify(failed("资源ID不能为空"))
  382. # 确保传入的ID为整数
  383. try:
  384. resource_id = int(resource_id)
  385. except (ValueError, TypeError):
  386. return jsonify(failed(f"资源ID必须为整数, 收到的是: {resource_id}"))
  387. # 记录请求信息
  388. logger.info(f"获取图谱请求,ID: {resource_id}")
  389. # 调用业务逻辑获取图谱
  390. graph_data = resource_kinship_graph(resource_id, meta)
  391. return jsonify(success(graph_data))
  392. except Exception as e:
  393. logger.error(f"获取数据资源亲缘关系图谱失败: {str(e)}")
  394. return jsonify(failed(str(e)))
  395. @bp.route('/save/metadata', methods=['POST'])
  396. def id_data_save():
  397. """保存数据资源关联的元数据"""
  398. try:
  399. # 获取参数
  400. if not request.json:
  401. return jsonify(failed('请求数据不能为空'))
  402. resource_id = request.json.get('id')
  403. metadata_list = request.json.get('data', [])
  404. if resource_id is None:
  405. return jsonify(failed("资源ID不能为空"))
  406. if not metadata_list:
  407. return jsonify(failed("元数据列表不能为空"))
  408. # 处理元数据保存
  409. with neo4j_driver.get_session() as session:
  410. # 获取数据资源名称
  411. resource_query = """
  412. MATCH (n:DataResource)
  413. WHERE id(n) = $resource_id
  414. RETURN n.name as resource_name
  415. """
  416. resource_result = session.run(resource_query, resource_id=int(resource_id))
  417. resource_record = resource_result.single()
  418. if not resource_record:
  419. return jsonify(failed(f"未找到ID为{resource_id}的数据资源"))
  420. resource_name = resource_record["resource_name"]
  421. # 先删除现有关系
  422. cypher_delete = """
  423. MATCH (n:DataResource)-[r:INCLUDES]->()
  424. WHERE id(n) = $resource_id
  425. DELETE r
  426. """
  427. session.run(cypher_delete, resource_id=int(resource_id))
  428. # 添加新关系
  429. for meta in metadata_list:
  430. # 创建元数据节点
  431. meta_cypher = """
  432. MERGE (m:DataMeta {name_zh: $name_zh})
  433. ON CREATE SET m.name_en = $name_en,
  434. m.create_time = $create_time,
  435. m.data_type = $type
  436. ON MATCH SET m.data_type = $type
  437. RETURN m
  438. """
  439. create_time = get_formatted_time()
  440. meta_result = session.run(
  441. meta_cypher,
  442. name_zh=meta["name_zh"],
  443. name_en=meta["name_en"],
  444. create_time=create_time,
  445. type=meta["data_type"]
  446. )
  447. meta_record = meta_result.single()
  448. if not meta_record:
  449. logger.error(f"创建元数据节点失败: {meta['name_zh']}")
  450. continue
  451. meta_node = meta_record["m"]
  452. meta_id = meta_node.id
  453. # 打印节点ID信息,便于调试
  454. logger.info(f"元数据节点ID: {meta_id}, 类型: {type(meta_id)}")
  455. logger.info(f"数据资源节点ID: {resource_id}, 类型: {type(resource_id)}")
  456. # 使用明确的属性名匹配而不是ID
  457. rel_cypher = """
  458. MATCH (a:DataResource {name: $r_name}), (m:DataMeta {name: $m_name})
  459. MERGE (a)-[r:INCLUDES]->(m)
  460. RETURN r
  461. """
  462. rel_result = session.run(
  463. rel_cypher,
  464. r_name=resource_name,
  465. m_name=meta["name"]
  466. )
  467. # 检查关系是否创建成功
  468. if rel_result.single():
  469. logger.info(f"成功创建关系: {resource_name} -> {meta['name']}")
  470. else:
  471. logger.warning(f"关系创建结果为空")
  472. # 额外验证关系是否创建
  473. verify_cypher = """
  474. MATCH (a:DataResource {name: $r_name})-[r:INCLUDES]->(m:DataMeta {name: $m_name})
  475. RETURN count(r) as rel_count
  476. """
  477. verify_result = session.run(
  478. verify_cypher,
  479. r_name=resource_name,
  480. m_name=meta["name"]
  481. )
  482. verify_record = verify_result.single()
  483. count = verify_record["rel_count"] if verify_record else 0
  484. logger.info(f"验证关系数量: {count}")
  485. return jsonify(success({"message": "元数据保存成功"}))
  486. except Exception as e:
  487. logger.error(f"保存数据资源关联的元数据失败: {str(e)}")
  488. return jsonify(failed(str(e)))
  489. @bp.route('/sql/test', methods=['POST'])
  490. def sql_test():
  491. """测试SQL查询"""
  492. try:
  493. # 获取参数
  494. if not request.json:
  495. return jsonify(failed('请求数据不能为空'))
  496. sql_query = request.json.get('sql', '')
  497. if not sql_query:
  498. return jsonify(failed("SQL查询不能为空"))
  499. # 解析SQL
  500. parsed_sql = select_sql(sql_query)
  501. if not parsed_sql:
  502. return jsonify(failed("解析SQL失败"))
  503. # 返回解析结果
  504. return jsonify(success(parsed_sql))
  505. except Exception as e:
  506. logger.error(f"测试SQL查询失败: {str(e)}")
  507. return jsonify(failed(str(e)))
  508. # 使用LLM识别DDL语句,用来代替原来的正则的方式
  509. # 用于在数据资源创建时,识别DDL语句 /api/resource/ddl/parse
  510. @bp.route('/ddl/parse', methods=['POST'])
  511. def ddl_identify():
  512. """识别DDL语句"""
  513. try:
  514. # 获取参数 - 支持两种方式:上传文件或JSON
  515. sql_content = ''
  516. # 检查是否有文件上传
  517. if 'file' in request.files:
  518. file = request.files['file']
  519. # 检查文件是否存在且文件名不为空
  520. if file and file.filename:
  521. # 检查是否是SQL文件
  522. if not file.filename.lower().endswith('.sql'):
  523. return jsonify(failed("只接受SQL文件"))
  524. # 读取文件内容
  525. sql_content = file.read().decode('utf-8')
  526. logger.info(f"从上传的文件中读取SQL内容,文件名: {file.filename}")
  527. # 如果没有文件上传,检查是否有JSON输入
  528. elif request.is_json and request.json:
  529. sql_content = request.json.get('sql', '')
  530. # 如果两种方式都没有提供SQL内容,则返回错误
  531. if not sql_content:
  532. return jsonify(failed("SQL内容不能为空,请上传SQL文件或提供SQL内容"))
  533. parser = DDLParser()
  534. # 提取创建表的DDL语句
  535. ddl_list = parser.parse_ddl(sql_content)
  536. if not ddl_list:
  537. return jsonify(failed("未找到有效的CREATE TABLE语句"))
  538. # 处理表的存在状态
  539. if isinstance(ddl_list, dict):
  540. # 获取所有表名
  541. table_names = list(ddl_list.keys())
  542. # 首先为所有表设置默认的exist状态
  543. for table_name in table_names:
  544. ddl_list[table_name]["exist"] = False
  545. if table_names:
  546. try:
  547. # 查询表是否存在
  548. with neo4j_driver.get_session() as session:
  549. table_query = """
  550. UNWIND $names AS name
  551. OPTIONAL MATCH (n:DataResource {name_en: name})
  552. RETURN name, n IS NOT NULL AS exists
  553. """
  554. table_results = session.run(table_query, names=table_names)
  555. # 更新存在的表的状态
  556. for record in table_results:
  557. table_name = record["name"]
  558. exists = record["exists"]
  559. if table_name in ddl_list:
  560. ddl_list[table_name]["exist"] = exists
  561. except Exception as e:
  562. logger.error(f"检查表存在状态失败: {str(e)}")
  563. # 如果查询失败,所有表保持默认的False状态
  564. logger.debug(f"识别到的DDL语句: {json.dumps(ddl_list, ensure_ascii=False)}")
  565. return jsonify(success(ddl_list))
  566. except Exception as e:
  567. logger.error(f"识别DDL语句失败: {str(e)}")
  568. logger.error(traceback.format_exc()) # 添加详细错误堆栈
  569. return jsonify(failed(str(e)))
  570. # 废弃的识别DDL语句方法,该API 与 ddl API 功能类似,但功能简化了
  571. @bp.route('/ddl/identify', methods=['POST'])
  572. def sql_ddl_identify():
  573. """识别DDL语句"""
  574. try:
  575. # 获取参数
  576. if not request.json:
  577. return jsonify(failed('请求数据不能为空'))
  578. sql_content = request.json.get('sql', '')
  579. if not sql_content:
  580. return jsonify(failed("SQL内容不能为空"))
  581. # 提取创建表的DDL语句
  582. create_ddl_list = select_create_ddl(sql_content)
  583. if not create_ddl_list:
  584. return jsonify(failed("未找到有效的CREATE TABLE语句"))
  585. return jsonify(success({"count": len(create_ddl_list)}))
  586. except Exception as e:
  587. logger.error(f"识别DDL语句失败: {str(e)}")
  588. return jsonify(failed(str(e)))
  589. @bp.route('/model/list', methods=['POST'])
  590. def resource_model_list():
  591. """获取模型资源列表"""
  592. try:
  593. # 获取分页和筛选参数
  594. if not request.json:
  595. return jsonify(failed('请求数据不能为空'))
  596. page = int(request.json.get('current', 1))
  597. page_size = int(request.json.get('size', 10))
  598. name_filter = request.json.get('name')
  599. # 调用业务逻辑查询模型资源列表
  600. resources, total_count = model_resource_list(page, page_size, name_filter)
  601. # 返回结果
  602. return jsonify(success({
  603. "records": resources,
  604. "total": total_count,
  605. "size": page_size,
  606. "current": page
  607. }))
  608. except Exception as e:
  609. logger.error(f"获取模型资源列表失败: {str(e)}")
  610. return jsonify(failed(str(e)))
  611. @bp.route('/detail', methods=['POST'])
  612. def data_resource_detail():
  613. """获取数据资源详情"""
  614. try:
  615. # 获取资源ID
  616. if not request.json:
  617. return jsonify(failed('请求数据不能为空'))
  618. resource_id = request.json.get('id')
  619. if resource_id is None:
  620. return jsonify(failed("资源ID不能为空"))
  621. # 确保传入的ID为整数
  622. try:
  623. resource_id = int(resource_id)
  624. except (ValueError, TypeError):
  625. return jsonify(failed(f"资源ID必须为整数, 收到的是: {resource_id}"))
  626. # 记录请求信息
  627. logger.info(f"获取资源详情请求,ID: {resource_id}")
  628. # 调用业务逻辑查询数据资源详情
  629. resource_data = handle_id_resource(resource_id)
  630. if not resource_data:
  631. logger.error(f"资源不存在,ID: {resource_id}")
  632. return jsonify(failed("资源不存在"))
  633. # 记录从handle_id_resource返回的数据
  634. logger.info(f"handle_id_resource返回数据,describe字段: {resource_data.get('describe')}")
  635. # 确保返回的数据格式符合要求
  636. response_data = {
  637. "parsed_data": resource_data.get("parsed_data", []),
  638. "tag": resource_data.get("tag", {"name": None, "id": None}),
  639. "leader": resource_data.get("leader", ""),
  640. "organization": resource_data.get("organization", ""),
  641. "name_zh": resource_data.get("name_zh", ""),
  642. "name_en": resource_data.get("name_en", ""),
  643. "data_sensitivity": resource_data.get("data_sensitivity", ""),
  644. "storage_location": resource_data.get("storage_location", "/"),
  645. "time": resource_data.get("time", ""),
  646. "type": resource_data.get("type", ""),
  647. "category": resource_data.get("category", ""),
  648. "url": resource_data.get("url", ""),
  649. "frequency": resource_data.get("frequency", ""),
  650. "status": resource_data.get("status", True),
  651. "id": resource_data.get("id"),
  652. "keywords": resource_data.get("keywords", []),
  653. "describe": resource_data.get("describe", "")
  654. }
  655. # 记录最终返回的数据
  656. logger.info(f"最终返回的response_data,describe字段: {response_data.get('describe')}")
  657. return jsonify(success(response_data))
  658. except Exception as e:
  659. logger.error(f"获取数据资源详情失败: {str(e)}")
  660. return jsonify(failed(str(e)))
  661. @bp.route('/config', methods=['GET'])
  662. @require_auth
  663. def get_resource_config():
  664. """获取数据资源配置信息"""
  665. config = get_minio_config()
  666. return jsonify({
  667. 'allowed_extensions': list(config['allowed_extensions']),
  668. 'bucket_name': config['bucket_name'],
  669. 'prefix': config['prefix']
  670. })