routes.py 61 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976
  1. """HTTP orchestration boundary for data-research ingestion jobs."""
  2. from __future__ import annotations
  3. import io
  4. import logging
  5. from flask import current_app, g, jsonify, request, send_file
  6. from app import db
  7. from app.api.data_development import bp
  8. from app.core.data_research.errors import DataResearchError
  9. from app.models.result import failed, success
  10. logger = logging.getLogger(__name__)
  11. def get_ingestion_service():
  12. from app.core.data_research.ingestion import IngestionService
  13. from app.core.data_research.repository import SqlAlchemyIngestionJobRepository
  14. return IngestionService(
  15. SqlAlchemyIngestionJobRepository(db.session),
  16. commit=db.session.commit,
  17. rollback=db.session.rollback,
  18. )
  19. def get_database_source_registration_service():
  20. from app.core.data_research.repository import (
  21. SqlAlchemyIngestionSourceRepository,
  22. )
  23. from app.core.data_research.sources import (
  24. DatabaseSourceRegistrationService,
  25. )
  26. from app.core.data_source.runtime import get_data_source_manager
  27. manager = get_data_source_manager()
  28. return DatabaseSourceRegistrationService(
  29. SqlAlchemyIngestionSourceRepository(db.session),
  30. definition_resolver=manager.definitions.get,
  31. commit=db.session.commit,
  32. rollback=db.session.rollback,
  33. )
  34. def get_catalog_snapshot_repository():
  35. from app.core.data_research.repository import (
  36. SqlAlchemyCatalogSnapshotRepository,
  37. )
  38. return SqlAlchemyCatalogSnapshotRepository(db.session)
  39. def get_catalog_ingestion_executor():
  40. from app.core.data_research.catalog.execution import (
  41. CatalogIngestionExecutor,
  42. )
  43. from app.core.data_research.catalog.mysql import MySqlCatalogCollector
  44. from app.core.data_research.catalog.postgresql import (
  45. PostgreSqlCatalogCollector,
  46. )
  47. from app.core.data_research.catalog.service import CatalogCollectionService
  48. from app.core.data_research.errors import IngestionSourceInvalid
  49. from app.core.data_source.runtime import get_data_source_manager
  50. manager = get_data_source_manager()
  51. def collector_resolver(database_type):
  52. if database_type == "postgresql":
  53. return PostgreSqlCatalogCollector()
  54. if database_type == "mysql":
  55. return MySqlCatalogCollector()
  56. raise IngestionSourceInvalid(
  57. f"database type {database_type} is not supported"
  58. )
  59. collector = CatalogCollectionService(
  60. manager,
  61. definition_resolver=manager.definitions.get,
  62. collector_resolver=collector_resolver,
  63. )
  64. return CatalogIngestionExecutor(
  65. get_ingestion_service(),
  66. collector,
  67. get_catalog_snapshot_repository(),
  68. commit=db.session.commit,
  69. rollback=db.session.rollback,
  70. )
  71. def get_data_element_service():
  72. from app.core.data_research.data_elements import DataElementService
  73. from app.core.data_research.repository import SqlAlchemyDataElementRepository
  74. from app.core.events.outbox import enqueue_outbox
  75. return DataElementService(
  76. SqlAlchemyDataElementRepository(db.session),
  77. outbox_enqueue=lambda **event: enqueue_outbox(db.session, **event),
  78. )
  79. def get_device_asset_service():
  80. from app.core.data_research.device_asset_repository import (
  81. SqlAlchemyDeviceAssetRepository,
  82. )
  83. from app.core.data_research.device_assets import DeviceAssetService
  84. return DeviceAssetService(
  85. SqlAlchemyDeviceAssetRepository(db.session),
  86. commit=db.session.commit,
  87. rollback=db.session.rollback,
  88. )
  89. def _device_ontology_authorizer(ontology_repository):
  90. from app.core.data_research.device_semantics import (
  91. DeviceOntologyPublicationAuthorizer,
  92. )
  93. from app.core.governance.responsibilities import (
  94. ResponsibilityService,
  95. SqlAlchemyResponsibilityRepository,
  96. )
  97. responsibilities = ResponsibilityService(
  98. SqlAlchemyResponsibilityRepository(db.session)
  99. )
  100. return DeviceOntologyPublicationAuthorizer(
  101. ontology_repository,
  102. responsibility_lookup=responsibilities.get,
  103. )
  104. def get_device_semantic_service():
  105. from app.core.data_research.device_semantics import DeviceSemanticService
  106. from app.core.data_research.ontology.repository import (
  107. SqlAlchemyOntologyRepository,
  108. )
  109. return DeviceSemanticService(
  110. SqlAlchemyOntologyRepository(db.session),
  111. commit=db.session.commit,
  112. rollback=db.session.rollback,
  113. )
  114. def get_device_semantic_code_service():
  115. from app.core.data_research.device_semantic_repository import (
  116. SqlAlchemyDeviceSemanticCodeRepository,
  117. )
  118. from app.core.data_research.device_semantics import (
  119. DeviceSemanticCodeService,
  120. )
  121. from app.core.data_research.ontology.repository import (
  122. SqlAlchemyOntologyRepository,
  123. )
  124. ontology_repository = SqlAlchemyOntologyRepository(db.session)
  125. authorizer = _device_ontology_authorizer(ontology_repository)
  126. return DeviceSemanticCodeService(
  127. SqlAlchemyDeviceSemanticCodeRepository(db.session),
  128. review_authorizer=authorizer.assert_accountable,
  129. commit=db.session.commit,
  130. rollback=db.session.rollback,
  131. )
  132. def get_device_entity_resolution_service():
  133. from app.core.data_research.device_entity_repository import (
  134. SqlAlchemyDeviceEntityResolutionRepository,
  135. )
  136. from app.core.data_research.device_entity_resolution import (
  137. DeviceEntityForbidden,
  138. DeviceEntityResolutionService,
  139. )
  140. from app.core.governance.responsibilities import (
  141. ResponsibilityService,
  142. SqlAlchemyResponsibilityRepository,
  143. )
  144. responsibilities = ResponsibilityService(
  145. SqlAlchemyResponsibilityRepository(db.session)
  146. )
  147. def assert_accountable(actor_uid):
  148. matrix = responsibilities.get(
  149. "device_mapping",
  150. "DEVICE_ENTITY_RESOLUTION",
  151. )
  152. accountable = [
  153. item
  154. for item in matrix.get("assignments", [])
  155. if item.get("responsibility_role") == "asset_manager"
  156. and item.get("raci_role") == "accountable"
  157. ]
  158. if len(accountable) != 1 or str(accountable[0].get("user_id")) != str(
  159. actor_uid
  160. ):
  161. raise DeviceEntityForbidden(
  162. "only the accountable device mapping asset manager may decide"
  163. )
  164. return DeviceEntityResolutionService(
  165. SqlAlchemyDeviceEntityResolutionRepository(db.session),
  166. review_authorizer=assert_accountable,
  167. auto_merge_enabled=current_app.config.get(
  168. "DEVICE_ENTITY_AUTO_MERGE_ENABLED",
  169. False,
  170. ),
  171. commit=db.session.commit,
  172. rollback=db.session.rollback,
  173. )
  174. def get_device_quality_service():
  175. from app.core.data_research.device_quality import DeviceQualityService
  176. from app.core.data_research.device_quality_repository import (
  177. SqlAlchemyDeviceQualityRepository,
  178. )
  179. from app.core.data_research.errors import DeviceQualityForbidden
  180. from app.core.governance.responsibilities import (
  181. ResponsibilityService,
  182. SqlAlchemyResponsibilityRepository,
  183. )
  184. responsibilities = ResponsibilityService(
  185. SqlAlchemyResponsibilityRepository(db.session)
  186. )
  187. def assert_accountable(actor_uid):
  188. matrix = responsibilities.get(
  189. "device_quality",
  190. "DEVICE_QUALITY",
  191. )
  192. accountable = [
  193. item
  194. for item in matrix.get("assignments", [])
  195. if item.get("responsibility_role") == "asset_manager"
  196. and item.get("raci_role") == "accountable"
  197. ]
  198. if len(accountable) != 1 or str(accountable[0].get("user_id")) != str(
  199. actor_uid
  200. ):
  201. raise DeviceQualityForbidden(
  202. "only the accountable device quality asset manager may publish"
  203. )
  204. return DeviceQualityService(
  205. SqlAlchemyDeviceQualityRepository(db.session),
  206. publish_authorizer=assert_accountable,
  207. commit=db.session.commit,
  208. rollback=db.session.rollback,
  209. )
  210. def get_candidate_decision_service():
  211. from app.core.data_research.candidate_decisions import CandidateDecisionService
  212. from app.core.data_research.data_elements import DataElementService
  213. from app.core.data_research.repository import (
  214. SqlAlchemyCandidateDecisionRepository,
  215. SqlAlchemyDataElementRepository,
  216. )
  217. return CandidateDecisionService(
  218. SqlAlchemyCandidateDecisionRepository(db.session),
  219. data_elements=DataElementService(SqlAlchemyDataElementRepository(db.session)),
  220. )
  221. def _artifact_storage():
  222. from minio import Minio
  223. from app.core.data_research.artifacts import MinioArtifactStorage
  224. client = Minio(
  225. current_app.config["MINIO_HOST"],
  226. access_key=current_app.config["MINIO_USER"],
  227. secret_key=current_app.config["MINIO_PASSWORD"],
  228. secure=bool(current_app.config.get("MINIO_SECURE")),
  229. )
  230. return MinioArtifactStorage(client, current_app.config["MINIO_BUCKET"])
  231. def get_artifact_service():
  232. from app.core.common.identifiers import new_governance_uid
  233. from app.core.data_research.artifacts import (
  234. ArtifactService,
  235. SqlAlchemyArtifactRepository,
  236. )
  237. from app.core.data_research.file_policy import FilePolicy
  238. return ArtifactService(
  239. SqlAlchemyArtifactRepository(db.session),
  240. _artifact_storage(),
  241. FilePolicy(
  242. max_bytes=int(current_app.config.get("DATA_RESEARCH_FILE_MAX_BYTES", 25 * 1024 * 1024)),
  243. max_pages=int(current_app.config.get("DATA_RESEARCH_FILE_MAX_PAGES", 500)),
  244. ),
  245. uid_factory=new_governance_uid,
  246. )
  247. def get_evidence_service():
  248. from app.core.data_research.artifacts import EvidenceService
  249. return EvidenceService(db.session, _artifact_storage())
  250. def get_ontology_service():
  251. from app.core.data_research.ontology.publication import (
  252. OntologyApplicationService,
  253. OntologyPublicationService,
  254. )
  255. from app.core.data_research.ontology.repository import SqlAlchemyOntologyRepository
  256. from app.core.events.outbox import enqueue_outbox
  257. repository = SqlAlchemyOntologyRepository(db.session)
  258. authorizer = _device_ontology_authorizer(repository)
  259. publication = OntologyPublicationService(
  260. repository,
  261. outbox_enqueue=lambda **event: enqueue_outbox(db.session, **event),
  262. publication_authorizer=authorizer,
  263. commit=db.session.commit,
  264. rollback=db.session.rollback,
  265. )
  266. return OntologyApplicationService(
  267. repository,
  268. publication=publication,
  269. commit=db.session.commit,
  270. rollback=db.session.rollback,
  271. )
  272. def get_ontology_dynamic_service():
  273. from app.core.data_research.ontology.change_sets import (
  274. SqlAlchemyDynamicOntologyService,
  275. )
  276. return SqlAlchemyDynamicOntologyService(db.session)
  277. def get_ontology_exchange_service():
  278. from app.core.data_research.ontology.exchange import OntologyExchangeService
  279. from app.core.data_research.ontology.repository import SqlAlchemyOntologyRepository
  280. return OntologyExchangeService(
  281. SqlAlchemyOntologyRepository(db.session),
  282. commit=db.session.commit,
  283. rollback=db.session.rollback,
  284. )
  285. def get_semantic_query_service():
  286. from app.core.data_research.ontology.query import (
  287. Neo4jSemanticRepository,
  288. SemanticQueryService,
  289. )
  290. from app.services.neo4j_driver import neo4j_driver
  291. return SemanticQueryService(Neo4jSemanticRepository(neo4j_driver))
  292. def _identity():
  293. return getattr(g, "current_user", {}) or {}
  294. def _record(record):
  295. return {
  296. "uid": str(record.uid),
  297. "source_uid": str(record.source_uid),
  298. "artifact_uid": str(record.artifact_uid) if record.artifact_uid else None,
  299. "job_type": record.job_type,
  300. "parser_version": record.parser_version,
  301. "status": record.status,
  302. "parameters": dict(record.parameters or {}),
  303. "statistics": dict(record.statistics or {}),
  304. "last_error": record.last_error,
  305. "attempt_count": int(record.attempt_count or 0),
  306. "failure_stage": record.failure_stage,
  307. "actor_uid": record.actor_uid,
  308. "created_at": record.created_at.isoformat() if record.created_at else None,
  309. "updated_at": record.updated_at.isoformat() if record.updated_at else None,
  310. "started_at": record.started_at.isoformat() if record.started_at else None,
  311. "finished_at": record.finished_at.isoformat() if record.finished_at else None,
  312. }
  313. def _catalog_snapshot(record):
  314. return {
  315. "uid": str(record.uid),
  316. "job_uid": str(record.job_uid),
  317. "source_uid": str(record.source_uid),
  318. "attempt": int(record.attempt),
  319. "database_type": record.database_type,
  320. "content_hash": record.content_hash,
  321. "snapshot": dict(record.snapshot or {}),
  322. "evidence_count": int(record.evidence_count),
  323. "created_at": (
  324. record.created_at.isoformat()
  325. if record.created_at
  326. else None
  327. ),
  328. }
  329. def _element(record):
  330. return {
  331. "uid": str(record.uid),
  332. "code": record.code,
  333. "status": record.status,
  334. "current_version": int(record.current_version),
  335. "snapshot": dict(record.snapshot or {}),
  336. "created_by": record.created_by,
  337. "updated_by": record.updated_by,
  338. }
  339. def _decision(record):
  340. return {
  341. "uid": str(record.uid),
  342. "candidate_uid": str(record.candidate_uid),
  343. "action": record.action,
  344. "data_element_uid": (
  345. str(record.data_element_uid) if record.data_element_uid else None
  346. ),
  347. "evidence_uids": list(record.evidence_uids),
  348. "actor_uid": record.actor_uid,
  349. "reason": record.reason,
  350. }
  351. def _artifact(record):
  352. return {
  353. "uid": str(record.uid),
  354. "source_uid": str(record.source_uid),
  355. "filename": record.filename,
  356. "media_type": record.media_type,
  357. "size_bytes": int(record.size_bytes),
  358. "content_hash": record.content_hash,
  359. "parser_version": record.parser_version,
  360. }
  361. def _ontology(record):
  362. return {
  363. "uid": str(record.uid),
  364. "code": record.code,
  365. "name": record.name,
  366. "owner_uid": record.owner_uid,
  367. "status": record.status,
  368. "draft_revision": int(record.draft_revision),
  369. "active_version_uid": record.active_version_uid,
  370. "domain_links": [
  371. {"domain_uid": link.domain_uid, "role": link.role}
  372. for link in record.domain_links
  373. ],
  374. }
  375. def _ontology_version(record):
  376. return {
  377. "uid": str(record.uid),
  378. "ontology_uid": str(record.ontology_uid),
  379. "version": int(record.version),
  380. "parent_version_uid": record.parent_version_uid,
  381. "status": record.status,
  382. "content_hash": record.content_hash,
  383. "graph_document": record.graph_document.to_dict(),
  384. "created_by": record.created_by,
  385. }
  386. def _iso(value):
  387. return value.isoformat() if value else None
  388. def _device_asset(record):
  389. return {
  390. "uid": str(record.uid),
  391. "asset_type": record.asset_type,
  392. "name": record.name,
  393. "status": record.status,
  394. "current_version": int(record.current_version),
  395. "location": record.location,
  396. "organization": record.organization,
  397. "responsible_person": record.responsible_person,
  398. "attributes": dict(record.attributes or {}),
  399. "created_by": record.created_by,
  400. "updated_by": record.updated_by,
  401. "created_at": _iso(record.created_at),
  402. "updated_at": _iso(record.updated_at),
  403. }
  404. def _device_asset_mapping(record):
  405. return {
  406. "uid": str(record.uid),
  407. "asset_uid": str(record.asset_uid),
  408. "source_uid": str(record.source_uid),
  409. "source_entity": record.source_entity,
  410. "asset_type": record.asset_type,
  411. "source_code": record.source_code,
  412. "source_updated_at": _iso(record.source_updated_at),
  413. "first_seen_at": _iso(record.first_seen_at),
  414. "last_seen_at": _iso(record.last_seen_at),
  415. }
  416. def _device_asset_detail(detail):
  417. data = _device_asset(detail.asset)
  418. data["source_mappings"] = [
  419. _device_asset_mapping(mapping)
  420. for mapping in detail.mappings
  421. ]
  422. return data
  423. def _device_asset_version(record):
  424. return {
  425. "uid": str(record.uid),
  426. "asset_uid": str(record.asset_uid),
  427. "version": int(record.version),
  428. "snapshot": dict(record.snapshot or {}),
  429. "source_mapping_uid": str(record.source_mapping_uid),
  430. "actor_uid": record.actor_uid,
  431. "created_at": _iso(record.created_at),
  432. }
  433. def _device_asset_import_result(result):
  434. return {
  435. "records": [
  436. {
  437. "action": item.action,
  438. "asset": _device_asset(item.asset),
  439. "source_mapping": _device_asset_mapping(item.mapping),
  440. }
  441. for item in result.items
  442. ],
  443. "created_count": int(result.created_count),
  444. "updated_count": int(result.updated_count),
  445. "unchanged_count": int(result.unchanged_count),
  446. }
  447. def _device_semantic_profile(result):
  448. if result is None:
  449. return {
  450. "bootstrapped": False,
  451. "ontology": None,
  452. "version": None,
  453. "profile": {
  454. "ready_to_publish": False,
  455. "class_count": 0,
  456. "relation_count": 0,
  457. "mapping_count": 0,
  458. "missing_classes": [],
  459. "missing_relations": [],
  460. "missing_mappings": [],
  461. },
  462. }
  463. profile = result.profile
  464. return {
  465. "bootstrapped": True,
  466. "created": bool(result.created),
  467. "ontology": _ontology(result.ontology),
  468. "version": _ontology_version(result.version),
  469. "profile": {
  470. "ready_to_publish": bool(profile.ready_to_publish),
  471. "class_count": int(profile.class_count),
  472. "relation_count": int(profile.relation_count),
  473. "mapping_count": int(profile.mapping_count),
  474. "missing_classes": list(profile.missing_classes),
  475. "missing_relations": list(profile.missing_relations),
  476. "missing_mappings": list(profile.missing_mappings),
  477. },
  478. }
  479. def _device_semantic_code(record):
  480. return {
  481. "uid": str(record.uid),
  482. "ontology_uid": str(record.ontology_uid),
  483. "code_type": record.code_type,
  484. "canonical_code": record.canonical_code,
  485. "canonical_name": record.canonical_name,
  486. "definition": record.definition,
  487. "status": record.status,
  488. "current_version": int(record.current_version),
  489. "source_mappings": [
  490. dict(item) for item in record.source_mappings
  491. ],
  492. "evidence_uids": list(record.evidence_uids),
  493. "suggestion_source": record.suggestion_source,
  494. "confidence": record.confidence,
  495. "created_by": record.created_by,
  496. "updated_by": record.updated_by,
  497. "created_at": _iso(record.created_at),
  498. "updated_at": _iso(record.updated_at),
  499. }
  500. def _device_semantic_version(record):
  501. return {
  502. "uid": str(record.uid),
  503. "code_uid": str(record.code_uid),
  504. "version": int(record.version),
  505. "snapshot": dict(record.snapshot),
  506. "created_by": record.created_by,
  507. "created_at": _iso(record.created_at),
  508. }
  509. def _device_semantic_review(record):
  510. return {
  511. "uid": str(record.uid),
  512. "code_uid": str(record.code_uid),
  513. "version": int(record.version),
  514. "decision": record.decision,
  515. "reason": record.reason,
  516. "actor_uid": record.actor_uid,
  517. "created_at": _iso(record.created_at),
  518. }
  519. def _device_entity_candidate(record):
  520. return {
  521. "uid": str(record.uid),
  522. "left_asset_uid": str(record.left_asset_uid),
  523. "right_asset_uid": str(record.right_asset_uid),
  524. "canonical_asset_uid": (
  525. str(record.canonical_asset_uid)
  526. if record.canonical_asset_uid
  527. else None
  528. ),
  529. "status": record.status,
  530. "suggestion_source": record.suggestion_source,
  531. "confidence": float(record.confidence),
  532. "explanation": [
  533. dict(item) for item in record.explanation
  534. ],
  535. "evidence_uids": list(record.evidence_uids),
  536. "model_provider": record.model_provider,
  537. "model_name": record.model_name,
  538. "current_version": int(record.current_version),
  539. "created_by": record.created_by,
  540. "reviewed_by": record.reviewed_by,
  541. "created_at": _iso(record.created_at),
  542. "updated_at": _iso(record.updated_at),
  543. }
  544. def _device_entity_review(record):
  545. return {
  546. "uid": str(record.uid),
  547. "candidate_uid": str(record.candidate_uid),
  548. "version": int(record.version),
  549. "decision": record.decision,
  550. "reason": record.reason,
  551. "actor_uid": record.actor_uid,
  552. "created_at": _iso(record.created_at),
  553. }
  554. def _device_entity_merge(record):
  555. return {
  556. "uid": str(record.uid),
  557. "candidate_uid": str(record.candidate_uid),
  558. "canonical_asset_uid": str(record.canonical_asset_uid),
  559. "member_asset_uid": str(record.member_asset_uid),
  560. "review_uid": str(record.review_uid),
  561. "snapshot": dict(record.snapshot),
  562. "actor_uid": record.actor_uid,
  563. "created_at": _iso(record.created_at),
  564. }
  565. def _device_entity_rollback(record):
  566. return {
  567. "uid": str(record.uid),
  568. "merge_uid": str(record.merge_uid),
  569. "candidate_uid": str(record.candidate_uid),
  570. "reason": record.reason,
  571. "snapshot": dict(record.snapshot),
  572. "actor_uid": record.actor_uid,
  573. "created_at": _iso(record.created_at),
  574. }
  575. def _device_entity_generation(result):
  576. return {
  577. "records": [
  578. _device_entity_candidate(record)
  579. for record in result.records
  580. ],
  581. "created_count": int(result.created_count),
  582. "existing_count": int(result.existing_count),
  583. "evaluated_pair_count": int(result.evaluated_pair_count),
  584. "auto_merged_count": int(result.auto_merged_count),
  585. "auto_merge_enabled": bool(
  586. current_app.config.get(
  587. "DEVICE_ENTITY_AUTO_MERGE_ENABLED",
  588. False,
  589. )
  590. ),
  591. }
  592. def _device_quality_version(record):
  593. if record is None:
  594. return None
  595. return {
  596. "uid": str(record.uid),
  597. "profile_uid": str(record.profile_uid),
  598. "version": int(record.version),
  599. "status": record.status,
  600. "rules": [dict(item) for item in record.rules],
  601. "content_hash": record.content_hash,
  602. "created_by": record.created_by,
  603. "created_at": _iso(record.created_at),
  604. "published_by": record.published_by,
  605. "published_at": _iso(record.published_at),
  606. }
  607. def _device_quality_run(record):
  608. return {
  609. "uid": str(record.uid),
  610. "policy_version_uid": str(record.policy_version_uid),
  611. "policy_hash": record.policy_hash,
  612. "source_uid": (
  613. str(record.source_uid) if record.source_uid else None
  614. ),
  615. "status": record.status,
  616. "total_assets": int(record.total_assets),
  617. "total_violations": int(record.total_violations),
  618. "score": float(record.score),
  619. "created_by": record.created_by,
  620. "created_at": _iso(record.created_at),
  621. }
  622. def _device_quality_rule_result(record):
  623. return {
  624. "uid": str(record.uid),
  625. "run_uid": str(record.run_uid),
  626. "rule_code": record.rule_code,
  627. "severity": record.severity,
  628. "weight": float(record.weight),
  629. "status": record.status,
  630. "evaluated_count": int(record.evaluated_count),
  631. "violation_count": int(record.violation_count),
  632. "sampled_count": int(record.sampled_count),
  633. "pass_rate": float(record.pass_rate),
  634. "weighted_score": float(record.weighted_score),
  635. "created_at": _iso(record.created_at),
  636. }
  637. def _device_quality_violation(record):
  638. return {
  639. "uid": str(record.uid),
  640. "run_uid": str(record.run_uid),
  641. "rule_code": record.rule_code,
  642. "severity": record.severity,
  643. "asset_uid": str(record.asset_uid),
  644. "field_name": record.field_name,
  645. "source_uid": (
  646. str(record.source_uid) if record.source_uid else None
  647. ),
  648. "source_mapping_uid": (
  649. str(record.source_mapping_uid)
  650. if record.source_mapping_uid
  651. else None
  652. ),
  653. "message": record.message,
  654. "evidence": dict(record.evidence or {}),
  655. "created_at": _iso(record.created_at),
  656. "expires_at": _iso(record.expires_at),
  657. }
  658. def _device_quality_asset_score(record):
  659. return {
  660. "uid": str(record.uid),
  661. "run_uid": str(record.run_uid),
  662. "asset_uid": str(record.asset_uid),
  663. "asset_type": record.asset_type,
  664. "status": record.status,
  665. "evaluated_rule_count": int(record.evaluated_rule_count),
  666. "violation_count": int(record.violation_count),
  667. "score": float(record.score),
  668. "created_at": _iso(record.created_at),
  669. }
  670. def _device_asset_page(name, *, default, maximum):
  671. from app.core.data_research.errors import DeviceAssetInvalid
  672. raw = request.args.get(name)
  673. try:
  674. value = default if raw in (None, "") else int(raw)
  675. except (TypeError, ValueError) as error:
  676. raise DeviceAssetInvalid(f"{name} must be an integer") from error
  677. if value < 1 or value > maximum:
  678. raise DeviceAssetInvalid(
  679. f"{name} must be between 1 and {maximum}"
  680. )
  681. return value
  682. def _error(error):
  683. if isinstance(error, DataResearchError):
  684. return (
  685. jsonify(
  686. failed(
  687. str(error),
  688. code=error.http_status,
  689. error={"code": error.code},
  690. )
  691. ),
  692. error.http_status,
  693. )
  694. logger.exception("data-research ingestion request failed")
  695. return (
  696. jsonify(
  697. failed(
  698. "数据采集任务处理失败",
  699. code=500,
  700. error={"code": "DATA_RESEARCH_ERROR"},
  701. )
  702. ),
  703. 500,
  704. )
  705. @bp.route("/ingestion-jobs", methods=["POST"])
  706. def create_ingestion_job():
  707. payload = request.get_json(silent=True) or {}
  708. try:
  709. actor_uid = _identity().get("id") or _identity().get("sub")
  710. if payload.get("job_type") == "catalog_collect":
  711. get_database_source_registration_service().ensure(
  712. payload.get("source_uid"),
  713. actor_uid=actor_uid,
  714. )
  715. record, created = get_ingestion_service().create_job(
  716. payload,
  717. actor_uid=actor_uid,
  718. )
  719. return jsonify(success(_record(record))), 201 if created else 200
  720. except Exception as error:
  721. return _error(error)
  722. @bp.route("/ingestion-jobs", methods=["GET"])
  723. def list_ingestion_jobs():
  724. filters = {
  725. name: request.args.get(name)
  726. for name in ("status", "source_uid")
  727. if request.args.get(name)
  728. }
  729. try:
  730. records = get_ingestion_service().list_jobs(filters)
  731. return jsonify(
  732. success({"records": [_record(item) for item in records], "total": len(records)})
  733. ), 200
  734. except Exception as error:
  735. return _error(error)
  736. @bp.route("/ingestion-jobs/<job_uid>", methods=["GET"])
  737. def get_ingestion_job(job_uid):
  738. try:
  739. return jsonify(success(_record(get_ingestion_service().get_job(job_uid)))), 200
  740. except Exception as error:
  741. return _error(error)
  742. @bp.route("/ingestion-jobs/<job_uid>/execute", methods=["POST"])
  743. def execute_ingestion_job(job_uid):
  744. try:
  745. record = get_catalog_ingestion_executor().execute(job_uid)
  746. return jsonify(success(_record(record))), 200
  747. except Exception as error:
  748. return _error(error)
  749. @bp.route(
  750. "/ingestion-jobs/<job_uid>/catalog-snapshots",
  751. methods=["GET"],
  752. )
  753. def list_catalog_snapshots(job_uid):
  754. try:
  755. records = get_catalog_snapshot_repository().list(job_uid)
  756. return jsonify(
  757. success(
  758. {
  759. "records": [
  760. _catalog_snapshot(record)
  761. for record in records
  762. ],
  763. "total": len(records),
  764. }
  765. )
  766. ), 200
  767. except Exception as error:
  768. return _error(error)
  769. @bp.route("/ingestion-jobs/<job_uid>/evidence", methods=["GET"])
  770. def list_ingestion_job_evidence(job_uid):
  771. try:
  772. records = get_evidence_service().list_for_job(job_uid)
  773. return jsonify(
  774. success({"records": records, "total": len(records)})
  775. ), 200
  776. except Exception as error:
  777. return _error(error)
  778. @bp.route("/ingestion-jobs/<job_uid>/retry", methods=["POST"])
  779. def retry_ingestion_job(job_uid):
  780. try:
  781. return jsonify(success(_record(get_ingestion_service().retry(job_uid)))), 200
  782. except Exception as error:
  783. return _error(error)
  784. @bp.route("/ingestion-jobs/<job_uid>/cancel", methods=["POST"])
  785. def cancel_ingestion_job(job_uid):
  786. try:
  787. service = get_ingestion_service()
  788. record = service.get_job(job_uid)
  789. identity = _identity()
  790. permissions = set(identity.get("permissions") or [])
  791. actor_uid = identity.get("id") or identity.get("sub")
  792. if record.actor_uid != actor_uid and "ingestion:admin" not in permissions:
  793. return jsonify(failed("权限不足", code=403)), 403
  794. return jsonify(success(_record(service.cancel(job_uid)))), 200
  795. except Exception as error:
  796. return _error(error)
  797. @bp.route("/data-elements", methods=["GET"])
  798. def list_data_elements():
  799. try:
  800. records = get_data_element_service().list(
  801. status=request.args.get("status"),
  802. business_domain_uid=request.args.get("business_domain_uid"),
  803. )
  804. return jsonify(
  805. success(
  806. {
  807. "records": [_element(item) for item in records],
  808. "total": len(records),
  809. }
  810. )
  811. ), 200
  812. except Exception as error:
  813. return _error(error)
  814. @bp.route("/device-assets", methods=["GET"])
  815. def list_device_assets():
  816. filters = {
  817. name: request.args.get(name)
  818. for name in ("keyword", "asset_type", "status", "source_uid")
  819. if request.args.get(name)
  820. }
  821. try:
  822. page = _device_asset_page(
  823. "page",
  824. default=1,
  825. maximum=1_000_000,
  826. )
  827. page_size = _device_asset_page(
  828. "page_size",
  829. default=20,
  830. maximum=100,
  831. )
  832. records, total = get_device_asset_service().search(
  833. filters,
  834. page=page,
  835. page_size=page_size,
  836. )
  837. return jsonify(
  838. success(
  839. {
  840. "records": [
  841. _device_asset_detail(record)
  842. for record in records
  843. ],
  844. "total": int(total),
  845. "page": page,
  846. "page_size": page_size,
  847. }
  848. )
  849. ), 200
  850. except Exception as error:
  851. return _error(error)
  852. @bp.route("/device-assets/import", methods=["POST"])
  853. def import_device_assets():
  854. try:
  855. result = get_device_asset_service().import_records(
  856. request.get_json(silent=True) or {},
  857. actor_uid=_identity().get("id") or _identity().get("sub"),
  858. )
  859. return jsonify(success(_device_asset_import_result(result))), 200
  860. except Exception as error:
  861. return _error(error)
  862. @bp.route("/device-assets/<asset_uid>", methods=["GET"])
  863. def get_device_asset(asset_uid):
  864. try:
  865. return jsonify(
  866. success(
  867. _device_asset_detail(
  868. get_device_asset_service().get(asset_uid)
  869. )
  870. )
  871. ), 200
  872. except Exception as error:
  873. return _error(error)
  874. @bp.route("/device-assets/<asset_uid>/versions", methods=["GET"])
  875. def list_device_asset_versions(asset_uid):
  876. try:
  877. records = get_device_asset_service().versions(asset_uid)
  878. return jsonify(
  879. success(
  880. {
  881. "records": [
  882. _device_asset_version(record)
  883. for record in records
  884. ],
  885. "total": len(records),
  886. }
  887. )
  888. ), 200
  889. except Exception as error:
  890. return _error(error)
  891. @bp.route("/device-semantics/bootstrap", methods=["POST"])
  892. def bootstrap_device_semantics():
  893. try:
  894. result = get_device_semantic_service().bootstrap(
  895. request.get_json(silent=True) or {},
  896. actor_uid=_identity().get("id") or _identity().get("sub"),
  897. )
  898. return (
  899. jsonify(success(_device_semantic_profile(result))),
  900. 201 if result.created else 200,
  901. )
  902. except Exception as error:
  903. return _error(error)
  904. @bp.route("/device-semantics/profile", methods=["GET"])
  905. def get_device_semantic_profile():
  906. try:
  907. result = get_device_semantic_service().profile()
  908. return jsonify(success(_device_semantic_profile(result))), 200
  909. except Exception as error:
  910. return _error(error)
  911. @bp.route("/device-semantics/codes", methods=["GET"])
  912. def list_device_semantic_codes():
  913. filters = {
  914. name: request.args.get(name)
  915. for name in ("ontology_uid", "code_type", "status", "keyword")
  916. if request.args.get(name)
  917. }
  918. try:
  919. records, total = get_device_semantic_code_service().search(
  920. filters,
  921. page=request.args.get("page", 1),
  922. page_size=request.args.get("page_size", 20),
  923. )
  924. return jsonify(
  925. success(
  926. {
  927. "records": [
  928. _device_semantic_code(record)
  929. for record in records
  930. ],
  931. "total": int(total),
  932. "page": int(request.args.get("page", 1)),
  933. "page_size": int(request.args.get("page_size", 20)),
  934. }
  935. )
  936. ), 200
  937. except Exception as error:
  938. return _error(error)
  939. @bp.route("/device-semantics/codes", methods=["POST"])
  940. def create_device_semantic_code():
  941. try:
  942. record = get_device_semantic_code_service().create(
  943. request.get_json(silent=True) or {},
  944. actor_uid=_identity().get("id") or _identity().get("sub"),
  945. )
  946. return jsonify(success(_device_semantic_code(record))), 201
  947. except Exception as error:
  948. return _error(error)
  949. @bp.route("/device-semantics/codes/<code_uid>", methods=["GET"])
  950. def get_device_semantic_code(code_uid):
  951. try:
  952. record = get_device_semantic_code_service().get(code_uid)
  953. return jsonify(success(_device_semantic_code(record))), 200
  954. except Exception as error:
  955. return _error(error)
  956. @bp.route(
  957. "/device-semantics/codes/<code_uid>/revisions",
  958. methods=["POST"],
  959. )
  960. def revise_device_semantic_code(code_uid):
  961. payload = request.get_json(silent=True) or {}
  962. try:
  963. record = get_device_semantic_code_service().revise(
  964. code_uid,
  965. payload,
  966. expected_version=payload.get("expected_version"),
  967. actor_uid=_identity().get("id") or _identity().get("sub"),
  968. )
  969. return jsonify(success(_device_semantic_code(record))), 200
  970. except Exception as error:
  971. return _error(error)
  972. @bp.route(
  973. "/device-semantics/codes/<code_uid>/submit",
  974. methods=["POST"],
  975. )
  976. def submit_device_semantic_code(code_uid):
  977. payload = request.get_json(silent=True) or {}
  978. try:
  979. record = get_device_semantic_code_service().submit(
  980. code_uid,
  981. expected_version=payload.get("expected_version"),
  982. actor_uid=_identity().get("id") or _identity().get("sub"),
  983. )
  984. return jsonify(success(_device_semantic_code(record))), 200
  985. except Exception as error:
  986. return _error(error)
  987. @bp.route(
  988. "/device-semantics/codes/<code_uid>/review",
  989. methods=["POST"],
  990. )
  991. def review_device_semantic_code(code_uid):
  992. try:
  993. record, review = get_device_semantic_code_service().review(
  994. code_uid,
  995. request.get_json(silent=True) or {},
  996. actor_uid=_identity().get("id") or _identity().get("sub"),
  997. )
  998. return jsonify(
  999. success(
  1000. {
  1001. "record": _device_semantic_code(record),
  1002. "review": _device_semantic_review(review),
  1003. }
  1004. )
  1005. ), 200
  1006. except Exception as error:
  1007. return _error(error)
  1008. @bp.route(
  1009. "/device-semantics/codes/<code_uid>/versions",
  1010. methods=["GET"],
  1011. )
  1012. def list_device_semantic_code_versions(code_uid):
  1013. try:
  1014. records = get_device_semantic_code_service().versions(code_uid)
  1015. return jsonify(
  1016. success(
  1017. {
  1018. "records": [
  1019. _device_semantic_version(record)
  1020. for record in records
  1021. ],
  1022. "total": len(records),
  1023. }
  1024. )
  1025. ), 200
  1026. except Exception as error:
  1027. return _error(error)
  1028. @bp.route(
  1029. "/device-semantics/codes/<code_uid>/reviews",
  1030. methods=["GET"],
  1031. )
  1032. def list_device_semantic_code_reviews(code_uid):
  1033. try:
  1034. records = get_device_semantic_code_service().reviews(code_uid)
  1035. return jsonify(
  1036. success(
  1037. {
  1038. "records": [
  1039. _device_semantic_review(record)
  1040. for record in records
  1041. ],
  1042. "total": len(records),
  1043. }
  1044. )
  1045. ), 200
  1046. except Exception as error:
  1047. return _error(error)
  1048. @bp.route("/device-entities/candidates", methods=["GET"])
  1049. def list_device_entity_candidates():
  1050. filters = {
  1051. name: request.args.get(name)
  1052. for name in ("status", "suggestion_source")
  1053. if request.args.get(name)
  1054. }
  1055. try:
  1056. records, total = get_device_entity_resolution_service().search(
  1057. filters,
  1058. page=request.args.get("page", 1),
  1059. page_size=request.args.get("page_size", 20),
  1060. )
  1061. return jsonify(
  1062. success(
  1063. {
  1064. "records": [
  1065. _device_entity_candidate(record)
  1066. for record in records
  1067. ],
  1068. "total": int(total),
  1069. "page": int(request.args.get("page", 1)),
  1070. "page_size": int(request.args.get("page_size", 20)),
  1071. "auto_merge_enabled": bool(
  1072. current_app.config.get(
  1073. "DEVICE_ENTITY_AUTO_MERGE_ENABLED",
  1074. False,
  1075. )
  1076. ),
  1077. }
  1078. )
  1079. ), 200
  1080. except Exception as error:
  1081. return _error(error)
  1082. @bp.route("/device-entities/candidates/generate", methods=["POST"])
  1083. def generate_device_entity_candidates():
  1084. try:
  1085. result = get_device_entity_resolution_service().generate(
  1086. request.get_json(silent=True) or {},
  1087. actor_uid=_identity().get("id") or _identity().get("sub"),
  1088. )
  1089. status = 201 if result.created_count else 200
  1090. return jsonify(success(_device_entity_generation(result))), status
  1091. except Exception as error:
  1092. return _error(error)
  1093. @bp.route("/device-entities/candidates", methods=["POST"])
  1094. def submit_device_entity_candidate():
  1095. try:
  1096. record = (
  1097. get_device_entity_resolution_service().submit_ai_candidate(
  1098. request.get_json(silent=True) or {},
  1099. actor_uid=_identity().get("id") or _identity().get("sub"),
  1100. )
  1101. )
  1102. return jsonify(success(_device_entity_candidate(record))), 201
  1103. except Exception as error:
  1104. return _error(error)
  1105. @bp.route("/device-entities/candidates/<candidate_uid>", methods=["GET"])
  1106. def get_device_entity_candidate(candidate_uid):
  1107. try:
  1108. record = get_device_entity_resolution_service().get(candidate_uid)
  1109. return jsonify(success(_device_entity_candidate(record))), 200
  1110. except Exception as error:
  1111. return _error(error)
  1112. @bp.route(
  1113. "/device-entities/candidates/<candidate_uid>/review",
  1114. methods=["POST"],
  1115. )
  1116. def review_device_entity_candidate(candidate_uid):
  1117. try:
  1118. candidate, review, merge = (
  1119. get_device_entity_resolution_service().review(
  1120. candidate_uid,
  1121. request.get_json(silent=True) or {},
  1122. actor_uid=_identity().get("id") or _identity().get("sub"),
  1123. )
  1124. )
  1125. return jsonify(
  1126. success(
  1127. {
  1128. "candidate": _device_entity_candidate(candidate),
  1129. "review": _device_entity_review(review),
  1130. "merge": (
  1131. _device_entity_merge(merge)
  1132. if merge is not None
  1133. else None
  1134. ),
  1135. }
  1136. )
  1137. ), 200
  1138. except Exception as error:
  1139. return _error(error)
  1140. @bp.route(
  1141. "/device-entities/candidates/<candidate_uid>/reviews",
  1142. methods=["GET"],
  1143. )
  1144. def list_device_entity_reviews(candidate_uid):
  1145. try:
  1146. records = get_device_entity_resolution_service().reviews(
  1147. candidate_uid
  1148. )
  1149. return jsonify(
  1150. success(
  1151. {
  1152. "records": [
  1153. _device_entity_review(record)
  1154. for record in records
  1155. ],
  1156. "total": len(records),
  1157. }
  1158. )
  1159. ), 200
  1160. except Exception as error:
  1161. return _error(error)
  1162. @bp.route(
  1163. "/device-entities/candidates/<candidate_uid>/merges",
  1164. methods=["GET"],
  1165. )
  1166. def list_device_entity_merges(candidate_uid):
  1167. try:
  1168. records = get_device_entity_resolution_service().merges(
  1169. candidate_uid
  1170. )
  1171. return jsonify(
  1172. success(
  1173. {
  1174. "records": [
  1175. _device_entity_merge(record)
  1176. for record in records
  1177. ],
  1178. "total": len(records),
  1179. }
  1180. )
  1181. ), 200
  1182. except Exception as error:
  1183. return _error(error)
  1184. @bp.route(
  1185. "/device-entities/merges/<merge_uid>/rollback",
  1186. methods=["POST"],
  1187. )
  1188. def rollback_device_entity_merge(merge_uid):
  1189. try:
  1190. candidate, rollback = (
  1191. get_device_entity_resolution_service().rollback(
  1192. merge_uid,
  1193. request.get_json(silent=True) or {},
  1194. actor_uid=_identity().get("id") or _identity().get("sub"),
  1195. )
  1196. )
  1197. return jsonify(
  1198. success(
  1199. {
  1200. "candidate": _device_entity_candidate(candidate),
  1201. "rollback": _device_entity_rollback(rollback),
  1202. }
  1203. )
  1204. ), 200
  1205. except Exception as error:
  1206. return _error(error)
  1207. @bp.route(
  1208. "/device-entities/merges/<merge_uid>/rollbacks",
  1209. methods=["GET"],
  1210. )
  1211. def list_device_entity_rollbacks(merge_uid):
  1212. try:
  1213. records = get_device_entity_resolution_service().rollbacks(
  1214. merge_uid
  1215. )
  1216. return jsonify(
  1217. success(
  1218. {
  1219. "records": [
  1220. _device_entity_rollback(record)
  1221. for record in records
  1222. ],
  1223. "total": len(records),
  1224. }
  1225. )
  1226. ), 200
  1227. except Exception as error:
  1228. return _error(error)
  1229. @bp.route("/device-quality/profile", methods=["GET"])
  1230. def get_device_quality_profile():
  1231. try:
  1232. result = get_device_quality_service().profile()
  1233. return jsonify(
  1234. success(
  1235. {
  1236. "profile_uid": result["profile_uid"],
  1237. "name": result["name"],
  1238. "latest_version": _device_quality_version(
  1239. result["latest_version"]
  1240. ),
  1241. "active_version": _device_quality_version(
  1242. result["active_version"]
  1243. ),
  1244. }
  1245. )
  1246. ), 200
  1247. except Exception as error:
  1248. return _error(error)
  1249. @bp.route("/device-quality/bootstrap", methods=["POST"])
  1250. def bootstrap_device_quality():
  1251. try:
  1252. record = get_device_quality_service().bootstrap(
  1253. actor_uid=_identity().get("id") or _identity().get("sub"),
  1254. )
  1255. return jsonify(success(_device_quality_version(record))), 201
  1256. except Exception as error:
  1257. return _error(error)
  1258. @bp.route("/device-quality/profile/versions", methods=["POST"])
  1259. def revise_device_quality_profile():
  1260. try:
  1261. body = request.get_json(silent=True) or {}
  1262. record = get_device_quality_service().revise(
  1263. rules=body.get("rules"),
  1264. expected_version=body.get("expected_version"),
  1265. actor_uid=_identity().get("id") or _identity().get("sub"),
  1266. )
  1267. return jsonify(success(_device_quality_version(record))), 201
  1268. except Exception as error:
  1269. return _error(error)
  1270. @bp.route("/device-quality/profile/versions", methods=["GET"])
  1271. def list_device_quality_versions():
  1272. try:
  1273. records = get_device_quality_service().versions()
  1274. return jsonify(
  1275. success(
  1276. {
  1277. "records": [
  1278. _device_quality_version(record)
  1279. for record in records
  1280. ],
  1281. "total": len(records),
  1282. }
  1283. )
  1284. ), 200
  1285. except Exception as error:
  1286. return _error(error)
  1287. @bp.route(
  1288. "/device-quality/profile/versions/<version_uid>/publish",
  1289. methods=["POST"],
  1290. )
  1291. def publish_device_quality_version(version_uid):
  1292. try:
  1293. record = get_device_quality_service().publish(
  1294. version_uid,
  1295. actor_uid=_identity().get("id") or _identity().get("sub"),
  1296. )
  1297. return jsonify(success(_device_quality_version(record))), 200
  1298. except Exception as error:
  1299. return _error(error)
  1300. @bp.route("/device-quality/runs", methods=["POST"])
  1301. def run_device_quality():
  1302. try:
  1303. body = request.get_json(silent=True) or {}
  1304. record = get_device_quality_service().run(
  1305. actor_uid=_identity().get("id") or _identity().get("sub"),
  1306. source_uid=body.get("source_uid"),
  1307. )
  1308. return jsonify(success(_device_quality_run(record))), 201
  1309. except Exception as error:
  1310. return _error(error)
  1311. @bp.route("/device-quality/runs", methods=["GET"])
  1312. def list_device_quality_runs():
  1313. try:
  1314. page = request.args.get("page", 1)
  1315. page_size = request.args.get("page_size", 20)
  1316. records, total = get_device_quality_service().runs(
  1317. page=page,
  1318. page_size=page_size,
  1319. )
  1320. return jsonify(
  1321. success(
  1322. {
  1323. "records": [
  1324. _device_quality_run(record)
  1325. for record in records
  1326. ],
  1327. "total": int(total),
  1328. "page": int(page),
  1329. "page_size": int(page_size),
  1330. }
  1331. )
  1332. ), 200
  1333. except Exception as error:
  1334. return _error(error)
  1335. @bp.route("/device-quality/runs/<run_uid>", methods=["GET"])
  1336. def get_device_quality_run(run_uid):
  1337. try:
  1338. record, results = get_device_quality_service().get_run(run_uid)
  1339. return jsonify(
  1340. success(
  1341. {
  1342. **_device_quality_run(record),
  1343. "rule_results": [
  1344. _device_quality_rule_result(item)
  1345. for item in results
  1346. ],
  1347. }
  1348. )
  1349. ), 200
  1350. except Exception as error:
  1351. return _error(error)
  1352. @bp.route(
  1353. "/device-quality/runs/<run_uid>/violations",
  1354. methods=["GET"],
  1355. )
  1356. def list_device_quality_violations(run_uid):
  1357. try:
  1358. page = request.args.get("page", 1)
  1359. page_size = request.args.get("page_size", 20)
  1360. records, total = get_device_quality_service().violations(
  1361. run_uid,
  1362. rule_code=request.args.get("rule_code"),
  1363. page=page,
  1364. page_size=page_size,
  1365. )
  1366. return jsonify(
  1367. success(
  1368. {
  1369. "records": [
  1370. _device_quality_violation(record)
  1371. for record in records
  1372. ],
  1373. "total": int(total),
  1374. "page": int(page),
  1375. "page_size": int(page_size),
  1376. }
  1377. )
  1378. ), 200
  1379. except Exception as error:
  1380. return _error(error)
  1381. @bp.route(
  1382. "/device-quality/runs/<run_uid>/asset-scores",
  1383. methods=["GET"],
  1384. )
  1385. def list_device_quality_asset_scores(run_uid):
  1386. try:
  1387. page = request.args.get("page", 1)
  1388. page_size = request.args.get("page_size", 20)
  1389. records, total = get_device_quality_service().asset_scores(
  1390. run_uid,
  1391. page=page,
  1392. page_size=page_size,
  1393. )
  1394. return jsonify(
  1395. success(
  1396. {
  1397. "records": [
  1398. _device_quality_asset_score(record)
  1399. for record in records
  1400. ],
  1401. "total": int(total),
  1402. "page": int(page),
  1403. "page_size": int(page_size),
  1404. }
  1405. )
  1406. ), 200
  1407. except Exception as error:
  1408. return _error(error)
  1409. @bp.route("/data-elements", methods=["POST"])
  1410. def create_data_element():
  1411. try:
  1412. record = get_data_element_service().create_draft(
  1413. request.get_json(silent=True) or {},
  1414. actor_uid=_identity().get("id") or _identity().get("sub"),
  1415. )
  1416. db.session.commit()
  1417. return jsonify(success(_element(record))), 201
  1418. except Exception as error:
  1419. db.session.rollback()
  1420. return _error(error)
  1421. @bp.route("/data-elements/<element_uid>/transitions", methods=["POST"])
  1422. def transition_data_element(element_uid):
  1423. payload = request.get_json(silent=True) or {}
  1424. identity = _identity()
  1425. target_status = str(payload.get("target_status") or "")
  1426. if (
  1427. target_status in {"published", "deprecated", "retired"}
  1428. and "data-elements:publish" not in set(identity.get("permissions") or [])
  1429. ):
  1430. return jsonify(failed("权限不足", code=403)), 403
  1431. try:
  1432. record = get_data_element_service().transition(
  1433. element_uid,
  1434. target_status,
  1435. expected_version=int(payload.get("expected_version")),
  1436. actor_uid=identity.get("id") or identity.get("sub"),
  1437. )
  1438. db.session.commit()
  1439. return jsonify(success(_element(record))), 200
  1440. except Exception as error:
  1441. db.session.rollback()
  1442. return _error(error)
  1443. @bp.route("/candidate-decisions", methods=["POST"])
  1444. def decide_candidates():
  1445. payload = request.get_json(silent=True) or {}
  1446. try:
  1447. records = get_candidate_decision_service().decide(
  1448. payload.get("decisions") or [],
  1449. actor_uid=_identity().get("id") or _identity().get("sub"),
  1450. )
  1451. return jsonify(success([_decision(record) for record in records])), 200
  1452. except Exception as error:
  1453. return _error(error)
  1454. @bp.route("/sources/files", methods=["POST"])
  1455. def upload_source_file():
  1456. uploaded = request.files.get("file")
  1457. if uploaded is None:
  1458. return jsonify(failed("缺少上传文件", code=400)), 400
  1459. try:
  1460. record, created = get_artifact_service().store(
  1461. request.form.get("source_uid"),
  1462. uploaded.filename,
  1463. uploaded.mimetype,
  1464. uploaded.read(),
  1465. request.form.get("parser_version") or "auto-v1",
  1466. )
  1467. db.session.commit()
  1468. return jsonify(success(_artifact(record))), 201 if created else 200
  1469. except Exception as error:
  1470. db.session.rollback()
  1471. return _error(error)
  1472. @bp.route("/evidence/<evidence_uid>", methods=["GET"])
  1473. def get_evidence(evidence_uid):
  1474. identity = _identity()
  1475. try:
  1476. service = get_evidence_service()
  1477. if request.args.get("download") in {"1", "true", "yes"}:
  1478. if "evidence:download" not in set(identity.get("permissions") or []):
  1479. return jsonify(failed("权限不足", code=403)), 403
  1480. content, filename, media_type = service.download(evidence_uid)
  1481. return send_file(
  1482. io.BytesIO(content),
  1483. mimetype=media_type,
  1484. as_attachment=True,
  1485. download_name=filename,
  1486. )
  1487. preview = service.preview(evidence_uid)
  1488. from app.core.data_research.artifacts import redact_excerpt
  1489. preview["excerpt"] = redact_excerpt(preview.get("excerpt"))
  1490. return jsonify(success(preview)), 200
  1491. except Exception as error:
  1492. return _error(error)
  1493. @bp.route("/ontologies", methods=["GET"])
  1494. def list_ontologies():
  1495. try:
  1496. return jsonify(success([_ontology(item) for item in get_ontology_service().list()])), 200
  1497. except Exception as error:
  1498. return _error(error)
  1499. @bp.route("/ontologies", methods=["POST"])
  1500. def create_ontology():
  1501. try:
  1502. record = get_ontology_service().create(
  1503. request.get_json(silent=True) or {},
  1504. _identity().get("id") or _identity().get("sub"),
  1505. )
  1506. return jsonify(success(_ontology(record))), 201
  1507. except Exception as error:
  1508. return _error(error)
  1509. @bp.route("/ontologies/<ontology_uid>", methods=["GET"])
  1510. def get_ontology(ontology_uid):
  1511. try:
  1512. return jsonify(success(_ontology(get_ontology_service().get(ontology_uid)))), 200
  1513. except Exception as error:
  1514. return _error(error)
  1515. @bp.route("/ontologies/<ontology_uid>/versions", methods=["GET"])
  1516. def list_ontology_versions(ontology_uid):
  1517. try:
  1518. return jsonify(
  1519. success(
  1520. [
  1521. _ontology_version(item)
  1522. for item in get_ontology_service().list_versions(ontology_uid)
  1523. ]
  1524. )
  1525. ), 200
  1526. except Exception as error:
  1527. return _error(error)
  1528. @bp.route("/ontologies/<ontology_uid>/graph", methods=["GET"])
  1529. def get_ontology_graph(ontology_uid):
  1530. try:
  1531. service = get_ontology_service()
  1532. ontology = service.get(ontology_uid)
  1533. version = service.latest_version(ontology_uid)
  1534. data = (
  1535. _ontology_version(version)
  1536. if version is not None
  1537. else {
  1538. "uid": None,
  1539. "ontology_uid": str(ontology_uid),
  1540. "version": 0,
  1541. "parent_version_uid": None,
  1542. "status": "draft",
  1543. "content_hash": None,
  1544. "graph_document": {
  1545. name: []
  1546. for name in (
  1547. "classes",
  1548. "properties",
  1549. "relations",
  1550. "constraints",
  1551. "domain_links",
  1552. "element_mappings",
  1553. )
  1554. },
  1555. "created_by": None,
  1556. }
  1557. )
  1558. response = jsonify(success(data))
  1559. response.headers["ETag"] = f'"{int(ontology.draft_revision)}"'
  1560. return response, 200
  1561. except Exception as error:
  1562. return _error(error)
  1563. @bp.route("/ontologies/<ontology_uid>/graph", methods=["PATCH"])
  1564. def save_ontology_graph(ontology_uid):
  1565. etag = str(request.headers.get("If-Match") or "").strip().strip('"')
  1566. if not etag.isdigit():
  1567. return jsonify(failed("缺少有效 If-Match 修订号", code=428)), 428
  1568. try:
  1569. version = get_ontology_service().save_draft(
  1570. ontology_uid,
  1571. request.get_json(silent=True) or {},
  1572. int(etag),
  1573. _identity().get("id") or _identity().get("sub"),
  1574. )
  1575. response = jsonify(success(_ontology_version(version)))
  1576. response.headers["ETag"] = f'"{int(etag) + 1}"'
  1577. return response, 200
  1578. except Exception as error:
  1579. return _error(error)
  1580. @bp.route("/ontologies/<ontology_uid>/validate", methods=["POST"])
  1581. def validate_ontology(ontology_uid):
  1582. try:
  1583. issues = get_ontology_service().validate(ontology_uid)
  1584. data = [
  1585. {"code": item.code, "message": item.message, "path": item.path}
  1586. for item in issues
  1587. ]
  1588. return jsonify(success(data)), 200
  1589. except Exception as error:
  1590. return _error(error)
  1591. @bp.route("/ontologies/<ontology_uid>/publish", methods=["POST"])
  1592. def publish_ontology(ontology_uid):
  1593. try:
  1594. version = get_ontology_service().publish(
  1595. ontology_uid,
  1596. request.headers.get("Idempotency-Key") or f"{ontology_uid}:publish",
  1597. _identity().get("id") or _identity().get("sub"),
  1598. )
  1599. return jsonify(success(_ontology_version(version))), 200
  1600. except Exception as error:
  1601. return _error(error)
  1602. @bp.route("/ontologies/<ontology_uid>/diff", methods=["GET"])
  1603. def diff_ontology(ontology_uid):
  1604. try:
  1605. return jsonify(success(get_ontology_service().diff(
  1606. ontology_uid, request.args.get("left"), request.args.get("right")
  1607. ))), 200
  1608. except Exception as error:
  1609. return _error(error)
  1610. @bp.route("/ontologies/<ontology_uid>/rollback", methods=["POST"])
  1611. def rollback_ontology(ontology_uid):
  1612. payload = request.get_json(silent=True) or {}
  1613. try:
  1614. version = get_ontology_service().rollback(
  1615. ontology_uid,
  1616. payload.get("target_version_uid"),
  1617. int(payload.get("expected_revision")),
  1618. _identity().get("id") or _identity().get("sub"),
  1619. )
  1620. return jsonify(success(_ontology_version(version))), 201
  1621. except Exception as error:
  1622. return _error(error)
  1623. @bp.route("/ontologies/<ontology_uid>/suggestions", methods=["POST"])
  1624. def generate_ontology_suggestions(ontology_uid):
  1625. try:
  1626. result = get_ontology_dynamic_service().generate(
  1627. ontology_uid,
  1628. request.get_json(silent=True) or {},
  1629. _identity().get("id") or _identity().get("sub"),
  1630. )
  1631. records = (
  1632. result.get("suggestions") or []
  1633. if isinstance(result, dict)
  1634. else result
  1635. )
  1636. suggestions = [
  1637. item
  1638. if isinstance(item, dict)
  1639. else {
  1640. "uid": item.uid,
  1641. "kind": item.kind,
  1642. "payload": item.payload,
  1643. "evidence_uids": list(item.evidence_uids),
  1644. "confidence": item.confidence,
  1645. "source": item.source,
  1646. "model_version": item.model_version,
  1647. "prompt_version": item.prompt_version,
  1648. }
  1649. for item in records
  1650. ]
  1651. data = {
  1652. "change_set_uid": (
  1653. result.get("change_set_uid")
  1654. if isinstance(result, dict)
  1655. else None
  1656. ),
  1657. "suggestions": suggestions,
  1658. }
  1659. return jsonify(success(data)), 201
  1660. except Exception as error:
  1661. return _error(error)
  1662. @bp.route(
  1663. "/ontologies/<ontology_uid>/change-sets/<change_set_uid>/decisions",
  1664. methods=["POST"],
  1665. )
  1666. def decide_ontology_change_set(ontology_uid, change_set_uid):
  1667. payload = request.get_json(silent=True) or {}
  1668. try:
  1669. result = get_ontology_dynamic_service().decide(
  1670. ontology_uid,
  1671. change_set_uid,
  1672. payload.get("decisions") or [],
  1673. _identity().get("id") or _identity().get("sub"),
  1674. )
  1675. return jsonify(success(result)), 200
  1676. except Exception as error:
  1677. return _error(error)
  1678. @bp.route("/ontologies/<ontology_uid>/export", methods=["GET"])
  1679. def export_ontology(ontology_uid):
  1680. try:
  1681. content, media_type, filename = get_ontology_exchange_service().export(
  1682. ontology_uid, request.args.get("format") or "json"
  1683. )
  1684. return send_file(
  1685. io.BytesIO(content),
  1686. mimetype=media_type,
  1687. as_attachment=True,
  1688. download_name=filename,
  1689. )
  1690. except Exception as error:
  1691. return _error(error)
  1692. @bp.route("/ontologies/import", methods=["POST"])
  1693. def import_ontology():
  1694. try:
  1695. uploaded = request.files.get("file")
  1696. content = uploaded.read() if uploaded is not None else request.get_data(cache=False)
  1697. result = get_ontology_exchange_service().import_document(
  1698. content,
  1699. request.args.get("format") or "json",
  1700. _identity().get("id") or _identity().get("sub"),
  1701. )
  1702. return jsonify(success(result)), 201
  1703. except Exception as error:
  1704. return _error(error)
  1705. @bp.route("/semantic/properties/<property_uid>", methods=["GET"])
  1706. def query_semantic_property(property_uid):
  1707. domain = str(request.args.get("business_domain_uid") or "").strip()
  1708. identity = _identity()
  1709. scoped = set(identity.get("business_domains") or [domain])
  1710. if "*" not in scoped and domain not in scoped:
  1711. return jsonify(failed("权限不足", code=403)), 403
  1712. try:
  1713. result = get_semantic_query_service().trace_property(
  1714. property_uid,
  1715. allowed_domains={domain},
  1716. limit=int(request.args.get("limit") or 20),
  1717. after_uid=request.args.get("after_uid"),
  1718. )
  1719. return jsonify(success(result)), 200
  1720. except Exception as error:
  1721. return _error(error)