product_engineering.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764
  1. #!/usr/bin/env python3
  2. """Evidence-first product engineering controls for P2-WP11.
  3. The commands in this module validate or render evidence. They do not promote
  4. objects, mutate customer source data, switch traffic, or downgrade databases.
  5. """
  6. from __future__ import annotations
  7. import argparse
  8. import ast
  9. import hashlib
  10. import json
  11. import re
  12. import subprocess
  13. import sys
  14. from pathlib import Path
  15. from typing import Any
  16. READINESS_CATEGORIES = (
  17. "service",
  18. "task",
  19. "database",
  20. "graph",
  21. "object_storage",
  22. "external_dependency",
  23. )
  24. PROMOTABLE_KINDS = {"asset", "standard", "ontology", "rule", "workflow"}
  25. ENVIRONMENT_ORDER = {"development": 10, "test": 20, "preproduction": 30}
  26. HEALTHY_STATES = {"healthy", "ready", "succeeded", "completed", "available"}
  27. SECRET_KEY_PATTERN = re.compile(
  28. r"(?:secret|password|token|credential|authorization|api[_-]?key)", re.I
  29. )
  30. SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
  31. COMPOSE_SERVICE_CATEGORIES = {
  32. "backend": "service",
  33. "frontend": "service",
  34. "runner": "task",
  35. "n8n": "task",
  36. "kestra": "task",
  37. "kestra-db-init": "task",
  38. "postgres": "database",
  39. "neo4j": "graph",
  40. "minio": "object_storage",
  41. "minio-init": "object_storage",
  42. "source-postgres": "external_dependency",
  43. "source-mysql": "external_dependency",
  44. }
  45. def _canonical_bytes(value: Any) -> bytes:
  46. return json.dumps(
  47. value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
  48. ).encode("utf-8")
  49. def _sha256_bytes(value: bytes) -> str:
  50. return hashlib.sha256(value).hexdigest()
  51. def _sha256_file(path: Path) -> str:
  52. return _sha256_bytes(path.read_bytes())
  53. def _tree_manifest(root: Path) -> dict[str, Any]:
  54. files: list[dict[str, str]] = []
  55. if not root.exists():
  56. return {"sha256": None, "file_count": 0}
  57. for path in sorted(root.rglob("*")):
  58. if not path.is_file():
  59. continue
  60. relative = path.relative_to(root)
  61. if "__pycache__" in relative.parts or path.suffix in {".pyc", ".pyo"}:
  62. continue
  63. files.append(
  64. {"path": relative.as_posix(), "sha256": _sha256_file(path)}
  65. )
  66. return {
  67. "sha256": _sha256_bytes(_canonical_bytes(files)),
  68. "file_count": len(files),
  69. }
  70. def _redact(value: Any) -> Any:
  71. if isinstance(value, dict):
  72. return {
  73. str(key): (
  74. "[REDACTED]"
  75. if SECRET_KEY_PATTERN.search(str(key))
  76. else _redact(item)
  77. )
  78. for key, item in value.items()
  79. }
  80. if isinstance(value, list):
  81. return [_redact(item) for item in value]
  82. return value
  83. def _read_json(path: Path) -> dict[str, Any]:
  84. value = json.loads(path.read_text(encoding="utf-8"))
  85. if not isinstance(value, dict):
  86. raise ValueError(f"JSON object required: {path}")
  87. return value
  88. def _write_json(value: Any, output: str | None = None) -> None:
  89. rendered = json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2) + "\n"
  90. if not output or output == "-":
  91. sys.stdout.write(rendered)
  92. return
  93. path = Path(output)
  94. path.parent.mkdir(parents=True, exist_ok=True)
  95. path.write_text(rendered, encoding="utf-8")
  96. def validate_environment_profiles(profiles_dir: Path) -> dict[str, Any]:
  97. profiles: list[dict[str, Any]] = []
  98. seen: dict[tuple[str, str], str] = {}
  99. collisions: list[dict[str, str]] = []
  100. expected = set(ENVIRONMENT_ORDER)
  101. for path in sorted(profiles_dir.glob("*.json")):
  102. profile = _read_json(path)
  103. environment = str(profile.get("environment", ""))
  104. if environment not in ENVIRONMENT_ORDER:
  105. raise ValueError(f"unknown environment in {path.name}: {environment}")
  106. if int(profile.get("promotion_order", -1)) != ENVIRONMENT_ORDER[environment]:
  107. raise ValueError(f"invalid promotion order for {environment}")
  108. required_text = ("project_name", "network_name", "data_namespace")
  109. for field in required_text:
  110. if not str(profile.get(field, "")).strip():
  111. raise ValueError(f"{field} is required for {environment}")
  112. key = (field, str(profile[field]))
  113. if key in seen:
  114. collisions.append(
  115. {
  116. "field": field,
  117. "value": str(profile[field]),
  118. "environments": f"{seen[key]},{environment}",
  119. }
  120. )
  121. seen[key] = environment
  122. ports = profile.get("ports")
  123. if not isinstance(ports, dict) or not ports:
  124. raise ValueError(f"ports are required for {environment}")
  125. for name, raw_port in sorted(ports.items()):
  126. port = int(raw_port)
  127. if not 1 <= port <= 65535:
  128. raise ValueError(f"invalid port {name}={port} for {environment}")
  129. key = ("published_port", str(port))
  130. if key in seen:
  131. collisions.append(
  132. {
  133. "field": "published_port",
  134. "value": str(port),
  135. "environments": f"{seen[key]},{environment}",
  136. }
  137. )
  138. seen[key] = environment
  139. controls = profile.get("controls")
  140. if not isinstance(controls, dict) or "auth_required" not in controls:
  141. raise ValueError(f"controls.auth_required is required for {environment}")
  142. profiles.append(
  143. {
  144. "environment": environment,
  145. "promotion_order": ENVIRONMENT_ORDER[environment],
  146. "project_name": profile["project_name"],
  147. "network_name": profile["network_name"],
  148. "data_namespace": profile["data_namespace"],
  149. "ports": dict(sorted((str(k), int(v)) for k, v in ports.items())),
  150. "controls": controls,
  151. "profile_sha256": _sha256_file(path),
  152. "path": path.name,
  153. }
  154. )
  155. found = {item["environment"] for item in profiles}
  156. if collisions:
  157. raise ValueError(f"environment collision detected: {collisions}")
  158. if found != expected:
  159. raise ValueError(
  160. "environment profile set must be development,test,preproduction; "
  161. f"found={','.join(sorted(found))}"
  162. )
  163. profiles.sort(key=lambda item: int(item["promotion_order"]))
  164. differences: list[dict[str, Any]] = []
  165. baseline = profiles[0]
  166. for profile in profiles[1:]:
  167. changed = [
  168. field
  169. for field in ("project_name", "network_name", "data_namespace", "ports", "controls")
  170. if profile[field] != baseline[field]
  171. ]
  172. differences.append(
  173. {
  174. "from": baseline["environment"],
  175. "to": profile["environment"],
  176. "changed_fields": changed,
  177. }
  178. )
  179. return {
  180. "schema_version": 1,
  181. "status": "ready",
  182. "promotion_path": [item["environment"] for item in profiles],
  183. "profiles": profiles,
  184. "collisions": [],
  185. "differences": differences,
  186. }
  187. def build_readiness_report(request: dict[str, Any]) -> dict[str, Any]:
  188. components = request.get("components")
  189. if not isinstance(components, list):
  190. raise ValueError("components must be a list")
  191. category_reports: dict[str, dict[str, Any]] = {}
  192. blocking: list[str] = []
  193. redacted_components: list[dict[str, Any]] = []
  194. for component in components:
  195. if not isinstance(component, dict):
  196. raise ValueError("each component must be an object")
  197. category = str(component.get("category", ""))
  198. if category not in READINESS_CATEGORIES:
  199. raise ValueError(f"unsupported readiness category: {category}")
  200. name = str(component.get("name", "")).strip()
  201. if not name:
  202. raise ValueError("component name is required")
  203. status = str(component.get("status", "unknown")).lower()
  204. required = bool(component.get("required", True))
  205. ready = status in HEALTHY_STATES
  206. if required and not ready:
  207. blocking.append(name)
  208. redacted_components.append(_redact(component))
  209. for category in READINESS_CATEGORIES:
  210. selected = [
  211. item for item in redacted_components if item["category"] == category
  212. ]
  213. required = [item for item in selected if item.get("required", True)]
  214. blocked = [
  215. item["name"]
  216. for item in required
  217. if str(item.get("status", "unknown")).lower() not in HEALTHY_STATES
  218. ]
  219. category_reports[category] = {
  220. "status": "ready" if required and not blocked else (
  221. "blocked" if blocked else "unconfigured"
  222. ),
  223. "component_count": len(selected),
  224. "required_count": len(required),
  225. "blocked_components": blocked,
  226. }
  227. unconfigured = sorted(
  228. category
  229. for category, report in category_reports.items()
  230. if report["status"] == "unconfigured"
  231. )
  232. return {
  233. "schema_version": 1,
  234. "environment": str(request.get("environment", "unknown")),
  235. "status": "blocked" if blocking or unconfigured else "ready",
  236. "blocking_components": sorted(blocking),
  237. "unconfigured_categories": unconfigured,
  238. "categories": category_reports,
  239. "components": redacted_components,
  240. }
  241. def build_compose_readiness_report(
  242. rows: list[dict[str, Any]], *, environment: str
  243. ) -> dict[str, Any]:
  244. components: list[dict[str, Any]] = []
  245. for row in rows:
  246. service = str(row.get("Service", ""))
  247. category = COMPOSE_SERVICE_CATEGORIES.get(service)
  248. if not category:
  249. continue
  250. state = str(row.get("State", "unknown")).lower()
  251. health = str(row.get("Health", "")).lower()
  252. exit_code = int(row.get("ExitCode", -1))
  253. completed = state == "exited" and exit_code == 0 and service.endswith("-init")
  254. ready = completed or (state == "running" and health in {"", "healthy"})
  255. components.append(
  256. {
  257. "name": service,
  258. "category": category,
  259. "status": "completed" if completed else ("healthy" if ready else "unhealthy"),
  260. "required": True,
  261. "details": {
  262. "state": state,
  263. "health": health or "not_reported",
  264. "exit_code": exit_code,
  265. "container_id": str(row.get("ID", ""))[:12],
  266. },
  267. }
  268. )
  269. report = build_readiness_report(
  270. {"environment": environment, "components": components}
  271. )
  272. report["collector"] = "docker_compose_ps"
  273. return report
  274. def collect_compose_readiness(
  275. compose_file: Path,
  276. *,
  277. project_name: str,
  278. environment: str,
  279. env_file: Path | None = None,
  280. ) -> dict[str, Any]:
  281. command = ["docker", "compose"]
  282. if env_file:
  283. command.extend(["--env-file", str(env_file)])
  284. command.extend(
  285. ["-p", project_name, "-f", str(compose_file), "ps", "-a", "--format", "json"]
  286. )
  287. completed = subprocess.run(command, check=True, capture_output=True, text=True)
  288. rendered = completed.stdout.strip()
  289. rows: list[dict[str, Any]] = []
  290. if rendered:
  291. try:
  292. parsed = json.loads(rendered)
  293. rows = parsed if isinstance(parsed, list) else [parsed]
  294. except json.JSONDecodeError:
  295. rows = [json.loads(line) for line in rendered.splitlines() if line.strip()]
  296. return build_compose_readiness_report(rows, environment=environment)
  297. def _promotion_digest_payload(bundle: dict[str, Any]) -> dict[str, Any]:
  298. return {
  299. "schema_version": bundle["schema_version"],
  300. "source_environment": bundle["source_environment"],
  301. "target_environment": bundle["target_environment"],
  302. "approval_ref": bundle["approval_ref"],
  303. "objects": bundle["objects"],
  304. }
  305. def create_promotion_bundle(request: dict[str, Any]) -> dict[str, Any]:
  306. source = str(request.get("source_environment", ""))
  307. target = str(request.get("target_environment", ""))
  308. if source not in ENVIRONMENT_ORDER or target not in ENVIRONMENT_ORDER:
  309. raise ValueError("known source and target environments are required")
  310. if ENVIRONMENT_ORDER[target] - ENVIRONMENT_ORDER[source] != 10:
  311. raise ValueError("promotion must target the adjacent environment")
  312. approval_ref = str(request.get("approval_ref", "")).strip()
  313. if not approval_ref:
  314. raise ValueError("approval_ref is required")
  315. objects = request.get("objects")
  316. if not isinstance(objects, list) or not objects:
  317. raise ValueError("at least one promotion object is required")
  318. normalized: list[dict[str, Any]] = []
  319. identities: set[tuple[str, str]] = set()
  320. for item in objects:
  321. if not isinstance(item, dict):
  322. raise ValueError("promotion object must be an object")
  323. kind = str(item.get("kind", ""))
  324. uid = str(item.get("uid", "")).strip()
  325. version = str(item.get("version", "")).strip()
  326. digest = str(item.get("payload_sha256", "")).lower()
  327. rollback_ref = str(item.get("rollback_ref", "")).strip()
  328. evidence_refs = item.get("evidence_refs")
  329. if kind not in PROMOTABLE_KINDS:
  330. raise ValueError(f"unsupported promotion kind: {kind}")
  331. if not uid or not version or not rollback_ref:
  332. raise ValueError("uid, version and rollback_ref are required")
  333. if not SHA256_PATTERN.fullmatch(digest):
  334. raise ValueError(f"invalid payload sha256 for {kind}:{uid}")
  335. if not isinstance(evidence_refs, list) or not evidence_refs:
  336. raise ValueError(f"evidence_refs are required for {kind}:{uid}")
  337. identity = (kind, uid)
  338. if identity in identities:
  339. raise ValueError(f"duplicate promotion object: {kind}:{uid}")
  340. identities.add(identity)
  341. normalized.append(
  342. {
  343. "kind": kind,
  344. "uid": uid,
  345. "version": version,
  346. "payload_sha256": digest,
  347. "rollback_ref": rollback_ref,
  348. "evidence_refs": sorted(str(value) for value in evidence_refs),
  349. }
  350. )
  351. normalized.sort(key=lambda item: (item["kind"], item["uid"]))
  352. bundle: dict[str, Any] = {
  353. "schema_version": 1,
  354. "status": "approved",
  355. "source_environment": source,
  356. "target_environment": target,
  357. "approval_ref": approval_ref,
  358. "objects": normalized,
  359. }
  360. bundle["bundle_sha256"] = _sha256_bytes(
  361. _canonical_bytes(_promotion_digest_payload(bundle))
  362. )
  363. return bundle
  364. def verify_promotion_bundle(bundle: dict[str, Any]) -> dict[str, Any]:
  365. expected = _sha256_bytes(_canonical_bytes(_promotion_digest_payload(bundle)))
  366. actual = str(bundle.get("bundle_sha256", ""))
  367. if actual != expected:
  368. raise ValueError(f"promotion bundle digest mismatch: expected={expected}, actual={actual}")
  369. return {
  370. "schema_version": 1,
  371. "status": "verified",
  372. "bundle_sha256": actual,
  373. "object_count": len(bundle.get("objects", [])),
  374. }
  375. def evaluate_customer_migration(request: dict[str, Any]) -> dict[str, Any]:
  376. reasons: list[str] = []
  377. mappings = request.get("mappings")
  378. if not isinstance(mappings, list):
  379. raise ValueError("mappings must be a list")
  380. incomplete = [
  381. str(item.get("source", "unknown"))
  382. for item in mappings
  383. if isinstance(item, dict)
  384. and item.get("required", True)
  385. and (not str(item.get("source", "")).strip() or not str(item.get("target", "")).strip())
  386. ]
  387. targets = [
  388. str(item.get("target", ""))
  389. for item in mappings
  390. if isinstance(item, dict) and str(item.get("target", "")).strip()
  391. ]
  392. duplicates = sorted({item for item in targets if targets.count(item) > 1})
  393. if incomplete or duplicates:
  394. reasons.append("mapping_incomplete")
  395. incremental = request.get("incremental") or {}
  396. latest = int(incremental.get("latest_sequence", 0))
  397. caught_up = int(incremental.get("caught_up_sequence", -1))
  398. lag = max(0, latest - caught_up)
  399. if lag:
  400. reasons.append("incremental_not_caught_up")
  401. reconciliation = request.get("reconciliation") or {}
  402. source_count = int(reconciliation.get("source_count", -1))
  403. target_count = int(reconciliation.get("target_count", -2))
  404. source_digest = str(reconciliation.get("source_digest", ""))
  405. target_digest = str(reconciliation.get("target_digest", ""))
  406. reconciled = (
  407. source_count >= 0
  408. and source_count == target_count
  409. and bool(source_digest)
  410. and source_digest == target_digest
  411. )
  412. if not reconciled:
  413. reasons.append("reconciliation_mismatch")
  414. rollback = request.get("rollback") or {}
  415. rollback_ready = bool(rollback.get("backup_ref") and rollback.get("release_ref"))
  416. if not rollback_ready:
  417. reasons.append("rollback_not_prepared")
  418. snapshot = request.get("source_snapshot") or {}
  419. if not snapshot.get("snapshot_ref"):
  420. reasons.append("source_snapshot_missing")
  421. return {
  422. "schema_version": 1,
  423. "migration_id": str(request.get("migration_id", "unknown")),
  424. "mode": "dry_run",
  425. "source_mutated": False,
  426. "status": "blocked" if reasons else "ready",
  427. "blocking_reasons": sorted(set(reasons)),
  428. "source_snapshot": _redact(snapshot),
  429. "mapping": {
  430. "status": "blocked" if incomplete or duplicates else "ready",
  431. "mapping_count": len(mappings),
  432. "incomplete_sources": incomplete,
  433. "duplicate_targets": duplicates,
  434. },
  435. "incremental_catchup": {
  436. "status": "caught_up" if lag == 0 else "lagging",
  437. "latest_sequence": latest,
  438. "caught_up_sequence": caught_up,
  439. "lag": lag,
  440. },
  441. "reconciliation": {
  442. "status": "matched" if reconciled else "mismatch",
  443. "source_count": source_count,
  444. "target_count": target_count,
  445. "source_digest": source_digest,
  446. "target_digest": target_digest,
  447. },
  448. "checkpoints": sorted(str(value) for value in request.get("checkpoints", [])),
  449. "rollback": {
  450. "status": "prepared" if rollback_ready else "blocked",
  451. "backup_ref": rollback.get("backup_ref"),
  452. "release_ref": rollback.get("release_ref"),
  453. "strategy": "restore_prior_release_and_pre_migration_backup",
  454. "database_downgrade": False,
  455. },
  456. }
  457. def _git_value(repo_root: Path, *arguments: str, default: str = "unknown") -> str:
  458. try:
  459. completed = subprocess.run(
  460. ["git", "-C", str(repo_root), *arguments],
  461. check=True,
  462. capture_output=True,
  463. text=True,
  464. )
  465. return completed.stdout.strip() or default
  466. except (OSError, subprocess.CalledProcessError):
  467. return default
  468. def _migration_head(repo_root: Path) -> str:
  469. revisions: dict[str, set[str]] = {}
  470. referenced: set[str] = set()
  471. for path in sorted((repo_root / "migrations/versions").glob("*.py")):
  472. tree = ast.parse(path.read_text(encoding="utf-8"))
  473. revision = ""
  474. parents: set[str] = set()
  475. for node in tree.body:
  476. if not isinstance(node, (ast.Assign, ast.AnnAssign)):
  477. continue
  478. target = node.targets[0] if isinstance(node, ast.Assign) else node.target
  479. if not isinstance(target, ast.Name):
  480. continue
  481. if target.id not in {"revision", "down_revision"}:
  482. continue
  483. value_node = node.value
  484. try:
  485. value = ast.literal_eval(value_node)
  486. except (ValueError, TypeError):
  487. continue
  488. if target.id == "revision":
  489. revision = str(value)
  490. elif value:
  491. if isinstance(value, (list, tuple)):
  492. parents.update(str(item) for item in value)
  493. else:
  494. parents.add(str(value))
  495. if revision:
  496. revisions[revision] = parents
  497. referenced.update(parents)
  498. heads = sorted(set(revisions) - referenced)
  499. if len(heads) != 1:
  500. raise ValueError(f"exactly one migration head required; found={heads}")
  501. return heads[0]
  502. def build_release_manifest(
  503. repo_root: Path, *, version: str, source_date_epoch: int
  504. ) -> dict[str, Any]:
  505. repo_root = repo_root.resolve()
  506. openapi = repo_root / "docs/architecture/OPENAPI.yaml"
  507. route_match = re.search(
  508. r"^x-route-count:\s*(\d+)\s*$",
  509. openapi.read_text(encoding="utf-8"),
  510. re.MULTILINE,
  511. )
  512. if not route_match:
  513. raise ValueError("OpenAPI x-route-count is missing")
  514. source_trees = {
  515. name: _tree_manifest(repo_root / name)
  516. for name in ("app", "database", "migrations")
  517. }
  518. release_copies: dict[str, dict[str, Any]] = {}
  519. for name, source in source_trees.items():
  520. release = _tree_manifest(repo_root / "deployment" / name)
  521. release_copies[name] = {
  522. **release,
  523. "matches_source": release["sha256"] == source["sha256"],
  524. }
  525. manifest = {
  526. "schema_version": 1,
  527. "version": version,
  528. "source_date_epoch": int(source_date_epoch),
  529. "source": {
  530. "commit": _git_value(repo_root, "rev-parse", "HEAD"),
  531. "tree": _git_value(repo_root, "rev-parse", "HEAD^{tree}"),
  532. },
  533. "source_trees": source_trees,
  534. "release_copies": release_copies,
  535. "migration": {
  536. "head": _migration_head(repo_root),
  537. "tree_sha256": source_trees["migrations"]["sha256"],
  538. },
  539. "openapi": {
  540. "route_count": int(route_match.group(1)),
  541. "sha256": _sha256_file(openapi),
  542. },
  543. "compose": {
  544. "sha256": _sha256_file(repo_root / "deploy/docker/docker-compose.yml")
  545. },
  546. "environment_profiles": _tree_manifest(repo_root / "deployment/environments"),
  547. "dependency_inputs": {
  548. "backend_sha256": _sha256_file(repo_root / "requirements.txt"),
  549. "frontend_sha256": _sha256_file(repo_root / "frontend/package-lock.json"),
  550. },
  551. }
  552. if not all(item["matches_source"] for item in release_copies.values()):
  553. raise ValueError("release copies do not match source trees")
  554. manifest["manifest_sha256"] = _sha256_bytes(_canonical_bytes(manifest))
  555. return manifest
  556. def _component(name: str, version: str, ecosystem: str) -> dict[str, str]:
  557. return {
  558. "type": "library",
  559. "name": name,
  560. "version": version,
  561. "purl": f"pkg:{ecosystem}/{name}@{version}",
  562. }
  563. def _sbom(name: str, components: list[dict[str, str]], input_digest: str) -> dict[str, Any]:
  564. components.sort(key=lambda item: (item["name"].lower(), item["version"]))
  565. return {
  566. "bomFormat": "CycloneDX",
  567. "specVersion": "1.5",
  568. "version": 1,
  569. "metadata": {
  570. "component": {"type": "application", "name": name},
  571. "properties": [
  572. {"name": "dataops:source-input-sha256", "value": input_digest}
  573. ],
  574. },
  575. "components": components,
  576. }
  577. def generate_backend_sbom(requirements: Path) -> dict[str, Any]:
  578. components: list[dict[str, str]] = []
  579. pattern = re.compile(r"^([A-Za-z0-9_.-]+)\s*(===|==|>=|~=|<=|>|<)\s*([^;\s]+)")
  580. for raw_line in requirements.read_text(encoding="utf-8").splitlines():
  581. line = raw_line.strip()
  582. if not line or line.startswith("#"):
  583. continue
  584. match = pattern.match(line)
  585. if not match:
  586. raise ValueError(f"unsupported requirement line: {line}")
  587. name, operator, version = match.groups()
  588. rendered_version = version if operator in {"==", "==="} else operator + version
  589. components.append(_component(name, rendered_version, "pypi"))
  590. return _sbom("dataops-platform-backend", components, _sha256_file(requirements))
  591. def generate_frontend_sbom(package_lock: Path) -> dict[str, Any]:
  592. lock = _read_json(package_lock)
  593. components: list[dict[str, str]] = []
  594. packages = lock.get("packages") or {}
  595. if not isinstance(packages, dict):
  596. raise ValueError("package-lock packages object is required")
  597. for package_path, value in packages.items():
  598. if not package_path or not isinstance(value, dict):
  599. continue
  600. name = str(value.get("name", ""))
  601. if not name and "node_modules/" in package_path:
  602. name = package_path.rsplit("node_modules/", 1)[-1]
  603. version = str(value.get("version", ""))
  604. if name and version:
  605. components.append(_component(name, version, "npm"))
  606. return _sbom("dataops-platform-frontend", components, _sha256_file(package_lock))
  607. def generate_sboms(repo_root: Path) -> dict[str, Any]:
  608. return {
  609. "backend": generate_backend_sbom(repo_root / "requirements.txt"),
  610. "frontend": generate_frontend_sbom(repo_root / "frontend/package-lock.json"),
  611. }
  612. def _parser() -> argparse.ArgumentParser:
  613. parser = argparse.ArgumentParser(description=__doc__)
  614. subparsers = parser.add_subparsers(dest="command", required=True)
  615. validate = subparsers.add_parser("validate-environments")
  616. validate.add_argument("--profiles-dir", type=Path, required=True)
  617. validate.add_argument("--output")
  618. readiness = subparsers.add_parser("readiness")
  619. readiness.add_argument("--input", type=Path, required=True)
  620. readiness.add_argument("--output")
  621. compose_readiness = subparsers.add_parser("compose-readiness")
  622. compose_readiness.add_argument("--compose-file", type=Path, required=True)
  623. compose_readiness.add_argument("--project-name", required=True)
  624. compose_readiness.add_argument("--environment", required=True)
  625. compose_readiness.add_argument("--env-file", type=Path)
  626. compose_readiness.add_argument("--output")
  627. promotion = subparsers.add_parser("promotion")
  628. promotion.add_argument("--input", type=Path, required=True)
  629. promotion.add_argument("--output")
  630. promotion.add_argument("--verify", action="store_true")
  631. migration = subparsers.add_parser("migration-dry-run")
  632. migration.add_argument("--input", type=Path, required=True)
  633. migration.add_argument("--output")
  634. manifest = subparsers.add_parser("manifest")
  635. manifest.add_argument("--repo-root", type=Path, required=True)
  636. manifest.add_argument("--version", required=True)
  637. manifest.add_argument("--source-date-epoch", type=int, required=True)
  638. manifest.add_argument("--output")
  639. sbom = subparsers.add_parser("sbom")
  640. sbom.add_argument("--repo-root", type=Path, required=True)
  641. sbom.add_argument("--output-dir", required=True)
  642. return parser
  643. def main(argv: list[str] | None = None) -> int:
  644. args = _parser().parse_args(argv)
  645. if args.command == "validate-environments":
  646. _write_json(validate_environment_profiles(args.profiles_dir), args.output)
  647. elif args.command == "readiness":
  648. _write_json(build_readiness_report(_read_json(args.input)), args.output)
  649. elif args.command == "compose-readiness":
  650. _write_json(
  651. collect_compose_readiness(
  652. args.compose_file,
  653. project_name=args.project_name,
  654. environment=args.environment,
  655. env_file=args.env_file,
  656. ),
  657. args.output,
  658. )
  659. elif args.command == "promotion":
  660. value = _read_json(args.input)
  661. report = verify_promotion_bundle(value) if args.verify else create_promotion_bundle(value)
  662. _write_json(report, args.output)
  663. elif args.command == "migration-dry-run":
  664. _write_json(evaluate_customer_migration(_read_json(args.input)), args.output)
  665. elif args.command == "manifest":
  666. _write_json(
  667. build_release_manifest(
  668. args.repo_root,
  669. version=args.version,
  670. source_date_epoch=args.source_date_epoch,
  671. ),
  672. args.output,
  673. )
  674. elif args.command == "sbom":
  675. sboms = generate_sboms(args.repo_root)
  676. if args.output_dir == "-":
  677. _write_json(sboms)
  678. else:
  679. output_dir = Path(args.output_dir)
  680. output_dir.mkdir(parents=True, exist_ok=True)
  681. _write_json(sboms["backend"], str(output_dir / "backend.cdx.json"))
  682. _write_json(sboms["frontend"], str(output_dir / "frontend.cdx.json"))
  683. return 0
  684. if __name__ == "__main__":
  685. try:
  686. raise SystemExit(main())
  687. except (OSError, ValueError, json.JSONDecodeError) as exc:
  688. print(f"ERROR: {exc}", file=sys.stderr)
  689. raise SystemExit(2)