expressions.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790
  1. """Closed, typed rule expressions for governed DataOps rules.
  2. The module deliberately owns both parsing and validation. It produces JSON
  3. only; expressions are never delegated to Python, SQL, JavaScript, or a model
  4. runtime. Compilers consume the resulting AST in a later release.
  5. """
  6. from __future__ import annotations
  7. import re
  8. from dataclasses import dataclass
  9. from datetime import date as Date
  10. from datetime import datetime
  11. from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
  12. from typing import Any
  13. from zoneinfo import ZoneInfo
  14. MAX_SOURCE_LENGTH = 20_000
  15. MAX_AST_DEPTH = 40
  16. MAX_AST_NODES = 500
  17. MAX_REGEX_LENGTH = 500
  18. ALLOWED_FUNCTIONS = frozenset(
  19. {
  20. "matches",
  21. "lower",
  22. "upper",
  23. "trim",
  24. "length",
  25. "coalesce",
  26. "date",
  27. "timestamp",
  28. "abs",
  29. "round",
  30. }
  31. )
  32. SUPPORTED_BACKENDS = frozenset({"postgresql", "mysql", "polars"})
  33. _FUNCTION_BACKENDS = {
  34. "matches": SUPPORTED_BACKENDS,
  35. # The reference oracle verifies Python/Polars Unicode case folding. SQL
  36. # engines require a deployment-pinned collation before they may opt in.
  37. "lower": frozenset({"polars"}),
  38. "upper": frozenset({"polars"}),
  39. "trim": SUPPORTED_BACKENDS,
  40. "length": SUPPORTED_BACKENDS,
  41. "coalesce": SUPPORTED_BACKENDS,
  42. "date": SUPPORTED_BACKENDS,
  43. # MySQL timezone conversion depends on server timezone tables; do not
  44. # advertise it before a deployment can prove that dependency.
  45. "timestamp": frozenset({"postgresql", "polars"}),
  46. "abs": SUPPORTED_BACKENDS,
  47. # Polars rounding mode cannot be assumed to match the V1 decimal policy.
  48. "round": frozenset({"postgresql", "mysql"}),
  49. }
  50. _IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$")
  51. _FIELD_TYPE_ALIASES = {
  52. "bigint": "integer",
  53. "bool": "boolean",
  54. "character varying": "string",
  55. "datetime": "timestamp",
  56. "int": "integer",
  57. "numeric": "decimal",
  58. "text": "string",
  59. "varchar": "string",
  60. }
  61. _FIELD_TYPES = {
  62. "binary",
  63. "boolean",
  64. "date",
  65. "decimal",
  66. "double",
  67. "float",
  68. "integer",
  69. "json",
  70. "string",
  71. "timestamp",
  72. "timestamptz",
  73. }
  74. _NUMERIC_TYPES = {"integer", "decimal", "float", "double"}
  75. _LITERAL_TYPES = {"boolean", "decimal", "integer", "null", "string"}
  76. _DECIMAL_LEXEME = re.compile(r"^[0-9]+\.[0-9]+$")
  77. _BINARY_OPERATORS = {
  78. "||",
  79. "&&",
  80. "==",
  81. "!=",
  82. "<",
  83. "<=",
  84. ">",
  85. ">=",
  86. "+",
  87. "-",
  88. "*",
  89. "/",
  90. "%",
  91. }
  92. _PRECEDENCE = {
  93. "||": 10,
  94. "&&": 20,
  95. "==": 30,
  96. "!=": 30,
  97. "<": 30,
  98. "<=": 30,
  99. ">": 30,
  100. ">=": 30,
  101. "+": 40,
  102. "-": 40,
  103. "*": 50,
  104. "/": 50,
  105. "%": 50,
  106. }
  107. @dataclass(frozen=True)
  108. class _Token:
  109. kind: str
  110. value: Any
  111. position: int
  112. def _error(position: int, message: str) -> ValueError:
  113. return ValueError(f"expression {message} at position {position}")
  114. class _Lexer:
  115. def __init__(self, source: str):
  116. self.source = source
  117. self.length = len(source)
  118. self.position = 0
  119. def tokens(self) -> list[_Token]:
  120. result: list[_Token] = []
  121. while self.position < self.length:
  122. char = self.source[self.position]
  123. if char.isspace():
  124. self.position += 1
  125. elif char in "'\"":
  126. result.append(self._string())
  127. elif char.isdigit():
  128. result.append(self._number())
  129. elif char.isascii() and (char.isalpha() or char == "_"):
  130. result.append(self._identifier())
  131. elif char in "(),":
  132. result.append(_Token(char, char, self.position))
  133. self.position += 1
  134. else:
  135. result.append(self._operator())
  136. result.append(_Token("EOF", None, self.position))
  137. return result
  138. def _string(self) -> _Token:
  139. start = self.position
  140. quote = self.source[self.position]
  141. self.position += 1
  142. pieces: list[str] = []
  143. escapes = {"n": "\n", "r": "\r", "t": "\t", "b": "\b", "f": "\f"}
  144. while self.position < self.length:
  145. char = self.source[self.position]
  146. self.position += 1
  147. if char == quote:
  148. value = "".join(pieces)
  149. if len(value) > 4_000:
  150. raise _error(start, "string literal exceeds 4000 characters")
  151. return _Token("LITERAL", ("string", value), start)
  152. if char != "\\":
  153. pieces.append(char)
  154. continue
  155. if self.position >= self.length:
  156. raise _error(start, "contains an unfinished string escape")
  157. escaped = self.source[self.position]
  158. self.position += 1
  159. if escaped in escapes:
  160. pieces.append(escapes[escaped])
  161. elif escaped in {"\\", "'", '\"'}:
  162. pieces.append(escaped)
  163. elif escaped == "u":
  164. encoded = self.source[self.position : self.position + 4]
  165. if len(encoded) != 4 or not re.fullmatch(r"[0-9a-fA-F]{4}", encoded):
  166. raise _error(start, "contains an invalid unicode escape")
  167. pieces.append(chr(int(encoded, 16)))
  168. self.position += 4
  169. else:
  170. raise _error(start, "contains an unsupported string escape")
  171. raise _error(start, "contains an unterminated string")
  172. def _number(self) -> _Token:
  173. start = self.position
  174. while self.position < self.length and self.source[self.position].isdigit():
  175. self.position += 1
  176. kind = "integer"
  177. if self.position < self.length and self.source[self.position] == ".":
  178. kind = "decimal"
  179. self.position += 1
  180. decimal_start = self.position
  181. while self.position < self.length and self.source[self.position].isdigit():
  182. self.position += 1
  183. if self.position == decimal_start:
  184. raise _error(start, "contains an invalid numeric literal")
  185. raw = self.source[start : self.position]
  186. # Decimal tokens keep their lexeme in the JSON AST. A float here
  187. # would destroy source precision before a compiler receives it.
  188. value: int | str = int(raw) if kind == "integer" else raw
  189. return _Token("LITERAL", (kind, value), start)
  190. def _identifier(self) -> _Token:
  191. start = self.position
  192. self.position += 1
  193. while self.position < self.length:
  194. char = self.source[self.position]
  195. if not (char.isascii() and (char.isalnum() or char == "_")):
  196. break
  197. self.position += 1
  198. value = self.source[start : self.position]
  199. if len(value) > 128:
  200. raise _error(start, "identifier exceeds 128 characters")
  201. lowered = value.lower()
  202. if lowered == "true":
  203. return _Token("LITERAL", ("boolean", True), start)
  204. if lowered == "false":
  205. return _Token("LITERAL", ("boolean", False), start)
  206. if lowered == "null":
  207. return _Token("LITERAL", ("null", None), start)
  208. return _Token("IDENT", value, start)
  209. def _operator(self) -> _Token:
  210. start = self.position
  211. for operator in ("&&", "||", "==", "!=", "<=", ">="):
  212. if self.source.startswith(operator, start):
  213. self.position += len(operator)
  214. return _Token("OP", operator, start)
  215. char = self.source[self.position]
  216. if char in "!<>+-*/%":
  217. self.position += 1
  218. return _Token("OP", char, start)
  219. raise _error(start, f"contains unexpected character {char!r}")
  220. class _Parser:
  221. def __init__(self, source: str):
  222. self.tokens = _Lexer(source).tokens()
  223. self.index = 0
  224. @property
  225. def current(self) -> _Token:
  226. return self.tokens[self.index]
  227. def consume(self, kind: str, value: str | None = None) -> _Token:
  228. token = self.current
  229. if token.kind != kind or (value is not None and token.value != value):
  230. expected = value if value is not None else kind
  231. raise _error(token.position, f"expected {expected!r}")
  232. self.index += 1
  233. return token
  234. def parse(self) -> dict[str, Any]:
  235. expression = self._expression(0)
  236. if self.current.kind != "EOF":
  237. raise _error(self.current.position, "contains unexpected trailing input")
  238. return expression
  239. def _expression(self, minimum_precedence: int) -> dict[str, Any]:
  240. token = self.current
  241. self.index += 1
  242. if token.kind == "LITERAL":
  243. literal_type, value = token.value
  244. left = {"kind": "literal", "type": literal_type, "value": value}
  245. elif token.kind == "IDENT":
  246. if self.current.kind == "(":
  247. if token.value not in ALLOWED_FUNCTIONS:
  248. raise _error(
  249. token.position,
  250. f"uses unsupported expression function {token.value!r}",
  251. )
  252. self.index += 1
  253. arguments: list[dict[str, Any]] = []
  254. if self.current.kind != ")":
  255. while True:
  256. arguments.append(self._expression(0))
  257. if self.current.kind != ",":
  258. break
  259. self.index += 1
  260. self.consume(")")
  261. left = {
  262. "kind": "call",
  263. "function": token.value,
  264. "arguments": arguments,
  265. }
  266. else:
  267. left = {"kind": "identifier", "name": token.value}
  268. elif token.kind == "OP" and token.value in {"!", "-"}:
  269. left = {
  270. "kind": "unary",
  271. "operator": token.value,
  272. "operand": self._expression(60),
  273. }
  274. elif token.kind == "(":
  275. left = self._expression(0)
  276. self.consume(")")
  277. else:
  278. raise _error(token.position, "contains an unexpected token")
  279. while self.current.kind == "OP":
  280. operator = self.current.value
  281. precedence = _PRECEDENCE.get(operator)
  282. if precedence is None or precedence < minimum_precedence:
  283. break
  284. self.index += 1
  285. left = {
  286. "kind": "binary",
  287. "operator": operator,
  288. "left": left,
  289. "right": self._expression(precedence + 1),
  290. }
  291. return left
  292. def parse_expression(source: str) -> dict:
  293. """Parse a V1 expression into the closed, JSON-serializable AST."""
  294. if not isinstance(source, str) or not source.strip():
  295. raise ValueError("expression source is required")
  296. if len(source) > MAX_SOURCE_LENGTH:
  297. raise ValueError(f"expression source exceeds {MAX_SOURCE_LENGTH} characters")
  298. ast = _Parser(source).parse()
  299. _validate_ast(ast)
  300. return ast
  301. def _validate_ast(ast: Any) -> None:
  302. count = 0
  303. def walk(node: Any, depth: int) -> None:
  304. nonlocal count
  305. if depth > MAX_AST_DEPTH:
  306. raise ValueError(f"expression AST depth exceeds {MAX_AST_DEPTH}")
  307. count += 1
  308. if count > MAX_AST_NODES:
  309. raise ValueError(f"expression AST node count exceeds {MAX_AST_NODES}")
  310. if not isinstance(node, dict):
  311. raise ValueError("expression AST node must be an object")
  312. kind = node.get("kind")
  313. if kind == "literal":
  314. if set(node) != {"kind", "type", "value"}:
  315. raise ValueError("literal AST node must have a closed shape")
  316. literal_type = node.get("type")
  317. value = node.get("value")
  318. if literal_type not in _LITERAL_TYPES:
  319. raise ValueError("literal AST node has an unsupported type")
  320. if literal_type == "null" and value is not None:
  321. raise ValueError("null literal must have a null value")
  322. if literal_type == "boolean" and not isinstance(value, bool):
  323. raise ValueError("boolean literal must have a boolean value")
  324. if literal_type == "string" and (
  325. not isinstance(value, str) or len(value) > 4_000
  326. ):
  327. raise ValueError("string literal is invalid")
  328. if literal_type == "integer" and (
  329. isinstance(value, bool) or not isinstance(value, int)
  330. ):
  331. raise ValueError("integer literal is invalid")
  332. if literal_type == "decimal" and (
  333. not isinstance(value, str)
  334. or _DECIMAL_LEXEME.fullmatch(value) is None
  335. ):
  336. raise ValueError("decimal literal is invalid")
  337. return
  338. if kind == "identifier":
  339. if set(node) != {"kind", "name"} or not isinstance(node.get("name"), str):
  340. raise ValueError("identifier AST node must have a closed shape")
  341. if _IDENTIFIER.fullmatch(node["name"]) is None:
  342. raise ValueError("identifier AST node has an invalid name")
  343. return
  344. if kind == "unary":
  345. if set(node) != {"kind", "operator", "operand"}:
  346. raise ValueError("unary AST node must have a closed shape")
  347. if node.get("operator") not in {"!", "-"}:
  348. raise ValueError("unary AST node has an unsupported operator")
  349. walk(node.get("operand"), depth + 1)
  350. return
  351. if kind == "binary":
  352. if set(node) != {"kind", "operator", "left", "right"}:
  353. raise ValueError("binary AST node must have a closed shape")
  354. if node.get("operator") not in _BINARY_OPERATORS:
  355. raise ValueError("binary AST node has an unsupported operator")
  356. walk(node.get("left"), depth + 1)
  357. walk(node.get("right"), depth + 1)
  358. return
  359. if kind == "call":
  360. if set(node) != {"kind", "function", "arguments"}:
  361. raise ValueError("call AST node must have a closed shape")
  362. function = node.get("function")
  363. arguments = node.get("arguments")
  364. if not isinstance(function, str) or _IDENTIFIER.fullmatch(function) is None:
  365. raise ValueError("call AST node has an invalid function")
  366. if function not in ALLOWED_FUNCTIONS:
  367. raise ValueError(f"unsupported expression function: {function}")
  368. if not isinstance(arguments, list) or len(arguments) > 100:
  369. raise ValueError("call AST arguments must be a bounded array")
  370. for argument in arguments:
  371. walk(argument, depth + 1)
  372. # This resource bound is independent of schema typing, so RuleSpec
  373. # normalization can reject an unsafe regex before compilation.
  374. if (
  375. function == "matches"
  376. and len(arguments) >= 2
  377. and arguments[1].get("kind") == "literal"
  378. and arguments[1].get("type") == "string"
  379. and len(arguments[1]["value"]) > MAX_REGEX_LENGTH
  380. ):
  381. raise ValueError(
  382. f"regex pattern exceeds {MAX_REGEX_LENGTH} characters"
  383. )
  384. return
  385. raise ValueError("expression AST node kind is unsupported")
  386. walk(ast, 1)
  387. def _normalized_fields(fields: Any) -> dict[str, str]:
  388. if not isinstance(fields, dict) or len(fields) > 1_000:
  389. raise ValueError("expression fields must be a bounded object")
  390. normalized: dict[str, str] = {}
  391. for name, field_type in fields.items():
  392. if not isinstance(name, str) or _IDENTIFIER.fullmatch(name) is None:
  393. raise ValueError("expression field names must be identifiers")
  394. if not isinstance(field_type, str):
  395. raise ValueError(f"expression field {name} type is invalid")
  396. result_type = _FIELD_TYPE_ALIASES.get(
  397. field_type.strip().lower(), field_type.strip().lower()
  398. )
  399. if result_type not in _FIELD_TYPES:
  400. raise ValueError(f"expression field {name} has an unsupported type")
  401. normalized[name] = result_type
  402. return normalized
  403. def _is_numeric(value_type: str) -> bool:
  404. return value_type in _NUMERIC_TYPES
  405. def _promote_numeric(left: str, right: str) -> str:
  406. order = {"integer": 0, "decimal": 1, "float": 2, "double": 3}
  407. return max((left, right), key=lambda value: order[value])
  408. def _compatible(left: str, right: str) -> bool:
  409. return left == "null" or right == "null" or left == right or (
  410. _is_numeric(left) and _is_numeric(right)
  411. )
  412. def _require_arguments(function: str, arguments: list[dict], expected: int | tuple[int, int]) -> None:
  413. if isinstance(expected, int):
  414. valid = len(arguments) == expected
  415. else:
  416. valid = expected[0] <= len(arguments) <= expected[1]
  417. if not valid:
  418. raise ValueError(f"function {function} received an unsupported argument count")
  419. def type_check_expression(ast: dict, fields: dict[str, str]) -> str:
  420. """Validate schema-bound identifiers and return the expression result type."""
  421. _validate_ast(ast)
  422. normalized_fields = _normalized_fields(fields)
  423. def check(node: dict) -> str:
  424. kind = node["kind"]
  425. if kind == "literal":
  426. return node["type"]
  427. if kind == "identifier":
  428. name = node["name"]
  429. if name not in normalized_fields:
  430. raise ValueError(f"unknown expression field: {name}")
  431. return normalized_fields[name]
  432. if kind == "unary":
  433. operand_type = check(node["operand"])
  434. operator = node["operator"]
  435. if operator == "!":
  436. if operand_type != "boolean":
  437. raise ValueError("logical not requires a boolean operand")
  438. return "boolean"
  439. if not _is_numeric(operand_type):
  440. raise ValueError("numeric negation requires a numeric operand")
  441. return operand_type
  442. if kind == "binary":
  443. left_type = check(node["left"])
  444. right_type = check(node["right"])
  445. operator = node["operator"]
  446. if operator in {"&&", "||"}:
  447. if left_type != "boolean" or right_type != "boolean":
  448. raise ValueError("logical operators require boolean operands")
  449. return "boolean"
  450. if operator in {"+", "-", "*", "/", "%"}:
  451. if not _is_numeric(left_type) or not _is_numeric(right_type):
  452. raise ValueError("arithmetic operators require numeric operands")
  453. return _promote_numeric(left_type, right_type)
  454. if not _compatible(left_type, right_type):
  455. raise ValueError(
  456. "comparison operands must have compatible types; use an explicit cast"
  457. )
  458. return "boolean"
  459. function = node["function"]
  460. arguments = node["arguments"]
  461. if function not in ALLOWED_FUNCTIONS:
  462. raise ValueError(f"unsupported expression function: {function}")
  463. argument_types = [check(argument) for argument in arguments]
  464. if function == "matches":
  465. _require_arguments(function, arguments, 2)
  466. if argument_types != ["string", "string"]:
  467. raise ValueError("matches requires string arguments")
  468. pattern = arguments[1]
  469. if pattern["kind"] != "literal" or pattern["type"] != "string":
  470. raise ValueError("matches pattern must be a string literal")
  471. if len(pattern["value"]) > MAX_REGEX_LENGTH:
  472. raise ValueError(f"regex pattern exceeds {MAX_REGEX_LENGTH} characters")
  473. try:
  474. re.compile(pattern["value"])
  475. except re.error as exc:
  476. raise ValueError("regex pattern is invalid") from exc
  477. return "boolean"
  478. if function in {"lower", "upper", "trim"}:
  479. _require_arguments(function, arguments, 1)
  480. if argument_types != ["string"]:
  481. raise ValueError(f"{function} requires a string argument")
  482. return "string"
  483. if function == "length":
  484. _require_arguments(function, arguments, 1)
  485. if argument_types != ["string"]:
  486. raise ValueError("length requires a string argument")
  487. return "integer"
  488. if function == "coalesce":
  489. _require_arguments(function, arguments, (2, 100))
  490. concrete = [value for value in argument_types if value != "null"]
  491. if not concrete:
  492. return "null"
  493. if not all(_compatible(concrete[0], value) for value in concrete[1:]):
  494. raise ValueError("coalesce arguments must have compatible types")
  495. if all(_is_numeric(value) for value in concrete):
  496. return max(concrete, key=lambda value: {"integer": 0, "decimal": 1, "float": 2, "double": 3}[value])
  497. return concrete[0]
  498. if function == "date":
  499. _require_arguments(function, arguments, 1)
  500. if argument_types[0] not in {"string", "date", "timestamp", "timestamptz"}:
  501. raise ValueError("date requires a string or temporal argument")
  502. return "date"
  503. if function == "timestamp":
  504. _require_arguments(function, arguments, 1)
  505. if argument_types[0] not in {"string", "date", "timestamp", "timestamptz"}:
  506. raise ValueError("timestamp requires a string or temporal argument")
  507. return "timestamp"
  508. if function == "abs":
  509. _require_arguments(function, arguments, 1)
  510. if not _is_numeric(argument_types[0]):
  511. raise ValueError("abs requires a numeric argument")
  512. return argument_types[0]
  513. _require_arguments(function, arguments, (1, 2))
  514. if not _is_numeric(argument_types[0]):
  515. raise ValueError("round requires a numeric value")
  516. if len(argument_types) == 2 and argument_types[1] != "integer":
  517. raise ValueError("round precision requires an integer")
  518. return argument_types[0]
  519. return check(ast)
  520. def validate_rule_expressions(
  521. rule_spec: dict[str, Any], fields: dict[str, str]
  522. ) -> frozenset[str]:
  523. """Type-check every canonical expression in a schema-bound RuleSpec.
  524. Callers must supply fields from a server-owned schema snapshot, never an
  525. authoring request. The return value is immutable-plan metadata rather
  526. than executable code and lets release reject unsupported backends early.
  527. """
  528. if not isinstance(rule_spec, dict) or not isinstance(
  529. rule_spec.get("steps"), list
  530. ):
  531. raise ValueError("rule spec steps are required for expression validation")
  532. supported_for_rule = SUPPORTED_BACKENDS
  533. for step in rule_spec["steps"]:
  534. if not isinstance(step, dict):
  535. raise ValueError("rule step must be an object")
  536. operation = step.get("op")
  537. ast = step.get("expression_ast")
  538. if operation in {"assert", "filter"} and ast is None:
  539. raise ValueError(f"{operation} expression is required")
  540. if ast is None:
  541. continue
  542. expression_type = type_check_expression(ast, fields)
  543. if operation in {"assert", "filter"} and expression_type != "boolean":
  544. raise ValueError(f"{operation} expression must return boolean")
  545. supported = backend_support(ast)
  546. if not supported:
  547. raise ValueError("expression has no supported execution backend")
  548. supported_for_rule = supported_for_rule & supported
  549. return frozenset(supported_for_rule)
  550. def _portable_regex(pattern: str) -> bool:
  551. """Keep only the portable regular-expression subset advertised by V1."""
  552. if len(pattern) > MAX_REGEX_LENGTH:
  553. return False
  554. if re.search(r"\(\?|\\[1-9]|\\[pP]", pattern):
  555. return False
  556. try:
  557. re.compile(pattern)
  558. except re.error:
  559. return False
  560. return True
  561. def backend_support(ast: dict) -> frozenset[str]:
  562. """Return backends that can preserve this grammar's V1 semantics.
  563. The result is intentionally a capability analysis, not a compiler. It
  564. validates the closed AST again so hand-authored JSON cannot bypass the
  565. function allowlist before a compiler sees it.
  566. """
  567. _validate_ast(ast)
  568. def support(node: dict) -> frozenset[str]:
  569. kind = node["kind"]
  570. if kind in {"literal", "identifier"}:
  571. return SUPPORTED_BACKENDS
  572. if kind == "unary":
  573. return support(node["operand"])
  574. if kind == "binary":
  575. supported = support(node["left"]) & support(node["right"])
  576. if node["operator"] == "/":
  577. return supported & frozenset({"polars"})
  578. return supported
  579. function = node["function"]
  580. supported = _FUNCTION_BACKENDS[function]
  581. for argument in node["arguments"]:
  582. supported = supported & support(argument)
  583. if function == "matches":
  584. arguments = node["arguments"]
  585. if (
  586. len(arguments) != 2
  587. or arguments[1]["kind"] != "literal"
  588. or arguments[1]["type"] != "string"
  589. or not _portable_regex(arguments[1]["value"])
  590. ):
  591. return frozenset()
  592. return frozenset(supported)
  593. return support(ast)
  594. def _decimal(value: Any) -> Decimal:
  595. if isinstance(value, Decimal):
  596. return value
  597. if isinstance(value, bool):
  598. raise ValueError("reference evaluator expected a numeric value")
  599. try:
  600. if isinstance(value, float):
  601. return Decimal(str(value))
  602. return Decimal(value)
  603. except (InvalidOperation, TypeError, ValueError) as exc:
  604. raise ValueError("reference evaluator expected a numeric value") from exc
  605. def _timestamp(value: Any, timezone: ZoneInfo) -> datetime:
  606. if isinstance(value, datetime):
  607. result = value
  608. elif isinstance(value, Date):
  609. result = datetime.combine(value, datetime.min.time())
  610. elif isinstance(value, str):
  611. try:
  612. result = datetime.fromisoformat(value.replace("Z", "+00:00"))
  613. except ValueError as exc:
  614. raise ValueError("reference evaluator received an invalid timestamp") from exc
  615. else:
  616. raise ValueError("reference evaluator expected a temporal value")
  617. if result.tzinfo is None:
  618. result = result.replace(tzinfo=timezone)
  619. return result
  620. def _sql_and(left: Any, right: Any) -> bool | None:
  621. if left is False or right is False:
  622. return False
  623. if left is None or right is None:
  624. return None
  625. return bool(left and right)
  626. def _sql_or(left: Any, right: Any) -> bool | None:
  627. if left is True or right is True:
  628. return True
  629. if left is None or right is None:
  630. return None
  631. return bool(left or right)
  632. def evaluate_expression(
  633. ast: dict, row: dict[str, Any], *, timezone: str = "UTC"
  634. ) -> Any:
  635. """Execute only the closed AST as a deterministic semantic test oracle.
  636. This is intentionally not a production runner. It has no access to a
  637. filesystem, network, Python callables, SQL, or model-generated code.
  638. """
  639. _validate_ast(ast)
  640. if not isinstance(row, dict):
  641. raise ValueError("reference evaluator row must be an object")
  642. try:
  643. timezone_info = ZoneInfo(timezone)
  644. except Exception as exc:
  645. raise ValueError("reference evaluator timezone is invalid") from exc
  646. def evaluate(node: dict) -> Any:
  647. kind = node["kind"]
  648. if kind == "literal":
  649. if node["type"] == "decimal":
  650. return Decimal(node["value"])
  651. return node["value"]
  652. if kind == "identifier":
  653. if node["name"] not in row:
  654. raise ValueError(f"reference evaluator missing field: {node['name']}")
  655. return row[node["name"]]
  656. if kind == "unary":
  657. value = evaluate(node["operand"])
  658. if value is None:
  659. return None
  660. return (not value) if node["operator"] == "!" else -_decimal(value)
  661. if kind == "binary":
  662. left = evaluate(node["left"])
  663. right = evaluate(node["right"])
  664. operator = node["operator"]
  665. if operator == "&&":
  666. return _sql_and(left, right)
  667. if operator == "||":
  668. return _sql_or(left, right)
  669. if left is None or right is None:
  670. return None
  671. if operator in {"+", "-", "*", "/", "%"}:
  672. left_decimal = _decimal(left)
  673. right_decimal = _decimal(right)
  674. return {
  675. "+": left_decimal + right_decimal,
  676. "-": left_decimal - right_decimal,
  677. "*": left_decimal * right_decimal,
  678. "/": left_decimal / right_decimal,
  679. "%": left_decimal % right_decimal,
  680. }[operator]
  681. if operator == "==":
  682. return left == right
  683. if operator == "!=":
  684. return left != right
  685. if operator == "<":
  686. return left < right
  687. if operator == "<=":
  688. return left <= right
  689. if operator == ">":
  690. return left > right
  691. return left >= right
  692. function = node["function"]
  693. arguments = [evaluate(argument) for argument in node["arguments"]]
  694. if function == "coalesce":
  695. return next((value for value in arguments if value is not None), None)
  696. if any(value is None for value in arguments):
  697. return None
  698. if function == "matches":
  699. return re.search(arguments[1], arguments[0]) is not None
  700. if function == "lower":
  701. return arguments[0].lower()
  702. if function == "upper":
  703. return arguments[0].upper()
  704. if function == "trim":
  705. return arguments[0].strip()
  706. if function == "length":
  707. return len(arguments[0])
  708. if function == "date":
  709. return _timestamp(arguments[0], timezone_info).astimezone(timezone_info).date()
  710. if function == "timestamp":
  711. return _timestamp(arguments[0], timezone_info)
  712. if function == "abs":
  713. return abs(_decimal(arguments[0]))
  714. precision = int(arguments[1]) if len(arguments) == 2 else 0
  715. return _decimal(arguments[0]).quantize(
  716. Decimal(1).scaleb(-precision), rounding=ROUND_HALF_UP
  717. )
  718. return evaluate(ast)