|
@@ -0,0 +1,829 @@
|
|
|
|
|
+"""SQLGlot compiler for one credential-free, deployment-bound SQL rule plan."""
|
|
|
|
|
+
|
|
|
|
|
+from __future__ import annotations
|
|
|
|
|
+
|
|
|
|
|
+import copy
|
|
|
|
|
+import hashlib
|
|
|
|
|
+import json
|
|
|
|
|
+import re
|
|
|
|
|
+from decimal import Decimal
|
|
|
|
|
+from typing import Any
|
|
|
|
|
+
|
|
|
|
|
+import sqlglot
|
|
|
|
|
+from sqlglot import exp
|
|
|
|
|
+
|
|
|
|
|
+from app.core.common.identifiers import ensure_governance_uid
|
|
|
|
|
+from app.core.data_rules.compilers.base import RuleCompiler
|
|
|
|
|
+from app.core.data_rules.contracts import read_rule_spec, rule_spec_hash
|
|
|
|
|
+from app.core.data_rules.execution_contracts import (
|
|
|
|
|
+ validate_dataset_binding,
|
|
|
|
|
+ validate_schema_snapshot,
|
|
|
|
|
+)
|
|
|
|
|
+from app.core.data_rules.expressions import (
|
|
|
|
|
+ backend_support,
|
|
|
|
|
+ type_check_expression,
|
|
|
|
|
+ validate_rule_expressions,
|
|
|
|
|
+)
|
|
|
|
|
+
|
|
|
|
|
+COMPILER_VERSION = "dataops-sqlglot-30.13.0"
|
|
|
|
|
+PLAN_SCHEMA_VERSION = "1.0"
|
|
|
|
|
+SUPPORTED_DIALECTS = {"postgresql", "mysql"}
|
|
|
|
|
+SUPPORTED_OPERATIONS = {
|
|
|
|
|
+ "assert",
|
|
|
|
|
+ "cast",
|
|
|
|
|
+ "deduplicate",
|
|
|
|
|
+ "derive",
|
|
|
|
|
+ "fill_null",
|
|
|
|
|
+ "filter",
|
|
|
|
|
+ "map_values",
|
|
|
|
|
+ "normalize_text",
|
|
|
|
|
+ "regex_replace",
|
|
|
|
|
+}
|
|
|
|
|
+CAPABILITY_KEYS = {
|
|
|
|
|
+ "dialect",
|
|
|
|
|
+ "timezone",
|
|
|
|
|
+ "collation",
|
|
|
|
|
+ "rounding_mode",
|
|
|
|
|
+ "regex_engine",
|
|
|
|
|
+}
|
|
|
|
|
+PLAN_KEYS = {
|
|
|
|
|
+ "schema_version",
|
|
|
|
|
+ "dialect",
|
|
|
|
|
+ "capabilities",
|
|
|
|
|
+ "data_source_uid",
|
|
|
|
|
+ "rule_version_id",
|
|
|
|
|
+ "input_binding_id",
|
|
|
|
|
+ "output_binding_id",
|
|
|
|
|
+ "statements",
|
|
|
|
|
+ "result_contract",
|
|
|
|
|
+}
|
|
|
|
|
+RESULT_CONTRACT = {
|
|
|
|
|
+ "rows_in": "counted",
|
|
|
|
|
+ "rows_out": "counted",
|
|
|
|
|
+ "rows_rejected": "counted",
|
|
|
|
|
+}
|
|
|
|
|
+_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$")
|
|
|
|
|
+_PORTABLE_REGEX_FORBIDDEN = re.compile(r"\(\?|\\[1-9]|\\[pP]")
|
|
|
|
|
+_NUMERIC_TYPES = {"integer", "decimal", "float", "double"}
|
|
|
|
|
+_TYPE_NAMES = {
|
|
|
|
|
+ "binary": "BINARY",
|
|
|
|
|
+ "boolean": "BOOLEAN",
|
|
|
|
|
+ "date": "DATE",
|
|
|
|
|
+ "decimal": "DECIMAL",
|
|
|
|
|
+ "double": "DOUBLE",
|
|
|
|
|
+ "float": "FLOAT",
|
|
|
|
|
+ "integer": "BIGINT",
|
|
|
|
|
+ "json": "JSON",
|
|
|
|
|
+ "string": "TEXT",
|
|
|
|
|
+ "timestamp": "TIMESTAMP",
|
|
|
|
|
+ "timestamptz": "TIMESTAMPTZ",
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _types_compatible(source_type: str, target_type: str) -> bool:
|
|
|
|
|
+ return source_type == target_type or {
|
|
|
|
|
+ source_type,
|
|
|
|
|
+ target_type,
|
|
|
|
|
+ } <= _NUMERIC_TYPES
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _value_matches_type(value: Any, field_type: str) -> bool:
|
|
|
|
|
+ if value is None:
|
|
|
|
|
+ return False
|
|
|
|
|
+ if field_type == "string":
|
|
|
|
|
+ return isinstance(value, str)
|
|
|
|
|
+ if field_type == "boolean":
|
|
|
|
|
+ return isinstance(value, bool)
|
|
|
|
|
+ if field_type == "integer":
|
|
|
|
|
+ return isinstance(value, int) and not isinstance(value, bool)
|
|
|
|
|
+ if field_type in _NUMERIC_TYPES:
|
|
|
|
|
+ if isinstance(value, bool):
|
|
|
|
|
+ return False
|
|
|
|
|
+ if isinstance(value, (int, float)):
|
|
|
|
|
+ return True
|
|
|
|
|
+ if isinstance(value, str):
|
|
|
|
|
+ try:
|
|
|
|
|
+ Decimal(value)
|
|
|
|
|
+ return True
|
|
|
|
|
+ except Exception:
|
|
|
|
|
+ return False
|
|
|
|
|
+ if field_type in {"date", "timestamp", "timestamptz"}:
|
|
|
|
|
+ return isinstance(value, str)
|
|
|
|
|
+ return False
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _canonical_json(value: Any) -> str:
|
|
|
|
|
+ try:
|
|
|
|
|
+ return json.dumps(
|
|
|
|
|
+ value,
|
|
|
|
|
+ sort_keys=True,
|
|
|
|
|
+ separators=(",", ":"),
|
|
|
|
|
+ ensure_ascii=False,
|
|
|
|
|
+ )
|
|
|
|
|
+ except (TypeError, ValueError) as exc:
|
|
|
|
|
+ raise ValueError("bound SQL plan must be JSON serializable") from exc
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _uid(value: Any, label: str) -> str:
|
|
|
|
|
+ try:
|
|
|
|
|
+ return ensure_governance_uid({"uid": str(value)})
|
|
|
|
|
+ except ValueError as exc:
|
|
|
|
|
+ raise ValueError(f"{label} must be a valid UUIDv7") from exc
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _dialect(value: Any) -> str:
|
|
|
|
|
+ normalized = str(value or "").strip().lower()
|
|
|
|
|
+ return "postgresql" if normalized == "postgres" else normalized
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _identifier(value: Any, label: str) -> str:
|
|
|
|
|
+ normalized = str(value or "")
|
|
|
|
|
+ if _IDENTIFIER.fullmatch(normalized) is None:
|
|
|
|
|
+ raise ValueError(f"{label} must be a SQL identifier")
|
|
|
|
|
+ return normalized
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _quoted_identifier(value: str) -> exp.Identifier:
|
|
|
|
|
+ return exp.Identifier(this=value, quoted=True)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _column(value: str) -> exp.Column:
|
|
|
|
|
+ return exp.Column(this=_quoted_identifier(value))
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _table(object_ref: Any) -> exp.Table:
|
|
|
|
|
+ parts = str(object_ref or "").split(".")
|
|
|
|
|
+ if len(parts) != 2:
|
|
|
|
|
+ raise ValueError("table object_ref must be schema.name")
|
|
|
|
|
+ schema_name = _identifier(parts[0], "table schema")
|
|
|
|
|
+ table_name = _identifier(parts[1], "table name")
|
|
|
|
|
+ return exp.Table(
|
|
|
|
|
+ this=_quoted_identifier(table_name),
|
|
|
|
|
+ db=_quoted_identifier(schema_name),
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _snapshot(value: Any, label: str) -> dict[str, Any]:
|
|
|
|
|
+ if not isinstance(value, dict):
|
|
|
|
|
+ raise ValueError(f"{label} must be an object")
|
|
|
|
|
+ snapshot_id = _uid(value.get("id"), f"{label} id")
|
|
|
|
|
+ normalized = validate_schema_snapshot(
|
|
|
|
|
+ {key: item for key, item in value.items() if key != "id"}
|
|
|
|
|
+ )
|
|
|
|
|
+ return {"id": snapshot_id, **normalized}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _binding(value: Any, label: str) -> dict[str, Any]:
|
|
|
|
|
+ if not isinstance(value, dict):
|
|
|
|
|
+ raise ValueError(f"{label} must be an object")
|
|
|
|
|
+ binding_id = _uid(value.get("id"), f"{label} id")
|
|
|
|
|
+ normalized = validate_dataset_binding(
|
|
|
|
|
+ {key: item for key, item in value.items() if key != "id"}
|
|
|
|
|
+ )
|
|
|
|
|
+ normalized["dialect"] = _dialect(normalized["dialect"])
|
|
|
|
|
+ return {"id": binding_id, **normalized}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _capabilities(
|
|
|
|
|
+ value: Any, *, dialect: str, timezone: str
|
|
|
|
|
+) -> dict[str, str]:
|
|
|
|
|
+ if not isinstance(value, dict) or set(value) != CAPABILITY_KEYS:
|
|
|
|
|
+ raise ValueError("SQL backend capabilities must have a closed shape")
|
|
|
|
|
+ capabilities = {
|
|
|
|
|
+ key: str(item).strip() for key, item in copy.deepcopy(value).items()
|
|
|
|
|
+ }
|
|
|
|
|
+ capabilities["dialect"] = _dialect(capabilities["dialect"])
|
|
|
|
|
+ if capabilities["dialect"] != dialect:
|
|
|
|
|
+ raise ValueError("SQL backend capability dialect does not match binding")
|
|
|
|
|
+ if capabilities["timezone"] != timezone:
|
|
|
|
|
+ raise ValueError("SQL backend timezone does not preserve rule semantics")
|
|
|
|
|
+ if not capabilities["collation"]:
|
|
|
|
|
+ raise ValueError("SQL backend collation capability is required")
|
|
|
|
|
+ if capabilities["rounding_mode"] != "half_away_from_zero":
|
|
|
|
|
+ raise ValueError("SQL backend rounding mode is unsupported")
|
|
|
|
|
+ expected_regex = "posix" if dialect == "postgresql" else "icu"
|
|
|
|
|
+ if capabilities["regex_engine"] != expected_regex:
|
|
|
|
|
+ raise ValueError("SQL backend regex engine is unsupported")
|
|
|
|
|
+ return capabilities
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class _ParameterStore:
|
|
|
|
|
+ def __init__(self):
|
|
|
|
|
+ self.values: dict[str, Any] = {}
|
|
|
|
|
+
|
|
|
|
|
+ def add(self, value: Any, *, value_type: str | None = None) -> exp.Expression:
|
|
|
|
|
+ name = f"rule_p_{len(self.values) + 1}"
|
|
|
|
|
+ if isinstance(value, Decimal):
|
|
|
|
|
+ value = str(value)
|
|
|
|
|
+ if not isinstance(value, (type(None), bool, int, float, str)):
|
|
|
|
|
+ raise ValueError("SQL parameter values must be JSON scalars")
|
|
|
|
|
+ self.values[name] = value
|
|
|
|
|
+ placeholder: exp.Expression = exp.Placeholder(this=name)
|
|
|
|
|
+ if value_type == "decimal":
|
|
|
|
|
+ placeholder = exp.Cast(
|
|
|
|
|
+ this=placeholder,
|
|
|
|
|
+ to=exp.DataType.build("DECIMAL"),
|
|
|
|
|
+ )
|
|
|
|
|
+ return placeholder
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class _ExpressionCompiler:
|
|
|
|
|
+ def __init__(
|
|
|
|
|
+ self,
|
|
|
|
|
+ *,
|
|
|
|
|
+ dialect: str,
|
|
|
|
|
+ fields: dict[str, str],
|
|
|
|
|
+ parameters: _ParameterStore,
|
|
|
|
|
+ ):
|
|
|
|
|
+ self.dialect = dialect
|
|
|
|
|
+ self.fields = fields
|
|
|
|
|
+ self.parameters = parameters
|
|
|
|
|
+
|
|
|
|
|
+ def compile(self, ast: dict[str, Any]) -> exp.Expression:
|
|
|
|
|
+ type_check_expression(ast, self.fields)
|
|
|
|
|
+ if self.dialect not in backend_support(ast):
|
|
|
|
|
+ raise ValueError("expression is unsupported by the bound SQL dialect")
|
|
|
|
|
+ return self._compile(ast)
|
|
|
|
|
+
|
|
|
|
|
+ def _compile(self, node: dict[str, Any]) -> exp.Expression:
|
|
|
|
|
+ kind = node["kind"]
|
|
|
|
|
+ if kind == "identifier":
|
|
|
|
|
+ return _column(_identifier(node["name"], "expression field"))
|
|
|
|
|
+ if kind == "literal":
|
|
|
|
|
+ literal_type = node["type"]
|
|
|
|
|
+ if literal_type == "null":
|
|
|
|
|
+ return exp.Null()
|
|
|
|
|
+ return self.parameters.add(
|
|
|
|
|
+ node["value"],
|
|
|
|
|
+ value_type=literal_type,
|
|
|
|
|
+ )
|
|
|
|
|
+ if kind == "unary":
|
|
|
|
|
+ operand = self._compile(node["operand"])
|
|
|
|
|
+ return (
|
|
|
|
|
+ exp.Not(this=operand)
|
|
|
|
|
+ if node["operator"] == "!"
|
|
|
|
|
+ else exp.Neg(this=operand)
|
|
|
|
|
+ )
|
|
|
|
|
+ if kind == "binary":
|
|
|
|
|
+ left_node = node["left"]
|
|
|
|
|
+ right_node = node["right"]
|
|
|
|
|
+ operator = node["operator"]
|
|
|
|
|
+ left = self._compile(left_node)
|
|
|
|
|
+ right = self._compile(right_node)
|
|
|
|
|
+ left_null = left_node.get("kind") == "literal" and left_node.get(
|
|
|
|
|
+ "type"
|
|
|
|
|
+ ) == "null"
|
|
|
|
|
+ right_null = right_node.get("kind") == "literal" and right_node.get(
|
|
|
|
|
+ "type"
|
|
|
|
|
+ ) == "null"
|
|
|
|
|
+ if operator in {"==", "!="} and (left_null or right_null):
|
|
|
|
|
+ concrete = right if left_null else left
|
|
|
|
|
+ predicate = exp.Is(this=concrete, expression=exp.Null())
|
|
|
|
|
+ return exp.Not(this=predicate) if operator == "!=" else predicate
|
|
|
|
|
+ operators = {
|
|
|
|
|
+ "||": exp.Or,
|
|
|
|
|
+ "&&": exp.And,
|
|
|
|
|
+ "==": exp.EQ,
|
|
|
|
|
+ "!=": exp.NEQ,
|
|
|
|
|
+ "<": exp.LT,
|
|
|
|
|
+ "<=": exp.LTE,
|
|
|
|
|
+ ">": exp.GT,
|
|
|
|
|
+ ">=": exp.GTE,
|
|
|
|
|
+ "+": exp.Add,
|
|
|
|
|
+ "-": exp.Sub,
|
|
|
|
|
+ "*": exp.Mul,
|
|
|
|
|
+ "/": exp.Div,
|
|
|
|
|
+ "%": exp.Mod,
|
|
|
|
|
+ }
|
|
|
|
|
+ return operators[operator](this=left, expression=right)
|
|
|
|
|
+ function = node["function"]
|
|
|
|
|
+ arguments = [self._compile(item) for item in node["arguments"]]
|
|
|
|
|
+ if function == "matches":
|
|
|
|
|
+ return exp.RegexpLike(this=arguments[0], expression=arguments[1])
|
|
|
|
|
+ if function == "trim":
|
|
|
|
|
+ return exp.Trim(this=arguments[0])
|
|
|
|
|
+ if function == "length":
|
|
|
|
|
+ return exp.Length(this=arguments[0])
|
|
|
|
|
+ if function == "coalesce":
|
|
|
|
|
+ return exp.Coalesce(this=arguments[0], expressions=arguments[1:])
|
|
|
|
|
+ if function == "abs":
|
|
|
|
|
+ return exp.Abs(this=arguments[0])
|
|
|
|
|
+ if function == "round":
|
|
|
|
|
+ return exp.Round(
|
|
|
|
|
+ this=arguments[0],
|
|
|
|
|
+ decimals=arguments[1] if len(arguments) == 2 else None,
|
|
|
|
|
+ )
|
|
|
|
|
+ if function == "date":
|
|
|
|
|
+ return exp.Cast(
|
|
|
|
|
+ this=arguments[0],
|
|
|
|
|
+ to=exp.DataType.build("DATE"),
|
|
|
|
|
+ )
|
|
|
|
|
+ if function == "timestamp" and self.dialect == "postgresql":
|
|
|
|
|
+ return exp.Cast(
|
|
|
|
|
+ this=arguments[0],
|
|
|
|
|
+ to=exp.DataType.build("TIMESTAMPTZ"),
|
|
|
|
|
+ )
|
|
|
|
|
+ raise ValueError(f"expression function {function} is not SQL executable")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _source_relation(
|
|
|
|
|
+ query: exp.Select, alias_number: int
|
|
|
|
|
+) -> exp.Subquery:
|
|
|
|
|
+ return exp.Subquery(
|
|
|
|
|
+ this=query,
|
|
|
|
|
+ alias=exp.TableAlias(
|
|
|
|
|
+ this=exp.Identifier(this=f"_dataops_s{alias_number}")
|
|
|
|
|
+ ),
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _project(
|
|
|
|
|
+ source: exp.Expression,
|
|
|
|
|
+ fields: list[str],
|
|
|
|
|
+ *,
|
|
|
|
|
+ replacements: dict[str, exp.Expression] | None = None,
|
|
|
|
|
+ where: exp.Expression | None = None,
|
|
|
|
|
+) -> exp.Select:
|
|
|
|
|
+ replacements = replacements or {}
|
|
|
|
|
+ projections = [
|
|
|
|
|
+ (replacements.get(name) or _column(name)).as_(
|
|
|
|
|
+ _quoted_identifier(name)
|
|
|
|
|
+ )
|
|
|
|
|
+ for name in fields
|
|
|
|
|
+ ]
|
|
|
|
|
+ query = exp.Select(expressions=projections).from_(source)
|
|
|
|
|
+ if where is not None:
|
|
|
|
|
+ query = query.where(where)
|
|
|
|
|
+ return query
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _portable_regex(pattern: Any) -> str:
|
|
|
|
|
+ if not isinstance(pattern, str) or len(pattern) > 500:
|
|
|
|
|
+ raise ValueError("regex pattern is invalid")
|
|
|
|
|
+ if _PORTABLE_REGEX_FORBIDDEN.search(pattern):
|
|
|
|
|
+ raise ValueError("regex pattern is not portable")
|
|
|
|
|
+ try:
|
|
|
|
|
+ re.compile(pattern)
|
|
|
|
|
+ except re.error as exc:
|
|
|
|
|
+ raise ValueError("regex pattern is invalid") from exc
|
|
|
|
|
+ return pattern
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _render(expression: exp.Expression, dialect: str) -> str:
|
|
|
|
|
+ sql_dialect = "postgres" if dialect == "postgresql" else dialect
|
|
|
|
|
+ statement = expression.sql(dialect=sql_dialect)
|
|
|
|
|
+ # SQLGlot renders PostgreSQL placeholders using DBAPI pyformat. The
|
|
|
|
|
+ # Runner stores SQLAlchemy named binds, so normalize only compiler-owned
|
|
|
|
|
+ # placeholder tokens after AST generation.
|
|
|
|
|
+ if dialect == "postgresql":
|
|
|
|
|
+ statement = re.sub(
|
|
|
|
|
+ r"%\(([A-Za-z_][A-Za-z0-9_]*)\)s",
|
|
|
|
|
+ r":\1",
|
|
|
|
|
+ statement,
|
|
|
|
|
+ )
|
|
|
|
|
+ return statement
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def validate_bound_sql_plan(value: Any) -> dict[str, Any]:
|
|
|
|
|
+ """Validate the exact executable plan shape and its SQLGlot AST."""
|
|
|
|
|
+
|
|
|
|
|
+ if not isinstance(value, dict) or set(value) != PLAN_KEYS:
|
|
|
|
|
+ raise ValueError("bound SQL plan must have a closed shape")
|
|
|
|
|
+ plan = copy.deepcopy(value)
|
|
|
|
|
+ if plan["schema_version"] != PLAN_SCHEMA_VERSION:
|
|
|
|
|
+ raise ValueError("unsupported bound SQL plan schema version")
|
|
|
|
|
+ dialect = _dialect(plan["dialect"])
|
|
|
|
|
+ if dialect not in SUPPORTED_DIALECTS:
|
|
|
|
|
+ raise ValueError("unsupported bound SQL plan dialect")
|
|
|
|
|
+ plan["dialect"] = dialect
|
|
|
|
|
+ plan["capabilities"] = _capabilities(
|
|
|
|
|
+ plan["capabilities"],
|
|
|
|
|
+ dialect=dialect,
|
|
|
|
|
+ timezone=str(plan["capabilities"].get("timezone") or ""),
|
|
|
|
|
+ )
|
|
|
|
|
+ for key in (
|
|
|
|
|
+ "data_source_uid",
|
|
|
|
|
+ "rule_version_id",
|
|
|
|
|
+ "input_binding_id",
|
|
|
|
|
+ "output_binding_id",
|
|
|
|
|
+ ):
|
|
|
|
|
+ plan[key] = _uid(plan[key], key)
|
|
|
|
|
+ statements = plan["statements"]
|
|
|
|
|
+ if not isinstance(statements, list) or len(statements) != 1:
|
|
|
|
|
+ raise ValueError("bound SQL plan must contain one statement")
|
|
|
|
|
+ statement = statements[0]
|
|
|
|
|
+ if not isinstance(statement, dict) or set(statement) != {
|
|
|
|
|
+ "purpose",
|
|
|
|
|
+ "sql",
|
|
|
|
|
+ "parameters",
|
|
|
|
|
+ }:
|
|
|
|
|
+ raise ValueError("bound SQL statement must have a closed shape")
|
|
|
|
|
+ if statement["purpose"] != "write":
|
|
|
|
|
+ raise ValueError("bound SQL plan must contain a write statement")
|
|
|
|
|
+ sql = statement["sql"]
|
|
|
|
|
+ if (
|
|
|
|
|
+ not isinstance(sql, str)
|
|
|
|
|
+ or not sql.strip()
|
|
|
|
|
+ or ";" in sql
|
|
|
|
|
+ or len(sql) > 100_000
|
|
|
|
|
+ ):
|
|
|
|
|
+ raise ValueError("bound SQL statement must be one bounded statement")
|
|
|
|
|
+ sql_dialect = "postgres" if dialect == "postgresql" else dialect
|
|
|
|
|
+ try:
|
|
|
|
|
+ expressions = sqlglot.parse(sql, read=sql_dialect)
|
|
|
|
|
+ except sqlglot.errors.ParseError as exc:
|
|
|
|
|
+ raise ValueError("bound SQL statement is invalid") from exc
|
|
|
|
|
+ if len(expressions) != 1 or not isinstance(expressions[0], exp.Insert):
|
|
|
|
|
+ raise ValueError("bound SQL statement must be one INSERT")
|
|
|
|
|
+ parsed = expressions[0]
|
|
|
|
|
+ if any(
|
|
|
|
|
+ isinstance(node, (exp.Command, exp.Delete, exp.Update))
|
|
|
|
|
+ for node in parsed.walk()
|
|
|
|
|
+ ):
|
|
|
|
|
+ raise ValueError("bound SQL statement contains unsupported operations")
|
|
|
|
|
+ source_tables = list(parsed.expression.find_all(exp.Table))
|
|
|
|
|
+ if len(source_tables) != 1:
|
|
|
|
|
+ raise ValueError("bound SQL plan must read exactly one source table")
|
|
|
|
|
+ parameters = statement["parameters"]
|
|
|
|
|
+ if not isinstance(parameters, dict) or len(parameters) > 500:
|
|
|
|
|
+ raise ValueError("bound SQL parameters must be a bounded object")
|
|
|
|
|
+ for name, item in parameters.items():
|
|
|
|
|
+ if _IDENTIFIER.fullmatch(str(name)) is None or not isinstance(
|
|
|
|
|
+ item, (type(None), bool, int, float, str)
|
|
|
|
|
+ ):
|
|
|
|
|
+ raise ValueError("bound SQL parameters are invalid")
|
|
|
|
|
+ placeholders = {
|
|
|
|
|
+ placeholder.name for placeholder in parsed.find_all(exp.Placeholder)
|
|
|
|
|
+ }
|
|
|
|
|
+ if placeholders != set(parameters):
|
|
|
|
|
+ raise ValueError("bound SQL placeholders do not match parameters")
|
|
|
|
|
+ if plan["result_contract"] != RESULT_CONTRACT:
|
|
|
|
|
+ raise ValueError("bound SQL result contract is unsupported")
|
|
|
|
|
+ _canonical_json(plan)
|
|
|
|
|
+ return plan
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def bound_sql_plan_hash(value: Any) -> str:
|
|
|
|
|
+ plan = validate_bound_sql_plan(value)
|
|
|
|
|
+ return hashlib.sha256(_canonical_json(plan).encode("utf-8")).hexdigest()
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def bound_sql_plan_relations(value: Any) -> dict[str, str]:
|
|
|
|
|
+ """Return the two physical relation names from a validated INSERT plan."""
|
|
|
|
|
+
|
|
|
|
|
+ plan = validate_bound_sql_plan(value)
|
|
|
|
|
+ sql_dialect = (
|
|
|
|
|
+ "postgres" if plan["dialect"] == "postgresql" else plan["dialect"]
|
|
|
|
|
+ )
|
|
|
|
|
+ insert = sqlglot.parse_one(
|
|
|
|
|
+ plan["statements"][0]["sql"],
|
|
|
|
|
+ read=sql_dialect,
|
|
|
|
|
+ )
|
|
|
|
|
+ target = insert.this
|
|
|
|
|
+ if not isinstance(target, exp.Schema) or not isinstance(
|
|
|
|
|
+ target.this, exp.Table
|
|
|
|
|
+ ):
|
|
|
|
|
+ raise ValueError("bound SQL target relation is invalid")
|
|
|
|
|
+ sources = list(insert.expression.find_all(exp.Table))
|
|
|
|
|
+ if len(sources) != 1:
|
|
|
|
|
+ raise ValueError("bound SQL source relation is invalid")
|
|
|
|
|
+
|
|
|
|
|
+ def object_ref(table: exp.Table) -> str:
|
|
|
|
|
+ if not table.db or not table.name:
|
|
|
|
|
+ raise ValueError("bound SQL relation must be schema-qualified")
|
|
|
|
|
+ return f"{table.db}.{table.name}"
|
|
|
|
|
+
|
|
|
|
|
+ return {
|
|
|
|
|
+ "input_object_ref": object_ref(sources[0]),
|
|
|
|
|
+ "output_object_ref": object_ref(target.this),
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class SqlGlotRuleCompiler(RuleCompiler):
|
|
|
|
|
+ """Compile the proven first operator slice into one INSERT ... SELECT."""
|
|
|
|
|
+
|
|
|
|
|
+ def __init__(self, dialect: str):
|
|
|
|
|
+ self.dialect = _dialect(dialect)
|
|
|
|
|
+ if self.dialect not in SUPPORTED_DIALECTS:
|
|
|
|
|
+ raise ValueError("unsupported SQLGlot compiler dialect")
|
|
|
|
|
+
|
|
|
|
|
+ def compile(
|
|
|
|
|
+ self,
|
|
|
|
|
+ *,
|
|
|
|
|
+ rule_version: dict[str, Any],
|
|
|
|
|
+ input_schema: dict[str, Any],
|
|
|
|
|
+ output_schema: dict[str, Any],
|
|
|
|
|
+ input_binding: dict[str, Any],
|
|
|
|
|
+ output_binding: dict[str, Any],
|
|
|
|
|
+ backend: dict[str, Any],
|
|
|
|
|
+ ) -> dict[str, Any]:
|
|
|
|
|
+ if not isinstance(rule_version, dict) or rule_version.get(
|
|
|
|
|
+ "status"
|
|
|
|
|
+ ) != "published":
|
|
|
|
|
+ raise ValueError("only published rule versions may be compiled")
|
|
|
|
|
+ rule_version_id = _uid(rule_version.get("id"), "rule_version_id")
|
|
|
|
|
+ spec = read_rule_spec(rule_version.get("rule_spec"))
|
|
|
|
|
+ if rule_version.get("spec_hash") != rule_spec_hash(spec):
|
|
|
|
|
+ raise ValueError("published rule version spec hash does not match")
|
|
|
|
|
+ source_schema = _snapshot(input_schema, "input schema")
|
|
|
|
|
+ target_schema = _snapshot(output_schema, "output schema")
|
|
|
|
|
+ source_binding = _binding(input_binding, "input binding")
|
|
|
|
|
+ target_binding = _binding(output_binding, "output binding")
|
|
|
|
|
+ if spec["input_schema_ref"] != source_schema["schema_ref"] or spec[
|
|
|
|
|
+ "output_schema_ref"
|
|
|
|
|
+ ] != target_schema["schema_ref"]:
|
|
|
|
|
+ raise ValueError("rule schema references do not match snapshots")
|
|
|
|
|
+ if source_binding["schema_snapshot_id"] != source_schema["id"] or target_binding[
|
|
|
|
|
+ "schema_snapshot_id"
|
|
|
|
|
+ ] != target_schema["id"]:
|
|
|
|
|
+ raise ValueError("binding schema snapshot does not match")
|
|
|
|
|
+ if source_binding["data_source_uid"] != target_binding["data_source_uid"]:
|
|
|
|
|
+ raise ValueError("cross-source rules require a batch compiler")
|
|
|
|
|
+ if source_binding["dialect"] != self.dialect or target_binding[
|
|
|
|
|
+ "dialect"
|
|
|
|
|
+ ] != self.dialect:
|
|
|
|
|
+ raise ValueError("binding dialect does not match compiler")
|
|
|
|
|
+ if source_binding["access_mode"] not in {"read", "read_write"}:
|
|
|
|
|
+ raise ValueError("input binding is not readable")
|
|
|
|
|
+ if target_binding["access_mode"] not in {"write", "read_write"}:
|
|
|
|
|
+ raise ValueError("output binding is not writable")
|
|
|
|
|
+ if source_binding["object_kind"] not in {"table", "view"} or target_binding[
|
|
|
|
|
+ "object_kind"
|
|
|
|
|
+ ] != "table":
|
|
|
|
|
+ raise ValueError("SQL rules require a table/view input and table output")
|
|
|
|
|
+ if target_binding["write_mode"] != "append":
|
|
|
|
|
+ raise ValueError("unsupported SQL output write mode")
|
|
|
|
|
+ capabilities = _capabilities(
|
|
|
|
|
+ backend,
|
|
|
|
|
+ dialect=self.dialect,
|
|
|
|
|
+ timezone=spec["timezone"],
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ field_types = {
|
|
|
|
|
+ field["name"]: field["type"] for field in source_schema["fields"]
|
|
|
|
|
+ }
|
|
|
|
|
+ supported = validate_rule_expressions(spec, field_types)
|
|
|
|
|
+ if self.dialect not in supported:
|
|
|
|
|
+ raise ValueError("rule expressions are unsupported by the bound dialect")
|
|
|
|
|
+ input_fields = [field["name"] for field in source_schema["fields"]]
|
|
|
|
|
+ output_fields = [field["name"] for field in target_schema["fields"]]
|
|
|
|
|
+ current_fields = list(input_fields)
|
|
|
|
|
+ source: exp.Expression = _table(source_binding["object_ref"])
|
|
|
|
|
+ parameters = _ParameterStore()
|
|
|
|
|
+ expression_compiler = _ExpressionCompiler(
|
|
|
|
|
+ dialect=self.dialect,
|
|
|
|
|
+ fields=field_types,
|
|
|
|
|
+ parameters=parameters,
|
|
|
|
|
+ )
|
|
|
|
|
+ alias_number = 0
|
|
|
|
|
+
|
|
|
|
|
+ def replace(
|
|
|
|
|
+ replacements: dict[str, exp.Expression],
|
|
|
|
|
+ *,
|
|
|
|
|
+ where: exp.Expression | None = None,
|
|
|
|
|
+ ) -> None:
|
|
|
|
|
+ nonlocal source, alias_number
|
|
|
|
|
+ query = _project(
|
|
|
|
|
+ source,
|
|
|
|
|
+ current_fields,
|
|
|
|
|
+ replacements=replacements,
|
|
|
|
|
+ where=where,
|
|
|
|
|
+ )
|
|
|
|
|
+ alias_number += 1
|
|
|
|
|
+ source = _source_relation(query, alias_number)
|
|
|
|
|
+
|
|
|
|
|
+ for step in spec["steps"]:
|
|
|
|
|
+ operation = step["op"]
|
|
|
|
|
+ if operation not in SUPPORTED_OPERATIONS:
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ f"unsupported SQL rule operation: {operation}"
|
|
|
|
|
+ )
|
|
|
|
|
+ if operation in {
|
|
|
|
|
+ "cast",
|
|
|
|
|
+ "fill_null",
|
|
|
|
|
+ "map_values",
|
|
|
|
|
+ "normalize_text",
|
|
|
|
|
+ "regex_replace",
|
|
|
|
|
+ }:
|
|
|
|
|
+ column_name = _identifier(step.get("column"), f"{operation} column")
|
|
|
|
|
+ if column_name not in current_fields:
|
|
|
|
|
+ raise ValueError(f"unknown SQL rule column: {column_name}")
|
|
|
|
|
+ if operation == "cast":
|
|
|
|
|
+ target_type = str(step.get("to") or "").strip().lower()
|
|
|
|
|
+ type_name = _TYPE_NAMES.get(target_type)
|
|
|
|
|
+ if type_name is None:
|
|
|
|
|
+ raise ValueError("unsupported SQL cast target")
|
|
|
|
|
+ replace(
|
|
|
|
|
+ {
|
|
|
|
|
+ column_name: exp.Cast(
|
|
|
|
|
+ this=_column(column_name),
|
|
|
|
|
+ to=exp.DataType.build(type_name),
|
|
|
|
|
+ )
|
|
|
|
|
+ }
|
|
|
|
|
+ )
|
|
|
|
|
+ field_types[column_name] = target_type
|
|
|
|
|
+ elif operation == "normalize_text":
|
|
|
|
|
+ if field_types[column_name] != "string":
|
|
|
|
|
+ raise ValueError("normalize_text requires a string field type")
|
|
|
|
|
+ if step.get("lowercase") or step.get("uppercase"):
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ "SQL case normalization requires a proven Unicode collation"
|
|
|
|
|
+ )
|
|
|
|
|
+ if not step.get("trim"):
|
|
|
|
|
+ raise ValueError("normalize_text has no proven SQL operation")
|
|
|
|
|
+ replace({column_name: exp.Trim(this=_column(column_name))})
|
|
|
|
|
+ elif operation == "regex_replace":
|
|
|
|
|
+ if field_types[column_name] != "string":
|
|
|
|
|
+ raise ValueError("regex_replace requires a string field type")
|
|
|
|
|
+ pattern = _portable_regex(step.get("pattern"))
|
|
|
|
|
+ replacement = step.get("replacement")
|
|
|
|
|
+ if not isinstance(replacement, str):
|
|
|
|
|
+ raise ValueError("regex replacement must be a string")
|
|
|
|
|
+ replace(
|
|
|
|
|
+ {
|
|
|
|
|
+ column_name: exp.RegexpReplace(
|
|
|
|
|
+ this=_column(column_name),
|
|
|
|
|
+ expression=parameters.add(pattern),
|
|
|
|
|
+ replacement=parameters.add(replacement),
|
|
|
|
|
+ )
|
|
|
|
|
+ }
|
|
|
|
|
+ )
|
|
|
|
|
+ elif operation == "fill_null":
|
|
|
|
|
+ if not _value_matches_type(
|
|
|
|
|
+ step.get("value"),
|
|
|
|
|
+ field_types[column_name],
|
|
|
|
|
+ ):
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ "fill_null value does not match the field type"
|
|
|
|
|
+ )
|
|
|
|
|
+ replace(
|
|
|
|
|
+ {
|
|
|
|
|
+ column_name: exp.Coalesce(
|
|
|
|
|
+ this=_column(column_name),
|
|
|
|
|
+ expressions=[parameters.add(step.get("value"))],
|
|
|
|
|
+ )
|
|
|
|
|
+ }
|
|
|
|
|
+ )
|
|
|
|
|
+ elif operation in {"filter", "assert"}:
|
|
|
|
|
+ if operation == "assert" and step.get("on_failure") != "reject":
|
|
|
|
|
+ raise ValueError("assert failure action is not SQL executable")
|
|
|
|
|
+ predicate = expression_compiler.compile(step["expression_ast"])
|
|
|
|
|
+ replace({}, where=predicate)
|
|
|
|
|
+ elif operation == "derive":
|
|
|
|
|
+ target = _identifier(step.get("target"), "derive target")
|
|
|
|
|
+ if target not in current_fields and target not in output_fields:
|
|
|
|
|
+ raise ValueError("derive target is not in the output schema")
|
|
|
|
|
+ result_type = type_check_expression(
|
|
|
|
|
+ step["expression_ast"],
|
|
|
|
|
+ field_types,
|
|
|
|
|
+ )
|
|
|
|
|
+ target_type = next(
|
|
|
|
|
+ field["type"]
|
|
|
|
|
+ for field in target_schema["fields"]
|
|
|
|
|
+ if field["name"] == target
|
|
|
|
|
+ )
|
|
|
|
|
+ if not _types_compatible(result_type, target_type):
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ "derive expression type does not match its target type"
|
|
|
|
|
+ )
|
|
|
|
|
+ derived = expression_compiler.compile(step["expression_ast"])
|
|
|
|
|
+ if target not in current_fields:
|
|
|
|
|
+ current_fields.append(target)
|
|
|
|
|
+ replace({target: derived})
|
|
|
|
|
+ field_types[target] = target_type
|
|
|
|
|
+ elif operation == "map_values":
|
|
|
|
|
+ if field_types[column_name] != "string":
|
|
|
|
|
+ raise ValueError("map_values requires a string field type")
|
|
|
|
|
+ mapping = step.get("mapping")
|
|
|
|
|
+ if not isinstance(mapping, dict) or not mapping:
|
|
|
|
|
+ raise ValueError("map_values mapping must be non-empty")
|
|
|
|
|
+ if not all(
|
|
|
|
|
+ isinstance(source_value, str)
|
|
|
|
|
+ and isinstance(target_value, str)
|
|
|
|
|
+ for source_value, target_value in mapping.items()
|
|
|
|
|
+ ):
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ "map_values values must match the string field type"
|
|
|
|
|
+ )
|
|
|
|
|
+ case = exp.Case()
|
|
|
|
|
+ for source_value, target_value in sorted(
|
|
|
|
|
+ mapping.items(), key=lambda item: str(item[0])
|
|
|
|
|
+ ):
|
|
|
|
|
+ case = case.when(
|
|
|
|
|
+ exp.EQ(
|
|
|
|
|
+ this=_column(column_name),
|
|
|
|
|
+ expression=parameters.add(source_value),
|
|
|
|
|
+ ),
|
|
|
|
|
+ parameters.add(target_value),
|
|
|
|
|
+ )
|
|
|
|
|
+ case = case.else_(_column(column_name))
|
|
|
|
|
+ replace({column_name: case})
|
|
|
|
|
+ elif operation == "deduplicate":
|
|
|
|
|
+ keys = [
|
|
|
|
|
+ _identifier(item, "deduplicate key")
|
|
|
|
|
+ for item in step.get("keys", [])
|
|
|
|
|
+ ]
|
|
|
|
|
+ order_by = [
|
|
|
|
|
+ _identifier(item, "deduplicate order field")
|
|
|
|
|
+ for item in step.get("order_by", [])
|
|
|
|
|
+ ]
|
|
|
|
|
+ if (
|
|
|
|
|
+ not keys
|
|
|
|
|
+ or not order_by
|
|
|
|
|
+ or not set(keys + order_by) <= set(current_fields)
|
|
|
|
|
+ ):
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ "deduplicate requires known keys and deterministic order"
|
|
|
|
|
+ )
|
|
|
|
|
+ descending = step.get("keep", "first") == "last"
|
|
|
|
|
+ row_number = exp.Window(
|
|
|
|
|
+ this=exp.RowNumber(),
|
|
|
|
|
+ partition_by=[_column(item) for item in keys],
|
|
|
|
|
+ order=exp.Order(
|
|
|
|
|
+ expressions=[
|
|
|
|
|
+ exp.Ordered(
|
|
|
|
|
+ this=_column(item),
|
|
|
|
|
+ desc=descending,
|
|
|
|
|
+ )
|
|
|
|
|
+ for item in order_by
|
|
|
|
|
+ ]
|
|
|
|
|
+ ),
|
|
|
|
|
+ ).as_("_dataops_row_number")
|
|
|
|
|
+ ranked = exp.Select(
|
|
|
|
|
+ expressions=[
|
|
|
|
|
+ *[
|
|
|
|
|
+ _column(name).as_(_quoted_identifier(name))
|
|
|
|
|
+ for name in current_fields
|
|
|
|
|
+ ],
|
|
|
|
|
+ row_number,
|
|
|
|
|
+ ]
|
|
|
|
|
+ ).from_(source)
|
|
|
|
|
+ alias_number += 1
|
|
|
|
|
+ ranked_source = _source_relation(ranked, alias_number)
|
|
|
|
|
+ deduplicated = _project(
|
|
|
|
|
+ ranked_source,
|
|
|
|
|
+ current_fields,
|
|
|
|
|
+ where=exp.EQ(
|
|
|
|
|
+ this=_column("_dataops_row_number"),
|
|
|
|
|
+ expression=exp.Literal.number(1),
|
|
|
|
|
+ ),
|
|
|
|
|
+ )
|
|
|
|
|
+ alias_number += 1
|
|
|
|
|
+ source = _source_relation(deduplicated, alias_number)
|
|
|
|
|
+
|
|
|
|
|
+ if not set(output_fields) <= set(current_fields):
|
|
|
|
|
+ raise ValueError("compiled SQL cannot produce the output schema")
|
|
|
|
|
+ output_types = {
|
|
|
|
|
+ field["name"]: field["type"] for field in target_schema["fields"]
|
|
|
|
|
+ }
|
|
|
|
|
+ if any(
|
|
|
|
|
+ not _types_compatible(field_types[name], output_types[name])
|
|
|
|
|
+ for name in output_fields
|
|
|
|
|
+ ):
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ "compiled SQL field types do not match the output schema"
|
|
|
|
|
+ )
|
|
|
|
|
+ final_select = exp.Select(
|
|
|
|
|
+ expressions=[_column(name) for name in output_fields]
|
|
|
|
|
+ ).from_(source)
|
|
|
|
|
+ target = exp.Schema(
|
|
|
|
|
+ this=_table(target_binding["object_ref"]),
|
|
|
|
|
+ expressions=[_quoted_identifier(name) for name in output_fields],
|
|
|
|
|
+ )
|
|
|
|
|
+ statement_ast = exp.Insert(this=target, expression=final_select)
|
|
|
|
|
+ statement = _render(statement_ast, self.dialect)
|
|
|
|
|
+ plan = validate_bound_sql_plan(
|
|
|
|
|
+ {
|
|
|
|
|
+ "schema_version": PLAN_SCHEMA_VERSION,
|
|
|
|
|
+ "dialect": self.dialect,
|
|
|
|
|
+ "capabilities": capabilities,
|
|
|
|
|
+ "data_source_uid": source_binding["data_source_uid"],
|
|
|
|
|
+ "rule_version_id": rule_version_id,
|
|
|
|
|
+ "input_binding_id": source_binding["id"],
|
|
|
|
|
+ "output_binding_id": target_binding["id"],
|
|
|
|
|
+ "statements": [
|
|
|
|
|
+ {
|
|
|
|
|
+ "purpose": "write",
|
|
|
|
|
+ "sql": statement,
|
|
|
|
|
+ "parameters": parameters.values,
|
|
|
|
|
+ }
|
|
|
|
|
+ ],
|
|
|
|
|
+ "result_contract": RESULT_CONTRACT,
|
|
|
|
|
+ }
|
|
|
|
|
+ )
|
|
|
|
|
+ return {
|
|
|
|
|
+ "backend": "sql_pushdown",
|
|
|
|
|
+ "compiler_version": COMPILER_VERSION,
|
|
|
|
|
+ "status": "compiled",
|
|
|
|
|
+ "plan": plan,
|
|
|
|
|
+ "plan_hash": bound_sql_plan_hash(plan),
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+__all__ = [
|
|
|
|
|
+ "COMPILER_VERSION",
|
|
|
|
|
+ "SqlGlotRuleCompiler",
|
|
|
|
|
+ "bound_sql_plan_hash",
|
|
|
|
|
+ "bound_sql_plan_relations",
|
|
|
|
|
+ "validate_bound_sql_plan",
|
|
|
|
|
+]
|