|
@@ -1,5 +1,6 @@
|
|
|
from __future__ import annotations
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
+import hashlib
|
|
|
import os
|
|
import os
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
|
|
|
@@ -14,6 +15,14 @@ from tests.core.data_rules.test_polars_compiler import (
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+def _sha256(path) -> str:
|
|
|
|
|
+ digest = hashlib.sha256()
|
|
|
|
|
+ with path.open("rb") as handle:
|
|
|
|
|
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
|
|
|
+ digest.update(chunk)
|
|
|
|
|
+ return digest.hexdigest()
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
def _compile_worker_plan(
|
|
def _compile_worker_plan(
|
|
|
*,
|
|
*,
|
|
|
input_schema,
|
|
input_schema,
|
|
@@ -201,7 +210,7 @@ def test_worker_rejects_golden_row_drift_inside_isolated_boundary(tmp_path):
|
|
|
"lookup_paths": {},
|
|
"lookup_paths": {},
|
|
|
"output_path": str(tmp_path / "golden-output.parquet"),
|
|
"output_path": str(tmp_path / "golden-output.parquet"),
|
|
|
"golden_path": str(golden_path),
|
|
"golden_path": str(golden_path),
|
|
|
- "golden_digest": "a" * 64,
|
|
|
|
|
|
|
+ "golden_digest": _sha256(golden_path),
|
|
|
"masking_policies": {},
|
|
"masking_policies": {},
|
|
|
},
|
|
},
|
|
|
memory_limit_bytes=plan["resource_limits"][
|
|
memory_limit_bytes=plan["resource_limits"][
|
|
@@ -239,10 +248,18 @@ def test_concurrent_large_golden_comparisons_fail_closed(tmp_path):
|
|
|
plan["resource_limits"]["memory_limit_bytes"] = 128 * 1024 * 1024
|
|
plan["resource_limits"]["memory_limit_bytes"] = 128 * 1024 * 1024
|
|
|
input_path = tmp_path / "large-input.parquet"
|
|
input_path = tmp_path / "large-input.parquet"
|
|
|
golden_path = tmp_path / "large-golden.parquet"
|
|
golden_path = tmp_path / "large-golden.parquet"
|
|
|
- pl.DataFrame({"value": ["actual"]}).write_parquet(input_path)
|
|
|
|
|
|
|
+ row_count = 200_000
|
|
|
pl.DataFrame(
|
|
pl.DataFrame(
|
|
|
- {"value": [f"expected-{index:08d}" for index in range(200_000)]}
|
|
|
|
|
|
|
+ {"value": [f"actual-{index:08d}" for index in range(row_count)]}
|
|
|
|
|
+ ).write_parquet(input_path)
|
|
|
|
|
+ pl.DataFrame(
|
|
|
|
|
+ {
|
|
|
|
|
+ "value": [
|
|
|
|
|
+ f"expected-{index:08d}" for index in range(row_count)
|
|
|
|
|
+ ]
|
|
|
|
|
+ }
|
|
|
).write_parquet(golden_path)
|
|
).write_parquet(golden_path)
|
|
|
|
|
+ golden_digest = _sha256(golden_path)
|
|
|
|
|
|
|
|
def compare(index):
|
|
def compare(index):
|
|
|
try:
|
|
try:
|
|
@@ -255,7 +272,7 @@ def test_concurrent_large_golden_comparisons_fail_closed(tmp_path):
|
|
|
tmp_path / f"large-output-{index}.parquet"
|
|
tmp_path / f"large-output-{index}.parquet"
|
|
|
),
|
|
),
|
|
|
"golden_path": str(golden_path),
|
|
"golden_path": str(golden_path),
|
|
|
- "golden_digest": "b" * 64,
|
|
|
|
|
|
|
+ "golden_digest": golden_digest,
|
|
|
"masking_policies": {},
|
|
"masking_policies": {},
|
|
|
},
|
|
},
|
|
|
memory_limit_bytes=plan["resource_limits"][
|
|
memory_limit_bytes=plan["resource_limits"][
|