Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
250 lines
8.9 KiB
Python
250 lines
8.9 KiB
Python
"""Tests for benchmark comparison and attribution logic.
|
|
|
|
Validates: Requirements 16.2, 16.3, 16.5
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from services.intelligence_pipeline_v3.benchmark.comparison import (
|
|
ComparisonReport,
|
|
ConfigDelta,
|
|
FieldDelta,
|
|
ResourceDelta,
|
|
compare_configurations,
|
|
)
|
|
from services.intelligence_pipeline_v3.benchmark.runner import (
|
|
BenchmarkDocumentResult,
|
|
BenchmarkRun,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_result(
|
|
doc_id: str,
|
|
*,
|
|
schema_valid: bool = True,
|
|
duration_ms: int = 100,
|
|
input_tokens: int = 500,
|
|
output_tokens: int = 200,
|
|
retries: int = 0,
|
|
error: str | None = None,
|
|
) -> BenchmarkDocumentResult:
|
|
"""Helper to create a BenchmarkDocumentResult."""
|
|
return BenchmarkDocumentResult(
|
|
document_id=doc_id,
|
|
raw_output='{"test": true}' if schema_valid else "invalid",
|
|
parsed_output={"test": True} if schema_valid else None,
|
|
schema_valid=schema_valid,
|
|
retries=retries,
|
|
duration_ms=duration_ms,
|
|
input_tokens=input_tokens,
|
|
output_tokens=output_tokens,
|
|
error=error,
|
|
)
|
|
|
|
|
|
def _make_run(
|
|
config_name: str,
|
|
results: list[BenchmarkDocumentResult],
|
|
) -> BenchmarkRun:
|
|
"""Helper to create a BenchmarkRun."""
|
|
return BenchmarkRun(
|
|
config_name=config_name,
|
|
document_ids=[r.document_id for r in results],
|
|
results=results,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCompareConfigurations:
|
|
"""Tests for compare_configurations function."""
|
|
|
|
def test_empty_comparison_runs(self) -> None:
|
|
baseline = _make_run("baseline_current", [
|
|
_make_result("doc1", schema_valid=True),
|
|
])
|
|
report = compare_configurations(baseline, [])
|
|
assert report.configs_compared == ["baseline_current"]
|
|
assert report.deltas == []
|
|
|
|
def test_basic_comparison_produces_deltas(self) -> None:
|
|
# Baseline: 50% schema validity
|
|
baseline = _make_run("baseline_current", [
|
|
_make_result("doc1", schema_valid=True),
|
|
_make_result("doc2", schema_valid=False, error="parse error"),
|
|
])
|
|
# Temp zero: 100% schema validity
|
|
temp_zero = _make_run("baseline_temp_zero", [
|
|
_make_result("doc1", schema_valid=True),
|
|
_make_result("doc2", schema_valid=True),
|
|
])
|
|
|
|
report = compare_configurations(baseline, [temp_zero])
|
|
|
|
assert len(report.configs_compared) == 2
|
|
assert len(report.deltas) == 1
|
|
delta = report.deltas[0]
|
|
assert delta.baseline_config == "baseline_current"
|
|
assert delta.comparison_config == "baseline_temp_zero"
|
|
assert len(delta.field_deltas) > 0
|
|
assert len(delta.resource_deltas) > 0
|
|
|
|
def test_schema_validity_improvement_detected(self) -> None:
|
|
baseline = _make_run("baseline_current", [
|
|
_make_result("doc1", schema_valid=True),
|
|
_make_result("doc2", schema_valid=False, error="err"),
|
|
_make_result("doc3", schema_valid=False, error="err"),
|
|
_make_result("doc4", schema_valid=True),
|
|
])
|
|
strict = _make_run("baseline_strict_schema", [
|
|
_make_result("doc1", schema_valid=True),
|
|
_make_result("doc2", schema_valid=True),
|
|
_make_result("doc3", schema_valid=True),
|
|
_make_result("doc4", schema_valid=True),
|
|
])
|
|
|
|
report = compare_configurations(baseline, [strict])
|
|
delta = report.deltas[0]
|
|
|
|
# Find the schema_validity_rate field delta
|
|
validity_delta = next(
|
|
(d for d in delta.field_deltas if d.field_name == "schema_validity_rate"),
|
|
None,
|
|
)
|
|
assert validity_delta is not None
|
|
assert validity_delta.improved is True
|
|
assert validity_delta.comparison_value == 1.0
|
|
assert validity_delta.baseline_value == 0.5
|
|
|
|
def test_attribution_with_incremental_improvement(self) -> None:
|
|
# Baseline: 50% valid
|
|
baseline = _make_run("baseline_current", [
|
|
_make_result("d1", schema_valid=True),
|
|
_make_result("d2", schema_valid=False, error="e"),
|
|
])
|
|
# Temp zero: 75% (fixes half the remaining)
|
|
# We simulate by 3/4 valid
|
|
temp_zero = _make_run("baseline_temp_zero", [
|
|
_make_result("d1", schema_valid=True),
|
|
_make_result("d2", schema_valid=True),
|
|
_make_result("d3", schema_valid=True),
|
|
_make_result("d4", schema_valid=False, error="e"),
|
|
])
|
|
# Strict schema: 100% valid
|
|
strict = _make_run("baseline_strict_schema", [
|
|
_make_result("d1", schema_valid=True),
|
|
_make_result("d2", schema_valid=True),
|
|
])
|
|
|
|
report = compare_configurations(baseline, [temp_zero, strict])
|
|
|
|
# Attribution should exist
|
|
assert "temperature_fix" in report.attribution_summary
|
|
assert "schema_constraint" in report.attribution_summary
|
|
|
|
# All attribution values should be between 0 and 1
|
|
for val in report.attribution_summary.values():
|
|
assert 0.0 <= val <= 1.0
|
|
|
|
def test_attribution_no_improvement(self) -> None:
|
|
# Both configurations have same validity
|
|
baseline = _make_run("baseline_current", [
|
|
_make_result("d1", schema_valid=True),
|
|
])
|
|
temp_zero = _make_run("baseline_temp_zero", [
|
|
_make_result("d1", schema_valid=True),
|
|
])
|
|
|
|
report = compare_configurations(baseline, [temp_zero])
|
|
|
|
# No improvement means zero attribution
|
|
assert report.attribution_summary.get("temperature_fix", 0.0) == 0.0
|
|
assert report.attribution_summary.get("schema_constraint", 0.0) == 0.0
|
|
|
|
def test_resource_improvement_lower_is_better(self) -> None:
|
|
baseline = _make_run("baseline_current", [
|
|
_make_result("d1", duration_ms=500, retries=3),
|
|
])
|
|
improved = _make_run("baseline_temp_zero", [
|
|
_make_result("d1", duration_ms=200, retries=0),
|
|
])
|
|
|
|
report = compare_configurations(baseline, [improved])
|
|
delta = report.deltas[0]
|
|
|
|
# Duration should show improvement (lower)
|
|
duration_delta = next(
|
|
(d for d in delta.resource_deltas if d.metric_name == "mean_duration_ms"),
|
|
None,
|
|
)
|
|
assert duration_delta is not None
|
|
assert duration_delta.improved is True
|
|
assert duration_delta.comparison_value < duration_delta.baseline_value
|
|
|
|
def test_multiple_comparisons(self) -> None:
|
|
baseline = _make_run("baseline_current", [
|
|
_make_result("d1", schema_valid=True),
|
|
])
|
|
comp1 = _make_run("baseline_temp_zero", [
|
|
_make_result("d1", schema_valid=True),
|
|
])
|
|
comp2 = _make_run("baseline_strict_schema", [
|
|
_make_result("d1", schema_valid=True),
|
|
])
|
|
|
|
report = compare_configurations(baseline, [comp1, comp2])
|
|
assert len(report.deltas) == 2
|
|
assert report.configs_compared == [
|
|
"baseline_current",
|
|
"baseline_temp_zero",
|
|
"baseline_strict_schema",
|
|
]
|
|
|
|
|
|
class TestComparisonReportModel:
|
|
"""Tests for the ComparisonReport Pydantic model."""
|
|
|
|
def test_serialization_roundtrip(self) -> None:
|
|
report = ComparisonReport(
|
|
configs_compared=["a", "b"],
|
|
deltas=[
|
|
ConfigDelta(
|
|
baseline_config="a",
|
|
comparison_config="b",
|
|
field_deltas=[
|
|
FieldDelta(
|
|
field_name="accuracy",
|
|
baseline_value=0.5,
|
|
comparison_value=0.8,
|
|
absolute_delta=0.3,
|
|
relative_delta_percent=60.0,
|
|
improved=True,
|
|
)
|
|
],
|
|
resource_deltas=[
|
|
ResourceDelta(
|
|
metric_name="latency_ms",
|
|
baseline_value=500.0,
|
|
comparison_value=300.0,
|
|
absolute_delta=-200.0,
|
|
relative_delta_percent=-40.0,
|
|
improved=True,
|
|
)
|
|
],
|
|
)
|
|
],
|
|
attribution_summary={"temperature_fix": 0.6, "schema_constraint": 0.4},
|
|
)
|
|
|
|
json_str = report.model_dump_json()
|
|
restored = ComparisonReport.model_validate_json(json_str)
|
|
assert restored.configs_compared == report.configs_compared
|
|
assert len(restored.deltas) == 1
|
|
assert restored.attribution_summary["temperature_fix"] == 0.6
|