"""Tests for benchmark comparison and attribution logic. Validates: Requirements 16.2, 16.3, 16.5 """ from __future__ import annotations from services.intelligence_pipeline_v3.benchmark.comparison import ( ComparisonReport, ConfigDelta, FieldDelta, ResourceDelta, compare_configurations, ) from services.intelligence_pipeline_v3.benchmark.runner import ( BenchmarkDocumentResult, BenchmarkRun, ) # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- def _make_result( doc_id: str, *, schema_valid: bool = True, duration_ms: int = 100, input_tokens: int = 500, output_tokens: int = 200, retries: int = 0, error: str | None = None, ) -> BenchmarkDocumentResult: """Helper to create a BenchmarkDocumentResult.""" return BenchmarkDocumentResult( document_id=doc_id, raw_output='{"test": true}' if schema_valid else "invalid", parsed_output={"test": True} if schema_valid else None, schema_valid=schema_valid, retries=retries, duration_ms=duration_ms, input_tokens=input_tokens, output_tokens=output_tokens, error=error, ) def _make_run( config_name: str, results: list[BenchmarkDocumentResult], ) -> BenchmarkRun: """Helper to create a BenchmarkRun.""" return BenchmarkRun( config_name=config_name, document_ids=[r.document_id for r in results], results=results, ) # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- class TestCompareConfigurations: """Tests for compare_configurations function.""" def test_empty_comparison_runs(self) -> None: baseline = _make_run("baseline_current", [ _make_result("doc1", schema_valid=True), ]) report = compare_configurations(baseline, []) assert report.configs_compared == ["baseline_current"] assert report.deltas == [] def test_basic_comparison_produces_deltas(self) -> None: # Baseline: 50% schema validity baseline = _make_run("baseline_current", [ _make_result("doc1", schema_valid=True), _make_result("doc2", schema_valid=False, error="parse error"), ]) # Temp zero: 100% schema validity temp_zero = _make_run("baseline_temp_zero", [ _make_result("doc1", schema_valid=True), _make_result("doc2", schema_valid=True), ]) report = compare_configurations(baseline, [temp_zero]) assert len(report.configs_compared) == 2 assert len(report.deltas) == 1 delta = report.deltas[0] assert delta.baseline_config == "baseline_current" assert delta.comparison_config == "baseline_temp_zero" assert len(delta.field_deltas) > 0 assert len(delta.resource_deltas) > 0 def test_schema_validity_improvement_detected(self) -> None: baseline = _make_run("baseline_current", [ _make_result("doc1", schema_valid=True), _make_result("doc2", schema_valid=False, error="err"), _make_result("doc3", schema_valid=False, error="err"), _make_result("doc4", schema_valid=True), ]) strict = _make_run("baseline_strict_schema", [ _make_result("doc1", schema_valid=True), _make_result("doc2", schema_valid=True), _make_result("doc3", schema_valid=True), _make_result("doc4", schema_valid=True), ]) report = compare_configurations(baseline, [strict]) delta = report.deltas[0] # Find the schema_validity_rate field delta validity_delta = next( (d for d in delta.field_deltas if d.field_name == "schema_validity_rate"), None, ) assert validity_delta is not None assert validity_delta.improved is True assert validity_delta.comparison_value == 1.0 assert validity_delta.baseline_value == 0.5 def test_attribution_with_incremental_improvement(self) -> None: # Baseline: 50% valid baseline = _make_run("baseline_current", [ _make_result("d1", schema_valid=True), _make_result("d2", schema_valid=False, error="e"), ]) # Temp zero: 75% (fixes half the remaining) # We simulate by 3/4 valid temp_zero = _make_run("baseline_temp_zero", [ _make_result("d1", schema_valid=True), _make_result("d2", schema_valid=True), _make_result("d3", schema_valid=True), _make_result("d4", schema_valid=False, error="e"), ]) # Strict schema: 100% valid strict = _make_run("baseline_strict_schema", [ _make_result("d1", schema_valid=True), _make_result("d2", schema_valid=True), ]) report = compare_configurations(baseline, [temp_zero, strict]) # Attribution should exist assert "temperature_fix" in report.attribution_summary assert "schema_constraint" in report.attribution_summary # All attribution values should be between 0 and 1 for val in report.attribution_summary.values(): assert 0.0 <= val <= 1.0 def test_attribution_no_improvement(self) -> None: # Both configurations have same validity baseline = _make_run("baseline_current", [ _make_result("d1", schema_valid=True), ]) temp_zero = _make_run("baseline_temp_zero", [ _make_result("d1", schema_valid=True), ]) report = compare_configurations(baseline, [temp_zero]) # No improvement means zero attribution assert report.attribution_summary.get("temperature_fix", 0.0) == 0.0 assert report.attribution_summary.get("schema_constraint", 0.0) == 0.0 def test_resource_improvement_lower_is_better(self) -> None: baseline = _make_run("baseline_current", [ _make_result("d1", duration_ms=500, retries=3), ]) improved = _make_run("baseline_temp_zero", [ _make_result("d1", duration_ms=200, retries=0), ]) report = compare_configurations(baseline, [improved]) delta = report.deltas[0] # Duration should show improvement (lower) duration_delta = next( (d for d in delta.resource_deltas if d.metric_name == "mean_duration_ms"), None, ) assert duration_delta is not None assert duration_delta.improved is True assert duration_delta.comparison_value < duration_delta.baseline_value def test_multiple_comparisons(self) -> None: baseline = _make_run("baseline_current", [ _make_result("d1", schema_valid=True), ]) comp1 = _make_run("baseline_temp_zero", [ _make_result("d1", schema_valid=True), ]) comp2 = _make_run("baseline_strict_schema", [ _make_result("d1", schema_valid=True), ]) report = compare_configurations(baseline, [comp1, comp2]) assert len(report.deltas) == 2 assert report.configs_compared == [ "baseline_current", "baseline_temp_zero", "baseline_strict_schema", ] class TestComparisonReportModel: """Tests for the ComparisonReport Pydantic model.""" def test_serialization_roundtrip(self) -> None: report = ComparisonReport( configs_compared=["a", "b"], deltas=[ ConfigDelta( baseline_config="a", comparison_config="b", field_deltas=[ FieldDelta( field_name="accuracy", baseline_value=0.5, comparison_value=0.8, absolute_delta=0.3, relative_delta_percent=60.0, improved=True, ) ], resource_deltas=[ ResourceDelta( metric_name="latency_ms", baseline_value=500.0, comparison_value=300.0, absolute_delta=-200.0, relative_delta_percent=-40.0, improved=True, ) ], ) ], attribution_summary={"temperature_fix": 0.6, "schema_constraint": 0.4}, ) json_str = report.model_dump_json() restored = ComparisonReport.model_validate_json(json_str) assert restored.configs_compared == report.configs_compared assert len(restored.deltas) == 1 assert restored.attribution_summary["temperature_fix"] == 0.6