"""Tests for benchmark comparison and attribution logic.

Validates: Requirements 16.2, 16.3, 16.5
"""
from __future__ import annotations

from services.intelligence_pipeline_v3.benchmark.comparison import (
    ComparisonReport,
    ConfigDelta,
    FieldDelta,
    ResourceDelta,
    compare_configurations,
)
from services.intelligence_pipeline_v3.benchmark.runner import (
    BenchmarkDocumentResult,
    BenchmarkRun,
)

# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


def _make_result(
    doc_id: str,
    *,
    schema_valid: bool = True,
    duration_ms: int = 100,
    input_tokens: int = 500,
    output_tokens: int = 200,
    retries: int = 0,
    error: str | None = None,
) -> BenchmarkDocumentResult:
    """Helper to create a BenchmarkDocumentResult."""
    return BenchmarkDocumentResult(
        document_id=doc_id,
        raw_output='{"test": true}' if schema_valid else "invalid",
        parsed_output={"test": True} if schema_valid else None,
        schema_valid=schema_valid,
        retries=retries,
        duration_ms=duration_ms,
        input_tokens=input_tokens,
        output_tokens=output_tokens,
        error=error,
    )


def _make_run(
    config_name: str,
    results: list[BenchmarkDocumentResult],
) -> BenchmarkRun:
    """Helper to create a BenchmarkRun."""
    return BenchmarkRun(
        config_name=config_name,
        document_ids=[r.document_id for r in results],
        results=results,
    )


# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------


class TestCompareConfigurations:
    """Tests for compare_configurations function."""

    def test_empty_comparison_runs(self) -> None:
        baseline = _make_run("baseline_current", [
            _make_result("doc1", schema_valid=True),
        ])
        report = compare_configurations(baseline, [])
        assert report.configs_compared == ["baseline_current"]
        assert report.deltas == []

    def test_basic_comparison_produces_deltas(self) -> None:
        # Baseline: 50% schema validity
        baseline = _make_run("baseline_current", [
            _make_result("doc1", schema_valid=True),
            _make_result("doc2", schema_valid=False, error="parse error"),
        ])
        # Temp zero: 100% schema validity
        temp_zero = _make_run("baseline_temp_zero", [
            _make_result("doc1", schema_valid=True),
            _make_result("doc2", schema_valid=True),
        ])

        report = compare_configurations(baseline, [temp_zero])

        assert len(report.configs_compared) == 2
        assert len(report.deltas) == 1
        delta = report.deltas[0]
        assert delta.baseline_config == "baseline_current"
        assert delta.comparison_config == "baseline_temp_zero"
        assert len(delta.field_deltas) > 0
        assert len(delta.resource_deltas) > 0

    def test_schema_validity_improvement_detected(self) -> None:
        baseline = _make_run("baseline_current", [
            _make_result("doc1", schema_valid=True),
            _make_result("doc2", schema_valid=False, error="err"),
            _make_result("doc3", schema_valid=False, error="err"),
            _make_result("doc4", schema_valid=True),
        ])
        strict = _make_run("baseline_strict_schema", [
            _make_result("doc1", schema_valid=True),
            _make_result("doc2", schema_valid=True),
            _make_result("doc3", schema_valid=True),
            _make_result("doc4", schema_valid=True),
        ])

        report = compare_configurations(baseline, [strict])
        delta = report.deltas[0]

        # Find the schema_validity_rate field delta
        validity_delta = next(
            (d for d in delta.field_deltas if d.field_name == "schema_validity_rate"),
            None,
        )
        assert validity_delta is not None
        assert validity_delta.improved is True
        assert validity_delta.comparison_value == 1.0
        assert validity_delta.baseline_value == 0.5

    def test_attribution_with_incremental_improvement(self) -> None:
        # Baseline: 50% valid
        baseline = _make_run("baseline_current", [
            _make_result("d1", schema_valid=True),
            _make_result("d2", schema_valid=False, error="e"),
        ])
        # Temp zero: 75% (fixes half the remaining)
        # We simulate by 3/4 valid
        temp_zero = _make_run("baseline_temp_zero", [
            _make_result("d1", schema_valid=True),
            _make_result("d2", schema_valid=True),
            _make_result("d3", schema_valid=True),
            _make_result("d4", schema_valid=False, error="e"),
        ])
        # Strict schema: 100% valid
        strict = _make_run("baseline_strict_schema", [
            _make_result("d1", schema_valid=True),
            _make_result("d2", schema_valid=True),
        ])

        report = compare_configurations(baseline, [temp_zero, strict])

        # Attribution should exist
        assert "temperature_fix" in report.attribution_summary
        assert "schema_constraint" in report.attribution_summary

        # All attribution values should be between 0 and 1
        for val in report.attribution_summary.values():
            assert 0.0 <= val <= 1.0

    def test_attribution_no_improvement(self) -> None:
        # Both configurations have same validity
        baseline = _make_run("baseline_current", [
            _make_result("d1", schema_valid=True),
        ])
        temp_zero = _make_run("baseline_temp_zero", [
            _make_result("d1", schema_valid=True),
        ])

        report = compare_configurations(baseline, [temp_zero])

        # No improvement means zero attribution
        assert report.attribution_summary.get("temperature_fix", 0.0) == 0.0
        assert report.attribution_summary.get("schema_constraint", 0.0) == 0.0

    def test_resource_improvement_lower_is_better(self) -> None:
        baseline = _make_run("baseline_current", [
            _make_result("d1", duration_ms=500, retries=3),
        ])
        improved = _make_run("baseline_temp_zero", [
            _make_result("d1", duration_ms=200, retries=0),
        ])

        report = compare_configurations(baseline, [improved])
        delta = report.deltas[0]

        # Duration should show improvement (lower)
        duration_delta = next(
            (d for d in delta.resource_deltas if d.metric_name == "mean_duration_ms"),
            None,
        )
        assert duration_delta is not None
        assert duration_delta.improved is True
        assert duration_delta.comparison_value < duration_delta.baseline_value

    def test_multiple_comparisons(self) -> None:
        baseline = _make_run("baseline_current", [
            _make_result("d1", schema_valid=True),
        ])
        comp1 = _make_run("baseline_temp_zero", [
            _make_result("d1", schema_valid=True),
        ])
        comp2 = _make_run("baseline_strict_schema", [
            _make_result("d1", schema_valid=True),
        ])

        report = compare_configurations(baseline, [comp1, comp2])
        assert len(report.deltas) == 2
        assert report.configs_compared == [
            "baseline_current",
            "baseline_temp_zero",
            "baseline_strict_schema",
        ]


class TestComparisonReportModel:
    """Tests for the ComparisonReport Pydantic model."""

    def test_serialization_roundtrip(self) -> None:
        report = ComparisonReport(
            configs_compared=["a", "b"],
            deltas=[
                ConfigDelta(
                    baseline_config="a",
                    comparison_config="b",
                    field_deltas=[
                        FieldDelta(
                            field_name="accuracy",
                            baseline_value=0.5,
                            comparison_value=0.8,
                            absolute_delta=0.3,
                            relative_delta_percent=60.0,
                            improved=True,
                        )
                    ],
                    resource_deltas=[
                        ResourceDelta(
                            metric_name="latency_ms",
                            baseline_value=500.0,
                            comparison_value=300.0,
                            absolute_delta=-200.0,
                            relative_delta_percent=-40.0,
                            improved=True,
                        )
                    ],
                )
            ],
            attribution_summary={"temperature_fix": 0.6, "schema_constraint": 0.4},
        )

        json_str = report.model_dump_json()
        restored = ComparisonReport.model_validate_json(json_str)
        assert restored.configs_compared == report.configs_compared
        assert len(restored.deltas) == 1
        assert restored.attribution_summary["temperature_fix"] == 0.6
