"""Unit tests for the per-document-type and per-difficulty report generator.

Tests the DocumentResult model, generate_evaluation_report(), and
format_report_markdown() function.
"""
from __future__ import annotations

from services.intelligence_pipeline_v3.evaluation.entity_metrics import (
    EntitySpan,
    MatchMode,
    TickerMention,
)
from services.intelligence_pipeline_v3.evaluation.event_metrics import (
    GoldEvent,
    PredictedEvent,
)
from services.intelligence_pipeline_v3.evaluation.evidence_metrics import (
    EvidenceSpan,
    ExtractionResult,
    FieldType,
)
from services.intelligence_pipeline_v3.evaluation.numeric_metrics import NumericFact
from services.intelligence_pipeline_v3.evaluation.report_generator import (
    Difficulty,
    DocumentResult,
    DocumentType,
    SafetyGateThresholds,
    format_report_markdown,
    generate_evaluation_report,
)
from services.intelligence_pipeline_v3.evaluation.resource_metrics import (
    StageTimingRecord,
)
from services.intelligence_pipeline_v3.evaluation.sentiment_metrics import (
    SentimentLabel,
    SentimentPrediction,
)
from services.intelligence_pipeline_v3.schemas.annotations import (
    EventClass,
)

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _make_doc(
    doc_id: str = "doc-1",
    doc_type: DocumentType = DocumentType.article,
    difficulty: Difficulty = Difficulty.easy,
    *,
    with_entities: bool = False,
    with_events: bool = False,
    with_numeric: bool = False,
    with_evidence: bool = False,
    with_sentiment: bool = False,
    with_timings: bool = False,
) -> DocumentResult:
    """Create a DocumentResult with optional populated metric inputs."""
    kwargs: dict = {
        "document_id": doc_id,
        "document_type": doc_type,
        "difficulty": difficulty,
    }

    if with_entities:
        kwargs["predicted_entities"] = [
            EntitySpan(text="Apple", entity_type="company", start_char=0, end_char=5),
            EntitySpan(text="iPhone", entity_type="product", start_char=10, end_char=16),
        ]
        kwargs["gold_entities"] = [
            EntitySpan(text="Apple", entity_type="company", start_char=0, end_char=5),
            EntitySpan(text="iPhone", entity_type="product", start_char=10, end_char=16),
        ]
        kwargs["predicted_tickers"] = [
            TickerMention(text="AAPL", ticker="AAPL", start_char=0, end_char=4),
        ]
        kwargs["gold_tickers"] = [
            TickerMention(text="AAPL", ticker="AAPL", start_char=0, end_char=4),
        ]

    if with_events:
        kwargs["predicted_events"] = [
            PredictedEvent(
                event_class=EventClass.EARNINGS_BEAT,
                evidence_ids=["ev1"],
                primary_company_ids=["comp1"],
            ),
        ]
        kwargs["gold_events"] = [
            GoldEvent(
                event_class=EventClass.EARNINGS_BEAT,
                evidence_ids=["ev1"],
                primary_company_ids=["comp1"],
            ),
        ]

    if with_numeric:
        kwargs["predicted_numeric_facts"] = [
            NumericFact(
                fact_type="eps",
                predicate="reported",
                literal_value="$1.50",
                normalized_value=1.50,
                unit="USD",
            ),
        ]
        kwargs["gold_numeric_facts"] = [
            NumericFact(
                fact_type="eps",
                predicate="reported",
                literal_value="$1.50",
                normalized_value=1.50,
                unit="USD",
            ),
        ]

    if with_evidence:
        kwargs["source_text"] = "Apple reported earnings beat expectations."
        kwargs["evidence_spans"] = [
            EvidenceSpan(
                span_id="span-1",
                text="Apple reported earnings beat",
                start_char=0,
                end_char=28,
            ),
        ]
        kwargs["extraction_results"] = [
            ExtractionResult(
                item_id="item-1",
                field_type=FieldType.entity,
                evidence_ids=["span-1"],
            ),
        ]

    if with_sentiment:
        kwargs["predicted_sentiments"] = [
            SentimentPrediction(
                company_entity_id="comp1",
                label=SentimentLabel.positive,
                positive_prob=0.8,
                negative_prob=0.1,
                neutral_prob=0.1,
            ),
        ]
        kwargs["gold_sentiments"] = [
            SentimentPrediction(
                company_entity_id="comp1",
                label=SentimentLabel.positive,
                positive_prob=0.9,
                negative_prob=0.05,
                neutral_prob=0.05,
            ),
        ]

    if with_timings:
        kwargs["stage_timings"] = [
            StageTimingRecord(
                document_id=doc_id,
                stage_name="extraction",
                start_time=100.0,
                end_time=101.5,
                input_tokens=500,
                output_tokens=200,
                cpu_seconds=1.2,
                gpu_seconds=0.3,
                gpu_memory_mb=4096.0,
            ),
            StageTimingRecord(
                document_id=doc_id,
                stage_name="sentiment",
                start_time=101.5,
                end_time=102.0,
                input_tokens=200,
                output_tokens=50,
                cpu_seconds=0.4,
                gpu_seconds=0.0,
            ),
        ]

    return DocumentResult(**kwargs)


# ---------------------------------------------------------------------------
# Tests — DocumentResult Model
# ---------------------------------------------------------------------------


class TestDocumentResult:
    """Tests for the DocumentResult model."""

    def test_minimal_creation(self):
        doc = DocumentResult(
            document_id="test-1",
            document_type=DocumentType.article,
            difficulty=Difficulty.easy,
        )
        assert doc.document_id == "test-1"
        assert doc.document_type == DocumentType.article
        assert doc.difficulty == Difficulty.easy
        assert doc.predicted_entities == []
        assert doc.stage_timings == []

    def test_all_document_types_valid(self):
        for dt in DocumentType:
            doc = DocumentResult(
                document_id="t",
                document_type=dt,
                difficulty=Difficulty.medium,
            )
            assert doc.document_type == dt

    def test_all_difficulties_valid(self):
        for d in Difficulty:
            doc = DocumentResult(
                document_id="t",
                document_type=DocumentType.filing,
                difficulty=d,
            )
            assert doc.difficulty == d


# ---------------------------------------------------------------------------
# Tests — generate_evaluation_report
# ---------------------------------------------------------------------------


class TestGenerateEvaluationReport:
    """Tests for the generate_evaluation_report function."""

    def test_empty_documents_list(self):
        report = generate_evaluation_report([])
        assert report.total_documents == 0
        assert report.overall.document_count == 0
        assert report.per_document_type == {}
        assert report.per_difficulty == {}
        assert report.safety_gate.passed is True

    def test_single_document_overall(self):
        doc = _make_doc(
            with_entities=True,
            with_events=True,
            with_numeric=True,
            with_evidence=True,
            with_sentiment=True,
            with_timings=True,
        )
        report = generate_evaluation_report([doc])
        assert report.total_documents == 1
        assert report.overall.document_count == 1
        assert report.overall.entity_metrics is not None
        assert report.overall.event_metrics is not None
        assert report.overall.numeric_metrics is not None
        assert report.overall.evidence_metrics is not None
        assert report.overall.sentiment_metrics is not None
        assert report.overall.resource_metrics is not None

    def test_groups_by_document_type(self):
        docs = [
            _make_doc("d1", DocumentType.article, Difficulty.easy, with_entities=True),
            _make_doc("d2", DocumentType.filing, Difficulty.easy, with_entities=True),
            _make_doc("d3", DocumentType.article, Difficulty.medium, with_entities=True),
        ]
        report = generate_evaluation_report(docs)
        assert report.total_documents == 3
        assert "article" in report.per_document_type
        assert "filing" in report.per_document_type
        assert report.per_document_type["article"].document_count == 2
        assert report.per_document_type["filing"].document_count == 1

    def test_groups_by_difficulty(self):
        docs = [
            _make_doc("d1", DocumentType.article, Difficulty.easy, with_entities=True),
            _make_doc("d2", DocumentType.article, Difficulty.hard, with_entities=True),
            _make_doc("d3", DocumentType.article, Difficulty.hard, with_entities=True),
        ]
        report = generate_evaluation_report(docs)
        assert "easy" in report.per_difficulty
        assert "hard" in report.per_difficulty
        assert report.per_difficulty["easy"].document_count == 1
        assert report.per_difficulty["hard"].document_count == 2

    def test_entity_metrics_perfect_match(self):
        doc = _make_doc(with_entities=True)
        report = generate_evaluation_report([doc])
        entity_report = report.overall.entity_metrics
        assert entity_report is not None
        assert entity_report.entity_metrics.overall.f1 == 1.0
        assert entity_report.ticker_metrics.overall.f1 == 1.0

    def test_event_metrics_perfect_match(self):
        doc = _make_doc(with_events=True)
        report = generate_evaluation_report([doc])
        event_report = report.overall.event_metrics
        assert event_report is not None
        # The predicted event matches the gold event (same class, overlapping evidence)
        assert event_report.event_metrics.micro.f1 > 0.0

    def test_numeric_metrics_exact_match(self):
        doc = _make_doc(with_numeric=True)
        report = generate_evaluation_report([doc])
        nm = report.overall.numeric_metrics
        assert nm is not None
        assert nm.exact_match_accuracy.accuracy == 1.0

    def test_evidence_metrics_valid_spans(self):
        doc = _make_doc(with_evidence=True)
        report = generate_evaluation_report([doc])
        ev = report.overall.evidence_metrics
        assert ev is not None
        assert ev.validity_rate == 1.0
        assert ev.support_rate == 1.0

    def test_sentiment_metrics_match(self):
        doc = _make_doc(with_sentiment=True)
        report = generate_evaluation_report([doc])
        sm = report.overall.sentiment_metrics
        assert sm is not None
        assert sm.f1_metrics.macro_f1 > 0.0

    def test_resource_metrics_present(self):
        doc = _make_doc(with_timings=True)
        report = generate_evaluation_report([doc])
        rm = report.overall.resource_metrics
        assert rm is not None
        assert rm.document_count == 1
        assert rm.latency.p50 > 0.0
        assert rm.throughput.total_documents == 1

    def test_empty_document_types_not_in_report(self):
        """Document types with no documents should not appear in per_document_type."""
        docs = [_make_doc("d1", DocumentType.article, Difficulty.easy)]
        report = generate_evaluation_report(docs)
        assert "filing" not in report.per_document_type
        assert "transcript" not in report.per_document_type

    def test_entity_match_mode_propagated(self):
        doc = _make_doc(with_entities=True)
        report_strict = generate_evaluation_report([doc], entity_match_mode=MatchMode.strict)
        report_relaxed = generate_evaluation_report([doc], entity_match_mode=MatchMode.relaxed)
        # Both should work; with perfect data, both should give same results
        assert report_strict.overall.entity_metrics is not None
        assert report_relaxed.overall.entity_metrics is not None


# ---------------------------------------------------------------------------
# Tests — Safety Gate
# ---------------------------------------------------------------------------


class TestSafetyGate:
    """Tests for the safety gate evaluation."""

    def test_all_pass_with_perfect_data(self):
        doc = _make_doc(
            with_entities=True,
            with_events=True,
            with_evidence=True,
            with_sentiment=True,
        )
        # Use relaxed ECE threshold since single-sample calibration can exceed defaults
        thresholds = SafetyGateThresholds(max_calibration_ece=0.3)
        report = generate_evaluation_report([doc], safety_thresholds=thresholds)
        assert report.safety_gate.passed is True
        assert all(report.safety_gate.checks.values())

    def test_custom_thresholds_fail(self):
        """Very high thresholds should cause failure on partial data."""
        # Create a doc with entity mismatch
        doc = DocumentResult(
            document_id="d1",
            document_type=DocumentType.article,
            difficulty=Difficulty.easy,
            predicted_entities=[
                EntitySpan(text="X", entity_type="company", start_char=0, end_char=1),
            ],
            gold_entities=[
                EntitySpan(text="Y", entity_type="company", start_char=5, end_char=6),
            ],
        )
        strict_thresholds = SafetyGateThresholds(min_entity_f1=0.9)
        report = generate_evaluation_report([doc], safety_thresholds=strict_thresholds)
        assert report.safety_gate.checks["entity_f1"] is False
        assert report.safety_gate.passed is False

    def test_safety_gate_details_populated(self):
        doc = _make_doc(with_entities=True, with_sentiment=True)
        report = generate_evaluation_report([doc])
        gate = report.safety_gate
        assert len(gate.checks) > 0
        assert len(gate.details) > 0
        # All details should be non-empty strings
        for detail in gate.details.values():
            assert isinstance(detail, str)
            assert len(detail) > 0


# ---------------------------------------------------------------------------
# Tests — format_report_markdown
# ---------------------------------------------------------------------------


class TestFormatReportMarkdown:
    """Tests for the markdown formatter."""

    def test_empty_report_produces_valid_markdown(self):
        report = generate_evaluation_report([])
        md = format_report_markdown(report)
        assert "# Intelligence Pipeline v3" in md
        assert "Safety Gate" in md
        assert "Total documents evaluated:** 0" in md

    def test_full_report_includes_all_sections(self):
        docs = [
            _make_doc(
                "d1", DocumentType.article, Difficulty.easy,
                with_entities=True, with_events=True,
                with_numeric=True, with_evidence=True,
                with_sentiment=True, with_timings=True,
            ),
            _make_doc(
                "d2", DocumentType.filing, Difficulty.hard,
                with_entities=True, with_events=True,
                with_numeric=True, with_evidence=True,
                with_sentiment=True, with_timings=True,
            ),
        ]
        report = generate_evaluation_report(docs)
        md = format_report_markdown(report)

        # Header
        assert "# Intelligence Pipeline v3 — Evaluation Report" in md
        # Safety gate
        assert "Safety Gate" in md
        assert "PASSED" in md or "FAILED" in md
        # Overall section
        assert "Overall Metrics" in md
        # Per type sections
        assert "Per Document Type" in md
        assert "article" in md
        assert "filing" in md
        # Per difficulty sections
        assert "Per Difficulty" in md
        assert "easy" in md
        assert "hard" in md
        # Metric sections
        assert "Entity Metrics" in md
        assert "Event & Relation Metrics" in md
        assert "Numeric Metrics" in md
        assert "Evidence Metrics" in md
        assert "Sentiment Metrics" in md
        assert "Resource Metrics" in md

    def test_markdown_contains_numeric_values(self):
        doc = _make_doc(with_timings=True)
        report = generate_evaluation_report([doc])
        md = format_report_markdown(report)
        # Should contain latency values
        assert "p50" in md or "Latency" in md
        assert "docs/min" in md

    def test_safety_gate_pass_icon(self):
        doc = _make_doc(with_entities=True, with_sentiment=True)
        report = generate_evaluation_report([doc])
        md = format_report_markdown(report)
        assert "✅" in md

    def test_safety_gate_fail_icon(self):
        doc = DocumentResult(
            document_id="d1",
            document_type=DocumentType.article,
            difficulty=Difficulty.easy,
            predicted_entities=[
                EntitySpan(text="X", entity_type="company", start_char=0, end_char=1),
            ],
            gold_entities=[
                EntitySpan(text="Y", entity_type="company", start_char=5, end_char=6),
            ],
        )
        thresholds = SafetyGateThresholds(min_entity_f1=0.9)
        report = generate_evaluation_report([doc], safety_thresholds=thresholds)
        md = format_report_markdown(report)
        assert "❌" in md


# ---------------------------------------------------------------------------
# Tests — Multi-document aggregation
# ---------------------------------------------------------------------------


class TestMultiDocumentAggregation:
    """Tests for correct metric aggregation across multiple documents."""

    def test_entities_aggregated_across_documents(self):
        """Entity counts from multiple docs should sum in the overall report."""
        doc1 = DocumentResult(
            document_id="d1",
            document_type=DocumentType.article,
            difficulty=Difficulty.easy,
            predicted_entities=[
                EntitySpan(text="Apple", entity_type="company", start_char=0, end_char=5),
            ],
            gold_entities=[
                EntitySpan(text="Apple", entity_type="company", start_char=0, end_char=5),
            ],
        )
        doc2 = DocumentResult(
            document_id="d2",
            document_type=DocumentType.article,
            difficulty=Difficulty.medium,
            predicted_entities=[
                EntitySpan(text="Google", entity_type="company", start_char=0, end_char=6),
            ],
            gold_entities=[
                EntitySpan(text="Google", entity_type="company", start_char=0, end_char=6),
            ],
        )
        report = generate_evaluation_report([doc1, doc2])
        overall_entities = report.overall.entity_metrics
        assert overall_entities is not None
        assert overall_entities.entity_metrics.overall.support_gold == 2
        assert overall_entities.entity_metrics.overall.f1 == 1.0

    def test_timings_aggregated_correctly(self):
        """Resource metrics should include all documents' timings."""
        doc1 = _make_doc("d1", DocumentType.article, Difficulty.easy, with_timings=True)
        doc2 = _make_doc("d2", DocumentType.filing, Difficulty.hard, with_timings=True)
        report = generate_evaluation_report([doc1, doc2])
        rm = report.overall.resource_metrics
        assert rm is not None
        assert rm.document_count == 2
        assert rm.throughput.total_documents == 2
