"""Per-document-type and per-difficulty evaluation report generator.

Groups evaluation results by document type and difficulty bucket,
runs all individual metric computations per group, and produces a
FullEvaluationReport with overall + per-type + per-difficulty breakdowns.

Validates: Requirements 16.3, 16.4
"""
from __future__ import annotations

from collections import defaultdict
from enum import Enum

from pydantic import BaseModel, Field

from services.intelligence_pipeline_v3.evaluation.entity_metrics import (
    EntityEvaluationReport,
    EntitySpan,
    MatchMode,
    TickerMention,
    evaluate_entities,
)
from services.intelligence_pipeline_v3.evaluation.event_metrics import (
    EventRelationEvaluationReport,
    GoldEvent,
    GoldRelation,
    PredictedEvent,
    PredictedRelation,
    evaluate_events_and_relations,
)
from services.intelligence_pipeline_v3.evaluation.evidence_metrics import (
    EvidenceMetricsResult,
    EvidenceSpan,
    ExtractionResult,
    evaluate_evidence,
)
from services.intelligence_pipeline_v3.evaluation.numeric_metrics import (
    NumericEvaluationReport,
    NumericFact,
    evaluate_numeric_facts,
)
from services.intelligence_pipeline_v3.evaluation.resource_metrics import (
    ResourceEvaluationReport,
    StageTimingRecord,
    evaluate_resources,
)
from services.intelligence_pipeline_v3.evaluation.sentiment_metrics import (
    SentimentEvaluationReport,
    SentimentPrediction,
    evaluate_sentiment,
)

# ---------------------------------------------------------------------------
# Domain Models
# ---------------------------------------------------------------------------


class DocumentType(str, Enum):
    """Document types in the evaluation corpus."""

    article = "article"
    filing = "filing"
    transcript = "transcript"
    press_release = "press_release"
    macro_event = "macro_event"


class Difficulty(str, Enum):
    """Difficulty buckets for evaluation stratification."""

    easy = "easy"
    medium = "medium"
    hard = "hard"


class DocumentResult(BaseModel):
    """Holds all metric inputs for a single evaluated document."""

    document_id: str
    document_type: DocumentType
    difficulty: Difficulty

    # Entity metric inputs
    predicted_entities: list[EntitySpan] = Field(default_factory=list)
    gold_entities: list[EntitySpan] = Field(default_factory=list)
    predicted_tickers: list[TickerMention] = Field(default_factory=list)
    gold_tickers: list[TickerMention] = Field(default_factory=list)

    # Event/relation metric inputs
    predicted_events: list[PredictedEvent] = Field(default_factory=list)
    gold_events: list[GoldEvent] = Field(default_factory=list)
    predicted_relations: list[PredictedRelation] = Field(default_factory=list)
    gold_relations: list[GoldRelation] = Field(default_factory=list)

    # Numeric metric inputs
    predicted_numeric_facts: list[NumericFact] = Field(default_factory=list)
    gold_numeric_facts: list[NumericFact] = Field(default_factory=list)

    # Evidence metric inputs
    evidence_spans: list[EvidenceSpan] = Field(default_factory=list)
    source_text: str = ""
    extraction_results: list[ExtractionResult] = Field(default_factory=list)

    # Sentiment metric inputs
    predicted_sentiments: list[SentimentPrediction] = Field(default_factory=list)
    gold_sentiments: list[SentimentPrediction] = Field(default_factory=list)

    # Resource metric inputs
    stage_timings: list[StageTimingRecord] = Field(default_factory=list)

    model_config = {"arbitrary_types_allowed": True}


# ---------------------------------------------------------------------------
# Safety Gate Models
# ---------------------------------------------------------------------------


class SafetyGateThresholds(BaseModel):
    """Configurable thresholds for safety gate pass/fail."""

    min_entity_f1: float = Field(default=0.7, ge=0.0, le=1.0)
    min_event_macro_f1: float = Field(default=0.5, ge=0.0, le=1.0)
    min_evidence_support_rate: float = Field(default=0.8, ge=0.0, le=1.0)
    max_unsupported_claim_rate: float = Field(default=0.2, ge=0.0, le=1.0)
    min_sentiment_macro_f1: float = Field(default=0.5, ge=0.0, le=1.0)
    max_calibration_ece: float = Field(default=0.15, ge=0.0, le=1.0)


class SafetyGateResult(BaseModel):
    """Result of safety gate evaluation."""

    passed: bool
    checks: dict[str, bool] = Field(default_factory=dict)
    details: dict[str, str] = Field(default_factory=dict)


# ---------------------------------------------------------------------------
# Report Models
# ---------------------------------------------------------------------------


class GroupMetrics(BaseModel):
    """Metrics for a single group (document type or difficulty bucket)."""

    group_name: str
    document_count: int = Field(ge=0)
    entity_metrics: EntityEvaluationReport | None = None
    event_metrics: EventRelationEvaluationReport | None = None
    numeric_metrics: NumericEvaluationReport | None = None
    evidence_metrics: EvidenceMetricsResult | None = None
    sentiment_metrics: SentimentEvaluationReport | None = None
    resource_metrics: ResourceEvaluationReport | None = None


class FullEvaluationReport(BaseModel):
    """Complete evaluation report with overall + per-type + per-difficulty breakdowns."""

    overall: GroupMetrics
    per_document_type: dict[str, GroupMetrics] = Field(default_factory=dict)
    per_difficulty: dict[str, GroupMetrics] = Field(default_factory=dict)
    safety_gate: SafetyGateResult
    total_documents: int = Field(ge=0)


# ---------------------------------------------------------------------------
# Core Computation
# ---------------------------------------------------------------------------


def _compute_group_metrics(
    group_name: str,
    documents: list[DocumentResult],
    entity_match_mode: MatchMode = MatchMode.strict,
) -> GroupMetrics:
    """Compute all metrics for a group of documents.

    Aggregates all individual document inputs into combined lists and
    runs each metric computation once for the group.
    """
    if not documents:
        return GroupMetrics(group_name=group_name, document_count=0)

    doc_count = len(documents)

    # Aggregate entity inputs
    all_pred_entities: list[EntitySpan] = []
    all_gold_entities: list[EntitySpan] = []
    all_pred_tickers: list[TickerMention] = []
    all_gold_tickers: list[TickerMention] = []

    for doc in documents:
        all_pred_entities.extend(doc.predicted_entities)
        all_gold_entities.extend(doc.gold_entities)
        all_pred_tickers.extend(doc.predicted_tickers)
        all_gold_tickers.extend(doc.gold_tickers)

    entity_report = evaluate_entities(
        predicted_entities=all_pred_entities,
        gold_entities=all_gold_entities,
        predicted_tickers=all_pred_tickers,
        gold_tickers=all_gold_tickers,
        mode=entity_match_mode,
        document_count=doc_count,
    )

    # Aggregate event/relation inputs
    all_pred_events: list[PredictedEvent] = []
    all_gold_events: list[GoldEvent] = []
    all_pred_relations: list[PredictedRelation] = []
    all_gold_relations: list[GoldRelation] = []

    for doc in documents:
        all_pred_events.extend(doc.predicted_events)
        all_gold_events.extend(doc.gold_events)
        all_pred_relations.extend(doc.predicted_relations)
        all_gold_relations.extend(doc.gold_relations)

    event_report = evaluate_events_and_relations(
        predicted_events=all_pred_events,
        gold_events=all_gold_events,
        predicted_relations=all_pred_relations,
        gold_relations=all_gold_relations,
        document_count=doc_count,
    )

    # Aggregate numeric inputs
    all_pred_numeric: list[NumericFact] = []
    all_gold_numeric: list[NumericFact] = []

    for doc in documents:
        all_pred_numeric.extend(doc.predicted_numeric_facts)
        all_gold_numeric.extend(doc.gold_numeric_facts)

    numeric_report = evaluate_numeric_facts(
        predicted=all_pred_numeric,
        gold=all_gold_numeric,
        document_count=doc_count,
    )

    # Aggregate evidence inputs — concatenate source texts with separator
    all_spans: list[EvidenceSpan] = []
    all_items: list[ExtractionResult] = []
    combined_source = ""

    for doc in documents:
        offset = len(combined_source)
        # Adjust span offsets for combined source
        for span in doc.evidence_spans:
            all_spans.append(
                EvidenceSpan(
                    span_id=span.span_id,
                    text=span.text,
                    start_char=span.start_char + offset,
                    end_char=span.end_char + offset,
                    document_id=span.document_id or doc.document_id,
                )
            )
        all_items.extend(doc.extraction_results)
        combined_source += doc.source_text

    evidence_report = evaluate_evidence(
        spans=all_spans,
        source_text=combined_source,
        items=all_items,
    )

    # Aggregate sentiment inputs
    all_pred_sentiments: list[SentimentPrediction] = []
    all_gold_sentiments: list[SentimentPrediction] = []

    for doc in documents:
        all_pred_sentiments.extend(doc.predicted_sentiments)
        all_gold_sentiments.extend(doc.gold_sentiments)

    sentiment_report = evaluate_sentiment(
        predicted=all_pred_sentiments,
        gold=all_gold_sentiments,
        document_count=doc_count,
    )

    # Aggregate resource inputs
    all_timings: list[StageTimingRecord] = []
    for doc in documents:
        all_timings.extend(doc.stage_timings)

    resource_report = evaluate_resources(records=all_timings)

    return GroupMetrics(
        group_name=group_name,
        document_count=doc_count,
        entity_metrics=entity_report,
        event_metrics=event_report,
        numeric_metrics=numeric_report,
        evidence_metrics=evidence_report,
        sentiment_metrics=sentiment_report,
        resource_metrics=resource_report,
    )


def _evaluate_safety_gate(
    overall: GroupMetrics,
    thresholds: SafetyGateThresholds,
) -> SafetyGateResult:
    """Evaluate safety gate thresholds against overall metrics."""
    checks: dict[str, bool] = {}
    details: dict[str, str] = {}

    # Entity F1
    if overall.entity_metrics:
        entity_f1 = overall.entity_metrics.entity_metrics.overall.f1
        passed_entity = entity_f1 >= thresholds.min_entity_f1
        checks["entity_f1"] = passed_entity
        details["entity_f1"] = (
            f"{entity_f1:.3f} {'≥' if passed_entity else '<'} {thresholds.min_entity_f1:.3f}"
        )
    else:
        checks["entity_f1"] = True
        details["entity_f1"] = "No entity data"

    # Event macro-F1
    if overall.event_metrics:
        event_f1 = overall.event_metrics.event_metrics.macro_f1
        passed_event = event_f1 >= thresholds.min_event_macro_f1
        checks["event_macro_f1"] = passed_event
        details["event_macro_f1"] = (
            f"{event_f1:.3f} {'≥' if passed_event else '<'} {thresholds.min_event_macro_f1:.3f}"
        )
    else:
        checks["event_macro_f1"] = True
        details["event_macro_f1"] = "No event data"

    # Evidence support rate
    if overall.evidence_metrics:
        support_rate = overall.evidence_metrics.support_rate
        passed_support = support_rate >= thresholds.min_evidence_support_rate
        checks["evidence_support_rate"] = passed_support
        details["evidence_support_rate"] = (
            f"{support_rate:.3f} {'≥' if passed_support else '<'} "
            f"{thresholds.min_evidence_support_rate:.3f}"
        )

        unsupported = overall.evidence_metrics.unsupported_claim_rate
        passed_unsupported = unsupported <= thresholds.max_unsupported_claim_rate
        checks["unsupported_claim_rate"] = passed_unsupported
        details["unsupported_claim_rate"] = (
            f"{unsupported:.3f} {'≤' if passed_unsupported else '>'} "
            f"{thresholds.max_unsupported_claim_rate:.3f}"
        )
    else:
        checks["evidence_support_rate"] = True
        checks["unsupported_claim_rate"] = True
        details["evidence_support_rate"] = "No evidence data"
        details["unsupported_claim_rate"] = "No evidence data"

    # Sentiment macro-F1
    if overall.sentiment_metrics:
        sent_f1 = overall.sentiment_metrics.f1_metrics.macro_f1
        passed_sent = sent_f1 >= thresholds.min_sentiment_macro_f1
        checks["sentiment_macro_f1"] = passed_sent
        details["sentiment_macro_f1"] = (
            f"{sent_f1:.3f} {'≥' if passed_sent else '<'} "
            f"{thresholds.min_sentiment_macro_f1:.3f}"
        )

        ece = overall.sentiment_metrics.calibration.ece
        passed_ece = ece <= thresholds.max_calibration_ece
        checks["calibration_ece"] = passed_ece
        details["calibration_ece"] = (
            f"{ece:.3f} {'≤' if passed_ece else '>'} {thresholds.max_calibration_ece:.3f}"
        )
    else:
        checks["sentiment_macro_f1"] = True
        checks["calibration_ece"] = True
        details["sentiment_macro_f1"] = "No sentiment data"
        details["calibration_ece"] = "No sentiment data"

    all_passed = all(checks.values())

    return SafetyGateResult(
        passed=all_passed,
        checks=checks,
        details=details,
    )


# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------


def generate_evaluation_report(
    documents: list[DocumentResult],
    entity_match_mode: MatchMode = MatchMode.strict,
    safety_thresholds: SafetyGateThresholds | None = None,
) -> FullEvaluationReport:
    """Generate a full evaluation report with per-type and per-difficulty breakdowns.

    Groups documents by document_type and difficulty, runs all metrics per group,
    and evaluates safety gate thresholds against overall results.

    Args:
        documents: List of DocumentResult objects with all metric inputs.
        entity_match_mode: Matching strategy for entity metrics.
        safety_thresholds: Configurable safety gate thresholds (uses defaults if None).

    Returns:
        FullEvaluationReport with overall, per-type, per-difficulty, and safety gate.
    """
    if safety_thresholds is None:
        safety_thresholds = SafetyGateThresholds()

    # Overall metrics
    overall = _compute_group_metrics("overall", documents, entity_match_mode)

    # Group by document type
    by_type: dict[str, list[DocumentResult]] = defaultdict(list)
    for doc in documents:
        by_type[doc.document_type.value].append(doc)

    per_document_type: dict[str, GroupMetrics] = {}
    for doc_type in DocumentType:
        type_docs = by_type.get(doc_type.value, [])
        if type_docs:
            per_document_type[doc_type.value] = _compute_group_metrics(
                doc_type.value, type_docs, entity_match_mode
            )

    # Group by difficulty
    by_difficulty: dict[str, list[DocumentResult]] = defaultdict(list)
    for doc in documents:
        by_difficulty[doc.difficulty.value].append(doc)

    per_difficulty: dict[str, GroupMetrics] = {}
    for diff in Difficulty:
        diff_docs = by_difficulty.get(diff.value, [])
        if diff_docs:
            per_difficulty[diff.value] = _compute_group_metrics(
                diff.value, diff_docs, entity_match_mode
            )

    # Safety gate evaluation
    safety_gate = _evaluate_safety_gate(overall, safety_thresholds)

    return FullEvaluationReport(
        overall=overall,
        per_document_type=per_document_type,
        per_difficulty=per_difficulty,
        safety_gate=safety_gate,
        total_documents=len(documents),
    )


# ---------------------------------------------------------------------------
# Markdown Formatter
# ---------------------------------------------------------------------------


def _format_prf1_row(label: str, p: float, r: float, f1: float, support: int) -> str:
    """Format a single PRF1 row for a markdown table."""
    return f"| {label} | {p:.3f} | {r:.3f} | {f1:.3f} | {support} |"


def _format_group_section(group: GroupMetrics, heading_level: int = 3) -> str:
    """Format a single group's metrics as markdown."""
    prefix = "#" * heading_level
    lines: list[str] = []
    lines.append(f"{prefix} {group.group_name} ({group.document_count} documents)")
    lines.append("")

    # Entity metrics
    if group.entity_metrics:
        em = group.entity_metrics
        lines.append(f"{prefix}# Entity Metrics")
        lines.append("")
        lines.append("| Metric | Precision | Recall | F1 | Support |")
        lines.append("|--------|-----------|--------|-----|---------|")
        o = em.entity_metrics.overall
        lines.append(_format_prf1_row("Entities (overall)", o.precision, o.recall, o.f1, o.support_gold))
        t = em.ticker_metrics.overall
        lines.append(_format_prf1_row("Tickers (overall)", t.precision, t.recall, t.f1, t.support_gold))
        lines.append("")
        lines.append(f"Ambiguity accuracy: {em.ambiguity_accuracy.accuracy:.3f}")
        lines.append("")

    # Event metrics
    if group.event_metrics:
        ev = group.event_metrics
        lines.append(f"{prefix}# Event & Relation Metrics")
        lines.append("")
        lines.append(f"- Event macro-F1: {ev.event_metrics.macro_f1:.3f}")
        micro = ev.event_metrics.micro
        lines.append(f"- Event micro-F1: {micro.f1:.3f} (P={micro.precision:.3f}, R={micro.recall:.3f})")
        lines.append(f"- Relation macro-F1: {ev.relation_metrics.macro_f1:.3f}")
        r_micro = ev.relation_metrics.micro
        lines.append(f"- Relation micro-F1: {r_micro.f1:.3f} (P={r_micro.precision:.3f}, R={r_micro.recall:.3f})")
        lines.append("")

    # Numeric metrics
    if group.numeric_metrics:
        nm = group.numeric_metrics
        lines.append(f"{prefix}# Numeric Metrics")
        lines.append("")
        lines.append(f"- Exact match accuracy: {nm.exact_match_accuracy.accuracy:.3f} ({nm.exact_match_accuracy.matches}/{nm.exact_match_accuracy.total})")
        lines.append(f"- Tolerance accuracy ({nm.tolerance_pct_used}%): {nm.tolerance_accuracy.accuracy:.3f} ({nm.tolerance_accuracy.matches}/{nm.tolerance_accuracy.total})")
        lines.append(f"- Unit consistency: {nm.unit_consistency.accuracy:.3f}")
        lines.append(f"- Period match: {nm.period_match.accuracy:.3f}")
        lines.append("")

    # Evidence metrics
    if group.evidence_metrics:
        ev = group.evidence_metrics
        lines.append(f"{prefix}# Evidence Metrics")
        lines.append("")
        lines.append(f"- Offset validity rate: {ev.validity_rate:.3f} ({ev.valid_spans}/{ev.total_spans})")
        lines.append(f"- Support rate: {ev.support_rate:.3f} ({ev.supported_items}/{ev.total_items})")
        lines.append(f"- Coverage score: {ev.coverage_score:.3f}")
        lines.append(f"- Orphan rate: {ev.orphan_rate:.3f} ({ev.orphan_spans} orphans)")
        lines.append(f"- Unsupported claim rate: {ev.unsupported_claim_rate:.3f}")
        lines.append("")

    # Sentiment metrics
    if group.sentiment_metrics:
        sm = group.sentiment_metrics
        lines.append(f"{prefix}# Sentiment Metrics")
        lines.append("")
        lines.append(f"- Macro-F1: {sm.f1_metrics.macro_f1:.3f}")
        lines.append(f"- Micro-F1: {sm.f1_metrics.micro_f1:.3f}")
        lines.append(f"- Direction accuracy: {sm.direction_accuracy.accuracy:.3f}")
        lines.append(f"- Calibration ECE: {sm.calibration.ece:.3f}")
        lines.append(f"- Brier score: {sm.calibration.brier_score:.3f}")
        lines.append("")

    # Resource metrics
    if group.resource_metrics:
        rm = group.resource_metrics
        lines.append(f"{prefix}# Resource Metrics")
        lines.append("")
        lines.append(f"- Latency p50: {rm.latency.p50:.2f}s, p95: {rm.latency.p95:.2f}s, p99: {rm.latency.p99:.2f}s")
        lines.append(f"- Throughput: {rm.throughput.documents_per_minute:.1f} docs/min")
        lines.append(f"- Total tokens: {rm.token_usage.total_tokens}")
        lines.append(f"- CPU: {rm.cpu.total_cpu_seconds:.1f}s total, {rm.cpu.mean_cpu_seconds_per_document:.2f}s/doc")
        lines.append(f"- GPU: {rm.gpu.total_gpu_seconds:.1f}s total, peak {rm.gpu.peak_gpu_memory_mb:.0f} MB")
        lines.append("")

    return "\n".join(lines)


def format_report_markdown(report: FullEvaluationReport) -> str:
    """Produce a readable markdown summary of the full evaluation report.

    Args:
        report: The complete evaluation report.

    Returns:
        Markdown-formatted string with all sections.
    """
    lines: list[str] = []

    lines.append("# Intelligence Pipeline v3 — Evaluation Report")
    lines.append("")
    lines.append(f"**Total documents evaluated:** {report.total_documents}")
    lines.append("")

    # Safety gate summary
    lines.append("## Safety Gate")
    lines.append("")
    gate = report.safety_gate
    status = "✅ PASSED" if gate.passed else "❌ FAILED"
    lines.append(f"**Status:** {status}")
    lines.append("")
    lines.append("| Check | Result | Details |")
    lines.append("|-------|--------|---------|")
    for check_name, passed in gate.checks.items():
        icon = "✅" if passed else "❌"
        detail = gate.details.get(check_name, "")
        lines.append(f"| {check_name} | {icon} | {detail} |")
    lines.append("")

    # Overall metrics
    lines.append("## Overall Metrics")
    lines.append("")
    lines.append(_format_group_section(report.overall, heading_level=3))

    # Per document type
    if report.per_document_type:
        lines.append("## Per Document Type")
        lines.append("")
        for doc_type, group in sorted(report.per_document_type.items()):
            lines.append(_format_group_section(group, heading_level=3))

    # Per difficulty
    if report.per_difficulty:
        lines.append("## Per Difficulty")
        lines.append("")
        for diff, group in sorted(report.per_difficulty.items()):
            lines.append(_format_group_section(group, heading_level=3))

    return "\n".join(lines)
