stonks-oracle/services/intelligence_pipeline_v3/fine_tuning/evaluation.py

"""Holdout evaluation and promotion gate checking for fine-tuned models.

Evaluates against frozen holdout and production artifact. A model is
promoted only when correctness gates pass — not merely when adjudication
rate falls.
"""

from __future__ import annotations

import enum
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any
from uuid import UUID, uuid4


class PromotionDecision(str, enum.Enum):
    """Decision on whether to promote a fine-tuned model."""

    PROMOTE = "promote"
    REJECT = "reject"
    NEEDS_REVIEW = "needs_review"


@dataclass
class EvaluationResult:
    """Results of evaluating a fine-tuned model against holdout data."""

    evaluation_id: UUID
    training_run_id: UUID
    model_version: str
    evaluated_at: datetime

    # Correctness metrics (what matters for promotion)
    entity_f1: float = 0.0
    entity_precision: float = 0.0
    entity_recall: float = 0.0
    event_f1: float = 0.0
    relation_f1: float = 0.0
    fact_exact_match: float = 0.0

    # Calibration metrics
    calibration_ece: float = 0.0
    brier_score: float = 0.0

    # Comparison with production model
    production_entity_f1: float = 0.0
    production_event_f1: float = 0.0
    entity_f1_delta: float = 0.0
    event_f1_delta: float = 0.0

    # Adjudication impact (reported but not a gate)
    adjudication_rate_before: float = 0.0
    adjudication_rate_after: float = 0.0
    adjudication_rate_delta: float = 0.0

    # Holdout details
    holdout_size: int = 0
    holdout_version: str = ""

    @classmethod
    def create(
        cls,
        training_run_id: UUID,
        model_version: str,
        **kwargs: Any,
    ) -> EvaluationResult:
        return cls(
            evaluation_id=uuid4(),
            training_run_id=training_run_id,
            model_version=model_version,
            evaluated_at=datetime.now(timezone.utc),
            **kwargs,
        )

    def passes_correctness_gates(
        self,
        min_entity_f1_delta: float = 0.0,
        min_event_f1_delta: float = -0.02,  # Allow tiny regression on events
        max_calibration_ece: float = 0.08,
    ) -> bool:
        """Check if correctness gates pass.

        Note: adjudication rate reduction is NOT a promotion gate.
        A model must pass field-level correctness regardless of
        adjudication impact.
        """
        # Entity F1 must not regress
        if self.entity_f1_delta < min_entity_f1_delta:
            return False

        # Event F1 must not regress significantly
        if self.event_f1_delta < min_event_f1_delta:
            return False

        # Calibration must remain acceptable
        if self.calibration_ece > max_calibration_ece:
            return False

        return True

    def promotion_decision(self) -> PromotionDecision:
        """Determine promotion decision based on gates."""
        if not self.passes_correctness_gates():
            return PromotionDecision.REJECT

        # If adjudication rate actually increases, flag for review
        if self.adjudication_rate_delta > 0.05:
            return PromotionDecision.NEEDS_REVIEW

        return PromotionDecision.PROMOTE

    def to_dict(self) -> dict[str, Any]:
        return {
            "evaluation_id": str(self.evaluation_id),
            "model_version": self.model_version,
            "entity_f1": self.entity_f1,
            "event_f1": self.event_f1,
            "relation_f1": self.relation_f1,
            "calibration_ece": self.calibration_ece,
            "entity_f1_delta": self.entity_f1_delta,
            "event_f1_delta": self.event_f1_delta,
            "adjudication_rate_delta": self.adjudication_rate_delta,
            "passes_correctness_gates": self.passes_correctness_gates(),
            "promotion_decision": self.promotion_decision().value,
        }


@dataclass
class ModelCard:
    """Model card for a trained specialist model artifact.

    Contains training range, dataset version, intended use, limitations,
    and evaluation results as required by Requirement 17.6.
    """

    card_id: UUID
    model_version: str
    base_model: str
    training_run_id: UUID
    created_at: datetime

    # Training details
    training_range: str = ""
    dataset_version: str = ""
    schema_version: str = ""
    total_training_examples: int = 0

    # Intended use
    intended_use: str = "Entity and event extraction for financial documents"
    entity_types: list[str] = field(default_factory=list)

    # Limitations
    limitations: list[str] = field(default_factory=lambda: [
        "Trained on English-language financial documents only",
        "Requires recalibration when new entity types are added",
        "Performance may degrade on document types not in training set",
    ])

    # Evaluation
    evaluation_results: EvaluationResult | None = None

    # Registry
    promoted: bool = False
    promoted_at: datetime | None = None
    deprecated: bool = False
    deprecated_at: datetime | None = None

    @classmethod
    def create(
        cls,
        model_version: str,
        base_model: str,
        training_run_id: UUID,
        **kwargs: Any,
    ) -> ModelCard:
        return cls(
            card_id=uuid4(),
            model_version=model_version,
            base_model=base_model,
            training_run_id=training_run_id,
            created_at=datetime.now(timezone.utc),
            **kwargs,
        )

    def promote(self) -> None:
        """Mark this model as promoted to production."""
        self.promoted = True
        self.promoted_at = datetime.now(timezone.utc)

    def deprecate(self) -> None:
        """Mark this model as deprecated."""
        self.deprecated = True
        self.deprecated_at = datetime.now(timezone.utc)

    def to_dict(self) -> dict[str, Any]:
        return {
            "card_id": str(self.card_id),
            "model_version": self.model_version,
            "base_model": self.base_model,
            "training_range": self.training_range,
            "dataset_version": self.dataset_version,
            "schema_version": self.schema_version,
            "total_training_examples": self.total_training_examples,
            "intended_use": self.intended_use,
            "entity_types": self.entity_types,
            "limitations": self.limitations,
            "promoted": self.promoted,
            "deprecated": self.deprecated,
        }