feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
@@ -0,0 +1,184 @@
+"""Tests for fine-tuning module — Task 50."""
+
+from __future__ import annotations
+
+from uuid import uuid4
+
+from services.intelligence_pipeline_v3.fine_tuning.evaluation import (
+    EvaluationResult,
+    ModelCard,
+    PromotionDecision,
+)
+from services.intelligence_pipeline_v3.fine_tuning.trainer import (
+    TrainingConfig,
+    TrainingRun,
+    TrainingStatus,
+)
+
+
+class TestTrainingRun:
+    """Task 50.1: Training pipeline."""
+
+    def test_create_training_run(self):
+        config = TrainingConfig(
+            base_model="GLiNER2-large",
+            schema_version="1.0",
+            dataset_version="v1",
+        )
+        run = TrainingRun.create(config)
+        assert run.status == TrainingStatus.PENDING
+        assert run.config.base_model == "GLiNER2-large"
+
+    def test_lifecycle(self):
+        config = TrainingConfig()
+        run = TrainingRun.create(config)
+        run.start()
+        assert run.status == TrainingStatus.PREPARING_DATA
+        assert run.started_at is not None
+        run.begin_training()
+        assert run.status == TrainingStatus.TRAINING
+        run.begin_evaluation()
+        assert run.status == TrainingStatus.EVALUATING
+        run.complete(
+            artifact_path="/models/gliner2-ft-v1",
+            model_version="gliner2-ft-v1.0",
+            train_loss=0.15,
+            validation_loss=0.20,
+            best_epoch=7,
+        )
+        assert run.status == TrainingStatus.COMPLETED
+        assert run.model_version == "gliner2-ft-v1.0"
+        assert run.duration_seconds is not None
+
+    def test_failure(self):
+        run = TrainingRun.create(TrainingConfig())
+        run.start()
+        run.fail("OOM error during training")
+        assert run.status == TrainingStatus.FAILED
+        assert "OOM" in run.errors[0]
+
+
+class TestEvaluation:
+    """Task 50.2: Holdout evaluation and promotion gates."""
+
+    def test_evaluation_passes_correctness_gates(self):
+        result = EvaluationResult.create(
+            training_run_id=uuid4(),
+            model_version="gliner2-ft-v1.0",
+            entity_f1=0.92,
+            event_f1=0.85,
+            entity_f1_delta=0.02,
+            event_f1_delta=0.01,
+            calibration_ece=0.05,
+        )
+        assert result.passes_correctness_gates()
+
+    def test_evaluation_fails_on_entity_regression(self):
+        result = EvaluationResult.create(
+            training_run_id=uuid4(),
+            model_version="gliner2-ft-bad",
+            entity_f1=0.80,
+            entity_f1_delta=-0.05,  # Regression
+            calibration_ece=0.05,
+        )
+        assert not result.passes_correctness_gates()
+
+    def test_evaluation_fails_on_high_calibration(self):
+        result = EvaluationResult.create(
+            training_run_id=uuid4(),
+            model_version="gliner2-ft-uncalibrated",
+            entity_f1=0.95,
+            entity_f1_delta=0.05,
+            calibration_ece=0.15,  # Too high
+        )
+        assert not result.passes_correctness_gates()
+
+    def test_promotion_not_based_on_adjudication_rate(self):
+        """Task 50.4: Promoted only when correctness gates pass,
+        not merely when adjudication rate falls.
+        """
+        result = EvaluationResult.create(
+            training_run_id=uuid4(),
+            model_version="gliner2-ft-fewer-adj",
+            entity_f1=0.80,
+            entity_f1_delta=-0.05,  # Regression!
+            event_f1_delta=-0.03,  # Regression!
+            calibration_ece=0.10,  # Too high!
+            adjudication_rate_before=0.40,
+            adjudication_rate_after=0.15,  # Great improvement
+            adjudication_rate_delta=-0.25,
+        )
+        # Despite great adjudication improvement, correctness fails
+        assert result.promotion_decision() == PromotionDecision.REJECT
+
+    def test_promote_when_all_gates_pass(self):
+        result = EvaluationResult.create(
+            training_run_id=uuid4(),
+            model_version="gliner2-ft-good",
+            entity_f1=0.94,
+            entity_f1_delta=0.02,
+            event_f1=0.88,
+            event_f1_delta=0.01,
+            calibration_ece=0.04,
+            adjudication_rate_delta=-0.10,
+        )
+        assert result.promotion_decision() == PromotionDecision.PROMOTE
+
+    def test_needs_review_on_adjudication_increase(self):
+        result = EvaluationResult.create(
+            training_run_id=uuid4(),
+            model_version="gliner2-ft-weird",
+            entity_f1=0.94,
+            entity_f1_delta=0.02,
+            event_f1_delta=0.01,
+            calibration_ece=0.04,
+            adjudication_rate_delta=0.10,  # Adjudication increased a lot
+        )
+        assert result.promotion_decision() == PromotionDecision.NEEDS_REVIEW
+
+
+class TestModelCard:
+    """Task 50: Model card with training metadata."""
+
+    def test_create_model_card(self):
+        card = ModelCard.create(
+            model_version="gliner2-ft-v1.0",
+            base_model="GLiNER2-large",
+            training_run_id=uuid4(),
+            training_range="2024-01 to 2024-06",
+            dataset_version="corpus-v1",
+        )
+        assert card.model_version == "gliner2-ft-v1.0"
+        assert card.base_model == "GLiNER2-large"
+        assert not card.promoted
+        assert not card.deprecated
+
+    def test_promote_and_deprecate(self):
+        card = ModelCard.create(
+            model_version="gliner2-ft-v1.0",
+            base_model="GLiNER2-large",
+            training_run_id=uuid4(),
+        )
+        card.promote()
+        assert card.promoted
+        assert card.promoted_at is not None
+        card.deprecate()
+        assert card.deprecated
+
+    def test_model_card_has_required_fields(self):
+        """Requirement 17.6: Model cards must include specific fields."""
+        card = ModelCard.create(
+            model_version="v1",
+            base_model="GLiNER2",
+            training_run_id=uuid4(),
+            training_range="2024-01 to 2024-06",
+            dataset_version="v1",
+            schema_version="1.0",
+            entity_types=["company", "event"],
+        )
+        d = card.to_dict()
+        assert "training_range" in d
+        assert "dataset_version" in d
+        assert "intended_use" in d
+        assert "limitations" in d
+        assert "entity_types" in d