Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
185 lines
6.1 KiB
Python
185 lines
6.1 KiB
Python
"""Tests for fine-tuning module — Task 50."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from uuid import uuid4
|
|
|
|
from services.intelligence_pipeline_v3.fine_tuning.evaluation import (
|
|
EvaluationResult,
|
|
ModelCard,
|
|
PromotionDecision,
|
|
)
|
|
from services.intelligence_pipeline_v3.fine_tuning.trainer import (
|
|
TrainingConfig,
|
|
TrainingRun,
|
|
TrainingStatus,
|
|
)
|
|
|
|
|
|
class TestTrainingRun:
|
|
"""Task 50.1: Training pipeline."""
|
|
|
|
def test_create_training_run(self):
|
|
config = TrainingConfig(
|
|
base_model="GLiNER2-large",
|
|
schema_version="1.0",
|
|
dataset_version="v1",
|
|
)
|
|
run = TrainingRun.create(config)
|
|
assert run.status == TrainingStatus.PENDING
|
|
assert run.config.base_model == "GLiNER2-large"
|
|
|
|
def test_lifecycle(self):
|
|
config = TrainingConfig()
|
|
run = TrainingRun.create(config)
|
|
run.start()
|
|
assert run.status == TrainingStatus.PREPARING_DATA
|
|
assert run.started_at is not None
|
|
run.begin_training()
|
|
assert run.status == TrainingStatus.TRAINING
|
|
run.begin_evaluation()
|
|
assert run.status == TrainingStatus.EVALUATING
|
|
run.complete(
|
|
artifact_path="/models/gliner2-ft-v1",
|
|
model_version="gliner2-ft-v1.0",
|
|
train_loss=0.15,
|
|
validation_loss=0.20,
|
|
best_epoch=7,
|
|
)
|
|
assert run.status == TrainingStatus.COMPLETED
|
|
assert run.model_version == "gliner2-ft-v1.0"
|
|
assert run.duration_seconds is not None
|
|
|
|
def test_failure(self):
|
|
run = TrainingRun.create(TrainingConfig())
|
|
run.start()
|
|
run.fail("OOM error during training")
|
|
assert run.status == TrainingStatus.FAILED
|
|
assert "OOM" in run.errors[0]
|
|
|
|
|
|
class TestEvaluation:
|
|
"""Task 50.2: Holdout evaluation and promotion gates."""
|
|
|
|
def test_evaluation_passes_correctness_gates(self):
|
|
result = EvaluationResult.create(
|
|
training_run_id=uuid4(),
|
|
model_version="gliner2-ft-v1.0",
|
|
entity_f1=0.92,
|
|
event_f1=0.85,
|
|
entity_f1_delta=0.02,
|
|
event_f1_delta=0.01,
|
|
calibration_ece=0.05,
|
|
)
|
|
assert result.passes_correctness_gates()
|
|
|
|
def test_evaluation_fails_on_entity_regression(self):
|
|
result = EvaluationResult.create(
|
|
training_run_id=uuid4(),
|
|
model_version="gliner2-ft-bad",
|
|
entity_f1=0.80,
|
|
entity_f1_delta=-0.05, # Regression
|
|
calibration_ece=0.05,
|
|
)
|
|
assert not result.passes_correctness_gates()
|
|
|
|
def test_evaluation_fails_on_high_calibration(self):
|
|
result = EvaluationResult.create(
|
|
training_run_id=uuid4(),
|
|
model_version="gliner2-ft-uncalibrated",
|
|
entity_f1=0.95,
|
|
entity_f1_delta=0.05,
|
|
calibration_ece=0.15, # Too high
|
|
)
|
|
assert not result.passes_correctness_gates()
|
|
|
|
def test_promotion_not_based_on_adjudication_rate(self):
|
|
"""Task 50.4: Promoted only when correctness gates pass,
|
|
not merely when adjudication rate falls.
|
|
"""
|
|
result = EvaluationResult.create(
|
|
training_run_id=uuid4(),
|
|
model_version="gliner2-ft-fewer-adj",
|
|
entity_f1=0.80,
|
|
entity_f1_delta=-0.05, # Regression!
|
|
event_f1_delta=-0.03, # Regression!
|
|
calibration_ece=0.10, # Too high!
|
|
adjudication_rate_before=0.40,
|
|
adjudication_rate_after=0.15, # Great improvement
|
|
adjudication_rate_delta=-0.25,
|
|
)
|
|
# Despite great adjudication improvement, correctness fails
|
|
assert result.promotion_decision() == PromotionDecision.REJECT
|
|
|
|
def test_promote_when_all_gates_pass(self):
|
|
result = EvaluationResult.create(
|
|
training_run_id=uuid4(),
|
|
model_version="gliner2-ft-good",
|
|
entity_f1=0.94,
|
|
entity_f1_delta=0.02,
|
|
event_f1=0.88,
|
|
event_f1_delta=0.01,
|
|
calibration_ece=0.04,
|
|
adjudication_rate_delta=-0.10,
|
|
)
|
|
assert result.promotion_decision() == PromotionDecision.PROMOTE
|
|
|
|
def test_needs_review_on_adjudication_increase(self):
|
|
result = EvaluationResult.create(
|
|
training_run_id=uuid4(),
|
|
model_version="gliner2-ft-weird",
|
|
entity_f1=0.94,
|
|
entity_f1_delta=0.02,
|
|
event_f1_delta=0.01,
|
|
calibration_ece=0.04,
|
|
adjudication_rate_delta=0.10, # Adjudication increased a lot
|
|
)
|
|
assert result.promotion_decision() == PromotionDecision.NEEDS_REVIEW
|
|
|
|
|
|
class TestModelCard:
|
|
"""Task 50: Model card with training metadata."""
|
|
|
|
def test_create_model_card(self):
|
|
card = ModelCard.create(
|
|
model_version="gliner2-ft-v1.0",
|
|
base_model="GLiNER2-large",
|
|
training_run_id=uuid4(),
|
|
training_range="2024-01 to 2024-06",
|
|
dataset_version="corpus-v1",
|
|
)
|
|
assert card.model_version == "gliner2-ft-v1.0"
|
|
assert card.base_model == "GLiNER2-large"
|
|
assert not card.promoted
|
|
assert not card.deprecated
|
|
|
|
def test_promote_and_deprecate(self):
|
|
card = ModelCard.create(
|
|
model_version="gliner2-ft-v1.0",
|
|
base_model="GLiNER2-large",
|
|
training_run_id=uuid4(),
|
|
)
|
|
card.promote()
|
|
assert card.promoted
|
|
assert card.promoted_at is not None
|
|
card.deprecate()
|
|
assert card.deprecated
|
|
|
|
def test_model_card_has_required_fields(self):
|
|
"""Requirement 17.6: Model cards must include specific fields."""
|
|
card = ModelCard.create(
|
|
model_version="v1",
|
|
base_model="GLiNER2",
|
|
training_run_id=uuid4(),
|
|
training_range="2024-01 to 2024-06",
|
|
dataset_version="v1",
|
|
schema_version="1.0",
|
|
entity_types=["company", "event"],
|
|
)
|
|
d = card.to_dict()
|
|
assert "training_range" in d
|
|
assert "dataset_version" in d
|
|
assert "intended_use" in d
|
|
assert "limitations" in d
|
|
assert "entity_types" in d
|