Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
105 lines
3.1 KiB
Python
105 lines
3.1 KiB
Python
"""Pydantic models for NuExtract benchmark and evaluation.
|
|
|
|
Defines structured result types, incremental value reporting,
|
|
and promotion gate thresholds.
|
|
|
|
Requirement: 6.6
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Literal
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
class ExtractedField(BaseModel):
|
|
"""A single field extracted by NuExtract."""
|
|
|
|
name: str
|
|
value: Any
|
|
start_char: int | None = None
|
|
end_char: int | None = None
|
|
confidence: float = 0.0
|
|
|
|
|
|
class NuExtractResult(BaseModel):
|
|
"""Result from NuExtract 1.5 Smol extraction.
|
|
|
|
Contains extracted fields with spans, confidence scores,
|
|
model lineage, and latency tracking.
|
|
"""
|
|
|
|
fields: list[ExtractedField] = Field(default_factory=list)
|
|
spans: list[dict[str, Any]] = Field(default_factory=list)
|
|
confidence: float = 0.0
|
|
model_version: str = "numind/NuExtract-1.5-smol"
|
|
latency_ms: float = 0.0
|
|
memory_mb: float = 0.0
|
|
document_type: str = ""
|
|
schema_used: dict[str, Any] = Field(default_factory=dict)
|
|
error: str | None = None
|
|
|
|
|
|
class IncrementalValueReport(BaseModel):
|
|
"""Report comparing NuExtract vs GLiNER2 + deterministic parsing per document type.
|
|
|
|
Tracks F1 scores for both approaches and computes the delta
|
|
to determine if NuExtract adds incremental value.
|
|
"""
|
|
|
|
document_type: Literal["filing", "transcript", "article", "press_release", "macro_event"]
|
|
gliner_f1: float = Field(ge=0.0, le=1.0)
|
|
nuextract_f1: float = Field(ge=0.0, le=1.0)
|
|
delta: float = Field(
|
|
description="nuextract_f1 - gliner_f1; positive means NuExtract is better"
|
|
)
|
|
nuextract_latency_ms: float = 0.0
|
|
nuextract_memory_mb: float = 0.0
|
|
gliner_latency_ms: float = 0.0
|
|
gliner_memory_mb: float = 0.0
|
|
sample_count: int = 0
|
|
promoted: bool = False
|
|
|
|
|
|
class PromotionGate(BaseModel):
|
|
"""Gate thresholds for promoting NuExtract for a document class.
|
|
|
|
NuExtract is only promoted for document classes where it beats
|
|
GLiNER2 + deterministic parsing by the configured minimums AND
|
|
stays within resource bounds.
|
|
"""
|
|
|
|
min_f1_improvement: float = Field(
|
|
default=0.05,
|
|
ge=0.0,
|
|
le=1.0,
|
|
description="Minimum F1 delta required for promotion",
|
|
)
|
|
max_latency_ms: float = Field(
|
|
default=5000.0,
|
|
gt=0.0,
|
|
description="Maximum acceptable p95 latency in milliseconds",
|
|
)
|
|
max_memory_mb: float = Field(
|
|
default=2048.0,
|
|
gt=0.0,
|
|
description="Maximum acceptable peak memory usage in MB",
|
|
)
|
|
min_sample_count: int = Field(
|
|
default=50,
|
|
ge=1,
|
|
description="Minimum sample count required for statistical confidence",
|
|
)
|
|
|
|
|
|
class BenchmarkReport(BaseModel):
|
|
"""Full benchmark report across all evaluated document types."""
|
|
|
|
reports: list[IncrementalValueReport] = Field(default_factory=list)
|
|
gate: PromotionGate = Field(default_factory=PromotionGate)
|
|
promoted_types: list[str] = Field(default_factory=list)
|
|
overall_nuextract_f1: float = 0.0
|
|
overall_gliner_f1: float = 0.0
|
|
overall_delta: float = 0.0
|
|
total_documents: int = 0
|