feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
"""Promotion evaluator for NuExtract document-class decisions.
|
||||
|
||||
Determines whether NuExtract should be promoted for specific document
|
||||
classes based on incremental value gates. NuExtract is only promoted
|
||||
where it demonstrably beats GLiNER2 + deterministic parsing.
|
||||
|
||||
Requirement: 6.6
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from services.intelligence_pipeline_v3.nuextract.models import (
|
||||
IncrementalValueReport,
|
||||
PromotionGate,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PromotionEvaluator:
|
||||
"""Evaluates whether NuExtract should be promoted for a document class.
|
||||
|
||||
Uses the configured gate thresholds to make promotion decisions:
|
||||
- F1 improvement must exceed minimum threshold
|
||||
- Latency must stay within maximum bounds
|
||||
- Memory must stay within maximum bounds
|
||||
- Sample count must meet minimum for statistical confidence
|
||||
|
||||
Parameters
|
||||
----------
|
||||
gate
|
||||
Promotion gate thresholds.
|
||||
"""
|
||||
|
||||
def __init__(self, gate: PromotionGate | None = None) -> None:
|
||||
self._gate = gate or PromotionGate()
|
||||
|
||||
@property
|
||||
def gate(self) -> PromotionGate:
|
||||
"""Return the current promotion gate configuration."""
|
||||
return self._gate
|
||||
|
||||
def evaluate(self, report: IncrementalValueReport) -> bool:
|
||||
"""Evaluate whether NuExtract should be promoted for this document type.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
report
|
||||
Incremental value report for a specific document type.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
True if NuExtract passes all gate thresholds.
|
||||
"""
|
||||
reasons = self.get_rejection_reasons(report)
|
||||
promoted = len(reasons) == 0
|
||||
|
||||
if promoted:
|
||||
logger.info(
|
||||
"NuExtract PROMOTED for %s: delta=%.4f, latency=%.1fms, memory=%.1fMB",
|
||||
report.document_type,
|
||||
report.delta,
|
||||
report.nuextract_latency_ms,
|
||||
report.nuextract_memory_mb,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"NuExtract NOT promoted for %s: %s",
|
||||
report.document_type,
|
||||
"; ".join(reasons),
|
||||
)
|
||||
|
||||
return promoted
|
||||
|
||||
def get_rejection_reasons(self, report: IncrementalValueReport) -> list[str]:
|
||||
"""Return list of reasons why promotion would be rejected.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
report
|
||||
Incremental value report for a specific document type.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
Empty list if promotion passes; otherwise reasons for rejection.
|
||||
"""
|
||||
reasons: list[str] = []
|
||||
|
||||
# Check minimum sample count
|
||||
if report.sample_count < self._gate.min_sample_count:
|
||||
reasons.append(
|
||||
f"Insufficient samples: {report.sample_count} < {self._gate.min_sample_count}"
|
||||
)
|
||||
|
||||
# Check F1 improvement
|
||||
if report.delta < self._gate.min_f1_improvement:
|
||||
reasons.append(
|
||||
f"F1 improvement too small: {report.delta:.4f} < {self._gate.min_f1_improvement:.4f}"
|
||||
)
|
||||
|
||||
# Check latency
|
||||
if report.nuextract_latency_ms > self._gate.max_latency_ms:
|
||||
reasons.append(
|
||||
f"Latency exceeds gate: {report.nuextract_latency_ms:.1f}ms > {self._gate.max_latency_ms:.1f}ms"
|
||||
)
|
||||
|
||||
# Check memory
|
||||
if report.nuextract_memory_mb > self._gate.max_memory_mb:
|
||||
reasons.append(
|
||||
f"Memory exceeds gate: {report.nuextract_memory_mb:.1f}MB > {self._gate.max_memory_mb:.1f}MB"
|
||||
)
|
||||
|
||||
return reasons
|
||||
Reference in New Issue
Block a user