Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
117 lines
3.5 KiB
Python
117 lines
3.5 KiB
Python
"""Promotion evaluator for NuExtract document-class decisions.
|
|
|
|
Determines whether NuExtract should be promoted for specific document
|
|
classes based on incremental value gates. NuExtract is only promoted
|
|
where it demonstrably beats GLiNER2 + deterministic parsing.
|
|
|
|
Requirement: 6.6
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
from services.intelligence_pipeline_v3.nuextract.models import (
|
|
IncrementalValueReport,
|
|
PromotionGate,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class PromotionEvaluator:
|
|
"""Evaluates whether NuExtract should be promoted for a document class.
|
|
|
|
Uses the configured gate thresholds to make promotion decisions:
|
|
- F1 improvement must exceed minimum threshold
|
|
- Latency must stay within maximum bounds
|
|
- Memory must stay within maximum bounds
|
|
- Sample count must meet minimum for statistical confidence
|
|
|
|
Parameters
|
|
----------
|
|
gate
|
|
Promotion gate thresholds.
|
|
"""
|
|
|
|
def __init__(self, gate: PromotionGate | None = None) -> None:
|
|
self._gate = gate or PromotionGate()
|
|
|
|
@property
|
|
def gate(self) -> PromotionGate:
|
|
"""Return the current promotion gate configuration."""
|
|
return self._gate
|
|
|
|
def evaluate(self, report: IncrementalValueReport) -> bool:
|
|
"""Evaluate whether NuExtract should be promoted for this document type.
|
|
|
|
Parameters
|
|
----------
|
|
report
|
|
Incremental value report for a specific document type.
|
|
|
|
Returns
|
|
-------
|
|
bool
|
|
True if NuExtract passes all gate thresholds.
|
|
"""
|
|
reasons = self.get_rejection_reasons(report)
|
|
promoted = len(reasons) == 0
|
|
|
|
if promoted:
|
|
logger.info(
|
|
"NuExtract PROMOTED for %s: delta=%.4f, latency=%.1fms, memory=%.1fMB",
|
|
report.document_type,
|
|
report.delta,
|
|
report.nuextract_latency_ms,
|
|
report.nuextract_memory_mb,
|
|
)
|
|
else:
|
|
logger.info(
|
|
"NuExtract NOT promoted for %s: %s",
|
|
report.document_type,
|
|
"; ".join(reasons),
|
|
)
|
|
|
|
return promoted
|
|
|
|
def get_rejection_reasons(self, report: IncrementalValueReport) -> list[str]:
|
|
"""Return list of reasons why promotion would be rejected.
|
|
|
|
Parameters
|
|
----------
|
|
report
|
|
Incremental value report for a specific document type.
|
|
|
|
Returns
|
|
-------
|
|
list[str]
|
|
Empty list if promotion passes; otherwise reasons for rejection.
|
|
"""
|
|
reasons: list[str] = []
|
|
|
|
# Check minimum sample count
|
|
if report.sample_count < self._gate.min_sample_count:
|
|
reasons.append(
|
|
f"Insufficient samples: {report.sample_count} < {self._gate.min_sample_count}"
|
|
)
|
|
|
|
# Check F1 improvement
|
|
if report.delta < self._gate.min_f1_improvement:
|
|
reasons.append(
|
|
f"F1 improvement too small: {report.delta:.4f} < {self._gate.min_f1_improvement:.4f}"
|
|
)
|
|
|
|
# Check latency
|
|
if report.nuextract_latency_ms > self._gate.max_latency_ms:
|
|
reasons.append(
|
|
f"Latency exceeds gate: {report.nuextract_latency_ms:.1f}ms > {self._gate.max_latency_ms:.1f}ms"
|
|
)
|
|
|
|
# Check memory
|
|
if report.nuextract_memory_mb > self._gate.max_memory_mb:
|
|
reasons.append(
|
|
f"Memory exceeds gate: {report.nuextract_memory_mb:.1f}MB > {self._gate.max_memory_mb:.1f}MB"
|
|
)
|
|
|
|
return reasons
|