feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,26 @@
|
||||
"""NuExtract 1.5 Smol benchmark and adapter package.
|
||||
|
||||
Evaluates NuExtract 1.5 Smol as an optional long-form or hierarchical
|
||||
fact-extraction stage. It is NOT an always-resident GPU model — deployment
|
||||
is CPU/on-demand only.
|
||||
|
||||
Requirement: 6.6
|
||||
"""
|
||||
|
||||
from services.intelligence_pipeline_v3.nuextract.adapter import NuExtractAdapter
|
||||
from services.intelligence_pipeline_v3.nuextract.benchmark import NuExtractBenchmark
|
||||
from services.intelligence_pipeline_v3.nuextract.models import (
|
||||
IncrementalValueReport,
|
||||
NuExtractResult,
|
||||
PromotionGate,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.nuextract.promotion import PromotionEvaluator
|
||||
|
||||
__all__ = [
|
||||
"NuExtractAdapter",
|
||||
"NuExtractBenchmark",
|
||||
"NuExtractResult",
|
||||
"IncrementalValueReport",
|
||||
"PromotionGate",
|
||||
"PromotionEvaluator",
|
||||
]
|
||||
@@ -0,0 +1,324 @@
|
||||
"""NuExtract 1.5 Smol adapter for on-demand CPU extraction.
|
||||
|
||||
Provides an isolated interface for NuExtract inference:
|
||||
- Production mode: loads numind/NuExtract-1.5-smol on CPU
|
||||
- Test mode: deterministic schema-based mock extraction
|
||||
|
||||
This adapter uses the shared InferenceGateway with a configurable
|
||||
target for CPU-only deployment. It is NOT always-resident — loaded
|
||||
on demand for benchmark or promoted document classes only.
|
||||
|
||||
Requirement: 6.6
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from services.intelligence_pipeline_v3.nuextract.models import (
|
||||
ExtractedField,
|
||||
NuExtractResult,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Pinned model configuration
|
||||
NUEXTRACT_MODEL_NAME = "numind/NuExtract-1.5-smol"
|
||||
NUEXTRACT_MODEL_VERSION = "numind/NuExtract-1.5-smol@v1.5"
|
||||
|
||||
|
||||
class NuExtractAdapter:
|
||||
"""Adapter for NuExtract 1.5 Smol hierarchical extraction.
|
||||
|
||||
Designed for CPU/on-demand use, not always-resident GPU deployment.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
test_mode
|
||||
When True, uses deterministic schema-based extraction rather than
|
||||
loading the model. Useful for testing without model dependencies.
|
||||
max_length
|
||||
Maximum input text length in characters. Longer texts are processed
|
||||
in segments.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
test_mode: bool = True,
|
||||
max_length: int = 16_000,
|
||||
) -> None:
|
||||
self._test_mode = test_mode
|
||||
self._max_length = max_length
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._model_name = NUEXTRACT_MODEL_NAME
|
||||
self._model_version = NUEXTRACT_MODEL_VERSION
|
||||
self._loaded = False
|
||||
|
||||
if not test_mode:
|
||||
self._load_model()
|
||||
|
||||
@property
|
||||
def model_version(self) -> str:
|
||||
"""Return the pinned model version string."""
|
||||
return self._model_version
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
"""Return the model name."""
|
||||
return self._model_name
|
||||
|
||||
@property
|
||||
def is_loaded(self) -> bool:
|
||||
"""Return whether the model is currently loaded."""
|
||||
return self._loaded
|
||||
|
||||
def _load_model(self) -> None:
|
||||
"""Load the NuExtract model and tokenizer for CPU inference."""
|
||||
try:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
logger.info("Loading NuExtract model (CPU): %s", self._model_name)
|
||||
self._tokenizer = AutoTokenizer.from_pretrained(self._model_name)
|
||||
self._model = AutoModelForCausalLM.from_pretrained(
|
||||
self._model_name,
|
||||
device_map="cpu",
|
||||
torch_dtype="auto",
|
||||
)
|
||||
self._model.eval()
|
||||
self._loaded = True
|
||||
logger.info("NuExtract model loaded successfully on CPU")
|
||||
except ImportError:
|
||||
raise RuntimeError(
|
||||
"transformers and torch are required for NuExtract inference. "
|
||||
"Install with: pip install transformers torch"
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to load NuExtract model: {e}") from e
|
||||
|
||||
def unload(self) -> None:
|
||||
"""Unload model to free memory (on-demand lifecycle)."""
|
||||
if self._model is not None:
|
||||
del self._model
|
||||
del self._tokenizer
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._loaded = False
|
||||
logger.info("NuExtract model unloaded")
|
||||
|
||||
async def extract(
|
||||
self,
|
||||
text: str,
|
||||
schema: dict[str, Any],
|
||||
document_type: str = "",
|
||||
) -> NuExtractResult:
|
||||
"""Extract structured fields from text using the given schema.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
text
|
||||
Source document text.
|
||||
schema
|
||||
JSON schema defining the fields to extract.
|
||||
document_type
|
||||
Document type identifier for reporting.
|
||||
|
||||
Returns
|
||||
-------
|
||||
NuExtractResult
|
||||
Extracted fields with confidence, latency, and memory metrics.
|
||||
"""
|
||||
start_time = time.perf_counter()
|
||||
|
||||
try:
|
||||
if self._test_mode:
|
||||
fields = self._extract_test_mode(text, schema)
|
||||
else:
|
||||
fields = self._extract_production(text, schema)
|
||||
|
||||
latency_ms = (time.perf_counter() - start_time) * 1000
|
||||
memory_mb = self._estimate_memory()
|
||||
|
||||
# Compute overall confidence as mean of field confidences
|
||||
confidence = 0.0
|
||||
if fields:
|
||||
confidence = sum(f.confidence for f in fields) / len(fields)
|
||||
|
||||
return NuExtractResult(
|
||||
fields=fields,
|
||||
spans=[
|
||||
{"start": f.start_char, "end": f.end_char, "field": f.name}
|
||||
for f in fields
|
||||
if f.start_char is not None
|
||||
],
|
||||
confidence=confidence,
|
||||
model_version=self._model_version,
|
||||
latency_ms=latency_ms,
|
||||
memory_mb=memory_mb,
|
||||
document_type=document_type,
|
||||
schema_used=schema,
|
||||
)
|
||||
except Exception as e:
|
||||
latency_ms = (time.perf_counter() - start_time) * 1000
|
||||
logger.error("NuExtract extraction failed: %s", e)
|
||||
return NuExtractResult(
|
||||
model_version=self._model_version,
|
||||
latency_ms=latency_ms,
|
||||
document_type=document_type,
|
||||
schema_used=schema,
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
def _extract_test_mode(
|
||||
self, text: str, schema: dict[str, Any]
|
||||
) -> list[ExtractedField]:
|
||||
"""Deterministic extraction for testing.
|
||||
|
||||
Matches schema field names against text content using simple
|
||||
pattern matching to simulate extraction behavior.
|
||||
"""
|
||||
fields: list[ExtractedField] = []
|
||||
properties = schema.get("properties", schema)
|
||||
|
||||
for field_name, field_spec in properties.items():
|
||||
# Simple pattern: look for the field name or related keywords in text
|
||||
pattern = re.compile(
|
||||
rf"\b{re.escape(field_name.replace('_', ' '))}[:\s]+([^\n.;]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
match = pattern.search(text)
|
||||
|
||||
if match:
|
||||
value = match.group(1).strip()
|
||||
fields.append(
|
||||
ExtractedField(
|
||||
name=field_name,
|
||||
value=value,
|
||||
start_char=match.start(1),
|
||||
end_char=match.end(1),
|
||||
confidence=0.85,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Try extracting from nearby context for hierarchical schemas
|
||||
if isinstance(field_spec, dict) and "properties" in field_spec:
|
||||
# Nested schema — attempt hierarchical extraction
|
||||
nested = self._extract_test_mode(text, field_spec)
|
||||
if nested:
|
||||
fields.append(
|
||||
ExtractedField(
|
||||
name=field_name,
|
||||
value={f.name: f.value for f in nested},
|
||||
confidence=sum(f.confidence for f in nested) / len(nested),
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Field not found in text
|
||||
fields.append(
|
||||
ExtractedField(
|
||||
name=field_name,
|
||||
value=None,
|
||||
confidence=0.0,
|
||||
)
|
||||
)
|
||||
|
||||
return fields
|
||||
|
||||
def _extract_production(
|
||||
self, text: str, schema: dict[str, Any]
|
||||
) -> list[ExtractedField]:
|
||||
"""Run NuExtract inference on text using the loaded model."""
|
||||
import json
|
||||
|
||||
import torch
|
||||
|
||||
if self._model is None or self._tokenizer is None:
|
||||
raise RuntimeError("Model not loaded. Initialize with test_mode=False.")
|
||||
|
||||
# Format input in NuExtract's expected format
|
||||
schema_str = json.dumps(schema, indent=2)
|
||||
prompt = f"<|input|>\n### Template:\n{schema_str}\n### Text:\n{text[:self._max_length]}\n<|output|>\n"
|
||||
|
||||
inputs = self._tokenizer(
|
||||
prompt,
|
||||
return_tensors="pt",
|
||||
truncation=True,
|
||||
max_length=4096,
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
outputs = self._model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=1024,
|
||||
temperature=0.0,
|
||||
do_sample=False,
|
||||
)
|
||||
|
||||
# Decode and parse the output
|
||||
generated = self._tokenizer.decode(
|
||||
outputs[0][inputs["input_ids"].shape[1]:],
|
||||
skip_special_tokens=True,
|
||||
)
|
||||
|
||||
return self._parse_output(generated, text, schema)
|
||||
|
||||
def _parse_output(
|
||||
self, output: str, source_text: str, schema: dict[str, Any]
|
||||
) -> list[ExtractedField]:
|
||||
"""Parse model output into structured fields with spans."""
|
||||
import json
|
||||
|
||||
fields: list[ExtractedField] = []
|
||||
|
||||
try:
|
||||
parsed = json.loads(output)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Failed to parse NuExtract output as JSON")
|
||||
return fields
|
||||
|
||||
properties = schema.get("properties", schema)
|
||||
for field_name in properties:
|
||||
if field_name in parsed:
|
||||
value = parsed[field_name]
|
||||
# Try to find the value in source text for span
|
||||
start_char = None
|
||||
end_char = None
|
||||
if isinstance(value, str) and value:
|
||||
idx = source_text.find(value)
|
||||
if idx >= 0:
|
||||
start_char = idx
|
||||
end_char = idx + len(value)
|
||||
|
||||
fields.append(
|
||||
ExtractedField(
|
||||
name=field_name,
|
||||
value=value,
|
||||
start_char=start_char,
|
||||
end_char=end_char,
|
||||
confidence=0.8,
|
||||
)
|
||||
)
|
||||
|
||||
return fields
|
||||
|
||||
def _estimate_memory(self) -> float:
|
||||
"""Estimate current memory usage in MB."""
|
||||
if self._test_mode:
|
||||
return 0.0
|
||||
|
||||
try:
|
||||
import torch
|
||||
|
||||
if torch.cuda.is_available():
|
||||
return torch.cuda.memory_allocated() / (1024 * 1024)
|
||||
# For CPU, estimate from model parameters
|
||||
if self._model is not None:
|
||||
param_bytes = sum(
|
||||
p.nelement() * p.element_size() for p in self._model.parameters()
|
||||
)
|
||||
return param_bytes / (1024 * 1024)
|
||||
except Exception:
|
||||
pass
|
||||
return 0.0
|
||||
@@ -0,0 +1,277 @@
|
||||
"""NuExtract benchmark comparing against GLiNER2 + deterministic parsing.
|
||||
|
||||
Evaluates NuExtract 1.5 Smol on hierarchical extraction for long filings
|
||||
and transcripts, measuring incremental correctness, CPU latency, and memory.
|
||||
|
||||
Reports per-document-type incremental value to determine which document
|
||||
classes benefit from NuExtract supplementation.
|
||||
|
||||
Requirement: 6.6
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
|
||||
from services.intelligence_pipeline_v3.nuextract.adapter import NuExtractAdapter
|
||||
from services.intelligence_pipeline_v3.nuextract.models import (
|
||||
BenchmarkReport,
|
||||
IncrementalValueReport,
|
||||
NuExtractResult,
|
||||
PromotionGate,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.nuextract.promotion import PromotionEvaluator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GoldDocument:
|
||||
"""A document from the gold corpus with ground-truth labels."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text: str,
|
||||
document_type: str,
|
||||
gold_fields: dict[str, Any],
|
||||
schema: dict[str, Any],
|
||||
document_id: str = "",
|
||||
) -> None:
|
||||
self.text = text
|
||||
self.document_type = document_type
|
||||
self.gold_fields = gold_fields
|
||||
self.schema = schema
|
||||
self.document_id = document_id
|
||||
|
||||
|
||||
class GLiNERResult:
|
||||
"""Simulated result from GLiNER2 + deterministic parsing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fields: dict[str, Any],
|
||||
latency_ms: float = 0.0,
|
||||
memory_mb: float = 0.0,
|
||||
) -> None:
|
||||
self.fields = fields
|
||||
self.latency_ms = latency_ms
|
||||
self.memory_mb = memory_mb
|
||||
|
||||
|
||||
class NuExtractBenchmark:
|
||||
"""Benchmark comparing NuExtract vs GLiNER2 + deterministic parsing.
|
||||
|
||||
Evaluates per-document-type to determine where NuExtract adds value.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
adapter
|
||||
NuExtractAdapter instance (test_mode or production).
|
||||
gate
|
||||
Promotion gate thresholds for deciding promotion.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
adapter: NuExtractAdapter | None = None,
|
||||
gate: PromotionGate | None = None,
|
||||
) -> None:
|
||||
self._adapter = adapter or NuExtractAdapter(test_mode=True)
|
||||
self._gate = gate or PromotionGate()
|
||||
self._evaluator = PromotionEvaluator(self._gate)
|
||||
|
||||
async def evaluate_against_gliner(
|
||||
self,
|
||||
documents: list[GoldDocument],
|
||||
gliner_results: list[GLiNERResult],
|
||||
) -> BenchmarkReport:
|
||||
"""Run full benchmark comparing NuExtract vs GLiNER2 + deterministic parsing.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
documents
|
||||
Gold corpus documents with ground-truth labels.
|
||||
gliner_results
|
||||
Pre-computed GLiNER2 + deterministic parsing results for each document.
|
||||
|
||||
Returns
|
||||
-------
|
||||
BenchmarkReport
|
||||
Full benchmark report with per-type results and promotion decisions.
|
||||
"""
|
||||
if len(documents) != len(gliner_results):
|
||||
raise ValueError(
|
||||
f"Document count ({len(documents)}) must match "
|
||||
f"GLiNER result count ({len(gliner_results)})"
|
||||
)
|
||||
|
||||
# Group by document type
|
||||
by_type: dict[str, list[tuple[GoldDocument, GLiNERResult]]] = defaultdict(list)
|
||||
for doc, gliner in zip(documents, gliner_results):
|
||||
by_type[doc.document_type].append((doc, gliner))
|
||||
|
||||
# Evaluate each document type
|
||||
reports: list[IncrementalValueReport] = []
|
||||
for doc_type, pairs in by_type.items():
|
||||
report = await self._evaluate_type(doc_type, pairs)
|
||||
reports.append(report)
|
||||
|
||||
# Determine promotions
|
||||
promoted_types: list[str] = []
|
||||
for report in reports:
|
||||
if self._evaluator.evaluate(report):
|
||||
report.promoted = True
|
||||
promoted_types.append(report.document_type)
|
||||
|
||||
# Compute overall metrics
|
||||
total_docs = len(documents)
|
||||
overall_nuextract_f1 = 0.0
|
||||
overall_gliner_f1 = 0.0
|
||||
if reports:
|
||||
weighted_nu = sum(r.nuextract_f1 * r.sample_count for r in reports)
|
||||
weighted_gl = sum(r.gliner_f1 * r.sample_count for r in reports)
|
||||
overall_nuextract_f1 = weighted_nu / total_docs if total_docs > 0 else 0.0
|
||||
overall_gliner_f1 = weighted_gl / total_docs if total_docs > 0 else 0.0
|
||||
|
||||
return BenchmarkReport(
|
||||
reports=reports,
|
||||
gate=self._gate,
|
||||
promoted_types=promoted_types,
|
||||
overall_nuextract_f1=overall_nuextract_f1,
|
||||
overall_gliner_f1=overall_gliner_f1,
|
||||
overall_delta=overall_nuextract_f1 - overall_gliner_f1,
|
||||
total_documents=total_docs,
|
||||
)
|
||||
|
||||
async def _evaluate_type(
|
||||
self,
|
||||
doc_type: str,
|
||||
pairs: list[tuple[GoldDocument, GLiNERResult]],
|
||||
) -> IncrementalValueReport:
|
||||
"""Evaluate NuExtract vs GLiNER for a single document type."""
|
||||
nuextract_scores: list[float] = []
|
||||
gliner_scores: list[float] = []
|
||||
nuextract_latencies: list[float] = []
|
||||
nuextract_memories: list[float] = []
|
||||
gliner_latencies: list[float] = []
|
||||
gliner_memories: list[float] = []
|
||||
|
||||
for doc, gliner_result in pairs:
|
||||
# Run NuExtract extraction
|
||||
nu_result = await self._adapter.extract(
|
||||
text=doc.text,
|
||||
schema=doc.schema,
|
||||
document_type=doc.document_type,
|
||||
)
|
||||
|
||||
# Compute F1 for NuExtract
|
||||
nu_f1 = self._compute_field_f1(nu_result, doc.gold_fields)
|
||||
nuextract_scores.append(nu_f1)
|
||||
nuextract_latencies.append(nu_result.latency_ms)
|
||||
nuextract_memories.append(nu_result.memory_mb)
|
||||
|
||||
# Compute F1 for GLiNER
|
||||
gl_f1 = self._compute_extraction_f1(gliner_result.fields, doc.gold_fields)
|
||||
gliner_scores.append(gl_f1)
|
||||
gliner_latencies.append(gliner_result.latency_ms)
|
||||
gliner_memories.append(gliner_result.memory_mb)
|
||||
|
||||
# Aggregate metrics
|
||||
n = len(pairs)
|
||||
avg_nu_f1 = sum(nuextract_scores) / n if n > 0 else 0.0
|
||||
avg_gl_f1 = sum(gliner_scores) / n if n > 0 else 0.0
|
||||
p95_nu_latency = _percentile(nuextract_latencies, 95)
|
||||
p95_gl_latency = _percentile(gliner_latencies, 95)
|
||||
max_nu_memory = max(nuextract_memories) if nuextract_memories else 0.0
|
||||
max_gl_memory = max(gliner_memories) if gliner_memories else 0.0
|
||||
|
||||
return IncrementalValueReport(
|
||||
document_type=doc_type,
|
||||
gliner_f1=avg_gl_f1,
|
||||
nuextract_f1=avg_nu_f1,
|
||||
delta=avg_nu_f1 - avg_gl_f1,
|
||||
nuextract_latency_ms=p95_nu_latency,
|
||||
nuextract_memory_mb=max_nu_memory,
|
||||
gliner_latency_ms=p95_gl_latency,
|
||||
gliner_memory_mb=max_gl_memory,
|
||||
sample_count=n,
|
||||
promoted=False,
|
||||
)
|
||||
|
||||
def _compute_field_f1(
|
||||
self, result: NuExtractResult, gold: dict[str, Any]
|
||||
) -> float:
|
||||
"""Compute F1 score for NuExtract result against gold labels."""
|
||||
if not gold:
|
||||
return 1.0 if not result.fields else 0.0
|
||||
|
||||
extracted_fields = {
|
||||
f.name: f.value for f in result.fields if f.value is not None
|
||||
}
|
||||
return self._compute_extraction_f1(extracted_fields, gold)
|
||||
|
||||
def _compute_extraction_f1(
|
||||
self, predicted: dict[str, Any], gold: dict[str, Any]
|
||||
) -> float:
|
||||
"""Compute field-level F1 between predicted and gold extractions."""
|
||||
if not gold and not predicted:
|
||||
return 1.0
|
||||
if not gold or not predicted:
|
||||
return 0.0
|
||||
|
||||
gold_set = set(gold.keys())
|
||||
pred_set = set(predicted.keys())
|
||||
|
||||
# True positives: predicted fields that match gold (key present AND value matches)
|
||||
tp = 0
|
||||
for key in gold_set & pred_set:
|
||||
if self._values_match(predicted[key], gold[key]):
|
||||
tp += 1
|
||||
|
||||
precision = tp / len(pred_set) if pred_set else 0.0
|
||||
recall = tp / len(gold_set) if gold_set else 0.0
|
||||
|
||||
if precision + recall == 0:
|
||||
return 0.0
|
||||
return 2 * precision * recall / (precision + recall)
|
||||
|
||||
def _values_match(self, predicted: Any, gold: Any) -> bool:
|
||||
"""Check if a predicted value matches gold (with tolerance)."""
|
||||
if predicted is None:
|
||||
return gold is None
|
||||
if gold is None:
|
||||
return False
|
||||
|
||||
# String comparison (case-insensitive, trimmed)
|
||||
if isinstance(gold, str) and isinstance(predicted, str):
|
||||
return predicted.strip().lower() == gold.strip().lower()
|
||||
|
||||
# Numeric comparison with tolerance
|
||||
if isinstance(gold, (int, float)) and isinstance(predicted, (int, float)):
|
||||
if gold == 0:
|
||||
return abs(predicted) < 1e-6
|
||||
return abs(predicted - gold) / abs(gold) < 0.05
|
||||
|
||||
# Dict comparison (recursive for hierarchical)
|
||||
if isinstance(gold, dict) and isinstance(predicted, dict):
|
||||
if not gold:
|
||||
return not predicted
|
||||
matches = sum(
|
||||
1
|
||||
for k in gold
|
||||
if k in predicted and self._values_match(predicted[k], gold[k])
|
||||
)
|
||||
return matches / len(gold) >= 0.5
|
||||
|
||||
# Fallback: equality
|
||||
return predicted == gold
|
||||
|
||||
|
||||
def _percentile(values: list[float], pct: int) -> float:
|
||||
"""Compute a percentile from a list of values."""
|
||||
if not values:
|
||||
return 0.0
|
||||
sorted_vals = sorted(values)
|
||||
idx = int(len(sorted_vals) * pct / 100)
|
||||
idx = min(idx, len(sorted_vals) - 1)
|
||||
return sorted_vals[idx]
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Pydantic models for NuExtract benchmark and evaluation.
|
||||
|
||||
Defines structured result types, incremental value reporting,
|
||||
and promotion gate thresholds.
|
||||
|
||||
Requirement: 6.6
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ExtractedField(BaseModel):
|
||||
"""A single field extracted by NuExtract."""
|
||||
|
||||
name: str
|
||||
value: Any
|
||||
start_char: int | None = None
|
||||
end_char: int | None = None
|
||||
confidence: float = 0.0
|
||||
|
||||
|
||||
class NuExtractResult(BaseModel):
|
||||
"""Result from NuExtract 1.5 Smol extraction.
|
||||
|
||||
Contains extracted fields with spans, confidence scores,
|
||||
model lineage, and latency tracking.
|
||||
"""
|
||||
|
||||
fields: list[ExtractedField] = Field(default_factory=list)
|
||||
spans: list[dict[str, Any]] = Field(default_factory=list)
|
||||
confidence: float = 0.0
|
||||
model_version: str = "numind/NuExtract-1.5-smol"
|
||||
latency_ms: float = 0.0
|
||||
memory_mb: float = 0.0
|
||||
document_type: str = ""
|
||||
schema_used: dict[str, Any] = Field(default_factory=dict)
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class IncrementalValueReport(BaseModel):
|
||||
"""Report comparing NuExtract vs GLiNER2 + deterministic parsing per document type.
|
||||
|
||||
Tracks F1 scores for both approaches and computes the delta
|
||||
to determine if NuExtract adds incremental value.
|
||||
"""
|
||||
|
||||
document_type: Literal["filing", "transcript", "article", "press_release", "macro_event"]
|
||||
gliner_f1: float = Field(ge=0.0, le=1.0)
|
||||
nuextract_f1: float = Field(ge=0.0, le=1.0)
|
||||
delta: float = Field(
|
||||
description="nuextract_f1 - gliner_f1; positive means NuExtract is better"
|
||||
)
|
||||
nuextract_latency_ms: float = 0.0
|
||||
nuextract_memory_mb: float = 0.0
|
||||
gliner_latency_ms: float = 0.0
|
||||
gliner_memory_mb: float = 0.0
|
||||
sample_count: int = 0
|
||||
promoted: bool = False
|
||||
|
||||
|
||||
class PromotionGate(BaseModel):
|
||||
"""Gate thresholds for promoting NuExtract for a document class.
|
||||
|
||||
NuExtract is only promoted for document classes where it beats
|
||||
GLiNER2 + deterministic parsing by the configured minimums AND
|
||||
stays within resource bounds.
|
||||
"""
|
||||
|
||||
min_f1_improvement: float = Field(
|
||||
default=0.05,
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
description="Minimum F1 delta required for promotion",
|
||||
)
|
||||
max_latency_ms: float = Field(
|
||||
default=5000.0,
|
||||
gt=0.0,
|
||||
description="Maximum acceptable p95 latency in milliseconds",
|
||||
)
|
||||
max_memory_mb: float = Field(
|
||||
default=2048.0,
|
||||
gt=0.0,
|
||||
description="Maximum acceptable peak memory usage in MB",
|
||||
)
|
||||
min_sample_count: int = Field(
|
||||
default=50,
|
||||
ge=1,
|
||||
description="Minimum sample count required for statistical confidence",
|
||||
)
|
||||
|
||||
|
||||
class BenchmarkReport(BaseModel):
|
||||
"""Full benchmark report across all evaluated document types."""
|
||||
|
||||
reports: list[IncrementalValueReport] = Field(default_factory=list)
|
||||
gate: PromotionGate = Field(default_factory=PromotionGate)
|
||||
promoted_types: list[str] = Field(default_factory=list)
|
||||
overall_nuextract_f1: float = 0.0
|
||||
overall_gliner_f1: float = 0.0
|
||||
overall_delta: float = 0.0
|
||||
total_documents: int = 0
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Promotion evaluator for NuExtract document-class decisions.
|
||||
|
||||
Determines whether NuExtract should be promoted for specific document
|
||||
classes based on incremental value gates. NuExtract is only promoted
|
||||
where it demonstrably beats GLiNER2 + deterministic parsing.
|
||||
|
||||
Requirement: 6.6
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from services.intelligence_pipeline_v3.nuextract.models import (
|
||||
IncrementalValueReport,
|
||||
PromotionGate,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PromotionEvaluator:
|
||||
"""Evaluates whether NuExtract should be promoted for a document class.
|
||||
|
||||
Uses the configured gate thresholds to make promotion decisions:
|
||||
- F1 improvement must exceed minimum threshold
|
||||
- Latency must stay within maximum bounds
|
||||
- Memory must stay within maximum bounds
|
||||
- Sample count must meet minimum for statistical confidence
|
||||
|
||||
Parameters
|
||||
----------
|
||||
gate
|
||||
Promotion gate thresholds.
|
||||
"""
|
||||
|
||||
def __init__(self, gate: PromotionGate | None = None) -> None:
|
||||
self._gate = gate or PromotionGate()
|
||||
|
||||
@property
|
||||
def gate(self) -> PromotionGate:
|
||||
"""Return the current promotion gate configuration."""
|
||||
return self._gate
|
||||
|
||||
def evaluate(self, report: IncrementalValueReport) -> bool:
|
||||
"""Evaluate whether NuExtract should be promoted for this document type.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
report
|
||||
Incremental value report for a specific document type.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
True if NuExtract passes all gate thresholds.
|
||||
"""
|
||||
reasons = self.get_rejection_reasons(report)
|
||||
promoted = len(reasons) == 0
|
||||
|
||||
if promoted:
|
||||
logger.info(
|
||||
"NuExtract PROMOTED for %s: delta=%.4f, latency=%.1fms, memory=%.1fMB",
|
||||
report.document_type,
|
||||
report.delta,
|
||||
report.nuextract_latency_ms,
|
||||
report.nuextract_memory_mb,
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"NuExtract NOT promoted for %s: %s",
|
||||
report.document_type,
|
||||
"; ".join(reasons),
|
||||
)
|
||||
|
||||
return promoted
|
||||
|
||||
def get_rejection_reasons(self, report: IncrementalValueReport) -> list[str]:
|
||||
"""Return list of reasons why promotion would be rejected.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
report
|
||||
Incremental value report for a specific document type.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[str]
|
||||
Empty list if promotion passes; otherwise reasons for rejection.
|
||||
"""
|
||||
reasons: list[str] = []
|
||||
|
||||
# Check minimum sample count
|
||||
if report.sample_count < self._gate.min_sample_count:
|
||||
reasons.append(
|
||||
f"Insufficient samples: {report.sample_count} < {self._gate.min_sample_count}"
|
||||
)
|
||||
|
||||
# Check F1 improvement
|
||||
if report.delta < self._gate.min_f1_improvement:
|
||||
reasons.append(
|
||||
f"F1 improvement too small: {report.delta:.4f} < {self._gate.min_f1_improvement:.4f}"
|
||||
)
|
||||
|
||||
# Check latency
|
||||
if report.nuextract_latency_ms > self._gate.max_latency_ms:
|
||||
reasons.append(
|
||||
f"Latency exceeds gate: {report.nuextract_latency_ms:.1f}ms > {self._gate.max_latency_ms:.1f}ms"
|
||||
)
|
||||
|
||||
# Check memory
|
||||
if report.nuextract_memory_mb > self._gate.max_memory_mb:
|
||||
reasons.append(
|
||||
f"Memory exceeds gate: {report.nuextract_memory_mb:.1f}MB > {self._gate.max_memory_mb:.1f}MB"
|
||||
)
|
||||
|
||||
return reasons
|
||||
Reference in New Issue
Block a user