feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,572 @@
|
||||
"""Tests for NuExtract 1.5 Smol benchmark and promotion logic.
|
||||
|
||||
Tests:
|
||||
- Adapter interface (test mode extraction)
|
||||
- Benchmark comparison logic
|
||||
- Promotion gate pass/fail
|
||||
- Per-document-type reporting
|
||||
|
||||
Requirement: 6.6
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from services.intelligence_pipeline_v3.nuextract.adapter import (
|
||||
NUEXTRACT_MODEL_VERSION,
|
||||
NuExtractAdapter,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.nuextract.benchmark import (
|
||||
GLiNERResult,
|
||||
GoldDocument,
|
||||
NuExtractBenchmark,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.nuextract.models import (
|
||||
BenchmarkReport,
|
||||
IncrementalValueReport,
|
||||
NuExtractResult,
|
||||
PromotionGate,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.nuextract.promotion import PromotionEvaluator
|
||||
|
||||
# --- Fixtures ---
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def adapter() -> NuExtractAdapter:
|
||||
"""Create a test-mode NuExtract adapter."""
|
||||
return NuExtractAdapter(test_mode=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def benchmark() -> NuExtractBenchmark:
|
||||
"""Create a benchmark instance with test-mode adapter."""
|
||||
return NuExtractBenchmark(
|
||||
adapter=NuExtractAdapter(test_mode=True),
|
||||
gate=PromotionGate(
|
||||
min_f1_improvement=0.05,
|
||||
max_latency_ms=5000.0,
|
||||
max_memory_mb=2048.0,
|
||||
min_sample_count=3,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def strict_gate() -> PromotionGate:
|
||||
"""Strict promotion gate that's hard to pass."""
|
||||
return PromotionGate(
|
||||
min_f1_improvement=0.20,
|
||||
max_latency_ms=100.0,
|
||||
max_memory_mb=512.0,
|
||||
min_sample_count=100,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def lenient_gate() -> PromotionGate:
|
||||
"""Lenient promotion gate that's easy to pass."""
|
||||
return PromotionGate(
|
||||
min_f1_improvement=0.01,
|
||||
max_latency_ms=10000.0,
|
||||
max_memory_mb=4096.0,
|
||||
min_sample_count=1,
|
||||
)
|
||||
|
||||
|
||||
def _make_filing_doc(revenue: str = "4.2 billion") -> GoldDocument:
|
||||
"""Create a sample filing document."""
|
||||
return GoldDocument(
|
||||
text=(
|
||||
f"Revenue: {revenue}\n"
|
||||
"Net Income: 1.3 billion\n"
|
||||
"Earnings Per Share: 2.45\n"
|
||||
"The company reported strong growth driven by cloud services."
|
||||
),
|
||||
document_type="filing",
|
||||
gold_fields={
|
||||
"revenue": revenue,
|
||||
"net_income": "1.3 billion",
|
||||
"earnings_per_share": "2.45",
|
||||
},
|
||||
schema={
|
||||
"properties": {
|
||||
"revenue": {"type": "string"},
|
||||
"net_income": {"type": "string"},
|
||||
"earnings_per_share": {"type": "string"},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _make_transcript_doc() -> GoldDocument:
|
||||
"""Create a sample transcript document."""
|
||||
return GoldDocument(
|
||||
text=(
|
||||
"CEO: We expect guidance of 5.0 to 5.2 billion for next quarter.\n"
|
||||
"CFO: Operating Margin: improved to 28 percent year over year.\n"
|
||||
"Analyst: What about the competitive landscape?\n"
|
||||
"CEO: We see strong demand across all segments."
|
||||
),
|
||||
document_type="transcript",
|
||||
gold_fields={
|
||||
"guidance": "5.0 to 5.2 billion",
|
||||
"operating_margin": "28 percent",
|
||||
},
|
||||
schema={
|
||||
"properties": {
|
||||
"guidance": {"type": "string"},
|
||||
"operating_margin": {"type": "string"},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _make_article_doc() -> GoldDocument:
|
||||
"""Create a sample article document."""
|
||||
return GoldDocument(
|
||||
text=(
|
||||
"Apple announced a new product line today. "
|
||||
"The stock price: rose 3.5% in after-hours trading. "
|
||||
"Analysts expect Revenue: 95 billion for the quarter."
|
||||
),
|
||||
document_type="article",
|
||||
gold_fields={
|
||||
"stock_price": "rose 3.5%",
|
||||
"revenue": "95 billion",
|
||||
},
|
||||
schema={
|
||||
"properties": {
|
||||
"stock_price": {"type": "string"},
|
||||
"revenue": {"type": "string"},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# --- Test Adapter Interface ---
|
||||
|
||||
|
||||
class TestNuExtractAdapter:
|
||||
"""Test the NuExtract adapter interface."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_returns_result(self, adapter: NuExtractAdapter) -> None:
|
||||
"""Adapter returns a valid NuExtractResult."""
|
||||
result = await adapter.extract(
|
||||
text="Revenue: 4.2 billion\nNet Income: 1.3 billion",
|
||||
schema={"properties": {"revenue": {"type": "string"}}},
|
||||
)
|
||||
assert isinstance(result, NuExtractResult)
|
||||
assert result.model_version == NUEXTRACT_MODEL_VERSION
|
||||
assert result.error is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_captures_latency(self, adapter: NuExtractAdapter) -> None:
|
||||
"""Extraction records latency in milliseconds."""
|
||||
result = await adapter.extract(
|
||||
text="Revenue: 10 million",
|
||||
schema={"properties": {"revenue": {"type": "string"}}},
|
||||
)
|
||||
assert result.latency_ms >= 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_finds_matching_fields(self, adapter: NuExtractAdapter) -> None:
|
||||
"""Adapter extracts fields that match schema keys in text."""
|
||||
result = await adapter.extract(
|
||||
text="Revenue: 4.2 billion\nEPS: 2.45",
|
||||
schema={
|
||||
"properties": {
|
||||
"revenue": {"type": "string"},
|
||||
"eps": {"type": "string"},
|
||||
}
|
||||
},
|
||||
)
|
||||
field_names = [f.name for f in result.fields]
|
||||
assert "revenue" in field_names
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_sets_document_type(self, adapter: NuExtractAdapter) -> None:
|
||||
"""Document type is preserved in result."""
|
||||
result = await adapter.extract(
|
||||
text="Some filing content",
|
||||
schema={"properties": {"field": {"type": "string"}}},
|
||||
document_type="filing",
|
||||
)
|
||||
assert result.document_type == "filing"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_stores_schema_used(self, adapter: NuExtractAdapter) -> None:
|
||||
"""Schema is stored in result for lineage."""
|
||||
schema = {"properties": {"revenue": {"type": "string"}}}
|
||||
result = await adapter.extract(text="Revenue: 100", schema=schema)
|
||||
assert result.schema_used == schema
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_handles_empty_text(self, adapter: NuExtractAdapter) -> None:
|
||||
"""Adapter handles empty text gracefully."""
|
||||
result = await adapter.extract(
|
||||
text="",
|
||||
schema={"properties": {"field": {"type": "string"}}},
|
||||
)
|
||||
assert isinstance(result, NuExtractResult)
|
||||
assert result.error is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_hierarchical_schema(self, adapter: NuExtractAdapter) -> None:
|
||||
"""Adapter handles nested/hierarchical schemas."""
|
||||
result = await adapter.extract(
|
||||
text="Revenue: 4.2 billion\nSegment growth: 15%",
|
||||
schema={
|
||||
"properties": {
|
||||
"financials": {
|
||||
"properties": {
|
||||
"revenue": {"type": "string"},
|
||||
"segment_growth": {"type": "string"},
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
assert isinstance(result, NuExtractResult)
|
||||
|
||||
def test_model_version_pinned(self, adapter: NuExtractAdapter) -> None:
|
||||
"""Model version is pinned and accessible."""
|
||||
assert adapter.model_version == NUEXTRACT_MODEL_VERSION
|
||||
assert "NuExtract" in adapter.model_name
|
||||
|
||||
def test_test_mode_does_not_load_model(self, adapter: NuExtractAdapter) -> None:
|
||||
"""Test mode doesn't attempt to load the real model."""
|
||||
assert not adapter.is_loaded
|
||||
|
||||
def test_unload_is_safe_in_test_mode(self, adapter: NuExtractAdapter) -> None:
|
||||
"""Unload is a no-op in test mode."""
|
||||
adapter.unload()
|
||||
assert not adapter.is_loaded
|
||||
|
||||
|
||||
# --- Test Benchmark Comparison Logic ---
|
||||
|
||||
|
||||
class TestBenchmarkComparison:
|
||||
"""Test the benchmark comparison between NuExtract and GLiNER2."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_benchmark_produces_report(self, benchmark: NuExtractBenchmark) -> None:
|
||||
"""Benchmark returns a complete BenchmarkReport."""
|
||||
docs = [_make_filing_doc(), _make_filing_doc("5.1 billion")]
|
||||
gliner_results = [
|
||||
GLiNERResult(fields={"revenue": "4.2 billion", "net_income": "1.3 billion"}),
|
||||
GLiNERResult(fields={"revenue": "5.1 billion", "net_income": "1.3 billion"}),
|
||||
]
|
||||
|
||||
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
||||
assert isinstance(report, BenchmarkReport)
|
||||
assert report.total_documents == 2
|
||||
assert len(report.reports) == 1 # One doc type: filing
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_benchmark_groups_by_document_type(self, benchmark: NuExtractBenchmark) -> None:
|
||||
"""Benchmark reports separately for each document type."""
|
||||
docs = [_make_filing_doc(), _make_transcript_doc(), _make_article_doc()]
|
||||
gliner_results = [
|
||||
GLiNERResult(fields={"revenue": "4.2 billion"}),
|
||||
GLiNERResult(fields={"guidance": "5.0 to 5.2 billion"}),
|
||||
GLiNERResult(fields={"stock_price": "rose 3.5%"}),
|
||||
]
|
||||
|
||||
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
||||
doc_types = {r.document_type for r in report.reports}
|
||||
assert "filing" in doc_types
|
||||
assert "transcript" in doc_types
|
||||
assert "article" in doc_types
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_benchmark_computes_f1_delta(self, benchmark: NuExtractBenchmark) -> None:
|
||||
"""Delta is computed as nuextract_f1 - gliner_f1."""
|
||||
docs = [_make_filing_doc(), _make_filing_doc(), _make_filing_doc()]
|
||||
gliner_results = [
|
||||
GLiNERResult(fields={"revenue": "4.2 billion"}),
|
||||
GLiNERResult(fields={"revenue": "4.2 billion"}),
|
||||
GLiNERResult(fields={"revenue": "4.2 billion"}),
|
||||
]
|
||||
|
||||
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
||||
for r in report.reports:
|
||||
assert r.delta == pytest.approx(r.nuextract_f1 - r.gliner_f1, abs=1e-6)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_benchmark_rejects_mismatched_lengths(self, benchmark: NuExtractBenchmark) -> None:
|
||||
"""Benchmark raises when document and result counts differ."""
|
||||
docs = [_make_filing_doc(), _make_filing_doc()]
|
||||
gliner_results = [GLiNERResult(fields={"revenue": "4.2 billion"})]
|
||||
|
||||
with pytest.raises(ValueError, match="must match"):
|
||||
await benchmark.evaluate_against_gliner(docs, gliner_results)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_benchmark_tracks_latency(self, benchmark: NuExtractBenchmark) -> None:
|
||||
"""Benchmark records latency metrics per type."""
|
||||
docs = [_make_filing_doc(), _make_filing_doc(), _make_filing_doc()]
|
||||
gliner_results = [
|
||||
GLiNERResult(fields={"revenue": "4.2 billion"}, latency_ms=50.0),
|
||||
GLiNERResult(fields={"revenue": "4.2 billion"}, latency_ms=60.0),
|
||||
GLiNERResult(fields={"revenue": "4.2 billion"}, latency_ms=55.0),
|
||||
]
|
||||
|
||||
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
||||
filing_report = report.reports[0]
|
||||
assert filing_report.gliner_latency_ms > 0.0
|
||||
assert filing_report.nuextract_latency_ms >= 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_benchmark_overall_metrics(self, benchmark: NuExtractBenchmark) -> None:
|
||||
"""Overall metrics are weighted averages across types."""
|
||||
docs = [_make_filing_doc(), _make_filing_doc(), _make_filing_doc()]
|
||||
gliner_results = [
|
||||
GLiNERResult(fields={"revenue": "4.2 billion", "net_income": "1.3 billion", "earnings_per_share": "2.45"}),
|
||||
GLiNERResult(fields={"revenue": "4.2 billion", "net_income": "1.3 billion", "earnings_per_share": "2.45"}),
|
||||
GLiNERResult(fields={"revenue": "4.2 billion", "net_income": "1.3 billion", "earnings_per_share": "2.45"}),
|
||||
]
|
||||
|
||||
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
||||
assert report.overall_gliner_f1 >= 0.0
|
||||
assert report.overall_nuextract_f1 >= 0.0
|
||||
assert report.overall_delta == pytest.approx(
|
||||
report.overall_nuextract_f1 - report.overall_gliner_f1, abs=1e-6
|
||||
)
|
||||
|
||||
|
||||
# --- Test Promotion Gate ---
|
||||
|
||||
|
||||
class TestPromotionGate:
|
||||
"""Test the promotion gate pass/fail logic."""
|
||||
|
||||
def test_promotion_passes_when_all_gates_met(self, lenient_gate: PromotionGate) -> None:
|
||||
"""Promotion passes when all thresholds are met."""
|
||||
evaluator = PromotionEvaluator(lenient_gate)
|
||||
report = IncrementalValueReport(
|
||||
document_type="filing",
|
||||
gliner_f1=0.80,
|
||||
nuextract_f1=0.85,
|
||||
delta=0.05,
|
||||
nuextract_latency_ms=200.0,
|
||||
nuextract_memory_mb=500.0,
|
||||
sample_count=100,
|
||||
)
|
||||
assert evaluator.evaluate(report) is True
|
||||
|
||||
def test_promotion_fails_insufficient_f1(self, strict_gate: PromotionGate) -> None:
|
||||
"""Promotion fails when F1 improvement is below threshold."""
|
||||
evaluator = PromotionEvaluator(strict_gate)
|
||||
report = IncrementalValueReport(
|
||||
document_type="filing",
|
||||
gliner_f1=0.80,
|
||||
nuextract_f1=0.82,
|
||||
delta=0.02, # Below 0.20 threshold
|
||||
nuextract_latency_ms=50.0,
|
||||
nuextract_memory_mb=200.0,
|
||||
sample_count=100,
|
||||
)
|
||||
assert evaluator.evaluate(report) is False
|
||||
|
||||
def test_promotion_fails_high_latency(self) -> None:
|
||||
"""Promotion fails when latency exceeds the gate."""
|
||||
gate = PromotionGate(
|
||||
min_f1_improvement=0.01,
|
||||
max_latency_ms=100.0,
|
||||
max_memory_mb=4096.0,
|
||||
min_sample_count=1,
|
||||
)
|
||||
evaluator = PromotionEvaluator(gate)
|
||||
report = IncrementalValueReport(
|
||||
document_type="transcript",
|
||||
gliner_f1=0.70,
|
||||
nuextract_f1=0.80,
|
||||
delta=0.10,
|
||||
nuextract_latency_ms=500.0, # Exceeds 100ms gate
|
||||
nuextract_memory_mb=200.0,
|
||||
sample_count=50,
|
||||
)
|
||||
assert evaluator.evaluate(report) is False
|
||||
|
||||
def test_promotion_fails_high_memory(self) -> None:
|
||||
"""Promotion fails when memory exceeds the gate."""
|
||||
gate = PromotionGate(
|
||||
min_f1_improvement=0.01,
|
||||
max_latency_ms=10000.0,
|
||||
max_memory_mb=512.0,
|
||||
min_sample_count=1,
|
||||
)
|
||||
evaluator = PromotionEvaluator(gate)
|
||||
report = IncrementalValueReport(
|
||||
document_type="article",
|
||||
gliner_f1=0.70,
|
||||
nuextract_f1=0.85,
|
||||
delta=0.15,
|
||||
nuextract_latency_ms=200.0,
|
||||
nuextract_memory_mb=1024.0, # Exceeds 512MB gate
|
||||
sample_count=50,
|
||||
)
|
||||
assert evaluator.evaluate(report) is False
|
||||
|
||||
def test_promotion_fails_insufficient_samples(self) -> None:
|
||||
"""Promotion fails when sample count is below minimum."""
|
||||
gate = PromotionGate(
|
||||
min_f1_improvement=0.01,
|
||||
max_latency_ms=10000.0,
|
||||
max_memory_mb=4096.0,
|
||||
min_sample_count=100,
|
||||
)
|
||||
evaluator = PromotionEvaluator(gate)
|
||||
report = IncrementalValueReport(
|
||||
document_type="filing",
|
||||
gliner_f1=0.70,
|
||||
nuextract_f1=0.90,
|
||||
delta=0.20,
|
||||
nuextract_latency_ms=200.0,
|
||||
nuextract_memory_mb=500.0,
|
||||
sample_count=10, # Below 100 minimum
|
||||
)
|
||||
assert evaluator.evaluate(report) is False
|
||||
|
||||
def test_rejection_reasons_reported(self) -> None:
|
||||
"""Evaluator provides specific rejection reasons."""
|
||||
gate = PromotionGate(
|
||||
min_f1_improvement=0.10,
|
||||
max_latency_ms=100.0,
|
||||
max_memory_mb=512.0,
|
||||
min_sample_count=50,
|
||||
)
|
||||
evaluator = PromotionEvaluator(gate)
|
||||
report = IncrementalValueReport(
|
||||
document_type="filing",
|
||||
gliner_f1=0.80,
|
||||
nuextract_f1=0.82,
|
||||
delta=0.02, # Below threshold
|
||||
nuextract_latency_ms=500.0, # Above threshold
|
||||
nuextract_memory_mb=1024.0, # Above threshold
|
||||
sample_count=10, # Below minimum
|
||||
)
|
||||
reasons = evaluator.get_rejection_reasons(report)
|
||||
assert len(reasons) == 4
|
||||
assert any("F1" in r for r in reasons)
|
||||
assert any("Latency" in r for r in reasons)
|
||||
assert any("Memory" in r for r in reasons)
|
||||
assert any("samples" in r.lower() for r in reasons)
|
||||
|
||||
def test_no_rejection_reasons_when_passing(self, lenient_gate: PromotionGate) -> None:
|
||||
"""No rejection reasons when all gates pass."""
|
||||
evaluator = PromotionEvaluator(lenient_gate)
|
||||
report = IncrementalValueReport(
|
||||
document_type="filing",
|
||||
gliner_f1=0.70,
|
||||
nuextract_f1=0.80,
|
||||
delta=0.10,
|
||||
nuextract_latency_ms=200.0,
|
||||
nuextract_memory_mb=500.0,
|
||||
sample_count=100,
|
||||
)
|
||||
reasons = evaluator.get_rejection_reasons(report)
|
||||
assert reasons == []
|
||||
|
||||
|
||||
# --- Test Per-Document-Type Reporting ---
|
||||
|
||||
|
||||
class TestPerDocumentTypeReporting:
|
||||
"""Test that benchmark produces correct per-type reports."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_promoted_types_listed(self) -> None:
|
||||
"""Promoted types appear in the benchmark report."""
|
||||
gate = PromotionGate(
|
||||
min_f1_improvement=0.0, # Accept any improvement
|
||||
max_latency_ms=10000.0,
|
||||
max_memory_mb=4096.0,
|
||||
min_sample_count=1,
|
||||
)
|
||||
benchmark = NuExtractBenchmark(
|
||||
adapter=NuExtractAdapter(test_mode=True),
|
||||
gate=gate,
|
||||
)
|
||||
|
||||
# Filing doc where NuExtract should find matches (schema keys appear in text)
|
||||
docs = [_make_filing_doc()]
|
||||
# GLiNER returns empty to ensure NuExtract has higher F1
|
||||
gliner_results = [GLiNERResult(fields={})]
|
||||
|
||||
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
||||
# With empty GLiNER results, NuExtract should score higher
|
||||
for r in report.reports:
|
||||
if r.nuextract_f1 > r.gliner_f1:
|
||||
assert r.document_type in report.promoted_types
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_promoted_types_excluded(self) -> None:
|
||||
"""Types that don't pass gates are not in promoted list."""
|
||||
gate = PromotionGate(
|
||||
min_f1_improvement=0.99, # Nearly impossible to pass
|
||||
max_latency_ms=10000.0,
|
||||
max_memory_mb=4096.0,
|
||||
min_sample_count=1,
|
||||
)
|
||||
benchmark = NuExtractBenchmark(
|
||||
adapter=NuExtractAdapter(test_mode=True),
|
||||
gate=gate,
|
||||
)
|
||||
|
||||
docs = [_make_filing_doc()]
|
||||
gliner_results = [
|
||||
GLiNERResult(
|
||||
fields={"revenue": "4.2 billion", "net_income": "1.3 billion", "earnings_per_share": "2.45"}
|
||||
)
|
||||
]
|
||||
|
||||
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
||||
assert report.promoted_types == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sample_count_per_type(self) -> None:
|
||||
"""Sample count reflects the number of documents per type."""
|
||||
benchmark = NuExtractBenchmark(
|
||||
adapter=NuExtractAdapter(test_mode=True),
|
||||
gate=PromotionGate(min_sample_count=1),
|
||||
)
|
||||
|
||||
docs = [
|
||||
_make_filing_doc(),
|
||||
_make_filing_doc("5.0 billion"),
|
||||
_make_transcript_doc(),
|
||||
]
|
||||
gliner_results = [
|
||||
GLiNERResult(fields={"revenue": "4.2 billion"}),
|
||||
GLiNERResult(fields={"revenue": "5.0 billion"}),
|
||||
GLiNERResult(fields={"guidance": "5.0 to 5.2 billion"}),
|
||||
]
|
||||
|
||||
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
||||
type_counts = {r.document_type: r.sample_count for r in report.reports}
|
||||
assert type_counts["filing"] == 2
|
||||
assert type_counts["transcript"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_f1_scores_bounded(self) -> None:
|
||||
"""F1 scores are always between 0 and 1."""
|
||||
benchmark = NuExtractBenchmark(
|
||||
adapter=NuExtractAdapter(test_mode=True),
|
||||
gate=PromotionGate(min_sample_count=1),
|
||||
)
|
||||
|
||||
docs = [_make_filing_doc(), _make_transcript_doc(), _make_article_doc()]
|
||||
gliner_results = [
|
||||
GLiNERResult(fields={"revenue": "wrong value"}),
|
||||
GLiNERResult(fields={"guidance": "wrong"}),
|
||||
GLiNERResult(fields={}),
|
||||
]
|
||||
|
||||
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
||||
for r in report.reports:
|
||||
assert 0.0 <= r.gliner_f1 <= 1.0
|
||||
assert 0.0 <= r.nuextract_f1 <= 1.0
|
||||
Reference in New Issue
Block a user