Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
573 lines
20 KiB
Python
573 lines
20 KiB
Python
"""Tests for NuExtract 1.5 Smol benchmark and promotion logic.
|
|
|
|
Tests:
|
|
- Adapter interface (test mode extraction)
|
|
- Benchmark comparison logic
|
|
- Promotion gate pass/fail
|
|
- Per-document-type reporting
|
|
|
|
Requirement: 6.6
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from services.intelligence_pipeline_v3.nuextract.adapter import (
|
|
NUEXTRACT_MODEL_VERSION,
|
|
NuExtractAdapter,
|
|
)
|
|
from services.intelligence_pipeline_v3.nuextract.benchmark import (
|
|
GLiNERResult,
|
|
GoldDocument,
|
|
NuExtractBenchmark,
|
|
)
|
|
from services.intelligence_pipeline_v3.nuextract.models import (
|
|
BenchmarkReport,
|
|
IncrementalValueReport,
|
|
NuExtractResult,
|
|
PromotionGate,
|
|
)
|
|
from services.intelligence_pipeline_v3.nuextract.promotion import PromotionEvaluator
|
|
|
|
# --- Fixtures ---
|
|
|
|
|
|
@pytest.fixture
|
|
def adapter() -> NuExtractAdapter:
|
|
"""Create a test-mode NuExtract adapter."""
|
|
return NuExtractAdapter(test_mode=True)
|
|
|
|
|
|
@pytest.fixture
|
|
def benchmark() -> NuExtractBenchmark:
|
|
"""Create a benchmark instance with test-mode adapter."""
|
|
return NuExtractBenchmark(
|
|
adapter=NuExtractAdapter(test_mode=True),
|
|
gate=PromotionGate(
|
|
min_f1_improvement=0.05,
|
|
max_latency_ms=5000.0,
|
|
max_memory_mb=2048.0,
|
|
min_sample_count=3,
|
|
),
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def strict_gate() -> PromotionGate:
|
|
"""Strict promotion gate that's hard to pass."""
|
|
return PromotionGate(
|
|
min_f1_improvement=0.20,
|
|
max_latency_ms=100.0,
|
|
max_memory_mb=512.0,
|
|
min_sample_count=100,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def lenient_gate() -> PromotionGate:
|
|
"""Lenient promotion gate that's easy to pass."""
|
|
return PromotionGate(
|
|
min_f1_improvement=0.01,
|
|
max_latency_ms=10000.0,
|
|
max_memory_mb=4096.0,
|
|
min_sample_count=1,
|
|
)
|
|
|
|
|
|
def _make_filing_doc(revenue: str = "4.2 billion") -> GoldDocument:
|
|
"""Create a sample filing document."""
|
|
return GoldDocument(
|
|
text=(
|
|
f"Revenue: {revenue}\n"
|
|
"Net Income: 1.3 billion\n"
|
|
"Earnings Per Share: 2.45\n"
|
|
"The company reported strong growth driven by cloud services."
|
|
),
|
|
document_type="filing",
|
|
gold_fields={
|
|
"revenue": revenue,
|
|
"net_income": "1.3 billion",
|
|
"earnings_per_share": "2.45",
|
|
},
|
|
schema={
|
|
"properties": {
|
|
"revenue": {"type": "string"},
|
|
"net_income": {"type": "string"},
|
|
"earnings_per_share": {"type": "string"},
|
|
}
|
|
},
|
|
)
|
|
|
|
|
|
def _make_transcript_doc() -> GoldDocument:
|
|
"""Create a sample transcript document."""
|
|
return GoldDocument(
|
|
text=(
|
|
"CEO: We expect guidance of 5.0 to 5.2 billion for next quarter.\n"
|
|
"CFO: Operating Margin: improved to 28 percent year over year.\n"
|
|
"Analyst: What about the competitive landscape?\n"
|
|
"CEO: We see strong demand across all segments."
|
|
),
|
|
document_type="transcript",
|
|
gold_fields={
|
|
"guidance": "5.0 to 5.2 billion",
|
|
"operating_margin": "28 percent",
|
|
},
|
|
schema={
|
|
"properties": {
|
|
"guidance": {"type": "string"},
|
|
"operating_margin": {"type": "string"},
|
|
}
|
|
},
|
|
)
|
|
|
|
|
|
def _make_article_doc() -> GoldDocument:
|
|
"""Create a sample article document."""
|
|
return GoldDocument(
|
|
text=(
|
|
"Apple announced a new product line today. "
|
|
"The stock price: rose 3.5% in after-hours trading. "
|
|
"Analysts expect Revenue: 95 billion for the quarter."
|
|
),
|
|
document_type="article",
|
|
gold_fields={
|
|
"stock_price": "rose 3.5%",
|
|
"revenue": "95 billion",
|
|
},
|
|
schema={
|
|
"properties": {
|
|
"stock_price": {"type": "string"},
|
|
"revenue": {"type": "string"},
|
|
}
|
|
},
|
|
)
|
|
|
|
|
|
# --- Test Adapter Interface ---
|
|
|
|
|
|
class TestNuExtractAdapter:
|
|
"""Test the NuExtract adapter interface."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_returns_result(self, adapter: NuExtractAdapter) -> None:
|
|
"""Adapter returns a valid NuExtractResult."""
|
|
result = await adapter.extract(
|
|
text="Revenue: 4.2 billion\nNet Income: 1.3 billion",
|
|
schema={"properties": {"revenue": {"type": "string"}}},
|
|
)
|
|
assert isinstance(result, NuExtractResult)
|
|
assert result.model_version == NUEXTRACT_MODEL_VERSION
|
|
assert result.error is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_captures_latency(self, adapter: NuExtractAdapter) -> None:
|
|
"""Extraction records latency in milliseconds."""
|
|
result = await adapter.extract(
|
|
text="Revenue: 10 million",
|
|
schema={"properties": {"revenue": {"type": "string"}}},
|
|
)
|
|
assert result.latency_ms >= 0.0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_finds_matching_fields(self, adapter: NuExtractAdapter) -> None:
|
|
"""Adapter extracts fields that match schema keys in text."""
|
|
result = await adapter.extract(
|
|
text="Revenue: 4.2 billion\nEPS: 2.45",
|
|
schema={
|
|
"properties": {
|
|
"revenue": {"type": "string"},
|
|
"eps": {"type": "string"},
|
|
}
|
|
},
|
|
)
|
|
field_names = [f.name for f in result.fields]
|
|
assert "revenue" in field_names
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_sets_document_type(self, adapter: NuExtractAdapter) -> None:
|
|
"""Document type is preserved in result."""
|
|
result = await adapter.extract(
|
|
text="Some filing content",
|
|
schema={"properties": {"field": {"type": "string"}}},
|
|
document_type="filing",
|
|
)
|
|
assert result.document_type == "filing"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_stores_schema_used(self, adapter: NuExtractAdapter) -> None:
|
|
"""Schema is stored in result for lineage."""
|
|
schema = {"properties": {"revenue": {"type": "string"}}}
|
|
result = await adapter.extract(text="Revenue: 100", schema=schema)
|
|
assert result.schema_used == schema
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_handles_empty_text(self, adapter: NuExtractAdapter) -> None:
|
|
"""Adapter handles empty text gracefully."""
|
|
result = await adapter.extract(
|
|
text="",
|
|
schema={"properties": {"field": {"type": "string"}}},
|
|
)
|
|
assert isinstance(result, NuExtractResult)
|
|
assert result.error is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_hierarchical_schema(self, adapter: NuExtractAdapter) -> None:
|
|
"""Adapter handles nested/hierarchical schemas."""
|
|
result = await adapter.extract(
|
|
text="Revenue: 4.2 billion\nSegment growth: 15%",
|
|
schema={
|
|
"properties": {
|
|
"financials": {
|
|
"properties": {
|
|
"revenue": {"type": "string"},
|
|
"segment_growth": {"type": "string"},
|
|
}
|
|
}
|
|
}
|
|
},
|
|
)
|
|
assert isinstance(result, NuExtractResult)
|
|
|
|
def test_model_version_pinned(self, adapter: NuExtractAdapter) -> None:
|
|
"""Model version is pinned and accessible."""
|
|
assert adapter.model_version == NUEXTRACT_MODEL_VERSION
|
|
assert "NuExtract" in adapter.model_name
|
|
|
|
def test_test_mode_does_not_load_model(self, adapter: NuExtractAdapter) -> None:
|
|
"""Test mode doesn't attempt to load the real model."""
|
|
assert not adapter.is_loaded
|
|
|
|
def test_unload_is_safe_in_test_mode(self, adapter: NuExtractAdapter) -> None:
|
|
"""Unload is a no-op in test mode."""
|
|
adapter.unload()
|
|
assert not adapter.is_loaded
|
|
|
|
|
|
# --- Test Benchmark Comparison Logic ---
|
|
|
|
|
|
class TestBenchmarkComparison:
|
|
"""Test the benchmark comparison between NuExtract and GLiNER2."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_benchmark_produces_report(self, benchmark: NuExtractBenchmark) -> None:
|
|
"""Benchmark returns a complete BenchmarkReport."""
|
|
docs = [_make_filing_doc(), _make_filing_doc("5.1 billion")]
|
|
gliner_results = [
|
|
GLiNERResult(fields={"revenue": "4.2 billion", "net_income": "1.3 billion"}),
|
|
GLiNERResult(fields={"revenue": "5.1 billion", "net_income": "1.3 billion"}),
|
|
]
|
|
|
|
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
|
assert isinstance(report, BenchmarkReport)
|
|
assert report.total_documents == 2
|
|
assert len(report.reports) == 1 # One doc type: filing
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_benchmark_groups_by_document_type(self, benchmark: NuExtractBenchmark) -> None:
|
|
"""Benchmark reports separately for each document type."""
|
|
docs = [_make_filing_doc(), _make_transcript_doc(), _make_article_doc()]
|
|
gliner_results = [
|
|
GLiNERResult(fields={"revenue": "4.2 billion"}),
|
|
GLiNERResult(fields={"guidance": "5.0 to 5.2 billion"}),
|
|
GLiNERResult(fields={"stock_price": "rose 3.5%"}),
|
|
]
|
|
|
|
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
|
doc_types = {r.document_type for r in report.reports}
|
|
assert "filing" in doc_types
|
|
assert "transcript" in doc_types
|
|
assert "article" in doc_types
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_benchmark_computes_f1_delta(self, benchmark: NuExtractBenchmark) -> None:
|
|
"""Delta is computed as nuextract_f1 - gliner_f1."""
|
|
docs = [_make_filing_doc(), _make_filing_doc(), _make_filing_doc()]
|
|
gliner_results = [
|
|
GLiNERResult(fields={"revenue": "4.2 billion"}),
|
|
GLiNERResult(fields={"revenue": "4.2 billion"}),
|
|
GLiNERResult(fields={"revenue": "4.2 billion"}),
|
|
]
|
|
|
|
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
|
for r in report.reports:
|
|
assert r.delta == pytest.approx(r.nuextract_f1 - r.gliner_f1, abs=1e-6)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_benchmark_rejects_mismatched_lengths(self, benchmark: NuExtractBenchmark) -> None:
|
|
"""Benchmark raises when document and result counts differ."""
|
|
docs = [_make_filing_doc(), _make_filing_doc()]
|
|
gliner_results = [GLiNERResult(fields={"revenue": "4.2 billion"})]
|
|
|
|
with pytest.raises(ValueError, match="must match"):
|
|
await benchmark.evaluate_against_gliner(docs, gliner_results)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_benchmark_tracks_latency(self, benchmark: NuExtractBenchmark) -> None:
|
|
"""Benchmark records latency metrics per type."""
|
|
docs = [_make_filing_doc(), _make_filing_doc(), _make_filing_doc()]
|
|
gliner_results = [
|
|
GLiNERResult(fields={"revenue": "4.2 billion"}, latency_ms=50.0),
|
|
GLiNERResult(fields={"revenue": "4.2 billion"}, latency_ms=60.0),
|
|
GLiNERResult(fields={"revenue": "4.2 billion"}, latency_ms=55.0),
|
|
]
|
|
|
|
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
|
filing_report = report.reports[0]
|
|
assert filing_report.gliner_latency_ms > 0.0
|
|
assert filing_report.nuextract_latency_ms >= 0.0
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_benchmark_overall_metrics(self, benchmark: NuExtractBenchmark) -> None:
|
|
"""Overall metrics are weighted averages across types."""
|
|
docs = [_make_filing_doc(), _make_filing_doc(), _make_filing_doc()]
|
|
gliner_results = [
|
|
GLiNERResult(fields={"revenue": "4.2 billion", "net_income": "1.3 billion", "earnings_per_share": "2.45"}),
|
|
GLiNERResult(fields={"revenue": "4.2 billion", "net_income": "1.3 billion", "earnings_per_share": "2.45"}),
|
|
GLiNERResult(fields={"revenue": "4.2 billion", "net_income": "1.3 billion", "earnings_per_share": "2.45"}),
|
|
]
|
|
|
|
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
|
assert report.overall_gliner_f1 >= 0.0
|
|
assert report.overall_nuextract_f1 >= 0.0
|
|
assert report.overall_delta == pytest.approx(
|
|
report.overall_nuextract_f1 - report.overall_gliner_f1, abs=1e-6
|
|
)
|
|
|
|
|
|
# --- Test Promotion Gate ---
|
|
|
|
|
|
class TestPromotionGate:
|
|
"""Test the promotion gate pass/fail logic."""
|
|
|
|
def test_promotion_passes_when_all_gates_met(self, lenient_gate: PromotionGate) -> None:
|
|
"""Promotion passes when all thresholds are met."""
|
|
evaluator = PromotionEvaluator(lenient_gate)
|
|
report = IncrementalValueReport(
|
|
document_type="filing",
|
|
gliner_f1=0.80,
|
|
nuextract_f1=0.85,
|
|
delta=0.05,
|
|
nuextract_latency_ms=200.0,
|
|
nuextract_memory_mb=500.0,
|
|
sample_count=100,
|
|
)
|
|
assert evaluator.evaluate(report) is True
|
|
|
|
def test_promotion_fails_insufficient_f1(self, strict_gate: PromotionGate) -> None:
|
|
"""Promotion fails when F1 improvement is below threshold."""
|
|
evaluator = PromotionEvaluator(strict_gate)
|
|
report = IncrementalValueReport(
|
|
document_type="filing",
|
|
gliner_f1=0.80,
|
|
nuextract_f1=0.82,
|
|
delta=0.02, # Below 0.20 threshold
|
|
nuextract_latency_ms=50.0,
|
|
nuextract_memory_mb=200.0,
|
|
sample_count=100,
|
|
)
|
|
assert evaluator.evaluate(report) is False
|
|
|
|
def test_promotion_fails_high_latency(self) -> None:
|
|
"""Promotion fails when latency exceeds the gate."""
|
|
gate = PromotionGate(
|
|
min_f1_improvement=0.01,
|
|
max_latency_ms=100.0,
|
|
max_memory_mb=4096.0,
|
|
min_sample_count=1,
|
|
)
|
|
evaluator = PromotionEvaluator(gate)
|
|
report = IncrementalValueReport(
|
|
document_type="transcript",
|
|
gliner_f1=0.70,
|
|
nuextract_f1=0.80,
|
|
delta=0.10,
|
|
nuextract_latency_ms=500.0, # Exceeds 100ms gate
|
|
nuextract_memory_mb=200.0,
|
|
sample_count=50,
|
|
)
|
|
assert evaluator.evaluate(report) is False
|
|
|
|
def test_promotion_fails_high_memory(self) -> None:
|
|
"""Promotion fails when memory exceeds the gate."""
|
|
gate = PromotionGate(
|
|
min_f1_improvement=0.01,
|
|
max_latency_ms=10000.0,
|
|
max_memory_mb=512.0,
|
|
min_sample_count=1,
|
|
)
|
|
evaluator = PromotionEvaluator(gate)
|
|
report = IncrementalValueReport(
|
|
document_type="article",
|
|
gliner_f1=0.70,
|
|
nuextract_f1=0.85,
|
|
delta=0.15,
|
|
nuextract_latency_ms=200.0,
|
|
nuextract_memory_mb=1024.0, # Exceeds 512MB gate
|
|
sample_count=50,
|
|
)
|
|
assert evaluator.evaluate(report) is False
|
|
|
|
def test_promotion_fails_insufficient_samples(self) -> None:
|
|
"""Promotion fails when sample count is below minimum."""
|
|
gate = PromotionGate(
|
|
min_f1_improvement=0.01,
|
|
max_latency_ms=10000.0,
|
|
max_memory_mb=4096.0,
|
|
min_sample_count=100,
|
|
)
|
|
evaluator = PromotionEvaluator(gate)
|
|
report = IncrementalValueReport(
|
|
document_type="filing",
|
|
gliner_f1=0.70,
|
|
nuextract_f1=0.90,
|
|
delta=0.20,
|
|
nuextract_latency_ms=200.0,
|
|
nuextract_memory_mb=500.0,
|
|
sample_count=10, # Below 100 minimum
|
|
)
|
|
assert evaluator.evaluate(report) is False
|
|
|
|
def test_rejection_reasons_reported(self) -> None:
|
|
"""Evaluator provides specific rejection reasons."""
|
|
gate = PromotionGate(
|
|
min_f1_improvement=0.10,
|
|
max_latency_ms=100.0,
|
|
max_memory_mb=512.0,
|
|
min_sample_count=50,
|
|
)
|
|
evaluator = PromotionEvaluator(gate)
|
|
report = IncrementalValueReport(
|
|
document_type="filing",
|
|
gliner_f1=0.80,
|
|
nuextract_f1=0.82,
|
|
delta=0.02, # Below threshold
|
|
nuextract_latency_ms=500.0, # Above threshold
|
|
nuextract_memory_mb=1024.0, # Above threshold
|
|
sample_count=10, # Below minimum
|
|
)
|
|
reasons = evaluator.get_rejection_reasons(report)
|
|
assert len(reasons) == 4
|
|
assert any("F1" in r for r in reasons)
|
|
assert any("Latency" in r for r in reasons)
|
|
assert any("Memory" in r for r in reasons)
|
|
assert any("samples" in r.lower() for r in reasons)
|
|
|
|
def test_no_rejection_reasons_when_passing(self, lenient_gate: PromotionGate) -> None:
|
|
"""No rejection reasons when all gates pass."""
|
|
evaluator = PromotionEvaluator(lenient_gate)
|
|
report = IncrementalValueReport(
|
|
document_type="filing",
|
|
gliner_f1=0.70,
|
|
nuextract_f1=0.80,
|
|
delta=0.10,
|
|
nuextract_latency_ms=200.0,
|
|
nuextract_memory_mb=500.0,
|
|
sample_count=100,
|
|
)
|
|
reasons = evaluator.get_rejection_reasons(report)
|
|
assert reasons == []
|
|
|
|
|
|
# --- Test Per-Document-Type Reporting ---
|
|
|
|
|
|
class TestPerDocumentTypeReporting:
|
|
"""Test that benchmark produces correct per-type reports."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_promoted_types_listed(self) -> None:
|
|
"""Promoted types appear in the benchmark report."""
|
|
gate = PromotionGate(
|
|
min_f1_improvement=0.0, # Accept any improvement
|
|
max_latency_ms=10000.0,
|
|
max_memory_mb=4096.0,
|
|
min_sample_count=1,
|
|
)
|
|
benchmark = NuExtractBenchmark(
|
|
adapter=NuExtractAdapter(test_mode=True),
|
|
gate=gate,
|
|
)
|
|
|
|
# Filing doc where NuExtract should find matches (schema keys appear in text)
|
|
docs = [_make_filing_doc()]
|
|
# GLiNER returns empty to ensure NuExtract has higher F1
|
|
gliner_results = [GLiNERResult(fields={})]
|
|
|
|
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
|
# With empty GLiNER results, NuExtract should score higher
|
|
for r in report.reports:
|
|
if r.nuextract_f1 > r.gliner_f1:
|
|
assert r.document_type in report.promoted_types
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_promoted_types_excluded(self) -> None:
|
|
"""Types that don't pass gates are not in promoted list."""
|
|
gate = PromotionGate(
|
|
min_f1_improvement=0.99, # Nearly impossible to pass
|
|
max_latency_ms=10000.0,
|
|
max_memory_mb=4096.0,
|
|
min_sample_count=1,
|
|
)
|
|
benchmark = NuExtractBenchmark(
|
|
adapter=NuExtractAdapter(test_mode=True),
|
|
gate=gate,
|
|
)
|
|
|
|
docs = [_make_filing_doc()]
|
|
gliner_results = [
|
|
GLiNERResult(
|
|
fields={"revenue": "4.2 billion", "net_income": "1.3 billion", "earnings_per_share": "2.45"}
|
|
)
|
|
]
|
|
|
|
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
|
assert report.promoted_types == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sample_count_per_type(self) -> None:
|
|
"""Sample count reflects the number of documents per type."""
|
|
benchmark = NuExtractBenchmark(
|
|
adapter=NuExtractAdapter(test_mode=True),
|
|
gate=PromotionGate(min_sample_count=1),
|
|
)
|
|
|
|
docs = [
|
|
_make_filing_doc(),
|
|
_make_filing_doc("5.0 billion"),
|
|
_make_transcript_doc(),
|
|
]
|
|
gliner_results = [
|
|
GLiNERResult(fields={"revenue": "4.2 billion"}),
|
|
GLiNERResult(fields={"revenue": "5.0 billion"}),
|
|
GLiNERResult(fields={"guidance": "5.0 to 5.2 billion"}),
|
|
]
|
|
|
|
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
|
type_counts = {r.document_type: r.sample_count for r in report.reports}
|
|
assert type_counts["filing"] == 2
|
|
assert type_counts["transcript"] == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_f1_scores_bounded(self) -> None:
|
|
"""F1 scores are always between 0 and 1."""
|
|
benchmark = NuExtractBenchmark(
|
|
adapter=NuExtractAdapter(test_mode=True),
|
|
gate=PromotionGate(min_sample_count=1),
|
|
)
|
|
|
|
docs = [_make_filing_doc(), _make_transcript_doc(), _make_article_doc()]
|
|
gliner_results = [
|
|
GLiNERResult(fields={"revenue": "wrong value"}),
|
|
GLiNERResult(fields={"guidance": "wrong"}),
|
|
GLiNERResult(fields={}),
|
|
]
|
|
|
|
report = await benchmark.evaluate_against_gliner(docs, gliner_results)
|
|
for r in report.reports:
|
|
assert 0.0 <= r.gliner_f1 <= 1.0
|
|
assert 0.0 <= r.nuextract_f1 <= 1.0
|