Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
125 lines
5.0 KiB
Python
125 lines
5.0 KiB
Python
"""Tests for active learning export — Task 49."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from services.intelligence_pipeline_v3.active_learning.exporter import (
|
|
ActiveLearningExporter,
|
|
ContentPolicy,
|
|
ExportConfig,
|
|
SelectionCriteria,
|
|
)
|
|
|
|
|
|
class TestActiveLearningExporter:
|
|
"""Task 49.1-49.3: Selection, filtering, versioned export."""
|
|
|
|
def test_select_low_confidence(self):
|
|
exporter = ActiveLearningExporter(config=ExportConfig())
|
|
record = exporter.select_record(
|
|
document_id="doc-001",
|
|
criteria=SelectionCriteria.LOW_CONFIDENCE,
|
|
source_spans=[{"text": "Apple beat Q4 estimates", "start": 0, "end": 23}],
|
|
document_type="news",
|
|
entity_labels=[{"type": "company", "text": "Apple"}],
|
|
confidence_scores={"entity_extraction": 0.3},
|
|
)
|
|
assert record is not None
|
|
assert record.selection_criteria == SelectionCriteria.LOW_CONFIDENCE
|
|
assert record.export_version == "1.0"
|
|
|
|
def test_select_adjudicated(self):
|
|
exporter = ActiveLearningExporter(config=ExportConfig())
|
|
record = exporter.select_record(
|
|
document_id="doc-002",
|
|
criteria=SelectionCriteria.ADJUDICATED,
|
|
source_spans=[{"text": "complex filing", "start": 0, "end": 14}],
|
|
adjudicator_decisions=[{"resolved_ticker": "AAPL", "confidence": 0.9}],
|
|
)
|
|
assert record is not None
|
|
assert record.adjudicator_decisions[0]["resolved_ticker"] == "AAPL"
|
|
|
|
def test_select_corrected(self):
|
|
exporter = ActiveLearningExporter(config=ExportConfig())
|
|
record = exporter.select_record(
|
|
document_id="doc-003",
|
|
criteria=SelectionCriteria.REVIEWER_CORRECTED,
|
|
source_spans=[{"text": "quarterly revenue", "start": 0, "end": 17}],
|
|
reviewer_corrections=[
|
|
{"field": "sentiment", "from": "positive", "to": "negative"}
|
|
],
|
|
)
|
|
assert record is not None
|
|
assert len(record.reviewer_corrections) == 1
|
|
|
|
def test_content_policy_redact(self):
|
|
config = ExportConfig(content_policy=ContentPolicy.REDACT_PII)
|
|
exporter = ActiveLearningExporter(config=config)
|
|
record = exporter.select_record(
|
|
document_id="doc-004",
|
|
criteria=SelectionCriteria.LOW_CONFIDENCE,
|
|
source_spans=[{"text": "John Smith at Apple", "start": 0, "end": 19}],
|
|
)
|
|
assert record is not None
|
|
# Spans are marked with policy applied
|
|
assert record.source_spans[0].get("content_policy_applied") == "redact_pii"
|
|
|
|
def test_content_policy_exclude(self):
|
|
config = ExportConfig(
|
|
content_policy=ContentPolicy.EXCLUDE,
|
|
sensitive_patterns=["classified"],
|
|
)
|
|
exporter = ActiveLearningExporter(config=config)
|
|
record = exporter.select_record(
|
|
document_id="doc-005",
|
|
criteria=SelectionCriteria.LOW_CONFIDENCE,
|
|
source_spans=[{"text": "This is classified information", "start": 0, "end": 30}],
|
|
)
|
|
assert record is None
|
|
assert exporter.total_excluded == 1
|
|
|
|
def test_max_export_count(self):
|
|
config = ExportConfig(max_export_count=2)
|
|
exporter = ActiveLearningExporter(config=config)
|
|
for i in range(5):
|
|
exporter.select_record(
|
|
document_id=f"doc-{i}",
|
|
criteria=SelectionCriteria.LOW_CONFIDENCE,
|
|
source_spans=[{"text": f"text {i}", "start": 0, "end": 5}],
|
|
)
|
|
assert exporter.total_exported == 2
|
|
|
|
def test_export_manifest(self):
|
|
config = ExportConfig(export_version="2.0")
|
|
exporter = ActiveLearningExporter(config=config)
|
|
exporter.select_record(
|
|
document_id="doc-001",
|
|
criteria=SelectionCriteria.LOW_CONFIDENCE,
|
|
source_spans=[{"text": "test", "start": 0, "end": 4}],
|
|
)
|
|
exporter.select_record(
|
|
document_id="doc-002",
|
|
criteria=SelectionCriteria.ADJUDICATED,
|
|
source_spans=[{"text": "test2", "start": 0, "end": 5}],
|
|
)
|
|
manifest = exporter.export_manifest()
|
|
assert manifest["export_version"] == "2.0"
|
|
assert manifest["total_records"] == 2
|
|
assert manifest["selection_criteria_distribution"]["low_confidence"] == 1
|
|
assert manifest["selection_criteria_distribution"]["adjudicated"] == 1
|
|
|
|
def test_versioned_format_includes_provenance(self):
|
|
exporter = ActiveLearningExporter(config=ExportConfig())
|
|
from uuid import uuid4
|
|
|
|
run_id = uuid4()
|
|
record = exporter.select_record(
|
|
document_id="doc-001",
|
|
criteria=SelectionCriteria.CONFLICTING,
|
|
source_spans=[{"text": "test", "start": 0, "end": 4}],
|
|
pipeline_run_id=run_id,
|
|
model_versions={"gliner": "2.0", "finbert": "1.1"},
|
|
)
|
|
assert record is not None
|
|
assert record.pipeline_run_id == run_id
|
|
assert record.model_versions["gliner"] == "2.0"
|