Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
159 lines
5.5 KiB
Python
159 lines
5.5 KiB
Python
"""Tests for audit/review module — Task 44."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from uuid import uuid4
|
|
|
|
from services.intelligence_pipeline_v3.audit.models import (
|
|
AuditRecord,
|
|
CorrectionEvent,
|
|
CorrectionType,
|
|
ReviewFilter,
|
|
ReviewStatus,
|
|
)
|
|
from services.intelligence_pipeline_v3.audit.store import AuditStore
|
|
|
|
|
|
class TestAuditRecord:
|
|
"""Task 44.1-44.2: Evidence display and specialist output tracking."""
|
|
|
|
def test_create_record(self):
|
|
record = AuditRecord.create(
|
|
document_id="doc-001",
|
|
run_id=uuid4(),
|
|
evidence_spans=[{"text": "Apple reported Q4 revenue", "start": 0, "end": 25}],
|
|
specialist_outputs={"sentiment": {"positive": 0.8}},
|
|
routing_reasons=["HIGH_CONFIDENCE"],
|
|
route_decision="fast_path",
|
|
)
|
|
assert record.document_id == "doc-001"
|
|
assert record.review_status == ReviewStatus.PENDING
|
|
assert len(record.evidence_spans) == 1
|
|
|
|
def test_add_correction(self):
|
|
record = AuditRecord.create(
|
|
document_id="doc-001", run_id=uuid4()
|
|
)
|
|
correction = CorrectionEvent.create(
|
|
record_id=record.record_id,
|
|
field_name="sentiment",
|
|
correction_type=CorrectionType.INCORRECT,
|
|
original_value="positive",
|
|
corrected_value="negative",
|
|
reviewer_id="reviewer-1",
|
|
)
|
|
record.add_correction(correction)
|
|
assert record.review_status == ReviewStatus.CORRECTED
|
|
assert len(record.corrections) == 1
|
|
|
|
def test_corrections_are_immutable(self):
|
|
correction = CorrectionEvent.create(
|
|
record_id=uuid4(),
|
|
field_name="ticker",
|
|
correction_type=CorrectionType.CORRECT,
|
|
original_value="AAPL",
|
|
reviewer_id="reviewer-1",
|
|
)
|
|
# Frozen dataclass — cannot modify
|
|
assert correction.event_id is not None
|
|
assert correction.timestamp is not None
|
|
|
|
def test_mark_reviewed(self):
|
|
record = AuditRecord.create(document_id="doc-001", run_id=uuid4())
|
|
record.mark_reviewed()
|
|
assert record.review_status == ReviewStatus.REVIEWED
|
|
|
|
def test_mark_confirmed(self):
|
|
record = AuditRecord.create(document_id="doc-001", run_id=uuid4())
|
|
record.mark_confirmed()
|
|
assert record.review_status == ReviewStatus.CONFIRMED
|
|
|
|
|
|
class TestReviewFilter:
|
|
"""Task 44.4: Filters for low confidence, unsupported claims, adjudicated."""
|
|
|
|
def test_filter_adjudicated(self):
|
|
f = ReviewFilter(is_adjudicated=True)
|
|
record_adj = AuditRecord.create(
|
|
document_id="doc-001",
|
|
run_id=uuid4(),
|
|
adjudicator_decision={"resolved": True},
|
|
)
|
|
record_fast = AuditRecord.create(
|
|
document_id="doc-002", run_id=uuid4()
|
|
)
|
|
assert f.matches(record_adj)
|
|
assert not f.matches(record_fast)
|
|
|
|
def test_filter_by_review_status(self):
|
|
f = ReviewFilter(review_status=ReviewStatus.CORRECTED)
|
|
record = AuditRecord.create(document_id="doc-001", run_id=uuid4())
|
|
assert not f.matches(record)
|
|
record.review_status = ReviewStatus.CORRECTED
|
|
assert f.matches(record)
|
|
|
|
def test_filter_unsupported_claims(self):
|
|
f = ReviewFilter(has_unsupported_claims=True)
|
|
record = AuditRecord.create(document_id="doc-001", run_id=uuid4())
|
|
assert not f.matches(record)
|
|
# Add unsupported correction
|
|
record.add_correction(
|
|
CorrectionEvent.create(
|
|
record_id=record.record_id,
|
|
field_name="fact",
|
|
correction_type=CorrectionType.UNSUPPORTED,
|
|
original_value="revenue beat",
|
|
reviewer_id="r1",
|
|
)
|
|
)
|
|
assert f.matches(record)
|
|
|
|
|
|
class TestAuditStore:
|
|
"""Task 44: Storage and retrieval."""
|
|
|
|
def test_store_and_retrieve(self):
|
|
store = AuditStore()
|
|
record = AuditRecord.create(document_id="doc-001", run_id=uuid4())
|
|
store.store(record)
|
|
assert store.get(record.record_id) is record
|
|
assert store.count() == 1
|
|
|
|
def test_get_by_document(self):
|
|
store = AuditStore()
|
|
run1 = uuid4()
|
|
run2 = uuid4()
|
|
store.store(AuditRecord.create(document_id="doc-001", run_id=run1))
|
|
store.store(AuditRecord.create(document_id="doc-001", run_id=run2))
|
|
store.store(AuditRecord.create(document_id="doc-002", run_id=uuid4()))
|
|
assert len(store.get_by_document("doc-001")) == 2
|
|
|
|
def test_add_correction_to_record(self):
|
|
store = AuditStore()
|
|
record = AuditRecord.create(document_id="doc-001", run_id=uuid4())
|
|
store.store(record)
|
|
correction = CorrectionEvent.create(
|
|
record_id=record.record_id,
|
|
field_name="ticker",
|
|
correction_type=CorrectionType.VALUE_OVERRIDE,
|
|
original_value="GOOG",
|
|
corrected_value="GOOGL",
|
|
reviewer_id="r1",
|
|
)
|
|
assert store.add_correction(record.record_id, correction)
|
|
assert store.correction_count() == 1
|
|
|
|
def test_filter(self):
|
|
store = AuditStore()
|
|
r1 = AuditRecord.create(
|
|
document_id="doc-001",
|
|
run_id=uuid4(),
|
|
adjudicator_decision={"x": 1},
|
|
)
|
|
r2 = AuditRecord.create(document_id="doc-002", run_id=uuid4())
|
|
store.store(r1)
|
|
store.store(r2)
|
|
results = store.filter(ReviewFilter(is_adjudicated=True))
|
|
assert len(results) == 1
|
|
assert results[0].document_id == "doc-001"
|