Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
143 lines
4.1 KiB
Python
143 lines
4.1 KiB
Python
"""Conservative confidence defaults for underrepresented classes.
|
|
|
|
When calibration data is insufficient for a specific document type or
|
|
event class, returns conservative values (0.3-0.5) and marks the result
|
|
as under-calibrated per Requirement 10.7.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
from services.intelligence_pipeline_v3.confidence.models import ConfidenceResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Conservative default probabilities by document type.
|
|
# These are intentionally low (0.3-0.5) to avoid overconfidence
|
|
# when insufficient calibration data exists.
|
|
_DOCUMENT_TYPE_DEFAULTS: dict[str, float] = {
|
|
"news": 0.45,
|
|
"filing": 0.40,
|
|
"transcript": 0.40,
|
|
"press_release": 0.45,
|
|
"macro_event": 0.35,
|
|
"unknown": 0.30,
|
|
}
|
|
|
|
# Conservative default probabilities by event class.
|
|
# More complex or rare event types get lower defaults.
|
|
_EVENT_CLASS_DEFAULTS: dict[str, float] = {
|
|
"earnings_beat": 0.50,
|
|
"earnings_miss": 0.50,
|
|
"guidance_raise": 0.45,
|
|
"guidance_cut": 0.45,
|
|
"merger_acquisition": 0.40,
|
|
"product_launch": 0.45,
|
|
"regulatory_action": 0.40,
|
|
"management_change": 0.45,
|
|
"legal_proceeding": 0.40,
|
|
"supply_chain": 0.35,
|
|
"rating_change": 0.45,
|
|
"dividend_change": 0.45,
|
|
"buyback": 0.45,
|
|
"macro_policy": 0.35,
|
|
"geopolitical": 0.30,
|
|
"sector_rotation": 0.35,
|
|
"unknown": 0.30,
|
|
}
|
|
|
|
# Features used when returning conservative defaults
|
|
_DEFAULT_FEATURES_USED = [
|
|
"document_type_prior",
|
|
"event_class_prior",
|
|
]
|
|
|
|
|
|
def get_default_confidence(
|
|
document_type: str,
|
|
event_class: str,
|
|
) -> ConfidenceResult:
|
|
"""Return a conservative confidence result for underrepresented classes.
|
|
|
|
Used when calibration data is insufficient for the given document type
|
|
and event class combination. Returns conservative probabilities (0.3-0.5)
|
|
and marks the result as under-calibrated.
|
|
|
|
Parameters
|
|
----------
|
|
document_type
|
|
The document type (news, filing, transcript, etc.).
|
|
event_class
|
|
The classified event type (earnings_beat, merger_acquisition, etc.).
|
|
|
|
Returns
|
|
-------
|
|
ConfidenceResult
|
|
A conservative confidence result with under_calibrated=True.
|
|
"""
|
|
doc_default = _DOCUMENT_TYPE_DEFAULTS.get(
|
|
document_type, _DOCUMENT_TYPE_DEFAULTS["unknown"]
|
|
)
|
|
event_default = _EVENT_CLASS_DEFAULTS.get(
|
|
event_class, _EVENT_CLASS_DEFAULTS["unknown"]
|
|
)
|
|
|
|
# Take the minimum of document and event defaults for extra conservatism
|
|
probability = min(doc_default, event_default)
|
|
|
|
logger.debug(
|
|
"Using conservative default confidence: doc_type=%s (%.2f), event=%s (%.2f) -> %.2f",
|
|
document_type,
|
|
doc_default,
|
|
event_class,
|
|
event_default,
|
|
probability,
|
|
)
|
|
|
|
return ConfidenceResult(
|
|
probability=probability,
|
|
features_used=_DEFAULT_FEATURES_USED,
|
|
is_calibrated=False,
|
|
under_calibrated=True,
|
|
calibration_version="conservative-default-v1",
|
|
)
|
|
|
|
|
|
def is_underrepresented(
|
|
document_type: str,
|
|
event_class: str,
|
|
min_samples: int = 30,
|
|
known_counts: dict[tuple[str, str], int] | None = None,
|
|
) -> bool:
|
|
"""Check if a document_type + event_class combination is underrepresented.
|
|
|
|
Parameters
|
|
----------
|
|
document_type
|
|
The document type.
|
|
event_class
|
|
The event class.
|
|
min_samples
|
|
Minimum number of calibration samples to consider a class well-represented.
|
|
known_counts
|
|
Optional mapping of (doc_type, event_class) -> sample count.
|
|
If None, treats any unknown combination as underrepresented.
|
|
|
|
Returns
|
|
-------
|
|
bool
|
|
True if the class has insufficient calibration data.
|
|
"""
|
|
if known_counts is None:
|
|
# Without explicit counts, use heuristic: unknown types are underrepresented
|
|
if document_type not in _DOCUMENT_TYPE_DEFAULTS:
|
|
return True
|
|
if event_class not in _EVENT_CLASS_DEFAULTS:
|
|
return True
|
|
return False
|
|
|
|
key = (document_type, event_class)
|
|
count = known_counts.get(key, 0)
|
|
return count < min_samples
|