feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,20 @@
|
||||
"""Legacy path deprecation tracking and cleanup management.
|
||||
|
||||
Tracks deprecated components (VLLMClient, v2 prompts, provider branching),
|
||||
validates that all consumers have migrated, and provides safe removal
|
||||
gating. Removal only proceeds after all downstream consumers read v3.
|
||||
"""
|
||||
|
||||
from services.intelligence_pipeline_v3.deprecation.tracker import (
|
||||
DeprecationEntry,
|
||||
DeprecationStatus,
|
||||
DeprecationTracker,
|
||||
MigrationReport,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DeprecationEntry",
|
||||
"DeprecationStatus",
|
||||
"DeprecationTracker",
|
||||
"MigrationReport",
|
||||
]
|
||||
@@ -0,0 +1,271 @@
|
||||
"""Deprecation tracker for legacy pipeline components.
|
||||
|
||||
Manages the lifecycle of deprecated components: VLLMClient, v2 prompts,
|
||||
provider branching, 8000-char truncation, environment/model defaults,
|
||||
provider free-text fields, and the compatibility adapter.
|
||||
|
||||
Removal only happens after all downstream consumers read v3 natively,
|
||||
validated by consumer audit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
|
||||
class DeprecationStatus(str, enum.Enum):
|
||||
"""Lifecycle status of a deprecated component."""
|
||||
|
||||
ACTIVE = "active" # Still in use
|
||||
DEPRECATED = "deprecated" # Marked for removal, consumers migrating
|
||||
MIGRATION_COMPLETE = "migration_complete" # All consumers migrated
|
||||
REMOVED = "removed" # Code removed
|
||||
ARCHIVED = "archived" # Final reports preserved
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeprecationEntry:
|
||||
"""A tracked deprecated component with migration status."""
|
||||
|
||||
entry_id: UUID
|
||||
component_name: str
|
||||
component_path: str # File/module path
|
||||
status: DeprecationStatus
|
||||
deprecated_at: datetime
|
||||
reason: str
|
||||
|
||||
# Consumer tracking
|
||||
known_consumers: list[str] = field(default_factory=list)
|
||||
migrated_consumers: list[str] = field(default_factory=list)
|
||||
|
||||
# Removal gates
|
||||
removal_approved: bool = False
|
||||
removal_approver: str = ""
|
||||
removed_at: datetime | None = None
|
||||
|
||||
# Migration tracking
|
||||
replacement: str = "" # What replaces this component
|
||||
migration_notes: str = ""
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls,
|
||||
component_name: str,
|
||||
component_path: str,
|
||||
reason: str,
|
||||
known_consumers: list[str] | None = None,
|
||||
replacement: str = "",
|
||||
) -> DeprecationEntry:
|
||||
return cls(
|
||||
entry_id=uuid4(),
|
||||
component_name=component_name,
|
||||
component_path=component_path,
|
||||
status=DeprecationStatus.DEPRECATED,
|
||||
deprecated_at=datetime.now(timezone.utc),
|
||||
reason=reason,
|
||||
known_consumers=known_consumers or [],
|
||||
replacement=replacement,
|
||||
)
|
||||
|
||||
@property
|
||||
def migration_progress(self) -> float:
|
||||
"""Fraction of consumers that have migrated (0.0-1.0)."""
|
||||
if not self.known_consumers:
|
||||
return 1.0
|
||||
return len(self.migrated_consumers) / len(self.known_consumers)
|
||||
|
||||
@property
|
||||
def all_consumers_migrated(self) -> bool:
|
||||
"""Whether all known consumers have migrated."""
|
||||
return set(self.known_consumers) <= set(self.migrated_consumers)
|
||||
|
||||
def mark_consumer_migrated(self, consumer: str) -> None:
|
||||
"""Record that a consumer has migrated off this component."""
|
||||
if consumer not in self.migrated_consumers:
|
||||
self.migrated_consumers.append(consumer)
|
||||
if self.all_consumers_migrated:
|
||||
self.status = DeprecationStatus.MIGRATION_COMPLETE
|
||||
|
||||
def approve_removal(self, approver: str) -> bool:
|
||||
"""Approve removal. Only valid if all consumers migrated.
|
||||
|
||||
Returns False if removal cannot be approved.
|
||||
"""
|
||||
if not self.all_consumers_migrated:
|
||||
return False
|
||||
self.removal_approved = True
|
||||
self.removal_approver = approver
|
||||
return True
|
||||
|
||||
def mark_removed(self) -> None:
|
||||
"""Record that the component has been removed from code."""
|
||||
self.status = DeprecationStatus.REMOVED
|
||||
self.removed_at = datetime.now(timezone.utc)
|
||||
|
||||
def archive(self) -> None:
|
||||
"""Archive after final migration reports preserved."""
|
||||
self.status = DeprecationStatus.ARCHIVED
|
||||
|
||||
|
||||
# Default deprecation entries for the v3 migration
|
||||
DEFAULT_DEPRECATIONS: list[dict[str, Any]] = [
|
||||
{
|
||||
"component_name": "VLLMClient",
|
||||
"component_path": "services/extractor/vllm_client.py",
|
||||
"reason": "Replaced by OpenAICompatibleClient via inference gateway",
|
||||
"known_consumers": [
|
||||
"services/extractor/llm_factory.py",
|
||||
"services/extractor/worker.py",
|
||||
],
|
||||
"replacement": "services/shared/inference/clients/openai_compatible.py",
|
||||
},
|
||||
{
|
||||
"component_name": "v2_extraction_prompt",
|
||||
"component_path": "services/extractor/prompts.py",
|
||||
"reason": "Monolithic prompt replaced by staged specialist extraction",
|
||||
"known_consumers": [
|
||||
"services/extractor/worker.py",
|
||||
],
|
||||
"replacement": "services/intelligence_pipeline_v3/adjudication/",
|
||||
},
|
||||
{
|
||||
"component_name": "provider_branching",
|
||||
"component_path": "services/extractor/llm_factory.py",
|
||||
"reason": "Duplicated if/else provider branching replaced by registry",
|
||||
"known_consumers": [
|
||||
"services/extractor/worker.py",
|
||||
"services/recommendation/thesis_llm.py",
|
||||
],
|
||||
"replacement": "services/shared/inference/registry.py",
|
||||
},
|
||||
{
|
||||
"component_name": "8000_char_truncation",
|
||||
"component_path": "services/extractor/prompts.py",
|
||||
"reason": "Truncation replaced by sentence-aware segmenter",
|
||||
"known_consumers": [
|
||||
"services/extractor/prompts.py",
|
||||
],
|
||||
"replacement": "services/intelligence_pipeline_v3/segmenter/",
|
||||
},
|
||||
{
|
||||
"component_name": "compatibility_adapter",
|
||||
"component_path": "services/intelligence_pipeline_v3/compatibility/",
|
||||
"reason": "Temporary adapter removed after all consumers read v3 natively",
|
||||
"known_consumers": [
|
||||
"services/aggregation/worker.py",
|
||||
"services/recommendation/",
|
||||
"services/query_api/",
|
||||
],
|
||||
"replacement": "Direct v3 intelligence records",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MigrationReport:
|
||||
"""Summary report of the deprecation/migration status."""
|
||||
|
||||
generated_at: datetime
|
||||
total_components: int = 0
|
||||
deprecated: int = 0
|
||||
migration_complete: int = 0
|
||||
removed: int = 0
|
||||
blocked_removals: list[str] = field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def generate(cls, entries: list[DeprecationEntry]) -> MigrationReport:
|
||||
report = cls(
|
||||
generated_at=datetime.now(timezone.utc),
|
||||
total_components=len(entries),
|
||||
)
|
||||
for entry in entries:
|
||||
if entry.status == DeprecationStatus.DEPRECATED:
|
||||
report.deprecated += 1
|
||||
if not entry.all_consumers_migrated:
|
||||
remaining = set(entry.known_consumers) - set(
|
||||
entry.migrated_consumers
|
||||
)
|
||||
report.blocked_removals.append(
|
||||
f"{entry.component_name}: waiting on {list(remaining)}"
|
||||
)
|
||||
elif entry.status == DeprecationStatus.MIGRATION_COMPLETE:
|
||||
report.migration_complete += 1
|
||||
elif entry.status in (
|
||||
DeprecationStatus.REMOVED,
|
||||
DeprecationStatus.ARCHIVED,
|
||||
):
|
||||
report.removed += 1
|
||||
return report
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"generated_at": self.generated_at.isoformat(),
|
||||
"total_components": self.total_components,
|
||||
"deprecated": self.deprecated,
|
||||
"migration_complete": self.migration_complete,
|
||||
"removed": self.removed,
|
||||
"blocked_removals": self.blocked_removals,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeprecationTracker:
|
||||
"""Tracks all deprecated components and their migration status.
|
||||
|
||||
Enforces that removal only happens after all consumers migrate
|
||||
and with explicit approval.
|
||||
"""
|
||||
|
||||
_entries: dict[str, DeprecationEntry] = field(default_factory=dict)
|
||||
|
||||
def add(self, entry: DeprecationEntry) -> None:
|
||||
"""Register a deprecated component."""
|
||||
self._entries[entry.component_name] = entry
|
||||
|
||||
def get(self, component_name: str) -> DeprecationEntry | None:
|
||||
return self._entries.get(component_name)
|
||||
|
||||
def mark_migrated(self, component_name: str, consumer: str) -> bool:
|
||||
"""Record a consumer migration. Returns False if component not found."""
|
||||
entry = self._entries.get(component_name)
|
||||
if entry is None:
|
||||
return False
|
||||
entry.mark_consumer_migrated(consumer)
|
||||
return True
|
||||
|
||||
def can_remove(self, component_name: str) -> bool:
|
||||
"""Check if a component can be safely removed."""
|
||||
entry = self._entries.get(component_name)
|
||||
if entry is None:
|
||||
return False
|
||||
return entry.all_consumers_migrated and entry.removal_approved
|
||||
|
||||
def approve_removal(self, component_name: str, approver: str) -> bool:
|
||||
"""Approve removal of a component."""
|
||||
entry = self._entries.get(component_name)
|
||||
if entry is None:
|
||||
return False
|
||||
return entry.approve_removal(approver)
|
||||
|
||||
def generate_report(self) -> MigrationReport:
|
||||
"""Generate a migration status report."""
|
||||
return MigrationReport.generate(list(self._entries.values()))
|
||||
|
||||
@property
|
||||
def all_entries(self) -> list[DeprecationEntry]:
|
||||
return list(self._entries.values())
|
||||
|
||||
@property
|
||||
def pending_removals(self) -> list[DeprecationEntry]:
|
||||
"""Entries that are ready for removal (migrated + approved)."""
|
||||
return [
|
||||
e
|
||||
for e in self._entries.values()
|
||||
if e.all_consumers_migrated
|
||||
and e.removal_approved
|
||||
and e.status != DeprecationStatus.REMOVED
|
||||
]
|
||||
Reference in New Issue
Block a user