Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
272 lines
9.3 KiB
Python
272 lines
9.3 KiB
Python
"""Deprecation tracker for legacy pipeline components.
|
|
|
|
Manages the lifecycle of deprecated components: VLLMClient, v2 prompts,
|
|
provider branching, 8000-char truncation, environment/model defaults,
|
|
provider free-text fields, and the compatibility adapter.
|
|
|
|
Removal only happens after all downstream consumers read v3 natively,
|
|
validated by consumer audit.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import enum
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
from uuid import UUID, uuid4
|
|
|
|
|
|
class DeprecationStatus(str, enum.Enum):
|
|
"""Lifecycle status of a deprecated component."""
|
|
|
|
ACTIVE = "active" # Still in use
|
|
DEPRECATED = "deprecated" # Marked for removal, consumers migrating
|
|
MIGRATION_COMPLETE = "migration_complete" # All consumers migrated
|
|
REMOVED = "removed" # Code removed
|
|
ARCHIVED = "archived" # Final reports preserved
|
|
|
|
|
|
@dataclass
|
|
class DeprecationEntry:
|
|
"""A tracked deprecated component with migration status."""
|
|
|
|
entry_id: UUID
|
|
component_name: str
|
|
component_path: str # File/module path
|
|
status: DeprecationStatus
|
|
deprecated_at: datetime
|
|
reason: str
|
|
|
|
# Consumer tracking
|
|
known_consumers: list[str] = field(default_factory=list)
|
|
migrated_consumers: list[str] = field(default_factory=list)
|
|
|
|
# Removal gates
|
|
removal_approved: bool = False
|
|
removal_approver: str = ""
|
|
removed_at: datetime | None = None
|
|
|
|
# Migration tracking
|
|
replacement: str = "" # What replaces this component
|
|
migration_notes: str = ""
|
|
|
|
@classmethod
|
|
def create(
|
|
cls,
|
|
component_name: str,
|
|
component_path: str,
|
|
reason: str,
|
|
known_consumers: list[str] | None = None,
|
|
replacement: str = "",
|
|
) -> DeprecationEntry:
|
|
return cls(
|
|
entry_id=uuid4(),
|
|
component_name=component_name,
|
|
component_path=component_path,
|
|
status=DeprecationStatus.DEPRECATED,
|
|
deprecated_at=datetime.now(timezone.utc),
|
|
reason=reason,
|
|
known_consumers=known_consumers or [],
|
|
replacement=replacement,
|
|
)
|
|
|
|
@property
|
|
def migration_progress(self) -> float:
|
|
"""Fraction of consumers that have migrated (0.0-1.0)."""
|
|
if not self.known_consumers:
|
|
return 1.0
|
|
return len(self.migrated_consumers) / len(self.known_consumers)
|
|
|
|
@property
|
|
def all_consumers_migrated(self) -> bool:
|
|
"""Whether all known consumers have migrated."""
|
|
return set(self.known_consumers) <= set(self.migrated_consumers)
|
|
|
|
def mark_consumer_migrated(self, consumer: str) -> None:
|
|
"""Record that a consumer has migrated off this component."""
|
|
if consumer not in self.migrated_consumers:
|
|
self.migrated_consumers.append(consumer)
|
|
if self.all_consumers_migrated:
|
|
self.status = DeprecationStatus.MIGRATION_COMPLETE
|
|
|
|
def approve_removal(self, approver: str) -> bool:
|
|
"""Approve removal. Only valid if all consumers migrated.
|
|
|
|
Returns False if removal cannot be approved.
|
|
"""
|
|
if not self.all_consumers_migrated:
|
|
return False
|
|
self.removal_approved = True
|
|
self.removal_approver = approver
|
|
return True
|
|
|
|
def mark_removed(self) -> None:
|
|
"""Record that the component has been removed from code."""
|
|
self.status = DeprecationStatus.REMOVED
|
|
self.removed_at = datetime.now(timezone.utc)
|
|
|
|
def archive(self) -> None:
|
|
"""Archive after final migration reports preserved."""
|
|
self.status = DeprecationStatus.ARCHIVED
|
|
|
|
|
|
# Default deprecation entries for the v3 migration
|
|
DEFAULT_DEPRECATIONS: list[dict[str, Any]] = [
|
|
{
|
|
"component_name": "VLLMClient",
|
|
"component_path": "services/extractor/vllm_client.py",
|
|
"reason": "Replaced by OpenAICompatibleClient via inference gateway",
|
|
"known_consumers": [
|
|
"services/extractor/llm_factory.py",
|
|
"services/extractor/worker.py",
|
|
],
|
|
"replacement": "services/shared/inference/clients/openai_compatible.py",
|
|
},
|
|
{
|
|
"component_name": "v2_extraction_prompt",
|
|
"component_path": "services/extractor/prompts.py",
|
|
"reason": "Monolithic prompt replaced by staged specialist extraction",
|
|
"known_consumers": [
|
|
"services/extractor/worker.py",
|
|
],
|
|
"replacement": "services/intelligence_pipeline_v3/adjudication/",
|
|
},
|
|
{
|
|
"component_name": "provider_branching",
|
|
"component_path": "services/extractor/llm_factory.py",
|
|
"reason": "Duplicated if/else provider branching replaced by registry",
|
|
"known_consumers": [
|
|
"services/extractor/worker.py",
|
|
"services/recommendation/thesis_llm.py",
|
|
],
|
|
"replacement": "services/shared/inference/registry.py",
|
|
},
|
|
{
|
|
"component_name": "8000_char_truncation",
|
|
"component_path": "services/extractor/prompts.py",
|
|
"reason": "Truncation replaced by sentence-aware segmenter",
|
|
"known_consumers": [
|
|
"services/extractor/prompts.py",
|
|
],
|
|
"replacement": "services/intelligence_pipeline_v3/segmenter/",
|
|
},
|
|
{
|
|
"component_name": "compatibility_adapter",
|
|
"component_path": "services/intelligence_pipeline_v3/compatibility/",
|
|
"reason": "Temporary adapter removed after all consumers read v3 natively",
|
|
"known_consumers": [
|
|
"services/aggregation/worker.py",
|
|
"services/recommendation/",
|
|
"services/query_api/",
|
|
],
|
|
"replacement": "Direct v3 intelligence records",
|
|
},
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class MigrationReport:
|
|
"""Summary report of the deprecation/migration status."""
|
|
|
|
generated_at: datetime
|
|
total_components: int = 0
|
|
deprecated: int = 0
|
|
migration_complete: int = 0
|
|
removed: int = 0
|
|
blocked_removals: list[str] = field(default_factory=list)
|
|
|
|
@classmethod
|
|
def generate(cls, entries: list[DeprecationEntry]) -> MigrationReport:
|
|
report = cls(
|
|
generated_at=datetime.now(timezone.utc),
|
|
total_components=len(entries),
|
|
)
|
|
for entry in entries:
|
|
if entry.status == DeprecationStatus.DEPRECATED:
|
|
report.deprecated += 1
|
|
if not entry.all_consumers_migrated:
|
|
remaining = set(entry.known_consumers) - set(
|
|
entry.migrated_consumers
|
|
)
|
|
report.blocked_removals.append(
|
|
f"{entry.component_name}: waiting on {list(remaining)}"
|
|
)
|
|
elif entry.status == DeprecationStatus.MIGRATION_COMPLETE:
|
|
report.migration_complete += 1
|
|
elif entry.status in (
|
|
DeprecationStatus.REMOVED,
|
|
DeprecationStatus.ARCHIVED,
|
|
):
|
|
report.removed += 1
|
|
return report
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"generated_at": self.generated_at.isoformat(),
|
|
"total_components": self.total_components,
|
|
"deprecated": self.deprecated,
|
|
"migration_complete": self.migration_complete,
|
|
"removed": self.removed,
|
|
"blocked_removals": self.blocked_removals,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class DeprecationTracker:
|
|
"""Tracks all deprecated components and their migration status.
|
|
|
|
Enforces that removal only happens after all consumers migrate
|
|
and with explicit approval.
|
|
"""
|
|
|
|
_entries: dict[str, DeprecationEntry] = field(default_factory=dict)
|
|
|
|
def add(self, entry: DeprecationEntry) -> None:
|
|
"""Register a deprecated component."""
|
|
self._entries[entry.component_name] = entry
|
|
|
|
def get(self, component_name: str) -> DeprecationEntry | None:
|
|
return self._entries.get(component_name)
|
|
|
|
def mark_migrated(self, component_name: str, consumer: str) -> bool:
|
|
"""Record a consumer migration. Returns False if component not found."""
|
|
entry = self._entries.get(component_name)
|
|
if entry is None:
|
|
return False
|
|
entry.mark_consumer_migrated(consumer)
|
|
return True
|
|
|
|
def can_remove(self, component_name: str) -> bool:
|
|
"""Check if a component can be safely removed."""
|
|
entry = self._entries.get(component_name)
|
|
if entry is None:
|
|
return False
|
|
return entry.all_consumers_migrated and entry.removal_approved
|
|
|
|
def approve_removal(self, component_name: str, approver: str) -> bool:
|
|
"""Approve removal of a component."""
|
|
entry = self._entries.get(component_name)
|
|
if entry is None:
|
|
return False
|
|
return entry.approve_removal(approver)
|
|
|
|
def generate_report(self) -> MigrationReport:
|
|
"""Generate a migration status report."""
|
|
return MigrationReport.generate(list(self._entries.values()))
|
|
|
|
@property
|
|
def all_entries(self) -> list[DeprecationEntry]:
|
|
return list(self._entries.values())
|
|
|
|
@property
|
|
def pending_removals(self) -> list[DeprecationEntry]:
|
|
"""Entries that are ready for removal (migrated + approved)."""
|
|
return [
|
|
e
|
|
for e in self._entries.values()
|
|
if e.all_consumers_migrated
|
|
and e.removal_approved
|
|
and e.status != DeprecationStatus.REMOVED
|
|
]
|