feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,200 @@
|
||||
"""Inference Gateway facade.
|
||||
|
||||
Wraps the factory + client lifecycle for all LLM consumers.
|
||||
Provides a single ``generate()`` method that dispatches to the correct
|
||||
client implementation based on the InferenceTarget protocol.
|
||||
|
||||
Manages client instances (one per endpoint, reusable) and provides
|
||||
explicit ``refresh_target()`` for configuration changes — no private
|
||||
``_config`` mutation.
|
||||
|
||||
Requirements: 2.1, 2.12, 13.6
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Protocol, runtime_checkable
|
||||
from uuid import UUID
|
||||
|
||||
from services.shared.inference.models import (
|
||||
InferenceResult,
|
||||
InferenceTarget,
|
||||
StructuredGenerationRequest,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("inference.gateway")
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class InferenceClient(Protocol):
|
||||
"""Protocol for inference client implementations."""
|
||||
|
||||
async def generate(self, request: StructuredGenerationRequest) -> InferenceResult:
|
||||
"""Send a structured generation request and return the result."""
|
||||
...
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Release underlying HTTP resources."""
|
||||
...
|
||||
|
||||
|
||||
class InferenceGateway:
|
||||
"""Gateway facade that routes inference requests to protocol-specific clients.
|
||||
|
||||
Key behaviours:
|
||||
- Maintains a client pool keyed by (endpoint_id, protocol)
|
||||
- Creates clients lazily on first request for a target
|
||||
- Reuses existing clients for the same endpoint
|
||||
- Provides ``refresh_target()`` to invalidate a cached client
|
||||
(replaces private ``_config`` mutation pattern)
|
||||
- Fails closed on unknown protocols
|
||||
|
||||
Usage::
|
||||
|
||||
gateway = InferenceGateway()
|
||||
result = await gateway.generate(target, request)
|
||||
# Later, when config changes:
|
||||
await gateway.refresh_target(target.endpoint_id)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Pool of active clients keyed by endpoint_id
|
||||
self._clients: dict[UUID, InferenceClient] = {}
|
||||
# Track the target associated with each client for refresh comparison
|
||||
self._targets: dict[UUID, InferenceTarget] = {}
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
target: InferenceTarget,
|
||||
request: StructuredGenerationRequest,
|
||||
) -> InferenceResult:
|
||||
"""Route a structured generation request to the appropriate client.
|
||||
|
||||
Creates or reuses a client for the target's endpoint. The result
|
||||
always includes endpoint_id, deployment_id, model, and protocol
|
||||
for lineage recording.
|
||||
|
||||
Args:
|
||||
target: Resolved inference target with protocol, endpoint, and capabilities.
|
||||
request: The structured generation request.
|
||||
|
||||
Returns:
|
||||
InferenceResult with content and full lineage metadata.
|
||||
|
||||
Raises:
|
||||
ValueError: If the target protocol is unknown (fail-closed).
|
||||
"""
|
||||
client = self._get_or_create_client(target)
|
||||
result = await client.generate(request)
|
||||
|
||||
# Ensure lineage fields are populated from target
|
||||
if result.endpoint_id is None:
|
||||
result.endpoint_id = target.endpoint_id
|
||||
if result.deployment_id is None:
|
||||
result.deployment_id = target.deployment_id
|
||||
if not result.model:
|
||||
result.model = target.model
|
||||
if not result.protocol or result.protocol != target.protocol:
|
||||
result.protocol = target.protocol
|
||||
|
||||
return result
|
||||
|
||||
async def refresh_target(self, endpoint_id: UUID) -> None:
|
||||
"""Invalidate and close the cached client for an endpoint.
|
||||
|
||||
Call this when endpoint configuration changes (model swap,
|
||||
URL change, credential rotation, etc.) instead of mutating
|
||||
private ``_config`` attributes.
|
||||
|
||||
The next ``generate()`` call for this endpoint will create
|
||||
a fresh client from the new target.
|
||||
"""
|
||||
client = self._clients.pop(endpoint_id, None)
|
||||
self._targets.pop(endpoint_id, None)
|
||||
if client is not None:
|
||||
logger.info("Refreshing client for endpoint %s", endpoint_id)
|
||||
try:
|
||||
await client.close()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Error closing client during refresh for endpoint %s",
|
||||
endpoint_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close all managed clients and release resources."""
|
||||
for endpoint_id, client in list(self._clients.items()):
|
||||
try:
|
||||
await client.close()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Error closing client for endpoint %s",
|
||||
endpoint_id,
|
||||
exc_info=True,
|
||||
)
|
||||
self._clients.clear()
|
||||
self._targets.clear()
|
||||
|
||||
def _get_or_create_client(self, target: InferenceTarget) -> InferenceClient:
|
||||
"""Get an existing client or create a new one for the target.
|
||||
|
||||
Clients are cached by endpoint_id. If the target has changed
|
||||
(different model, URL, etc.), the old client is replaced.
|
||||
"""
|
||||
endpoint_id = target.endpoint_id
|
||||
existing = self._clients.get(endpoint_id)
|
||||
|
||||
if existing is not None:
|
||||
# Reuse if target hasn't changed
|
||||
cached_target = self._targets.get(endpoint_id)
|
||||
if cached_target is target or cached_target == target:
|
||||
return existing
|
||||
|
||||
# Create a new client for the target's protocol
|
||||
client = self._create_client(target)
|
||||
self._clients[endpoint_id] = client
|
||||
self._targets[endpoint_id] = target
|
||||
return client
|
||||
|
||||
def _create_client(self, target: InferenceTarget) -> InferenceClient:
|
||||
"""Create a protocol-specific client for the target.
|
||||
|
||||
Raises:
|
||||
ValueError: If the protocol is unknown (fail-closed per Req 2.6).
|
||||
"""
|
||||
protocol = target.protocol
|
||||
|
||||
if protocol == "openai_chat":
|
||||
from services.shared.inference.clients.openai_compatible import (
|
||||
OpenAICompatibleClient,
|
||||
)
|
||||
logger.info(
|
||||
"Creating OpenAICompatibleClient for endpoint %s (model=%s)",
|
||||
target.endpoint_id,
|
||||
target.model,
|
||||
)
|
||||
return OpenAICompatibleClient(target)
|
||||
|
||||
if protocol == "ollama_native":
|
||||
from services.shared.inference.clients.ollama_native import (
|
||||
OllamaNativeClient,
|
||||
)
|
||||
logger.info(
|
||||
"Creating OllamaNativeClient for endpoint %s (model=%s)",
|
||||
target.endpoint_id,
|
||||
target.model,
|
||||
)
|
||||
return OllamaNativeClient(target)
|
||||
|
||||
# Unknown protocol — fail closed (Requirement 2.6)
|
||||
raise ValueError(
|
||||
f"Unknown inference protocol '{protocol}' for endpoint {target.endpoint_id}. "
|
||||
"Supported protocols: 'ollama_native', 'openai_chat', 'specialist_http'. "
|
||||
"The gateway will NOT silently route to a default provider."
|
||||
)
|
||||
|
||||
@property
|
||||
def active_endpoints(self) -> list[UUID]:
|
||||
"""Return the list of endpoint IDs with active cached clients."""
|
||||
return list(self._clients.keys())
|
||||
Reference in New Issue
Block a user