"""Inference Gateway facade. Wraps the factory + client lifecycle for all LLM consumers. Provides a single ``generate()`` method that dispatches to the correct client implementation based on the InferenceTarget protocol. Manages client instances (one per endpoint, reusable) and provides explicit ``refresh_target()`` for configuration changes — no private ``_config`` mutation. Requirements: 2.1, 2.12, 13.6 """ from __future__ import annotations import logging from typing import Protocol, runtime_checkable from uuid import UUID from services.shared.inference.models import ( InferenceResult, InferenceTarget, StructuredGenerationRequest, ) logger = logging.getLogger("inference.gateway") @runtime_checkable class InferenceClient(Protocol): """Protocol for inference client implementations.""" async def generate(self, request: StructuredGenerationRequest) -> InferenceResult: """Send a structured generation request and return the result.""" ... async def close(self) -> None: """Release underlying HTTP resources.""" ... class InferenceGateway: """Gateway facade that routes inference requests to protocol-specific clients. Key behaviours: - Maintains a client pool keyed by (endpoint_id, protocol) - Creates clients lazily on first request for a target - Reuses existing clients for the same endpoint - Provides ``refresh_target()`` to invalidate a cached client (replaces private ``_config`` mutation pattern) - Fails closed on unknown protocols Usage:: gateway = InferenceGateway() result = await gateway.generate(target, request) # Later, when config changes: await gateway.refresh_target(target.endpoint_id) """ def __init__(self) -> None: # Pool of active clients keyed by endpoint_id self._clients: dict[UUID, InferenceClient] = {} # Track the target associated with each client for refresh comparison self._targets: dict[UUID, InferenceTarget] = {} async def generate( self, target: InferenceTarget, request: StructuredGenerationRequest, ) -> InferenceResult: """Route a structured generation request to the appropriate client. Creates or reuses a client for the target's endpoint. The result always includes endpoint_id, deployment_id, model, and protocol for lineage recording. Args: target: Resolved inference target with protocol, endpoint, and capabilities. request: The structured generation request. Returns: InferenceResult with content and full lineage metadata. Raises: ValueError: If the target protocol is unknown (fail-closed). """ client = self._get_or_create_client(target) result = await client.generate(request) # Ensure lineage fields are populated from target if result.endpoint_id is None: result.endpoint_id = target.endpoint_id if result.deployment_id is None: result.deployment_id = target.deployment_id if not result.model: result.model = target.model if not result.protocol or result.protocol != target.protocol: result.protocol = target.protocol return result async def refresh_target(self, endpoint_id: UUID) -> None: """Invalidate and close the cached client for an endpoint. Call this when endpoint configuration changes (model swap, URL change, credential rotation, etc.) instead of mutating private ``_config`` attributes. The next ``generate()`` call for this endpoint will create a fresh client from the new target. """ client = self._clients.pop(endpoint_id, None) self._targets.pop(endpoint_id, None) if client is not None: logger.info("Refreshing client for endpoint %s", endpoint_id) try: await client.close() except Exception: logger.warning( "Error closing client during refresh for endpoint %s", endpoint_id, exc_info=True, ) async def close(self) -> None: """Close all managed clients and release resources.""" for endpoint_id, client in list(self._clients.items()): try: await client.close() except Exception: logger.warning( "Error closing client for endpoint %s", endpoint_id, exc_info=True, ) self._clients.clear() self._targets.clear() def _get_or_create_client(self, target: InferenceTarget) -> InferenceClient: """Get an existing client or create a new one for the target. Clients are cached by endpoint_id. If the target has changed (different model, URL, etc.), the old client is replaced. """ endpoint_id = target.endpoint_id existing = self._clients.get(endpoint_id) if existing is not None: # Reuse if target hasn't changed cached_target = self._targets.get(endpoint_id) if cached_target is target or cached_target == target: return existing # Create a new client for the target's protocol client = self._create_client(target) self._clients[endpoint_id] = client self._targets[endpoint_id] = target return client def _create_client(self, target: InferenceTarget) -> InferenceClient: """Create a protocol-specific client for the target. Raises: ValueError: If the protocol is unknown (fail-closed per Req 2.6). """ protocol = target.protocol if protocol == "openai_chat": from services.shared.inference.clients.openai_compatible import ( OpenAICompatibleClient, ) logger.info( "Creating OpenAICompatibleClient for endpoint %s (model=%s)", target.endpoint_id, target.model, ) return OpenAICompatibleClient(target) if protocol == "ollama_native": from services.shared.inference.clients.ollama_native import ( OllamaNativeClient, ) logger.info( "Creating OllamaNativeClient for endpoint %s (model=%s)", target.endpoint_id, target.model, ) return OllamaNativeClient(target) # Unknown protocol — fail closed (Requirement 2.6) raise ValueError( f"Unknown inference protocol '{protocol}' for endpoint {target.endpoint_id}. " "Supported protocols: 'ollama_native', 'openai_chat', 'specialist_http'. " "The gateway will NOT silently route to a default provider." ) @property def active_endpoints(self) -> list[UUID]: """Return the list of endpoint IDs with active cached clients.""" return list(self._clients.keys())