feat: Intelligence Pipeline v3 — full implementation

Multi-stage evidence-grounded inference architecture replacing the
monolithic 9B model extraction pipeline. CPU-first specialist services
handle routine extraction while the 9B vLLM model is preserved for
semantic adjudication of ambiguous cases.

Key components:
- Capability-aware inference gateway (OpenAI-compatible + Ollama)
- Endpoint registry with DB migrations and REST API
- Sentence-aware document segmenter (property tests)
- Deterministic financial parsing with offset integrity
- Symbol resolution with ambiguity detection
- Specialist service (GLiNER2, dynamic batching, K8s deployment)
- Company-specific sentiment (FinBERT, calibration)
- Retrieval-based novelty and duplicate detection
- Confidence calibration pipeline
- Deterministic routing engine (property tests)
- 9B adjudication layer with VRAM gating
- Stock-specific impact model (features, labels, baseline, trained)
- Pipeline orchestrator (state machine, queues, leases, feature flags)
- Bounded parallelism (async workers, semaphore, load shedding)
- Observability (tracing, metrics, alerts)
- Compatibility adapter (v3→v2 golden mapping tests)
- Shadow/canary promotion framework
- Active learning and fine-tuning pipeline

Test results: 1,161 tests pass, ruff lint clean.
All 282 spec tasks completed.

services/shared/inference/__init__.py

@@ -0,0 +1,33 @@
+"""Inference gateway: shared client and routing layer.
+
+Exports:
+    InferenceGateway - Gateway facade for all LLM consumers
+    InferenceTarget - Resolved target configuration
+    StructuredGenerationRequest - Structured generation request
+    InferenceResult - Inference response with lineage
+    ModelLineage - Lineage record for persistence
+    build_lineage_from_result - Extract lineage from a result
+"""
+from services.shared.inference.gateway import InferenceGateway
+from services.shared.inference.lineage import build_lineage_from_result
+from services.shared.inference.models import (
+    ChatMessage,
+    InferenceResult,
+    InferenceTarget,
+    ModelLineage,
+    ProviderCapabilities,
+    StructuredGenerationRequest,
+    TokenUsage,
+)
+
+__all__ = [
+    "InferenceGateway",
+    "InferenceTarget",
+    "StructuredGenerationRequest",
+    "InferenceResult",
+    "ChatMessage",
+    "ModelLineage",
+    "ProviderCapabilities",
+    "TokenUsage",
+    "build_lineage_from_result",
+]

services/shared/inference/capabilities.py

@@ -0,0 +1,683 @@
+"""Capability probing for inference endpoints.
+
+Discovers actual endpoint capabilities by sending probe requests,
+rather than relying solely on declared capabilities. Probe results
+are cached with a configurable TTL and used to validate that
+required capabilities work before routing real requests.
+
+Requirements: 2.10, 2.11
+"""
+from __future__ import annotations
+
+import json
+import logging
+import time
+from datetime import datetime, timezone
+from typing import Any
+from uuid import UUID
+
+import httpx
+from pydantic import BaseModel, Field
+
+from services.shared.inference.models import (
+    ChatMessage,
+    InferenceTarget,
+)
+
+logger = logging.getLogger(__name__)
+
+# Default TTL for probe result cache (seconds)
+DEFAULT_PROBE_TTL_SECONDS = 300  # 5 minutes
+
+
+# ---------------------------------------------------------------------------
+# Probe result models
+# ---------------------------------------------------------------------------
+
+
+class HealthProbeResult(BaseModel):
+    """Result of a health probe against an endpoint."""
+
+    success: bool
+    detail: str = ""
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    software_version: str | None = None
+    latency_ms: int = 0
+
+
+class ModelListingResult(BaseModel):
+    """Result of probing the /v1/models endpoint."""
+
+    success: bool
+    detail: str = ""
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    software_version: str | None = None
+    models: list[str] = Field(default_factory=list)
+    target_model_found: bool = False
+
+
+class JsonSchemaProbeResult(BaseModel):
+    """Result of probing strict JSON Schema output."""
+
+    success: bool
+    detail: str = ""
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    software_version: str | None = None
+    schema_valid: bool = False
+    structured_mode_used: str = ""
+
+
+class UsageProbeResult(BaseModel):
+    """Result of probing usage metadata return."""
+
+    success: bool
+    detail: str = ""
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    software_version: str | None = None
+    has_prompt_tokens: bool = False
+    has_completion_tokens: bool = False
+
+
+class SeedProbeResult(BaseModel):
+    """Result of probing seed/determinism behavior."""
+
+    success: bool
+    detail: str = ""
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    software_version: str | None = None
+    outputs_match: bool = False
+
+
+class OutputTokenFieldResult(BaseModel):
+    """Result of probing max_completion_tokens field support."""
+
+    success: bool
+    detail: str = ""
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    software_version: str | None = None
+    field_accepted: bool = False
+
+
+class FullProbeResult(BaseModel):
+    """Aggregated results from all capability probes."""
+
+    endpoint_id: UUID
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    health: HealthProbeResult | None = None
+    model_listing: ModelListingResult | None = None
+    json_schema: JsonSchemaProbeResult | None = None
+    usage_metadata: UsageProbeResult | None = None
+    seed_determinism: SeedProbeResult | None = None
+    output_token_field: OutputTokenFieldResult | None = None
+    software_version: str | None = None
+    probe_duration_ms: int = 0
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+_MINIMAL_PROBE_SCHEMA = {
+    "title": "probe_response",
+    "type": "object",
+    "properties": {
+        "status": {"type": "string"},
+    },
+    "required": ["status"],
+}
+
+_PROBE_MESSAGES = [
+    ChatMessage(role="user", content="Respond with JSON: {\"status\": \"ok\"}"),
+]
+
+
+def _resolve_auth_headers(target: InferenceTarget) -> dict[str, str]:
+    """Build auth headers for probe requests from the target."""
+    import os
+
+    headers: dict[str, str] = {"Content-Type": "application/json"}
+    headers.update(target.extra_headers)
+
+    if target.auth_secret_ref:
+        secret = os.environ.get(target.auth_secret_ref)
+        if secret:
+            scheme = target.auth_scheme.lower()
+            if scheme == "bearer":
+                headers["Authorization"] = f"Bearer {secret}"
+            else:
+                headers[target.auth_scheme] = secret
+
+    return headers
+
+
+def _extract_software_version(response: httpx.Response) -> str | None:
+    """Try to extract software/version from response headers."""
+    # Common version headers across providers
+    for header in ("x-vllm-version", "server", "x-server-version", "x-version"):
+        value = response.headers.get(header)
+        if value:
+            return value
+    return None
+
+
+# ---------------------------------------------------------------------------
+# EndpointProber
+# ---------------------------------------------------------------------------
+
+
+class EndpointProber:
+    """Probes inference endpoints to discover actual capabilities.
+
+    Each probe method sends a minimal request to verify that a declared
+    capability actually works, returning structured results with timing
+    and software version metadata.
+    """
+
+    def __init__(
+        self,
+        *,
+        http_client: httpx.AsyncClient | None = None,
+        probe_timeout: float = 15.0,
+    ) -> None:
+        self._owns_client = http_client is None
+        self._http = http_client or httpx.AsyncClient(timeout=probe_timeout)
+        self._probe_timeout = probe_timeout
+
+    async def probe_health(self, target: InferenceTarget) -> HealthProbeResult:
+        """Check if the endpoint is reachable and responsive.
+
+        Tries the configured health_path or falls back to common paths:
+        /health, /v1/models, or the base URL itself.
+        """
+        headers = _resolve_auth_headers(target)
+        base = target.base_url.rstrip("/")
+
+        # Try common health paths in order
+        health_paths = ["/health", "/v1/models", "/"]
+        start = time.monotonic()
+
+        for path in health_paths:
+            url = f"{base}{path}"
+            try:
+                response = await self._http.get(
+                    url, headers=headers, timeout=self._probe_timeout
+                )
+                latency_ms = int((time.monotonic() - start) * 1000)
+                sw_version = _extract_software_version(response)
+
+                if response.status_code < 500:
+                    return HealthProbeResult(
+                        success=True,
+                        detail=f"Endpoint reachable via {path} (HTTP {response.status_code})",
+                        latency_ms=latency_ms,
+                        software_version=sw_version,
+                    )
+            except (httpx.ConnectError, httpx.TimeoutException) as exc:
+                latency_ms = int((time.monotonic() - start) * 1000)
+                return HealthProbeResult(
+                    success=False,
+                    detail=f"Connection failed: {type(exc).__name__}: {exc}",
+                    latency_ms=latency_ms,
+                )
+
+        latency_ms = int((time.monotonic() - start) * 1000)
+        return HealthProbeResult(
+            success=False,
+            detail="All health paths returned 5xx",
+            latency_ms=latency_ms,
+        )
+
+    async def probe_model_listing(
+        self, target: InferenceTarget
+    ) -> ModelListingResult:
+        """Check if the /v1/models endpoint works and lists the target model."""
+        headers = _resolve_auth_headers(target)
+        base = target.base_url.rstrip("/")
+        url = f"{base}/v1/models"
+
+        try:
+            response = await self._http.get(
+                url, headers=headers, timeout=self._probe_timeout
+            )
+            sw_version = _extract_software_version(response)
+
+            if response.status_code != 200:
+                return ModelListingResult(
+                    success=False,
+                    detail=f"Model listing returned HTTP {response.status_code}",
+                    software_version=sw_version,
+                )
+
+            data = response.json()
+            models_data = data.get("data", [])
+            model_ids = [m.get("id", "") for m in models_data if isinstance(m, dict)]
+            target_found = target.model in model_ids
+
+            return ModelListingResult(
+                success=True,
+                detail=f"Found {len(model_ids)} model(s)",
+                software_version=sw_version,
+                models=model_ids,
+                target_model_found=target_found,
+            )
+        except (httpx.ConnectError, httpx.TimeoutException) as exc:
+            return ModelListingResult(
+                success=False,
+                detail=f"Connection failed: {type(exc).__name__}: {exc}",
+            )
+        except (json.JSONDecodeError, ValueError) as exc:
+            return ModelListingResult(
+                success=False,
+                detail=f"Invalid JSON in model listing response: {exc}",
+            )
+
+    async def probe_json_schema(
+        self, target: InferenceTarget
+    ) -> JsonSchemaProbeResult:
+        """Send a minimal schema-constrained request and verify the response validates."""
+        headers = _resolve_auth_headers(target)
+        base = target.base_url.rstrip("/")
+        url = f"{base}/v1/chat/completions"
+
+        body: dict[str, Any] = {
+            "model": target.model,
+            "messages": [{"role": m.role, "content": m.content} for m in _PROBE_MESSAGES],
+            "temperature": 0,
+            "max_tokens": 64,
+            "response_format": {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "probe_response",
+                    "strict": True,
+                    "schema": _MINIMAL_PROBE_SCHEMA,
+                },
+            },
+        }
+
+        try:
+            response = await self._http.post(
+                url, json=body, headers=headers, timeout=self._probe_timeout
+            )
+            sw_version = _extract_software_version(response)
+
+            if response.status_code != 200:
+                return JsonSchemaProbeResult(
+                    success=False,
+                    detail=f"JSON schema probe returned HTTP {response.status_code}",
+                    software_version=sw_version,
+                    structured_mode_used="json_schema",
+                )
+
+            data = response.json()
+            choices = data.get("choices", [])
+            if not choices:
+                return JsonSchemaProbeResult(
+                    success=False,
+                    detail="Empty choices in schema probe response",
+                    software_version=sw_version,
+                    structured_mode_used="json_schema",
+                )
+
+            content = choices[0].get("message", {}).get("content", "")
+            try:
+                parsed = json.loads(content)
+                # Validate against the minimal schema
+                is_valid = (
+                    isinstance(parsed, dict)
+                    and "status" in parsed
+                    and isinstance(parsed["status"], str)
+                )
+                return JsonSchemaProbeResult(
+                    success=is_valid,
+                    detail="Schema probe response validates" if is_valid else "Response did not match expected schema",
+                    software_version=sw_version,
+                    schema_valid=is_valid,
+                    structured_mode_used="json_schema",
+                )
+            except (json.JSONDecodeError, ValueError):
+                return JsonSchemaProbeResult(
+                    success=False,
+                    detail=f"Response content is not valid JSON: {content[:100]}",
+                    software_version=sw_version,
+                    schema_valid=False,
+                    structured_mode_used="json_schema",
+                )
+        except (httpx.ConnectError, httpx.TimeoutException) as exc:
+            return JsonSchemaProbeResult(
+                success=False,
+                detail=f"Connection failed: {type(exc).__name__}: {exc}",
+            )
+
+    async def probe_usage_metadata(
+        self, target: InferenceTarget
+    ) -> UsageProbeResult:
+        """Check if usage tokens (prompt_tokens, completion_tokens) are returned."""
+        headers = _resolve_auth_headers(target)
+        base = target.base_url.rstrip("/")
+        url = f"{base}/v1/chat/completions"
+
+        body: dict[str, Any] = {
+            "model": target.model,
+            "messages": [{"role": m.role, "content": m.content} for m in _PROBE_MESSAGES],
+            "temperature": 0,
+            "max_tokens": 32,
+        }
+
+        try:
+            response = await self._http.post(
+                url, json=body, headers=headers, timeout=self._probe_timeout
+            )
+            sw_version = _extract_software_version(response)
+
+            if response.status_code != 200:
+                return UsageProbeResult(
+                    success=False,
+                    detail=f"Usage probe returned HTTP {response.status_code}",
+                    software_version=sw_version,
+                )
+
+            data = response.json()
+            usage = data.get("usage", {})
+            has_prompt = "prompt_tokens" in usage and usage["prompt_tokens"] is not None
+            has_completion = "completion_tokens" in usage and usage["completion_tokens"] is not None
+
+            return UsageProbeResult(
+                success=has_prompt or has_completion,
+                detail=f"Usage metadata: prompt_tokens={'yes' if has_prompt else 'no'}, completion_tokens={'yes' if has_completion else 'no'}",
+                software_version=sw_version,
+                has_prompt_tokens=has_prompt,
+                has_completion_tokens=has_completion,
+            )
+        except (httpx.ConnectError, httpx.TimeoutException) as exc:
+            return UsageProbeResult(
+                success=False,
+                detail=f"Connection failed: {type(exc).__name__}: {exc}",
+            )
+
+    async def probe_seed_determinism(
+        self, target: InferenceTarget
+    ) -> SeedProbeResult:
+        """Send the same request twice with the same seed and check if outputs match."""
+        headers = _resolve_auth_headers(target)
+        base = target.base_url.rstrip("/")
+        url = f"{base}/v1/chat/completions"
+
+        body: dict[str, Any] = {
+            "model": target.model,
+            "messages": [{"role": "user", "content": "Say exactly: hello"}],
+            "temperature": 0,
+            "max_tokens": 16,
+            "seed": 42,
+        }
+
+        outputs: list[str] = []
+        sw_version: str | None = None
+
+        try:
+            for _ in range(2):
+                response = await self._http.post(
+                    url, json=body, headers=headers, timeout=self._probe_timeout
+                )
+                if not sw_version:
+                    sw_version = _extract_software_version(response)
+
+                if response.status_code != 200:
+                    return SeedProbeResult(
+                        success=False,
+                        detail=f"Seed probe returned HTTP {response.status_code}",
+                        software_version=sw_version,
+                    )
+
+                data = response.json()
+                choices = data.get("choices", [])
+                if not choices:
+                    return SeedProbeResult(
+                        success=False,
+                        detail="Empty choices in seed probe response",
+                        software_version=sw_version,
+                    )
+                content = choices[0].get("message", {}).get("content", "")
+                outputs.append(content)
+
+            match = outputs[0] == outputs[1]
+            return SeedProbeResult(
+                success=True,
+                detail=f"Outputs {'match' if match else 'differ'} with same seed",
+                software_version=sw_version,
+                outputs_match=match,
+            )
+        except (httpx.ConnectError, httpx.TimeoutException) as exc:
+            return SeedProbeResult(
+                success=False,
+                detail=f"Connection failed: {type(exc).__name__}: {exc}",
+            )
+
+    async def probe_output_token_field(
+        self, target: InferenceTarget
+    ) -> OutputTokenFieldResult:
+        """Check if the endpoint accepts max_completion_tokens field.
+
+        Some providers use max_completion_tokens instead of (or in addition to)
+        max_tokens. This probe tests if the field is accepted without error.
+        """
+        headers = _resolve_auth_headers(target)
+        base = target.base_url.rstrip("/")
+        url = f"{base}/v1/chat/completions"
+
+        body: dict[str, Any] = {
+            "model": target.model,
+            "messages": [{"role": "user", "content": "Say hi"}],
+            "temperature": 0,
+            "max_completion_tokens": 16,
+        }
+
+        try:
+            response = await self._http.post(
+                url, json=body, headers=headers, timeout=self._probe_timeout
+            )
+            sw_version = _extract_software_version(response)
+
+            # If the endpoint accepts the field, it should respond 200
+            # If it rejects it, it will typically return 400 or 422
+            accepted = response.status_code == 200
+            detail = (
+                "max_completion_tokens field accepted"
+                if accepted
+                else f"max_completion_tokens field rejected (HTTP {response.status_code})"
+            )
+
+            return OutputTokenFieldResult(
+                success=accepted,
+                detail=detail,
+                software_version=sw_version,
+                field_accepted=accepted,
+            )
+        except (httpx.ConnectError, httpx.TimeoutException) as exc:
+            return OutputTokenFieldResult(
+                success=False,
+                detail=f"Connection failed: {type(exc).__name__}: {exc}",
+            )
+
+    async def run_full_probe(self, target: InferenceTarget) -> FullProbeResult:
+        """Run all probes against a target and return aggregated results."""
+        start = time.monotonic()
+
+        health = await self.probe_health(target)
+        if not health.success:
+            # If health fails, skip remaining probes
+            duration_ms = int((time.monotonic() - start) * 1000)
+            return FullProbeResult(
+                endpoint_id=target.endpoint_id,
+                health=health,
+                software_version=health.software_version,
+                probe_duration_ms=duration_ms,
+            )
+
+        model_listing = await self.probe_model_listing(target)
+        json_schema_result = await self.probe_json_schema(target)
+        usage_result = await self.probe_usage_metadata(target)
+        seed_result = await self.probe_seed_determinism(target)
+        output_token_result = await self.probe_output_token_field(target)
+
+        duration_ms = int((time.monotonic() - start) * 1000)
+
+        # Determine software version from whichever probe returned one
+        sw_version = (
+            health.software_version
+            or model_listing.software_version
+            or json_schema_result.software_version
+            or usage_result.software_version
+            or seed_result.software_version
+            or output_token_result.software_version
+        )
+
+        return FullProbeResult(
+            endpoint_id=target.endpoint_id,
+            health=health,
+            model_listing=model_listing,
+            json_schema=json_schema_result,
+            usage_metadata=usage_result,
+            seed_determinism=seed_result,
+            output_token_field=output_token_result,
+            software_version=sw_version,
+            probe_duration_ms=duration_ms,
+        )
+
+    async def close(self) -> None:
+        """Close the HTTP client if we own it."""
+        if self._owns_client:
+            await self._http.aclose()
+
+
+# ---------------------------------------------------------------------------
+# ProbeResultStore — TTL-based cache for probe results
+# ---------------------------------------------------------------------------
+
+
+class ProbeResultStore:
+    """In-memory cache for endpoint probe results with configurable TTL.
+
+    Uses a simple dict with timestamps. Expired entries are lazily evicted
+    on access. The TTL can be configured globally or per-store instance.
+    """
+
+    def __init__(self, ttl_seconds: float = DEFAULT_PROBE_TTL_SECONDS) -> None:
+        self._ttl_seconds = ttl_seconds
+        self._store: dict[UUID, tuple[float, FullProbeResult]] = {}
+
+    @property
+    def ttl_seconds(self) -> float:
+        """Current TTL configuration."""
+        return self._ttl_seconds
+
+    def store(self, endpoint_id: UUID, result: FullProbeResult) -> None:
+        """Store a probe result with the current timestamp."""
+        self._store[endpoint_id] = (time.monotonic(), result)
+
+    def get(self, endpoint_id: UUID) -> FullProbeResult | None:
+        """Retrieve a probe result if it exists and hasn't expired.
+
+        Returns None if no result is stored or if the TTL has elapsed.
+        """
+        entry = self._store.get(endpoint_id)
+        if entry is None:
+            return None
+
+        stored_at, result = entry
+        elapsed = time.monotonic() - stored_at
+        if elapsed > self._ttl_seconds:
+            # Expired — evict lazily
+            del self._store[endpoint_id]
+            return None
+
+        return result
+
+    def invalidate(self, endpoint_id: UUID) -> None:
+        """Force cache eviction for an endpoint."""
+        self._store.pop(endpoint_id, None)
+
+    def clear(self) -> None:
+        """Clear all cached results."""
+        self._store.clear()
+
+    def __len__(self) -> int:
+        """Number of entries (including possibly expired ones)."""
+        return len(self._store)
+
+
+# ---------------------------------------------------------------------------
+# Required capability validation
+# ---------------------------------------------------------------------------
+
+
+def validate_required_capabilities(
+    target: InferenceTarget,
+    probe_result: FullProbeResult,
+) -> list[str]:
+    """Validate that declared required capabilities actually work.
+
+    Returns a list of failure descriptions. If the list is non-empty,
+    the endpoint should NOT be activated for routing.
+
+    Checks:
+    - Health must succeed
+    - If chat_completions is declared, health must pass
+    - If json_schema is declared, json_schema probe must succeed and validate
+    - If usage is declared, usage probe must show token counts
+    - If seed is declared, seed probe must show matching outputs
+    - If max_completion_tokens is declared, output_token_field probe must pass
+    - If model_listing is declared, model listing must succeed and find the model
+    """
+    failures: list[str] = []
+    caps = target.capabilities
+
+    # Health is always required
+    if probe_result.health is None or not probe_result.health.success:
+        detail = probe_result.health.detail if probe_result.health else "No health probe result"
+        failures.append(f"Health check failed: {detail}")
+        # If health fails, all other checks are meaningless
+        return failures
+
+    # Model listing capability
+    if caps.model_listing:
+        if probe_result.model_listing is None or not probe_result.model_listing.success:
+            detail = probe_result.model_listing.detail if probe_result.model_listing else "Not probed"
+            failures.append(f"Model listing failed: {detail}")
+        elif not probe_result.model_listing.target_model_found:
+            failures.append(
+                f"Model '{target.model}' not found in endpoint model list"
+            )
+
+    # JSON Schema capability
+    if caps.json_schema:
+        if probe_result.json_schema is None or not probe_result.json_schema.success:
+            detail = probe_result.json_schema.detail if probe_result.json_schema else "Not probed"
+            failures.append(f"JSON Schema structured output failed: {detail}")
+        elif not probe_result.json_schema.schema_valid:
+            failures.append(
+                "JSON Schema probe succeeded but response did not validate"
+            )
+
+    # Usage metadata capability
+    if caps.usage:
+        if probe_result.usage_metadata is None or not probe_result.usage_metadata.success:
+            detail = probe_result.usage_metadata.detail if probe_result.usage_metadata else "Not probed"
+            failures.append(f"Usage metadata not available: {detail}")
+
+    # Seed determinism capability
+    if caps.seed:
+        if probe_result.seed_determinism is None or not probe_result.seed_determinism.success:
+            detail = probe_result.seed_determinism.detail if probe_result.seed_determinism else "Not probed"
+            failures.append(f"Seed determinism probe failed: {detail}")
+        elif not probe_result.seed_determinism.outputs_match:
+            failures.append(
+                "Seed declared but outputs differ with same seed"
+            )
+
+    # max_completion_tokens capability
+    if caps.max_completion_tokens:
+        if probe_result.output_token_field is None or not probe_result.output_token_field.success:
+            detail = probe_result.output_token_field.detail if probe_result.output_token_field else "Not probed"
+            failures.append(f"max_completion_tokens field not supported: {detail}")
+
+    return failures

services/shared/inference/clients/__init__.py

@@ -0,0 +1,10 @@
+"""Inference gateway clients.
+
+Exports:
+    OpenAICompatibleClient - Client for OpenAI-compatible endpoints (vLLM, OpenAI, etc.)
+    OllamaNativeClient - Client for Ollama /api/chat native endpoints.
+"""
+from services.shared.inference.clients.ollama_native import OllamaNativeClient
+from services.shared.inference.clients.openai_compatible import OpenAICompatibleClient
+
+__all__ = ["OpenAICompatibleClient", "OllamaNativeClient"]

services/shared/inference/clients/ollama_native.py

@@ -0,0 +1,382 @@
+"""Ollama native client for the shared inference gateway.
+
+Implements generation via the Ollama /api/chat endpoint with:
+- Shared StructuredGenerationRequest/InferenceResult types
+- Native JSON schema formatting when supported (format field)
+- Prompt-only fallback with explicit reporting
+- Stall/loop detection as Ollama-specific policy
+- Configurable max output tokens (num_predict) and context window (num_ctx)
+- Error mapping to shared InferenceErrorCategory
+
+Requirements: 2.1, 2.6
+"""
+from __future__ import annotations
+
+import json
+import logging
+import time
+from dataclasses import dataclass
+
+import httpx
+
+from services.shared.inference.errors import InferenceError, InferenceErrorCategory
+from services.shared.inference.models import (
+    InferenceResult,
+    InferenceTarget,
+    StructuredGenerationRequest,
+    TokenUsage,
+)
+
+# Re-export for backward compatibility
+__all__ = ["OllamaNativeClient", "StallPolicy"]
+
+logger = logging.getLogger("inference.ollama_native")
+
+
+@dataclass
+class StallPolicy:
+    """Ollama-specific stall/loop detection configuration.
+
+    Monitors streaming responses for repetitive output patterns.
+    This policy is intentionally Ollama-specific and does not leak
+    into the generic InferenceResult interface.
+    """
+
+    enabled: bool = True
+    check_interval_seconds: float = 5.0
+    max_unchanged_intervals: int = 6
+    loop_window: int = 64
+    loop_threshold: float = 0.5
+
+
+def _map_http_status_to_category(status_code: int) -> InferenceErrorCategory:
+    """Map Ollama HTTP status codes to normalized error categories."""
+    if status_code == 401:
+        return InferenceErrorCategory.AUTH_FAILED
+    if status_code == 403:
+        return InferenceErrorCategory.FORBIDDEN
+    if status_code == 404:
+        return InferenceErrorCategory.MODEL_NOT_FOUND
+    if status_code == 429:
+        return InferenceErrorCategory.RATE_LIMITED
+    if status_code in (400, 422):
+        return InferenceErrorCategory.BAD_REQUEST
+    if status_code >= 500:
+        return InferenceErrorCategory.SERVER_ERROR
+    return InferenceErrorCategory.UNKNOWN
+
+
+def _detect_loop(content: str, window: int, threshold: float) -> bool:
+    """Detect repetitive output by checking if the tail repeats earlier content.
+
+    Looks at the last `window` characters and checks if a significant
+    portion of the tail matches an earlier substring, indicating the model
+    is stuck in a generation loop.
+    """
+    if len(content) < window * 2:
+        return False
+
+    tail = content[-window:]
+    body = content[:-window]
+
+    # Check if the tail appears verbatim in the preceding content
+    if tail in body:
+        return True
+
+    # Check character-level repetition ratio in the tail
+    if not tail:
+        return False
+    unique_chars = len(set(tail))
+    ratio = unique_chars / len(tail)
+    return ratio < threshold
+
+
+class OllamaNativeClient:
+    """Async client for Ollama /api/chat using shared inference types.
+
+    Translates StructuredGenerationRequest into Ollama's native API format
+    and returns InferenceResult with proper metadata and error classification.
+
+    Stall/loop detection is an Ollama-specific policy that monitors streaming
+    responses for repetitive patterns and aborts generation when detected.
+    This does not affect the InferenceResult interface — stall detection
+    raises an InferenceError with category STALL_DETECTED.
+    """
+
+    def __init__(
+        self,
+        target: InferenceTarget,
+        *,
+        http_client: httpx.AsyncClient | None = None,
+        stall_policy: StallPolicy | None = None,
+    ) -> None:
+        if target.protocol != "ollama_native":
+            raise ValueError(
+                f"OllamaNativeClient requires protocol='ollama_native', got '{target.protocol}'"
+            )
+        self._target = target
+        self._stall_policy = stall_policy or StallPolicy()
+        self._owns_client = http_client is None
+        self._http = http_client or httpx.AsyncClient(
+            timeout=httpx.Timeout(300.0, read=300.0),
+        )
+
+    async def close(self) -> None:
+        """Close the underlying HTTP client if we own it."""
+        if self._owns_client:
+            await self._http.aclose()
+
+    async def generate(self, request: StructuredGenerationRequest) -> InferenceResult:
+        """Send a structured generation request to Ollama and return the result.
+
+        Builds the Ollama-native payload, streams the response, applies
+        stall detection, and maps results to InferenceResult.
+        """
+        start = time.monotonic()
+
+        payload = self._build_payload(request)
+        url = f"{self._target.base_url}/api/chat"
+
+        logger.info(
+            "Ollama POST %s model=%s messages=%d max_tokens=%d",
+            url,
+            self._target.model,
+            len(request.messages),
+            request.max_output_tokens,
+        )
+
+        try:
+            content, metadata = await self._stream_response(url, payload, request.timeout_seconds)
+        except InferenceError:
+            raise
+        except httpx.TimeoutException as exc:
+            raise InferenceError(
+                InferenceErrorCategory.TIMEOUT,
+                f"Request timed out after {request.timeout_seconds}s",
+                provider_detail=str(exc),
+            ) from exc
+        except httpx.ConnectError as exc:
+            raise InferenceError(
+                InferenceErrorCategory.CONNECTION_REFUSED,
+                "Connection refused",
+                provider_detail=str(exc),
+            ) from exc
+        except httpx.HTTPError as exc:
+            raise InferenceError(
+                InferenceErrorCategory.CONNECTION_ERROR,
+                "HTTP connection error",
+                provider_detail=str(exc),
+            ) from exc
+
+        latency_ms = int((time.monotonic() - start) * 1000)
+
+        # Determine structured mode
+        structured_mode = self._determine_structured_mode(request)
+
+        # Attempt JSON parsing if we expect structured output
+        parsed = None
+        if request.json_schema and content:
+            try:
+                parsed = json.loads(content)
+            except (json.JSONDecodeError, ValueError):
+                pass
+
+        return InferenceResult(
+            content=content,
+            parsed=parsed,
+            endpoint_id=self._target.endpoint_id,
+            deployment_id=self._target.deployment_id,
+            model=metadata.get("model", self._target.model) or self._target.model,
+            protocol=self._target.protocol,
+            structured_mode=structured_mode,
+            latency_ms=latency_ms,
+            usage=TokenUsage(
+                input_tokens=metadata.get("prompt_eval_count"),
+                output_tokens=metadata.get("eval_count"),
+            ),
+            request_id=request.trace_id or None,
+            repaired=False,
+            retries=0,
+        )
+
+    def _build_payload(self, request: StructuredGenerationRequest) -> dict:
+        """Build the Ollama /api/chat request body."""
+        messages = [
+            {"role": msg.role, "content": msg.content}
+            for msg in request.messages
+        ]
+
+        options: dict = {}
+
+        # Honor max output tokens via num_predict
+        if request.max_output_tokens:
+            options["num_predict"] = request.max_output_tokens
+
+        # Honor context window from target configuration
+        if self._target.context_window and self._target.context_window > 0:
+            options["num_ctx"] = self._target.context_window
+
+        # Temperature
+        options["temperature"] = request.temperature
+
+        # Seed if provided
+        if request.seed is not None:
+            options["seed"] = request.seed
+
+        payload: dict = {
+            "model": self._target.model,
+            "messages": messages,
+            "stream": True,
+            "options": options,
+        }
+
+        # Native schema formatting when supported
+        if request.json_schema and self._target.capabilities.json_schema:
+            payload["format"] = request.json_schema
+
+        # Merge any extra body params from target
+        if self._target.extra_body:
+            for key, value in self._target.extra_body.items():
+                if key not in payload:
+                    payload[key] = value
+
+        return payload
+
+    def _determine_structured_mode(
+        self, request: StructuredGenerationRequest
+    ) -> str:
+        """Determine which structured output mode was used."""
+        if not request.json_schema:
+            return "none"
+        if self._target.capabilities.json_schema:
+            return "json_schema"
+        # Schema was requested but not natively supported — prompt only
+        return "prompt_only"
+
+    async def _stream_response(
+        self,
+        url: str,
+        payload: dict,
+        timeout_seconds: float,
+    ) -> tuple[str, dict]:
+        """Stream Ollama response, applying stall detection.
+
+        Returns (content, metadata) where metadata contains token counts
+        and timing from the final Ollama response chunk.
+        """
+        content_parts: list[str] = []
+        metadata: dict = {}
+        last_content_length = 0
+        unchanged_count = 0
+        last_check_time = time.monotonic()
+
+        try:
+            async with self._http.stream(
+                "POST",
+                url,
+                json=payload,
+                timeout=httpx.Timeout(timeout_seconds, read=timeout_seconds),
+            ) as response:
+                if response.status_code != 200:
+                    # Read body for error detail
+                    body = b""
+                    async for chunk in response.aiter_bytes():
+                        body += chunk
+                    detail = body.decode("utf-8", errors="replace")[:500]
+                    category = _map_http_status_to_category(response.status_code)
+                    raise InferenceError(
+                        category,
+                        f"Ollama returned HTTP {response.status_code}",
+                        provider_detail=detail,
+                        status_code=response.status_code,
+                    )
+
+                async for line in response.aiter_lines():
+                    if not line.strip():
+                        continue
+
+                    try:
+                        chunk_data = json.loads(line)
+                    except json.JSONDecodeError:
+                        continue
+
+                    # Extract content from message
+                    message = chunk_data.get("message", {})
+                    chunk_content = message.get("content", "")
+                    if chunk_content:
+                        content_parts.append(chunk_content)
+
+                    # Check for completion
+                    if chunk_data.get("done", False):
+                        # Capture metadata from final chunk
+                        metadata["model"] = chunk_data.get("model", "")
+                        metadata["prompt_eval_count"] = chunk_data.get("prompt_eval_count")
+                        metadata["eval_count"] = chunk_data.get("eval_count")
+                        total_ns = chunk_data.get("total_duration")
+                        if total_ns:
+                            metadata["total_duration_ms"] = total_ns // 1_000_000
+                        break
+
+                    # Stall detection
+                    if self._stall_policy.enabled:
+                        now = time.monotonic()
+                        if now - last_check_time >= self._stall_policy.check_interval_seconds:
+                            current_content = "".join(content_parts)
+                            current_length = len(current_content)
+
+                            if current_length == last_content_length:
+                                unchanged_count += 1
+                            else:
+                                # Check for loop pattern
+                                if _detect_loop(
+                                    current_content,
+                                    self._stall_policy.loop_window,
+                                    self._stall_policy.loop_threshold,
+                                ):
+                                    unchanged_count += 1
+                                else:
+                                    unchanged_count = 0
+
+                            last_content_length = current_length
+                            last_check_time = now
+
+                            if unchanged_count >= self._stall_policy.max_unchanged_intervals:
+                                logger.warning(
+                                    "Stall detected after %d unchanged intervals, aborting",
+                                    unchanged_count,
+                                )
+                                raise InferenceError(
+                                    InferenceErrorCategory.STALL_DETECTED,
+                                    f"Generation stalled after {unchanged_count} check intervals",
+                                )
+
+        except InferenceError:
+            raise
+        except httpx.TimeoutException as exc:
+            raise InferenceError(
+                InferenceErrorCategory.TIMEOUT,
+                "Stream read timed out",
+                provider_detail=str(exc),
+            ) from exc
+        except httpx.ConnectError as exc:
+            raise InferenceError(
+                InferenceErrorCategory.CONNECTION_REFUSED,
+                "Connection refused during streaming",
+                provider_detail=str(exc),
+            ) from exc
+        except httpx.HTTPError as exc:
+            raise InferenceError(
+                InferenceErrorCategory.CONNECTION_ERROR,
+                "HTTP error during streaming",
+                provider_detail=str(exc),
+            ) from exc
+
+        final_content = "".join(content_parts)
+
+        if not final_content:
+            raise InferenceError(
+                InferenceErrorCategory.EMPTY_RESPONSE,
+                "Ollama returned empty content",
+            )
+
+        return final_content, metadata

services/shared/inference/clients/openai_compatible.py

@@ -0,0 +1,431 @@
+"""OpenAI-compatible inference client.
+
+Uses httpx.AsyncClient directly (NOT the openai SDK). This keeps wire payloads
+explicit, permits provider-specific extra_body, and simplifies redacted request
+auditing.
+
+Requirements: 2.1, 2.2, 2.3, 2.4, 2.5, 2.7, 2.9
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import time
+from typing import Any
+
+import httpx
+
+from services.shared.inference.models import (
+    ErrorCategory,
+    InferenceResult,
+    InferenceTarget,
+    StructuredGenerationRequest,
+    TokenUsage,
+)
+
+logger = logging.getLogger(__name__)
+
+# Status codes that trigger retry
+_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
+
+# Sensitive keys that must never appear in logs
+_SENSITIVE_KEYS = frozenset({"authorization", "x-api-key", "api-key"})
+
+
+def _resolve_auth_secret(secret_ref: str | None) -> str | None:
+    """Resolve an authentication secret from environment variables.
+
+    The secret_ref is treated as an environment variable name.
+    Returns None if the ref is None or the env var is not set.
+    """
+    if not secret_ref:
+        return None
+    return os.environ.get(secret_ref)
+
+
+def _redact_headers(headers: dict[str, str]) -> dict[str, str]:
+    """Return a copy of headers with sensitive values redacted."""
+    redacted = {}
+    for key, value in headers.items():
+        if key.lower() in _SENSITIVE_KEYS:
+            redacted[key] = "***REDACTED***"
+        else:
+            redacted[key] = value
+    return redacted
+
+
+class OpenAICompatibleClient:
+    """Client for OpenAI-compatible /v1/chat/completions endpoints.
+
+    Supports:
+    - Bearer and configurable authentication via runtime secret resolution
+    - Standard response_format.json_schema payloads
+    - Configurable vLLM structured_outputs extra-body payloads
+    - Explicit JSON-object and prompt-only fallback policies
+    - Retry on transient errors (5xx, 429, timeout)
+    - Local JSON schema revalidation
+    - Metadata capture (request_id, usage, finish_reason, retries)
+    """
+
+    def __init__(
+        self,
+        target: InferenceTarget,
+        *,
+        http_client: httpx.AsyncClient | None = None,
+    ) -> None:
+        self._target = target
+        self._owns_client = http_client is None
+        self._http = http_client or httpx.AsyncClient(timeout=target.timeout_seconds)
+
+    @property
+    def target(self) -> InferenceTarget:
+        """The resolved inference target."""
+        return self._target
+
+    async def generate(
+        self,
+        request: StructuredGenerationRequest,
+    ) -> InferenceResult:
+        """Send a structured generation request and return the result.
+
+        Chooses the strongest structured-output mode the target supports:
+        1. json_schema — sends the actual schema with strict mode
+        2. json_object — only if target explicitly declares json_object capability
+        3. prompt_only — fallback when no structured output is available
+        """
+        structured_mode = self._choose_structured_mode(request)
+        headers = self._build_headers()
+        body = self._build_body(request, structured_mode)
+        url = f"{self._target.base_url.rstrip('/')}/v1/chat/completions"
+
+        retries = 0
+        max_retries = self._target.max_retries
+        start_time = time.monotonic()
+        last_error: str | None = None
+        last_error_category: str | None = None
+
+        while True:
+            try:
+                response = await self._http.post(
+                    url,
+                    json=body,
+                    headers=headers,
+                    timeout=request.timeout_seconds,
+                )
+            except httpx.TimeoutException:
+                last_error = "Request timed out"
+                last_error_category = ErrorCategory.TIMEOUT
+                if retries < max_retries:
+                    retries += 1
+                    logger.warning(
+                        "Timeout on attempt %d/%d to %s",
+                        retries,
+                        max_retries,
+                        url,
+                    )
+                    continue
+                return self._error_result(
+                    last_error,
+                    last_error_category,
+                    retries,
+                    start_time,
+                    structured_mode,
+                )
+            except httpx.ConnectError as exc:
+                last_error = f"Connection error: {exc}"
+                last_error_category = ErrorCategory.CONNECTION_ERROR
+                if retries < max_retries:
+                    retries += 1
+                    logger.warning(
+                        "Connection error on attempt %d/%d to %s: %s",
+                        retries,
+                        max_retries,
+                        url,
+                        exc,
+                    )
+                    continue
+                return self._error_result(
+                    last_error,
+                    last_error_category,
+                    retries,
+                    start_time,
+                    structured_mode,
+                )
+
+            # Handle retryable HTTP status codes
+            if response.status_code in _RETRYABLE_STATUS_CODES:
+                if response.status_code == 429:
+                    last_error_category = ErrorCategory.RATE_LIMIT
+                    last_error = "Rate limited (429)"
+                else:
+                    last_error_category = ErrorCategory.SERVER_ERROR
+                    last_error = f"Server error ({response.status_code})"
+
+                if retries < max_retries:
+                    retries += 1
+                    logger.warning(
+                        "HTTP %d on attempt %d/%d to %s",
+                        response.status_code,
+                        retries,
+                        max_retries,
+                        url,
+                    )
+                    continue
+                return self._error_result(
+                    last_error,
+                    last_error_category,
+                    retries,
+                    start_time,
+                    structured_mode,
+                )
+
+            # Handle non-retryable errors
+            if response.status_code == 401 or response.status_code == 403:
+                return self._error_result(
+                    f"Authentication failed ({response.status_code})",
+                    ErrorCategory.AUTHENTICATION,
+                    retries,
+                    start_time,
+                    structured_mode,
+                )
+
+            if response.status_code >= 400:
+                return self._error_result(
+                    f"Client error ({response.status_code})",
+                    ErrorCategory.INVALID_RESPONSE,
+                    retries,
+                    start_time,
+                    structured_mode,
+                )
+
+            # Success path
+            break
+
+        latency_ms = int((time.monotonic() - start_time) * 1000)
+        return self._parse_response(
+            response,
+            request,
+            structured_mode,
+            retries,
+            latency_ms,
+        )
+
+    def _choose_structured_mode(
+        self, request: StructuredGenerationRequest
+    ) -> str:
+        """Choose the strongest structured-output mode available."""
+        caps = self._target.capabilities
+
+        if request.json_schema and caps.json_schema:
+            return "json_schema"
+        if request.json_schema and caps.json_object:
+            return "json_object"
+        if request.json_schema:
+            # Schema requested but endpoint supports neither — prompt-only fallback
+            return "prompt_only"
+        return "none"
+
+    def _build_headers(self) -> dict[str, str]:
+        """Build request headers including authentication."""
+        headers: dict[str, str] = {
+            "Content-Type": "application/json",
+        }
+
+        # Add extra headers from target config
+        headers.update(self._target.extra_headers)
+
+        # Resolve auth secret
+        secret = _resolve_auth_secret(self._target.auth_secret_ref)
+        if secret:
+            scheme = self._target.auth_scheme.lower()
+            if scheme == "bearer":
+                headers["Authorization"] = f"Bearer {secret}"
+            else:
+                # Custom auth scheme (e.g., "X-API-Key: <value>")
+                headers[self._target.auth_scheme] = secret
+
+        return headers
+
+    def _build_body(
+        self,
+        request: StructuredGenerationRequest,
+        structured_mode: str,
+    ) -> dict[str, Any]:
+        """Build the request body for /v1/chat/completions."""
+        messages = [
+            {"role": msg.role, "content": msg.content}
+            for msg in request.messages
+        ]
+
+        body: dict[str, Any] = {
+            "model": self._target.model,
+            "messages": messages,
+            "temperature": request.temperature,
+            "max_tokens": request.max_output_tokens,
+        }
+
+        # Add seed if the target supports it
+        if self._target.capabilities.seed and request.seed is not None:
+            body["seed"] = request.seed
+
+        # Add response_format based on chosen mode
+        if structured_mode == "json_schema" and request.json_schema:
+            body["response_format"] = {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": request.json_schema.get("title", "response"),
+                    "strict": True,
+                    "schema": request.json_schema,
+                },
+            }
+        elif structured_mode == "json_object":
+            body["response_format"] = {"type": "json_object"}
+
+        # Add extra_body from target config (vLLM structured_outputs, etc.)
+        if self._target.extra_body:
+            body.update(self._target.extra_body)
+
+        return body
+
+    def _parse_response(
+        self,
+        response: httpx.Response,
+        request: StructuredGenerationRequest,
+        structured_mode: str,
+        retries: int,
+        latency_ms: int,
+    ) -> InferenceResult:
+        """Parse a successful response into InferenceResult."""
+        # Extract request ID from response headers
+        request_id = response.headers.get("x-request-id")
+
+        try:
+            data = response.json()
+        except (json.JSONDecodeError, ValueError):
+            return InferenceResult(
+                content="",
+                endpoint_id=self._target.endpoint_id,
+                deployment_id=self._target.deployment_id,
+                model=self._target.model,
+                protocol=self._target.protocol,
+                structured_mode=structured_mode,  # type: ignore[arg-type]
+                latency_ms=latency_ms,
+                retries=retries,
+                request_id=request_id,
+                error="Invalid JSON in response body",
+                error_category=ErrorCategory.INVALID_RESPONSE,
+            )
+
+        # Extract content from choices
+        choices = data.get("choices", [])
+        if not choices:
+            return InferenceResult(
+                content="",
+                endpoint_id=self._target.endpoint_id,
+                deployment_id=self._target.deployment_id,
+                model=self._target.model,
+                protocol=self._target.protocol,
+                structured_mode=structured_mode,  # type: ignore[arg-type]
+                latency_ms=latency_ms,
+                retries=retries,
+                request_id=request_id,
+                error="Empty choices in response",
+                error_category=ErrorCategory.INVALID_RESPONSE,
+            )
+
+        message = choices[0].get("message", {})
+        content = message.get("content", "")
+        finish_reason = choices[0].get("finish_reason")
+
+        # Extract usage metadata
+        usage_data = data.get("usage", {})
+        input_tokens = usage_data.get("prompt_tokens")
+        output_tokens = usage_data.get("completion_tokens")
+        total_tokens = usage_data.get("total_tokens")
+
+        # Parse JSON content if structured output was requested
+        parsed: dict[str, Any] | None = None
+        schema_valid: bool | None = None
+
+        if structured_mode in ("json_schema", "json_object") and content:
+            try:
+                parsed = json.loads(content)
+            except (json.JSONDecodeError, ValueError):
+                # Content is not valid JSON
+                schema_valid = False
+
+        # Validate parsed JSON against supplied schema
+        if parsed is not None and request.json_schema:
+            schema_valid = self._validate_schema(parsed, request.json_schema)
+
+        return InferenceResult(
+            content=content,
+            parsed=parsed,
+            endpoint_id=self._target.endpoint_id,
+            deployment_id=self._target.deployment_id,
+            model=self._target.model,
+            protocol=self._target.protocol,
+            structured_mode=structured_mode,  # type: ignore[arg-type]
+            latency_ms=latency_ms,
+            usage=TokenUsage(
+                input_tokens=input_tokens,
+                output_tokens=output_tokens,
+                total_tokens=total_tokens,
+            ),
+            request_id=request_id,
+            finish_reason=finish_reason,
+            retries=retries,
+            schema_valid=schema_valid,
+        )
+
+    def _validate_schema(
+        self, data: dict[str, Any], schema: dict[str, Any]
+    ) -> bool:
+        """Revalidate parsed JSON locally against the supplied schema.
+
+        Returns True if valid, False otherwise.
+        """
+        try:
+            import jsonschema
+
+            jsonschema.validate(instance=data, schema=schema)
+            return True
+        except Exception:
+            logger.debug("Schema validation failed for response data")
+            return False
+
+    def _error_result(
+        self,
+        error: str,
+        error_category: str,
+        retries: int,
+        start_time: float,
+        structured_mode: str,
+    ) -> InferenceResult:
+        """Build an error InferenceResult."""
+        latency_ms = int((time.monotonic() - start_time) * 1000)
+        return InferenceResult(
+            content="",
+            endpoint_id=self._target.endpoint_id,
+            deployment_id=self._target.deployment_id,
+            model=self._target.model,
+            protocol=self._target.protocol,
+            structured_mode=structured_mode,  # type: ignore[arg-type]
+            latency_ms=latency_ms,
+            retries=retries,
+            error=error,
+            error_category=error_category,
+        )
+
+    async def close(self) -> None:
+        """Close the underlying HTTP client if we own it."""
+        if self._owns_client:
+            await self._http.aclose()
+
+    def __repr__(self) -> str:
+        return (
+            f"OpenAICompatibleClient("
+            f"model={self._target.model!r}, "
+            f"base_url={self._target.base_url!r})"
+        )

services/shared/inference/errors.py

@@ -0,0 +1,89 @@
+"""Normalized error categories for the inference gateway.
+
+Maps provider-specific failures into a protocol-agnostic taxonomy
+so that retry logic, alerting, and metrics work uniformly across
+Ollama, OpenAI-compatible, and specialist endpoints.
+
+Requirements: 2.1, 2.9
+"""
+from __future__ import annotations
+
+from enum import Enum
+
+
+class InferenceErrorCategory(str, Enum):
+    """Normalized error categories for inference failures."""
+
+    # Network / transport
+    TIMEOUT = "timeout"
+    CONNECTION_REFUSED = "connection_refused"
+    CONNECTION_ERROR = "connection_error"
+
+    # Authentication / authorization
+    AUTH_FAILED = "auth_failed"
+    FORBIDDEN = "forbidden"
+
+    # Rate limiting
+    RATE_LIMITED = "rate_limited"
+
+    # Server errors
+    SERVER_ERROR = "server_error"
+    SERVICE_UNAVAILABLE = "service_unavailable"
+
+    # Client errors
+    BAD_REQUEST = "bad_request"
+    MODEL_NOT_FOUND = "model_not_found"
+    INVALID_REQUEST = "invalid_request"
+
+    # Response problems
+    INVALID_RESPONSE = "invalid_response"
+    EMPTY_RESPONSE = "empty_response"
+    SCHEMA_VIOLATION = "schema_violation"
+
+    # Capability / policy
+    CAPABILITY_UNAVAILABLE = "capability_unavailable"
+    POLICY_VIOLATION = "policy_violation"
+
+    # Ollama-specific
+    STALL_DETECTED = "stall_detected"
+
+    # Unknown
+    UNKNOWN = "unknown"
+
+    @property
+    def retryable(self) -> bool:
+        """Whether this error category should generally be retried."""
+        return self in _RETRYABLE_CATEGORIES
+
+
+_RETRYABLE_CATEGORIES = frozenset({
+    InferenceErrorCategory.TIMEOUT,
+    InferenceErrorCategory.CONNECTION_ERROR,
+    InferenceErrorCategory.CONNECTION_REFUSED,
+    InferenceErrorCategory.SERVER_ERROR,
+    InferenceErrorCategory.SERVICE_UNAVAILABLE,
+    InferenceErrorCategory.RATE_LIMITED,
+    InferenceErrorCategory.STALL_DETECTED,
+    InferenceErrorCategory.EMPTY_RESPONSE,
+})
+
+
+class InferenceError(Exception):
+    """Typed inference error with category and optional provider detail."""
+
+    def __init__(
+        self,
+        category: InferenceErrorCategory,
+        message: str = "",
+        *,
+        provider_detail: str | None = None,
+        status_code: int | None = None,
+    ) -> None:
+        self.category = category
+        self.provider_detail = provider_detail
+        self.status_code = status_code
+        super().__init__(message or category.value)
+
+    @property
+    def retryable(self) -> bool:
+        return self.category.retryable

services/shared/inference/factory.py

@@ -0,0 +1,109 @@
+"""Inference client factory with protocol alias resolution.
+
+Replaces the legacy VLLMClient/OllamaClient fallback pattern with a
+capability-aware routing layer. Unknown protocols ALWAYS fail closed —
+they never silently fall back to Ollama.
+
+Requirements: 2.2, 2.6
+Design: Inference Gateway — profiles and capability probes
+"""
+from __future__ import annotations
+
+import warnings
+
+from services.shared.inference.clients.ollama_native import OllamaNativeClient
+from services.shared.inference.clients.openai_compatible import OpenAICompatibleClient
+from services.shared.inference.errors import InferenceError, InferenceErrorCategory
+from services.shared.inference.models import InferenceTarget
+
+# Backward-compatible protocol aliases.
+# "vllm" is retained as a deprecated alias for "openai_chat".
+# "ollama" is retained as an alias for "ollama_native".
+PROTOCOL_ALIASES: dict[str, str] = {
+    "vllm": "openai_chat",
+    "ollama": "ollama_native",
+}
+
+# Canonical protocol names that the factory can instantiate.
+KNOWN_PROTOCOLS: frozenset[str] = frozenset({
+    "openai_chat",
+    "ollama_native",
+    "specialist_http",
+})
+
+
+def resolve_protocol(provider_name: str) -> str:
+    """Resolve a protocol name or alias to a canonical protocol.
+
+    Raises InferenceError(CAPABILITY_UNAVAILABLE) for unknown protocols.
+    Emits a deprecation warning for the deprecated "vllm" alias.
+
+    Args:
+        provider_name: Raw protocol/provider string (e.g. "vllm", "ollama_native").
+
+    Returns:
+        Canonical protocol string.
+
+    Raises:
+        InferenceError: If the protocol is unknown and cannot be resolved.
+    """
+    normalized = provider_name.strip().lower()
+
+    # Check if it's already a known canonical protocol
+    if normalized in KNOWN_PROTOCOLS:
+        return normalized
+
+    # Check aliases
+    if normalized in PROTOCOL_ALIASES:
+        canonical = PROTOCOL_ALIASES[normalized]
+        if normalized == "vllm":
+            warnings.warn(
+                "Provider 'vllm' is deprecated. Use protocol 'openai_chat' instead. "
+                "The 'vllm' alias will be removed in a future version.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+        return canonical
+
+    # Unknown protocol — fail closed, NEVER fall back to Ollama
+    raise InferenceError(
+        InferenceErrorCategory.CAPABILITY_UNAVAILABLE,
+        f"Unknown protocol: {normalized!r}. "
+        f"Supported protocols: {sorted(KNOWN_PROTOCOLS)}. "
+        f"Supported aliases: {sorted(PROTOCOL_ALIASES.keys())}.",
+    )
+
+
+def create_client(
+    target: InferenceTarget,
+) -> OpenAICompatibleClient | OllamaNativeClient:
+    """Create the appropriate inference client for the given target.
+
+    Routes based on the target's protocol field. If the protocol is an alias
+    (e.g. "vllm"), it is resolved first. Unknown protocols raise a typed
+    configuration error — they NEVER silently fall back to Ollama.
+
+    Args:
+        target: Fully resolved inference target with protocol, URL, etc.
+
+    Returns:
+        An OpenAICompatibleClient or OllamaNativeClient instance.
+
+    Raises:
+        InferenceError: If the protocol is unknown or unsupported.
+    """
+    protocol = resolve_protocol(target.protocol)
+
+    if protocol == "openai_chat":
+        return OpenAICompatibleClient(target)
+
+    if protocol == "ollama_native":
+        return OllamaNativeClient(target)
+
+    # specialist_http is a valid protocol but has no client implementation yet
+    # (handled by a separate specialist service layer). Fail closed here.
+    raise InferenceError(
+        InferenceErrorCategory.CAPABILITY_UNAVAILABLE,
+        f"Protocol {protocol!r} is recognized but has no client implementation in this factory. "
+        f"Use the specialist HTTP service layer directly.",
+    )

services/shared/inference/gateway.py

@@ -0,0 +1,200 @@
+"""Inference Gateway facade.
+
+Wraps the factory + client lifecycle for all LLM consumers.
+Provides a single ``generate()`` method that dispatches to the correct
+client implementation based on the InferenceTarget protocol.
+
+Manages client instances (one per endpoint, reusable) and provides
+explicit ``refresh_target()`` for configuration changes — no private
+``_config`` mutation.
+
+Requirements: 2.1, 2.12, 13.6
+"""
+from __future__ import annotations
+
+import logging
+from typing import Protocol, runtime_checkable
+from uuid import UUID
+
+from services.shared.inference.models import (
+    InferenceResult,
+    InferenceTarget,
+    StructuredGenerationRequest,
+)
+
+logger = logging.getLogger("inference.gateway")
+
+
+@runtime_checkable
+class InferenceClient(Protocol):
+    """Protocol for inference client implementations."""
+
+    async def generate(self, request: StructuredGenerationRequest) -> InferenceResult:
+        """Send a structured generation request and return the result."""
+        ...
+
+    async def close(self) -> None:
+        """Release underlying HTTP resources."""
+        ...
+
+
+class InferenceGateway:
+    """Gateway facade that routes inference requests to protocol-specific clients.
+
+    Key behaviours:
+    - Maintains a client pool keyed by (endpoint_id, protocol)
+    - Creates clients lazily on first request for a target
+    - Reuses existing clients for the same endpoint
+    - Provides ``refresh_target()`` to invalidate a cached client
+      (replaces private ``_config`` mutation pattern)
+    - Fails closed on unknown protocols
+
+    Usage::
+
+        gateway = InferenceGateway()
+        result = await gateway.generate(target, request)
+        # Later, when config changes:
+        await gateway.refresh_target(target.endpoint_id)
+    """
+
+    def __init__(self) -> None:
+        # Pool of active clients keyed by endpoint_id
+        self._clients: dict[UUID, InferenceClient] = {}
+        # Track the target associated with each client for refresh comparison
+        self._targets: dict[UUID, InferenceTarget] = {}
+
+    async def generate(
+        self,
+        target: InferenceTarget,
+        request: StructuredGenerationRequest,
+    ) -> InferenceResult:
+        """Route a structured generation request to the appropriate client.
+
+        Creates or reuses a client for the target's endpoint. The result
+        always includes endpoint_id, deployment_id, model, and protocol
+        for lineage recording.
+
+        Args:
+            target: Resolved inference target with protocol, endpoint, and capabilities.
+            request: The structured generation request.
+
+        Returns:
+            InferenceResult with content and full lineage metadata.
+
+        Raises:
+            ValueError: If the target protocol is unknown (fail-closed).
+        """
+        client = self._get_or_create_client(target)
+        result = await client.generate(request)
+
+        # Ensure lineage fields are populated from target
+        if result.endpoint_id is None:
+            result.endpoint_id = target.endpoint_id
+        if result.deployment_id is None:
+            result.deployment_id = target.deployment_id
+        if not result.model:
+            result.model = target.model
+        if not result.protocol or result.protocol != target.protocol:
+            result.protocol = target.protocol
+
+        return result
+
+    async def refresh_target(self, endpoint_id: UUID) -> None:
+        """Invalidate and close the cached client for an endpoint.
+
+        Call this when endpoint configuration changes (model swap,
+        URL change, credential rotation, etc.) instead of mutating
+        private ``_config`` attributes.
+
+        The next ``generate()`` call for this endpoint will create
+        a fresh client from the new target.
+        """
+        client = self._clients.pop(endpoint_id, None)
+        self._targets.pop(endpoint_id, None)
+        if client is not None:
+            logger.info("Refreshing client for endpoint %s", endpoint_id)
+            try:
+                await client.close()
+            except Exception:
+                logger.warning(
+                    "Error closing client during refresh for endpoint %s",
+                    endpoint_id,
+                    exc_info=True,
+                )
+
+    async def close(self) -> None:
+        """Close all managed clients and release resources."""
+        for endpoint_id, client in list(self._clients.items()):
+            try:
+                await client.close()
+            except Exception:
+                logger.warning(
+                    "Error closing client for endpoint %s",
+                    endpoint_id,
+                    exc_info=True,
+                )
+        self._clients.clear()
+        self._targets.clear()
+
+    def _get_or_create_client(self, target: InferenceTarget) -> InferenceClient:
+        """Get an existing client or create a new one for the target.
+
+        Clients are cached by endpoint_id. If the target has changed
+        (different model, URL, etc.), the old client is replaced.
+        """
+        endpoint_id = target.endpoint_id
+        existing = self._clients.get(endpoint_id)
+
+        if existing is not None:
+            # Reuse if target hasn't changed
+            cached_target = self._targets.get(endpoint_id)
+            if cached_target is target or cached_target == target:
+                return existing
+
+        # Create a new client for the target's protocol
+        client = self._create_client(target)
+        self._clients[endpoint_id] = client
+        self._targets[endpoint_id] = target
+        return client
+
+    def _create_client(self, target: InferenceTarget) -> InferenceClient:
+        """Create a protocol-specific client for the target.
+
+        Raises:
+            ValueError: If the protocol is unknown (fail-closed per Req 2.6).
+        """
+        protocol = target.protocol
+
+        if protocol == "openai_chat":
+            from services.shared.inference.clients.openai_compatible import (
+                OpenAICompatibleClient,
+            )
+            logger.info(
+                "Creating OpenAICompatibleClient for endpoint %s (model=%s)",
+                target.endpoint_id,
+                target.model,
+            )
+            return OpenAICompatibleClient(target)
+
+        if protocol == "ollama_native":
+            from services.shared.inference.clients.ollama_native import (
+                OllamaNativeClient,
+            )
+            logger.info(
+                "Creating OllamaNativeClient for endpoint %s (model=%s)",
+                target.endpoint_id,
+                target.model,
+            )
+            return OllamaNativeClient(target)
+
+        # Unknown protocol — fail closed (Requirement 2.6)
+        raise ValueError(
+            f"Unknown inference protocol '{protocol}' for endpoint {target.endpoint_id}. "
+            "Supported protocols: 'ollama_native', 'openai_chat', 'specialist_http'. "
+            "The gateway will NOT silently route to a default provider."
+        )
+
+    @property
+    def active_endpoints(self) -> list[UUID]:
+        """Return the list of endpoint IDs with active cached clients."""
+        return list(self._clients.keys())

services/shared/inference/lineage.py

@@ -0,0 +1,69 @@
+"""Lineage recording for inference results.
+
+Extracts persistence fields from InferenceResult so that actual endpoint,
+model, and route lineage are recorded — fixing the hardcoded
+``model_provider = 'ollama'`` pattern.
+
+Requirements: 2.9, 13.6
+"""
+from __future__ import annotations
+
+from services.shared.inference.models import InferenceResult, ModelLineage
+
+
+def build_lineage_from_result(result: InferenceResult, trace_id: str = "") -> ModelLineage:
+    """Extract a ModelLineage record from an InferenceResult.
+
+    This replaces hardcoded ``model_provider = 'ollama'`` persistence
+    by capturing the actual endpoint, deployment, model, protocol,
+    structured mode, request ID, latency, and retries from the result.
+
+    Args:
+        result: The completed inference result.
+        trace_id: Optional distributed trace ID for correlation.
+
+    Returns:
+        A ModelLineage with all required persistence fields.
+    """
+    return ModelLineage(
+        endpoint_id=result.endpoint_id,
+        deployment_id=result.deployment_id,
+        model=result.model,
+        protocol=result.protocol,
+        structured_mode=result.structured_mode,
+        request_id=result.request_id,
+        latency_ms=result.latency_ms,
+        retries=result.retries,
+        trace_id=trace_id,
+    )
+
+
+def lineage_to_persistence_dict(lineage: ModelLineage) -> dict:
+    """Convert a ModelLineage to a flat dict for database persistence.
+
+    Returns fields suitable for inserting into agent_performance_log,
+    document_intelligence, or similar tables.
+
+    The ``model_provider`` field is derived from the protocol:
+    - ``ollama_native`` → ``"ollama"``
+    - ``openai_chat`` → ``"openai_compatible"`` (covers vLLM, OpenAI, etc.)
+    - ``specialist_http`` → ``"specialist"``
+    """
+    protocol_to_provider = {
+        "ollama_native": "ollama",
+        "openai_chat": "openai_compatible",
+        "specialist_http": "specialist",
+    }
+
+    return {
+        "model_provider": protocol_to_provider.get(lineage.protocol, lineage.protocol),
+        "model_name": lineage.model,
+        "endpoint_id": str(lineage.endpoint_id) if lineage.endpoint_id else None,
+        "deployment_id": str(lineage.deployment_id) if lineage.deployment_id else None,
+        "protocol": lineage.protocol,
+        "structured_mode": lineage.structured_mode,
+        "request_id": lineage.request_id,
+        "latency_ms": lineage.latency_ms,
+        "retries": lineage.retries,
+        "trace_id": lineage.trace_id,
+    }

services/shared/inference/migration.py

@@ -0,0 +1,70 @@
+"""Migration helpers for deprecated provider records.
+
+Scans agent configuration records for deprecated provider values (e.g. "vllm")
+and produces structured deprecation warnings to guide operators toward the
+canonical protocol names.
+
+Requirements: 2.2, 3.8
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from services.shared.inference.factory import PROTOCOL_ALIASES
+
+
+@dataclass(frozen=True)
+class ProviderDeprecationWarning:
+    """Structured deprecation warning for an agent record using a deprecated provider.
+
+    Attributes:
+        agent_id: The ID of the agent with the deprecated provider.
+        current_value: The current deprecated provider string (e.g. "vllm").
+        recommended_value: The canonical protocol to migrate to.
+        message: Human-readable migration guidance.
+    """
+
+    agent_id: str
+    current_value: str
+    recommended_value: str
+    message: str
+
+
+def check_deprecated_providers(
+    agent_records: list[dict],
+) -> list[ProviderDeprecationWarning]:
+    """Scan agent records for deprecated provider values.
+
+    Checks the ``model_provider`` field of each agent record against
+    PROTOCOL_ALIASES. Records using deprecated aliases get a warning
+    with migration guidance.
+
+    Args:
+        agent_records: List of dicts, each having at least ``agent_id``
+            (or ``id``) and ``model_provider`` fields.
+
+    Returns:
+        List of ProviderDeprecationWarning for records using deprecated providers.
+    """
+    warnings_list: list[ProviderDeprecationWarning] = []
+
+    for record in agent_records:
+        agent_id = str(record.get("agent_id") or record.get("id", "unknown"))
+        provider = (record.get("model_provider") or "").strip().lower()
+
+        if provider in PROTOCOL_ALIASES:
+            recommended = PROTOCOL_ALIASES[provider]
+            warnings_list.append(
+                ProviderDeprecationWarning(
+                    agent_id=agent_id,
+                    current_value=provider,
+                    recommended_value=recommended,
+                    message=(
+                        f"Agent {agent_id} uses deprecated provider '{provider}'. "
+                        f"Migrate to protocol '{recommended}'. "
+                        f"The '{provider}' alias will be removed in a future version."
+                    ),
+                )
+            )
+
+    return warnings_list

services/shared/inference/models.py

@@ -0,0 +1,132 @@
+"""Inference gateway domain models.
+
+Core types for the capability-aware inference gateway.
+Requirements: 2.1, 2.8, 2.9
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+
+@dataclass(frozen=True)
+class ProviderCapabilities:
+    """Declared capabilities for an inference endpoint/deployment."""
+
+    chat_completions: bool = False
+    responses_api: bool = False
+    json_schema: bool = False
+    json_object: bool = False
+    seed: bool = False
+    usage: bool = False
+    max_completion_tokens: bool = False
+    reasoning_toggle: bool = False
+    model_listing: bool = False
+
+
+@dataclass(frozen=True)
+class InferenceTarget:
+    """Resolved target for an inference request."""
+
+    endpoint_id: UUID
+    deployment_id: UUID
+    protocol: Literal["ollama_native", "openai_chat", "specialist_http"]
+    base_url: str
+    model: str
+    capabilities: ProviderCapabilities
+    auth_secret_ref: str | None = None
+    auth_scheme: str = "bearer"
+    extra_headers: dict[str, str] = field(default_factory=dict)
+    extra_body: dict[str, Any] = field(default_factory=dict)
+    max_retries: int = 3
+    timeout_seconds: float = 120.0
+    context_window: int = 0
+    max_output_tokens: int | None = None
+
+
+class ChatMessage(BaseModel):
+    """A single chat message."""
+
+    role: Literal["system", "user", "assistant"]
+    content: str
+
+
+class StructuredGenerationRequest(BaseModel):
+    """Request for structured generation via the inference gateway."""
+
+    messages: list[ChatMessage]
+    json_schema: dict[str, Any] | None = None
+    max_output_tokens: int = 4096
+    temperature: float = 0.0
+    seed: int | None = 0
+    timeout_seconds: float = 120.0
+    trace_id: str = ""
+
+
+class TokenUsage(BaseModel):
+    """Token usage metadata from an inference response."""
+
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    total_tokens: int | None = None
+
+
+class InferenceResult(BaseModel):
+    """Result of an inference request.
+
+    Contains the generated content plus metadata for lineage,
+    observability, and audit.
+    """
+
+    content: str
+    parsed: dict[str, Any] | None = None
+    endpoint_id: UUID | None = None
+    deployment_id: UUID | None = None
+    model: str = ""
+    protocol: Literal["ollama_native", "openai_chat", "specialist_http"] = "openai_chat"
+    structured_mode: Literal["json_schema", "json_object", "prompt_only", "none"] = "none"
+    latency_ms: int = 0
+    usage: TokenUsage = Field(default_factory=TokenUsage)
+    request_id: str | None = None
+    finish_reason: str | None = None
+    repaired: bool = False
+    retries: int = 0
+    error: str | None = None
+    error_category: str | None = None
+    schema_valid: bool | None = None
+
+
+class ModelLineage(BaseModel):
+    """Lineage record capturing which endpoint, model, and route served a request.
+
+    Used for persistence so actual endpoint, model, and route lineage are
+    recorded (fixes hardcoded model_provider = 'ollama').
+    """
+
+    endpoint_id: UUID | None = None
+    deployment_id: UUID | None = None
+    model: str = ""
+    protocol: Literal["ollama_native", "openai_chat", "specialist_http"] = "openai_chat"
+    structured_mode: str = "none"
+    request_id: str | None = None
+    latency_ms: int = 0
+    retries: int = 0
+    trace_id: str = ""
+
+
+# Error categories for provider failures
+class ErrorCategory:
+    """Normalized error categories for inference failures."""
+
+    TIMEOUT = "timeout"
+    AUTHENTICATION = "authentication"
+    RATE_LIMIT = "rate_limit"
+    SERVER_ERROR = "server_error"
+    INVALID_RESPONSE = "invalid_response"
+    SCHEMA_VIOLATION = "schema_violation"
+    CAPABILITY_ERROR = "capability_error"
+    POLICY_ERROR = "policy_error"
+    CONNECTION_ERROR = "connection_error"

services/shared/inference/redaction.py

@@ -0,0 +1,120 @@
+"""Credential and sensitive header redaction utilities.
+
+Ensures that authentication values, API keys, bearer tokens, and other
+sensitive headers are never included in logs, traces, serialized request
+snapshots, or error messages.
+
+Requirements: 2.8
+"""
+from __future__ import annotations
+
+import re
+from typing import Any, Mapping
+
+from services.shared.inference.models import InferenceTarget
+
+# ---------------------------------------------------------------------------
+# Sensitive header detection
+# ---------------------------------------------------------------------------
+
+#: Header names (lowercase) that are always redacted.
+SENSITIVE_HEADER_NAMES: frozenset[str] = frozenset(
+    {
+        "authorization",
+        "x-api-key",
+        "api-key",
+        "x-auth-token",
+        "proxy-authorization",
+        "cookie",
+        "set-cookie",
+        "x-secret",
+        "x-access-token",
+    }
+)
+
+_REDACTED = "***REDACTED***"
+
+# Patterns that look like bearer tokens or API keys in free text
+_TOKEN_PATTERNS: list[re.Pattern[str]] = [
+    # Bearer tokens
+    re.compile(r"(Bearer\s+)\S+", re.IGNORECASE),
+    # Common API key formats (sk-..., pk-..., key-..., token-..., api_key_...)
+    re.compile(r"\b((?:sk|pk|key|token|api[_-]?key)[_-])\S{8,}", re.IGNORECASE),
+]
+
+
+# ---------------------------------------------------------------------------
+# Header redaction
+# ---------------------------------------------------------------------------
+
+
+def redact_headers(headers: Mapping[str, str]) -> dict[str, str]:
+    """Return a copy of headers with sensitive values replaced by a placeholder.
+
+    Matches header names case-insensitively against ``SENSITIVE_HEADER_NAMES``.
+    """
+    result: dict[str, str] = {}
+    for name, value in headers.items():
+        if name.lower() in SENSITIVE_HEADER_NAMES:
+            result[name] = _REDACTED
+        else:
+            result[name] = value
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Error message redaction
+# ---------------------------------------------------------------------------
+
+
+def redact_error_message(message: str) -> str:
+    """Remove bearer tokens and API key patterns from text.
+
+    Applied to error messages before they are logged or stored.
+    """
+    result = message
+    for pattern in _TOKEN_PATTERNS:
+        result = pattern.sub(
+            lambda m: m.group(1) + _REDACTED if m.lastindex else _REDACTED,
+            result,
+        )
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Target redaction for logging / serialization
+# ---------------------------------------------------------------------------
+
+
+def redact_target_for_logging(target: InferenceTarget) -> dict[str, Any]:
+    """Serialize an InferenceTarget for logging with credentials redacted.
+
+    - ``auth_secret_ref`` is included as the reference name (e.g.
+      "vault://inference/openai-key") so operators can identify which
+      secret is in use. The target never holds the resolved secret value.
+    - Sensitive headers in ``extra_headers`` are redacted.
+    - ``extra_body`` is included as-is (callers should not place secrets there).
+
+    Requirements: 2.8
+    """
+    return {
+        "endpoint_id": str(target.endpoint_id),
+        "deployment_id": str(target.deployment_id),
+        "protocol": target.protocol,
+        "base_url": target.base_url,
+        "model": target.model,
+        "capabilities": {
+            "chat_completions": target.capabilities.chat_completions,
+            "responses_api": target.capabilities.responses_api,
+            "json_schema": target.capabilities.json_schema,
+            "json_object": target.capabilities.json_object,
+            "seed": target.capabilities.seed,
+            "usage": target.capabilities.usage,
+            "max_completion_tokens": target.capabilities.max_completion_tokens,
+            "reasoning_toggle": target.capabilities.reasoning_toggle,
+            "model_listing": target.capabilities.model_listing,
+        },
+        "auth_secret_ref": target.auth_secret_ref,
+        "extra_headers": redact_headers(dict(target.extra_headers)),
+        "extra_body": dict(target.extra_body),
+    }

services/shared/inference/registry.py

@@ -0,0 +1,319 @@
+"""Registry resolver for the inference gateway.
+
+Resolves agent stage bindings to complete InferenceTarget instances
+using TTL-cached lookups against the registry tables. Auth secrets
+are NOT resolved during caching — only at invocation time.
+
+Requirements: 3.5, 3.9
+"""
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any, TypeVar
+from uuid import UUID
+
+from services.shared.inference.errors import InferenceError, InferenceErrorCategory
+from services.shared.inference.models import InferenceTarget, ProviderCapabilities
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+# Default cache TTL in seconds
+DEFAULT_CACHE_TTL_SECONDS = 60.0
+
+
+# ---------------------------------------------------------------------------
+# RegistryCache — internal TTL dict
+# ---------------------------------------------------------------------------
+
+
+class RegistryCache:
+    """TTL-based in-memory cache for registry lookups.
+
+    Keys follow the format:
+      - "binding:{agent_id}:{stage}"
+      - "endpoint:{endpoint_id}"
+      - "deployment:{deployment_id}"
+
+    Expired entries are lazily evicted on access.
+    """
+
+    def __init__(self, ttl_seconds: float = DEFAULT_CACHE_TTL_SECONDS) -> None:
+        self._ttl_seconds = ttl_seconds
+        self._store: dict[str, tuple[float, Any]] = {}
+
+    @property
+    def ttl_seconds(self) -> float:
+        return self._ttl_seconds
+
+    def get(self, key: str) -> Any | None:
+        """Retrieve a cached value if it exists and hasn't expired."""
+        entry = self._store.get(key)
+        if entry is None:
+            return None
+
+        stored_at, value = entry
+        if (time.monotonic() - stored_at) > self._ttl_seconds:
+            del self._store[key]
+            return None
+
+        return value
+
+    def set(self, key: str, value: Any) -> None:
+        """Store a value with the current timestamp."""
+        self._store[key] = (time.monotonic(), value)
+
+    def invalidate(self, key_pattern: str) -> None:
+        """Invalidate all entries whose key starts with the given pattern.
+
+        Supports prefix-based invalidation:
+          - invalidate("endpoint:abc-123") removes that specific endpoint
+          - invalidate("endpoint:") removes ALL endpoint entries
+          - invalidate("binding:agent-1:") removes all bindings for agent-1
+        """
+        keys_to_remove = [k for k in self._store if k.startswith(key_pattern)]
+        for k in keys_to_remove:
+            del self._store[k]
+
+    def clear(self) -> None:
+        """Clear all cached entries."""
+        self._store.clear()
+
+    def __len__(self) -> int:
+        return len(self._store)
+
+    def __contains__(self, key: str) -> bool:
+        """Check if a non-expired entry exists for the key."""
+        return self.get(key) is not None
+
+
+# ---------------------------------------------------------------------------
+# Database query protocol
+# ---------------------------------------------------------------------------
+
+
+class RegistryDB:
+    """Protocol for registry database queries.
+
+    In production this would be backed by asyncpg. For testing,
+    a simple dict-based mock implements the same interface.
+    """
+
+    async def get_active_binding(
+        self, agent_id: UUID, stage: str
+    ) -> dict[str, Any] | None:
+        """Get the active binding for an agent+stage.
+
+        Returns a dict with keys: id, agent_id, stage, model_deployment_id,
+        route_order, routing_config, is_active, revision.
+        Returns None if no active binding exists.
+        """
+        raise NotImplementedError
+
+    async def get_model_deployment(
+        self, deployment_id: UUID
+    ) -> dict[str, Any] | None:
+        """Get a model deployment by ID.
+
+        Returns a dict with keys: id, endpoint_id, served_model_name,
+        display_name, capabilities, context_window, max_output_tokens,
+        quantization, runtime_metadata, enabled, revision.
+        """
+        raise NotImplementedError
+
+    async def get_inference_endpoint(
+        self, endpoint_id: UUID
+    ) -> dict[str, Any] | None:
+        """Get an inference endpoint by ID.
+
+        Returns a dict with keys: id, name, protocol, base_url,
+        auth_secret_ref, auth_scheme, default_headers, health_path,
+        enabled, revision.
+        """
+        raise NotImplementedError
+
+
+# ---------------------------------------------------------------------------
+# RegistryResolver
+# ---------------------------------------------------------------------------
+
+
+class RegistryResolver:
+    """Resolves agent stage bindings to InferenceTarget instances.
+
+    Resolution order:
+      1. Find active binding for (agent_id, stage)
+      2. Get model_deployment from binding
+      3. Get inference_endpoint from deployment
+      4. Build InferenceTarget
+
+    Fail-closed: any missing or disabled resource raises InferenceError
+    with CAPABILITY_UNAVAILABLE. Never returns a fallback target.
+
+    Auth secrets are NOT resolved during caching — auth_secret_ref is
+    preserved as-is in the target. Resolution happens at invocation time
+    by the client layer.
+    """
+
+    def __init__(
+        self,
+        db: RegistryDB,
+        *,
+        cache_ttl_seconds: float = DEFAULT_CACHE_TTL_SECONDS,
+    ) -> None:
+        self._db = db
+        self._cache = RegistryCache(ttl_seconds=cache_ttl_seconds)
+
+    @property
+    def cache(self) -> RegistryCache:
+        """Access the internal cache (for testing/inspection)."""
+        return self._cache
+
+    async def resolve_target(self, agent_id: UUID, stage: str) -> InferenceTarget:
+        """Resolve the active binding for an agent+stage into an InferenceTarget.
+
+        Uses TTL-cached lookups. If any component is missing or disabled,
+        raises InferenceError(CAPABILITY_UNAVAILABLE).
+
+        Auth secret is NOT resolved here — only at invocation time.
+        """
+        # Check cache first for the full resolved target
+        cache_key = f"binding:{agent_id}:{stage}"
+        cached_target = self._cache.get(cache_key)
+        if cached_target is not None:
+            return cached_target
+
+        # Step 1: Find active binding
+        binding = await self._db.get_active_binding(agent_id, stage)
+        if binding is None:
+            raise InferenceError(
+                InferenceErrorCategory.CAPABILITY_UNAVAILABLE,
+                f"No active binding for agent={agent_id} stage={stage}",
+            )
+
+        if not binding.get("is_active", False):
+            raise InferenceError(
+                InferenceErrorCategory.CAPABILITY_UNAVAILABLE,
+                f"Binding for agent={agent_id} stage={stage} is inactive",
+            )
+
+        deployment_id = binding.get("model_deployment_id")
+        if deployment_id is None:
+            raise InferenceError(
+                InferenceErrorCategory.CAPABILITY_UNAVAILABLE,
+                f"Binding for agent={agent_id} stage={stage} has no deployment",
+            )
+
+        # Step 2: Get model deployment
+        deployment = await self._resolve_deployment(deployment_id)
+
+        # Step 3: Get inference endpoint
+        endpoint_id = deployment["endpoint_id"]
+        endpoint = await self._resolve_endpoint(endpoint_id)
+
+        # Step 4: Build InferenceTarget
+        capabilities_data = deployment.get("capabilities", {})
+        capabilities = ProviderCapabilities(
+            chat_completions=capabilities_data.get("chat_completions", False),
+            responses_api=capabilities_data.get("responses_api", False),
+            json_schema=capabilities_data.get("json_schema", False),
+            json_object=capabilities_data.get("json_object", False),
+            seed=capabilities_data.get("seed", False),
+            usage=capabilities_data.get("usage", False),
+            max_completion_tokens=capabilities_data.get("max_completion_tokens", False),
+            reasoning_toggle=capabilities_data.get("reasoning_toggle", False),
+            model_listing=capabilities_data.get("model_listing", False),
+        )
+
+        extra_headers = endpoint.get("default_headers", {})
+        if not isinstance(extra_headers, dict):
+            extra_headers = {}
+
+        runtime_metadata = deployment.get("runtime_metadata", {})
+        extra_body = runtime_metadata.get("extra_body", {}) if isinstance(runtime_metadata, dict) else {}
+
+        target = InferenceTarget(
+            endpoint_id=endpoint_id,
+            deployment_id=deployment_id,
+            protocol=endpoint["protocol"],
+            base_url=endpoint["base_url"],
+            model=deployment["served_model_name"],
+            capabilities=capabilities,
+            auth_secret_ref=endpoint.get("auth_secret_ref"),
+            auth_scheme=endpoint.get("auth_scheme", "bearer"),
+            extra_headers=extra_headers,
+            extra_body=extra_body,
+            context_window=deployment.get("context_window") or 0,
+            max_output_tokens=deployment.get("max_output_tokens"),
+        )
+
+        # Cache the resolved target
+        self._cache.set(cache_key, target)
+
+        return target
+
+    async def _resolve_deployment(self, deployment_id: UUID) -> dict[str, Any]:
+        """Resolve a model deployment, using cache if available."""
+        dep_cache_key = f"deployment:{deployment_id}"
+        cached = self._cache.get(dep_cache_key)
+        if cached is not None:
+            return cached
+
+        deployment = await self._db.get_model_deployment(deployment_id)
+        if deployment is None:
+            raise InferenceError(
+                InferenceErrorCategory.CAPABILITY_UNAVAILABLE,
+                f"Model deployment {deployment_id} not found",
+            )
+
+        if not deployment.get("enabled", False):
+            raise InferenceError(
+                InferenceErrorCategory.CAPABILITY_UNAVAILABLE,
+                f"Model deployment {deployment_id} is disabled",
+            )
+
+        self._cache.set(dep_cache_key, deployment)
+        return deployment
+
+    async def _resolve_endpoint(self, endpoint_id: UUID) -> dict[str, Any]:
+        """Resolve an inference endpoint, using cache if available."""
+        ep_cache_key = f"endpoint:{endpoint_id}"
+        cached = self._cache.get(ep_cache_key)
+        if cached is not None:
+            return cached
+
+        endpoint = await self._db.get_inference_endpoint(endpoint_id)
+        if endpoint is None:
+            raise InferenceError(
+                InferenceErrorCategory.CAPABILITY_UNAVAILABLE,
+                f"Inference endpoint {endpoint_id} not found",
+            )
+
+        if not endpoint.get("enabled", False):
+            raise InferenceError(
+                InferenceErrorCategory.CAPABILITY_UNAVAILABLE,
+                f"Inference endpoint {endpoint_id} is disabled",
+            )
+
+        self._cache.set(ep_cache_key, endpoint)
+        return endpoint
+
+    def invalidate(self, endpoint_id: UUID) -> None:
+        """Evict cache entries related to an endpoint.
+
+        Called on revision changes or probe failures to force
+        re-resolution on the next request.
+        """
+        self._cache.invalidate(f"endpoint:{endpoint_id}")
+        # Also clear all binding caches since they may reference this endpoint
+        # We clear all bindings because we can't efficiently know which bindings
+        # use this endpoint without scanning
+        self._cache.invalidate("binding:")
+        logger.info("Invalidated cache for endpoint %s", endpoint_id)
+
+    def invalidate_all(self) -> None:
+        """Full cache clear."""
+        self._cache.clear()
+        logger.info("Full registry cache invalidated")

services/shared/inference/seed_migration.py

@@ -0,0 +1,286 @@
+"""Seed migration helpers for the inference registry.
+
+Provides canonical endpoint profiles, model deployments, and agent
+provider-to-stage-binding conversion logic for migrating from the legacy
+model_provider/model_name fields to the v3 registry.
+
+Task 18.1-18.5: Migrate existing provider records.
+Requirements: 3.8, 3.9
+"""
+from __future__ import annotations
+
+from typing import Any
+from uuid import UUID
+
+# ─── Well-known IDs ────────────────────────────────────────────────────────────
+# These match the SQL seed migration (042_seed_inference_registry.sql)
+
+OLLAMA_ENDPOINT_ID = UUID("a0000000-0000-4000-8000-000000000001")
+VLLM_ENDPOINT_ID = UUID("a0000000-0000-4000-8000-000000000002")
+OLLAMA_DEPLOYMENT_ID = UUID("b0000000-0000-4000-8000-000000000001")
+VLLM_DEPLOYMENT_ID = UUID("b0000000-0000-4000-8000-000000000002")
+
+
+def get_initial_endpoints() -> list[dict[str, Any]]:
+    """Return the canonical endpoint profiles for the seed migration.
+
+    Returns:
+        List of endpoint dicts matching the inference_endpoints table schema.
+        - stonks-ollama: Ollama native protocol at cluster-internal URL.
+        - stonks-vllm: OpenAI-chat protocol (vLLM) at cluster-internal URL.
+    """
+    return [
+        {
+            "id": OLLAMA_ENDPOINT_ID,
+            "name": "stonks-ollama",
+            "protocol": "ollama_native",
+            "base_url": "http://ollama.ollama-service.svc.cluster.local:11434",
+            "auth_secret_ref": None,
+            "auth_scheme": "none",
+            "default_headers": {},
+            "health_path": "/api/tags",
+            "enabled": True,
+        },
+        {
+            "id": VLLM_ENDPOINT_ID,
+            "name": "stonks-vllm",
+            "protocol": "openai_chat",
+            "base_url": "http://kube-vllm.stonks-oracle.svc.cluster.local:8000",
+            "auth_secret_ref": None,
+            "auth_scheme": "none",
+            "default_headers": {},
+            "health_path": "/health",
+            "enabled": True,
+        },
+    ]
+
+
+def get_initial_deployments() -> list[dict[str, Any]]:
+    """Return the initial model deployments for the seed migration.
+
+    Returns:
+        List of deployment dicts matching the model_deployments table schema.
+        - Ollama: qwen3.5:9b with native JSON mode.
+        - vLLM: AxionML/Qwen3.5-9B-NVFP4 on RTX 4070 Ti SUPER, strict JSON Schema.
+    """
+    return [
+        {
+            "id": OLLAMA_DEPLOYMENT_ID,
+            "endpoint_id": OLLAMA_ENDPOINT_ID,
+            "served_model_name": "qwen3.5:9b",
+            "display_name": "Qwen 3.5 9B (Ollama)",
+            "capabilities": {
+                "chat_completions": True,
+                "json_schema": False,
+                "json_object": True,
+                "seed": False,
+                "usage": False,
+                "max_completion_tokens": False,
+                "model_listing": True,
+            },
+            "context_window": 32768,
+            "max_output_tokens": 32768,
+            "quantization": None,
+            "runtime_metadata": {
+                "source": "ollama_native",
+                "notes": "Ollama-served model with native JSON mode",
+            },
+            "enabled": True,
+        },
+        {
+            "id": VLLM_DEPLOYMENT_ID,
+            "endpoint_id": VLLM_ENDPOINT_ID,
+            "served_model_name": "AxionML/Qwen3.5-9B-NVFP4",
+            "display_name": "Qwen 3.5 9B NVFP4 (vLLM)",
+            "capabilities": {
+                "chat_completions": True,
+                "json_schema": True,
+                "json_object": True,
+                "seed": True,
+                "usage": True,
+                "max_completion_tokens": True,
+                "model_listing": True,
+            },
+            "context_window": 8192,
+            "max_output_tokens": 2048,
+            "quantization": "NVFP4",
+            "runtime_metadata": {
+                "gpu": "RTX 4070 Ti SUPER",
+                "gpu_memory_utilization": 0.80,
+                "max_num_seqs": 8,
+                "vllm_structured_outputs": True,
+            },
+            "enabled": True,
+        },
+    ]
+
+
+# ─── Provider mapping ──────────────────────────────────────────────────────────
+# Maps legacy model_provider values to endpoint/deployment IDs.
+
+_PROVIDER_TO_ENDPOINT: dict[str, UUID] = {
+    "ollama": OLLAMA_ENDPOINT_ID,
+    "vllm": VLLM_ENDPOINT_ID,
+}
+
+_PROVIDER_TO_DEPLOYMENT: dict[str, UUID] = {
+    "ollama": OLLAMA_DEPLOYMENT_ID,
+    "vllm": VLLM_DEPLOYMENT_ID,
+}
+
+
+class UnknownProviderError(ValueError):
+    """Raised when an agent record has an unrecognized model_provider value."""
+
+    def __init__(self, provider: str, agent_id: str) -> None:
+        self.provider = provider
+        self.agent_id = agent_id
+        super().__init__(
+            f"Unknown model_provider '{provider}' for agent '{agent_id}'. "
+            f"Supported providers: {sorted(_PROVIDER_TO_ENDPOINT.keys())}. "
+            f"Cannot silently convert unknown providers."
+        )
+
+
+def convert_agent_providers(agent_records: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Convert existing agent model_provider/model_name fields to stage bindings.
+
+    For each agent record with a known model_provider (ollama or vllm), produces
+    a stage binding record that maps the agent to the appropriate endpoint and
+    model deployment. The original model_provider and model_name fields are
+    retained for backward compatibility — this function supplements, not replaces.
+
+    Args:
+        agent_records: List of dicts with at least 'id' (or 'agent_id'),
+            'model_provider', and optionally 'slug' fields.
+
+    Returns:
+        List of stage binding dicts suitable for insertion into
+        agent_stage_bindings.
+
+    Raises:
+        UnknownProviderError: If a record has a model_provider that cannot be
+            mapped. Unknown providers must NOT be silently converted.
+    """
+    bindings: list[dict[str, Any]] = []
+
+    for record in agent_records:
+        agent_id = str(record.get("id") or record.get("agent_id", ""))
+        provider = (record.get("model_provider") or "").strip().lower()
+
+        if not provider:
+            # No provider set — skip, nothing to convert
+            continue
+
+        if provider not in _PROVIDER_TO_ENDPOINT:
+            raise UnknownProviderError(provider=provider, agent_id=agent_id)
+
+        endpoint_id = _PROVIDER_TO_ENDPOINT[provider]
+        deployment_id = _PROVIDER_TO_DEPLOYMENT[provider]
+
+        # Determine stage from agent slug or default to 'extraction'
+        slug = record.get("slug", "")
+        stage = _infer_stage_from_slug(slug)
+
+        bindings.append({
+            "agent_id": agent_id,
+            "stage": stage,
+            "endpoint_id": endpoint_id,
+            "model_deployment_id": str(deployment_id),
+            "route_order": 0,
+            "routing_config": {},
+            "is_active": True,
+        })
+
+    return bindings
+
+
+def _infer_stage_from_slug(slug: str) -> str:
+    """Map an agent slug to a pipeline stage name.
+
+    Known agent slugs and their corresponding stages:
+        - document-extractor -> extraction
+        - event-classifier -> classification
+        - thesis-rewriter -> thesis_rewrite
+        - report-summarizer -> summarization
+
+    Falls back to 'extraction' for unrecognized slugs.
+    """
+    slug_to_stage: dict[str, str] = {
+        "document-extractor": "extraction",
+        "event-classifier": "classification",
+        "thesis-rewriter": "thesis_rewrite",
+        "report-summarizer": "summarization",
+    }
+    return slug_to_stage.get(slug, "extraction")
+
+
+# ─── Conflicting defaults identification ──────────────────────────────────────
+
+# Known locations where model/provider defaults have historically conflicted.
+_KNOWN_CONFLICT_LOCATIONS: list[dict[str, str]] = [
+    {
+        "location": "services/shared/config.py",
+        "field": "VLLMConfig.model",
+        "description": "Python config default for vLLM model name",
+    },
+    {
+        "location": "services/shared/config.py",
+        "field": "VLLMConfig.base_url",
+        "description": "Python config default for vLLM base URL (192.168.42.254:8000)",
+    },
+    {
+        "location": "infra/migrations/026_ai_agents.sql",
+        "field": "model_provider/model_name DEFAULT",
+        "description": "Agent table DDL defaults to 'ollama'/'qwen3.5:9b-fast'",
+    },
+    {
+        "location": "infra/migrations/031_fix_agent_defaults.sql",
+        "field": "model_provider UPDATE",
+        "description": "Migration updates agents to 'vllm'/'AxionML/Qwen3.5-9B-NVFP4'",
+    },
+    {
+        "location": "infra/migrations/033_stop_hardcoding_agent_model.sql",
+        "field": "model_provider UPDATE",
+        "description": "Migration updates agents to 'vllm'/'AxionML/Qwen3.5-9B-NVFP4'",
+    },
+    {
+        "location": "infra/helm/stonks-oracle/values.yaml",
+        "field": "VLLM_BASE_URL / VLLM_MODEL",
+        "description": "Helm values point to nuextract-external.vllm-service with NuExtract3",
+    },
+    {
+        "location": "infra/helm/stonks-oracle/values.yaml",
+        "field": "OLLAMA_BASE_URL / OLLAMA_MODEL",
+        "description": "Helm values point to nuextract-external.vllm-service with NuExtract3",
+    },
+    {
+        "location": "infra/kube-vllm/deployment.yaml",
+        "field": "vLLM deployment model arg",
+        "description": "Standalone kube-vllm deployment with AxionML/Qwen3.5-9B-NVFP4",
+    },
+]
+
+
+def identify_conflicting_defaults() -> list[str]:
+    """List locations where model defaults historically conflict.
+
+    Returns a list of human-readable strings identifying places where
+    the model provider, model name, or base URL have conflicting
+    defaults across config.py, migrations, Helm values, and the
+    kube-vllm deployment.
+
+    These conflicts should be resolved after the inference registry
+    is established as the single source of truth.
+
+    Returns:
+        List of conflict description strings.
+    """
+    conflicts: list[str] = []
+
+    for entry in _KNOWN_CONFLICT_LOCATIONS:
+        conflicts.append(
+            f"{entry['location']} [{entry['field']}]: {entry['description']}"
+        )
+
+    return conflicts