feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,10 @@
|
||||
"""Inference gateway clients.
|
||||
|
||||
Exports:
|
||||
OpenAICompatibleClient - Client for OpenAI-compatible endpoints (vLLM, OpenAI, etc.)
|
||||
OllamaNativeClient - Client for Ollama /api/chat native endpoints.
|
||||
"""
|
||||
from services.shared.inference.clients.ollama_native import OllamaNativeClient
|
||||
from services.shared.inference.clients.openai_compatible import OpenAICompatibleClient
|
||||
|
||||
__all__ = ["OpenAICompatibleClient", "OllamaNativeClient"]
|
||||
@@ -0,0 +1,382 @@
|
||||
"""Ollama native client for the shared inference gateway.
|
||||
|
||||
Implements generation via the Ollama /api/chat endpoint with:
|
||||
- Shared StructuredGenerationRequest/InferenceResult types
|
||||
- Native JSON schema formatting when supported (format field)
|
||||
- Prompt-only fallback with explicit reporting
|
||||
- Stall/loop detection as Ollama-specific policy
|
||||
- Configurable max output tokens (num_predict) and context window (num_ctx)
|
||||
- Error mapping to shared InferenceErrorCategory
|
||||
|
||||
Requirements: 2.1, 2.6
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
||||
from services.shared.inference.errors import InferenceError, InferenceErrorCategory
|
||||
from services.shared.inference.models import (
|
||||
InferenceResult,
|
||||
InferenceTarget,
|
||||
StructuredGenerationRequest,
|
||||
TokenUsage,
|
||||
)
|
||||
|
||||
# Re-export for backward compatibility
|
||||
__all__ = ["OllamaNativeClient", "StallPolicy"]
|
||||
|
||||
logger = logging.getLogger("inference.ollama_native")
|
||||
|
||||
|
||||
@dataclass
|
||||
class StallPolicy:
|
||||
"""Ollama-specific stall/loop detection configuration.
|
||||
|
||||
Monitors streaming responses for repetitive output patterns.
|
||||
This policy is intentionally Ollama-specific and does not leak
|
||||
into the generic InferenceResult interface.
|
||||
"""
|
||||
|
||||
enabled: bool = True
|
||||
check_interval_seconds: float = 5.0
|
||||
max_unchanged_intervals: int = 6
|
||||
loop_window: int = 64
|
||||
loop_threshold: float = 0.5
|
||||
|
||||
|
||||
def _map_http_status_to_category(status_code: int) -> InferenceErrorCategory:
|
||||
"""Map Ollama HTTP status codes to normalized error categories."""
|
||||
if status_code == 401:
|
||||
return InferenceErrorCategory.AUTH_FAILED
|
||||
if status_code == 403:
|
||||
return InferenceErrorCategory.FORBIDDEN
|
||||
if status_code == 404:
|
||||
return InferenceErrorCategory.MODEL_NOT_FOUND
|
||||
if status_code == 429:
|
||||
return InferenceErrorCategory.RATE_LIMITED
|
||||
if status_code in (400, 422):
|
||||
return InferenceErrorCategory.BAD_REQUEST
|
||||
if status_code >= 500:
|
||||
return InferenceErrorCategory.SERVER_ERROR
|
||||
return InferenceErrorCategory.UNKNOWN
|
||||
|
||||
|
||||
def _detect_loop(content: str, window: int, threshold: float) -> bool:
|
||||
"""Detect repetitive output by checking if the tail repeats earlier content.
|
||||
|
||||
Looks at the last `window` characters and checks if a significant
|
||||
portion of the tail matches an earlier substring, indicating the model
|
||||
is stuck in a generation loop.
|
||||
"""
|
||||
if len(content) < window * 2:
|
||||
return False
|
||||
|
||||
tail = content[-window:]
|
||||
body = content[:-window]
|
||||
|
||||
# Check if the tail appears verbatim in the preceding content
|
||||
if tail in body:
|
||||
return True
|
||||
|
||||
# Check character-level repetition ratio in the tail
|
||||
if not tail:
|
||||
return False
|
||||
unique_chars = len(set(tail))
|
||||
ratio = unique_chars / len(tail)
|
||||
return ratio < threshold
|
||||
|
||||
|
||||
class OllamaNativeClient:
|
||||
"""Async client for Ollama /api/chat using shared inference types.
|
||||
|
||||
Translates StructuredGenerationRequest into Ollama's native API format
|
||||
and returns InferenceResult with proper metadata and error classification.
|
||||
|
||||
Stall/loop detection is an Ollama-specific policy that monitors streaming
|
||||
responses for repetitive patterns and aborts generation when detected.
|
||||
This does not affect the InferenceResult interface — stall detection
|
||||
raises an InferenceError with category STALL_DETECTED.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
target: InferenceTarget,
|
||||
*,
|
||||
http_client: httpx.AsyncClient | None = None,
|
||||
stall_policy: StallPolicy | None = None,
|
||||
) -> None:
|
||||
if target.protocol != "ollama_native":
|
||||
raise ValueError(
|
||||
f"OllamaNativeClient requires protocol='ollama_native', got '{target.protocol}'"
|
||||
)
|
||||
self._target = target
|
||||
self._stall_policy = stall_policy or StallPolicy()
|
||||
self._owns_client = http_client is None
|
||||
self._http = http_client or httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(300.0, read=300.0),
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close the underlying HTTP client if we own it."""
|
||||
if self._owns_client:
|
||||
await self._http.aclose()
|
||||
|
||||
async def generate(self, request: StructuredGenerationRequest) -> InferenceResult:
|
||||
"""Send a structured generation request to Ollama and return the result.
|
||||
|
||||
Builds the Ollama-native payload, streams the response, applies
|
||||
stall detection, and maps results to InferenceResult.
|
||||
"""
|
||||
start = time.monotonic()
|
||||
|
||||
payload = self._build_payload(request)
|
||||
url = f"{self._target.base_url}/api/chat"
|
||||
|
||||
logger.info(
|
||||
"Ollama POST %s model=%s messages=%d max_tokens=%d",
|
||||
url,
|
||||
self._target.model,
|
||||
len(request.messages),
|
||||
request.max_output_tokens,
|
||||
)
|
||||
|
||||
try:
|
||||
content, metadata = await self._stream_response(url, payload, request.timeout_seconds)
|
||||
except InferenceError:
|
||||
raise
|
||||
except httpx.TimeoutException as exc:
|
||||
raise InferenceError(
|
||||
InferenceErrorCategory.TIMEOUT,
|
||||
f"Request timed out after {request.timeout_seconds}s",
|
||||
provider_detail=str(exc),
|
||||
) from exc
|
||||
except httpx.ConnectError as exc:
|
||||
raise InferenceError(
|
||||
InferenceErrorCategory.CONNECTION_REFUSED,
|
||||
"Connection refused",
|
||||
provider_detail=str(exc),
|
||||
) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise InferenceError(
|
||||
InferenceErrorCategory.CONNECTION_ERROR,
|
||||
"HTTP connection error",
|
||||
provider_detail=str(exc),
|
||||
) from exc
|
||||
|
||||
latency_ms = int((time.monotonic() - start) * 1000)
|
||||
|
||||
# Determine structured mode
|
||||
structured_mode = self._determine_structured_mode(request)
|
||||
|
||||
# Attempt JSON parsing if we expect structured output
|
||||
parsed = None
|
||||
if request.json_schema and content:
|
||||
try:
|
||||
parsed = json.loads(content)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
|
||||
return InferenceResult(
|
||||
content=content,
|
||||
parsed=parsed,
|
||||
endpoint_id=self._target.endpoint_id,
|
||||
deployment_id=self._target.deployment_id,
|
||||
model=metadata.get("model", self._target.model) or self._target.model,
|
||||
protocol=self._target.protocol,
|
||||
structured_mode=structured_mode,
|
||||
latency_ms=latency_ms,
|
||||
usage=TokenUsage(
|
||||
input_tokens=metadata.get("prompt_eval_count"),
|
||||
output_tokens=metadata.get("eval_count"),
|
||||
),
|
||||
request_id=request.trace_id or None,
|
||||
repaired=False,
|
||||
retries=0,
|
||||
)
|
||||
|
||||
def _build_payload(self, request: StructuredGenerationRequest) -> dict:
|
||||
"""Build the Ollama /api/chat request body."""
|
||||
messages = [
|
||||
{"role": msg.role, "content": msg.content}
|
||||
for msg in request.messages
|
||||
]
|
||||
|
||||
options: dict = {}
|
||||
|
||||
# Honor max output tokens via num_predict
|
||||
if request.max_output_tokens:
|
||||
options["num_predict"] = request.max_output_tokens
|
||||
|
||||
# Honor context window from target configuration
|
||||
if self._target.context_window and self._target.context_window > 0:
|
||||
options["num_ctx"] = self._target.context_window
|
||||
|
||||
# Temperature
|
||||
options["temperature"] = request.temperature
|
||||
|
||||
# Seed if provided
|
||||
if request.seed is not None:
|
||||
options["seed"] = request.seed
|
||||
|
||||
payload: dict = {
|
||||
"model": self._target.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"options": options,
|
||||
}
|
||||
|
||||
# Native schema formatting when supported
|
||||
if request.json_schema and self._target.capabilities.json_schema:
|
||||
payload["format"] = request.json_schema
|
||||
|
||||
# Merge any extra body params from target
|
||||
if self._target.extra_body:
|
||||
for key, value in self._target.extra_body.items():
|
||||
if key not in payload:
|
||||
payload[key] = value
|
||||
|
||||
return payload
|
||||
|
||||
def _determine_structured_mode(
|
||||
self, request: StructuredGenerationRequest
|
||||
) -> str:
|
||||
"""Determine which structured output mode was used."""
|
||||
if not request.json_schema:
|
||||
return "none"
|
||||
if self._target.capabilities.json_schema:
|
||||
return "json_schema"
|
||||
# Schema was requested but not natively supported — prompt only
|
||||
return "prompt_only"
|
||||
|
||||
async def _stream_response(
|
||||
self,
|
||||
url: str,
|
||||
payload: dict,
|
||||
timeout_seconds: float,
|
||||
) -> tuple[str, dict]:
|
||||
"""Stream Ollama response, applying stall detection.
|
||||
|
||||
Returns (content, metadata) where metadata contains token counts
|
||||
and timing from the final Ollama response chunk.
|
||||
"""
|
||||
content_parts: list[str] = []
|
||||
metadata: dict = {}
|
||||
last_content_length = 0
|
||||
unchanged_count = 0
|
||||
last_check_time = time.monotonic()
|
||||
|
||||
try:
|
||||
async with self._http.stream(
|
||||
"POST",
|
||||
url,
|
||||
json=payload,
|
||||
timeout=httpx.Timeout(timeout_seconds, read=timeout_seconds),
|
||||
) as response:
|
||||
if response.status_code != 200:
|
||||
# Read body for error detail
|
||||
body = b""
|
||||
async for chunk in response.aiter_bytes():
|
||||
body += chunk
|
||||
detail = body.decode("utf-8", errors="replace")[:500]
|
||||
category = _map_http_status_to_category(response.status_code)
|
||||
raise InferenceError(
|
||||
category,
|
||||
f"Ollama returned HTTP {response.status_code}",
|
||||
provider_detail=detail,
|
||||
status_code=response.status_code,
|
||||
)
|
||||
|
||||
async for line in response.aiter_lines():
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
try:
|
||||
chunk_data = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
# Extract content from message
|
||||
message = chunk_data.get("message", {})
|
||||
chunk_content = message.get("content", "")
|
||||
if chunk_content:
|
||||
content_parts.append(chunk_content)
|
||||
|
||||
# Check for completion
|
||||
if chunk_data.get("done", False):
|
||||
# Capture metadata from final chunk
|
||||
metadata["model"] = chunk_data.get("model", "")
|
||||
metadata["prompt_eval_count"] = chunk_data.get("prompt_eval_count")
|
||||
metadata["eval_count"] = chunk_data.get("eval_count")
|
||||
total_ns = chunk_data.get("total_duration")
|
||||
if total_ns:
|
||||
metadata["total_duration_ms"] = total_ns // 1_000_000
|
||||
break
|
||||
|
||||
# Stall detection
|
||||
if self._stall_policy.enabled:
|
||||
now = time.monotonic()
|
||||
if now - last_check_time >= self._stall_policy.check_interval_seconds:
|
||||
current_content = "".join(content_parts)
|
||||
current_length = len(current_content)
|
||||
|
||||
if current_length == last_content_length:
|
||||
unchanged_count += 1
|
||||
else:
|
||||
# Check for loop pattern
|
||||
if _detect_loop(
|
||||
current_content,
|
||||
self._stall_policy.loop_window,
|
||||
self._stall_policy.loop_threshold,
|
||||
):
|
||||
unchanged_count += 1
|
||||
else:
|
||||
unchanged_count = 0
|
||||
|
||||
last_content_length = current_length
|
||||
last_check_time = now
|
||||
|
||||
if unchanged_count >= self._stall_policy.max_unchanged_intervals:
|
||||
logger.warning(
|
||||
"Stall detected after %d unchanged intervals, aborting",
|
||||
unchanged_count,
|
||||
)
|
||||
raise InferenceError(
|
||||
InferenceErrorCategory.STALL_DETECTED,
|
||||
f"Generation stalled after {unchanged_count} check intervals",
|
||||
)
|
||||
|
||||
except InferenceError:
|
||||
raise
|
||||
except httpx.TimeoutException as exc:
|
||||
raise InferenceError(
|
||||
InferenceErrorCategory.TIMEOUT,
|
||||
"Stream read timed out",
|
||||
provider_detail=str(exc),
|
||||
) from exc
|
||||
except httpx.ConnectError as exc:
|
||||
raise InferenceError(
|
||||
InferenceErrorCategory.CONNECTION_REFUSED,
|
||||
"Connection refused during streaming",
|
||||
provider_detail=str(exc),
|
||||
) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise InferenceError(
|
||||
InferenceErrorCategory.CONNECTION_ERROR,
|
||||
"HTTP error during streaming",
|
||||
provider_detail=str(exc),
|
||||
) from exc
|
||||
|
||||
final_content = "".join(content_parts)
|
||||
|
||||
if not final_content:
|
||||
raise InferenceError(
|
||||
InferenceErrorCategory.EMPTY_RESPONSE,
|
||||
"Ollama returned empty content",
|
||||
)
|
||||
|
||||
return final_content, metadata
|
||||
@@ -0,0 +1,431 @@
|
||||
"""OpenAI-compatible inference client.
|
||||
|
||||
Uses httpx.AsyncClient directly (NOT the openai SDK). This keeps wire payloads
|
||||
explicit, permits provider-specific extra_body, and simplifies redacted request
|
||||
auditing.
|
||||
|
||||
Requirements: 2.1, 2.2, 2.3, 2.4, 2.5, 2.7, 2.9
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from services.shared.inference.models import (
|
||||
ErrorCategory,
|
||||
InferenceResult,
|
||||
InferenceTarget,
|
||||
StructuredGenerationRequest,
|
||||
TokenUsage,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Status codes that trigger retry
|
||||
_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
|
||||
|
||||
# Sensitive keys that must never appear in logs
|
||||
_SENSITIVE_KEYS = frozenset({"authorization", "x-api-key", "api-key"})
|
||||
|
||||
|
||||
def _resolve_auth_secret(secret_ref: str | None) -> str | None:
|
||||
"""Resolve an authentication secret from environment variables.
|
||||
|
||||
The secret_ref is treated as an environment variable name.
|
||||
Returns None if the ref is None or the env var is not set.
|
||||
"""
|
||||
if not secret_ref:
|
||||
return None
|
||||
return os.environ.get(secret_ref)
|
||||
|
||||
|
||||
def _redact_headers(headers: dict[str, str]) -> dict[str, str]:
|
||||
"""Return a copy of headers with sensitive values redacted."""
|
||||
redacted = {}
|
||||
for key, value in headers.items():
|
||||
if key.lower() in _SENSITIVE_KEYS:
|
||||
redacted[key] = "***REDACTED***"
|
||||
else:
|
||||
redacted[key] = value
|
||||
return redacted
|
||||
|
||||
|
||||
class OpenAICompatibleClient:
|
||||
"""Client for OpenAI-compatible /v1/chat/completions endpoints.
|
||||
|
||||
Supports:
|
||||
- Bearer and configurable authentication via runtime secret resolution
|
||||
- Standard response_format.json_schema payloads
|
||||
- Configurable vLLM structured_outputs extra-body payloads
|
||||
- Explicit JSON-object and prompt-only fallback policies
|
||||
- Retry on transient errors (5xx, 429, timeout)
|
||||
- Local JSON schema revalidation
|
||||
- Metadata capture (request_id, usage, finish_reason, retries)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
target: InferenceTarget,
|
||||
*,
|
||||
http_client: httpx.AsyncClient | None = None,
|
||||
) -> None:
|
||||
self._target = target
|
||||
self._owns_client = http_client is None
|
||||
self._http = http_client or httpx.AsyncClient(timeout=target.timeout_seconds)
|
||||
|
||||
@property
|
||||
def target(self) -> InferenceTarget:
|
||||
"""The resolved inference target."""
|
||||
return self._target
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
request: StructuredGenerationRequest,
|
||||
) -> InferenceResult:
|
||||
"""Send a structured generation request and return the result.
|
||||
|
||||
Chooses the strongest structured-output mode the target supports:
|
||||
1. json_schema — sends the actual schema with strict mode
|
||||
2. json_object — only if target explicitly declares json_object capability
|
||||
3. prompt_only — fallback when no structured output is available
|
||||
"""
|
||||
structured_mode = self._choose_structured_mode(request)
|
||||
headers = self._build_headers()
|
||||
body = self._build_body(request, structured_mode)
|
||||
url = f"{self._target.base_url.rstrip('/')}/v1/chat/completions"
|
||||
|
||||
retries = 0
|
||||
max_retries = self._target.max_retries
|
||||
start_time = time.monotonic()
|
||||
last_error: str | None = None
|
||||
last_error_category: str | None = None
|
||||
|
||||
while True:
|
||||
try:
|
||||
response = await self._http.post(
|
||||
url,
|
||||
json=body,
|
||||
headers=headers,
|
||||
timeout=request.timeout_seconds,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
last_error = "Request timed out"
|
||||
last_error_category = ErrorCategory.TIMEOUT
|
||||
if retries < max_retries:
|
||||
retries += 1
|
||||
logger.warning(
|
||||
"Timeout on attempt %d/%d to %s",
|
||||
retries,
|
||||
max_retries,
|
||||
url,
|
||||
)
|
||||
continue
|
||||
return self._error_result(
|
||||
last_error,
|
||||
last_error_category,
|
||||
retries,
|
||||
start_time,
|
||||
structured_mode,
|
||||
)
|
||||
except httpx.ConnectError as exc:
|
||||
last_error = f"Connection error: {exc}"
|
||||
last_error_category = ErrorCategory.CONNECTION_ERROR
|
||||
if retries < max_retries:
|
||||
retries += 1
|
||||
logger.warning(
|
||||
"Connection error on attempt %d/%d to %s: %s",
|
||||
retries,
|
||||
max_retries,
|
||||
url,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
return self._error_result(
|
||||
last_error,
|
||||
last_error_category,
|
||||
retries,
|
||||
start_time,
|
||||
structured_mode,
|
||||
)
|
||||
|
||||
# Handle retryable HTTP status codes
|
||||
if response.status_code in _RETRYABLE_STATUS_CODES:
|
||||
if response.status_code == 429:
|
||||
last_error_category = ErrorCategory.RATE_LIMIT
|
||||
last_error = "Rate limited (429)"
|
||||
else:
|
||||
last_error_category = ErrorCategory.SERVER_ERROR
|
||||
last_error = f"Server error ({response.status_code})"
|
||||
|
||||
if retries < max_retries:
|
||||
retries += 1
|
||||
logger.warning(
|
||||
"HTTP %d on attempt %d/%d to %s",
|
||||
response.status_code,
|
||||
retries,
|
||||
max_retries,
|
||||
url,
|
||||
)
|
||||
continue
|
||||
return self._error_result(
|
||||
last_error,
|
||||
last_error_category,
|
||||
retries,
|
||||
start_time,
|
||||
structured_mode,
|
||||
)
|
||||
|
||||
# Handle non-retryable errors
|
||||
if response.status_code == 401 or response.status_code == 403:
|
||||
return self._error_result(
|
||||
f"Authentication failed ({response.status_code})",
|
||||
ErrorCategory.AUTHENTICATION,
|
||||
retries,
|
||||
start_time,
|
||||
structured_mode,
|
||||
)
|
||||
|
||||
if response.status_code >= 400:
|
||||
return self._error_result(
|
||||
f"Client error ({response.status_code})",
|
||||
ErrorCategory.INVALID_RESPONSE,
|
||||
retries,
|
||||
start_time,
|
||||
structured_mode,
|
||||
)
|
||||
|
||||
# Success path
|
||||
break
|
||||
|
||||
latency_ms = int((time.monotonic() - start_time) * 1000)
|
||||
return self._parse_response(
|
||||
response,
|
||||
request,
|
||||
structured_mode,
|
||||
retries,
|
||||
latency_ms,
|
||||
)
|
||||
|
||||
def _choose_structured_mode(
|
||||
self, request: StructuredGenerationRequest
|
||||
) -> str:
|
||||
"""Choose the strongest structured-output mode available."""
|
||||
caps = self._target.capabilities
|
||||
|
||||
if request.json_schema and caps.json_schema:
|
||||
return "json_schema"
|
||||
if request.json_schema and caps.json_object:
|
||||
return "json_object"
|
||||
if request.json_schema:
|
||||
# Schema requested but endpoint supports neither — prompt-only fallback
|
||||
return "prompt_only"
|
||||
return "none"
|
||||
|
||||
def _build_headers(self) -> dict[str, str]:
|
||||
"""Build request headers including authentication."""
|
||||
headers: dict[str, str] = {
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# Add extra headers from target config
|
||||
headers.update(self._target.extra_headers)
|
||||
|
||||
# Resolve auth secret
|
||||
secret = _resolve_auth_secret(self._target.auth_secret_ref)
|
||||
if secret:
|
||||
scheme = self._target.auth_scheme.lower()
|
||||
if scheme == "bearer":
|
||||
headers["Authorization"] = f"Bearer {secret}"
|
||||
else:
|
||||
# Custom auth scheme (e.g., "X-API-Key: <value>")
|
||||
headers[self._target.auth_scheme] = secret
|
||||
|
||||
return headers
|
||||
|
||||
def _build_body(
|
||||
self,
|
||||
request: StructuredGenerationRequest,
|
||||
structured_mode: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the request body for /v1/chat/completions."""
|
||||
messages = [
|
||||
{"role": msg.role, "content": msg.content}
|
||||
for msg in request.messages
|
||||
]
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"model": self._target.model,
|
||||
"messages": messages,
|
||||
"temperature": request.temperature,
|
||||
"max_tokens": request.max_output_tokens,
|
||||
}
|
||||
|
||||
# Add seed if the target supports it
|
||||
if self._target.capabilities.seed and request.seed is not None:
|
||||
body["seed"] = request.seed
|
||||
|
||||
# Add response_format based on chosen mode
|
||||
if structured_mode == "json_schema" and request.json_schema:
|
||||
body["response_format"] = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": request.json_schema.get("title", "response"),
|
||||
"strict": True,
|
||||
"schema": request.json_schema,
|
||||
},
|
||||
}
|
||||
elif structured_mode == "json_object":
|
||||
body["response_format"] = {"type": "json_object"}
|
||||
|
||||
# Add extra_body from target config (vLLM structured_outputs, etc.)
|
||||
if self._target.extra_body:
|
||||
body.update(self._target.extra_body)
|
||||
|
||||
return body
|
||||
|
||||
def _parse_response(
|
||||
self,
|
||||
response: httpx.Response,
|
||||
request: StructuredGenerationRequest,
|
||||
structured_mode: str,
|
||||
retries: int,
|
||||
latency_ms: int,
|
||||
) -> InferenceResult:
|
||||
"""Parse a successful response into InferenceResult."""
|
||||
# Extract request ID from response headers
|
||||
request_id = response.headers.get("x-request-id")
|
||||
|
||||
try:
|
||||
data = response.json()
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return InferenceResult(
|
||||
content="",
|
||||
endpoint_id=self._target.endpoint_id,
|
||||
deployment_id=self._target.deployment_id,
|
||||
model=self._target.model,
|
||||
protocol=self._target.protocol,
|
||||
structured_mode=structured_mode, # type: ignore[arg-type]
|
||||
latency_ms=latency_ms,
|
||||
retries=retries,
|
||||
request_id=request_id,
|
||||
error="Invalid JSON in response body",
|
||||
error_category=ErrorCategory.INVALID_RESPONSE,
|
||||
)
|
||||
|
||||
# Extract content from choices
|
||||
choices = data.get("choices", [])
|
||||
if not choices:
|
||||
return InferenceResult(
|
||||
content="",
|
||||
endpoint_id=self._target.endpoint_id,
|
||||
deployment_id=self._target.deployment_id,
|
||||
model=self._target.model,
|
||||
protocol=self._target.protocol,
|
||||
structured_mode=structured_mode, # type: ignore[arg-type]
|
||||
latency_ms=latency_ms,
|
||||
retries=retries,
|
||||
request_id=request_id,
|
||||
error="Empty choices in response",
|
||||
error_category=ErrorCategory.INVALID_RESPONSE,
|
||||
)
|
||||
|
||||
message = choices[0].get("message", {})
|
||||
content = message.get("content", "")
|
||||
finish_reason = choices[0].get("finish_reason")
|
||||
|
||||
# Extract usage metadata
|
||||
usage_data = data.get("usage", {})
|
||||
input_tokens = usage_data.get("prompt_tokens")
|
||||
output_tokens = usage_data.get("completion_tokens")
|
||||
total_tokens = usage_data.get("total_tokens")
|
||||
|
||||
# Parse JSON content if structured output was requested
|
||||
parsed: dict[str, Any] | None = None
|
||||
schema_valid: bool | None = None
|
||||
|
||||
if structured_mode in ("json_schema", "json_object") and content:
|
||||
try:
|
||||
parsed = json.loads(content)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
# Content is not valid JSON
|
||||
schema_valid = False
|
||||
|
||||
# Validate parsed JSON against supplied schema
|
||||
if parsed is not None and request.json_schema:
|
||||
schema_valid = self._validate_schema(parsed, request.json_schema)
|
||||
|
||||
return InferenceResult(
|
||||
content=content,
|
||||
parsed=parsed,
|
||||
endpoint_id=self._target.endpoint_id,
|
||||
deployment_id=self._target.deployment_id,
|
||||
model=self._target.model,
|
||||
protocol=self._target.protocol,
|
||||
structured_mode=structured_mode, # type: ignore[arg-type]
|
||||
latency_ms=latency_ms,
|
||||
usage=TokenUsage(
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=total_tokens,
|
||||
),
|
||||
request_id=request_id,
|
||||
finish_reason=finish_reason,
|
||||
retries=retries,
|
||||
schema_valid=schema_valid,
|
||||
)
|
||||
|
||||
def _validate_schema(
|
||||
self, data: dict[str, Any], schema: dict[str, Any]
|
||||
) -> bool:
|
||||
"""Revalidate parsed JSON locally against the supplied schema.
|
||||
|
||||
Returns True if valid, False otherwise.
|
||||
"""
|
||||
try:
|
||||
import jsonschema
|
||||
|
||||
jsonschema.validate(instance=data, schema=schema)
|
||||
return True
|
||||
except Exception:
|
||||
logger.debug("Schema validation failed for response data")
|
||||
return False
|
||||
|
||||
def _error_result(
|
||||
self,
|
||||
error: str,
|
||||
error_category: str,
|
||||
retries: int,
|
||||
start_time: float,
|
||||
structured_mode: str,
|
||||
) -> InferenceResult:
|
||||
"""Build an error InferenceResult."""
|
||||
latency_ms = int((time.monotonic() - start_time) * 1000)
|
||||
return InferenceResult(
|
||||
content="",
|
||||
endpoint_id=self._target.endpoint_id,
|
||||
deployment_id=self._target.deployment_id,
|
||||
model=self._target.model,
|
||||
protocol=self._target.protocol,
|
||||
structured_mode=structured_mode, # type: ignore[arg-type]
|
||||
latency_ms=latency_ms,
|
||||
retries=retries,
|
||||
error=error,
|
||||
error_category=error_category,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close the underlying HTTP client if we own it."""
|
||||
if self._owns_client:
|
||||
await self._http.aclose()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"OpenAICompatibleClient("
|
||||
f"model={self._target.model!r}, "
|
||||
f"base_url={self._target.base_url!r})"
|
||||
)
|
||||
Reference in New Issue
Block a user