feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,8 @@
|
||||
"""Inference Registry API service.
|
||||
|
||||
FastAPI router for managing inference_endpoints, model_deployments,
|
||||
and agent_stage_bindings. Auth secret values are NEVER returned in
|
||||
responses.
|
||||
|
||||
Requirements: 3.6, 3.7
|
||||
"""
|
||||
@@ -0,0 +1,634 @@
|
||||
"""FastAPI router for the inference registry API.
|
||||
|
||||
Manages inference_endpoints, model_deployments, and agent_stage_bindings.
|
||||
Auth secret values are NEVER returned in any response.
|
||||
|
||||
Endpoints:
|
||||
- CRUD for inference_endpoints (19.1)
|
||||
- probe, enable, disable, test-structured-output actions (19.2)
|
||||
- Protocol/endpoint/deployment selectors (19.3)
|
||||
- Display last probe, capabilities, limits, bindings (19.4)
|
||||
- External egress confirmation (19.5)
|
||||
|
||||
Requirements: 3.6, 3.7
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from services.inference_registry.schemas import (
|
||||
BindingCreate,
|
||||
BindingResponse,
|
||||
DeploymentCreate,
|
||||
DeploymentResponse,
|
||||
EgressConfirmation,
|
||||
EndpointCreate,
|
||||
EndpointListResponse,
|
||||
EndpointResponse,
|
||||
EndpointUpdate,
|
||||
ProbeResponse,
|
||||
StructuredOutputTestRequest,
|
||||
StructuredOutputTestResponse,
|
||||
)
|
||||
from services.inference_registry.security import (
|
||||
is_external_endpoint,
|
||||
redact_binding,
|
||||
redact_deployment,
|
||||
redact_endpoint,
|
||||
redact_endpoint_for_list,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/inference", tags=["inference-registry"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dependency injection protocol for database access
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class InferenceRegistryDB:
|
||||
"""Protocol for inference registry database operations.
|
||||
|
||||
In production, backed by asyncpg pool. In tests, a mock implements this.
|
||||
"""
|
||||
|
||||
async def list_endpoints(self) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_endpoint(self, endpoint_id: uuid.UUID) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def create_endpoint(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def update_endpoint(
|
||||
self, endpoint_id: uuid.UUID, data: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def disable_endpoint(self, endpoint_id: uuid.UUID) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_deployments(
|
||||
self, endpoint_id: uuid.UUID | None = None
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_deployment(self, deployment_id: uuid.UUID) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def create_deployment(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_bindings(
|
||||
self, agent_id: uuid.UUID | None = None,
|
||||
endpoint_id: uuid.UUID | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def create_binding(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_bindings_for_endpoint(
|
||||
self, endpoint_id: uuid.UUID
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_last_probe(
|
||||
self, endpoint_id: uuid.UUID
|
||||
) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def store_probe_result(
|
||||
self, endpoint_id: uuid.UUID, result: dict[str, Any]
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_egress_confirmation(
|
||||
self, endpoint_id: uuid.UUID
|
||||
) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
async def store_egress_confirmation(
|
||||
self, endpoint_id: uuid.UUID
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
# Global DB instance (set during app startup)
|
||||
_db: InferenceRegistryDB | None = None
|
||||
|
||||
|
||||
def set_db(db: InferenceRegistryDB) -> None:
|
||||
"""Set the database dependency for the router."""
|
||||
global _db
|
||||
_db = db
|
||||
|
||||
|
||||
def get_db() -> InferenceRegistryDB:
|
||||
"""Get the database dependency."""
|
||||
if _db is None:
|
||||
raise HTTPException(503, "Database not initialized")
|
||||
return _db
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoint CRUD (19.1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/endpoints", response_model=list[EndpointListResponse])
|
||||
async def list_endpoints(db: InferenceRegistryDB = Depends(get_db)):
|
||||
"""List all inference endpoints with secrets redacted."""
|
||||
endpoints = await db.list_endpoints()
|
||||
return [redact_endpoint_for_list(ep) for ep in endpoints]
|
||||
|
||||
|
||||
@router.get("/endpoints/{endpoint_id}", response_model=EndpointResponse)
|
||||
async def get_endpoint(
|
||||
endpoint_id: uuid.UUID,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Get endpoint detail with secrets redacted.
|
||||
|
||||
Includes last probe results, capabilities, and active stage bindings (19.4).
|
||||
"""
|
||||
endpoint = await db.get_endpoint(endpoint_id)
|
||||
if endpoint is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
|
||||
response = redact_endpoint(endpoint)
|
||||
|
||||
# Attach last probe result (19.4)
|
||||
probe_data = await db.get_last_probe(endpoint_id)
|
||||
if probe_data:
|
||||
response.last_probe = ProbeResponse(**probe_data)
|
||||
|
||||
# Attach capabilities from deployments
|
||||
deployments = await db.list_deployments(endpoint_id=endpoint_id)
|
||||
if deployments:
|
||||
# Aggregate capabilities from all deployments
|
||||
combined_caps: dict[str, Any] = {}
|
||||
for dep in deployments:
|
||||
caps = dep.get("capabilities", {})
|
||||
for k, v in caps.items():
|
||||
if v:
|
||||
combined_caps[k] = True
|
||||
response.capabilities = combined_caps
|
||||
|
||||
# Attach active bindings (19.4)
|
||||
bindings = await db.get_bindings_for_endpoint(endpoint_id)
|
||||
if bindings:
|
||||
response.active_bindings = [redact_binding(b) for b in bindings]
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@router.post("/endpoints", response_model=EndpointResponse, status_code=201)
|
||||
async def create_endpoint(
|
||||
body: EndpointCreate,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Create a new inference endpoint.
|
||||
|
||||
Validates protocol and URL format. External endpoints require
|
||||
egress confirmation before they can be enabled (19.5).
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
endpoint_data = {
|
||||
"id": uuid.uuid4(),
|
||||
"name": body.name,
|
||||
"protocol": body.protocol,
|
||||
"base_url": body.base_url,
|
||||
"auth_secret_ref": body.auth_secret_ref,
|
||||
"auth_scheme": body.auth_scheme,
|
||||
"default_headers": body.default_headers,
|
||||
"health_path": body.health_path,
|
||||
"enabled": body.enabled,
|
||||
"revision": 1,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
|
||||
# If external, require egress confirmation before enabling (19.5)
|
||||
if body.enabled and is_external_endpoint(body.base_url):
|
||||
endpoint_data["enabled"] = False # Will need confirm-egress call
|
||||
|
||||
created = await db.create_endpoint(endpoint_data)
|
||||
return redact_endpoint(created)
|
||||
|
||||
|
||||
@router.put("/endpoints/{endpoint_id}", response_model=EndpointResponse)
|
||||
async def update_endpoint(
|
||||
endpoint_id: uuid.UUID,
|
||||
body: EndpointUpdate,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Update an existing inference endpoint."""
|
||||
existing = await db.get_endpoint(endpoint_id)
|
||||
if existing is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
|
||||
update_data: dict[str, Any] = {}
|
||||
for field_name in (
|
||||
"name", "protocol", "base_url", "auth_secret_ref",
|
||||
"auth_scheme", "default_headers", "health_path", "enabled",
|
||||
):
|
||||
value = getattr(body, field_name)
|
||||
if value is not None:
|
||||
update_data[field_name] = value
|
||||
|
||||
if update_data:
|
||||
update_data["updated_at"] = datetime.now(timezone.utc)
|
||||
update_data["revision"] = existing.get("revision", 1) + 1
|
||||
|
||||
updated = await db.update_endpoint(endpoint_id, update_data)
|
||||
if updated is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
return redact_endpoint(updated)
|
||||
|
||||
|
||||
@router.delete("/endpoints/{endpoint_id}", response_model=EndpointResponse)
|
||||
async def delete_endpoint(
|
||||
endpoint_id: uuid.UUID,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Soft-delete (disable) an inference endpoint."""
|
||||
disabled = await db.disable_endpoint(endpoint_id)
|
||||
if disabled is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
return redact_endpoint(disabled)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoint actions (19.2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/endpoints/{endpoint_id}/probe", response_model=ProbeResponse)
|
||||
async def probe_endpoint(
|
||||
endpoint_id: uuid.UUID,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Run a capability probe against the endpoint.
|
||||
|
||||
Performs health check, model listing, JSON Schema test,
|
||||
usage metadata check, seed determinism check, and output-token field check.
|
||||
"""
|
||||
endpoint = await db.get_endpoint(endpoint_id)
|
||||
if endpoint is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
|
||||
# Import the prober
|
||||
from services.shared.inference.capabilities import EndpointProber, FullProbeResult
|
||||
from services.shared.inference.models import InferenceTarget, ProviderCapabilities
|
||||
|
||||
# Get a deployment to probe with (need a model name)
|
||||
deployments = await db.list_deployments(endpoint_id=endpoint_id)
|
||||
model_name = "default"
|
||||
caps_data: dict[str, Any] = {}
|
||||
deployment_id = uuid.uuid4()
|
||||
|
||||
if deployments:
|
||||
model_name = deployments[0].get("served_model_name", "default")
|
||||
caps_data = deployments[0].get("capabilities", {})
|
||||
deployment_id = deployments[0]["id"]
|
||||
|
||||
capabilities = ProviderCapabilities(
|
||||
chat_completions=caps_data.get("chat_completions", True),
|
||||
responses_api=caps_data.get("responses_api", False),
|
||||
json_schema=caps_data.get("json_schema", False),
|
||||
json_object=caps_data.get("json_object", False),
|
||||
seed=caps_data.get("seed", False),
|
||||
usage=caps_data.get("usage", False),
|
||||
max_completion_tokens=caps_data.get("max_completion_tokens", False),
|
||||
reasoning_toggle=caps_data.get("reasoning_toggle", False),
|
||||
model_listing=caps_data.get("model_listing", False),
|
||||
)
|
||||
|
||||
target = InferenceTarget(
|
||||
endpoint_id=endpoint_id,
|
||||
deployment_id=deployment_id,
|
||||
protocol=endpoint["protocol"],
|
||||
base_url=endpoint["base_url"],
|
||||
model=model_name,
|
||||
capabilities=capabilities,
|
||||
auth_secret_ref=endpoint.get("auth_secret_ref"),
|
||||
auth_scheme=endpoint.get("auth_scheme", "bearer"),
|
||||
extra_headers=endpoint.get("default_headers") or {},
|
||||
)
|
||||
|
||||
prober = EndpointProber()
|
||||
try:
|
||||
result: FullProbeResult = await prober.run_full_probe(target)
|
||||
finally:
|
||||
await prober.close()
|
||||
|
||||
# Build probe response
|
||||
probe_response = ProbeResponse(
|
||||
endpoint_id=endpoint_id,
|
||||
timestamp=result.timestamp,
|
||||
software_version=result.software_version,
|
||||
probe_duration_ms=result.probe_duration_ms,
|
||||
health_success=result.health.success if result.health else False,
|
||||
health_detail=result.health.detail if result.health else "",
|
||||
model_listing_success=(
|
||||
result.model_listing.success if result.model_listing else None
|
||||
),
|
||||
json_schema_success=(
|
||||
result.json_schema.success if result.json_schema else None
|
||||
),
|
||||
usage_success=(
|
||||
result.usage_metadata.success if result.usage_metadata else None
|
||||
),
|
||||
seed_success=(
|
||||
result.seed_determinism.success if result.seed_determinism else None
|
||||
),
|
||||
output_token_field_success=(
|
||||
result.output_token_field.success if result.output_token_field else None
|
||||
),
|
||||
)
|
||||
|
||||
# Store probe result
|
||||
await db.store_probe_result(endpoint_id, probe_response.model_dump())
|
||||
|
||||
return probe_response
|
||||
|
||||
|
||||
@router.post("/endpoints/{endpoint_id}/enable", response_model=EndpointResponse)
|
||||
async def enable_endpoint(
|
||||
endpoint_id: uuid.UUID,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Enable an inference endpoint.
|
||||
|
||||
External endpoints require egress confirmation first (19.5).
|
||||
"""
|
||||
endpoint = await db.get_endpoint(endpoint_id)
|
||||
if endpoint is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
|
||||
# Check if external endpoint needs egress confirmation (19.5)
|
||||
if is_external_endpoint(endpoint["base_url"]):
|
||||
has_confirmation = await db.get_egress_confirmation(endpoint_id)
|
||||
if not has_confirmation:
|
||||
raise HTTPException(
|
||||
403,
|
||||
"External endpoint requires egress confirmation. "
|
||||
"POST /api/inference/endpoints/{id}/confirm-egress first.",
|
||||
)
|
||||
|
||||
update_data = {
|
||||
"enabled": True,
|
||||
"updated_at": datetime.now(timezone.utc),
|
||||
"revision": endpoint.get("revision", 1) + 1,
|
||||
}
|
||||
updated = await db.update_endpoint(endpoint_id, update_data)
|
||||
if updated is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
return redact_endpoint(updated)
|
||||
|
||||
|
||||
@router.post("/endpoints/{endpoint_id}/disable", response_model=EndpointResponse)
|
||||
async def disable_endpoint_action(
|
||||
endpoint_id: uuid.UUID,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Disable an inference endpoint."""
|
||||
endpoint = await db.get_endpoint(endpoint_id)
|
||||
if endpoint is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
|
||||
update_data = {
|
||||
"enabled": False,
|
||||
"updated_at": datetime.now(timezone.utc),
|
||||
"revision": endpoint.get("revision", 1) + 1,
|
||||
}
|
||||
updated = await db.update_endpoint(endpoint_id, update_data)
|
||||
if updated is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
return redact_endpoint(updated)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/endpoints/{endpoint_id}/test-structured-output",
|
||||
response_model=StructuredOutputTestResponse,
|
||||
)
|
||||
async def test_structured_output(
|
||||
endpoint_id: uuid.UUID,
|
||||
body: StructuredOutputTestRequest | None = None,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Test JSON Schema structured output on an endpoint.
|
||||
|
||||
Sends a minimal schema-constrained request and validates the response.
|
||||
"""
|
||||
if body is None:
|
||||
body = StructuredOutputTestRequest()
|
||||
|
||||
endpoint = await db.get_endpoint(endpoint_id)
|
||||
if endpoint is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
|
||||
from services.shared.inference.capabilities import EndpointProber
|
||||
from services.shared.inference.models import InferenceTarget, ProviderCapabilities
|
||||
|
||||
# Get a deployment to test with
|
||||
deployments = await db.list_deployments(endpoint_id=endpoint_id)
|
||||
model_name = "default"
|
||||
deployment_id = uuid.uuid4()
|
||||
if deployments:
|
||||
model_name = deployments[0].get("served_model_name", "default")
|
||||
deployment_id = deployments[0]["id"]
|
||||
|
||||
target = InferenceTarget(
|
||||
endpoint_id=endpoint_id,
|
||||
deployment_id=deployment_id,
|
||||
protocol=endpoint["protocol"],
|
||||
base_url=endpoint["base_url"],
|
||||
model=model_name,
|
||||
capabilities=ProviderCapabilities(
|
||||
chat_completions=True,
|
||||
json_schema=True,
|
||||
),
|
||||
auth_secret_ref=endpoint.get("auth_secret_ref"),
|
||||
auth_scheme=endpoint.get("auth_scheme", "bearer"),
|
||||
extra_headers=endpoint.get("default_headers") or {},
|
||||
)
|
||||
|
||||
prober = EndpointProber()
|
||||
try:
|
||||
result = await prober.probe_json_schema(target)
|
||||
finally:
|
||||
await prober.close()
|
||||
|
||||
return StructuredOutputTestResponse(
|
||||
success=result.success,
|
||||
structured_mode=result.structured_mode_used,
|
||||
content=result.detail,
|
||||
schema_valid=result.schema_valid,
|
||||
error=None if result.success else result.detail,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# External egress confirmation (19.5)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/endpoints/{endpoint_id}/confirm-egress", response_model=EndpointResponse)
|
||||
async def confirm_egress(
|
||||
endpoint_id: uuid.UUID,
|
||||
body: EgressConfirmation,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Confirm external endpoint egress enablement.
|
||||
|
||||
Required before enabling an endpoint with a non-cluster URL.
|
||||
The request body must contain {"confirmed": true}.
|
||||
"""
|
||||
endpoint = await db.get_endpoint(endpoint_id)
|
||||
if endpoint is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
|
||||
if not is_external_endpoint(endpoint["base_url"]):
|
||||
raise HTTPException(400, "Endpoint is not external; no egress confirmation needed")
|
||||
|
||||
# body.confirmed is already validated by pydantic to be True
|
||||
await db.store_egress_confirmation(endpoint_id)
|
||||
|
||||
# Now enable the endpoint
|
||||
update_data = {
|
||||
"enabled": True,
|
||||
"updated_at": datetime.now(timezone.utc),
|
||||
"revision": endpoint.get("revision", 1) + 1,
|
||||
}
|
||||
updated = await db.update_endpoint(endpoint_id, update_data)
|
||||
if updated is None:
|
||||
raise HTTPException(404, "Endpoint not found")
|
||||
return redact_endpoint(updated)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deployments (19.3, 19.4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/deployments", response_model=list[DeploymentResponse])
|
||||
async def list_deployments(
|
||||
endpoint_id: uuid.UUID | None = None,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""List model deployments, optionally filtered by endpoint."""
|
||||
deployments = await db.list_deployments(endpoint_id=endpoint_id)
|
||||
return [redact_deployment(d) for d in deployments]
|
||||
|
||||
|
||||
@router.get("/deployments/{deployment_id}", response_model=DeploymentResponse)
|
||||
async def get_deployment(
|
||||
deployment_id: uuid.UUID,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Get a model deployment with capabilities and limits (19.4)."""
|
||||
deployment = await db.get_deployment(deployment_id)
|
||||
if deployment is None:
|
||||
raise HTTPException(404, "Deployment not found")
|
||||
return redact_deployment(deployment)
|
||||
|
||||
|
||||
@router.post("/deployments", response_model=DeploymentResponse, status_code=201)
|
||||
async def create_deployment(
|
||||
body: DeploymentCreate,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Create a new model deployment."""
|
||||
# Verify endpoint exists
|
||||
endpoint = await db.get_endpoint(body.endpoint_id)
|
||||
if endpoint is None:
|
||||
raise HTTPException(404, "Referenced endpoint not found")
|
||||
|
||||
deployment_data = {
|
||||
"id": uuid.uuid4(),
|
||||
"endpoint_id": body.endpoint_id,
|
||||
"served_model_name": body.served_model_name,
|
||||
"display_name": body.display_name,
|
||||
"capabilities": body.capabilities,
|
||||
"context_window": body.context_window,
|
||||
"max_output_tokens": body.max_output_tokens,
|
||||
"quantization": body.quantization,
|
||||
"runtime_metadata": body.runtime_metadata,
|
||||
"enabled": body.enabled,
|
||||
"revision": 1,
|
||||
}
|
||||
|
||||
created = await db.create_deployment(deployment_data)
|
||||
return redact_deployment(created)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bindings (19.3, 19.4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/bindings", response_model=list[BindingResponse])
|
||||
async def list_bindings(
|
||||
agent_id: uuid.UUID | None = None,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""List agent stage bindings."""
|
||||
bindings = await db.list_bindings(agent_id=agent_id)
|
||||
return [redact_binding(b) for b in bindings]
|
||||
|
||||
|
||||
@router.post("/bindings", response_model=BindingResponse, status_code=201)
|
||||
async def create_binding(
|
||||
body: BindingCreate,
|
||||
db: InferenceRegistryDB = Depends(get_db),
|
||||
):
|
||||
"""Create an agent stage binding."""
|
||||
# Verify deployment exists if provided
|
||||
if body.model_deployment_id:
|
||||
deployment = await db.get_deployment(body.model_deployment_id)
|
||||
if deployment is None:
|
||||
raise HTTPException(404, "Referenced deployment not found")
|
||||
|
||||
binding_data = {
|
||||
"id": uuid.uuid4(),
|
||||
"agent_id": body.agent_id,
|
||||
"stage": body.stage,
|
||||
"model_deployment_id": body.model_deployment_id,
|
||||
"route_order": body.route_order,
|
||||
"routing_config": body.routing_config,
|
||||
"is_active": body.is_active,
|
||||
"revision": 1,
|
||||
}
|
||||
|
||||
created = await db.create_binding(binding_data)
|
||||
return redact_binding(created)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Selectors (19.3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/protocols")
|
||||
async def list_protocols():
|
||||
"""Return available protocol options for endpoint creation.
|
||||
|
||||
Replaces free-text provider inputs with controlled selectors.
|
||||
"""
|
||||
return {
|
||||
"protocols": [
|
||||
{"value": "ollama_native", "label": "Ollama Native", "description": "Ollama /api/chat endpoint"},
|
||||
{"value": "openai_chat", "label": "OpenAI Compatible", "description": "OpenAI /v1/chat/completions (vLLM, OpenAI, LM Studio, SGLang)"},
|
||||
{"value": "specialist_http", "label": "Specialist HTTP", "description": "Typed non-generative endpoints (GLiNER, FinBERT)"},
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
"""Pydantic request/response models for the inference registry API.
|
||||
|
||||
All response models EXCLUDE actual auth_secret_ref values.
|
||||
Instead they show a status string: "configured" or "not_configured".
|
||||
|
||||
Requirements: 3.6, 3.7
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
VALID_PROTOCOLS = ("ollama_native", "openai_chat", "specialist_http")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoint schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class EndpointCreate(BaseModel):
|
||||
"""Request body for creating an inference endpoint."""
|
||||
|
||||
name: str = Field(..., min_length=1, max_length=255)
|
||||
protocol: Literal["ollama_native", "openai_chat", "specialist_http"]
|
||||
base_url: str = Field(..., min_length=1)
|
||||
auth_secret_ref: str | None = None
|
||||
auth_scheme: str = "bearer"
|
||||
default_headers: dict[str, str] = Field(default_factory=dict)
|
||||
health_path: str | None = None
|
||||
enabled: bool = True
|
||||
|
||||
@field_validator("base_url")
|
||||
@classmethod
|
||||
def validate_url(cls, v: str) -> str:
|
||||
"""Validate that base_url looks like a valid URL."""
|
||||
if not v.startswith(("http://", "https://")):
|
||||
raise ValueError("base_url must start with http:// or https://")
|
||||
return v.rstrip("/")
|
||||
|
||||
@field_validator("protocol")
|
||||
@classmethod
|
||||
def validate_protocol(cls, v: str) -> str:
|
||||
if v not in VALID_PROTOCOLS:
|
||||
raise ValueError(f"protocol must be one of {VALID_PROTOCOLS}")
|
||||
return v
|
||||
|
||||
|
||||
class EndpointUpdate(BaseModel):
|
||||
"""Request body for updating an inference endpoint."""
|
||||
|
||||
name: str | None = None
|
||||
protocol: Literal["ollama_native", "openai_chat", "specialist_http"] | None = None
|
||||
base_url: str | None = None
|
||||
auth_secret_ref: str | None = Field(default=None)
|
||||
auth_scheme: str | None = None
|
||||
default_headers: dict[str, str] | None = None
|
||||
health_path: str | None = None
|
||||
enabled: bool | None = None
|
||||
|
||||
@field_validator("base_url")
|
||||
@classmethod
|
||||
def validate_url(cls, v: str | None) -> str | None:
|
||||
if v is not None:
|
||||
if not v.startswith(("http://", "https://")):
|
||||
raise ValueError("base_url must start with http:// or https://")
|
||||
return v.rstrip("/")
|
||||
return v
|
||||
|
||||
@field_validator("protocol")
|
||||
@classmethod
|
||||
def validate_protocol(cls, v: str | None) -> str | None:
|
||||
if v is not None and v not in VALID_PROTOCOLS:
|
||||
raise ValueError(f"protocol must be one of {VALID_PROTOCOLS}")
|
||||
return v
|
||||
|
||||
|
||||
class EndpointResponse(BaseModel):
|
||||
"""Response model for an inference endpoint. NEVER includes auth_secret_ref value."""
|
||||
|
||||
id: UUID
|
||||
name: str
|
||||
protocol: str
|
||||
base_url: str
|
||||
auth_secret_status: str = "not_configured" # "configured" or "not_configured"
|
||||
auth_scheme: str = "bearer"
|
||||
default_headers: dict[str, str] = Field(default_factory=dict)
|
||||
health_path: str | None = None
|
||||
enabled: bool = True
|
||||
revision: int = 1
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
last_probe: ProbeResponse | None = None
|
||||
capabilities: dict[str, Any] | None = None
|
||||
active_bindings: list[BindingResponse] | None = None
|
||||
|
||||
|
||||
class EndpointListResponse(BaseModel):
|
||||
"""Response model for listing endpoints."""
|
||||
|
||||
id: UUID
|
||||
name: str
|
||||
protocol: str
|
||||
base_url: str
|
||||
auth_secret_status: str = "not_configured"
|
||||
enabled: bool = True
|
||||
revision: int = 1
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deployment schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class DeploymentCreate(BaseModel):
|
||||
"""Request body for creating a model deployment."""
|
||||
|
||||
endpoint_id: UUID
|
||||
served_model_name: str = Field(..., min_length=1)
|
||||
display_name: str = Field(..., min_length=1)
|
||||
capabilities: dict[str, Any] = Field(default_factory=dict)
|
||||
context_window: int | None = None
|
||||
max_output_tokens: int | None = None
|
||||
quantization: str | None = None
|
||||
runtime_metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
class DeploymentResponse(BaseModel):
|
||||
"""Response model for a model deployment."""
|
||||
|
||||
id: UUID
|
||||
endpoint_id: UUID
|
||||
served_model_name: str
|
||||
display_name: str
|
||||
capabilities: dict[str, Any] = Field(default_factory=dict)
|
||||
context_window: int | None = None
|
||||
max_output_tokens: int | None = None
|
||||
quantization: str | None = None
|
||||
runtime_metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
enabled: bool = True
|
||||
revision: int = 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Binding schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class BindingCreate(BaseModel):
|
||||
"""Request body for creating an agent stage binding."""
|
||||
|
||||
agent_id: UUID
|
||||
stage: str = Field(..., min_length=1)
|
||||
model_deployment_id: UUID | None = None
|
||||
route_order: int = 0
|
||||
routing_config: dict[str, Any] = Field(default_factory=dict)
|
||||
is_active: bool = True
|
||||
|
||||
|
||||
class BindingResponse(BaseModel):
|
||||
"""Response model for an agent stage binding."""
|
||||
|
||||
id: UUID
|
||||
agent_id: UUID
|
||||
stage: str
|
||||
model_deployment_id: UUID | None = None
|
||||
route_order: int = 0
|
||||
routing_config: dict[str, Any] = Field(default_factory=dict)
|
||||
is_active: bool = True
|
||||
revision: int = 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Probe schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ProbeResponse(BaseModel):
|
||||
"""Response model for probe results."""
|
||||
|
||||
endpoint_id: UUID
|
||||
timestamp: datetime | None = None
|
||||
software_version: str | None = None
|
||||
probe_duration_ms: int = 0
|
||||
health_success: bool = False
|
||||
health_detail: str = ""
|
||||
model_listing_success: bool | None = None
|
||||
json_schema_success: bool | None = None
|
||||
usage_success: bool | None = None
|
||||
seed_success: bool | None = None
|
||||
output_token_field_success: bool | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Egress confirmation schema
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class EgressConfirmation(BaseModel):
|
||||
"""Request body for confirming external endpoint egress enablement.
|
||||
|
||||
Requires explicit confirmed=true flag.
|
||||
"""
|
||||
|
||||
confirmed: bool = Field(
|
||||
...,
|
||||
description="Must be explicitly set to true to confirm external egress enablement",
|
||||
)
|
||||
|
||||
@field_validator("confirmed")
|
||||
@classmethod
|
||||
def must_be_true(cls, v: bool) -> bool:
|
||||
if not v:
|
||||
raise ValueError("confirmed must be true to enable external egress")
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Structured output test schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class StructuredOutputTestRequest(BaseModel):
|
||||
"""Request body for testing structured output on an endpoint."""
|
||||
|
||||
json_schema: dict[str, Any] = Field(
|
||||
default_factory=lambda: {
|
||||
"type": "object",
|
||||
"properties": {"status": {"type": "string"}},
|
||||
"required": ["status"],
|
||||
}
|
||||
)
|
||||
prompt: str = 'Respond with JSON: {"status": "ok"}'
|
||||
|
||||
|
||||
class StructuredOutputTestResponse(BaseModel):
|
||||
"""Response for structured output test."""
|
||||
|
||||
success: bool
|
||||
structured_mode: str = ""
|
||||
content: str = ""
|
||||
parsed: dict[str, Any] | None = None
|
||||
schema_valid: bool = False
|
||||
latency_ms: int = 0
|
||||
error: str | None = None
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Security helpers for the inference registry API.
|
||||
|
||||
Ensures auth_secret_ref values are NEVER exposed in API responses.
|
||||
Replaces the actual secret reference with a status string.
|
||||
|
||||
Requirements: 3.6
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from services.inference_registry.schemas import (
|
||||
BindingResponse,
|
||||
DeploymentResponse,
|
||||
EndpointListResponse,
|
||||
EndpointResponse,
|
||||
)
|
||||
|
||||
|
||||
def redact_endpoint(endpoint_dict: dict[str, Any]) -> EndpointResponse:
|
||||
"""Convert a raw endpoint dict to an EndpointResponse with secrets redacted.
|
||||
|
||||
The auth_secret_ref value is replaced with a status string:
|
||||
- "configured" if a secret reference exists
|
||||
- "not_configured" if no secret reference is set
|
||||
|
||||
The actual secret ref value is NEVER included in the response.
|
||||
"""
|
||||
auth_secret_ref = endpoint_dict.get("auth_secret_ref")
|
||||
auth_secret_status = "configured" if auth_secret_ref else "not_configured"
|
||||
|
||||
return EndpointResponse(
|
||||
id=endpoint_dict["id"],
|
||||
name=endpoint_dict["name"],
|
||||
protocol=endpoint_dict["protocol"],
|
||||
base_url=endpoint_dict["base_url"],
|
||||
auth_secret_status=auth_secret_status,
|
||||
auth_scheme=endpoint_dict.get("auth_scheme", "bearer"),
|
||||
default_headers=endpoint_dict.get("default_headers") or {},
|
||||
health_path=endpoint_dict.get("health_path"),
|
||||
enabled=endpoint_dict.get("enabled", True),
|
||||
revision=endpoint_dict.get("revision", 1),
|
||||
created_at=endpoint_dict.get("created_at"),
|
||||
updated_at=endpoint_dict.get("updated_at"),
|
||||
)
|
||||
|
||||
|
||||
def redact_endpoint_for_list(endpoint_dict: dict[str, Any]) -> EndpointListResponse:
|
||||
"""Convert a raw endpoint dict to a list response with secrets redacted."""
|
||||
auth_secret_ref = endpoint_dict.get("auth_secret_ref")
|
||||
auth_secret_status = "configured" if auth_secret_ref else "not_configured"
|
||||
|
||||
return EndpointListResponse(
|
||||
id=endpoint_dict["id"],
|
||||
name=endpoint_dict["name"],
|
||||
protocol=endpoint_dict["protocol"],
|
||||
base_url=endpoint_dict["base_url"],
|
||||
auth_secret_status=auth_secret_status,
|
||||
enabled=endpoint_dict.get("enabled", True),
|
||||
revision=endpoint_dict.get("revision", 1),
|
||||
created_at=endpoint_dict.get("created_at"),
|
||||
updated_at=endpoint_dict.get("updated_at"),
|
||||
)
|
||||
|
||||
|
||||
def redact_deployment(deployment_dict: dict[str, Any]) -> DeploymentResponse:
|
||||
"""Convert a raw deployment dict to a DeploymentResponse."""
|
||||
return DeploymentResponse(
|
||||
id=deployment_dict["id"],
|
||||
endpoint_id=deployment_dict["endpoint_id"],
|
||||
served_model_name=deployment_dict["served_model_name"],
|
||||
display_name=deployment_dict["display_name"],
|
||||
capabilities=deployment_dict.get("capabilities") or {},
|
||||
context_window=deployment_dict.get("context_window"),
|
||||
max_output_tokens=deployment_dict.get("max_output_tokens"),
|
||||
quantization=deployment_dict.get("quantization"),
|
||||
runtime_metadata=deployment_dict.get("runtime_metadata") or {},
|
||||
enabled=deployment_dict.get("enabled", True),
|
||||
revision=deployment_dict.get("revision", 1),
|
||||
)
|
||||
|
||||
|
||||
def redact_binding(binding_dict: dict[str, Any]) -> BindingResponse:
|
||||
"""Convert a raw binding dict to a BindingResponse."""
|
||||
return BindingResponse(
|
||||
id=binding_dict["id"],
|
||||
agent_id=binding_dict["agent_id"],
|
||||
stage=binding_dict["stage"],
|
||||
model_deployment_id=binding_dict.get("model_deployment_id"),
|
||||
route_order=binding_dict.get("route_order", 0),
|
||||
routing_config=binding_dict.get("routing_config") or {},
|
||||
is_active=binding_dict.get("is_active", True),
|
||||
revision=binding_dict.get("revision", 1),
|
||||
)
|
||||
|
||||
|
||||
def is_external_endpoint(base_url: str) -> bool:
|
||||
"""Determine if an endpoint URL points to an external (non-cluster) service.
|
||||
|
||||
External endpoints require egress confirmation before enablement.
|
||||
Local/cluster endpoints match:
|
||||
- localhost / 127.0.0.1
|
||||
- *.svc.cluster.local (Kubernetes internal)
|
||||
- 10.x.x.x / 192.168.x.x (private network)
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(base_url)
|
||||
hostname = parsed.hostname or ""
|
||||
|
||||
# Cluster-local patterns
|
||||
if hostname in ("localhost", "127.0.0.1", "::1"):
|
||||
return False
|
||||
if hostname.endswith(".svc.cluster.local"):
|
||||
return False
|
||||
if hostname.startswith("10.") or hostname.startswith("192.168."):
|
||||
return False
|
||||
# Additional private ranges
|
||||
if hostname.startswith("172."):
|
||||
parts = hostname.split(".")
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
second = int(parts[1])
|
||||
if 16 <= second <= 31:
|
||||
return False
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return True
|
||||
Reference in New Issue
Block a user