"""FastAPI router for the inference registry API. Manages inference_endpoints, model_deployments, and agent_stage_bindings. Auth secret values are NEVER returned in any response. Endpoints: - CRUD for inference_endpoints (19.1) - probe, enable, disable, test-structured-output actions (19.2) - Protocol/endpoint/deployment selectors (19.3) - Display last probe, capabilities, limits, bindings (19.4) - External egress confirmation (19.5) Requirements: 3.6, 3.7 """ from __future__ import annotations import logging import uuid from datetime import datetime, timezone from typing import Any from fastapi import APIRouter, Depends, HTTPException from services.inference_registry.schemas import ( BindingCreate, BindingResponse, DeploymentCreate, DeploymentResponse, EgressConfirmation, EndpointCreate, EndpointListResponse, EndpointResponse, EndpointUpdate, ProbeResponse, StructuredOutputTestRequest, StructuredOutputTestResponse, ) from services.inference_registry.security import ( is_external_endpoint, redact_binding, redact_deployment, redact_endpoint, redact_endpoint_for_list, ) logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/inference", tags=["inference-registry"]) # --------------------------------------------------------------------------- # Dependency injection protocol for database access # --------------------------------------------------------------------------- class InferenceRegistryDB: """Protocol for inference registry database operations. In production, backed by asyncpg pool. In tests, a mock implements this. """ async def list_endpoints(self) -> list[dict[str, Any]]: raise NotImplementedError async def get_endpoint(self, endpoint_id: uuid.UUID) -> dict[str, Any] | None: raise NotImplementedError async def create_endpoint(self, data: dict[str, Any]) -> dict[str, Any]: raise NotImplementedError async def update_endpoint( self, endpoint_id: uuid.UUID, data: dict[str, Any] ) -> dict[str, Any] | None: raise NotImplementedError async def disable_endpoint(self, endpoint_id: uuid.UUID) -> dict[str, Any] | None: raise NotImplementedError async def list_deployments( self, endpoint_id: uuid.UUID | None = None ) -> list[dict[str, Any]]: raise NotImplementedError async def get_deployment(self, deployment_id: uuid.UUID) -> dict[str, Any] | None: raise NotImplementedError async def create_deployment(self, data: dict[str, Any]) -> dict[str, Any]: raise NotImplementedError async def list_bindings( self, agent_id: uuid.UUID | None = None, endpoint_id: uuid.UUID | None = None, ) -> list[dict[str, Any]]: raise NotImplementedError async def create_binding(self, data: dict[str, Any]) -> dict[str, Any]: raise NotImplementedError async def get_bindings_for_endpoint( self, endpoint_id: uuid.UUID ) -> list[dict[str, Any]]: raise NotImplementedError async def get_last_probe( self, endpoint_id: uuid.UUID ) -> dict[str, Any] | None: raise NotImplementedError async def store_probe_result( self, endpoint_id: uuid.UUID, result: dict[str, Any] ) -> None: raise NotImplementedError async def get_egress_confirmation( self, endpoint_id: uuid.UUID ) -> bool: raise NotImplementedError async def store_egress_confirmation( self, endpoint_id: uuid.UUID ) -> None: raise NotImplementedError # Global DB instance (set during app startup) _db: InferenceRegistryDB | None = None def set_db(db: InferenceRegistryDB) -> None: """Set the database dependency for the router.""" global _db _db = db def get_db() -> InferenceRegistryDB: """Get the database dependency.""" if _db is None: raise HTTPException(503, "Database not initialized") return _db # --------------------------------------------------------------------------- # Endpoint CRUD (19.1) # --------------------------------------------------------------------------- @router.get("/endpoints", response_model=list[EndpointListResponse]) async def list_endpoints(db: InferenceRegistryDB = Depends(get_db)): """List all inference endpoints with secrets redacted.""" endpoints = await db.list_endpoints() return [redact_endpoint_for_list(ep) for ep in endpoints] @router.get("/endpoints/{endpoint_id}", response_model=EndpointResponse) async def get_endpoint( endpoint_id: uuid.UUID, db: InferenceRegistryDB = Depends(get_db), ): """Get endpoint detail with secrets redacted. Includes last probe results, capabilities, and active stage bindings (19.4). """ endpoint = await db.get_endpoint(endpoint_id) if endpoint is None: raise HTTPException(404, "Endpoint not found") response = redact_endpoint(endpoint) # Attach last probe result (19.4) probe_data = await db.get_last_probe(endpoint_id) if probe_data: response.last_probe = ProbeResponse(**probe_data) # Attach capabilities from deployments deployments = await db.list_deployments(endpoint_id=endpoint_id) if deployments: # Aggregate capabilities from all deployments combined_caps: dict[str, Any] = {} for dep in deployments: caps = dep.get("capabilities", {}) for k, v in caps.items(): if v: combined_caps[k] = True response.capabilities = combined_caps # Attach active bindings (19.4) bindings = await db.get_bindings_for_endpoint(endpoint_id) if bindings: response.active_bindings = [redact_binding(b) for b in bindings] return response @router.post("/endpoints", response_model=EndpointResponse, status_code=201) async def create_endpoint( body: EndpointCreate, db: InferenceRegistryDB = Depends(get_db), ): """Create a new inference endpoint. Validates protocol and URL format. External endpoints require egress confirmation before they can be enabled (19.5). """ now = datetime.now(timezone.utc) endpoint_data = { "id": uuid.uuid4(), "name": body.name, "protocol": body.protocol, "base_url": body.base_url, "auth_secret_ref": body.auth_secret_ref, "auth_scheme": body.auth_scheme, "default_headers": body.default_headers, "health_path": body.health_path, "enabled": body.enabled, "revision": 1, "created_at": now, "updated_at": now, } # If external, require egress confirmation before enabling (19.5) if body.enabled and is_external_endpoint(body.base_url): endpoint_data["enabled"] = False # Will need confirm-egress call created = await db.create_endpoint(endpoint_data) return redact_endpoint(created) @router.put("/endpoints/{endpoint_id}", response_model=EndpointResponse) async def update_endpoint( endpoint_id: uuid.UUID, body: EndpointUpdate, db: InferenceRegistryDB = Depends(get_db), ): """Update an existing inference endpoint.""" existing = await db.get_endpoint(endpoint_id) if existing is None: raise HTTPException(404, "Endpoint not found") update_data: dict[str, Any] = {} for field_name in ( "name", "protocol", "base_url", "auth_secret_ref", "auth_scheme", "default_headers", "health_path", "enabled", ): value = getattr(body, field_name) if value is not None: update_data[field_name] = value if update_data: update_data["updated_at"] = datetime.now(timezone.utc) update_data["revision"] = existing.get("revision", 1) + 1 updated = await db.update_endpoint(endpoint_id, update_data) if updated is None: raise HTTPException(404, "Endpoint not found") return redact_endpoint(updated) @router.delete("/endpoints/{endpoint_id}", response_model=EndpointResponse) async def delete_endpoint( endpoint_id: uuid.UUID, db: InferenceRegistryDB = Depends(get_db), ): """Soft-delete (disable) an inference endpoint.""" disabled = await db.disable_endpoint(endpoint_id) if disabled is None: raise HTTPException(404, "Endpoint not found") return redact_endpoint(disabled) # --------------------------------------------------------------------------- # Endpoint actions (19.2) # --------------------------------------------------------------------------- @router.post("/endpoints/{endpoint_id}/probe", response_model=ProbeResponse) async def probe_endpoint( endpoint_id: uuid.UUID, db: InferenceRegistryDB = Depends(get_db), ): """Run a capability probe against the endpoint. Performs health check, model listing, JSON Schema test, usage metadata check, seed determinism check, and output-token field check. """ endpoint = await db.get_endpoint(endpoint_id) if endpoint is None: raise HTTPException(404, "Endpoint not found") # Import the prober from services.shared.inference.capabilities import EndpointProber, FullProbeResult from services.shared.inference.models import InferenceTarget, ProviderCapabilities # Get a deployment to probe with (need a model name) deployments = await db.list_deployments(endpoint_id=endpoint_id) model_name = "default" caps_data: dict[str, Any] = {} deployment_id = uuid.uuid4() if deployments: model_name = deployments[0].get("served_model_name", "default") caps_data = deployments[0].get("capabilities", {}) deployment_id = deployments[0]["id"] capabilities = ProviderCapabilities( chat_completions=caps_data.get("chat_completions", True), responses_api=caps_data.get("responses_api", False), json_schema=caps_data.get("json_schema", False), json_object=caps_data.get("json_object", False), seed=caps_data.get("seed", False), usage=caps_data.get("usage", False), max_completion_tokens=caps_data.get("max_completion_tokens", False), reasoning_toggle=caps_data.get("reasoning_toggle", False), model_listing=caps_data.get("model_listing", False), ) target = InferenceTarget( endpoint_id=endpoint_id, deployment_id=deployment_id, protocol=endpoint["protocol"], base_url=endpoint["base_url"], model=model_name, capabilities=capabilities, auth_secret_ref=endpoint.get("auth_secret_ref"), auth_scheme=endpoint.get("auth_scheme", "bearer"), extra_headers=endpoint.get("default_headers") or {}, ) prober = EndpointProber() try: result: FullProbeResult = await prober.run_full_probe(target) finally: await prober.close() # Build probe response probe_response = ProbeResponse( endpoint_id=endpoint_id, timestamp=result.timestamp, software_version=result.software_version, probe_duration_ms=result.probe_duration_ms, health_success=result.health.success if result.health else False, health_detail=result.health.detail if result.health else "", model_listing_success=( result.model_listing.success if result.model_listing else None ), json_schema_success=( result.json_schema.success if result.json_schema else None ), usage_success=( result.usage_metadata.success if result.usage_metadata else None ), seed_success=( result.seed_determinism.success if result.seed_determinism else None ), output_token_field_success=( result.output_token_field.success if result.output_token_field else None ), ) # Store probe result await db.store_probe_result(endpoint_id, probe_response.model_dump()) return probe_response @router.post("/endpoints/{endpoint_id}/enable", response_model=EndpointResponse) async def enable_endpoint( endpoint_id: uuid.UUID, db: InferenceRegistryDB = Depends(get_db), ): """Enable an inference endpoint. External endpoints require egress confirmation first (19.5). """ endpoint = await db.get_endpoint(endpoint_id) if endpoint is None: raise HTTPException(404, "Endpoint not found") # Check if external endpoint needs egress confirmation (19.5) if is_external_endpoint(endpoint["base_url"]): has_confirmation = await db.get_egress_confirmation(endpoint_id) if not has_confirmation: raise HTTPException( 403, "External endpoint requires egress confirmation. " "POST /api/inference/endpoints/{id}/confirm-egress first.", ) update_data = { "enabled": True, "updated_at": datetime.now(timezone.utc), "revision": endpoint.get("revision", 1) + 1, } updated = await db.update_endpoint(endpoint_id, update_data) if updated is None: raise HTTPException(404, "Endpoint not found") return redact_endpoint(updated) @router.post("/endpoints/{endpoint_id}/disable", response_model=EndpointResponse) async def disable_endpoint_action( endpoint_id: uuid.UUID, db: InferenceRegistryDB = Depends(get_db), ): """Disable an inference endpoint.""" endpoint = await db.get_endpoint(endpoint_id) if endpoint is None: raise HTTPException(404, "Endpoint not found") update_data = { "enabled": False, "updated_at": datetime.now(timezone.utc), "revision": endpoint.get("revision", 1) + 1, } updated = await db.update_endpoint(endpoint_id, update_data) if updated is None: raise HTTPException(404, "Endpoint not found") return redact_endpoint(updated) @router.post( "/endpoints/{endpoint_id}/test-structured-output", response_model=StructuredOutputTestResponse, ) async def test_structured_output( endpoint_id: uuid.UUID, body: StructuredOutputTestRequest | None = None, db: InferenceRegistryDB = Depends(get_db), ): """Test JSON Schema structured output on an endpoint. Sends a minimal schema-constrained request and validates the response. """ if body is None: body = StructuredOutputTestRequest() endpoint = await db.get_endpoint(endpoint_id) if endpoint is None: raise HTTPException(404, "Endpoint not found") from services.shared.inference.capabilities import EndpointProber from services.shared.inference.models import InferenceTarget, ProviderCapabilities # Get a deployment to test with deployments = await db.list_deployments(endpoint_id=endpoint_id) model_name = "default" deployment_id = uuid.uuid4() if deployments: model_name = deployments[0].get("served_model_name", "default") deployment_id = deployments[0]["id"] target = InferenceTarget( endpoint_id=endpoint_id, deployment_id=deployment_id, protocol=endpoint["protocol"], base_url=endpoint["base_url"], model=model_name, capabilities=ProviderCapabilities( chat_completions=True, json_schema=True, ), auth_secret_ref=endpoint.get("auth_secret_ref"), auth_scheme=endpoint.get("auth_scheme", "bearer"), extra_headers=endpoint.get("default_headers") or {}, ) prober = EndpointProber() try: result = await prober.probe_json_schema(target) finally: await prober.close() return StructuredOutputTestResponse( success=result.success, structured_mode=result.structured_mode_used, content=result.detail, schema_valid=result.schema_valid, error=None if result.success else result.detail, ) # --------------------------------------------------------------------------- # External egress confirmation (19.5) # --------------------------------------------------------------------------- @router.post("/endpoints/{endpoint_id}/confirm-egress", response_model=EndpointResponse) async def confirm_egress( endpoint_id: uuid.UUID, body: EgressConfirmation, db: InferenceRegistryDB = Depends(get_db), ): """Confirm external endpoint egress enablement. Required before enabling an endpoint with a non-cluster URL. The request body must contain {"confirmed": true}. """ endpoint = await db.get_endpoint(endpoint_id) if endpoint is None: raise HTTPException(404, "Endpoint not found") if not is_external_endpoint(endpoint["base_url"]): raise HTTPException(400, "Endpoint is not external; no egress confirmation needed") # body.confirmed is already validated by pydantic to be True await db.store_egress_confirmation(endpoint_id) # Now enable the endpoint update_data = { "enabled": True, "updated_at": datetime.now(timezone.utc), "revision": endpoint.get("revision", 1) + 1, } updated = await db.update_endpoint(endpoint_id, update_data) if updated is None: raise HTTPException(404, "Endpoint not found") return redact_endpoint(updated) # --------------------------------------------------------------------------- # Deployments (19.3, 19.4) # --------------------------------------------------------------------------- @router.get("/deployments", response_model=list[DeploymentResponse]) async def list_deployments( endpoint_id: uuid.UUID | None = None, db: InferenceRegistryDB = Depends(get_db), ): """List model deployments, optionally filtered by endpoint.""" deployments = await db.list_deployments(endpoint_id=endpoint_id) return [redact_deployment(d) for d in deployments] @router.get("/deployments/{deployment_id}", response_model=DeploymentResponse) async def get_deployment( deployment_id: uuid.UUID, db: InferenceRegistryDB = Depends(get_db), ): """Get a model deployment with capabilities and limits (19.4).""" deployment = await db.get_deployment(deployment_id) if deployment is None: raise HTTPException(404, "Deployment not found") return redact_deployment(deployment) @router.post("/deployments", response_model=DeploymentResponse, status_code=201) async def create_deployment( body: DeploymentCreate, db: InferenceRegistryDB = Depends(get_db), ): """Create a new model deployment.""" # Verify endpoint exists endpoint = await db.get_endpoint(body.endpoint_id) if endpoint is None: raise HTTPException(404, "Referenced endpoint not found") deployment_data = { "id": uuid.uuid4(), "endpoint_id": body.endpoint_id, "served_model_name": body.served_model_name, "display_name": body.display_name, "capabilities": body.capabilities, "context_window": body.context_window, "max_output_tokens": body.max_output_tokens, "quantization": body.quantization, "runtime_metadata": body.runtime_metadata, "enabled": body.enabled, "revision": 1, } created = await db.create_deployment(deployment_data) return redact_deployment(created) # --------------------------------------------------------------------------- # Bindings (19.3, 19.4) # --------------------------------------------------------------------------- @router.get("/bindings", response_model=list[BindingResponse]) async def list_bindings( agent_id: uuid.UUID | None = None, db: InferenceRegistryDB = Depends(get_db), ): """List agent stage bindings.""" bindings = await db.list_bindings(agent_id=agent_id) return [redact_binding(b) for b in bindings] @router.post("/bindings", response_model=BindingResponse, status_code=201) async def create_binding( body: BindingCreate, db: InferenceRegistryDB = Depends(get_db), ): """Create an agent stage binding.""" # Verify deployment exists if provided if body.model_deployment_id: deployment = await db.get_deployment(body.model_deployment_id) if deployment is None: raise HTTPException(404, "Referenced deployment not found") binding_data = { "id": uuid.uuid4(), "agent_id": body.agent_id, "stage": body.stage, "model_deployment_id": body.model_deployment_id, "route_order": body.route_order, "routing_config": body.routing_config, "is_active": body.is_active, "revision": 1, } created = await db.create_binding(binding_data) return redact_binding(created) # --------------------------------------------------------------------------- # Selectors (19.3) # --------------------------------------------------------------------------- @router.get("/protocols") async def list_protocols(): """Return available protocol options for endpoint creation. Replaces free-text provider inputs with controlled selectors. """ return { "protocols": [ {"value": "ollama_native", "label": "Ollama Native", "description": "Ollama /api/chat endpoint"}, {"value": "openai_chat", "label": "OpenAI Compatible", "description": "OpenAI /v1/chat/completions (vLLM, OpenAI, LM Studio, SGLang)"}, {"value": "specialist_http", "label": "Specialist HTTP", "description": "Typed non-generative endpoints (GLiNER, FinBERT)"}, ] }