Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
1016 lines
35 KiB
Python
1016 lines
35 KiB
Python
"""Tests for capability probing module.
|
|
|
|
Tests against mocked HTTP transport validating all probe behaviors,
|
|
result storage with TTL, and required-capability validation.
|
|
|
|
Requirements: 2.10, 2.11
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from services.shared.inference.capabilities import (
|
|
DEFAULT_PROBE_TTL_SECONDS,
|
|
EndpointProber,
|
|
FullProbeResult,
|
|
HealthProbeResult,
|
|
JsonSchemaProbeResult,
|
|
ModelListingResult,
|
|
OutputTokenFieldResult,
|
|
ProbeResultStore,
|
|
SeedProbeResult,
|
|
UsageProbeResult,
|
|
validate_required_capabilities,
|
|
)
|
|
from services.shared.inference.models import (
|
|
InferenceTarget,
|
|
ProviderCapabilities,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_target(
|
|
*,
|
|
json_schema: bool = True,
|
|
json_object: bool = False,
|
|
seed: bool = True,
|
|
usage: bool = True,
|
|
max_completion_tokens: bool = False,
|
|
model_listing: bool = True,
|
|
model: str = "test-model",
|
|
base_url: str = "http://test-endpoint:8000",
|
|
auth_secret_ref: str | None = None,
|
|
) -> InferenceTarget:
|
|
"""Build a test InferenceTarget."""
|
|
return InferenceTarget(
|
|
endpoint_id=uuid4(),
|
|
deployment_id=uuid4(),
|
|
protocol="openai_chat",
|
|
base_url=base_url,
|
|
model=model,
|
|
capabilities=ProviderCapabilities(
|
|
chat_completions=True,
|
|
json_schema=json_schema,
|
|
json_object=json_object,
|
|
seed=seed,
|
|
usage=usage,
|
|
max_completion_tokens=max_completion_tokens,
|
|
model_listing=model_listing,
|
|
),
|
|
auth_secret_ref=auth_secret_ref,
|
|
)
|
|
|
|
|
|
def _chat_response(
|
|
content: str = '{"status": "ok"}',
|
|
*,
|
|
usage: dict | None = None,
|
|
finish_reason: str = "stop",
|
|
status: int = 200,
|
|
headers: dict | None = None,
|
|
) -> httpx.Response:
|
|
"""Build a mock OpenAI-compatible chat completions response."""
|
|
body: dict = {
|
|
"id": "chatcmpl-probe",
|
|
"object": "chat.completion",
|
|
"choices": [
|
|
{
|
|
"index": 0,
|
|
"message": {"role": "assistant", "content": content},
|
|
"finish_reason": finish_reason,
|
|
}
|
|
],
|
|
}
|
|
if usage:
|
|
body["usage"] = usage
|
|
resp_headers = headers or {}
|
|
return httpx.Response(status, json=body, headers=resp_headers)
|
|
|
|
|
|
def _models_response(
|
|
model_ids: list[str] | None = None,
|
|
status: int = 200,
|
|
headers: dict | None = None,
|
|
) -> httpx.Response:
|
|
"""Build a mock /v1/models response."""
|
|
if model_ids is None:
|
|
model_ids = ["test-model"]
|
|
body = {
|
|
"data": [{"id": mid, "object": "model"} for mid in model_ids],
|
|
}
|
|
return httpx.Response(status, json=body, headers=headers or {})
|
|
|
|
|
|
# ===========================================================================
|
|
# 13.1: Probe health and model listing
|
|
# ===========================================================================
|
|
|
|
|
|
class TestProbeHealth:
|
|
"""Tests for probe_health method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_success_via_health_path(self):
|
|
"""Health probe succeeds when /health returns 200."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if "/health" in str(request.url):
|
|
return httpx.Response(200, json={"status": "ok"})
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_health(target)
|
|
|
|
assert result.success is True
|
|
assert "/health" in result.detail
|
|
assert result.latency_ms >= 0
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_success_via_models_fallback(self):
|
|
"""Health probe falls back to /v1/models when /health fails."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
url = str(request.url)
|
|
if url.endswith("/health"):
|
|
return httpx.Response(500)
|
|
if "/v1/models" in url:
|
|
return httpx.Response(
|
|
200,
|
|
json={"data": []},
|
|
headers={"server": "vllm/0.6.0"},
|
|
)
|
|
return httpx.Response(500)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_health(target)
|
|
|
|
assert result.success is True
|
|
assert "/v1/models" in result.detail
|
|
assert result.software_version == "vllm/0.6.0"
|
|
await prober.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_failure_connection_refused(self):
|
|
"""Health probe fails when endpoint is unreachable."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise httpx.ConnectError("Connection refused")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_health(target)
|
|
|
|
assert result.success is False
|
|
assert "Connection failed" in result.detail
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_failure_timeout(self):
|
|
"""Health probe fails on timeout."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise httpx.ReadTimeout("timed out")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_health(target)
|
|
|
|
assert result.success is False
|
|
assert "Connection failed" in result.detail
|
|
await prober.close()
|
|
|
|
|
|
class TestProbeModelListing:
|
|
"""Tests for probe_model_listing method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_model_listing_success_with_target_model(self):
|
|
"""Model listing succeeds and finds the target model."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return _models_response(["test-model", "other-model"])
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_model_listing(target)
|
|
|
|
assert result.success is True
|
|
assert result.target_model_found is True
|
|
assert "test-model" in result.models
|
|
assert "other-model" in result.models
|
|
await prober.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_model_listing_model_not_found(self):
|
|
"""Model listing succeeds but target model is not in the list."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return _models_response(["other-model-a", "other-model-b"])
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_model_listing(target)
|
|
|
|
assert result.success is True
|
|
assert result.target_model_found is False
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_model_listing_http_error(self):
|
|
"""Model listing fails on non-200 response."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(403, text="Forbidden")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_model_listing(target)
|
|
|
|
assert result.success is False
|
|
assert "403" in result.detail
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_model_listing_connection_error(self):
|
|
"""Model listing fails when endpoint is unreachable."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise httpx.ConnectError("refused")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_model_listing(target)
|
|
|
|
assert result.success is False
|
|
assert "Connection failed" in result.detail
|
|
await prober.close()
|
|
|
|
|
|
# ===========================================================================
|
|
# 13.2: Probe strict JSON Schema with a minimal schema
|
|
# ===========================================================================
|
|
|
|
|
|
class TestProbeJsonSchema:
|
|
"""Tests for probe_json_schema method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_json_schema_probe_success(self):
|
|
"""Schema probe succeeds when endpoint returns valid schema-conforming JSON."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
payload = json.loads(request.content)
|
|
# Verify request includes response_format
|
|
assert payload["response_format"]["type"] == "json_schema"
|
|
return _chat_response('{"status": "ok"}')
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_json_schema(target)
|
|
|
|
assert result.success is True
|
|
assert result.schema_valid is True
|
|
assert result.structured_mode_used == "json_schema"
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_json_schema_probe_invalid_response(self):
|
|
"""Schema probe fails when response doesn't match expected schema."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
# Returns JSON but wrong schema (missing "status" field)
|
|
return _chat_response('{"wrong_field": 123}')
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_json_schema(target)
|
|
|
|
assert result.success is False
|
|
assert result.schema_valid is False
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_json_schema_probe_non_json_response(self):
|
|
"""Schema probe fails when response is not valid JSON."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return _chat_response("This is plain text, not JSON")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_json_schema(target)
|
|
|
|
assert result.success is False
|
|
assert result.schema_valid is False
|
|
assert "not valid JSON" in result.detail
|
|
await prober.close()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_json_schema_probe_http_error(self):
|
|
"""Schema probe fails on non-200 response."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(400, json={"error": "bad request"})
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_json_schema(target)
|
|
|
|
assert result.success is False
|
|
assert "400" in result.detail
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_json_schema_probe_empty_choices(self):
|
|
"""Schema probe fails when response has empty choices."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
body = {"choices": []}
|
|
return httpx.Response(200, json=body)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_json_schema(target)
|
|
|
|
assert result.success is False
|
|
assert "Empty choices" in result.detail
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_json_schema_probe_connection_error(self):
|
|
"""Schema probe fails on connection error."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise httpx.ConnectError("refused")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_json_schema(target)
|
|
|
|
assert result.success is False
|
|
assert "Connection failed" in result.detail
|
|
await prober.close()
|
|
|
|
|
|
# ===========================================================================
|
|
# 13.3: Probe usage metadata, seed behavior, and output-token field
|
|
# ===========================================================================
|
|
|
|
|
|
class TestProbeUsageMetadata:
|
|
"""Tests for probe_usage_metadata method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_usage_metadata_both_present(self):
|
|
"""Usage probe succeeds when both token counts are present."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return _chat_response(
|
|
"hi",
|
|
usage={"prompt_tokens": 10, "completion_tokens": 5},
|
|
)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_usage_metadata(target)
|
|
|
|
assert result.success is True
|
|
assert result.has_prompt_tokens is True
|
|
assert result.has_completion_tokens is True
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_usage_metadata_none_present(self):
|
|
"""Usage probe fails when no usage tokens are returned."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return _chat_response("hi") # No usage field
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_usage_metadata(target)
|
|
|
|
assert result.success is False
|
|
assert result.has_prompt_tokens is False
|
|
assert result.has_completion_tokens is False
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_usage_metadata_http_error(self):
|
|
"""Usage probe fails on HTTP error."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(500, text="Internal error")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_usage_metadata(target)
|
|
|
|
assert result.success is False
|
|
assert "500" in result.detail
|
|
await prober.close()
|
|
|
|
|
|
class TestProbeSeedDeterminism:
|
|
"""Tests for probe_seed_determinism method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_seed_determinism_outputs_match(self):
|
|
"""Seed probe reports match when outputs are identical."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return _chat_response("hello")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_seed_determinism(target)
|
|
|
|
assert result.success is True
|
|
assert result.outputs_match is True
|
|
assert "match" in result.detail
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_seed_determinism_outputs_differ(self):
|
|
"""Seed probe reports no match when outputs differ."""
|
|
call_count = 0
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
nonlocal call_count
|
|
call_count += 1
|
|
content = f"response-{call_count}"
|
|
return _chat_response(content)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_seed_determinism(target)
|
|
|
|
assert result.success is True
|
|
assert result.outputs_match is False
|
|
assert "differ" in result.detail
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_seed_determinism_http_error(self):
|
|
"""Seed probe fails on HTTP error."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(500, text="error")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_seed_determinism(target)
|
|
|
|
assert result.success is False
|
|
assert "500" in result.detail
|
|
await prober.close()
|
|
|
|
|
|
class TestProbeOutputTokenField:
|
|
"""Tests for probe_output_token_field method."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_output_token_field_accepted(self):
|
|
"""Output token field probe succeeds when server returns 200."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
payload = json.loads(request.content)
|
|
assert "max_completion_tokens" in payload
|
|
return _chat_response("hi")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_output_token_field(target)
|
|
|
|
assert result.success is True
|
|
assert result.field_accepted is True
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_output_token_field_rejected(self):
|
|
"""Output token field probe fails when server rejects the field."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
400,
|
|
json={"error": "unknown field: max_completion_tokens"},
|
|
)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_output_token_field(target)
|
|
|
|
assert result.success is False
|
|
assert result.field_accepted is False
|
|
assert "rejected" in result.detail
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_output_token_field_connection_error(self):
|
|
"""Output token field probe fails on connection error."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise httpx.ConnectError("refused")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.probe_output_token_field(target)
|
|
|
|
assert result.success is False
|
|
assert "Connection failed" in result.detail
|
|
await prober.close()
|
|
|
|
|
|
# ===========================================================================
|
|
# 13.4: Store probe results and software/version metadata with TTL
|
|
# ===========================================================================
|
|
|
|
|
|
class TestProbeResultStore:
|
|
"""Tests for ProbeResultStore TTL-based cache."""
|
|
|
|
def test_store_and_retrieve(self):
|
|
"""Can store and retrieve a probe result within TTL."""
|
|
store = ProbeResultStore(ttl_seconds=60)
|
|
endpoint_id = uuid4()
|
|
result = FullProbeResult(
|
|
endpoint_id=endpoint_id,
|
|
health=HealthProbeResult(success=True, detail="ok"),
|
|
software_version="vllm/0.6.0",
|
|
)
|
|
|
|
store.store(endpoint_id, result)
|
|
retrieved = store.get(endpoint_id)
|
|
|
|
assert retrieved is not None
|
|
assert retrieved.endpoint_id == endpoint_id
|
|
assert retrieved.software_version == "vllm/0.6.0"
|
|
|
|
def test_retrieve_nonexistent_returns_none(self):
|
|
"""Getting a non-existent entry returns None."""
|
|
store = ProbeResultStore(ttl_seconds=60)
|
|
assert store.get(uuid4()) is None
|
|
|
|
def test_ttl_expiry(self, monkeypatch):
|
|
"""Expired entries return None on retrieval."""
|
|
store = ProbeResultStore(ttl_seconds=1)
|
|
endpoint_id = uuid4()
|
|
result = FullProbeResult(endpoint_id=endpoint_id)
|
|
|
|
store.store(endpoint_id, result)
|
|
|
|
# Monkey-patch time.monotonic to simulate TTL expiry
|
|
original_monotonic = time.monotonic
|
|
start = original_monotonic()
|
|
monkeypatch.setattr(
|
|
time, "monotonic", lambda: start + 2.0
|
|
)
|
|
|
|
# Need to re-store with a fixed time reference
|
|
# Actually, let's use a different approach: store, then move time forward
|
|
store._store[endpoint_id] = (start - 2.0, result)
|
|
retrieved = store.get(endpoint_id)
|
|
|
|
assert retrieved is None
|
|
# Entry should be evicted
|
|
assert endpoint_id not in store._store
|
|
|
|
def test_invalidate(self):
|
|
"""Invalidate removes an entry."""
|
|
store = ProbeResultStore(ttl_seconds=60)
|
|
endpoint_id = uuid4()
|
|
result = FullProbeResult(endpoint_id=endpoint_id)
|
|
|
|
store.store(endpoint_id, result)
|
|
store.invalidate(endpoint_id)
|
|
|
|
assert store.get(endpoint_id) is None
|
|
|
|
def test_invalidate_nonexistent_is_safe(self):
|
|
"""Invalidating a non-existent entry doesn't raise."""
|
|
store = ProbeResultStore(ttl_seconds=60)
|
|
store.invalidate(uuid4()) # Should not raise
|
|
|
|
|
|
def test_clear(self):
|
|
"""Clear removes all entries."""
|
|
store = ProbeResultStore(ttl_seconds=60)
|
|
for _ in range(5):
|
|
eid = uuid4()
|
|
store.store(eid, FullProbeResult(endpoint_id=eid))
|
|
|
|
assert len(store) == 5
|
|
store.clear()
|
|
assert len(store) == 0
|
|
|
|
def test_default_ttl(self):
|
|
"""Default TTL is 5 minutes."""
|
|
store = ProbeResultStore()
|
|
assert store.ttl_seconds == DEFAULT_PROBE_TTL_SECONDS
|
|
assert store.ttl_seconds == 300
|
|
|
|
def test_software_version_stored(self):
|
|
"""Software version metadata is preserved in stored results."""
|
|
store = ProbeResultStore(ttl_seconds=60)
|
|
endpoint_id = uuid4()
|
|
result = FullProbeResult(
|
|
endpoint_id=endpoint_id,
|
|
health=HealthProbeResult(
|
|
success=True,
|
|
software_version="vllm/0.6.1",
|
|
),
|
|
software_version="vllm/0.6.1",
|
|
)
|
|
|
|
store.store(endpoint_id, result)
|
|
retrieved = store.get(endpoint_id)
|
|
|
|
assert retrieved is not None
|
|
assert retrieved.software_version == "vllm/0.6.1"
|
|
assert retrieved.health is not None
|
|
assert retrieved.health.software_version == "vllm/0.6.1"
|
|
|
|
|
|
# ===========================================================================
|
|
# 13.5: Refuse activation when declared required capabilities fail
|
|
# ===========================================================================
|
|
|
|
|
|
class TestValidateRequiredCapabilities:
|
|
"""Tests for validate_required_capabilities function."""
|
|
|
|
def test_all_capabilities_pass(self):
|
|
"""No failures when all probes pass for declared capabilities."""
|
|
target = _make_target(
|
|
json_schema=True,
|
|
seed=True,
|
|
usage=True,
|
|
max_completion_tokens=True,
|
|
model_listing=True,
|
|
)
|
|
probe_result = FullProbeResult(
|
|
endpoint_id=target.endpoint_id,
|
|
health=HealthProbeResult(success=True, detail="ok"),
|
|
model_listing=ModelListingResult(
|
|
success=True, target_model_found=True, models=["test-model"]
|
|
),
|
|
json_schema=JsonSchemaProbeResult(
|
|
success=True, schema_valid=True
|
|
),
|
|
usage_metadata=UsageProbeResult(
|
|
success=True, has_prompt_tokens=True, has_completion_tokens=True
|
|
),
|
|
seed_determinism=SeedProbeResult(
|
|
success=True, outputs_match=True
|
|
),
|
|
output_token_field=OutputTokenFieldResult(
|
|
success=True, field_accepted=True
|
|
),
|
|
)
|
|
|
|
failures = validate_required_capabilities(target, probe_result)
|
|
assert failures == []
|
|
|
|
def test_health_failure_blocks_all(self):
|
|
"""Health failure returns immediately without checking others."""
|
|
target = _make_target()
|
|
probe_result = FullProbeResult(
|
|
endpoint_id=target.endpoint_id,
|
|
health=HealthProbeResult(
|
|
success=False, detail="Connection refused"
|
|
),
|
|
)
|
|
|
|
failures = validate_required_capabilities(target, probe_result)
|
|
|
|
assert len(failures) == 1
|
|
assert "Health check failed" in failures[0]
|
|
|
|
def test_json_schema_failure(self):
|
|
"""JSON schema failure is reported when declared."""
|
|
target = _make_target(json_schema=True)
|
|
probe_result = FullProbeResult(
|
|
endpoint_id=target.endpoint_id,
|
|
health=HealthProbeResult(success=True),
|
|
model_listing=ModelListingResult(
|
|
success=True, target_model_found=True, models=["test-model"]
|
|
),
|
|
json_schema=JsonSchemaProbeResult(
|
|
success=False, detail="HTTP 400"
|
|
),
|
|
usage_metadata=UsageProbeResult(success=True),
|
|
seed_determinism=SeedProbeResult(
|
|
success=True, outputs_match=True
|
|
),
|
|
)
|
|
|
|
failures = validate_required_capabilities(target, probe_result)
|
|
|
|
assert any("JSON Schema" in f for f in failures)
|
|
|
|
|
|
def test_json_schema_passes_but_invalid_response(self):
|
|
"""Schema probe succeeded but response didn't validate."""
|
|
target = _make_target(json_schema=True)
|
|
probe_result = FullProbeResult(
|
|
endpoint_id=target.endpoint_id,
|
|
health=HealthProbeResult(success=True),
|
|
model_listing=ModelListingResult(
|
|
success=True, target_model_found=True, models=["test-model"]
|
|
),
|
|
json_schema=JsonSchemaProbeResult(
|
|
success=True, schema_valid=False
|
|
),
|
|
usage_metadata=UsageProbeResult(success=True),
|
|
seed_determinism=SeedProbeResult(
|
|
success=True, outputs_match=True
|
|
),
|
|
)
|
|
|
|
failures = validate_required_capabilities(target, probe_result)
|
|
|
|
assert any("did not validate" in f for f in failures)
|
|
|
|
def test_model_not_found_in_listing(self):
|
|
"""Model not found in endpoint listing is a failure."""
|
|
target = _make_target(model_listing=True)
|
|
probe_result = FullProbeResult(
|
|
endpoint_id=target.endpoint_id,
|
|
health=HealthProbeResult(success=True),
|
|
model_listing=ModelListingResult(
|
|
success=True,
|
|
target_model_found=False,
|
|
models=["other-model"],
|
|
),
|
|
json_schema=JsonSchemaProbeResult(success=True, schema_valid=True),
|
|
usage_metadata=UsageProbeResult(success=True),
|
|
seed_determinism=SeedProbeResult(
|
|
success=True, outputs_match=True
|
|
),
|
|
)
|
|
|
|
failures = validate_required_capabilities(target, probe_result)
|
|
|
|
assert any("not found" in f for f in failures)
|
|
|
|
def test_seed_declared_but_outputs_differ(self):
|
|
"""Seed failure when outputs don't match."""
|
|
target = _make_target(seed=True, model_listing=False)
|
|
probe_result = FullProbeResult(
|
|
endpoint_id=target.endpoint_id,
|
|
health=HealthProbeResult(success=True),
|
|
json_schema=JsonSchemaProbeResult(success=True, schema_valid=True),
|
|
usage_metadata=UsageProbeResult(success=True),
|
|
seed_determinism=SeedProbeResult(
|
|
success=True, outputs_match=False
|
|
),
|
|
)
|
|
|
|
failures = validate_required_capabilities(target, probe_result)
|
|
|
|
assert any("outputs differ" in f for f in failures)
|
|
|
|
|
|
def test_usage_not_available(self):
|
|
"""Usage failure when declared but probe finds no tokens."""
|
|
target = _make_target(usage=True, model_listing=False, seed=False)
|
|
probe_result = FullProbeResult(
|
|
endpoint_id=target.endpoint_id,
|
|
health=HealthProbeResult(success=True),
|
|
json_schema=JsonSchemaProbeResult(success=True, schema_valid=True),
|
|
usage_metadata=UsageProbeResult(
|
|
success=False, detail="No usage in response"
|
|
),
|
|
)
|
|
|
|
failures = validate_required_capabilities(target, probe_result)
|
|
|
|
assert any("Usage metadata" in f for f in failures)
|
|
|
|
def test_max_completion_tokens_not_supported(self):
|
|
"""max_completion_tokens failure when declared but not accepted."""
|
|
target = _make_target(
|
|
max_completion_tokens=True,
|
|
model_listing=False,
|
|
seed=False,
|
|
usage=False,
|
|
json_schema=False,
|
|
)
|
|
probe_result = FullProbeResult(
|
|
endpoint_id=target.endpoint_id,
|
|
health=HealthProbeResult(success=True),
|
|
output_token_field=OutputTokenFieldResult(
|
|
success=False, detail="HTTP 400"
|
|
),
|
|
)
|
|
|
|
failures = validate_required_capabilities(target, probe_result)
|
|
|
|
assert any("max_completion_tokens" in f for f in failures)
|
|
|
|
def test_undeclared_capabilities_not_checked(self):
|
|
"""Capabilities not declared in target are not validated."""
|
|
# Target declares NO capabilities except chat_completions
|
|
target = _make_target(
|
|
json_schema=False,
|
|
json_object=False,
|
|
seed=False,
|
|
usage=False,
|
|
max_completion_tokens=False,
|
|
model_listing=False,
|
|
)
|
|
# Even though probes are missing/failed, no failures reported
|
|
probe_result = FullProbeResult(
|
|
endpoint_id=target.endpoint_id,
|
|
health=HealthProbeResult(success=True),
|
|
)
|
|
|
|
failures = validate_required_capabilities(target, probe_result)
|
|
assert failures == []
|
|
|
|
def test_multiple_failures_reported(self):
|
|
"""Multiple capability failures are all reported."""
|
|
target = _make_target(
|
|
json_schema=True,
|
|
seed=True,
|
|
usage=True,
|
|
max_completion_tokens=True,
|
|
model_listing=True,
|
|
)
|
|
# Everything fails
|
|
probe_result = FullProbeResult(
|
|
endpoint_id=target.endpoint_id,
|
|
health=HealthProbeResult(success=True),
|
|
model_listing=ModelListingResult(success=False, detail="error"),
|
|
json_schema=JsonSchemaProbeResult(success=False, detail="error"),
|
|
usage_metadata=UsageProbeResult(success=False, detail="error"),
|
|
seed_determinism=SeedProbeResult(success=False, detail="error"),
|
|
output_token_field=OutputTokenFieldResult(
|
|
success=False, detail="error"
|
|
),
|
|
)
|
|
|
|
failures = validate_required_capabilities(target, probe_result)
|
|
|
|
# Should report all failures
|
|
assert len(failures) >= 4
|
|
|
|
|
|
# ===========================================================================
|
|
# Integration: run_full_probe
|
|
# ===========================================================================
|
|
|
|
|
|
class TestRunFullProbe:
|
|
"""Tests for the full probe orchestration."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_full_probe_all_pass(self):
|
|
"""Full probe runs all sub-probes and returns aggregated results."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
url = str(request.url)
|
|
if "/health" in url:
|
|
return httpx.Response(
|
|
200,
|
|
json={"status": "ok"},
|
|
headers={"x-vllm-version": "0.6.1"},
|
|
)
|
|
if "/v1/models" in url and request.method == "GET":
|
|
return _models_response(
|
|
["test-model"],
|
|
headers={"x-vllm-version": "0.6.1"},
|
|
)
|
|
if "/v1/chat/completions" in url:
|
|
payload = json.loads(request.content)
|
|
return _chat_response(
|
|
'{"status": "ok"}',
|
|
usage={"prompt_tokens": 5, "completion_tokens": 3},
|
|
headers={"x-vllm-version": "0.6.1"},
|
|
)
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.run_full_probe(target)
|
|
|
|
assert result.endpoint_id == target.endpoint_id
|
|
assert result.health is not None and result.health.success
|
|
assert result.model_listing is not None and result.model_listing.success
|
|
assert result.json_schema is not None and result.json_schema.success
|
|
assert result.usage_metadata is not None and result.usage_metadata.success
|
|
assert result.seed_determinism is not None and result.seed_determinism.success
|
|
assert result.output_token_field is not None
|
|
assert result.software_version == "0.6.1"
|
|
assert result.probe_duration_ms >= 0
|
|
await prober.close()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_full_probe_stops_on_health_failure(self):
|
|
"""Full probe short-circuits when health fails."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise httpx.ConnectError("refused")
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
http = httpx.AsyncClient(transport=transport)
|
|
prober = EndpointProber(http_client=http)
|
|
target = _make_target()
|
|
|
|
result = await prober.run_full_probe(target)
|
|
|
|
assert result.health is not None and not result.health.success
|
|
# Other probes should not have been run
|
|
assert result.model_listing is None
|
|
assert result.json_schema is None
|
|
assert result.usage_metadata is None
|
|
assert result.seed_determinism is None
|
|
assert result.output_token_field is None
|
|
await prober.close()
|