"""Tests for capability probing module. Tests against mocked HTTP transport validating all probe behaviors, result storage with TTL, and required-capability validation. Requirements: 2.10, 2.11 """ from __future__ import annotations import json import time from uuid import uuid4 import httpx import pytest from services.shared.inference.capabilities import ( DEFAULT_PROBE_TTL_SECONDS, EndpointProber, FullProbeResult, HealthProbeResult, JsonSchemaProbeResult, ModelListingResult, OutputTokenFieldResult, ProbeResultStore, SeedProbeResult, UsageProbeResult, validate_required_capabilities, ) from services.shared.inference.models import ( InferenceTarget, ProviderCapabilities, ) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _make_target( *, json_schema: bool = True, json_object: bool = False, seed: bool = True, usage: bool = True, max_completion_tokens: bool = False, model_listing: bool = True, model: str = "test-model", base_url: str = "http://test-endpoint:8000", auth_secret_ref: str | None = None, ) -> InferenceTarget: """Build a test InferenceTarget.""" return InferenceTarget( endpoint_id=uuid4(), deployment_id=uuid4(), protocol="openai_chat", base_url=base_url, model=model, capabilities=ProviderCapabilities( chat_completions=True, json_schema=json_schema, json_object=json_object, seed=seed, usage=usage, max_completion_tokens=max_completion_tokens, model_listing=model_listing, ), auth_secret_ref=auth_secret_ref, ) def _chat_response( content: str = '{"status": "ok"}', *, usage: dict | None = None, finish_reason: str = "stop", status: int = 200, headers: dict | None = None, ) -> httpx.Response: """Build a mock OpenAI-compatible chat completions response.""" body: dict = { "id": "chatcmpl-probe", "object": "chat.completion", "choices": [ { "index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": finish_reason, } ], } if usage: body["usage"] = usage resp_headers = headers or {} return httpx.Response(status, json=body, headers=resp_headers) def _models_response( model_ids: list[str] | None = None, status: int = 200, headers: dict | None = None, ) -> httpx.Response: """Build a mock /v1/models response.""" if model_ids is None: model_ids = ["test-model"] body = { "data": [{"id": mid, "object": "model"} for mid in model_ids], } return httpx.Response(status, json=body, headers=headers or {}) # =========================================================================== # 13.1: Probe health and model listing # =========================================================================== class TestProbeHealth: """Tests for probe_health method.""" @pytest.mark.asyncio async def test_health_success_via_health_path(self): """Health probe succeeds when /health returns 200.""" def handler(request: httpx.Request) -> httpx.Response: if "/health" in str(request.url): return httpx.Response(200, json={"status": "ok"}) return httpx.Response(404) transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_health(target) assert result.success is True assert "/health" in result.detail assert result.latency_ms >= 0 await prober.close() @pytest.mark.asyncio async def test_health_success_via_models_fallback(self): """Health probe falls back to /v1/models when /health fails.""" def handler(request: httpx.Request) -> httpx.Response: url = str(request.url) if url.endswith("/health"): return httpx.Response(500) if "/v1/models" in url: return httpx.Response( 200, json={"data": []}, headers={"server": "vllm/0.6.0"}, ) return httpx.Response(500) transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_health(target) assert result.success is True assert "/v1/models" in result.detail assert result.software_version == "vllm/0.6.0" await prober.close() @pytest.mark.asyncio async def test_health_failure_connection_refused(self): """Health probe fails when endpoint is unreachable.""" def handler(request: httpx.Request) -> httpx.Response: raise httpx.ConnectError("Connection refused") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_health(target) assert result.success is False assert "Connection failed" in result.detail await prober.close() @pytest.mark.asyncio async def test_health_failure_timeout(self): """Health probe fails on timeout.""" def handler(request: httpx.Request) -> httpx.Response: raise httpx.ReadTimeout("timed out") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_health(target) assert result.success is False assert "Connection failed" in result.detail await prober.close() class TestProbeModelListing: """Tests for probe_model_listing method.""" @pytest.mark.asyncio async def test_model_listing_success_with_target_model(self): """Model listing succeeds and finds the target model.""" def handler(request: httpx.Request) -> httpx.Response: return _models_response(["test-model", "other-model"]) transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_model_listing(target) assert result.success is True assert result.target_model_found is True assert "test-model" in result.models assert "other-model" in result.models await prober.close() @pytest.mark.asyncio async def test_model_listing_model_not_found(self): """Model listing succeeds but target model is not in the list.""" def handler(request: httpx.Request) -> httpx.Response: return _models_response(["other-model-a", "other-model-b"]) transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_model_listing(target) assert result.success is True assert result.target_model_found is False await prober.close() @pytest.mark.asyncio async def test_model_listing_http_error(self): """Model listing fails on non-200 response.""" def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(403, text="Forbidden") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_model_listing(target) assert result.success is False assert "403" in result.detail await prober.close() @pytest.mark.asyncio async def test_model_listing_connection_error(self): """Model listing fails when endpoint is unreachable.""" def handler(request: httpx.Request) -> httpx.Response: raise httpx.ConnectError("refused") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_model_listing(target) assert result.success is False assert "Connection failed" in result.detail await prober.close() # =========================================================================== # 13.2: Probe strict JSON Schema with a minimal schema # =========================================================================== class TestProbeJsonSchema: """Tests for probe_json_schema method.""" @pytest.mark.asyncio async def test_json_schema_probe_success(self): """Schema probe succeeds when endpoint returns valid schema-conforming JSON.""" def handler(request: httpx.Request) -> httpx.Response: payload = json.loads(request.content) # Verify request includes response_format assert payload["response_format"]["type"] == "json_schema" return _chat_response('{"status": "ok"}') transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_json_schema(target) assert result.success is True assert result.schema_valid is True assert result.structured_mode_used == "json_schema" await prober.close() @pytest.mark.asyncio async def test_json_schema_probe_invalid_response(self): """Schema probe fails when response doesn't match expected schema.""" def handler(request: httpx.Request) -> httpx.Response: # Returns JSON but wrong schema (missing "status" field) return _chat_response('{"wrong_field": 123}') transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_json_schema(target) assert result.success is False assert result.schema_valid is False await prober.close() @pytest.mark.asyncio async def test_json_schema_probe_non_json_response(self): """Schema probe fails when response is not valid JSON.""" def handler(request: httpx.Request) -> httpx.Response: return _chat_response("This is plain text, not JSON") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_json_schema(target) assert result.success is False assert result.schema_valid is False assert "not valid JSON" in result.detail await prober.close() @pytest.mark.asyncio async def test_json_schema_probe_http_error(self): """Schema probe fails on non-200 response.""" def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(400, json={"error": "bad request"}) transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_json_schema(target) assert result.success is False assert "400" in result.detail await prober.close() @pytest.mark.asyncio async def test_json_schema_probe_empty_choices(self): """Schema probe fails when response has empty choices.""" def handler(request: httpx.Request) -> httpx.Response: body = {"choices": []} return httpx.Response(200, json=body) transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_json_schema(target) assert result.success is False assert "Empty choices" in result.detail await prober.close() @pytest.mark.asyncio async def test_json_schema_probe_connection_error(self): """Schema probe fails on connection error.""" def handler(request: httpx.Request) -> httpx.Response: raise httpx.ConnectError("refused") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_json_schema(target) assert result.success is False assert "Connection failed" in result.detail await prober.close() # =========================================================================== # 13.3: Probe usage metadata, seed behavior, and output-token field # =========================================================================== class TestProbeUsageMetadata: """Tests for probe_usage_metadata method.""" @pytest.mark.asyncio async def test_usage_metadata_both_present(self): """Usage probe succeeds when both token counts are present.""" def handler(request: httpx.Request) -> httpx.Response: return _chat_response( "hi", usage={"prompt_tokens": 10, "completion_tokens": 5}, ) transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_usage_metadata(target) assert result.success is True assert result.has_prompt_tokens is True assert result.has_completion_tokens is True await prober.close() @pytest.mark.asyncio async def test_usage_metadata_none_present(self): """Usage probe fails when no usage tokens are returned.""" def handler(request: httpx.Request) -> httpx.Response: return _chat_response("hi") # No usage field transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_usage_metadata(target) assert result.success is False assert result.has_prompt_tokens is False assert result.has_completion_tokens is False await prober.close() @pytest.mark.asyncio async def test_usage_metadata_http_error(self): """Usage probe fails on HTTP error.""" def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(500, text="Internal error") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_usage_metadata(target) assert result.success is False assert "500" in result.detail await prober.close() class TestProbeSeedDeterminism: """Tests for probe_seed_determinism method.""" @pytest.mark.asyncio async def test_seed_determinism_outputs_match(self): """Seed probe reports match when outputs are identical.""" def handler(request: httpx.Request) -> httpx.Response: return _chat_response("hello") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_seed_determinism(target) assert result.success is True assert result.outputs_match is True assert "match" in result.detail await prober.close() @pytest.mark.asyncio async def test_seed_determinism_outputs_differ(self): """Seed probe reports no match when outputs differ.""" call_count = 0 def handler(request: httpx.Request) -> httpx.Response: nonlocal call_count call_count += 1 content = f"response-{call_count}" return _chat_response(content) transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_seed_determinism(target) assert result.success is True assert result.outputs_match is False assert "differ" in result.detail await prober.close() @pytest.mark.asyncio async def test_seed_determinism_http_error(self): """Seed probe fails on HTTP error.""" def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(500, text="error") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_seed_determinism(target) assert result.success is False assert "500" in result.detail await prober.close() class TestProbeOutputTokenField: """Tests for probe_output_token_field method.""" @pytest.mark.asyncio async def test_output_token_field_accepted(self): """Output token field probe succeeds when server returns 200.""" def handler(request: httpx.Request) -> httpx.Response: payload = json.loads(request.content) assert "max_completion_tokens" in payload return _chat_response("hi") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_output_token_field(target) assert result.success is True assert result.field_accepted is True await prober.close() @pytest.mark.asyncio async def test_output_token_field_rejected(self): """Output token field probe fails when server rejects the field.""" def handler(request: httpx.Request) -> httpx.Response: return httpx.Response( 400, json={"error": "unknown field: max_completion_tokens"}, ) transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_output_token_field(target) assert result.success is False assert result.field_accepted is False assert "rejected" in result.detail await prober.close() @pytest.mark.asyncio async def test_output_token_field_connection_error(self): """Output token field probe fails on connection error.""" def handler(request: httpx.Request) -> httpx.Response: raise httpx.ConnectError("refused") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.probe_output_token_field(target) assert result.success is False assert "Connection failed" in result.detail await prober.close() # =========================================================================== # 13.4: Store probe results and software/version metadata with TTL # =========================================================================== class TestProbeResultStore: """Tests for ProbeResultStore TTL-based cache.""" def test_store_and_retrieve(self): """Can store and retrieve a probe result within TTL.""" store = ProbeResultStore(ttl_seconds=60) endpoint_id = uuid4() result = FullProbeResult( endpoint_id=endpoint_id, health=HealthProbeResult(success=True, detail="ok"), software_version="vllm/0.6.0", ) store.store(endpoint_id, result) retrieved = store.get(endpoint_id) assert retrieved is not None assert retrieved.endpoint_id == endpoint_id assert retrieved.software_version == "vllm/0.6.0" def test_retrieve_nonexistent_returns_none(self): """Getting a non-existent entry returns None.""" store = ProbeResultStore(ttl_seconds=60) assert store.get(uuid4()) is None def test_ttl_expiry(self, monkeypatch): """Expired entries return None on retrieval.""" store = ProbeResultStore(ttl_seconds=1) endpoint_id = uuid4() result = FullProbeResult(endpoint_id=endpoint_id) store.store(endpoint_id, result) # Monkey-patch time.monotonic to simulate TTL expiry original_monotonic = time.monotonic start = original_monotonic() monkeypatch.setattr( time, "monotonic", lambda: start + 2.0 ) # Need to re-store with a fixed time reference # Actually, let's use a different approach: store, then move time forward store._store[endpoint_id] = (start - 2.0, result) retrieved = store.get(endpoint_id) assert retrieved is None # Entry should be evicted assert endpoint_id not in store._store def test_invalidate(self): """Invalidate removes an entry.""" store = ProbeResultStore(ttl_seconds=60) endpoint_id = uuid4() result = FullProbeResult(endpoint_id=endpoint_id) store.store(endpoint_id, result) store.invalidate(endpoint_id) assert store.get(endpoint_id) is None def test_invalidate_nonexistent_is_safe(self): """Invalidating a non-existent entry doesn't raise.""" store = ProbeResultStore(ttl_seconds=60) store.invalidate(uuid4()) # Should not raise def test_clear(self): """Clear removes all entries.""" store = ProbeResultStore(ttl_seconds=60) for _ in range(5): eid = uuid4() store.store(eid, FullProbeResult(endpoint_id=eid)) assert len(store) == 5 store.clear() assert len(store) == 0 def test_default_ttl(self): """Default TTL is 5 minutes.""" store = ProbeResultStore() assert store.ttl_seconds == DEFAULT_PROBE_TTL_SECONDS assert store.ttl_seconds == 300 def test_software_version_stored(self): """Software version metadata is preserved in stored results.""" store = ProbeResultStore(ttl_seconds=60) endpoint_id = uuid4() result = FullProbeResult( endpoint_id=endpoint_id, health=HealthProbeResult( success=True, software_version="vllm/0.6.1", ), software_version="vllm/0.6.1", ) store.store(endpoint_id, result) retrieved = store.get(endpoint_id) assert retrieved is not None assert retrieved.software_version == "vllm/0.6.1" assert retrieved.health is not None assert retrieved.health.software_version == "vllm/0.6.1" # =========================================================================== # 13.5: Refuse activation when declared required capabilities fail # =========================================================================== class TestValidateRequiredCapabilities: """Tests for validate_required_capabilities function.""" def test_all_capabilities_pass(self): """No failures when all probes pass for declared capabilities.""" target = _make_target( json_schema=True, seed=True, usage=True, max_completion_tokens=True, model_listing=True, ) probe_result = FullProbeResult( endpoint_id=target.endpoint_id, health=HealthProbeResult(success=True, detail="ok"), model_listing=ModelListingResult( success=True, target_model_found=True, models=["test-model"] ), json_schema=JsonSchemaProbeResult( success=True, schema_valid=True ), usage_metadata=UsageProbeResult( success=True, has_prompt_tokens=True, has_completion_tokens=True ), seed_determinism=SeedProbeResult( success=True, outputs_match=True ), output_token_field=OutputTokenFieldResult( success=True, field_accepted=True ), ) failures = validate_required_capabilities(target, probe_result) assert failures == [] def test_health_failure_blocks_all(self): """Health failure returns immediately without checking others.""" target = _make_target() probe_result = FullProbeResult( endpoint_id=target.endpoint_id, health=HealthProbeResult( success=False, detail="Connection refused" ), ) failures = validate_required_capabilities(target, probe_result) assert len(failures) == 1 assert "Health check failed" in failures[0] def test_json_schema_failure(self): """JSON schema failure is reported when declared.""" target = _make_target(json_schema=True) probe_result = FullProbeResult( endpoint_id=target.endpoint_id, health=HealthProbeResult(success=True), model_listing=ModelListingResult( success=True, target_model_found=True, models=["test-model"] ), json_schema=JsonSchemaProbeResult( success=False, detail="HTTP 400" ), usage_metadata=UsageProbeResult(success=True), seed_determinism=SeedProbeResult( success=True, outputs_match=True ), ) failures = validate_required_capabilities(target, probe_result) assert any("JSON Schema" in f for f in failures) def test_json_schema_passes_but_invalid_response(self): """Schema probe succeeded but response didn't validate.""" target = _make_target(json_schema=True) probe_result = FullProbeResult( endpoint_id=target.endpoint_id, health=HealthProbeResult(success=True), model_listing=ModelListingResult( success=True, target_model_found=True, models=["test-model"] ), json_schema=JsonSchemaProbeResult( success=True, schema_valid=False ), usage_metadata=UsageProbeResult(success=True), seed_determinism=SeedProbeResult( success=True, outputs_match=True ), ) failures = validate_required_capabilities(target, probe_result) assert any("did not validate" in f for f in failures) def test_model_not_found_in_listing(self): """Model not found in endpoint listing is a failure.""" target = _make_target(model_listing=True) probe_result = FullProbeResult( endpoint_id=target.endpoint_id, health=HealthProbeResult(success=True), model_listing=ModelListingResult( success=True, target_model_found=False, models=["other-model"], ), json_schema=JsonSchemaProbeResult(success=True, schema_valid=True), usage_metadata=UsageProbeResult(success=True), seed_determinism=SeedProbeResult( success=True, outputs_match=True ), ) failures = validate_required_capabilities(target, probe_result) assert any("not found" in f for f in failures) def test_seed_declared_but_outputs_differ(self): """Seed failure when outputs don't match.""" target = _make_target(seed=True, model_listing=False) probe_result = FullProbeResult( endpoint_id=target.endpoint_id, health=HealthProbeResult(success=True), json_schema=JsonSchemaProbeResult(success=True, schema_valid=True), usage_metadata=UsageProbeResult(success=True), seed_determinism=SeedProbeResult( success=True, outputs_match=False ), ) failures = validate_required_capabilities(target, probe_result) assert any("outputs differ" in f for f in failures) def test_usage_not_available(self): """Usage failure when declared but probe finds no tokens.""" target = _make_target(usage=True, model_listing=False, seed=False) probe_result = FullProbeResult( endpoint_id=target.endpoint_id, health=HealthProbeResult(success=True), json_schema=JsonSchemaProbeResult(success=True, schema_valid=True), usage_metadata=UsageProbeResult( success=False, detail="No usage in response" ), ) failures = validate_required_capabilities(target, probe_result) assert any("Usage metadata" in f for f in failures) def test_max_completion_tokens_not_supported(self): """max_completion_tokens failure when declared but not accepted.""" target = _make_target( max_completion_tokens=True, model_listing=False, seed=False, usage=False, json_schema=False, ) probe_result = FullProbeResult( endpoint_id=target.endpoint_id, health=HealthProbeResult(success=True), output_token_field=OutputTokenFieldResult( success=False, detail="HTTP 400" ), ) failures = validate_required_capabilities(target, probe_result) assert any("max_completion_tokens" in f for f in failures) def test_undeclared_capabilities_not_checked(self): """Capabilities not declared in target are not validated.""" # Target declares NO capabilities except chat_completions target = _make_target( json_schema=False, json_object=False, seed=False, usage=False, max_completion_tokens=False, model_listing=False, ) # Even though probes are missing/failed, no failures reported probe_result = FullProbeResult( endpoint_id=target.endpoint_id, health=HealthProbeResult(success=True), ) failures = validate_required_capabilities(target, probe_result) assert failures == [] def test_multiple_failures_reported(self): """Multiple capability failures are all reported.""" target = _make_target( json_schema=True, seed=True, usage=True, max_completion_tokens=True, model_listing=True, ) # Everything fails probe_result = FullProbeResult( endpoint_id=target.endpoint_id, health=HealthProbeResult(success=True), model_listing=ModelListingResult(success=False, detail="error"), json_schema=JsonSchemaProbeResult(success=False, detail="error"), usage_metadata=UsageProbeResult(success=False, detail="error"), seed_determinism=SeedProbeResult(success=False, detail="error"), output_token_field=OutputTokenFieldResult( success=False, detail="error" ), ) failures = validate_required_capabilities(target, probe_result) # Should report all failures assert len(failures) >= 4 # =========================================================================== # Integration: run_full_probe # =========================================================================== class TestRunFullProbe: """Tests for the full probe orchestration.""" @pytest.mark.asyncio async def test_full_probe_all_pass(self): """Full probe runs all sub-probes and returns aggregated results.""" def handler(request: httpx.Request) -> httpx.Response: url = str(request.url) if "/health" in url: return httpx.Response( 200, json={"status": "ok"}, headers={"x-vllm-version": "0.6.1"}, ) if "/v1/models" in url and request.method == "GET": return _models_response( ["test-model"], headers={"x-vllm-version": "0.6.1"}, ) if "/v1/chat/completions" in url: payload = json.loads(request.content) return _chat_response( '{"status": "ok"}', usage={"prompt_tokens": 5, "completion_tokens": 3}, headers={"x-vllm-version": "0.6.1"}, ) return httpx.Response(404) transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.run_full_probe(target) assert result.endpoint_id == target.endpoint_id assert result.health is not None and result.health.success assert result.model_listing is not None and result.model_listing.success assert result.json_schema is not None and result.json_schema.success assert result.usage_metadata is not None and result.usage_metadata.success assert result.seed_determinism is not None and result.seed_determinism.success assert result.output_token_field is not None assert result.software_version == "0.6.1" assert result.probe_duration_ms >= 0 await prober.close() @pytest.mark.asyncio async def test_full_probe_stops_on_health_failure(self): """Full probe short-circuits when health fails.""" def handler(request: httpx.Request) -> httpx.Response: raise httpx.ConnectError("refused") transport = httpx.MockTransport(handler) http = httpx.AsyncClient(transport=transport) prober = EndpointProber(http_client=http) target = _make_target() result = await prober.run_full_probe(target) assert result.health is not None and not result.health.success # Other probes should not have been run assert result.model_listing is None assert result.json_schema is None assert result.usage_metadata is None assert result.seed_determinism is None assert result.output_token_field is None await prober.close()