feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,191 @@
|
||||
"""Property-based tests for provider routing and protocol resolution.
|
||||
|
||||
Feature: intelligence-pipeline-v3
|
||||
|
||||
Validates that the inference factory:
|
||||
- Never silently falls back to Ollama for unknown protocols
|
||||
- Correctly resolves "vllm" to "openai_chat"
|
||||
- Fails closed with a typed error for any unrecognized protocol
|
||||
|
||||
Uses Hypothesis to verify these properties hold for arbitrary string inputs.
|
||||
|
||||
**Validates: Requirements 2.2, 2.6**
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from hypothesis import given, settings
|
||||
from hypothesis import strategies as st
|
||||
|
||||
from services.shared.inference.clients.ollama_native import OllamaNativeClient
|
||||
from services.shared.inference.clients.openai_compatible import OpenAICompatibleClient
|
||||
from services.shared.inference.errors import InferenceError, InferenceErrorCategory
|
||||
from services.shared.inference.factory import (
|
||||
KNOWN_PROTOCOLS,
|
||||
PROTOCOL_ALIASES,
|
||||
create_client,
|
||||
resolve_protocol,
|
||||
)
|
||||
from services.shared.inference.models import InferenceTarget, ProviderCapabilities
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_target(protocol: str) -> InferenceTarget:
|
||||
"""Build a minimal InferenceTarget for testing protocol routing."""
|
||||
return InferenceTarget(
|
||||
endpoint_id=uuid4(),
|
||||
deployment_id=uuid4(),
|
||||
protocol=protocol, # type: ignore[arg-type]
|
||||
base_url="http://localhost:11434",
|
||||
model="test-model",
|
||||
capabilities=ProviderCapabilities(
|
||||
chat_completions=True,
|
||||
json_schema=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _is_known_or_alias(s: str) -> bool:
|
||||
"""Check if a string is a known protocol or alias after normalization."""
|
||||
normalized = s.strip().lower()
|
||||
return normalized in KNOWN_PROTOCOLS or normalized in PROTOCOL_ALIASES
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Property tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPropertyUnknownProtocolsFailClosed:
|
||||
"""Property: Unknown protocols always raise InferenceError.
|
||||
|
||||
For any string that is NOT in PROTOCOL_ALIASES keys AND not a known
|
||||
protocol, resolve_protocol() MUST raise InferenceError with
|
||||
CAPABILITY_UNAVAILABLE category.
|
||||
|
||||
**Validates: Requirements 2.6**
|
||||
"""
|
||||
|
||||
@given(protocol=st.text(min_size=0, max_size=50))
|
||||
@settings(max_examples=100)
|
||||
def test_unknown_protocol_raises_error(self, protocol: str):
|
||||
"""**Validates: Requirements 2.6**
|
||||
|
||||
Any string not in known protocols or aliases must raise InferenceError.
|
||||
"""
|
||||
if _is_known_or_alias(protocol):
|
||||
return # skip known protocols — those should resolve fine
|
||||
|
||||
try:
|
||||
resolve_protocol(protocol)
|
||||
raise AssertionError(
|
||||
f"resolve_protocol({protocol!r}) did not raise for unknown protocol"
|
||||
)
|
||||
except InferenceError as exc:
|
||||
assert exc.category == InferenceErrorCategory.CAPABILITY_UNAVAILABLE, (
|
||||
f"Expected CAPABILITY_UNAVAILABLE, got {exc.category} for {protocol!r}"
|
||||
)
|
||||
|
||||
|
||||
class TestPropertyUnknownProtocolNeverProducesOllama:
|
||||
"""Property: No unknown protocol input to create_client() produces an Ollama client.
|
||||
|
||||
For any string that is NOT a known protocol or alias, create_client()
|
||||
must raise InferenceError — it must NEVER return an OllamaNativeClient
|
||||
as a silent fallback.
|
||||
|
||||
**Validates: Requirements 2.6**
|
||||
"""
|
||||
|
||||
@given(protocol=st.text(min_size=0, max_size=50))
|
||||
@settings(max_examples=100)
|
||||
def test_unknown_protocol_never_returns_ollama_client(self, protocol: str):
|
||||
"""**Validates: Requirements 2.6**
|
||||
|
||||
create_client() with an unknown protocol must never return an
|
||||
OllamaNativeClient instance. It must raise InferenceError.
|
||||
"""
|
||||
if _is_known_or_alias(protocol):
|
||||
return # skip valid protocols
|
||||
|
||||
target = _make_target(protocol)
|
||||
try:
|
||||
client = create_client(target)
|
||||
# If we got here, the factory returned a client for an unknown protocol
|
||||
assert not isinstance(client, OllamaNativeClient), (
|
||||
f"create_client() returned OllamaNativeClient for unknown protocol {protocol!r}"
|
||||
)
|
||||
assert not isinstance(client, OpenAICompatibleClient), (
|
||||
f"create_client() returned a client for unknown protocol {protocol!r} "
|
||||
f"instead of raising InferenceError"
|
||||
)
|
||||
raise AssertionError(
|
||||
f"create_client() returned {type(client)} for unknown protocol {protocol!r} "
|
||||
f"instead of raising InferenceError"
|
||||
)
|
||||
except InferenceError:
|
||||
pass # Expected behavior — unknown protocol fails closed
|
||||
|
||||
|
||||
class TestPropertyVLLMResolvesToOpenAIChat:
|
||||
"""Property: "vllm" always resolves to "openai_chat".
|
||||
|
||||
The backward-compatible alias must consistently map to the
|
||||
canonical openai_chat protocol regardless of whitespace or casing.
|
||||
|
||||
**Validates: Requirements 2.2**
|
||||
"""
|
||||
|
||||
@given(
|
||||
padding_left=st.text(
|
||||
alphabet=st.sampled_from([" ", "\t"]),
|
||||
min_size=0,
|
||||
max_size=5,
|
||||
),
|
||||
padding_right=st.text(
|
||||
alphabet=st.sampled_from([" ", "\t"]),
|
||||
min_size=0,
|
||||
max_size=5,
|
||||
),
|
||||
)
|
||||
@settings(max_examples=100)
|
||||
def test_vllm_always_resolves_to_openai_chat(
|
||||
self, padding_left: str, padding_right: str
|
||||
):
|
||||
"""**Validates: Requirements 2.2**
|
||||
|
||||
"vllm" with arbitrary surrounding whitespace always resolves to "openai_chat".
|
||||
"""
|
||||
import warnings as _warnings
|
||||
|
||||
protocol_input = f"{padding_left}vllm{padding_right}"
|
||||
with _warnings.catch_warnings():
|
||||
_warnings.simplefilter("ignore", DeprecationWarning)
|
||||
result = resolve_protocol(protocol_input)
|
||||
|
||||
assert result == "openai_chat", (
|
||||
f"Expected 'openai_chat' for input {protocol_input!r}, got {result!r}"
|
||||
)
|
||||
|
||||
@given(
|
||||
case_variant=st.sampled_from(["vllm", "VLLM", "Vllm", "vLLM", "VlLm"]),
|
||||
)
|
||||
@settings(max_examples=100)
|
||||
def test_vllm_case_insensitive(self, case_variant: str):
|
||||
"""**Validates: Requirements 2.2**
|
||||
|
||||
"vllm" in any case resolves to "openai_chat".
|
||||
"""
|
||||
import warnings as _warnings
|
||||
|
||||
with _warnings.catch_warnings():
|
||||
_warnings.simplefilter("ignore", DeprecationWarning)
|
||||
result = resolve_protocol(case_variant)
|
||||
|
||||
assert result == "openai_chat", (
|
||||
f"Expected 'openai_chat' for input {case_variant!r}, got {result!r}"
|
||||
)
|
||||
Reference in New Issue
Block a user