Celes Renata
a72f336ad1
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
287 lines
10 KiB
Python
287 lines
10 KiB
Python
"""Seed migration helpers for the inference registry.
|
|
|
|
Provides canonical endpoint profiles, model deployments, and agent
|
|
provider-to-stage-binding conversion logic for migrating from the legacy
|
|
model_provider/model_name fields to the v3 registry.
|
|
|
|
Task 18.1-18.5: Migrate existing provider records.
|
|
Requirements: 3.8, 3.9
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from uuid import UUID
|
|
|
|
# ─── Well-known IDs ────────────────────────────────────────────────────────────
|
|
# These match the SQL seed migration (042_seed_inference_registry.sql)
|
|
|
|
OLLAMA_ENDPOINT_ID = UUID("a0000000-0000-4000-8000-000000000001")
|
|
VLLM_ENDPOINT_ID = UUID("a0000000-0000-4000-8000-000000000002")
|
|
OLLAMA_DEPLOYMENT_ID = UUID("b0000000-0000-4000-8000-000000000001")
|
|
VLLM_DEPLOYMENT_ID = UUID("b0000000-0000-4000-8000-000000000002")
|
|
|
|
|
|
def get_initial_endpoints() -> list[dict[str, Any]]:
|
|
"""Return the canonical endpoint profiles for the seed migration.
|
|
|
|
Returns:
|
|
List of endpoint dicts matching the inference_endpoints table schema.
|
|
- stonks-ollama: Ollama native protocol at cluster-internal URL.
|
|
- stonks-vllm: OpenAI-chat protocol (vLLM) at cluster-internal URL.
|
|
"""
|
|
return [
|
|
{
|
|
"id": OLLAMA_ENDPOINT_ID,
|
|
"name": "stonks-ollama",
|
|
"protocol": "ollama_native",
|
|
"base_url": "http://ollama.ollama-service.svc.cluster.local:11434",
|
|
"auth_secret_ref": None,
|
|
"auth_scheme": "none",
|
|
"default_headers": {},
|
|
"health_path": "/api/tags",
|
|
"enabled": True,
|
|
},
|
|
{
|
|
"id": VLLM_ENDPOINT_ID,
|
|
"name": "stonks-vllm",
|
|
"protocol": "openai_chat",
|
|
"base_url": "http://kube-vllm.stonks-oracle.svc.cluster.local:8000",
|
|
"auth_secret_ref": None,
|
|
"auth_scheme": "none",
|
|
"default_headers": {},
|
|
"health_path": "/health",
|
|
"enabled": True,
|
|
},
|
|
]
|
|
|
|
|
|
def get_initial_deployments() -> list[dict[str, Any]]:
|
|
"""Return the initial model deployments for the seed migration.
|
|
|
|
Returns:
|
|
List of deployment dicts matching the model_deployments table schema.
|
|
- Ollama: qwen3.5:9b with native JSON mode.
|
|
- vLLM: AxionML/Qwen3.5-9B-NVFP4 on RTX 4070 Ti SUPER, strict JSON Schema.
|
|
"""
|
|
return [
|
|
{
|
|
"id": OLLAMA_DEPLOYMENT_ID,
|
|
"endpoint_id": OLLAMA_ENDPOINT_ID,
|
|
"served_model_name": "qwen3.5:9b",
|
|
"display_name": "Qwen 3.5 9B (Ollama)",
|
|
"capabilities": {
|
|
"chat_completions": True,
|
|
"json_schema": False,
|
|
"json_object": True,
|
|
"seed": False,
|
|
"usage": False,
|
|
"max_completion_tokens": False,
|
|
"model_listing": True,
|
|
},
|
|
"context_window": 32768,
|
|
"max_output_tokens": 32768,
|
|
"quantization": None,
|
|
"runtime_metadata": {
|
|
"source": "ollama_native",
|
|
"notes": "Ollama-served model with native JSON mode",
|
|
},
|
|
"enabled": True,
|
|
},
|
|
{
|
|
"id": VLLM_DEPLOYMENT_ID,
|
|
"endpoint_id": VLLM_ENDPOINT_ID,
|
|
"served_model_name": "AxionML/Qwen3.5-9B-NVFP4",
|
|
"display_name": "Qwen 3.5 9B NVFP4 (vLLM)",
|
|
"capabilities": {
|
|
"chat_completions": True,
|
|
"json_schema": True,
|
|
"json_object": True,
|
|
"seed": True,
|
|
"usage": True,
|
|
"max_completion_tokens": True,
|
|
"model_listing": True,
|
|
},
|
|
"context_window": 8192,
|
|
"max_output_tokens": 2048,
|
|
"quantization": "NVFP4",
|
|
"runtime_metadata": {
|
|
"gpu": "RTX 4070 Ti SUPER",
|
|
"gpu_memory_utilization": 0.80,
|
|
"max_num_seqs": 8,
|
|
"vllm_structured_outputs": True,
|
|
},
|
|
"enabled": True,
|
|
},
|
|
]
|
|
|
|
|
|
# ─── Provider mapping ──────────────────────────────────────────────────────────
|
|
# Maps legacy model_provider values to endpoint/deployment IDs.
|
|
|
|
_PROVIDER_TO_ENDPOINT: dict[str, UUID] = {
|
|
"ollama": OLLAMA_ENDPOINT_ID,
|
|
"vllm": VLLM_ENDPOINT_ID,
|
|
}
|
|
|
|
_PROVIDER_TO_DEPLOYMENT: dict[str, UUID] = {
|
|
"ollama": OLLAMA_DEPLOYMENT_ID,
|
|
"vllm": VLLM_DEPLOYMENT_ID,
|
|
}
|
|
|
|
|
|
class UnknownProviderError(ValueError):
|
|
"""Raised when an agent record has an unrecognized model_provider value."""
|
|
|
|
def __init__(self, provider: str, agent_id: str) -> None:
|
|
self.provider = provider
|
|
self.agent_id = agent_id
|
|
super().__init__(
|
|
f"Unknown model_provider '{provider}' for agent '{agent_id}'. "
|
|
f"Supported providers: {sorted(_PROVIDER_TO_ENDPOINT.keys())}. "
|
|
f"Cannot silently convert unknown providers."
|
|
)
|
|
|
|
|
|
def convert_agent_providers(agent_records: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
"""Convert existing agent model_provider/model_name fields to stage bindings.
|
|
|
|
For each agent record with a known model_provider (ollama or vllm), produces
|
|
a stage binding record that maps the agent to the appropriate endpoint and
|
|
model deployment. The original model_provider and model_name fields are
|
|
retained for backward compatibility — this function supplements, not replaces.
|
|
|
|
Args:
|
|
agent_records: List of dicts with at least 'id' (or 'agent_id'),
|
|
'model_provider', and optionally 'slug' fields.
|
|
|
|
Returns:
|
|
List of stage binding dicts suitable for insertion into
|
|
agent_stage_bindings.
|
|
|
|
Raises:
|
|
UnknownProviderError: If a record has a model_provider that cannot be
|
|
mapped. Unknown providers must NOT be silently converted.
|
|
"""
|
|
bindings: list[dict[str, Any]] = []
|
|
|
|
for record in agent_records:
|
|
agent_id = str(record.get("id") or record.get("agent_id", ""))
|
|
provider = (record.get("model_provider") or "").strip().lower()
|
|
|
|
if not provider:
|
|
# No provider set — skip, nothing to convert
|
|
continue
|
|
|
|
if provider not in _PROVIDER_TO_ENDPOINT:
|
|
raise UnknownProviderError(provider=provider, agent_id=agent_id)
|
|
|
|
endpoint_id = _PROVIDER_TO_ENDPOINT[provider]
|
|
deployment_id = _PROVIDER_TO_DEPLOYMENT[provider]
|
|
|
|
# Determine stage from agent slug or default to 'extraction'
|
|
slug = record.get("slug", "")
|
|
stage = _infer_stage_from_slug(slug)
|
|
|
|
bindings.append({
|
|
"agent_id": agent_id,
|
|
"stage": stage,
|
|
"endpoint_id": endpoint_id,
|
|
"model_deployment_id": str(deployment_id),
|
|
"route_order": 0,
|
|
"routing_config": {},
|
|
"is_active": True,
|
|
})
|
|
|
|
return bindings
|
|
|
|
|
|
def _infer_stage_from_slug(slug: str) -> str:
|
|
"""Map an agent slug to a pipeline stage name.
|
|
|
|
Known agent slugs and their corresponding stages:
|
|
- document-extractor -> extraction
|
|
- event-classifier -> classification
|
|
- thesis-rewriter -> thesis_rewrite
|
|
- report-summarizer -> summarization
|
|
|
|
Falls back to 'extraction' for unrecognized slugs.
|
|
"""
|
|
slug_to_stage: dict[str, str] = {
|
|
"document-extractor": "extraction",
|
|
"event-classifier": "classification",
|
|
"thesis-rewriter": "thesis_rewrite",
|
|
"report-summarizer": "summarization",
|
|
}
|
|
return slug_to_stage.get(slug, "extraction")
|
|
|
|
|
|
# ─── Conflicting defaults identification ──────────────────────────────────────
|
|
|
|
# Known locations where model/provider defaults have historically conflicted.
|
|
_KNOWN_CONFLICT_LOCATIONS: list[dict[str, str]] = [
|
|
{
|
|
"location": "services/shared/config.py",
|
|
"field": "VLLMConfig.model",
|
|
"description": "Python config default for vLLM model name",
|
|
},
|
|
{
|
|
"location": "services/shared/config.py",
|
|
"field": "VLLMConfig.base_url",
|
|
"description": "Python config default for vLLM base URL (192.168.42.254:8000)",
|
|
},
|
|
{
|
|
"location": "infra/migrations/026_ai_agents.sql",
|
|
"field": "model_provider/model_name DEFAULT",
|
|
"description": "Agent table DDL defaults to 'ollama'/'qwen3.5:9b-fast'",
|
|
},
|
|
{
|
|
"location": "infra/migrations/031_fix_agent_defaults.sql",
|
|
"field": "model_provider UPDATE",
|
|
"description": "Migration updates agents to 'vllm'/'AxionML/Qwen3.5-9B-NVFP4'",
|
|
},
|
|
{
|
|
"location": "infra/migrations/033_stop_hardcoding_agent_model.sql",
|
|
"field": "model_provider UPDATE",
|
|
"description": "Migration updates agents to 'vllm'/'AxionML/Qwen3.5-9B-NVFP4'",
|
|
},
|
|
{
|
|
"location": "infra/helm/stonks-oracle/values.yaml",
|
|
"field": "VLLM_BASE_URL / VLLM_MODEL",
|
|
"description": "Helm values point to nuextract-external.vllm-service with NuExtract3",
|
|
},
|
|
{
|
|
"location": "infra/helm/stonks-oracle/values.yaml",
|
|
"field": "OLLAMA_BASE_URL / OLLAMA_MODEL",
|
|
"description": "Helm values point to nuextract-external.vllm-service with NuExtract3",
|
|
},
|
|
{
|
|
"location": "infra/kube-vllm/deployment.yaml",
|
|
"field": "vLLM deployment model arg",
|
|
"description": "Standalone kube-vllm deployment with AxionML/Qwen3.5-9B-NVFP4",
|
|
},
|
|
]
|
|
|
|
|
|
def identify_conflicting_defaults() -> list[str]:
|
|
"""List locations where model defaults historically conflict.
|
|
|
|
Returns a list of human-readable strings identifying places where
|
|
the model provider, model name, or base URL have conflicting
|
|
defaults across config.py, migrations, Helm values, and the
|
|
kube-vllm deployment.
|
|
|
|
These conflicts should be resolved after the inference registry
|
|
is established as the single source of truth.
|
|
|
|
Returns:
|
|
List of conflict description strings.
|
|
"""
|
|
conflicts: list[str] = []
|
|
|
|
for entry in _KNOWN_CONFLICT_LOCATIONS:
|
|
conflicts.append(
|
|
f"{entry['location']} [{entry['field']}]: {entry['description']}"
|
|
)
|
|
|
|
return conflicts
|