admin/stonks-oracle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Intelligence Pipeline v3 — full implementation

Browse Source 
Multi-stage evidence-grounded inference architecture replacing the
monolithic 9B model extraction pipeline. CPU-first specialist services
handle routine extraction while the 9B vLLM model is preserved for
semantic adjudication of ambiguous cases.

Key components:
- Capability-aware inference gateway (OpenAI-compatible + Ollama)
- Endpoint registry with DB migrations and REST API
- Sentence-aware document segmenter (property tests)
- Deterministic financial parsing with offset integrity
- Symbol resolution with ambiguity detection
- Specialist service (GLiNER2, dynamic batching, K8s deployment)
- Company-specific sentiment (FinBERT, calibration)
- Retrieval-based novelty and duplicate detection
- Confidence calibration pipeline
- Deterministic routing engine (property tests)
- 9B adjudication layer with VRAM gating
- Stock-specific impact model (features, labels, baseline, trained)
- Pipeline orchestrator (state machine, queues, leases, feature flags)
- Bounded parallelism (async workers, semaphore, load shedding)
- Observability (tracing, metrics, alerts)
- Compatibility adapter (v3→v2 golden mapping tests)
- Shadow/canary promotion framework
- Active learning and fine-tuning pipeline

Test results: 1,161 tests pass, ruff lint clean.
All 282 spec tasks completed.
This commit is contained in:
Celes Renata
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/intelligence_pipeline_v3/benchmark/__init__.py
+46
View File
@@ -0,0 +1,46 @@
"""Benchmark configuration and comparison framework for Intelligence Pipeline v3.
Defines extraction configurations for controlled comparison between the current
production pipeline and corrected variants. Supports attribution of improvement
sources (temperature fix, schema constraints, architecture changes).
Validates: Requirements 16.2, 16.3, 16.5
"""
from services.intelligence_pipeline_v3.benchmark.comparison import (
ComparisonReport,
ConfigDelta,
FieldDelta,
ResourceDelta,
compare_configurations,
)
from services.intelligence_pipeline_v3.benchmark.configurations import (
BASELINE_CURRENT,
BASELINE_STRICT_SCHEMA,
BASELINE_TEMP_ZERO,
BenchmarkConfig,
StructuredOutputMode,
list_configurations,
)
from services.intelligence_pipeline_v3.benchmark.runner import (
BenchmarkDocumentResult,
BenchmarkRun,
BenchmarkRunner,
)
__all__ = [
"BASELINE_CURRENT",
"BASELINE_STRICT_SCHEMA",
"BASELINE_TEMP_ZERO",
"BenchmarkConfig",
"BenchmarkDocumentResult",
"BenchmarkRun",
"BenchmarkRunner",
"ComparisonReport",
"ConfigDelta",
"FieldDelta",
"ResourceDelta",
"StructuredOutputMode",
"compare_configurations",
"list_configurations",
]
services/intelligence_pipeline_v3/benchmark/comparison.py
+294
View File
@@ -0,0 +1,294 @@
"""Comparison and attribution for benchmark configurations.
Produces delta tables and attribution reports to quantify how much of
the apparent architecture gain comes from fixing the current request alone
(temperature, schema constraints) versus the full v3 architecture.
Validates: Requirements 16.2, 16.3, 16.5
"""
from __future__ import annotations
from pydantic import BaseModel, Field
from services.intelligence_pipeline_v3.benchmark.runner import BenchmarkRun
# ---------------------------------------------------------------------------
# Result Models
# ---------------------------------------------------------------------------
class FieldDelta(BaseModel):
"""Per-field improvement between two configurations."""
field_name: str = Field(description="Name of the compared field/metric")
baseline_value: float = Field(description="Value in the baseline configuration")
comparison_value: float = Field(description="Value in the compared configuration")
absolute_delta: float = Field(description="comparison - baseline")
relative_delta_percent: float = Field(
description="Percentage change from baseline ((comp - base) / base * 100)",
)
improved: bool = Field(
description="Whether the delta represents improvement (higher is better assumed unless inverted)",
)
class ResourceDelta(BaseModel):
"""Resource usage comparison between configurations."""
metric_name: str = Field(description="Resource metric name")
baseline_value: float = Field(description="Baseline resource usage")
comparison_value: float = Field(description="Compared configuration resource usage")
absolute_delta: float = Field(description="comparison - baseline")
relative_delta_percent: float = Field(description="Percentage change")
improved: bool = Field(
description="Whether the delta represents improvement (lower is better for resources)",
)
class ConfigDelta(BaseModel):
"""Comparison results between a baseline and one other configuration."""
baseline_config: str = Field(description="Baseline configuration name")
comparison_config: str = Field(description="Compared configuration name")
field_deltas: list[FieldDelta] = Field(default_factory=list)
resource_deltas: list[ResourceDelta] = Field(default_factory=list)
class ComparisonReport(BaseModel):
"""Full comparison report across multiple configurations.
Attributes:
configs_compared: Names of all configurations in this comparison.
deltas: Per-configuration comparison against the baseline.
attribution_summary: Human-readable attribution of improvement sources.
"""
configs_compared: list[str] = Field(
description="All configuration names included in this comparison",
)
deltas: list[ConfigDelta] = Field(
default_factory=list,
description="Delta tables for each non-baseline config vs baseline",
)
attribution_summary: dict[str, float] = Field(
default_factory=dict,
description=(
"Attribution percentages: maps source (e.g. 'temperature_fix', "
"'schema_constraint', 'architecture') to fraction of total improvement"
),
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _compute_field_delta(
field_name: str,
baseline_val: float,
comparison_val: float,
*,
higher_is_better: bool = True,
) -> FieldDelta:
"""Compute a single field delta with direction awareness."""
absolute = comparison_val - baseline_val
relative = (
(absolute / baseline_val * 100.0) if baseline_val != 0.0 else 0.0
)
improved = absolute > 0.0 if higher_is_better else absolute < 0.0
return FieldDelta(
field_name=field_name,
baseline_value=baseline_val,
comparison_value=comparison_val,
absolute_delta=absolute,
relative_delta_percent=relative,
improved=improved,
)
def _compute_resource_delta(
metric_name: str,
baseline_val: float,
comparison_val: float,
) -> ResourceDelta:
"""Compute a resource delta (lower is better)."""
absolute = comparison_val - baseline_val
relative = (
(absolute / baseline_val * 100.0) if baseline_val != 0.0 else 0.0
)
improved = absolute < 0.0 # Lower resource usage is better
return ResourceDelta(
metric_name=metric_name,
baseline_value=baseline_val,
comparison_value=comparison_val,
absolute_delta=absolute,
relative_delta_percent=relative,
improved=improved,
)
def _run_metrics(run: BenchmarkRun) -> dict[str, float]:
"""Extract summary metrics from a benchmark run."""
n = len(run.results) or 1 # Avoid division by zero
return {
"schema_validity_rate": run.schema_validity_rate,
"success_count": float(run.success_count),
"failure_count": float(run.failure_count),
"mean_duration_ms": run.mean_duration_ms,
"total_input_tokens": float(run.total_input_tokens),
"total_output_tokens": float(run.total_output_tokens),
"mean_retries": sum(r.retries for r in run.results) / n,
}
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
def compare_configurations(
baseline_run: BenchmarkRun,
comparison_runs: list[BenchmarkRun],
) -> ComparisonReport:
"""Compare benchmark runs to produce delta tables and attribution.
Computes per-field and per-resource deltas between the baseline and
each comparison configuration, then attributes improvement sources.
Args:
baseline_run: The baseline (typically BASELINE_CURRENT) run results.
comparison_runs: One or more comparison configuration runs.
Returns:
ComparisonReport with deltas and attribution percentages.
"""
configs_compared = [baseline_run.config_name] + [
r.config_name for r in comparison_runs
]
baseline_metrics = _run_metrics(baseline_run)
deltas: list[ConfigDelta] = []
# Fields where higher is better
_higher_is_better = {"schema_validity_rate", "success_count"}
# Fields where lower is better (resource-like)
_resource_fields = {
"mean_duration_ms",
"total_input_tokens",
"total_output_tokens",
"mean_retries",
"failure_count",
}
for comp_run in comparison_runs:
comp_metrics = _run_metrics(comp_run)
field_deltas: list[FieldDelta] = []
resource_deltas: list[ResourceDelta] = []
for metric_name, baseline_val in baseline_metrics.items():
comp_val = comp_metrics[metric_name]
if metric_name in _resource_fields:
resource_deltas.append(
_compute_resource_delta(metric_name, baseline_val, comp_val)
)
else:
field_deltas.append(
_compute_field_delta(
metric_name,
baseline_val,
comp_val,
higher_is_better=(metric_name in _higher_is_better),
)
)
deltas.append(
ConfigDelta(
baseline_config=baseline_run.config_name,
comparison_config=comp_run.config_name,
field_deltas=field_deltas,
resource_deltas=resource_deltas,
)
)
# Attribution: estimate how much improvement comes from each fix
attribution = _compute_attribution(baseline_metrics, comparison_runs)
return ComparisonReport(
configs_compared=configs_compared,
deltas=deltas,
attribution_summary=attribution,
)
def _compute_attribution(
baseline_metrics: dict[str, float],
comparison_runs: list[BenchmarkRun],
) -> dict[str, float]:
"""Compute attribution percentages for improvement sources.
Uses schema_validity_rate as the primary improvement signal.
Attribution is computed as the fraction of total improvement each
configuration step contributes.
Returns a dict mapping source labels to fraction (0.0-1.0).
"""
attribution: dict[str, float] = {}
if not comparison_runs:
return attribution
baseline_validity = baseline_metrics["schema_validity_rate"]
# Find temp_zero and strict_schema runs by config name
temp_zero_validity: float | None = None
strict_schema_validity: float | None = None
for run in comparison_runs:
run_metrics = _run_metrics(run)
if "temp_zero" in run.config_name:
temp_zero_validity = run_metrics["schema_validity_rate"]
elif "strict_schema" in run.config_name:
strict_schema_validity = run_metrics["schema_validity_rate"]
# Compute incremental gains
# Total improvement = strict_schema - baseline (or best comparison - baseline)
best_validity = max(
_run_metrics(r)["schema_validity_rate"] for r in comparison_runs
)
total_improvement = best_validity - baseline_validity
if total_improvement <= 0.0:
# No improvement detected; equal attribution
attribution["temperature_fix"] = 0.0
attribution["schema_constraint"] = 0.0
return attribution
# Temperature fix contribution
if temp_zero_validity is not None:
temp_gain = temp_zero_validity - baseline_validity
attribution["temperature_fix"] = max(0.0, temp_gain / total_improvement)
else:
attribution["temperature_fix"] = 0.0
# Schema constraint contribution (incremental over temp fix)
if strict_schema_validity is not None and temp_zero_validity is not None:
schema_gain = strict_schema_validity - temp_zero_validity
attribution["schema_constraint"] = max(0.0, schema_gain / total_improvement)
elif strict_schema_validity is not None:
schema_gain = strict_schema_validity - baseline_validity
attribution["schema_constraint"] = max(0.0, schema_gain / total_improvement)
else:
attribution["schema_constraint"] = 0.0
# Remaining is attributed to other factors
accounted = attribution.get("temperature_fix", 0.0) + attribution.get(
"schema_constraint", 0.0
)
attribution["other"] = max(0.0, 1.0 - accounted)
return attribution
services/intelligence_pipeline_v3/benchmark/configurations.py
+134
View File
@@ -0,0 +1,134 @@
"""Benchmark configuration definitions for controlled extraction comparisons.
Defines the standard configurations used to attribute improvement sources:
- BASELINE_CURRENT: Current production settings (temperature 0.7, no schema constraint)
- BASELINE_TEMP_ZERO: Same model, temperature 0, no schema constraint
- BASELINE_STRICT_SCHEMA: Same model, temperature 0, strict JSON Schema
Validates: Requirements 16.2, 16.3, 16.5
"""
from __future__ import annotations
from enum import Enum
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
class StructuredOutputMode(str, Enum):
"""Structured output constraint modes for extraction."""
NONE = "none"
JSON_OBJECT = "json_object"
JSON_SCHEMA = "json_schema"
class BenchmarkConfig(BaseModel):
"""Configuration for a single benchmark extraction run.
Captures all parameters that affect extraction behavior so that
differences between runs can be attributed to specific settings.
"""
model_config = ConfigDict(frozen=True)
config_name: str = Field(
description="Unique identifier for this configuration",
)
description: str = Field(
description="Human-readable description of what this configuration tests",
)
model_name: str = Field(
description="Served model name (e.g. 'AxionML/Qwen3.5-9B-NVFP4')",
)
temperature: float = Field(
ge=0.0,
le=2.0,
description="Sampling temperature; 0.0 = deterministic",
)
max_output_tokens: int = Field(
gt=0,
description="Maximum tokens in generated output",
)
structured_output_mode: StructuredOutputMode = Field(
description="How output structure is constrained",
)
seed: int | None = Field(
default=None,
description="Random seed for reproducibility (None = not pinned)",
)
additional_params: dict[str, Any] = Field(
default_factory=dict,
description="Provider-specific extra parameters",
)
# ---------------------------------------------------------------------------
# Standard Benchmark Configurations
# ---------------------------------------------------------------------------
# The 9B model currently deployed on the cluster
_DEFAULT_MODEL = "AxionML/Qwen3.5-9B-NVFP4"
_DEFAULT_MAX_OUTPUT_TOKENS = 2048
BASELINE_CURRENT = BenchmarkConfig(
config_name="baseline_current",
description=(
"Current production settings: temperature 0.7, response_format json_object "
"only (schema not enforced on generation), no seed pinning. "
"Represents the unchanged request as deployed."
),
model_name=_DEFAULT_MODEL,
temperature=0.7,
max_output_tokens=_DEFAULT_MAX_OUTPUT_TOKENS,
structured_output_mode=StructuredOutputMode.JSON_OBJECT,
seed=None,
additional_params={},
)
BASELINE_TEMP_ZERO = BenchmarkConfig(
config_name="baseline_temp_zero",
description=(
"Same 9B model with temperature set to 0.0 for deterministic generation. "
"Still uses json_object mode without strict schema enforcement. "
"Isolates the effect of removing sampling stochasticity."
),
model_name=_DEFAULT_MODEL,
temperature=0.0,
max_output_tokens=_DEFAULT_MAX_OUTPUT_TOKENS,
structured_output_mode=StructuredOutputMode.JSON_OBJECT,
seed=0,
additional_params={},
)
BASELINE_STRICT_SCHEMA = BenchmarkConfig(
config_name="baseline_strict_schema",
description=(
"Same 9B model with temperature 0.0 AND strict JSON Schema output "
"enforcement via vLLM structured output backend. "
"Isolates the combined effect of deterministic generation plus "
"grammar-constrained decoding."
),
model_name=_DEFAULT_MODEL,
temperature=0.0,
max_output_tokens=_DEFAULT_MAX_OUTPUT_TOKENS,
structured_output_mode=StructuredOutputMode.JSON_SCHEMA,
seed=0,
additional_params={},
)
# Registry of all standard configurations
_STANDARD_CONFIGURATIONS: dict[str, BenchmarkConfig] = {
BASELINE_CURRENT.config_name: BASELINE_CURRENT,
BASELINE_TEMP_ZERO.config_name: BASELINE_TEMP_ZERO,
BASELINE_STRICT_SCHEMA.config_name: BASELINE_STRICT_SCHEMA,
}
def list_configurations() -> list[BenchmarkConfig]:
"""Return all registered benchmark configurations.
Returns:
List of BenchmarkConfig instances in definition order.
"""
return list(_STANDARD_CONFIGURATIONS.values())
services/intelligence_pipeline_v3/benchmark/runner.py
+296
View File
@@ -0,0 +1,296 @@
"""Benchmark runner scaffold for extraction configuration comparisons.
Provides the framework for running extraction benchmarks across different
configurations. The actual model invocations require the cluster, but
results can be stored and compared locally.
Validates: Requirements 16.2, 16.3
"""
from __future__ import annotations
import json
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from pydantic import BaseModel, Field
from services.intelligence_pipeline_v3.benchmark.configurations import (
BenchmarkConfig,
)
# ---------------------------------------------------------------------------
# Result Models
# ---------------------------------------------------------------------------
class BenchmarkDocumentResult(BaseModel):
"""Result of running one document through one benchmark configuration."""
document_id: str = Field(description="Identifier of the source document")
raw_output: str | None = Field(
default=None,
description="Raw model output text (before parsing)",
)
parsed_output: dict[str, Any] | None = Field(
default=None,
description="Parsed JSON output if extraction succeeded",
)
schema_valid: bool = Field(
default=False,
description="Whether the output passed JSON Schema validation",
)
retries: int = Field(
default=0,
ge=0,
description="Number of retries needed to get valid output",
)
duration_ms: int = Field(
default=0,
ge=0,
description="Total wall-clock time in milliseconds",
)
input_tokens: int = Field(
default=0,
ge=0,
description="Input tokens consumed",
)
output_tokens: int = Field(
default=0,
ge=0,
description="Output tokens generated",
)
error: str | None = Field(
default=None,
description="Error message if extraction failed",
)
class BenchmarkRun(BaseModel):
"""A complete benchmark run: one configuration applied to multiple documents."""
config_name: str = Field(description="Configuration used for this run")
timestamp: datetime = Field(
default_factory=lambda: datetime.now(timezone.utc),
description="When this run was executed",
)
document_ids: list[str] = Field(
default_factory=list,
description="Documents included in this run",
)
results: list[BenchmarkDocumentResult] = Field(
default_factory=list,
description="Per-document results",
)
@property
def success_count(self) -> int:
"""Number of documents that produced valid output."""
return sum(1 for r in self.results if r.schema_valid and r.error is None)
@property
def failure_count(self) -> int:
"""Number of documents that failed or produced invalid output."""
return len(self.results) - self.success_count
@property
def schema_validity_rate(self) -> float:
"""Fraction of results that passed schema validation."""
if not self.results:
return 0.0
return self.success_count / len(self.results)
@property
def mean_duration_ms(self) -> float:
"""Average duration across all results."""
if not self.results:
return 0.0
return sum(r.duration_ms for r in self.results) / len(self.results)
@property
def total_input_tokens(self) -> int:
"""Total input tokens across all results."""
return sum(r.input_tokens for r in self.results)
@property
def total_output_tokens(self) -> int:
"""Total output tokens across all results."""
return sum(r.output_tokens for r in self.results)
# ---------------------------------------------------------------------------
# Artifact Storage
# ---------------------------------------------------------------------------
_DEFAULT_ARTIFACT_DIR = Path("artifacts/benchmark")
def _ensure_artifact_dir(base: Path) -> Path:
"""Create artifact directory if it does not exist."""
base.mkdir(parents=True, exist_ok=True)
return base
def save_benchmark_run(
run: BenchmarkRun,
artifact_dir: Path | None = None,
) -> Path:
"""Persist a benchmark run as a JSON artifact.
Args:
run: The benchmark run to save.
artifact_dir: Directory to write to. Defaults to artifacts/benchmark/.
Returns:
Path to the written JSON file.
"""
base = artifact_dir or _DEFAULT_ARTIFACT_DIR
_ensure_artifact_dir(base)
ts = run.timestamp.strftime("%Y%m%d_%H%M%S")
filename = f"{run.config_name}_{ts}.json"
path = base / filename
path.write_text(
run.model_dump_json(indent=2),
encoding="utf-8",
)
return path
def load_benchmark_run(path: Path) -> BenchmarkRun:
"""Load a benchmark run from a JSON artifact.
Args:
path: Path to the JSON artifact file.
Returns:
Deserialized BenchmarkRun.
"""
data = json.loads(path.read_text(encoding="utf-8"))
return BenchmarkRun.model_validate(data)
# ---------------------------------------------------------------------------
# Runner
# ---------------------------------------------------------------------------
class BenchmarkRunner:
"""Runs extraction benchmarks using a given configuration.
The runner provides the scaffolding for executing benchmarks.
Actual LLM invocation is delegated to an inference callable.
When no inference callable is provided, results are recorded as
errors (useful for dry-run / configuration testing).
"""
def __init__(
self,
config: BenchmarkConfig,
artifact_dir: Path | None = None,
inference_fn: Any | None = None,
) -> None:
"""Initialize the benchmark runner.
Args:
config: Benchmark configuration to use for all runs.
artifact_dir: Where to store result artifacts.
inference_fn: Optional async callable(document_text, config) -> dict.
If None, documents are recorded as not-run errors.
"""
self.config = config
self.artifact_dir = artifact_dir or _DEFAULT_ARTIFACT_DIR
self._inference_fn = inference_fn
async def run_single_document(
self,
document_id: str,
document_text: str,
json_schema: dict[str, Any] | None = None,
) -> BenchmarkDocumentResult:
"""Run a single document through the configured extraction.
Args:
document_id: Unique document identifier.
document_text: Full document text to extract from.
json_schema: Optional JSON Schema for validation.
Returns:
BenchmarkDocumentResult with extraction outcome.
"""
if self._inference_fn is None:
return BenchmarkDocumentResult(
document_id=document_id,
error="No inference function configured (dry-run mode)",
)
start = time.perf_counter()
try:
result = await self._inference_fn(document_text, self.config)
duration_ms = int((time.perf_counter() - start) * 1000)
raw_output = result.get("raw_output", "")
parsed_output = result.get("parsed_output")
schema_valid = result.get("schema_valid", False)
input_tokens = result.get("input_tokens", 0)
output_tokens = result.get("output_tokens", 0)
retries = result.get("retries", 0)
return BenchmarkDocumentResult(
document_id=document_id,
raw_output=raw_output,
parsed_output=parsed_output,
schema_valid=schema_valid,
retries=retries,
duration_ms=duration_ms,
input_tokens=input_tokens,
output_tokens=output_tokens,
)
except Exception as exc:
duration_ms = int((time.perf_counter() - start) * 1000)
return BenchmarkDocumentResult(
document_id=document_id,
duration_ms=duration_ms,
error=str(exc),
)
async def run_batch(
self,
documents: list[tuple[str, str]],
json_schema: dict[str, Any] | None = None,
save_artifacts: bool = True,
) -> BenchmarkRun:
"""Run a batch of documents through the configured extraction.
Args:
documents: List of (document_id, document_text) tuples.
json_schema: Optional JSON Schema for validation.
save_artifacts: Whether to persist results as JSON artifacts.
Returns:
BenchmarkRun with all document results.
"""
results: list[BenchmarkDocumentResult] = []
document_ids: list[str] = []
for doc_id, doc_text in documents:
document_ids.append(doc_id)
result = await self.run_single_document(
document_id=doc_id,
document_text=doc_text,
json_schema=json_schema,
)
results.append(result)
run = BenchmarkRun(
config_name=self.config.config_name,
document_ids=document_ids,
results=results,
)
if save_artifacts:
save_benchmark_run(run, self.artifact_dir)
return run