feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,660 @@
|
||||
"""Latency, throughput, token, CPU, GPU, and memory resource metrics.
|
||||
|
||||
Implements evaluation metrics for pipeline resource consumption and efficiency.
|
||||
Supports per-document and per-stage breakdowns with percentile calculations.
|
||||
|
||||
Validates: Requirements 16.3, 16.4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StageTimingRecord:
|
||||
"""A single stage execution record with resource measurements.
|
||||
|
||||
Captures timing, token usage, and hardware resource consumption
|
||||
for one processing stage of one document.
|
||||
"""
|
||||
|
||||
document_id: str
|
||||
stage_name: str
|
||||
start_time: float # Unix timestamp (seconds)
|
||||
end_time: float # Unix timestamp (seconds)
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
gpu_memory_mb: float = 0.0
|
||||
cpu_seconds: float = 0.0
|
||||
gpu_seconds: float = 0.0
|
||||
|
||||
@property
|
||||
def duration_seconds(self) -> float:
|
||||
"""Wall-clock duration of this stage in seconds."""
|
||||
return self.end_time - self.start_time
|
||||
|
||||
@property
|
||||
def total_tokens(self) -> int:
|
||||
"""Sum of input and output tokens."""
|
||||
return self.input_tokens + self.output_tokens
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Percentile Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_percentile(values: list[float], percentile: float) -> float:
|
||||
"""Compute a percentile from a sorted list without numpy.
|
||||
|
||||
Uses linear interpolation between nearest ranks.
|
||||
|
||||
Args:
|
||||
values: List of numeric values (need not be pre-sorted).
|
||||
percentile: Percentile to compute (0-100).
|
||||
|
||||
Returns:
|
||||
The interpolated percentile value.
|
||||
|
||||
Raises:
|
||||
ValueError: If values is empty or percentile is out of range.
|
||||
"""
|
||||
if not values:
|
||||
raise ValueError("Cannot compute percentile of empty list")
|
||||
if not (0.0 <= percentile <= 100.0):
|
||||
raise ValueError(f"Percentile must be between 0 and 100, got {percentile}")
|
||||
|
||||
sorted_values = sorted(values)
|
||||
n = len(sorted_values)
|
||||
|
||||
if n == 1:
|
||||
return sorted_values[0]
|
||||
|
||||
# Compute the rank (0-indexed fractional position)
|
||||
rank = (percentile / 100.0) * (n - 1)
|
||||
lower_idx = int(rank)
|
||||
upper_idx = lower_idx + 1
|
||||
fraction = rank - lower_idx
|
||||
|
||||
if upper_idx >= n:
|
||||
return sorted_values[-1]
|
||||
|
||||
return sorted_values[lower_idx] + fraction * (
|
||||
sorted_values[upper_idx] - sorted_values[lower_idx]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LatencyPercentiles(BaseModel):
|
||||
"""Latency percentile distribution in seconds."""
|
||||
|
||||
p50: float = Field(ge=0.0)
|
||||
p90: float = Field(ge=0.0)
|
||||
p95: float = Field(ge=0.0)
|
||||
p99: float = Field(ge=0.0)
|
||||
mean: float = Field(ge=0.0)
|
||||
max: float = Field(ge=0.0)
|
||||
min: float = Field(ge=0.0)
|
||||
count: int = Field(ge=0)
|
||||
|
||||
|
||||
class ThroughputMetrics(BaseModel):
|
||||
"""Document throughput measurements."""
|
||||
|
||||
documents_per_minute: float = Field(ge=0.0)
|
||||
documents_per_hour: float = Field(ge=0.0)
|
||||
total_documents: int = Field(ge=0)
|
||||
total_wall_seconds: float = Field(ge=0.0)
|
||||
|
||||
|
||||
class TokenUsageMetrics(BaseModel):
|
||||
"""Token consumption statistics."""
|
||||
|
||||
total_input_tokens: int = Field(ge=0)
|
||||
total_output_tokens: int = Field(ge=0)
|
||||
total_tokens: int = Field(ge=0)
|
||||
mean_input_tokens_per_document: float = Field(ge=0.0)
|
||||
mean_output_tokens_per_document: float = Field(ge=0.0)
|
||||
mean_total_tokens_per_document: float = Field(ge=0.0)
|
||||
per_stage: dict[str, "StageTokenUsage"] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class StageTokenUsage(BaseModel):
|
||||
"""Token usage breakdown for a single stage."""
|
||||
|
||||
total_input_tokens: int = Field(ge=0)
|
||||
total_output_tokens: int = Field(ge=0)
|
||||
total_tokens: int = Field(ge=0)
|
||||
mean_input_tokens: float = Field(ge=0.0)
|
||||
mean_output_tokens: float = Field(ge=0.0)
|
||||
mean_total_tokens: float = Field(ge=0.0)
|
||||
count: int = Field(ge=0)
|
||||
|
||||
|
||||
class CpuMetrics(BaseModel):
|
||||
"""CPU resource consumption metrics."""
|
||||
|
||||
total_cpu_seconds: float = Field(ge=0.0)
|
||||
mean_cpu_seconds_per_document: float = Field(ge=0.0)
|
||||
peak_cpu_seconds: float = Field(ge=0.0, description="Max CPU-seconds for a single document")
|
||||
|
||||
|
||||
class GpuMetrics(BaseModel):
|
||||
"""GPU resource consumption metrics."""
|
||||
|
||||
total_gpu_seconds: float = Field(ge=0.0)
|
||||
mean_gpu_seconds_per_document: float = Field(ge=0.0)
|
||||
peak_gpu_memory_mb: float = Field(ge=0.0)
|
||||
mean_gpu_memory_mb: float = Field(ge=0.0)
|
||||
gpu_utilization_percent: float = Field(
|
||||
ge=0.0, le=100.0,
|
||||
description="Percentage of total wall time spent on GPU",
|
||||
)
|
||||
|
||||
|
||||
class MemoryMetrics(BaseModel):
|
||||
"""Memory consumption metrics."""
|
||||
|
||||
peak_rss_memory_mb: float = Field(ge=0.0)
|
||||
mean_working_set_mb: float = Field(ge=0.0)
|
||||
|
||||
|
||||
class EfficiencyMetrics(BaseModel):
|
||||
"""Efficiency ratio metrics."""
|
||||
|
||||
tokens_per_second: float = Field(ge=0.0)
|
||||
documents_per_gpu_second: float = Field(ge=0.0)
|
||||
fast_path_cpu_seconds: float = Field(ge=0.0)
|
||||
adjudication_cpu_seconds: float = Field(ge=0.0)
|
||||
fast_path_gpu_seconds: float = Field(ge=0.0)
|
||||
adjudication_gpu_seconds: float = Field(ge=0.0)
|
||||
fast_path_fraction: float = Field(
|
||||
ge=0.0, le=1.0,
|
||||
description="Fraction of total resource usage from fast-path stages",
|
||||
)
|
||||
adjudication_fraction: float = Field(
|
||||
ge=0.0, le=1.0,
|
||||
description="Fraction of total resource usage from adjudication stages",
|
||||
)
|
||||
|
||||
|
||||
class StageLatencyBreakdown(BaseModel):
|
||||
"""Per-stage latency statistics."""
|
||||
|
||||
stage_name: str
|
||||
latency: LatencyPercentiles
|
||||
invocation_count: int = Field(ge=0)
|
||||
|
||||
|
||||
class ResourceEvaluationReport(BaseModel):
|
||||
"""Complete resource evaluation report."""
|
||||
|
||||
latency: LatencyPercentiles
|
||||
per_stage_latency: list[StageLatencyBreakdown] = Field(default_factory=list)
|
||||
throughput: ThroughputMetrics
|
||||
token_usage: TokenUsageMetrics
|
||||
cpu: CpuMetrics
|
||||
gpu: GpuMetrics
|
||||
memory: MemoryMetrics
|
||||
efficiency: EfficiencyMetrics
|
||||
document_count: int = Field(ge=0)
|
||||
|
||||
|
||||
# Rebuild model to resolve forward references
|
||||
TokenUsageMetrics.model_rebuild()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Computation Logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Stages considered as "adjudication" for resource split calculations
|
||||
ADJUDICATION_STAGES: frozenset[str] = frozenset({
|
||||
"adjudication",
|
||||
"adjudicator",
|
||||
"9b_adjudication",
|
||||
"semantic_adjudication",
|
||||
})
|
||||
|
||||
|
||||
def _is_adjudication_stage(stage_name: str) -> bool:
|
||||
"""Determine if a stage belongs to the adjudication path."""
|
||||
lower = stage_name.lower()
|
||||
return lower in ADJUDICATION_STAGES or "adjudicat" in lower
|
||||
|
||||
|
||||
def _compute_latency_percentiles(durations: list[float]) -> LatencyPercentiles:
|
||||
"""Compute latency percentile distribution from a list of durations."""
|
||||
if not durations:
|
||||
return LatencyPercentiles(
|
||||
p50=0.0, p90=0.0, p95=0.0, p99=0.0,
|
||||
mean=0.0, max=0.0, min=0.0, count=0,
|
||||
)
|
||||
|
||||
return LatencyPercentiles(
|
||||
p50=compute_percentile(durations, 50.0),
|
||||
p90=compute_percentile(durations, 90.0),
|
||||
p95=compute_percentile(durations, 95.0),
|
||||
p99=compute_percentile(durations, 99.0),
|
||||
mean=sum(durations) / len(durations),
|
||||
max=max(durations),
|
||||
min=min(durations),
|
||||
count=len(durations),
|
||||
)
|
||||
|
||||
|
||||
def _compute_document_durations(
|
||||
records: list[StageTimingRecord],
|
||||
) -> dict[str, float]:
|
||||
"""Compute total wall-clock duration per document.
|
||||
|
||||
Uses min(start_time) to max(end_time) for each document to handle
|
||||
overlapping/parallel stages.
|
||||
"""
|
||||
doc_times: dict[str, tuple[float, float]] = {}
|
||||
for r in records:
|
||||
if r.document_id not in doc_times:
|
||||
doc_times[r.document_id] = (r.start_time, r.end_time)
|
||||
else:
|
||||
existing = doc_times[r.document_id]
|
||||
doc_times[r.document_id] = (
|
||||
min(existing[0], r.start_time),
|
||||
max(existing[1], r.end_time),
|
||||
)
|
||||
return {doc_id: end - start for doc_id, (start, end) in doc_times.items()}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_latency_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> tuple[LatencyPercentiles, list[StageLatencyBreakdown]]:
|
||||
"""Compute per-document and per-stage latency metrics.
|
||||
|
||||
Per-document latency is the wall-clock time from the earliest stage
|
||||
start to the latest stage end for each document.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
Tuple of (overall document latency, per-stage breakdown).
|
||||
"""
|
||||
if not records:
|
||||
return (
|
||||
LatencyPercentiles(
|
||||
p50=0.0, p90=0.0, p95=0.0, p99=0.0,
|
||||
mean=0.0, max=0.0, min=0.0, count=0,
|
||||
),
|
||||
[],
|
||||
)
|
||||
|
||||
# Per-document latency
|
||||
doc_durations = _compute_document_durations(records)
|
||||
overall = _compute_latency_percentiles(list(doc_durations.values()))
|
||||
|
||||
# Per-stage latency
|
||||
stage_durations: dict[str, list[float]] = {}
|
||||
for r in records:
|
||||
stage_durations.setdefault(r.stage_name, []).append(r.duration_seconds)
|
||||
|
||||
per_stage = [
|
||||
StageLatencyBreakdown(
|
||||
stage_name=stage,
|
||||
latency=_compute_latency_percentiles(durations),
|
||||
invocation_count=len(durations),
|
||||
)
|
||||
for stage, durations in sorted(stage_durations.items())
|
||||
]
|
||||
|
||||
return overall, per_stage
|
||||
|
||||
|
||||
def compute_throughput_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> ThroughputMetrics:
|
||||
"""Compute document throughput from timing records.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
ThroughputMetrics with documents/minute and documents/hour.
|
||||
"""
|
||||
if not records:
|
||||
return ThroughputMetrics(
|
||||
documents_per_minute=0.0,
|
||||
documents_per_hour=0.0,
|
||||
total_documents=0,
|
||||
total_wall_seconds=0.0,
|
||||
)
|
||||
|
||||
doc_ids = {r.document_id for r in records}
|
||||
total_docs = len(doc_ids)
|
||||
|
||||
# Total wall time: earliest start to latest end across all records
|
||||
earliest = min(r.start_time for r in records)
|
||||
latest = max(r.end_time for r in records)
|
||||
total_wall = latest - earliest
|
||||
|
||||
if total_wall <= 0.0:
|
||||
return ThroughputMetrics(
|
||||
documents_per_minute=0.0,
|
||||
documents_per_hour=0.0,
|
||||
total_documents=total_docs,
|
||||
total_wall_seconds=0.0,
|
||||
)
|
||||
|
||||
docs_per_second = total_docs / total_wall
|
||||
|
||||
return ThroughputMetrics(
|
||||
documents_per_minute=docs_per_second * 60.0,
|
||||
documents_per_hour=docs_per_second * 3600.0,
|
||||
total_documents=total_docs,
|
||||
total_wall_seconds=total_wall,
|
||||
)
|
||||
|
||||
|
||||
def compute_token_usage_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> TokenUsageMetrics:
|
||||
"""Compute token usage statistics per document and per stage.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
TokenUsageMetrics with aggregate and per-stage breakdowns.
|
||||
"""
|
||||
if not records:
|
||||
return TokenUsageMetrics(
|
||||
total_input_tokens=0,
|
||||
total_output_tokens=0,
|
||||
total_tokens=0,
|
||||
mean_input_tokens_per_document=0.0,
|
||||
mean_output_tokens_per_document=0.0,
|
||||
mean_total_tokens_per_document=0.0,
|
||||
per_stage={},
|
||||
)
|
||||
|
||||
total_input = sum(r.input_tokens for r in records)
|
||||
total_output = sum(r.output_tokens for r in records)
|
||||
total = total_input + total_output
|
||||
|
||||
doc_ids = {r.document_id for r in records}
|
||||
n_docs = len(doc_ids)
|
||||
|
||||
# Per-stage breakdown
|
||||
stage_records: dict[str, list[StageTimingRecord]] = {}
|
||||
for r in records:
|
||||
stage_records.setdefault(r.stage_name, []).append(r)
|
||||
|
||||
per_stage: dict[str, StageTokenUsage] = {}
|
||||
for stage, stage_recs in sorted(stage_records.items()):
|
||||
s_input = sum(r.input_tokens for r in stage_recs)
|
||||
s_output = sum(r.output_tokens for r in stage_recs)
|
||||
s_total = s_input + s_output
|
||||
count = len(stage_recs)
|
||||
per_stage[stage] = StageTokenUsage(
|
||||
total_input_tokens=s_input,
|
||||
total_output_tokens=s_output,
|
||||
total_tokens=s_total,
|
||||
mean_input_tokens=s_input / count if count > 0 else 0.0,
|
||||
mean_output_tokens=s_output / count if count > 0 else 0.0,
|
||||
mean_total_tokens=s_total / count if count > 0 else 0.0,
|
||||
count=count,
|
||||
)
|
||||
|
||||
return TokenUsageMetrics(
|
||||
total_input_tokens=total_input,
|
||||
total_output_tokens=total_output,
|
||||
total_tokens=total,
|
||||
mean_input_tokens_per_document=total_input / n_docs if n_docs > 0 else 0.0,
|
||||
mean_output_tokens_per_document=total_output / n_docs if n_docs > 0 else 0.0,
|
||||
mean_total_tokens_per_document=total / n_docs if n_docs > 0 else 0.0,
|
||||
per_stage=per_stage,
|
||||
)
|
||||
|
||||
|
||||
def compute_cpu_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> CpuMetrics:
|
||||
"""Compute CPU resource consumption metrics.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
CpuMetrics with totals and per-document statistics.
|
||||
"""
|
||||
if not records:
|
||||
return CpuMetrics(
|
||||
total_cpu_seconds=0.0,
|
||||
mean_cpu_seconds_per_document=0.0,
|
||||
peak_cpu_seconds=0.0,
|
||||
)
|
||||
|
||||
total_cpu = sum(r.cpu_seconds for r in records)
|
||||
|
||||
# Per-document CPU totals
|
||||
doc_cpu: dict[str, float] = {}
|
||||
for r in records:
|
||||
doc_cpu[r.document_id] = doc_cpu.get(r.document_id, 0.0) + r.cpu_seconds
|
||||
|
||||
n_docs = len(doc_cpu)
|
||||
peak = max(doc_cpu.values()) if doc_cpu else 0.0
|
||||
|
||||
return CpuMetrics(
|
||||
total_cpu_seconds=total_cpu,
|
||||
mean_cpu_seconds_per_document=total_cpu / n_docs if n_docs > 0 else 0.0,
|
||||
peak_cpu_seconds=peak,
|
||||
)
|
||||
|
||||
|
||||
def compute_gpu_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> GpuMetrics:
|
||||
"""Compute GPU resource consumption metrics.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
GpuMetrics with totals, peaks, and utilization percentage.
|
||||
"""
|
||||
if not records:
|
||||
return GpuMetrics(
|
||||
total_gpu_seconds=0.0,
|
||||
mean_gpu_seconds_per_document=0.0,
|
||||
peak_gpu_memory_mb=0.0,
|
||||
mean_gpu_memory_mb=0.0,
|
||||
gpu_utilization_percent=0.0,
|
||||
)
|
||||
|
||||
total_gpu = sum(r.gpu_seconds for r in records)
|
||||
|
||||
# Per-document GPU totals
|
||||
doc_gpu: dict[str, float] = {}
|
||||
for r in records:
|
||||
doc_gpu[r.document_id] = doc_gpu.get(r.document_id, 0.0) + r.gpu_seconds
|
||||
|
||||
n_docs = len(doc_gpu)
|
||||
|
||||
# GPU memory stats
|
||||
gpu_mem_values = [r.gpu_memory_mb for r in records if r.gpu_memory_mb > 0.0]
|
||||
peak_gpu_mem = max(gpu_mem_values) if gpu_mem_values else 0.0
|
||||
mean_gpu_mem = (
|
||||
sum(gpu_mem_values) / len(gpu_mem_values) if gpu_mem_values else 0.0
|
||||
)
|
||||
|
||||
# GPU utilization: fraction of wall time spent on GPU work
|
||||
earliest = min(r.start_time for r in records)
|
||||
latest = max(r.end_time for r in records)
|
||||
total_wall = latest - earliest
|
||||
|
||||
utilization = (
|
||||
(total_gpu / total_wall) * 100.0 if total_wall > 0.0 else 0.0
|
||||
)
|
||||
# Cap at 100% (parallel GPU stages could theoretically exceed wall time)
|
||||
utilization = min(utilization, 100.0)
|
||||
|
||||
return GpuMetrics(
|
||||
total_gpu_seconds=total_gpu,
|
||||
mean_gpu_seconds_per_document=total_gpu / n_docs if n_docs > 0 else 0.0,
|
||||
peak_gpu_memory_mb=peak_gpu_mem,
|
||||
mean_gpu_memory_mb=mean_gpu_mem,
|
||||
gpu_utilization_percent=utilization,
|
||||
)
|
||||
|
||||
|
||||
def compute_memory_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
rss_samples_mb: list[float] | None = None,
|
||||
) -> MemoryMetrics:
|
||||
"""Compute memory consumption metrics.
|
||||
|
||||
Uses gpu_memory_mb as a proxy for working set if no explicit RSS
|
||||
samples are provided. When rss_samples_mb is given, it takes
|
||||
precedence for peak and mean calculations.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
rss_samples_mb: Optional explicit RSS memory samples in MB.
|
||||
|
||||
Returns:
|
||||
MemoryMetrics with peak and mean working set.
|
||||
"""
|
||||
if rss_samples_mb:
|
||||
return MemoryMetrics(
|
||||
peak_rss_memory_mb=max(rss_samples_mb),
|
||||
mean_working_set_mb=sum(rss_samples_mb) / len(rss_samples_mb),
|
||||
)
|
||||
|
||||
if not records:
|
||||
return MemoryMetrics(peak_rss_memory_mb=0.0, mean_working_set_mb=0.0)
|
||||
|
||||
# Use gpu_memory_mb as working set proxy
|
||||
mem_values = [r.gpu_memory_mb for r in records if r.gpu_memory_mb > 0.0]
|
||||
if not mem_values:
|
||||
return MemoryMetrics(peak_rss_memory_mb=0.0, mean_working_set_mb=0.0)
|
||||
|
||||
return MemoryMetrics(
|
||||
peak_rss_memory_mb=max(mem_values),
|
||||
mean_working_set_mb=sum(mem_values) / len(mem_values),
|
||||
)
|
||||
|
||||
|
||||
def compute_efficiency_metrics(
|
||||
records: list[StageTimingRecord],
|
||||
) -> EfficiencyMetrics:
|
||||
"""Compute efficiency ratios including tokens/second and resource splits.
|
||||
|
||||
Fast-path vs adjudication split is determined by stage name matching.
|
||||
|
||||
Args:
|
||||
records: Stage timing records.
|
||||
|
||||
Returns:
|
||||
EfficiencyMetrics with ratios and resource splits.
|
||||
"""
|
||||
if not records:
|
||||
return EfficiencyMetrics(
|
||||
tokens_per_second=0.0,
|
||||
documents_per_gpu_second=0.0,
|
||||
fast_path_cpu_seconds=0.0,
|
||||
adjudication_cpu_seconds=0.0,
|
||||
fast_path_gpu_seconds=0.0,
|
||||
adjudication_gpu_seconds=0.0,
|
||||
fast_path_fraction=0.0,
|
||||
adjudication_fraction=0.0,
|
||||
)
|
||||
|
||||
total_tokens = sum(r.total_tokens for r in records)
|
||||
total_wall = max(r.end_time for r in records) - min(r.start_time for r in records)
|
||||
total_gpu = sum(r.gpu_seconds for r in records)
|
||||
n_docs = len({r.document_id for r in records})
|
||||
|
||||
tokens_per_second = total_tokens / total_wall if total_wall > 0.0 else 0.0
|
||||
docs_per_gpu_second = n_docs / total_gpu if total_gpu > 0.0 else 0.0
|
||||
|
||||
# Resource split
|
||||
fast_cpu = 0.0
|
||||
adj_cpu = 0.0
|
||||
fast_gpu = 0.0
|
||||
adj_gpu = 0.0
|
||||
|
||||
for r in records:
|
||||
if _is_adjudication_stage(r.stage_name):
|
||||
adj_cpu += r.cpu_seconds
|
||||
adj_gpu += r.gpu_seconds
|
||||
else:
|
||||
fast_cpu += r.cpu_seconds
|
||||
fast_gpu += r.gpu_seconds
|
||||
|
||||
total_resource = fast_cpu + adj_cpu + fast_gpu + adj_gpu
|
||||
fast_total = fast_cpu + fast_gpu
|
||||
adj_total = adj_cpu + adj_gpu
|
||||
|
||||
fast_fraction = fast_total / total_resource if total_resource > 0.0 else 0.0
|
||||
adj_fraction = adj_total / total_resource if total_resource > 0.0 else 0.0
|
||||
|
||||
return EfficiencyMetrics(
|
||||
tokens_per_second=tokens_per_second,
|
||||
documents_per_gpu_second=docs_per_gpu_second,
|
||||
fast_path_cpu_seconds=fast_cpu,
|
||||
adjudication_cpu_seconds=adj_cpu,
|
||||
fast_path_gpu_seconds=fast_gpu,
|
||||
adjudication_gpu_seconds=adj_gpu,
|
||||
fast_path_fraction=fast_fraction,
|
||||
adjudication_fraction=adj_fraction,
|
||||
)
|
||||
|
||||
|
||||
def evaluate_resources(
|
||||
records: list[StageTimingRecord],
|
||||
rss_samples_mb: list[float] | None = None,
|
||||
) -> ResourceEvaluationReport:
|
||||
"""Run full resource evaluation producing a complete report.
|
||||
|
||||
Args:
|
||||
records: List of stage timing records from pipeline execution.
|
||||
rss_samples_mb: Optional explicit RSS memory samples.
|
||||
|
||||
Returns:
|
||||
ResourceEvaluationReport with all resource metrics.
|
||||
"""
|
||||
latency, per_stage_latency = compute_latency_metrics(records)
|
||||
throughput = compute_throughput_metrics(records)
|
||||
token_usage = compute_token_usage_metrics(records)
|
||||
cpu = compute_cpu_metrics(records)
|
||||
gpu = compute_gpu_metrics(records)
|
||||
memory = compute_memory_metrics(records, rss_samples_mb)
|
||||
efficiency = compute_efficiency_metrics(records)
|
||||
|
||||
doc_count = len({r.document_id for r in records}) if records else 0
|
||||
|
||||
return ResourceEvaluationReport(
|
||||
latency=latency,
|
||||
per_stage_latency=per_stage_latency,
|
||||
throughput=throughput,
|
||||
token_usage=token_usage,
|
||||
cpu=cpu,
|
||||
gpu=gpu,
|
||||
memory=memory,
|
||||
efficiency=efficiency,
|
||||
document_count=doc_count,
|
||||
)
|
||||
Reference in New Issue
Block a user