admin/stonks-oracle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Intelligence Pipeline v3 — full implementation

Browse Source 
Multi-stage evidence-grounded inference architecture replacing the
monolithic 9B model extraction pipeline. CPU-first specialist services
handle routine extraction while the 9B vLLM model is preserved for
semantic adjudication of ambiguous cases.

Key components:
- Capability-aware inference gateway (OpenAI-compatible + Ollama)
- Endpoint registry with DB migrations and REST API
- Sentence-aware document segmenter (property tests)
- Deterministic financial parsing with offset integrity
- Symbol resolution with ambiguity detection
- Specialist service (GLiNER2, dynamic batching, K8s deployment)
- Company-specific sentiment (FinBERT, calibration)
- Retrieval-based novelty and duplicate detection
- Confidence calibration pipeline
- Deterministic routing engine (property tests)
- 9B adjudication layer with VRAM gating
- Stock-specific impact model (features, labels, baseline, trained)
- Pipeline orchestrator (state machine, queues, leases, feature flags)
- Bounded parallelism (async workers, semaphore, load shedding)
- Observability (tracing, metrics, alerts)
- Compatibility adapter (v3→v2 golden mapping tests)
- Shadow/canary promotion framework
- Active learning and fine-tuning pipeline

Test results: 1,161 tests pass, ruff lint clean.
All 282 spec tasks completed.
This commit is contained in:
Celes Renata
2026-07-13 02:14:59 +00:00
parent 84634a365e
commit a72f336ad1
227 changed files with 50403 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/intelligence_pipeline_v3/schemas/__init__.py
+66
View File
@@ -0,0 +1,66 @@
"""V3 annotation schema — entity, event, relation, sentiment, and evidence models."""
from services.intelligence_pipeline_v3.schemas.annotations import (
AmbiguityMarker,
AmbiguityType,
AnnotatedDocument,
AnnotationMetadata,
CompanySentimentAnnotation,
DirectEffect,
EntityAnnotation,
EntityType,
EventAnnotation,
EventClass,
EvidenceSpanAnnotation,
InferredExposure,
NumericFactAnnotation,
PeriodAnnotation,
PeriodType,
RelationAnnotation,
RelationType,
SentimentLabel,
)
from services.intelligence_pipeline_v3.schemas.safety import (
SAFETY_CRITICAL_FIELDS,
SafetyCriticalField,
SafetyGateResult,
check_safety_gates,
)
from services.intelligence_pipeline_v3.schemas.validators import (
ValidationError as AnnotationValidationError,
)
from services.intelligence_pipeline_v3.schemas.validators import (
ValidationResult,
validate_annotation,
)
__all__ = [
# Annotation models
"AmbiguityMarker",
"AmbiguityType",
"AnnotatedDocument",
"AnnotationMetadata",
"CompanySentimentAnnotation",
"DirectEffect",
"EntityAnnotation",
"EntityType",
"EvidenceSpanAnnotation",
"EventAnnotation",
"EventClass",
"InferredExposure",
"NumericFactAnnotation",
"PeriodAnnotation",
"PeriodType",
"RelationAnnotation",
"RelationType",
"SentimentLabel",
# Safety
"SAFETY_CRITICAL_FIELDS",
"SafetyCriticalField",
"SafetyGateResult",
"check_safety_gates",
# Validators
"AnnotationValidationError",
"ValidationResult",
"validate_annotation",
]
services/intelligence_pipeline_v3/schemas/annotations.py
+398
View File
@@ -0,0 +1,398 @@
"""V3 annotation schema — labels for entities, events, relations, facts, and evidence.
This module defines the complete annotation schema for the Intelligence Pipeline v3
Gold Corpus. Every extracted field is traceable to source evidence via character offsets.
Schema version: 1.0.0
"""
from __future__ import annotations
import uuid
from datetime import date, datetime
from enum import Enum
from typing import Literal
from pydantic import BaseModel, Field, model_validator
# ---------------------------------------------------------------------------
# Enumerations
# ---------------------------------------------------------------------------
class EntityType(str, Enum):
"""Recognized entity types in the v3 pipeline."""
COMPANY = "company"
PERSON = "person"
PRODUCT = "product"
EVENT = "event"
FINANCIAL_METRIC = "financial_metric"
DATE = "date"
PERCENTAGE = "percentage"
CURRENCY = "currency"
RELATIONSHIP = "relationship"
class EventClass(str, Enum):
"""Versioned event taxonomy for market-relevant occurrences."""
EARNINGS_BEAT = "earnings_beat"
EARNINGS_MISS = "earnings_miss"
GUIDANCE_RAISE = "guidance_raise"
GUIDANCE_CUT = "guidance_cut"
MA_ANNOUNCEMENT = "ma_announcement"
LEGAL_REGULATORY = "legal_regulatory"
PRODUCT_LAUNCH = "product_launch"
SUPPLY_CHAIN = "supply_chain"
RATING_CHANGE = "rating_change"
MANAGEMENT_CHANGE = "management_change"
MACRO_EVENT = "macro_event"
DIVIDEND_CHANGE = "dividend_change"
BUYBACK = "buyback"
class SentimentLabel(str, Enum):
"""Sentiment classification for company-linked evidence groups."""
POSITIVE = "positive"
NEGATIVE = "negative"
NEUTRAL = "neutral"
MIXED = "mixed"
class RelationType(str, Enum):
"""Relation types between entities or between events and companies."""
DIRECTLY_AFFECTS = "directly_affects"
INFERRED_EXPOSURE = "inferred_exposure"
COMPETES_WITH = "competes_with"
SUPPLIES = "supplies"
class PeriodType(str, Enum):
"""Financial period type identifiers."""
FISCAL_QUARTER = "fiscal_quarter"
FISCAL_YEAR = "fiscal_year"
CALENDAR_QUARTER = "calendar_quarter"
CALENDAR_YEAR = "calendar_year"
TRAILING_TWELVE_MONTHS = "ttm"
YEAR_TO_DATE = "ytd"
CUSTOM = "custom"
class AmbiguityType(str, Enum):
"""Ambiguity reasons that trigger adjudication routing."""
UNRESOLVED_ALIAS = "unresolved_alias"
MULTIPLE_PRIMARY_COMPANIES = "multiple_primary_companies"
CONTRADICTORY_NUMERIC_FACTS = "contradictory_numeric_facts"
CONFLICTING_SENTIMENT = "conflicting_sentiment"
IMPLIED_CAUSAL_IMPACT = "implied_causal_impact"
GUIDANCE_VS_CONSENSUS_REQUIRES_REASONING = "guidance_vs_consensus_requires_reasoning"
MATERIAL_FIELD_MISSING = "material_field_missing"
EVIDENCE_COVERAGE_BELOW_THRESHOLD = "evidence_coverage_below_threshold"
CALIBRATED_CONFIDENCE_BELOW_THRESHOLD = "calibrated_confidence_below_threshold"
LONG_DOCUMENT_CROSS_CHUNK_RELATION = "long_document_cross_chunk_relation"
# ---------------------------------------------------------------------------
# Evidence Span
# ---------------------------------------------------------------------------
class EvidenceSpanAnnotation(BaseModel):
"""Exact source text with stable character offsets.
Every extracted fact, entity, or relation MUST reference at least one evidence span.
Offsets are zero-based and refer to the original (pre-chunking) document text.
"""
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
chunk_id: str | None = Field(
default=None, description="Chunk ID if document was segmented."
)
start_char: int = Field(ge=0, description="Zero-based start character offset.")
end_char: int = Field(ge=0, description="Zero-based end character offset (exclusive).")
text: str = Field(min_length=1, description="Exact source text at this span.")
checksum: str | None = Field(
default=None,
description="SHA-256 hex digest of the span text for integrity verification.",
)
@model_validator(mode="after")
def end_after_start(self) -> "EvidenceSpanAnnotation":
if self.end_char <= self.start_char:
raise ValueError(
f"end_char ({self.end_char}) must be greater than start_char ({self.start_char})"
)
return self
# ---------------------------------------------------------------------------
# Entity Annotation
# ---------------------------------------------------------------------------
class EntityAnnotation(BaseModel):
"""A labeled entity mention with optional canonical resolution."""
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
entity_type: EntityType
literal_text: str = Field(min_length=1, description="Exact surface form as it appears.")
canonical_id: str | None = Field(
default=None,
description="UUID of the canonical company/entity from the symbol registry.",
)
canonical_name: str | None = Field(
default=None, description="Resolved canonical name (e.g., ticker or full name)."
)
evidence_ids: list[str] = Field(
min_length=1,
description="References to EvidenceSpanAnnotation IDs supporting this entity.",
)
confidence: float = Field(ge=0.0, le=1.0, description="Annotator certainty [0, 1].")
derivation: str = Field(
default="manual",
description="How this label was derived: manual, deterministic, specialist, adjudicated.",
)
# ---------------------------------------------------------------------------
# Event Annotation
# ---------------------------------------------------------------------------
class EventAnnotation(BaseModel):
"""A market-relevant event classification anchored to evidence."""
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
event_class: EventClass
description: str = Field(
default="", description="Brief human-readable description of the event."
)
primary_company_ids: list[str] = Field(
default_factory=list,
description="Entity annotation IDs of directly affected companies.",
)
evidence_ids: list[str] = Field(
min_length=1, description="Evidence spans supporting this event classification."
)
confidence: float = Field(ge=0.0, le=1.0)
derivation: str = Field(default="manual")
# ---------------------------------------------------------------------------
# Relation Annotation
# ---------------------------------------------------------------------------
class RelationAnnotation(BaseModel):
"""A typed relation between two entities or between an event and an entity."""
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
relation_type: RelationType
source_id: str = Field(description="Entity or event annotation ID (subject).")
target_id: str = Field(description="Entity annotation ID (object).")
evidence_ids: list[str] = Field(min_length=1)
confidence: float = Field(ge=0.0, le=1.0)
derivation: str = Field(default="manual")
# ---------------------------------------------------------------------------
# Numeric Fact Annotation
# ---------------------------------------------------------------------------
class NumericFactAnnotation(BaseModel):
"""A numeric fact (EPS, revenue, percentage change, etc.) extracted from the document."""
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
fact_type: str = Field(
description="Category: eps, revenue, percentage_change, price_target, guidance, market_cap, etc."
)
subject_entity_id: str | None = Field(
default=None, description="Entity annotation ID this fact belongs to."
)
predicate: str = Field(
description="Semantic predicate: reported, expected, raised_to, cut_to, beat_by, etc."
)
literal_value: str = Field(description="Exact textual representation from the source.")
normalized_value: float | None = Field(
default=None, description="Numeric value after normalization."
)
unit: str | None = Field(
default=None, description="Unit: USD, %, bps, shares, etc."
)
period: "PeriodAnnotation | None" = Field(
default=None, description="Financial period this fact applies to."
)
evidence_ids: list[str] = Field(min_length=1)
confidence: float = Field(ge=0.0, le=1.0)
derivation: str = Field(default="manual")
# ---------------------------------------------------------------------------
# Period Annotation
# ---------------------------------------------------------------------------
class PeriodAnnotation(BaseModel):
"""Financial or calendar period reference."""
period_type: PeriodType
fiscal_year: int | None = Field(default=None, description="e.g. 2024")
fiscal_quarter: int | None = Field(default=None, ge=1, le=4)
start_date: date | None = None
end_date: date | None = None
literal_text: str | None = Field(
default=None, description="Original text describing the period."
)
# ---------------------------------------------------------------------------
# Sentiment Annotation
# ---------------------------------------------------------------------------
class CompanySentimentAnnotation(BaseModel):
"""Company-specific sentiment with probability distribution.
Mixed sentiment is derived from disagreement across evidence groups — it is NOT
an unconstrained fourth softmax label.
"""
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
company_entity_id: str = Field(
description="Entity annotation ID of the company this sentiment applies to."
)
label: SentimentLabel
positive_probability: float = Field(ge=0.0, le=1.0)
negative_probability: float = Field(ge=0.0, le=1.0)
neutral_probability: float = Field(ge=0.0, le=1.0)
evidence_ids: list[str] = Field(min_length=1)
confidence: float = Field(ge=0.0, le=1.0)
derivation: str = Field(default="manual")
@model_validator(mode="after")
def probabilities_sum_to_one(self) -> "CompanySentimentAnnotation":
total = (
self.positive_probability
+ self.negative_probability
+ self.neutral_probability
)
if abs(total - 1.0) > 0.01:
raise ValueError(
f"Sentiment probabilities must sum to ~1.0, got {total:.4f}"
)
return self
# ---------------------------------------------------------------------------
# Direct Effects and Inferred Exposure
# ---------------------------------------------------------------------------
class DirectEffect(BaseModel):
"""An event directly affecting a specific company, backed by explicit evidence."""
event_id: str = Field(description="Event annotation ID.")
company_entity_id: str = Field(description="Entity annotation ID of the affected company.")
evidence_ids: list[str] = Field(min_length=1)
confidence: float = Field(ge=0.0, le=1.0)
class InferredExposure(BaseModel):
"""An inferred (not explicitly stated) exposure of a company to an event.
Inferred exposures have separate confidence and do NOT enter primary extraction.
They flow through the interpolation/propagation architecture with distinct provenance.
"""
event_id: str = Field(description="Event annotation ID.")
company_entity_id: str = Field(description="Entity annotation ID.")
reasoning: str = Field(
description="Brief explanation of the inference chain (e.g., supply-chain relationship)."
)
evidence_ids: list[str] = Field(
default_factory=list,
description="Supporting evidence (may be empty for purely inferred relations).",
)
confidence: float = Field(ge=0.0, le=1.0)
# ---------------------------------------------------------------------------
# Ambiguity Marker
# ---------------------------------------------------------------------------
class AmbiguityMarker(BaseModel):
"""Flags cases that require adjudication routing.
These markers determine whether a document is processed via the fast path
or routed to the 9B model for semantic adjudication.
"""
ambiguity_type: AmbiguityType
description: str = Field(
default="", description="Human-readable description of the ambiguity."
)
affected_entity_ids: list[str] = Field(default_factory=list)
affected_event_ids: list[str] = Field(default_factory=list)
severity: Literal["low", "medium", "high"] = Field(
default="medium",
description="Impact on extraction confidence.",
)
# ---------------------------------------------------------------------------
# Annotation Metadata
# ---------------------------------------------------------------------------
class AnnotationMetadata(BaseModel):
"""Metadata for a complete document annotation."""
schema_version: str = Field(default="1.0.0")
annotator_id: str = Field(description="Identifier of the annotator (human or system).")
annotation_date: datetime = Field(default_factory=lambda: datetime.now(tz=datetime.now().astimezone().tzinfo))
review_status: Literal["draft", "reviewed", "adjudicated", "gold"] = Field(
default="draft"
)
reviewer_id: str | None = None
review_date: datetime | None = None
notes: str = Field(default="")
# ---------------------------------------------------------------------------
# Top-Level Annotated Document
# ---------------------------------------------------------------------------
class AnnotatedDocument(BaseModel):
"""Complete v3 annotation for a single source document.
This is the top-level container used in the Gold Corpus. Every field references
evidence spans for traceability.
"""
document_id: str = Field(description="UUID of the source document.")
document_type: str = Field(description="article, filing, transcript, press_release, macro_event")
source_text: str = Field(description="Full original document text (offsets reference this).")
metadata: AnnotationMetadata
# Core annotations
evidence_spans: list[EvidenceSpanAnnotation] = Field(default_factory=list)
entities: list[EntityAnnotation] = Field(default_factory=list)
events: list[EventAnnotation] = Field(default_factory=list)
relations: list[RelationAnnotation] = Field(default_factory=list)
numeric_facts: list[NumericFactAnnotation] = Field(default_factory=list)
sentiments: list[CompanySentimentAnnotation] = Field(default_factory=list)
# Effects and exposure
direct_effects: list[DirectEffect] = Field(default_factory=list)
inferred_exposures: list[InferredExposure] = Field(default_factory=list)
# Routing
ambiguity_markers: list[AmbiguityMarker] = Field(default_factory=list)
services/intelligence_pipeline_v3/schemas/safety.py
+149
View File
@@ -0,0 +1,149 @@
"""Safety-critical field definitions for v3 promotion gates.
Fields marked as safety-critical MUST pass their quality gates before pipeline
outputs are allowed to influence production aggregation or trading decisions.
A single safety-critical failure blocks promotion for the affected document type.
Schema version: 1.0.0
"""
from __future__ import annotations
from dataclasses import dataclass
from enum import Enum
from typing import TYPE_CHECKING
if TYPE_CHECKING:
pass
class SafetyCriticalField(str, Enum):
"""Fields whose incorrect extraction can materially affect trading decisions."""
# Company identity — wrong ticker attribution can cause trades on the wrong security
COMPANY_IDENTITY = "company_identity"
# Event classification — misclassifying earnings_beat as earnings_miss inverts signals
EVENT_CLASS = "event_class"
# Sentiment direction — wrong sentiment directly affects position direction
SENTIMENT_DIRECTION = "sentiment_direction"
# Numeric fact accuracy — wrong EPS or revenue magnitude affects impact estimation
NUMERIC_FACT_VALUE = "numeric_fact_value"
# Direct effect attribution — attributing an event to the wrong company creates false signals
DIRECT_EFFECT_ATTRIBUTION = "direct_effect_attribution"
# Evidence support — claims without valid evidence spans are unverifiable
EVIDENCE_SUPPORT = "evidence_support"
# Confidence calibration — overconfident scores bypass appropriate review thresholds
CONFIDENCE_CALIBRATION = "confidence_calibration"
# Map each safety-critical field to its minimum required quality metric for promotion
SAFETY_CRITICAL_FIELDS: dict[SafetyCriticalField, dict[str, float]] = {
SafetyCriticalField.COMPANY_IDENTITY: {
"precision": 0.95,
"recall": 0.90,
"f1": 0.92,
},
SafetyCriticalField.EVENT_CLASS: {
"macro_f1": 0.85,
"per_class_min_f1": 0.70,
},
SafetyCriticalField.SENTIMENT_DIRECTION: {
"macro_f1": 0.85,
"direction_accuracy": 0.90,
},
SafetyCriticalField.NUMERIC_FACT_VALUE: {
"exact_match": 0.80,
"tolerance_match_5pct": 0.92,
},
SafetyCriticalField.DIRECT_EFFECT_ATTRIBUTION: {
"precision": 0.93,
"recall": 0.88,
},
SafetyCriticalField.EVIDENCE_SUPPORT: {
"support_rate": 0.95,
"offset_validity": 0.98,
},
SafetyCriticalField.CONFIDENCE_CALIBRATION: {
"ece": 0.05, # Expected Calibration Error — lower is better
"brier_score": 0.15, # Lower is better
},
}
@dataclass
class SafetyGateResult:
"""Result of checking a single safety-critical field against its gate."""
field: SafetyCriticalField
passed: bool
metric_name: str
required_value: float
actual_value: float
is_lower_better: bool = False
@property
def margin(self) -> float:
"""How far above (or below for lower-is-better) the threshold."""
if self.is_lower_better:
return self.required_value - self.actual_value
return self.actual_value - self.required_value
def check_safety_gates(
metrics: dict[SafetyCriticalField, dict[str, float]],
) -> list[SafetyGateResult]:
"""Check all safety-critical fields against their promotion thresholds.
Args:
metrics: Measured quality metrics per safety-critical field.
Keys match SafetyCriticalField, values are metric_name -> value dicts.
Returns:
List of SafetyGateResult for each check performed.
Any result with passed=False blocks promotion.
"""
lower_is_better = {"ece", "brier_score"}
results: list[SafetyGateResult] = []
for field, thresholds in SAFETY_CRITICAL_FIELDS.items():
measured = metrics.get(field, {})
for metric_name, required in thresholds.items():
actual = measured.get(metric_name)
if actual is None:
# Missing metric fails the gate
results.append(
SafetyGateResult(
field=field,
passed=False,
metric_name=metric_name,
required_value=required,
actual_value=float("nan"),
is_lower_better=metric_name in lower_is_better,
)
)
continue
is_lower = metric_name in lower_is_better
if is_lower:
passed = actual <= required
else:
passed = actual >= required
results.append(
SafetyGateResult(
field=field,
passed=passed,
metric_name=metric_name,
required_value=required,
actual_value=actual,
is_lower_better=is_lower,
)
)
return results
services/intelligence_pipeline_v3/schemas/samples.py
+465
View File
@@ -0,0 +1,465 @@
"""Sample annotations as test fixtures for the v3 annotation schema.
These samples demonstrate correct annotation format and serve as regression
fixtures for the validator. They cover representative document types and
complexity levels from the Gold Corpus.
Schema version: 1.0.0
"""
from __future__ import annotations
from datetime import datetime, timezone
from services.intelligence_pipeline_v3.schemas.annotations import (
AmbiguityMarker,
AmbiguityType,
AnnotatedDocument,
AnnotationMetadata,
CompanySentimentAnnotation,
DirectEffect,
EntityAnnotation,
EntityType,
EventAnnotation,
EventClass,
EvidenceSpanAnnotation,
InferredExposure,
NumericFactAnnotation,
PeriodAnnotation,
PeriodType,
RelationAnnotation,
RelationType,
SentimentLabel,
)
# ---------------------------------------------------------------------------
# Sample 1: Simple earnings beat article (single company, fast path)
# ---------------------------------------------------------------------------
_EARNINGS_TEXT = (
"Apple Inc. reported quarterly earnings of $1.52 per share, "
"beating the consensus estimate of $1.43 by $0.09. "
"Revenue came in at $94.9 billion, above expectations of $92.1 billion. "
"The company raised its dividend by 4% to $0.26 per share."
)
def build_sample_earnings_beat() -> AnnotatedDocument:
"""Single-company earnings beat with numeric facts and clear sentiment."""
ev_apple = EvidenceSpanAnnotation(
id="ev-001",
start_char=0,
end_char=10,
text="Apple Inc.",
)
ev_eps = EvidenceSpanAnnotation(
id="ev-002",
start_char=11,
end_char=108,
text="reported quarterly earnings of $1.52 per share, beating the consensus estimate of $1.43 by $0.09.",
)
ev_revenue = EvidenceSpanAnnotation(
id="ev-003",
start_char=109,
end_char=179,
text="Revenue came in at $94.9 billion, above expectations of $92.1 billion.",
)
ev_dividend = EvidenceSpanAnnotation(
id="ev-004",
start_char=180,
end_char=237,
text="The company raised its dividend by 4% to $0.26 per share.",
)
entity_apple = EntityAnnotation(
id="ent-001",
entity_type=EntityType.COMPANY,
literal_text="Apple Inc.",
canonical_id="aapl-uuid",
canonical_name="AAPL",
evidence_ids=["ev-001"],
confidence=1.0,
derivation="deterministic",
)
event_beat = EventAnnotation(
id="evt-001",
event_class=EventClass.EARNINGS_BEAT,
description="Apple Q1 FY2025 earnings beat consensus by $0.09/share",
primary_company_ids=["ent-001"],
evidence_ids=["ev-002"],
confidence=0.98,
derivation="specialist",
)
event_dividend = EventAnnotation(
id="evt-002",
event_class=EventClass.DIVIDEND_CHANGE,
description="Apple raises dividend by 4%",
primary_company_ids=["ent-001"],
evidence_ids=["ev-004"],
confidence=0.95,
derivation="specialist",
)
fact_eps = NumericFactAnnotation(
id="fact-001",
fact_type="eps",
subject_entity_id="ent-001",
predicate="reported",
literal_value="$1.52 per share",
normalized_value=1.52,
unit="USD",
period=PeriodAnnotation(
period_type=PeriodType.FISCAL_QUARTER,
fiscal_year=2025,
fiscal_quarter=1,
literal_text="quarterly",
),
evidence_ids=["ev-002"],
confidence=0.99,
derivation="deterministic",
)
fact_revenue = NumericFactAnnotation(
id="fact-002",
fact_type="revenue",
subject_entity_id="ent-001",
predicate="reported",
literal_value="$94.9 billion",
normalized_value=94_900_000_000,
unit="USD",
evidence_ids=["ev-003"],
confidence=0.99,
derivation="deterministic",
)
sentiment = CompanySentimentAnnotation(
id="sent-001",
company_entity_id="ent-001",
label=SentimentLabel.POSITIVE,
positive_probability=0.88,
negative_probability=0.04,
neutral_probability=0.08,
evidence_ids=["ev-002", "ev-003", "ev-004"],
confidence=0.92,
derivation="specialist",
)
direct = DirectEffect(
event_id="evt-001",
company_entity_id="ent-001",
evidence_ids=["ev-002"],
confidence=0.98,
)
return AnnotatedDocument(
document_id="doc-sample-001",
document_type="article",
source_text=_EARNINGS_TEXT,
metadata=AnnotationMetadata(
schema_version="1.0.0",
annotator_id="gold-annotator-1",
annotation_date=datetime(2025, 1, 15, tzinfo=timezone.utc),
review_status="gold",
reviewer_id="senior-reviewer-1",
review_date=datetime(2025, 1, 16, tzinfo=timezone.utc),
),
evidence_spans=[ev_apple, ev_eps, ev_revenue, ev_dividend],
entities=[entity_apple],
events=[event_beat, event_dividend],
relations=[],
numeric_facts=[fact_eps, fact_revenue],
sentiments=[sentiment],
direct_effects=[direct],
inferred_exposures=[],
ambiguity_markers=[],
)
# ---------------------------------------------------------------------------
# Sample 2: Multi-company competitive article (requires adjudication)
# ---------------------------------------------------------------------------
_MULTI_COMPANY_TEXT = (
"Microsoft announced a $10 billion investment in OpenAI, "
"intensifying competition with Google in the AI space. "
"Analysts expect this deal to pressure Alphabet's cloud revenue growth, "
"though some see it as validation of the broader AI investment thesis."
)
def build_sample_multi_company_competitive() -> AnnotatedDocument:
"""Multi-company article with competing sentiments and inferred exposure."""
ev_msft = EvidenceSpanAnnotation(
id="ev-101",
start_char=0,
end_char=9,
text="Microsoft",
)
ev_deal = EvidenceSpanAnnotation(
id="ev-102",
start_char=10,
end_char=55,
text="announced a $10 billion investment in OpenAI,",
)
ev_competition = EvidenceSpanAnnotation(
id="ev-103",
start_char=56,
end_char=109,
text="intensifying competition with Google in the AI space.",
)
ev_pressure = EvidenceSpanAnnotation(
id="ev-104",
start_char=110,
end_char=180,
text="Analysts expect this deal to pressure Alphabet's cloud revenue growth,",
)
ev_validation = EvidenceSpanAnnotation(
id="ev-105",
start_char=181,
end_char=250,
text="though some see it as validation of the broader AI investment thesis.",
)
ent_msft = EntityAnnotation(
id="ent-101",
entity_type=EntityType.COMPANY,
literal_text="Microsoft",
canonical_id="msft-uuid",
canonical_name="MSFT",
evidence_ids=["ev-101"],
confidence=1.0,
derivation="deterministic",
)
ent_goog = EntityAnnotation(
id="ent-102",
entity_type=EntityType.COMPANY,
literal_text="Google",
canonical_id="googl-uuid",
canonical_name="GOOGL",
evidence_ids=["ev-103"],
confidence=0.98,
derivation="deterministic",
)
ent_alphabet = EntityAnnotation(
id="ent-103",
entity_type=EntityType.COMPANY,
literal_text="Alphabet",
canonical_id="googl-uuid",
canonical_name="GOOGL",
evidence_ids=["ev-104"],
confidence=0.97,
derivation="specialist",
)
event_ma = EventAnnotation(
id="evt-101",
event_class=EventClass.MA_ANNOUNCEMENT,
description="Microsoft $10B investment in OpenAI",
primary_company_ids=["ent-101"],
evidence_ids=["ev-102"],
confidence=0.96,
derivation="specialist",
)
rel_competes = RelationAnnotation(
id="rel-101",
relation_type=RelationType.COMPETES_WITH,
source_id="ent-101",
target_id="ent-102",
evidence_ids=["ev-103"],
confidence=0.90,
derivation="specialist",
)
fact_amount = NumericFactAnnotation(
id="fact-101",
fact_type="investment_amount",
subject_entity_id="ent-101",
predicate="invested",
literal_value="$10 billion",
normalized_value=10_000_000_000,
unit="USD",
evidence_ids=["ev-102"],
confidence=0.99,
derivation="deterministic",
)
sentiment_msft = CompanySentimentAnnotation(
id="sent-101",
company_entity_id="ent-101",
label=SentimentLabel.POSITIVE,
positive_probability=0.75,
negative_probability=0.05,
neutral_probability=0.20,
evidence_ids=["ev-102"],
confidence=0.85,
derivation="specialist",
)
sentiment_goog = CompanySentimentAnnotation(
id="sent-102",
company_entity_id="ent-102",
label=SentimentLabel.MIXED,
positive_probability=0.30,
negative_probability=0.45,
neutral_probability=0.25,
evidence_ids=["ev-103", "ev-104", "ev-105"],
confidence=0.70,
derivation="specialist",
)
direct_msft = DirectEffect(
event_id="evt-101",
company_entity_id="ent-101",
evidence_ids=["ev-102"],
confidence=0.96,
)
inferred_goog = InferredExposure(
event_id="evt-101",
company_entity_id="ent-102",
reasoning="Competitive pressure from Microsoft's AI investment threatens Google's cloud market share",
evidence_ids=["ev-103", "ev-104"],
confidence=0.72,
)
ambiguity = AmbiguityMarker(
ambiguity_type=AmbiguityType.CONFLICTING_SENTIMENT,
description="Alphabet sentiment is mixed — competitive pressure vs. AI thesis validation",
affected_entity_ids=["ent-102", "ent-103"],
severity="medium",
)
return AnnotatedDocument(
document_id="doc-sample-002",
document_type="article",
source_text=_MULTI_COMPANY_TEXT,
metadata=AnnotationMetadata(
schema_version="1.0.0",
annotator_id="gold-annotator-2",
annotation_date=datetime(2025, 1, 20, tzinfo=timezone.utc),
review_status="gold",
reviewer_id="senior-reviewer-1",
review_date=datetime(2025, 1, 21, tzinfo=timezone.utc),
),
evidence_spans=[ev_msft, ev_deal, ev_competition, ev_pressure, ev_validation],
entities=[ent_msft, ent_goog, ent_alphabet],
events=[event_ma],
relations=[rel_competes],
numeric_facts=[fact_amount],
sentiments=[sentiment_msft, sentiment_goog],
direct_effects=[direct_msft],
inferred_exposures=[inferred_goog],
ambiguity_markers=[ambiguity],
)
# ---------------------------------------------------------------------------
# Sample 3: Macro event with inferred sector exposure
# ---------------------------------------------------------------------------
_MACRO_TEXT = (
"The Federal Reserve raised interest rates by 25 basis points to 5.50%, "
"citing persistent inflation concerns. Markets sold off broadly, "
"with technology stocks leading the decline."
)
def build_sample_macro_event() -> AnnotatedDocument:
"""Macro event with sector-level inferred exposure and no single primary company."""
ev_rate = EvidenceSpanAnnotation(
id="ev-202",
start_char=0,
end_char=70,
text="The Federal Reserve raised interest rates by 25 basis points to 5.50%,",
)
ev_inflation = EvidenceSpanAnnotation(
id="ev-203",
start_char=71,
end_char=108,
text="citing persistent inflation concerns.",
)
ev_selloff = EvidenceSpanAnnotation(
id="ev-204",
start_char=109,
end_char=178,
text="Markets sold off broadly, with technology stocks leading the decline.",
)
ent_fed = EntityAnnotation(
id="ent-201",
entity_type=EntityType.COMPANY,
literal_text="The Federal Reserve",
canonical_id=None,
canonical_name="Federal Reserve",
evidence_ids=["ev-202"],
confidence=1.0,
derivation="deterministic",
)
event_macro = EventAnnotation(
id="evt-201",
event_class=EventClass.MACRO_EVENT,
description="Fed raises rates 25bps to 5.50%",
primary_company_ids=[],
evidence_ids=["ev-202", "ev-203", "ev-204"],
confidence=0.99,
derivation="deterministic",
)
fact_rate = NumericFactAnnotation(
id="fact-201",
fact_type="interest_rate_change",
subject_entity_id="ent-201",
predicate="raised_by",
literal_value="25 basis points",
normalized_value=0.25,
unit="percentage_points",
evidence_ids=["ev-202"],
confidence=0.99,
derivation="deterministic",
)
fact_level = NumericFactAnnotation(
id="fact-202",
fact_type="interest_rate_level",
subject_entity_id="ent-201",
predicate="to",
literal_value="5.50%",
normalized_value=5.50,
unit="%",
evidence_ids=["ev-202"],
confidence=0.99,
derivation="deterministic",
)
return AnnotatedDocument(
document_id="doc-sample-003",
document_type="macro_event",
source_text=_MACRO_TEXT,
metadata=AnnotationMetadata(
schema_version="1.0.0",
annotator_id="gold-annotator-1",
annotation_date=datetime(2025, 2, 1, tzinfo=timezone.utc),
review_status="gold",
),
evidence_spans=[ev_rate, ev_inflation, ev_selloff],
entities=[ent_fed],
events=[event_macro],
relations=[],
numeric_facts=[fact_rate, fact_level],
sentiments=[],
direct_effects=[],
inferred_exposures=[],
ambiguity_markers=[],
)
# All sample builders for easy iteration
SAMPLE_BUILDERS = [
build_sample_earnings_beat,
build_sample_multi_company_competitive,
build_sample_macro_event,
]
services/intelligence_pipeline_v3/schemas/validators.py
+328
View File
@@ -0,0 +1,328 @@
"""Schema validators for v3 annotations.
Validates completeness, cross-references, evidence coverage, and offset integrity
for annotated documents before they enter the Gold Corpus or production pipeline.
Schema version: 1.0.0
"""
from __future__ import annotations
from dataclasses import dataclass, field
from enum import Enum
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from services.intelligence_pipeline_v3.schemas.annotations import AnnotatedDocument
class ValidationSeverity(str, Enum):
ERROR = "error"
WARNING = "warning"
@dataclass
class ValidationError:
"""A single validation issue found in an annotation."""
severity: ValidationSeverity
field_path: str
message: str
entity_id: str | None = None
@dataclass
class ValidationResult:
"""Complete validation result for an annotated document."""
valid: bool
errors: list[ValidationError] = field(default_factory=list)
warnings: list[ValidationError] = field(default_factory=list)
@property
def error_count(self) -> int:
return len(self.errors)
@property
def warning_count(self) -> int:
return len(self.warnings)
def validate_annotation(doc: "AnnotatedDocument") -> ValidationResult:
"""Validate completeness and cross-references of an annotated document.
Checks performed:
1. All evidence_ids referenced by entities/events/relations/facts exist in evidence_spans
2. All entity IDs referenced by events/relations/effects exist in entities
3. Evidence span offsets are within the source_text bounds
4. Evidence span text matches the source_text at the given offsets
5. Sentiment probabilities are valid
6. No orphaned evidence spans (warning only)
7. Direct effects reference valid event and entity IDs
8. Inferred exposures reference valid event and entity IDs
"""
errors: list[ValidationError] = []
warnings: list[ValidationError] = []
# Build lookup indexes
evidence_ids = {span.id for span in doc.evidence_spans}
entity_ids = {entity.id for entity in doc.entities}
event_ids = {event.id for event in doc.events}
all_annotation_ids = evidence_ids | entity_ids | event_ids
# Track which evidence spans are referenced
referenced_evidence: set[str] = set()
# 1. Validate evidence spans themselves
for i, span in enumerate(doc.evidence_spans):
path = f"evidence_spans[{i}]"
# Offset bounds check
if span.start_char >= len(doc.source_text):
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.start_char",
message=f"start_char {span.start_char} exceeds source_text length {len(doc.source_text)}",
entity_id=span.id,
)
)
elif span.end_char > len(doc.source_text):
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.end_char",
message=f"end_char {span.end_char} exceeds source_text length {len(doc.source_text)}",
entity_id=span.id,
)
)
else:
# Text match check
expected_text = doc.source_text[span.start_char : span.end_char]
if expected_text != span.text:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.text",
message=(
f"Span text does not match source_text at offsets "
f"[{span.start_char}:{span.end_char}]. "
f"Expected: {expected_text!r}, got: {span.text!r}"
),
entity_id=span.id,
)
)
# 2. Validate entity evidence references
for i, entity in enumerate(doc.entities):
path = f"entities[{i}]"
for eid in entity.evidence_ids:
referenced_evidence.add(eid)
if eid not in evidence_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.evidence_ids",
message=f"References non-existent evidence span: {eid}",
entity_id=entity.id,
)
)
# 3. Validate event evidence and entity references
for i, event in enumerate(doc.events):
path = f"events[{i}]"
for eid in event.evidence_ids:
referenced_evidence.add(eid)
if eid not in evidence_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.evidence_ids",
message=f"References non-existent evidence span: {eid}",
entity_id=event.id,
)
)
for cid in event.primary_company_ids:
if cid not in entity_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.primary_company_ids",
message=f"References non-existent entity: {cid}",
entity_id=event.id,
)
)
# 4. Validate relation references
for i, rel in enumerate(doc.relations):
path = f"relations[{i}]"
for eid in rel.evidence_ids:
referenced_evidence.add(eid)
if eid not in evidence_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.evidence_ids",
message=f"References non-existent evidence span: {eid}",
entity_id=rel.id,
)
)
if rel.source_id not in all_annotation_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.source_id",
message=f"source_id references non-existent annotation: {rel.source_id}",
entity_id=rel.id,
)
)
if rel.target_id not in entity_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.target_id",
message=f"target_id references non-existent entity: {rel.target_id}",
entity_id=rel.id,
)
)
# 5. Validate numeric fact references
for i, fact in enumerate(doc.numeric_facts):
path = f"numeric_facts[{i}]"
for eid in fact.evidence_ids:
referenced_evidence.add(eid)
if eid not in evidence_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.evidence_ids",
message=f"References non-existent evidence span: {eid}",
entity_id=fact.id,
)
)
if fact.subject_entity_id and fact.subject_entity_id not in entity_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.subject_entity_id",
message=f"References non-existent entity: {fact.subject_entity_id}",
entity_id=fact.id,
)
)
# 6. Validate sentiment references
for i, sent in enumerate(doc.sentiments):
path = f"sentiments[{i}]"
for eid in sent.evidence_ids:
referenced_evidence.add(eid)
if eid not in evidence_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.evidence_ids",
message=f"References non-existent evidence span: {eid}",
entity_id=sent.id,
)
)
if sent.company_entity_id not in entity_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.company_entity_id",
message=f"References non-existent entity: {sent.company_entity_id}",
entity_id=sent.id,
)
)
# 7. Validate direct effects
for i, effect in enumerate(doc.direct_effects):
path = f"direct_effects[{i}]"
for eid in effect.evidence_ids:
referenced_evidence.add(eid)
if eid not in evidence_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.evidence_ids",
message=f"References non-existent evidence span: {eid}",
)
)
if effect.event_id not in event_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.event_id",
message=f"References non-existent event: {effect.event_id}",
)
)
if effect.company_entity_id not in entity_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.company_entity_id",
message=f"References non-existent entity: {effect.company_entity_id}",
)
)
# 8. Validate inferred exposures
for i, exposure in enumerate(doc.inferred_exposures):
path = f"inferred_exposures[{i}]"
for eid in exposure.evidence_ids:
referenced_evidence.add(eid)
if eid not in evidence_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.evidence_ids",
message=f"References non-existent evidence span: {eid}",
)
)
if exposure.event_id not in event_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.event_id",
message=f"References non-existent event: {exposure.event_id}",
)
)
if exposure.company_entity_id not in entity_ids:
errors.append(
ValidationError(
severity=ValidationSeverity.ERROR,
field_path=f"{path}.company_entity_id",
message=f"References non-existent entity: {exposure.company_entity_id}",
)
)
# 9. Check for orphaned evidence spans (warning)
orphaned = evidence_ids - referenced_evidence
for eid in orphaned:
warnings.append(
ValidationError(
severity=ValidationSeverity.WARNING,
field_path="evidence_spans",
message=f"Evidence span {eid} is not referenced by any annotation.",
entity_id=eid,
)
)
# 10. Check minimum annotation completeness
if not doc.entities:
warnings.append(
ValidationError(
severity=ValidationSeverity.WARNING,
field_path="entities",
message="Document has no entity annotations.",
)
)
if not doc.events:
warnings.append(
ValidationError(
severity=ValidationSeverity.WARNING,
field_path="events",
message="Document has no event annotations.",
)
)
is_valid = len(errors) == 0
return ValidationResult(valid=is_valid, errors=errors, warnings=warnings)