feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,465 @@
|
||||
"""Sample annotations as test fixtures for the v3 annotation schema.
|
||||
|
||||
These samples demonstrate correct annotation format and serve as regression
|
||||
fixtures for the validator. They cover representative document types and
|
||||
complexity levels from the Gold Corpus.
|
||||
|
||||
Schema version: 1.0.0
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from services.intelligence_pipeline_v3.schemas.annotations import (
|
||||
AmbiguityMarker,
|
||||
AmbiguityType,
|
||||
AnnotatedDocument,
|
||||
AnnotationMetadata,
|
||||
CompanySentimentAnnotation,
|
||||
DirectEffect,
|
||||
EntityAnnotation,
|
||||
EntityType,
|
||||
EventAnnotation,
|
||||
EventClass,
|
||||
EvidenceSpanAnnotation,
|
||||
InferredExposure,
|
||||
NumericFactAnnotation,
|
||||
PeriodAnnotation,
|
||||
PeriodType,
|
||||
RelationAnnotation,
|
||||
RelationType,
|
||||
SentimentLabel,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sample 1: Simple earnings beat article (single company, fast path)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_EARNINGS_TEXT = (
|
||||
"Apple Inc. reported quarterly earnings of $1.52 per share, "
|
||||
"beating the consensus estimate of $1.43 by $0.09. "
|
||||
"Revenue came in at $94.9 billion, above expectations of $92.1 billion. "
|
||||
"The company raised its dividend by 4% to $0.26 per share."
|
||||
)
|
||||
|
||||
|
||||
def build_sample_earnings_beat() -> AnnotatedDocument:
|
||||
"""Single-company earnings beat with numeric facts and clear sentiment."""
|
||||
ev_apple = EvidenceSpanAnnotation(
|
||||
id="ev-001",
|
||||
start_char=0,
|
||||
end_char=10,
|
||||
text="Apple Inc.",
|
||||
)
|
||||
ev_eps = EvidenceSpanAnnotation(
|
||||
id="ev-002",
|
||||
start_char=11,
|
||||
end_char=108,
|
||||
text="reported quarterly earnings of $1.52 per share, beating the consensus estimate of $1.43 by $0.09.",
|
||||
)
|
||||
ev_revenue = EvidenceSpanAnnotation(
|
||||
id="ev-003",
|
||||
start_char=109,
|
||||
end_char=179,
|
||||
text="Revenue came in at $94.9 billion, above expectations of $92.1 billion.",
|
||||
)
|
||||
ev_dividend = EvidenceSpanAnnotation(
|
||||
id="ev-004",
|
||||
start_char=180,
|
||||
end_char=237,
|
||||
text="The company raised its dividend by 4% to $0.26 per share.",
|
||||
)
|
||||
|
||||
entity_apple = EntityAnnotation(
|
||||
id="ent-001",
|
||||
entity_type=EntityType.COMPANY,
|
||||
literal_text="Apple Inc.",
|
||||
canonical_id="aapl-uuid",
|
||||
canonical_name="AAPL",
|
||||
evidence_ids=["ev-001"],
|
||||
confidence=1.0,
|
||||
derivation="deterministic",
|
||||
)
|
||||
|
||||
event_beat = EventAnnotation(
|
||||
id="evt-001",
|
||||
event_class=EventClass.EARNINGS_BEAT,
|
||||
description="Apple Q1 FY2025 earnings beat consensus by $0.09/share",
|
||||
primary_company_ids=["ent-001"],
|
||||
evidence_ids=["ev-002"],
|
||||
confidence=0.98,
|
||||
derivation="specialist",
|
||||
)
|
||||
|
||||
event_dividend = EventAnnotation(
|
||||
id="evt-002",
|
||||
event_class=EventClass.DIVIDEND_CHANGE,
|
||||
description="Apple raises dividend by 4%",
|
||||
primary_company_ids=["ent-001"],
|
||||
evidence_ids=["ev-004"],
|
||||
confidence=0.95,
|
||||
derivation="specialist",
|
||||
)
|
||||
|
||||
fact_eps = NumericFactAnnotation(
|
||||
id="fact-001",
|
||||
fact_type="eps",
|
||||
subject_entity_id="ent-001",
|
||||
predicate="reported",
|
||||
literal_value="$1.52 per share",
|
||||
normalized_value=1.52,
|
||||
unit="USD",
|
||||
period=PeriodAnnotation(
|
||||
period_type=PeriodType.FISCAL_QUARTER,
|
||||
fiscal_year=2025,
|
||||
fiscal_quarter=1,
|
||||
literal_text="quarterly",
|
||||
),
|
||||
evidence_ids=["ev-002"],
|
||||
confidence=0.99,
|
||||
derivation="deterministic",
|
||||
)
|
||||
|
||||
fact_revenue = NumericFactAnnotation(
|
||||
id="fact-002",
|
||||
fact_type="revenue",
|
||||
subject_entity_id="ent-001",
|
||||
predicate="reported",
|
||||
literal_value="$94.9 billion",
|
||||
normalized_value=94_900_000_000,
|
||||
unit="USD",
|
||||
evidence_ids=["ev-003"],
|
||||
confidence=0.99,
|
||||
derivation="deterministic",
|
||||
)
|
||||
|
||||
sentiment = CompanySentimentAnnotation(
|
||||
id="sent-001",
|
||||
company_entity_id="ent-001",
|
||||
label=SentimentLabel.POSITIVE,
|
||||
positive_probability=0.88,
|
||||
negative_probability=0.04,
|
||||
neutral_probability=0.08,
|
||||
evidence_ids=["ev-002", "ev-003", "ev-004"],
|
||||
confidence=0.92,
|
||||
derivation="specialist",
|
||||
)
|
||||
|
||||
direct = DirectEffect(
|
||||
event_id="evt-001",
|
||||
company_entity_id="ent-001",
|
||||
evidence_ids=["ev-002"],
|
||||
confidence=0.98,
|
||||
)
|
||||
|
||||
return AnnotatedDocument(
|
||||
document_id="doc-sample-001",
|
||||
document_type="article",
|
||||
source_text=_EARNINGS_TEXT,
|
||||
metadata=AnnotationMetadata(
|
||||
schema_version="1.0.0",
|
||||
annotator_id="gold-annotator-1",
|
||||
annotation_date=datetime(2025, 1, 15, tzinfo=timezone.utc),
|
||||
review_status="gold",
|
||||
reviewer_id="senior-reviewer-1",
|
||||
review_date=datetime(2025, 1, 16, tzinfo=timezone.utc),
|
||||
),
|
||||
evidence_spans=[ev_apple, ev_eps, ev_revenue, ev_dividend],
|
||||
entities=[entity_apple],
|
||||
events=[event_beat, event_dividend],
|
||||
relations=[],
|
||||
numeric_facts=[fact_eps, fact_revenue],
|
||||
sentiments=[sentiment],
|
||||
direct_effects=[direct],
|
||||
inferred_exposures=[],
|
||||
ambiguity_markers=[],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sample 2: Multi-company competitive article (requires adjudication)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MULTI_COMPANY_TEXT = (
|
||||
"Microsoft announced a $10 billion investment in OpenAI, "
|
||||
"intensifying competition with Google in the AI space. "
|
||||
"Analysts expect this deal to pressure Alphabet's cloud revenue growth, "
|
||||
"though some see it as validation of the broader AI investment thesis."
|
||||
)
|
||||
|
||||
|
||||
def build_sample_multi_company_competitive() -> AnnotatedDocument:
|
||||
"""Multi-company article with competing sentiments and inferred exposure."""
|
||||
ev_msft = EvidenceSpanAnnotation(
|
||||
id="ev-101",
|
||||
start_char=0,
|
||||
end_char=9,
|
||||
text="Microsoft",
|
||||
)
|
||||
ev_deal = EvidenceSpanAnnotation(
|
||||
id="ev-102",
|
||||
start_char=10,
|
||||
end_char=55,
|
||||
text="announced a $10 billion investment in OpenAI,",
|
||||
)
|
||||
ev_competition = EvidenceSpanAnnotation(
|
||||
id="ev-103",
|
||||
start_char=56,
|
||||
end_char=109,
|
||||
text="intensifying competition with Google in the AI space.",
|
||||
)
|
||||
ev_pressure = EvidenceSpanAnnotation(
|
||||
id="ev-104",
|
||||
start_char=110,
|
||||
end_char=180,
|
||||
text="Analysts expect this deal to pressure Alphabet's cloud revenue growth,",
|
||||
)
|
||||
ev_validation = EvidenceSpanAnnotation(
|
||||
id="ev-105",
|
||||
start_char=181,
|
||||
end_char=250,
|
||||
text="though some see it as validation of the broader AI investment thesis.",
|
||||
)
|
||||
|
||||
ent_msft = EntityAnnotation(
|
||||
id="ent-101",
|
||||
entity_type=EntityType.COMPANY,
|
||||
literal_text="Microsoft",
|
||||
canonical_id="msft-uuid",
|
||||
canonical_name="MSFT",
|
||||
evidence_ids=["ev-101"],
|
||||
confidence=1.0,
|
||||
derivation="deterministic",
|
||||
)
|
||||
ent_goog = EntityAnnotation(
|
||||
id="ent-102",
|
||||
entity_type=EntityType.COMPANY,
|
||||
literal_text="Google",
|
||||
canonical_id="googl-uuid",
|
||||
canonical_name="GOOGL",
|
||||
evidence_ids=["ev-103"],
|
||||
confidence=0.98,
|
||||
derivation="deterministic",
|
||||
)
|
||||
ent_alphabet = EntityAnnotation(
|
||||
id="ent-103",
|
||||
entity_type=EntityType.COMPANY,
|
||||
literal_text="Alphabet",
|
||||
canonical_id="googl-uuid",
|
||||
canonical_name="GOOGL",
|
||||
evidence_ids=["ev-104"],
|
||||
confidence=0.97,
|
||||
derivation="specialist",
|
||||
)
|
||||
|
||||
event_ma = EventAnnotation(
|
||||
id="evt-101",
|
||||
event_class=EventClass.MA_ANNOUNCEMENT,
|
||||
description="Microsoft $10B investment in OpenAI",
|
||||
primary_company_ids=["ent-101"],
|
||||
evidence_ids=["ev-102"],
|
||||
confidence=0.96,
|
||||
derivation="specialist",
|
||||
)
|
||||
|
||||
rel_competes = RelationAnnotation(
|
||||
id="rel-101",
|
||||
relation_type=RelationType.COMPETES_WITH,
|
||||
source_id="ent-101",
|
||||
target_id="ent-102",
|
||||
evidence_ids=["ev-103"],
|
||||
confidence=0.90,
|
||||
derivation="specialist",
|
||||
)
|
||||
|
||||
fact_amount = NumericFactAnnotation(
|
||||
id="fact-101",
|
||||
fact_type="investment_amount",
|
||||
subject_entity_id="ent-101",
|
||||
predicate="invested",
|
||||
literal_value="$10 billion",
|
||||
normalized_value=10_000_000_000,
|
||||
unit="USD",
|
||||
evidence_ids=["ev-102"],
|
||||
confidence=0.99,
|
||||
derivation="deterministic",
|
||||
)
|
||||
|
||||
sentiment_msft = CompanySentimentAnnotation(
|
||||
id="sent-101",
|
||||
company_entity_id="ent-101",
|
||||
label=SentimentLabel.POSITIVE,
|
||||
positive_probability=0.75,
|
||||
negative_probability=0.05,
|
||||
neutral_probability=0.20,
|
||||
evidence_ids=["ev-102"],
|
||||
confidence=0.85,
|
||||
derivation="specialist",
|
||||
)
|
||||
|
||||
sentiment_goog = CompanySentimentAnnotation(
|
||||
id="sent-102",
|
||||
company_entity_id="ent-102",
|
||||
label=SentimentLabel.MIXED,
|
||||
positive_probability=0.30,
|
||||
negative_probability=0.45,
|
||||
neutral_probability=0.25,
|
||||
evidence_ids=["ev-103", "ev-104", "ev-105"],
|
||||
confidence=0.70,
|
||||
derivation="specialist",
|
||||
)
|
||||
|
||||
direct_msft = DirectEffect(
|
||||
event_id="evt-101",
|
||||
company_entity_id="ent-101",
|
||||
evidence_ids=["ev-102"],
|
||||
confidence=0.96,
|
||||
)
|
||||
|
||||
inferred_goog = InferredExposure(
|
||||
event_id="evt-101",
|
||||
company_entity_id="ent-102",
|
||||
reasoning="Competitive pressure from Microsoft's AI investment threatens Google's cloud market share",
|
||||
evidence_ids=["ev-103", "ev-104"],
|
||||
confidence=0.72,
|
||||
)
|
||||
|
||||
ambiguity = AmbiguityMarker(
|
||||
ambiguity_type=AmbiguityType.CONFLICTING_SENTIMENT,
|
||||
description="Alphabet sentiment is mixed — competitive pressure vs. AI thesis validation",
|
||||
affected_entity_ids=["ent-102", "ent-103"],
|
||||
severity="medium",
|
||||
)
|
||||
|
||||
return AnnotatedDocument(
|
||||
document_id="doc-sample-002",
|
||||
document_type="article",
|
||||
source_text=_MULTI_COMPANY_TEXT,
|
||||
metadata=AnnotationMetadata(
|
||||
schema_version="1.0.0",
|
||||
annotator_id="gold-annotator-2",
|
||||
annotation_date=datetime(2025, 1, 20, tzinfo=timezone.utc),
|
||||
review_status="gold",
|
||||
reviewer_id="senior-reviewer-1",
|
||||
review_date=datetime(2025, 1, 21, tzinfo=timezone.utc),
|
||||
),
|
||||
evidence_spans=[ev_msft, ev_deal, ev_competition, ev_pressure, ev_validation],
|
||||
entities=[ent_msft, ent_goog, ent_alphabet],
|
||||
events=[event_ma],
|
||||
relations=[rel_competes],
|
||||
numeric_facts=[fact_amount],
|
||||
sentiments=[sentiment_msft, sentiment_goog],
|
||||
direct_effects=[direct_msft],
|
||||
inferred_exposures=[inferred_goog],
|
||||
ambiguity_markers=[ambiguity],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sample 3: Macro event with inferred sector exposure
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MACRO_TEXT = (
|
||||
"The Federal Reserve raised interest rates by 25 basis points to 5.50%, "
|
||||
"citing persistent inflation concerns. Markets sold off broadly, "
|
||||
"with technology stocks leading the decline."
|
||||
)
|
||||
|
||||
|
||||
def build_sample_macro_event() -> AnnotatedDocument:
|
||||
"""Macro event with sector-level inferred exposure and no single primary company."""
|
||||
ev_rate = EvidenceSpanAnnotation(
|
||||
id="ev-202",
|
||||
start_char=0,
|
||||
end_char=70,
|
||||
text="The Federal Reserve raised interest rates by 25 basis points to 5.50%,",
|
||||
)
|
||||
ev_inflation = EvidenceSpanAnnotation(
|
||||
id="ev-203",
|
||||
start_char=71,
|
||||
end_char=108,
|
||||
text="citing persistent inflation concerns.",
|
||||
)
|
||||
ev_selloff = EvidenceSpanAnnotation(
|
||||
id="ev-204",
|
||||
start_char=109,
|
||||
end_char=178,
|
||||
text="Markets sold off broadly, with technology stocks leading the decline.",
|
||||
)
|
||||
|
||||
ent_fed = EntityAnnotation(
|
||||
id="ent-201",
|
||||
entity_type=EntityType.COMPANY,
|
||||
literal_text="The Federal Reserve",
|
||||
canonical_id=None,
|
||||
canonical_name="Federal Reserve",
|
||||
evidence_ids=["ev-202"],
|
||||
confidence=1.0,
|
||||
derivation="deterministic",
|
||||
)
|
||||
|
||||
event_macro = EventAnnotation(
|
||||
id="evt-201",
|
||||
event_class=EventClass.MACRO_EVENT,
|
||||
description="Fed raises rates 25bps to 5.50%",
|
||||
primary_company_ids=[],
|
||||
evidence_ids=["ev-202", "ev-203", "ev-204"],
|
||||
confidence=0.99,
|
||||
derivation="deterministic",
|
||||
)
|
||||
|
||||
fact_rate = NumericFactAnnotation(
|
||||
id="fact-201",
|
||||
fact_type="interest_rate_change",
|
||||
subject_entity_id="ent-201",
|
||||
predicate="raised_by",
|
||||
literal_value="25 basis points",
|
||||
normalized_value=0.25,
|
||||
unit="percentage_points",
|
||||
evidence_ids=["ev-202"],
|
||||
confidence=0.99,
|
||||
derivation="deterministic",
|
||||
)
|
||||
|
||||
fact_level = NumericFactAnnotation(
|
||||
id="fact-202",
|
||||
fact_type="interest_rate_level",
|
||||
subject_entity_id="ent-201",
|
||||
predicate="to",
|
||||
literal_value="5.50%",
|
||||
normalized_value=5.50,
|
||||
unit="%",
|
||||
evidence_ids=["ev-202"],
|
||||
confidence=0.99,
|
||||
derivation="deterministic",
|
||||
)
|
||||
|
||||
return AnnotatedDocument(
|
||||
document_id="doc-sample-003",
|
||||
document_type="macro_event",
|
||||
source_text=_MACRO_TEXT,
|
||||
metadata=AnnotationMetadata(
|
||||
schema_version="1.0.0",
|
||||
annotator_id="gold-annotator-1",
|
||||
annotation_date=datetime(2025, 2, 1, tzinfo=timezone.utc),
|
||||
review_status="gold",
|
||||
),
|
||||
evidence_spans=[ev_rate, ev_inflation, ev_selloff],
|
||||
entities=[ent_fed],
|
||||
events=[event_macro],
|
||||
relations=[],
|
||||
numeric_facts=[fact_rate, fact_level],
|
||||
sentiments=[],
|
||||
direct_effects=[],
|
||||
inferred_exposures=[],
|
||||
ambiguity_markers=[],
|
||||
)
|
||||
|
||||
|
||||
# All sample builders for easy iteration
|
||||
SAMPLE_BUILDERS = [
|
||||
build_sample_earnings_beat,
|
||||
build_sample_multi_company_competitive,
|
||||
build_sample_macro_event,
|
||||
]
|
||||
Reference in New Issue
Block a user