admin/stonks-oracle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: competitive intelligence & historical pattern matching layer

Browse Source
This commit is contained in:
Celes Renata
2026-04-14 19:42:48 +00:00
parent b478022ba3
commit f7a11d14ea
203 changed files with 20155 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/lake_publisher/jobs.py
+240
View File
@@ -39,12 +39,17 @@ from services.lake_publisher.worker import (
publish_document_extractions_batch,
publish_document_fact,
publish_documents_batch,
publish_global_event_fact,
publish_macro_impact_fact,
publish_market_bar,
publish_market_quote,
publish_pnl_daily,
publish_positions_daily_batch,
publish_trade_fill,
publish_trade_order,
publish_trend_projection_fact,
publish_competitor_relationship_fact,
publish_competitive_signal_fact,
)
from services.shared.config import load_config
from services.shared.db import get_minio, get_pg_pool, get_redis
@@ -164,6 +169,57 @@ ORDER BY di.created_at
LIMIT 500
"""
_FETCH_GLOBAL_EVENT = """
SELECT
ge.id, ge.event_types, ge.severity, ge.affected_regions,
ge.affected_sectors, ge.affected_commodities, ge.summary,
ge.estimated_duration, ge.confidence, ge.source_document_id,
ge.created_at
FROM global_events ge
WHERE ge.id = $1::uuid
"""
_FETCH_MACRO_IMPACTS_FOR_EVENT = """
SELECT
mir.event_id, mir.company_id, mir.ticker,
mir.macro_impact_score, mir.impact_direction,
mir.contributing_factors, mir.confidence, mir.computed_at
FROM macro_impact_records mir
WHERE mir.event_id = $1::uuid
"""
_FETCH_TREND_PROJECTION = """
SELECT
tp.id, tp.trend_window_id, tp.projected_direction,
tp.projected_strength, tp.projected_confidence,
tp.projection_horizon, tp.driving_factors,
tp.macro_contribution_pct, tp.diverges_from_current,
tp.computed_at,
tw.ticker
FROM trend_projections tp
JOIN trend_windows tw ON tw.id = tp.trend_window_id
WHERE tp.trend_window_id = $1::uuid
"""
_FETCH_COMPETITOR_RELATIONSHIP = """
SELECT
cr.id, cr.company_a_id, cr.company_b_id,
cr.relationship_type, cr.strength, cr.bidirectional,
cr.source, cr.active, cr.created_at
FROM competitor_relationships cr
WHERE cr.id = $1::uuid
"""
_FETCH_COMPETITIVE_SIGNALS_FOR_DOCUMENT = """
SELECT
csr.id, csr.source_document_id, csr.source_ticker,
csr.target_ticker, csr.catalyst_type, csr.pattern_confidence,
csr.signal_direction, csr.signal_strength,
csr.relationship_strength, csr.computed_at
FROM competitive_signal_records csr
WHERE csr.source_document_id = $1::uuid
"""
# ---------------------------------------------------------------------------
# Job handlers — each transforms operational rows into lake facts
@@ -510,6 +566,165 @@ async def publish_bulk_extractions_job(
return [ref] if ref else []
async def publish_global_event_job(
pool: asyncpg.Pool,
minio_client: Minio,
entity_id: str,
) -> str:
"""Publish a global event fact from PostgreSQL to the lake."""
row = await pool.fetchrow(_FETCH_GLOBAL_EVENT, entity_id)
if row is None:
logger.warning("Global event %s not found, skipping lake publish", entity_id)
return ""
event_types = row["event_types"] or []
affected_regions = row["affected_regions"] or []
affected_sectors = row["affected_sectors"] or []
affected_commodities = row["affected_commodities"] or []
return publish_global_event_fact(
client=minio_client,
event_id=str(row["id"]),
event_types=list(event_types),
severity=row["severity"] or "low",
affected_regions=list(affected_regions),
affected_sectors=list(affected_sectors),
affected_commodities=list(affected_commodities),
summary=row["summary"] or "",
estimated_duration=row["estimated_duration"] or "short_term",
confidence=float(row["confidence"] or 0.0),
source_document_id=str(row["source_document_id"]) if row["source_document_id"] else "",
created_at=row["created_at"],
)
async def publish_macro_impacts_job(
pool: asyncpg.Pool,
minio_client: Minio,
entity_id: str,
) -> list[str]:
"""Publish macro impact facts for a global event from PostgreSQL to the lake."""
rows = await pool.fetch(_FETCH_MACRO_IMPACTS_FOR_EVENT, entity_id)
if not rows:
logger.info("No macro impact records for event %s", entity_id)
return []
refs: list[str] = []
for row in rows:
factors = row["contributing_factors"]
if isinstance(factors, str):
try:
factors = json.loads(factors)
except (json.JSONDecodeError, TypeError):
factors = [factors] if factors else []
elif factors is None:
factors = []
ref = publish_macro_impact_fact(
client=minio_client,
event_id=str(row["event_id"]),
company_id=str(row["company_id"]),
ticker=row["ticker"],
macro_impact_score=float(row["macro_impact_score"] or 0.0),
impact_direction=row["impact_direction"] or "neutral",
contributing_factors=list(factors),
confidence=float(row["confidence"] or 0.0),
computed_at=row["computed_at"],
)
refs.append(ref)
return refs
async def publish_trend_projection_job(
pool: asyncpg.Pool,
minio_client: Minio,
entity_id: str,
) -> str:
"""Publish a trend projection fact from PostgreSQL to the lake."""
row = await pool.fetchrow(_FETCH_TREND_PROJECTION, entity_id)
if row is None:
logger.warning("Trend projection for window %s not found", entity_id)
return ""
factors = row["driving_factors"]
if isinstance(factors, str):
try:
factors = json.loads(factors)
except (json.JSONDecodeError, TypeError):
factors = [factors] if factors else []
elif factors is None:
factors = []
return publish_trend_projection_fact(
client=minio_client,
trend_window_id=str(row["trend_window_id"]),
ticker=row["ticker"] or "",
projected_direction=row["projected_direction"] or "neutral",
projected_strength=float(row["projected_strength"] or 0.0),
projected_confidence=float(row["projected_confidence"] or 0.0),
projection_horizon=row["projection_horizon"] or "7d",
driving_factors=list(factors),
macro_contribution_pct=float(row["macro_contribution_pct"] or 0.0),
diverges_from_current=bool(row["diverges_from_current"]),
computed_at=row["computed_at"],
)
async def publish_competitor_relationship_job(
pool: asyncpg.Pool,
minio_client: Minio,
entity_id: str,
) -> str:
"""Publish a competitor relationship fact from PostgreSQL to the lake."""
row = await pool.fetchrow(_FETCH_COMPETITOR_RELATIONSHIP, entity_id)
if row is None:
logger.warning("Competitor relationship %s not found, skipping lake publish", entity_id)
return ""
return publish_competitor_relationship_fact(
client=minio_client,
relationship_id=str(row["id"]),
company_a_id=str(row["company_a_id"]),
company_b_id=str(row["company_b_id"]),
relationship_type=row["relationship_type"],
strength=float(row["strength"]),
bidirectional=bool(row["bidirectional"]),
source=row["source"],
active=bool(row["active"]),
created_at=row["created_at"],
)
async def publish_competitive_signals_job(
pool: asyncpg.Pool,
minio_client: Minio,
entity_id: str,
) -> list[str]:
"""Publish competitive signal facts for a document from PostgreSQL to the lake."""
rows = await pool.fetch(_FETCH_COMPETITIVE_SIGNALS_FOR_DOCUMENT, entity_id)
if not rows:
logger.info("No competitive signals for document %s", entity_id)
return []
refs: list[str] = []
for row in rows:
ref = publish_competitive_signal_fact(
client=minio_client,
signal_id=str(row["id"]),
source_document_id=str(row["source_document_id"]),
source_ticker=row["source_ticker"],
target_ticker=row["target_ticker"],
catalyst_type=row["catalyst_type"],
pattern_confidence=float(row["pattern_confidence"]),
signal_direction=row["signal_direction"],
signal_strength=float(row["signal_strength"]),
relationship_strength=float(row["relationship_strength"]),
computed_at=row["computed_at"],
)
refs.append(ref)
return refs
# ---------------------------------------------------------------------------
# Job dispatcher
# ---------------------------------------------------------------------------
@@ -525,6 +740,11 @@ JOB_TYPES = {
"company_event",
"bulk_documents",
"bulk_extractions",
"global_event",
"macro_impact",
"trend_projection",
"competitor_relationship",
"competitive_signal",
}
@@ -594,6 +814,26 @@ async def dispatch_job(
refs = await publish_bulk_extractions_job(pool, minio_client, since)
result["refs"] = refs
elif job_type == "global_event":
ref = await publish_global_event_job(pool, minio_client, entity_id)
result["refs"] = [ref] if ref else []
elif job_type == "macro_impact":
refs = await publish_macro_impacts_job(pool, minio_client, entity_id)
result["refs"] = refs
elif job_type == "trend_projection":
ref = await publish_trend_projection_job(pool, minio_client, entity_id)
result["refs"] = [ref] if ref else []
elif job_type == "competitor_relationship":
ref = await publish_competitor_relationship_job(pool, minio_client, entity_id)
result["refs"] = [ref] if ref else []
elif job_type == "competitive_signal":
refs = await publish_competitive_signals_job(pool, minio_client, entity_id)
result["refs"] = refs
else:
result["error"] = f"Unknown job_type: {job_type}"
logger.warning("Unknown lake publish job type: %s", job_type)