admin/stonks-oracle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: competitive intelligence & historical pattern matching layer

Browse Source
This commit is contained in:
Celes Renata
2026-04-14 19:42:48 +00:00
parent b478022ba3
commit f7a11d14ea
203 changed files with 20155 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/lake_publisher/iceberg.py
+21
View File
@@ -124,6 +124,27 @@ TABLE_SCHEMAS: dict[str, pa.Schema] = {
"model_performance": MODEL_PERFORMANCE_SCHEMA,
}
# Lazily register schemas defined in worker.py to avoid circular imports.
# These are added after the initial dict definition.
def _register_worker_schemas() -> None:
from services.lake_publisher.worker import (
COMPETITOR_RELATIONSHIPS_SCHEMA,
COMPETITIVE_SIGNALS_SCHEMA,
GLOBAL_EVENTS_SCHEMA,
MACRO_IMPACTS_SCHEMA,
TREND_PROJECTIONS_SCHEMA,
)
TABLE_SCHEMAS["competitor_relationships"] = COMPETITOR_RELATIONSHIPS_SCHEMA
TABLE_SCHEMAS["competitive_signals"] = COMPETITIVE_SIGNALS_SCHEMA
TABLE_SCHEMAS["global_events"] = GLOBAL_EVENTS_SCHEMA
TABLE_SCHEMAS["macro_impacts"] = MACRO_IMPACTS_SCHEMA
TABLE_SCHEMAS["trend_projections"] = TREND_PROJECTIONS_SCHEMA
try:
_register_worker_schemas()
except ImportError:
pass # worker.py not available in minimal test environments
@dataclass(frozen=True)
class IcebergTableDef:
services/lake_publisher/jobs.py
+240
View File
@@ -39,12 +39,17 @@ from services.lake_publisher.worker import (
publish_document_extractions_batch,
publish_document_fact,
publish_documents_batch,
publish_global_event_fact,
publish_macro_impact_fact,
publish_market_bar,
publish_market_quote,
publish_pnl_daily,
publish_positions_daily_batch,
publish_trade_fill,
publish_trade_order,
publish_trend_projection_fact,
publish_competitor_relationship_fact,
publish_competitive_signal_fact,
)
from services.shared.config import load_config
from services.shared.db import get_minio, get_pg_pool, get_redis
@@ -164,6 +169,57 @@ ORDER BY di.created_at
LIMIT 500
"""
_FETCH_GLOBAL_EVENT = """
SELECT
ge.id, ge.event_types, ge.severity, ge.affected_regions,
ge.affected_sectors, ge.affected_commodities, ge.summary,
ge.estimated_duration, ge.confidence, ge.source_document_id,
ge.created_at
FROM global_events ge
WHERE ge.id = $1::uuid
"""
_FETCH_MACRO_IMPACTS_FOR_EVENT = """
SELECT
mir.event_id, mir.company_id, mir.ticker,
mir.macro_impact_score, mir.impact_direction,
mir.contributing_factors, mir.confidence, mir.computed_at
FROM macro_impact_records mir
WHERE mir.event_id = $1::uuid
"""
_FETCH_TREND_PROJECTION = """
SELECT
tp.id, tp.trend_window_id, tp.projected_direction,
tp.projected_strength, tp.projected_confidence,
tp.projection_horizon, tp.driving_factors,
tp.macro_contribution_pct, tp.diverges_from_current,
tp.computed_at,
tw.ticker
FROM trend_projections tp
JOIN trend_windows tw ON tw.id = tp.trend_window_id
WHERE tp.trend_window_id = $1::uuid
"""
_FETCH_COMPETITOR_RELATIONSHIP = """
SELECT
cr.id, cr.company_a_id, cr.company_b_id,
cr.relationship_type, cr.strength, cr.bidirectional,
cr.source, cr.active, cr.created_at
FROM competitor_relationships cr
WHERE cr.id = $1::uuid
"""
_FETCH_COMPETITIVE_SIGNALS_FOR_DOCUMENT = """
SELECT
csr.id, csr.source_document_id, csr.source_ticker,
csr.target_ticker, csr.catalyst_type, csr.pattern_confidence,
csr.signal_direction, csr.signal_strength,
csr.relationship_strength, csr.computed_at
FROM competitive_signal_records csr
WHERE csr.source_document_id = $1::uuid
"""
# ---------------------------------------------------------------------------
# Job handlers — each transforms operational rows into lake facts
@@ -510,6 +566,165 @@ async def publish_bulk_extractions_job(
return [ref] if ref else []
async def publish_global_event_job(
pool: asyncpg.Pool,
minio_client: Minio,
entity_id: str,
) -> str:
"""Publish a global event fact from PostgreSQL to the lake."""
row = await pool.fetchrow(_FETCH_GLOBAL_EVENT, entity_id)
if row is None:
logger.warning("Global event %s not found, skipping lake publish", entity_id)
return ""
event_types = row["event_types"] or []
affected_regions = row["affected_regions"] or []
affected_sectors = row["affected_sectors"] or []
affected_commodities = row["affected_commodities"] or []
return publish_global_event_fact(
client=minio_client,
event_id=str(row["id"]),
event_types=list(event_types),
severity=row["severity"] or "low",
affected_regions=list(affected_regions),
affected_sectors=list(affected_sectors),
affected_commodities=list(affected_commodities),
summary=row["summary"] or "",
estimated_duration=row["estimated_duration"] or "short_term",
confidence=float(row["confidence"] or 0.0),
source_document_id=str(row["source_document_id"]) if row["source_document_id"] else "",
created_at=row["created_at"],
)
async def publish_macro_impacts_job(
pool: asyncpg.Pool,
minio_client: Minio,
entity_id: str,
) -> list[str]:
"""Publish macro impact facts for a global event from PostgreSQL to the lake."""
rows = await pool.fetch(_FETCH_MACRO_IMPACTS_FOR_EVENT, entity_id)
if not rows:
logger.info("No macro impact records for event %s", entity_id)
return []
refs: list[str] = []
for row in rows:
factors = row["contributing_factors"]
if isinstance(factors, str):
try:
factors = json.loads(factors)
except (json.JSONDecodeError, TypeError):
factors = [factors] if factors else []
elif factors is None:
factors = []
ref = publish_macro_impact_fact(
client=minio_client,
event_id=str(row["event_id"]),
company_id=str(row["company_id"]),
ticker=row["ticker"],
macro_impact_score=float(row["macro_impact_score"] or 0.0),
impact_direction=row["impact_direction"] or "neutral",
contributing_factors=list(factors),
confidence=float(row["confidence"] or 0.0),
computed_at=row["computed_at"],
)
refs.append(ref)
return refs
async def publish_trend_projection_job(
pool: asyncpg.Pool,
minio_client: Minio,
entity_id: str,
) -> str:
"""Publish a trend projection fact from PostgreSQL to the lake."""
row = await pool.fetchrow(_FETCH_TREND_PROJECTION, entity_id)
if row is None:
logger.warning("Trend projection for window %s not found", entity_id)
return ""
factors = row["driving_factors"]
if isinstance(factors, str):
try:
factors = json.loads(factors)
except (json.JSONDecodeError, TypeError):
factors = [factors] if factors else []
elif factors is None:
factors = []
return publish_trend_projection_fact(
client=minio_client,
trend_window_id=str(row["trend_window_id"]),
ticker=row["ticker"] or "",
projected_direction=row["projected_direction"] or "neutral",
projected_strength=float(row["projected_strength"] or 0.0),
projected_confidence=float(row["projected_confidence"] or 0.0),
projection_horizon=row["projection_horizon"] or "7d",
driving_factors=list(factors),
macro_contribution_pct=float(row["macro_contribution_pct"] or 0.0),
diverges_from_current=bool(row["diverges_from_current"]),
computed_at=row["computed_at"],
)
async def publish_competitor_relationship_job(
pool: asyncpg.Pool,
minio_client: Minio,
entity_id: str,
) -> str:
"""Publish a competitor relationship fact from PostgreSQL to the lake."""
row = await pool.fetchrow(_FETCH_COMPETITOR_RELATIONSHIP, entity_id)
if row is None:
logger.warning("Competitor relationship %s not found, skipping lake publish", entity_id)
return ""
return publish_competitor_relationship_fact(
client=minio_client,
relationship_id=str(row["id"]),
company_a_id=str(row["company_a_id"]),
company_b_id=str(row["company_b_id"]),
relationship_type=row["relationship_type"],
strength=float(row["strength"]),
bidirectional=bool(row["bidirectional"]),
source=row["source"],
active=bool(row["active"]),
created_at=row["created_at"],
)
async def publish_competitive_signals_job(
pool: asyncpg.Pool,
minio_client: Minio,
entity_id: str,
) -> list[str]:
"""Publish competitive signal facts for a document from PostgreSQL to the lake."""
rows = await pool.fetch(_FETCH_COMPETITIVE_SIGNALS_FOR_DOCUMENT, entity_id)
if not rows:
logger.info("No competitive signals for document %s", entity_id)
return []
refs: list[str] = []
for row in rows:
ref = publish_competitive_signal_fact(
client=minio_client,
signal_id=str(row["id"]),
source_document_id=str(row["source_document_id"]),
source_ticker=row["source_ticker"],
target_ticker=row["target_ticker"],
catalyst_type=row["catalyst_type"],
pattern_confidence=float(row["pattern_confidence"]),
signal_direction=row["signal_direction"],
signal_strength=float(row["signal_strength"]),
relationship_strength=float(row["relationship_strength"]),
computed_at=row["computed_at"],
)
refs.append(ref)
return refs
# ---------------------------------------------------------------------------
# Job dispatcher
# ---------------------------------------------------------------------------
@@ -525,6 +740,11 @@ JOB_TYPES = {
"company_event",
"bulk_documents",
"bulk_extractions",
"global_event",
"macro_impact",
"trend_projection",
"competitor_relationship",
"competitive_signal",
}
@@ -594,6 +814,26 @@ async def dispatch_job(
refs = await publish_bulk_extractions_job(pool, minio_client, since)
result["refs"] = refs
elif job_type == "global_event":
ref = await publish_global_event_job(pool, minio_client, entity_id)
result["refs"] = [ref] if ref else []
elif job_type == "macro_impact":
refs = await publish_macro_impacts_job(pool, minio_client, entity_id)
result["refs"] = refs
elif job_type == "trend_projection":
ref = await publish_trend_projection_job(pool, minio_client, entity_id)
result["refs"] = [ref] if ref else []
elif job_type == "competitor_relationship":
ref = await publish_competitor_relationship_job(pool, minio_client, entity_id)
result["refs"] = [ref] if ref else []
elif job_type == "competitive_signal":
refs = await publish_competitive_signals_job(pool, minio_client, entity_id)
result["refs"] = refs
else:
result["error"] = f"Unknown job_type: {job_type}"
logger.warning("Unknown lake publish job type: %s", job_type)
services/lake_publisher/partitions.py
+5
View File
@@ -55,6 +55,11 @@ TABLE_PARTITIONS: dict[str, PartitionSpec] = {
"pnl_daily": PartitionSpec("pnl_daily"),
"prediction_vs_outcome": PartitionSpec("prediction_vs_outcome", extra_keys=("model_version",)),
"model_performance": PartitionSpec("model_performance", extra_keys=("model_version",)),
"global_events": PartitionSpec("global_events"),
"macro_impacts": PartitionSpec("macro_impacts", extra_keys=("ticker",)),
"trend_projections": PartitionSpec("trend_projections", extra_keys=("ticker",)),
"competitor_relationships": PartitionSpec("competitor_relationships"),
"competitive_signals": PartitionSpec("competitive_signals", extra_keys=("target_ticker",)),
}
services/lake_publisher/worker.py
+370
View File
@@ -1226,3 +1226,373 @@ def publish_prediction_vs_outcome_batch(
) -> str:
"""Publish a batch of prediction vs outcome rows as a single Parquet file."""
return _publish_batch(client, "prediction_vs_outcome", rows, PREDICTION_VS_OUTCOME_SCHEMA, dt)
# --- global_events fact table ---
GLOBAL_EVENTS_SCHEMA = pa.schema([
("event_id", pa.string()),
("event_types", pa.string()),
("severity", pa.string()),
("affected_regions", pa.string()),
("affected_sectors", pa.string()),
("affected_commodities", pa.string()),
("summary", pa.string()),
("estimated_duration", pa.string()),
("confidence", pa.float64()),
("source_document_id", pa.string()),
("created_at", pa.timestamp("us", tz="UTC")),
("dt", pa.date32()),
])
def publish_global_event_fact(
client: Minio,
event_id: str,
event_types: list[str],
severity: str,
affected_regions: list[str],
affected_sectors: list[str],
affected_commodities: list[str],
summary: str,
estimated_duration: str,
confidence: float,
source_document_id: str,
created_at: datetime,
) -> str:
"""Publish a single global event fact to MinIO.
Writes a Parquet file to:
s3://stonks-lakehouse/warehouse/global_events/dt={date}/part-{uuid}.parquet
Returns the s3:// URI of the written object.
Requirements: 7.3, 12.6
Design ref: Analytical Lake Datasets (lake.global_events)
"""
row: dict[str, object] = {
"event_id": event_id,
"event_types": ", ".join(event_types),
"severity": severity,
"affected_regions": ", ".join(affected_regions),
"affected_sectors": ", ".join(affected_sectors),
"affected_commodities": ", ".join(affected_commodities),
"summary": summary,
"estimated_duration": estimated_duration,
"confidence": confidence,
"source_document_id": source_document_id,
"created_at": created_at,
**partition_values(created_at),
}
table = pa.Table.from_pylist([row], schema=GLOBAL_EVENTS_SCHEMA)
parquet_bytes = _write_parquet_bytes(table)
path = _partition_path("global_events", created_at)
_put_lakehouse_object(client, "global_events", path, parquet_bytes)
ref = s3_uri(path)
logger.info("Published global_event fact %s: %s", event_id, ref)
return ref
# --- macro_impacts fact table ---
MACRO_IMPACTS_SCHEMA = pa.schema([
("event_id", pa.string()),
("company_id", pa.string()),
("ticker", pa.string()),
("macro_impact_score", pa.float64()),
("impact_direction", pa.string()),
("contributing_factors", pa.string()),
("confidence", pa.float64()),
("computed_at", pa.timestamp("us", tz="UTC")),
("dt", pa.date32()),
])
def publish_macro_impact_fact(
client: Minio,
event_id: str,
company_id: str,
ticker: str,
macro_impact_score: float,
impact_direction: str,
contributing_factors: list[str],
confidence: float,
computed_at: datetime,
) -> str:
"""Publish a single macro impact fact to MinIO.
Writes a Parquet file to:
s3://stonks-lakehouse/warehouse/macro_impacts/dt={date}/ticker={ticker}/part-{uuid}.parquet
Returns the s3:// URI of the written object.
Requirements: 7.3, 12.6
Design ref: Analytical Lake Datasets (lake.macro_impacts)
"""
extra = {"ticker": ticker}
row: dict[str, object] = {
"event_id": event_id,
"company_id": company_id,
"ticker": ticker,
"macro_impact_score": macro_impact_score,
"impact_direction": impact_direction,
"contributing_factors": ", ".join(contributing_factors),
"confidence": confidence,
"computed_at": computed_at,
**partition_values(computed_at, extra),
}
table = pa.Table.from_pylist([row], schema=MACRO_IMPACTS_SCHEMA)
parquet_bytes = _write_parquet_bytes(table)
path = _partition_path("macro_impacts", computed_at, extra_partitions=extra)
_put_lakehouse_object(client, "macro_impacts", path, parquet_bytes)
ref = s3_uri(path)
logger.info("Published macro_impact fact for %s/%s: %s", ticker, event_id, ref)
return ref
# --- trend_projections fact table ---
TREND_PROJECTIONS_SCHEMA = pa.schema([
("trend_window_id", pa.string()),
("ticker", pa.string()),
("projected_direction", pa.string()),
("projected_strength", pa.float64()),
("projected_confidence", pa.float64()),
("projection_horizon", pa.string()),
("driving_factors", pa.string()),
("macro_contribution_pct", pa.float64()),
("diverges_from_current", pa.bool_()),
("computed_at", pa.timestamp("us", tz="UTC")),
("dt", pa.date32()),
])
def publish_trend_projection_fact(
client: Minio,
trend_window_id: str,
ticker: str,
projected_direction: str,
projected_strength: float,
projected_confidence: float,
projection_horizon: str,
driving_factors: list[str],
macro_contribution_pct: float,
diverges_from_current: bool,
computed_at: datetime,
) -> str:
"""Publish a single trend projection fact to MinIO.
Writes a Parquet file to:
s3://stonks-lakehouse/warehouse/trend_projections/dt={date}/ticker={ticker}/part-{uuid}.parquet
Returns the s3:// URI of the written object.
Requirements: 7.3, 12.6
Design ref: Analytical Lake Datasets (lake.trend_projections)
"""
extra = {"ticker": ticker}
row: dict[str, object] = {
"trend_window_id": trend_window_id,
"ticker": ticker,
"projected_direction": projected_direction,
"projected_strength": projected_strength,
"projected_confidence": projected_confidence,
"projection_horizon": projection_horizon,
"driving_factors": ", ".join(driving_factors),
"macro_contribution_pct": macro_contribution_pct,
"diverges_from_current": diverges_from_current,
"computed_at": computed_at,
**partition_values(computed_at, extra),
}
table = pa.Table.from_pylist([row], schema=TREND_PROJECTIONS_SCHEMA)
parquet_bytes = _write_parquet_bytes(table)
path = _partition_path("trend_projections", computed_at, extra_partitions=extra)
_put_lakehouse_object(client, "trend_projections", path, parquet_bytes)
ref = s3_uri(path)
logger.info("Published trend_projection fact for %s: %s", ticker, ref)
return ref
# --- Batch publishers for macro fact tables ---
def publish_global_events_batch(
client: Minio,
rows: list[dict[str, object]],
dt: datetime,
) -> str:
"""Publish a batch of global event rows as a single Parquet file."""
return _publish_batch(client, "global_events", rows, GLOBAL_EVENTS_SCHEMA, dt)
def publish_macro_impacts_batch(
client: Minio,
rows: list[dict[str, object]],
dt: datetime,
ticker: str = "",
) -> str:
"""Publish a batch of macro impact rows as a single Parquet file."""
extra = {"ticker": ticker} if ticker else None
return _publish_batch(client, "macro_impacts", rows, MACRO_IMPACTS_SCHEMA, dt, extra)
def publish_trend_projections_batch(
client: Minio,
rows: list[dict[str, object]],
dt: datetime,
ticker: str = "",
) -> str:
"""Publish a batch of trend projection rows as a single Parquet file."""
extra = {"ticker": ticker} if ticker else None
return _publish_batch(client, "trend_projections", rows, TREND_PROJECTIONS_SCHEMA, dt, extra)
# --- competitor_relationships fact table ---
COMPETITOR_RELATIONSHIPS_SCHEMA = pa.schema([
("id", pa.string()),
("company_a_id", pa.string()),
("company_b_id", pa.string()),
("relationship_type", pa.string()),
("strength", pa.float64()),
("bidirectional", pa.bool_()),
("source", pa.string()),
("active", pa.bool_()),
("created_at", pa.timestamp("us", tz="UTC")),
("dt", pa.date32()),
])
def publish_competitor_relationship_fact(
client: Minio,
relationship_id: str,
company_a_id: str,
company_b_id: str,
relationship_type: str,
strength: float,
bidirectional: bool,
source: str,
active: bool,
created_at: datetime,
) -> str:
"""Publish a single competitor relationship fact to MinIO.
Writes a Parquet file to:
s3://stonks-lakehouse/warehouse/competitor_relationships/dt={date}/part-{uuid}.parquet
Returns the s3:// URI of the written object.
Requirements: 7.3
Design ref: Analytical Lake Datasets (lake.competitor_relationships)
"""
row: dict[str, object] = {
"id": relationship_id,
"company_a_id": company_a_id,
"company_b_id": company_b_id,
"relationship_type": relationship_type,
"strength": strength,
"bidirectional": bidirectional,
"source": source,
"active": active,
"created_at": created_at,
**partition_values(created_at),
}
table = pa.Table.from_pylist([row], schema=COMPETITOR_RELATIONSHIPS_SCHEMA)
parquet_bytes = _write_parquet_bytes(table)
path = _partition_path("competitor_relationships", created_at)
_put_lakehouse_object(client, "competitor_relationships", path, parquet_bytes)
ref = s3_uri(path)
logger.info("Published competitor_relationship fact %s: %s", relationship_id, ref)
return ref
def publish_competitor_relationships_batch(
client: Minio,
rows: list[dict[str, object]],
dt: datetime,
) -> str:
"""Publish a batch of competitor relationship rows as a single Parquet file."""
return _publish_batch(client, "competitor_relationships", rows, COMPETITOR_RELATIONSHIPS_SCHEMA, dt)
# --- competitive_signals fact table ---
COMPETITIVE_SIGNALS_SCHEMA = pa.schema([
("id", pa.string()),
("source_document_id", pa.string()),
("source_ticker", pa.string()),
("target_ticker", pa.string()),
("catalyst_type", pa.string()),
("pattern_confidence", pa.float64()),
("signal_direction", pa.string()),
("signal_strength", pa.float64()),
("relationship_strength", pa.float64()),
("computed_at", pa.timestamp("us", tz="UTC")),
("dt", pa.date32()),
])
def publish_competitive_signal_fact(
client: Minio,
signal_id: str,
source_document_id: str,
source_ticker: str,
target_ticker: str,
catalyst_type: str,
pattern_confidence: float,
signal_direction: str,
signal_strength: float,
relationship_strength: float,
computed_at: datetime,
) -> str:
"""Publish a single competitive signal fact to MinIO.
Writes a Parquet file to:
s3://stonks-lakehouse/warehouse/competitive_signals/dt={date}/target_ticker={ticker}/part-{uuid}.parquet
Returns the s3:// URI of the written object.
Requirements: 7.4
Design ref: Analytical Lake Datasets (lake.competitive_signals)
"""
extra = {"target_ticker": target_ticker}
row: dict[str, object] = {
"id": signal_id,
"source_document_id": source_document_id,
"source_ticker": source_ticker,
"target_ticker": target_ticker,
"catalyst_type": catalyst_type,
"pattern_confidence": pattern_confidence,
"signal_direction": signal_direction,
"signal_strength": signal_strength,
"relationship_strength": relationship_strength,
"computed_at": computed_at,
**partition_values(computed_at, extra),
}
table = pa.Table.from_pylist([row], schema=COMPETITIVE_SIGNALS_SCHEMA)
parquet_bytes = _write_parquet_bytes(table)
path = _partition_path("competitive_signals", computed_at, extra_partitions=extra)
_put_lakehouse_object(client, "competitive_signals", path, parquet_bytes)
ref = s3_uri(path)
logger.info("Published competitive_signal fact for %s→%s: %s", source_ticker, target_ticker, ref)
return ref
def publish_competitive_signals_batch(
client: Minio,
rows: list[dict[str, object]],
dt: datetime,
target_ticker: str = "",
) -> str:
"""Publish a batch of competitive signal rows as a single Parquet file."""
extra = {"target_ticker": target_ticker} if target_ticker else None
return _publish_batch(client, "competitive_signals", rows, COMPETITIVE_SIGNALS_SCHEMA, dt, extra)