feat: trading feedback engine — periodic performance reports with AI summarization
ci/woodpecker/push/test Pipeline was successful
ci/woodpecker/push/build-2 Pipeline was successful
ci/woodpecker/push/build-3 Pipeline was successful
ci/woodpecker/push/build-1 Pipeline was successful
ci/woodpecker/push/finalize Pipeline was successful
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
Build and Push / beta-gate (push) Has been cancelled
ci/woodpecker/push/test Pipeline was successful
ci/woodpecker/push/build-2 Pipeline was successful
ci/woodpecker/push/build-3 Pipeline was successful
ci/woodpecker/push/build-1 Pipeline was successful
ci/woodpecker/push/finalize Pipeline was successful
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
Build and Push / beta-gate (push) Has been cancelled
- Migration 038: trading_reports table + report-summarizer agent seed
- 6 reporting modules: models, collector, sections, validator, summarizer, generator
- API endpoints: GET /api/reports (paginated, filterable), GET /api/reports/{id}
- Frontend hooks: useReports, useReport with TanStack Query
- Scheduler: daily (after 16:30 ET) and weekly (Saturday) report triggers
- Redis queue consumer for async report generation with retry/dedup
- 5 property-based tests (chunking, serialization, validation, accuracy, deltas)
- 109 unit/integration tests across all modules
- 6 frontend hook tests with MSW mocks
This commit is contained in:
+172
-1
@@ -10,8 +10,9 @@ import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Optional
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import asyncpg
|
||||
import redis.asyncio as aioredis
|
||||
@@ -26,6 +27,7 @@ from services.shared.redis_keys import (
|
||||
QUEUE_INGESTION,
|
||||
QUEUE_MACRO_CLASSIFICATION,
|
||||
QUEUE_PREFIX,
|
||||
QUEUE_REPORT_GENERATION,
|
||||
lock_key,
|
||||
queue_key,
|
||||
rate_limit_key,
|
||||
@@ -498,6 +500,163 @@ async def schedule_cycle(pool: asyncpg.Pool, rds: aioredis.Redis) -> int:
|
||||
return enqueued
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report generation: queue consumer + scheduled triggers
|
||||
# Requirements: 6.1, 6.2, 6.3, 6.4, 6.5
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Eastern Time zone for market-close checks
|
||||
_ET = ZoneInfo("America/New_York")
|
||||
|
||||
# How often to check the report generation queue (every N cycles)
|
||||
# 15s tick × 4 cycles = ~1 minute
|
||||
REPORT_CONSUMER_CYCLE_INTERVAL: int = 4
|
||||
|
||||
# How often to check report scheduling triggers (every N cycles)
|
||||
# 15s tick × 20 cycles = ~5 minutes
|
||||
REPORT_SCHEDULE_CYCLE_INTERVAL: int = 20
|
||||
|
||||
# Redis key prefix for report schedule dedup markers
|
||||
_REPORT_DEDUPE_PREFIX = f"{QUEUE_PREFIX}:report_dedupe"
|
||||
_REPORT_DEDUPE_TTL = 86400 # 24 hours — prevents re-enqueuing same report within a day
|
||||
|
||||
|
||||
def _report_dedupe_key(report_type: str, period_start: str, period_end: str) -> str:
|
||||
"""Build a Redis key for deduplicating report schedule triggers."""
|
||||
return f"{_REPORT_DEDUPE_PREFIX}:{report_type}:{period_start}:{period_end}"
|
||||
|
||||
|
||||
async def consume_report_generation_jobs(
|
||||
pool: asyncpg.Pool,
|
||||
rds: aioredis.Redis,
|
||||
) -> int:
|
||||
"""Pop and process jobs from the report generation queue.
|
||||
|
||||
Pops up to 5 jobs per invocation to avoid blocking the scheduler loop.
|
||||
Each job is deserialized and handed to process_report_job from the
|
||||
reporting generator module.
|
||||
|
||||
Returns the number of jobs processed.
|
||||
"""
|
||||
from services.reporting.generator import process_report_job
|
||||
|
||||
report_queue = queue_key(QUEUE_REPORT_GENERATION)
|
||||
processed = 0
|
||||
|
||||
for _ in range(5):
|
||||
raw = await rds.lpop(report_queue)
|
||||
if raw is None:
|
||||
break
|
||||
|
||||
try:
|
||||
job = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.error("Invalid report generation job payload: %s", raw)
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
"Processing report generation job: type=%s period=%s to %s",
|
||||
job.get("report_type"),
|
||||
job.get("period_start"),
|
||||
job.get("period_end"),
|
||||
)
|
||||
|
||||
try:
|
||||
await process_report_job(pool, job)
|
||||
processed += 1
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to process report generation job: %s", job,
|
||||
)
|
||||
|
||||
if processed > 0:
|
||||
logger.info("Processed %d report generation jobs", processed)
|
||||
return processed
|
||||
|
||||
|
||||
async def maybe_enqueue_daily_report(
|
||||
rds: aioredis.Redis,
|
||||
now_et: datetime,
|
||||
) -> bool:
|
||||
"""Enqueue a daily report job if it's after 16:30 ET on a weekday.
|
||||
|
||||
Uses a Redis dedupe key to avoid re-enqueuing the same daily report.
|
||||
Returns True if a job was enqueued, False otherwise.
|
||||
"""
|
||||
# Only on weekdays (Mon=0 .. Fri=4)
|
||||
if now_et.weekday() > 4:
|
||||
return False
|
||||
|
||||
# Only after 16:30 ET
|
||||
if now_et.hour < 16 or (now_et.hour == 16 and now_et.minute < 30):
|
||||
return False
|
||||
|
||||
today = now_et.date()
|
||||
period_start = today.isoformat()
|
||||
period_end = today.isoformat()
|
||||
|
||||
dedupe = _report_dedupe_key("daily", period_start, period_end)
|
||||
created = await rds.set(dedupe, "1", nx=True, ex=_REPORT_DEDUPE_TTL)
|
||||
if not created:
|
||||
return False
|
||||
|
||||
job = json.dumps({
|
||||
"report_type": "daily",
|
||||
"period_start": period_start,
|
||||
"period_end": period_end,
|
||||
})
|
||||
await rds.rpush(queue_key(QUEUE_REPORT_GENERATION), job)
|
||||
logger.info("Enqueued daily report for %s", period_start)
|
||||
return True
|
||||
|
||||
|
||||
async def maybe_enqueue_weekly_report(
|
||||
rds: aioredis.Redis,
|
||||
now_et: datetime,
|
||||
) -> bool:
|
||||
"""Enqueue a weekly report job on Saturday.
|
||||
|
||||
Covers the previous Monday through Friday.
|
||||
Uses a Redis dedupe key to avoid re-enqueuing the same weekly report.
|
||||
Returns True if a job was enqueued, False otherwise.
|
||||
"""
|
||||
# Only on Saturday (weekday() == 5)
|
||||
if now_et.weekday() != 5:
|
||||
return False
|
||||
|
||||
today = now_et.date()
|
||||
# Previous Monday = today - 5 days, previous Friday = today - 1 day
|
||||
period_start = (today - timedelta(days=5)).isoformat()
|
||||
period_end = (today - timedelta(days=1)).isoformat()
|
||||
|
||||
dedupe = _report_dedupe_key("weekly", period_start, period_end)
|
||||
created = await rds.set(dedupe, "1", nx=True, ex=_REPORT_DEDUPE_TTL)
|
||||
if not created:
|
||||
return False
|
||||
|
||||
job = json.dumps({
|
||||
"report_type": "weekly",
|
||||
"period_start": period_start,
|
||||
"period_end": period_end,
|
||||
})
|
||||
await rds.rpush(queue_key(QUEUE_REPORT_GENERATION), job)
|
||||
logger.info(
|
||||
"Enqueued weekly report for %s to %s", period_start, period_end,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def check_report_schedule(rds: aioredis.Redis) -> None:
|
||||
"""Check if daily or weekly report triggers should fire.
|
||||
|
||||
Called periodically from the main loop. Uses Eastern Time to determine
|
||||
market close (16:30 ET) and day of week.
|
||||
"""
|
||||
now_et = datetime.now(tz=_ET)
|
||||
await maybe_enqueue_daily_report(rds, now_et)
|
||||
await maybe_enqueue_weekly_report(rds, now_et)
|
||||
|
||||
|
||||
async def enqueue_periodic_aggregation(pool: asyncpg.Pool, rds: aioredis.Redis) -> int:
|
||||
"""Enqueue aggregation jobs for all active tickers.
|
||||
|
||||
@@ -544,6 +703,8 @@ async def main() -> None:
|
||||
retry_counter = 0
|
||||
cleanup_counter = 0
|
||||
aggregation_counter = 0
|
||||
report_consumer_counter = 0
|
||||
report_schedule_counter = 0
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
@@ -576,6 +737,16 @@ async def main() -> None:
|
||||
if aggregation_counter >= AGGREGATION_CYCLE_INTERVAL:
|
||||
aggregation_counter = 0
|
||||
await enqueue_periodic_aggregation(pool, rds)
|
||||
# Consume report generation jobs (~1 minute)
|
||||
report_consumer_counter += 1
|
||||
if report_consumer_counter >= REPORT_CONSUMER_CYCLE_INTERVAL:
|
||||
report_consumer_counter = 0
|
||||
await consume_report_generation_jobs(pool, rds)
|
||||
# Check report schedule triggers (~5 minutes)
|
||||
report_schedule_counter += 1
|
||||
if report_schedule_counter >= REPORT_SCHEDULE_CYCLE_INTERVAL:
|
||||
report_schedule_counter = 0
|
||||
await check_report_schedule(rds)
|
||||
finally:
|
||||
await release_lock(rds, "scheduler_cycle")
|
||||
except Exception:
|
||||
|
||||
Reference in New Issue
Block a user