feat: trading feedback engine — periodic performance reports with AI summarization
ci/woodpecker/push/test Pipeline was successful
ci/woodpecker/push/build-2 Pipeline was successful
ci/woodpecker/push/build-3 Pipeline was successful
ci/woodpecker/push/build-1 Pipeline was successful
ci/woodpecker/push/finalize Pipeline was successful
Build and Push / lint-and-test (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.adapters.broker_adapter name:broker-adapter]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.aggregation.worker name:aggregation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.extractor.worker name:extractor]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.ingestion.worker name:ingestion]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.lake_publisher.worker name:lake-publisher]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.parser.worker name:parser]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.recommendation.worker name:recommendation]) (push) Has been cancelled
Build and Push / build-services (map[cmd:python -m services.scheduler.app name:scheduler]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.api.app:app --host 0.0.0.0 --port 8000 name:query-api]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.risk.app:app --host 0.0.0.0 --port 8000 name:risk]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.symbol_registry.app:app --host 0.0.0.0 --port 8000 name:symbol-registry]) (push) Has been cancelled
Build and Push / build-services (map[cmd:uvicorn services.trading.app:app --host 0.0.0.0 --port 8000 name:trading-engine]) (push) Has been cancelled
Build and Push / build-dashboard (push) Has been cancelled
Build and Push / build-superset (push) Has been cancelled
Build and Push / integration-test (push) Has been cancelled
Build and Push / beta-gate (push) Has been cancelled

- Migration 038: trading_reports table + report-summarizer agent seed
- 6 reporting modules: models, collector, sections, validator, summarizer, generator
- API endpoints: GET /api/reports (paginated, filterable), GET /api/reports/{id}
- Frontend hooks: useReports, useReport with TanStack Query
- Scheduler: daily (after 16:30 ET) and weekly (Saturday) report triggers
- Redis queue consumer for async report generation with retry/dedup
- 5 property-based tests (chunking, serialization, validation, accuracy, deltas)
- 109 unit/integration tests across all modules
- 6 frontend hook tests with MSW mocks
This commit is contained in:
Celes Renata
2026-05-01 22:13:09 +00:00
parent 376fcb4bb4
commit bc077bfcc8
28 changed files with 6771 additions and 1 deletions
+172 -1
View File
@@ -10,8 +10,9 @@ import asyncio
import json
import logging
import os
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from typing import Any, Optional
from zoneinfo import ZoneInfo
import asyncpg
import redis.asyncio as aioredis
@@ -26,6 +27,7 @@ from services.shared.redis_keys import (
QUEUE_INGESTION,
QUEUE_MACRO_CLASSIFICATION,
QUEUE_PREFIX,
QUEUE_REPORT_GENERATION,
lock_key,
queue_key,
rate_limit_key,
@@ -498,6 +500,163 @@ async def schedule_cycle(pool: asyncpg.Pool, rds: aioredis.Redis) -> int:
return enqueued
# ---------------------------------------------------------------------------
# Report generation: queue consumer + scheduled triggers
# Requirements: 6.1, 6.2, 6.3, 6.4, 6.5
# ---------------------------------------------------------------------------
# Eastern Time zone for market-close checks
_ET = ZoneInfo("America/New_York")
# How often to check the report generation queue (every N cycles)
# 15s tick × 4 cycles = ~1 minute
REPORT_CONSUMER_CYCLE_INTERVAL: int = 4
# How often to check report scheduling triggers (every N cycles)
# 15s tick × 20 cycles = ~5 minutes
REPORT_SCHEDULE_CYCLE_INTERVAL: int = 20
# Redis key prefix for report schedule dedup markers
_REPORT_DEDUPE_PREFIX = f"{QUEUE_PREFIX}:report_dedupe"
_REPORT_DEDUPE_TTL = 86400 # 24 hours — prevents re-enqueuing same report within a day
def _report_dedupe_key(report_type: str, period_start: str, period_end: str) -> str:
"""Build a Redis key for deduplicating report schedule triggers."""
return f"{_REPORT_DEDUPE_PREFIX}:{report_type}:{period_start}:{period_end}"
async def consume_report_generation_jobs(
pool: asyncpg.Pool,
rds: aioredis.Redis,
) -> int:
"""Pop and process jobs from the report generation queue.
Pops up to 5 jobs per invocation to avoid blocking the scheduler loop.
Each job is deserialized and handed to process_report_job from the
reporting generator module.
Returns the number of jobs processed.
"""
from services.reporting.generator import process_report_job
report_queue = queue_key(QUEUE_REPORT_GENERATION)
processed = 0
for _ in range(5):
raw = await rds.lpop(report_queue)
if raw is None:
break
try:
job = json.loads(raw)
except (json.JSONDecodeError, TypeError):
logger.error("Invalid report generation job payload: %s", raw)
continue
logger.info(
"Processing report generation job: type=%s period=%s to %s",
job.get("report_type"),
job.get("period_start"),
job.get("period_end"),
)
try:
await process_report_job(pool, job)
processed += 1
except Exception:
logger.exception(
"Failed to process report generation job: %s", job,
)
if processed > 0:
logger.info("Processed %d report generation jobs", processed)
return processed
async def maybe_enqueue_daily_report(
rds: aioredis.Redis,
now_et: datetime,
) -> bool:
"""Enqueue a daily report job if it's after 16:30 ET on a weekday.
Uses a Redis dedupe key to avoid re-enqueuing the same daily report.
Returns True if a job was enqueued, False otherwise.
"""
# Only on weekdays (Mon=0 .. Fri=4)
if now_et.weekday() > 4:
return False
# Only after 16:30 ET
if now_et.hour < 16 or (now_et.hour == 16 and now_et.minute < 30):
return False
today = now_et.date()
period_start = today.isoformat()
period_end = today.isoformat()
dedupe = _report_dedupe_key("daily", period_start, period_end)
created = await rds.set(dedupe, "1", nx=True, ex=_REPORT_DEDUPE_TTL)
if not created:
return False
job = json.dumps({
"report_type": "daily",
"period_start": period_start,
"period_end": period_end,
})
await rds.rpush(queue_key(QUEUE_REPORT_GENERATION), job)
logger.info("Enqueued daily report for %s", period_start)
return True
async def maybe_enqueue_weekly_report(
rds: aioredis.Redis,
now_et: datetime,
) -> bool:
"""Enqueue a weekly report job on Saturday.
Covers the previous Monday through Friday.
Uses a Redis dedupe key to avoid re-enqueuing the same weekly report.
Returns True if a job was enqueued, False otherwise.
"""
# Only on Saturday (weekday() == 5)
if now_et.weekday() != 5:
return False
today = now_et.date()
# Previous Monday = today - 5 days, previous Friday = today - 1 day
period_start = (today - timedelta(days=5)).isoformat()
period_end = (today - timedelta(days=1)).isoformat()
dedupe = _report_dedupe_key("weekly", period_start, period_end)
created = await rds.set(dedupe, "1", nx=True, ex=_REPORT_DEDUPE_TTL)
if not created:
return False
job = json.dumps({
"report_type": "weekly",
"period_start": period_start,
"period_end": period_end,
})
await rds.rpush(queue_key(QUEUE_REPORT_GENERATION), job)
logger.info(
"Enqueued weekly report for %s to %s", period_start, period_end,
)
return True
async def check_report_schedule(rds: aioredis.Redis) -> None:
"""Check if daily or weekly report triggers should fire.
Called periodically from the main loop. Uses Eastern Time to determine
market close (16:30 ET) and day of week.
"""
now_et = datetime.now(tz=_ET)
await maybe_enqueue_daily_report(rds, now_et)
await maybe_enqueue_weekly_report(rds, now_et)
async def enqueue_periodic_aggregation(pool: asyncpg.Pool, rds: aioredis.Redis) -> int:
"""Enqueue aggregation jobs for all active tickers.
@@ -544,6 +703,8 @@ async def main() -> None:
retry_counter = 0
cleanup_counter = 0
aggregation_counter = 0
report_consumer_counter = 0
report_schedule_counter = 0
try:
while True:
try:
@@ -576,6 +737,16 @@ async def main() -> None:
if aggregation_counter >= AGGREGATION_CYCLE_INTERVAL:
aggregation_counter = 0
await enqueue_periodic_aggregation(pool, rds)
# Consume report generation jobs (~1 minute)
report_consumer_counter += 1
if report_consumer_counter >= REPORT_CONSUMER_CYCLE_INTERVAL:
report_consumer_counter = 0
await consume_report_generation_jobs(pool, rds)
# Check report schedule triggers (~5 minutes)
report_schedule_counter += 1
if report_schedule_counter >= REPORT_SCHEDULE_CYCLE_INTERVAL:
report_schedule_counter = 0
await check_report_schedule(rds)
finally:
await release_lock(rds, "scheduler_cycle")
except Exception: