phase 14-15: docker build validation and helm deployment
This commit is contained in:
Celes Renata
@@ -0,0 +1,832 @@
|
||||
"""Broker adapter service - standalone worker for sandbox order execution.
|
||||
|
||||
Runs the Alpaca broker adapter in sandbox (paper) mode, processing order
|
||||
requests from the broker queue, evaluating them through the risk engine,
|
||||
submitting to Alpaca's paper trading API, and persisting the full audit trail.
|
||||
|
||||
Also periodically syncs positions and account state from Alpaca.
|
||||
|
||||
Implements idempotent order submission keys and duplicate prevention:
|
||||
- Deterministic idempotency key generation from job attributes
|
||||
- Redis-based fast-path duplicate detection before broker submission
|
||||
- PostgreSQL UNIQUE constraint on idempotency_key as durable fallback
|
||||
|
||||
Requirements: 2.4, 8.1, 8.3, 8.5
|
||||
Design: Section 4.9 - Broker Adapter
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import asyncpg
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
from services.adapters.broker_adapter import (
|
||||
AlpacaBrokerAdapter,
|
||||
OrderRequest,
|
||||
OrderResponse,
|
||||
OrderSide,
|
||||
OrderStatus,
|
||||
OrderType,
|
||||
TradingMode,
|
||||
)
|
||||
from services.risk.engine import (
|
||||
AccountRiskState,
|
||||
PortfolioRiskConfig,
|
||||
ProposedOrder,
|
||||
evaluate_order,
|
||||
)
|
||||
from services.risk.approval import (
|
||||
ApprovalRequest,
|
||||
ApprovalStatus,
|
||||
compute_expiry,
|
||||
create_approval_request,
|
||||
requires_approval,
|
||||
)
|
||||
from services.shared.audit import (
|
||||
audit_approval_requested,
|
||||
audit_duplicate_prevented,
|
||||
audit_order_filled,
|
||||
audit_order_rejected,
|
||||
audit_order_submitted,
|
||||
audit_risk_evaluated,
|
||||
)
|
||||
from services.lake_publisher.worker import (
|
||||
publish_trade_order,
|
||||
publish_trade_fill,
|
||||
publish_positions_daily_batch,
|
||||
LAKEHOUSE_BUCKET,
|
||||
)
|
||||
from services.shared.config import load_config
|
||||
from services.shared.db import get_pg_pool, get_redis
|
||||
from services.shared.logging import Span, new_trace_id, set_trace_context, setup_logging
|
||||
from services.shared.metrics import (
|
||||
ORDERS_DUPLICATES_PREVENTED,
|
||||
ORDERS_FILLED,
|
||||
ORDERS_REJECTED,
|
||||
ORDERS_SUBMITTED,
|
||||
POSITIONS_SYNCED,
|
||||
RISK_CHECK_FAILURES,
|
||||
RISK_EVALUATIONS_TOTAL,
|
||||
)
|
||||
from services.shared.redis_keys import QUEUE_BROKER, queue_key
|
||||
|
||||
logger = logging.getLogger("broker_service")
|
||||
|
||||
POSITION_SYNC_INTERVAL = 60 # seconds
|
||||
|
||||
# Redis TTL for idempotency markers (24 hours)
|
||||
ORDER_IDEMPOTENCY_TTL = 86400
|
||||
ORDER_IDEMPOTENCY_PREFIX = "stonks:order_idempotency"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DB persistence helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_UPSERT_BROKER_ACCOUNT = """
|
||||
INSERT INTO broker_accounts (id, provider, account_id, mode, config, active)
|
||||
VALUES ($1::uuid, $2, $3, $4, $5::jsonb, TRUE)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
config = EXCLUDED.config,
|
||||
mode = EXCLUDED.mode,
|
||||
active = TRUE
|
||||
"""
|
||||
|
||||
_INSERT_ORDER = """
|
||||
INSERT INTO orders (
|
||||
id, recommendation_id, broker_account_id, ticker, side, order_type,
|
||||
quantity, limit_price, stop_price, status, idempotency_key,
|
||||
broker_order_id, decision_trace, submitted_at, filled_at,
|
||||
fill_price, fill_quantity
|
||||
) VALUES (
|
||||
$1::uuid, $2, $3::uuid, $4, $5, $6,
|
||||
$7, $8, $9, $10, $11,
|
||||
$12, $13::jsonb, $14, $15,
|
||||
$16, $17
|
||||
)
|
||||
ON CONFLICT (idempotency_key) DO UPDATE SET
|
||||
status = EXCLUDED.status,
|
||||
broker_order_id = EXCLUDED.broker_order_id,
|
||||
filled_at = EXCLUDED.filled_at,
|
||||
fill_price = EXCLUDED.fill_price,
|
||||
fill_quantity = EXCLUDED.fill_quantity,
|
||||
updated_at = NOW()
|
||||
"""
|
||||
|
||||
_INSERT_ORDER_EVENT = """
|
||||
INSERT INTO order_events (order_id, event_type, data, broker_timestamp)
|
||||
VALUES ($1::uuid, $2, $3::jsonb, $4)
|
||||
"""
|
||||
|
||||
_INSERT_RISK_EVALUATION = """
|
||||
INSERT INTO risk_evaluations (id, recommendation_id, eligible, allowed_mode, rejection_reasons, risk_checks, evaluated_at)
|
||||
VALUES ($1::uuid, $2::uuid, $3, $4, $5::jsonb, $6::jsonb, $7)
|
||||
"""
|
||||
|
||||
_UPSERT_POSITION = """
|
||||
INSERT INTO positions (broker_account_id, ticker, quantity, avg_entry_price, current_price, unrealized_pnl, updated_at)
|
||||
VALUES ($1::uuid, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (broker_account_id, ticker)
|
||||
DO UPDATE SET
|
||||
quantity = EXCLUDED.quantity,
|
||||
avg_entry_price = EXCLUDED.avg_entry_price,
|
||||
current_price = EXCLUDED.current_price,
|
||||
unrealized_pnl = EXCLUDED.unrealized_pnl,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
"""
|
||||
|
||||
_LOAD_RISK_CONFIG = """
|
||||
SELECT config FROM risk_configs WHERE active = TRUE ORDER BY updated_at DESC LIMIT 1
|
||||
"""
|
||||
|
||||
_LOAD_DAILY_SNAPSHOT = """
|
||||
SELECT portfolio_value, daily_pnl, daily_trade_count, positions_by_sector
|
||||
FROM daily_risk_snapshots
|
||||
WHERE account_id = $1 AND snapshot_date = CURRENT_DATE
|
||||
LIMIT 1
|
||||
"""
|
||||
|
||||
_CHECK_ORDER_BY_IDEMPOTENCY_KEY = """
|
||||
SELECT id, status, broker_order_id FROM orders
|
||||
WHERE idempotency_key = $1
|
||||
LIMIT 1
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Idempotency helpers (Requirement 8.5)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_idempotency_key(job: dict[str, Any]) -> str:
|
||||
"""Generate a deterministic idempotency key from job attributes.
|
||||
|
||||
If the job already carries an explicit idempotency_key, use it.
|
||||
Otherwise, derive a stable key from the combination of
|
||||
recommendation_id, ticker, side, quantity, and order_type so that
|
||||
replayed queue messages produce the same key and are detected as
|
||||
duplicates.
|
||||
"""
|
||||
explicit = job.get("idempotency_key")
|
||||
if explicit:
|
||||
return str(explicit)
|
||||
|
||||
# Build a deterministic key from job content
|
||||
parts = [
|
||||
str(job.get("recommendation_id", "")),
|
||||
str(job.get("ticker", "")),
|
||||
str(job.get("side", "buy")),
|
||||
str(job.get("quantity", 0)),
|
||||
str(job.get("order_type", "market")),
|
||||
str(job.get("limit_price", "")),
|
||||
str(job.get("stop_price", "")),
|
||||
]
|
||||
raw = "|".join(parts)
|
||||
return hashlib.sha256(raw.encode()).hexdigest()[:40]
|
||||
|
||||
|
||||
def _redis_idempotency_key(idempotency_key: str) -> str:
|
||||
"""Build the Redis key for an order idempotency marker."""
|
||||
return f"{ORDER_IDEMPOTENCY_PREFIX}:{idempotency_key}"
|
||||
|
||||
|
||||
async def check_idempotency_redis(
|
||||
rds: aioredis.Redis,
|
||||
idempotency_key: str,
|
||||
) -> str | None:
|
||||
"""Fast-path: check Redis for a previously processed idempotency key.
|
||||
|
||||
Returns the existing order_id if found, None otherwise.
|
||||
"""
|
||||
redis_key = _redis_idempotency_key(idempotency_key)
|
||||
cached = await rds.get(redis_key)
|
||||
if cached:
|
||||
return str(cached)
|
||||
return None
|
||||
|
||||
|
||||
async def check_idempotency_db(
|
||||
pool: asyncpg.Pool,
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Durable fallback: check PostgreSQL for an existing order with this key.
|
||||
|
||||
Returns a dict with id, status, broker_order_id if found, None otherwise.
|
||||
"""
|
||||
row = await pool.fetchrow(_CHECK_ORDER_BY_IDEMPOTENCY_KEY, idempotency_key)
|
||||
if row:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"status": str(row["status"]),
|
||||
"broker_order_id": str(row["broker_order_id"] or ""),
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
async def mark_idempotency_redis(
|
||||
rds: aioredis.Redis,
|
||||
idempotency_key: str,
|
||||
order_id: str,
|
||||
) -> None:
|
||||
"""Set the Redis idempotency marker after an order is processed."""
|
||||
redis_key = _redis_idempotency_key(idempotency_key)
|
||||
await rds.set(redis_key, order_id, ex=ORDER_IDEMPOTENCY_TTL)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core service logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_order_request(job: dict[str, Any]) -> OrderRequest:
|
||||
"""Build an OrderRequest from a broker queue job payload."""
|
||||
side = OrderSide.SELL if job.get("side", "buy") == "sell" else OrderSide.BUY
|
||||
order_type_str = job.get("order_type", "market")
|
||||
order_type_map = {
|
||||
"market": OrderType.MARKET,
|
||||
"limit": OrderType.LIMIT,
|
||||
"stop": OrderType.STOP,
|
||||
"stop_limit": OrderType.STOP_LIMIT,
|
||||
}
|
||||
return OrderRequest(
|
||||
ticker=job["ticker"],
|
||||
side=side,
|
||||
quantity=float(job.get("quantity", 0)),
|
||||
order_type=order_type_map.get(order_type_str, OrderType.MARKET),
|
||||
limit_price=job.get("limit_price"),
|
||||
stop_price=job.get("stop_price"),
|
||||
time_in_force=job.get("time_in_force", "day"),
|
||||
idempotency_key=generate_idempotency_key(job),
|
||||
)
|
||||
|
||||
|
||||
def build_proposed_order(job: dict[str, Any]) -> ProposedOrder:
|
||||
"""Build a ProposedOrder for risk evaluation from a broker queue job."""
|
||||
return ProposedOrder(
|
||||
recommendation_id=job.get("recommendation_id"),
|
||||
ticker=job["ticker"],
|
||||
sector=job.get("sector", ""),
|
||||
action=job.get("side", "buy"),
|
||||
quantity=float(job.get("quantity", 0)),
|
||||
estimated_value=float(job.get("estimated_value", 0)),
|
||||
confidence=float(job.get("confidence", 0)),
|
||||
)
|
||||
|
||||
|
||||
async def load_risk_config(pool: asyncpg.Pool) -> PortfolioRiskConfig:
|
||||
"""Load the active risk configuration from the database."""
|
||||
row = await pool.fetchrow(_LOAD_RISK_CONFIG)
|
||||
if row and row["config"]:
|
||||
data = row["config"] if isinstance(row["config"], dict) else json.loads(row["config"])
|
||||
return PortfolioRiskConfig.from_db_json(data)
|
||||
return PortfolioRiskConfig()
|
||||
|
||||
|
||||
async def load_account_risk_state(
|
||||
pool: asyncpg.Pool,
|
||||
adapter: AlpacaBrokerAdapter,
|
||||
account_uuid: str,
|
||||
) -> AccountRiskState:
|
||||
"""Build an AccountRiskState from the broker and daily snapshot."""
|
||||
state = AccountRiskState(account_id=account_uuid)
|
||||
|
||||
# Get live account info from Alpaca
|
||||
try:
|
||||
acct = await adapter.get_account()
|
||||
state.portfolio_value = acct.portfolio_value
|
||||
state.cash = acct.cash
|
||||
state.buying_power = acct.buying_power
|
||||
except Exception as e:
|
||||
logger.warning("Failed to fetch account from Alpaca: %s", e)
|
||||
|
||||
# Get positions from Alpaca
|
||||
try:
|
||||
positions = await adapter.get_positions()
|
||||
for pos in positions:
|
||||
state.positions_by_symbol[pos.ticker] = pos.market_value
|
||||
state.open_position_count = len(positions)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to fetch positions from Alpaca: %s", e)
|
||||
|
||||
# Overlay daily snapshot from DB
|
||||
row = await pool.fetchrow(_LOAD_DAILY_SNAPSHOT, account_uuid)
|
||||
if row:
|
||||
state.daily_pnl = float(row["daily_pnl"] or 0)
|
||||
state.daily_trade_count = int(row["daily_trade_count"] or 0)
|
||||
sector_data = row["positions_by_sector"]
|
||||
if sector_data:
|
||||
state.positions_by_sector = (
|
||||
sector_data if isinstance(sector_data, dict) else json.loads(sector_data)
|
||||
)
|
||||
|
||||
return state
|
||||
|
||||
|
||||
async def persist_order(
|
||||
pool: asyncpg.Pool,
|
||||
order_id: str,
|
||||
order: OrderRequest,
|
||||
resp: OrderResponse,
|
||||
account_uuid: str,
|
||||
risk_eval: dict[str, Any],
|
||||
recommendation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Persist order, events, and risk evaluation to PostgreSQL."""
|
||||
now = datetime.now(timezone.utc)
|
||||
filled_at = now if resp.status == OrderStatus.FILLED else None
|
||||
|
||||
decision_trace = {
|
||||
"risk_evaluation": risk_eval,
|
||||
"order_request": order.to_dict(),
|
||||
"broker_response": resp.to_dict(),
|
||||
}
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.execute(
|
||||
_INSERT_ORDER,
|
||||
order_id,
|
||||
recommendation_id,
|
||||
account_uuid,
|
||||
order.ticker,
|
||||
order.side.value,
|
||||
order.order_type.value,
|
||||
order.quantity,
|
||||
order.limit_price,
|
||||
order.stop_price,
|
||||
resp.status.value,
|
||||
order.idempotency_key,
|
||||
resp.broker_order_id,
|
||||
json.dumps(decision_trace),
|
||||
resp.submitted_at or now,
|
||||
filled_at,
|
||||
resp.filled_avg_price,
|
||||
resp.filled_quantity,
|
||||
)
|
||||
|
||||
# Record order events
|
||||
for event_type in ["submitted"]:
|
||||
await conn.execute(
|
||||
_INSERT_ORDER_EVENT,
|
||||
order_id,
|
||||
event_type,
|
||||
json.dumps({"ticker": order.ticker, "side": order.side.value}),
|
||||
now,
|
||||
)
|
||||
|
||||
if resp.status == OrderStatus.FILLED:
|
||||
await conn.execute(
|
||||
_INSERT_ORDER_EVENT,
|
||||
order_id,
|
||||
"fill",
|
||||
json.dumps({
|
||||
"fill_price": resp.filled_avg_price,
|
||||
"fill_qty": resp.filled_quantity,
|
||||
}),
|
||||
now,
|
||||
)
|
||||
elif resp.status == OrderStatus.REJECTED:
|
||||
await conn.execute(
|
||||
_INSERT_ORDER_EVENT,
|
||||
order_id,
|
||||
"rejected",
|
||||
json.dumps({"error": resp.error}),
|
||||
now,
|
||||
)
|
||||
|
||||
|
||||
async def sync_positions(
|
||||
adapter: AlpacaBrokerAdapter,
|
||||
pool: asyncpg.Pool,
|
||||
account_uuid: str,
|
||||
minio_client: Any | None = None,
|
||||
) -> None:
|
||||
"""Sync current positions from Alpaca to PostgreSQL and publish to lake."""
|
||||
now = datetime.now(timezone.utc)
|
||||
try:
|
||||
positions = await adapter.get_positions()
|
||||
async with pool.acquire() as conn:
|
||||
for pos in positions:
|
||||
await conn.execute(
|
||||
_UPSERT_POSITION,
|
||||
account_uuid,
|
||||
pos.ticker,
|
||||
pos.quantity,
|
||||
pos.avg_entry_price,
|
||||
pos.current_price,
|
||||
pos.unrealized_pnl,
|
||||
now,
|
||||
)
|
||||
logger.info("Synced %d positions from Alpaca", len(positions))
|
||||
POSITIONS_SYNCED.inc()
|
||||
|
||||
# Publish positions snapshot to analytical lake
|
||||
if minio_client is not None and positions:
|
||||
try:
|
||||
pos_dicts = [
|
||||
{
|
||||
"ticker": p.ticker,
|
||||
"quantity": p.quantity,
|
||||
"avg_entry_price": p.avg_entry_price,
|
||||
"close_price": p.current_price,
|
||||
"unrealized_pnl": p.unrealized_pnl,
|
||||
}
|
||||
for p in positions
|
||||
]
|
||||
publish_positions_daily_batch(
|
||||
minio_client, pos_dicts, account_uuid, now,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to publish positions to lake: %s", e)
|
||||
except Exception as e:
|
||||
logger.error("Position sync failed: %s", e)
|
||||
|
||||
|
||||
async def register_broker_account(
|
||||
pool: asyncpg.Pool,
|
||||
account_uuid: str,
|
||||
adapter: AlpacaBrokerAdapter,
|
||||
) -> None:
|
||||
"""Register or update the broker account in PostgreSQL."""
|
||||
try:
|
||||
acct = await adapter.get_account()
|
||||
config_json = json.dumps({
|
||||
"provider": "alpaca",
|
||||
"buying_power": acct.buying_power,
|
||||
"cash": acct.cash,
|
||||
"portfolio_value": acct.portfolio_value,
|
||||
})
|
||||
await pool.execute(
|
||||
_UPSERT_BROKER_ACCOUNT,
|
||||
account_uuid,
|
||||
"alpaca",
|
||||
acct.account_id or account_uuid,
|
||||
adapter.mode.value,
|
||||
config_json,
|
||||
)
|
||||
logger.info(
|
||||
"Registered Alpaca account: id=%s mode=%s portfolio=%.2f",
|
||||
acct.account_id, adapter.mode.value, acct.portfolio_value,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Failed to register broker account: %s", e)
|
||||
|
||||
|
||||
async def process_order_job(
|
||||
job: dict[str, Any],
|
||||
adapter: AlpacaBrokerAdapter,
|
||||
pool: asyncpg.Pool,
|
||||
account_uuid: str,
|
||||
rds: aioredis.Redis | None = None,
|
||||
minio_client: Any | None = None,
|
||||
) -> None:
|
||||
"""Process a single order job from the broker queue.
|
||||
|
||||
1. Generate deterministic idempotency key
|
||||
2. Check Redis + DB for duplicate (Req 8.5)
|
||||
3. Build proposed order and run risk evaluation
|
||||
4. If risk passes, submit to Alpaca
|
||||
5. Persist order, events, and risk evaluation
|
||||
6. Set Redis idempotency marker
|
||||
"""
|
||||
ticker = job.get("ticker", "???")
|
||||
order_id = str(uuid.uuid4())
|
||||
idempotency_key = generate_idempotency_key(job)
|
||||
|
||||
# --- Duplicate prevention (Requirement 8.5) ---
|
||||
# Fast path: Redis check
|
||||
if rds is not None:
|
||||
existing_order_id = await check_idempotency_redis(rds, idempotency_key)
|
||||
if existing_order_id:
|
||||
logger.info(
|
||||
"Duplicate order detected (redis) for %s key=%s existing=%s",
|
||||
ticker, idempotency_key[:16], existing_order_id,
|
||||
)
|
||||
ORDERS_DUPLICATES_PREVENTED.labels(detected_via="redis").inc()
|
||||
await audit_duplicate_prevented(
|
||||
pool, existing_order_id, ticker, idempotency_key, detected_via="redis",
|
||||
)
|
||||
return
|
||||
|
||||
# Durable fallback: DB check
|
||||
existing = await check_idempotency_db(pool, idempotency_key)
|
||||
if existing:
|
||||
logger.info(
|
||||
"Duplicate order detected (db) for %s key=%s existing=%s status=%s",
|
||||
ticker, idempotency_key[:16], existing["id"], existing["status"],
|
||||
)
|
||||
ORDERS_DUPLICATES_PREVENTED.labels(detected_via="db").inc()
|
||||
await audit_duplicate_prevented(
|
||||
pool, existing["id"], ticker, idempotency_key, detected_via="db",
|
||||
)
|
||||
# Warm Redis cache for future fast-path hits
|
||||
if rds is not None:
|
||||
await mark_idempotency_redis(rds, idempotency_key, existing["id"])
|
||||
return
|
||||
|
||||
# Risk evaluation
|
||||
risk_config = await load_risk_config(pool)
|
||||
risk_state = await load_account_risk_state(pool, adapter, account_uuid)
|
||||
proposed = build_proposed_order(job)
|
||||
evaluation = evaluate_order(proposed, risk_config, risk_state)
|
||||
|
||||
risk_eval_dict = {
|
||||
"evaluation_id": evaluation.evaluation_id,
|
||||
"eligible": evaluation.eligible,
|
||||
"allowed_mode": evaluation.allowed_mode.value,
|
||||
"rejection_reasons": evaluation.rejection_reasons,
|
||||
"checks": [c.model_dump(mode="json") for c in evaluation.checks],
|
||||
}
|
||||
|
||||
# Persist risk evaluation
|
||||
rec_id = job.get("recommendation_id")
|
||||
try:
|
||||
await pool.execute(
|
||||
_INSERT_RISK_EVALUATION,
|
||||
evaluation.evaluation_id,
|
||||
rec_id,
|
||||
evaluation.eligible,
|
||||
evaluation.allowed_mode.value,
|
||||
json.dumps(evaluation.rejection_reasons),
|
||||
json.dumps(risk_eval_dict["checks"]),
|
||||
evaluation.evaluated_at,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to persist risk evaluation: %s", e)
|
||||
|
||||
# Audit: risk evaluation result
|
||||
await audit_risk_evaluated(
|
||||
pool,
|
||||
evaluation_id=evaluation.evaluation_id,
|
||||
recommendation_id=rec_id,
|
||||
ticker=ticker,
|
||||
eligible=evaluation.eligible,
|
||||
allowed_mode=evaluation.allowed_mode.value,
|
||||
rejection_reasons=evaluation.rejection_reasons,
|
||||
check_count=len(evaluation.checks),
|
||||
)
|
||||
|
||||
if not evaluation.eligible:
|
||||
RISK_EVALUATIONS_TOTAL.labels(result="rejected").inc()
|
||||
for check in evaluation.checks:
|
||||
if check.result.value == "fail":
|
||||
RISK_CHECK_FAILURES.labels(check_name=check.check_name).inc()
|
||||
ORDERS_REJECTED.labels(reason_category="risk_engine").inc()
|
||||
logger.info(
|
||||
"Order rejected by risk engine for %s: %s",
|
||||
ticker, evaluation.rejection_reasons,
|
||||
)
|
||||
# Persist the rejected order for audit
|
||||
order_req = build_order_request(job)
|
||||
rejected_resp = OrderResponse(
|
||||
broker_order_id="",
|
||||
status=OrderStatus.REJECTED,
|
||||
ticker=ticker,
|
||||
side=OrderSide.SELL if job.get("side") == "sell" else OrderSide.BUY,
|
||||
quantity=float(job.get("quantity", 0)),
|
||||
error=f"Risk rejected: {'; '.join(evaluation.rejection_reasons)}",
|
||||
)
|
||||
await persist_order(
|
||||
pool, order_id, order_req, rejected_resp,
|
||||
account_uuid, risk_eval_dict, rec_id,
|
||||
)
|
||||
# Publish rejected order fact to analytical lake
|
||||
if minio_client is not None:
|
||||
try:
|
||||
publish_trade_order(
|
||||
minio_client, order_id, ticker,
|
||||
side=job.get("side", "buy"),
|
||||
order_type=job.get("order_type", "market"),
|
||||
quantity=float(job.get("quantity", 0)),
|
||||
limit_price=job.get("limit_price"),
|
||||
status="rejected",
|
||||
broker_account=account_uuid,
|
||||
submitted_at=datetime.now(timezone.utc),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to publish rejected order to lake: %s", e)
|
||||
# Audit: order rejected by risk engine
|
||||
await audit_order_rejected(
|
||||
pool, order_id, ticker,
|
||||
reason=f"Risk rejected: {'; '.join(evaluation.rejection_reasons)}",
|
||||
source="risk_engine",
|
||||
)
|
||||
# Mark idempotency even for rejected orders to prevent reprocessing
|
||||
if rds is not None:
|
||||
await mark_idempotency_redis(rds, idempotency_key, order_id)
|
||||
return
|
||||
|
||||
# --- Operator approval gate (Requirement 8.2) ---
|
||||
if requires_approval(risk_config, evaluation.allowed_mode):
|
||||
expiry = compute_expiry(risk_config)
|
||||
approval_req = ApprovalRequest(
|
||||
order_job=job,
|
||||
recommendation_id=rec_id,
|
||||
ticker=ticker,
|
||||
side=job.get("side", "buy"),
|
||||
quantity=float(job.get("quantity", 0)),
|
||||
estimated_value=float(job.get("estimated_value", 0)),
|
||||
risk_evaluation_id=evaluation.evaluation_id,
|
||||
expires_at=expiry,
|
||||
)
|
||||
try:
|
||||
await create_approval_request(pool, approval_req)
|
||||
logger.info(
|
||||
"Order for %s held for operator approval (id=%s, expires=%s)",
|
||||
ticker, approval_req.approval_id, expiry.isoformat(),
|
||||
)
|
||||
await audit_approval_requested(
|
||||
pool,
|
||||
approval_id=approval_req.approval_id,
|
||||
ticker=ticker,
|
||||
side=approval_req.side,
|
||||
quantity=approval_req.quantity,
|
||||
estimated_value=approval_req.estimated_value,
|
||||
recommendation_id=rec_id,
|
||||
expires_at=expiry.isoformat(),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Failed to create approval request for %s: %s", ticker, e)
|
||||
# Do NOT mark idempotency — the job will be re-submitted after approval
|
||||
return
|
||||
|
||||
# Submit to Alpaca
|
||||
order_req = build_order_request(job)
|
||||
RISK_EVALUATIONS_TOTAL.labels(result="passed").inc()
|
||||
|
||||
# Audit: order submitted to broker
|
||||
await audit_order_submitted(
|
||||
pool,
|
||||
order_id=order_id,
|
||||
ticker=ticker,
|
||||
side=order_req.side.value,
|
||||
quantity=order_req.quantity,
|
||||
order_type=order_req.order_type.value,
|
||||
idempotency_key=order_req.idempotency_key,
|
||||
recommendation_id=rec_id,
|
||||
evaluation_id=evaluation.evaluation_id,
|
||||
)
|
||||
|
||||
resp = await adapter.submit_order(order_req)
|
||||
|
||||
await persist_order(
|
||||
pool, order_id, order_req, resp,
|
||||
account_uuid, risk_eval_dict, rec_id,
|
||||
)
|
||||
|
||||
# Publish order fact to analytical lake
|
||||
if minio_client is not None:
|
||||
try:
|
||||
publish_trade_order(
|
||||
minio_client, order_id, ticker,
|
||||
side=order_req.side.value,
|
||||
order_type=order_req.order_type.value,
|
||||
quantity=order_req.quantity,
|
||||
limit_price=order_req.limit_price,
|
||||
status=resp.status.value,
|
||||
broker_account=account_uuid,
|
||||
submitted_at=resp.submitted_at or datetime.now(timezone.utc),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to publish order to lake: %s", e)
|
||||
|
||||
# Publish fill fact if the order was filled
|
||||
if resp.status == OrderStatus.FILLED and resp.filled_avg_price is not None:
|
||||
try:
|
||||
fill_id = str(uuid.uuid4())
|
||||
publish_trade_fill(
|
||||
minio_client, fill_id, order_id, ticker,
|
||||
side=order_req.side.value,
|
||||
fill_price=resp.filled_avg_price,
|
||||
fill_quantity=resp.filled_quantity,
|
||||
broker_account=account_uuid,
|
||||
filled_at=datetime.now(timezone.utc),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to publish fill to lake: %s", e)
|
||||
|
||||
# Mark idempotency after successful persistence
|
||||
if rds is not None:
|
||||
await mark_idempotency_redis(rds, idempotency_key, order_id)
|
||||
|
||||
if resp.ok:
|
||||
mode = "paper" if adapter.mode == TradingMode.PAPER else "live"
|
||||
ORDERS_SUBMITTED.labels(
|
||||
side=order_req.side.value,
|
||||
order_type=order_req.order_type.value,
|
||||
mode=mode,
|
||||
).inc()
|
||||
logger.info(
|
||||
"Order submitted to Alpaca: %s %s %.0f %s @ %s | broker_id=%s",
|
||||
resp.status.value, order_req.side.value, order_req.quantity,
|
||||
ticker, resp.filled_avg_price, resp.broker_order_id,
|
||||
)
|
||||
# Audit: order filled
|
||||
if resp.status == OrderStatus.FILLED:
|
||||
ORDERS_FILLED.labels(side=order_req.side.value).inc()
|
||||
await audit_order_filled(
|
||||
pool, order_id, ticker,
|
||||
side=order_req.side.value,
|
||||
fill_quantity=resp.filled_quantity,
|
||||
fill_price=resp.filled_avg_price,
|
||||
broker_order_id=resp.broker_order_id,
|
||||
)
|
||||
else:
|
||||
ORDERS_REJECTED.labels(reason_category="broker").inc()
|
||||
logger.warning(
|
||||
"Order failed for %s: %s (status=%s)",
|
||||
ticker, resp.error, resp.status.value,
|
||||
)
|
||||
# Audit: order rejected by broker
|
||||
await audit_order_rejected(
|
||||
pool, order_id, ticker,
|
||||
reason=resp.error or f"Broker status: {resp.status.value}",
|
||||
source="broker",
|
||||
)
|
||||
|
||||
|
||||
|
||||
async def position_sync_loop(
|
||||
adapter: AlpacaBrokerAdapter,
|
||||
pool: asyncpg.Pool,
|
||||
account_uuid: str,
|
||||
minio_client: Any | None = None,
|
||||
) -> None:
|
||||
"""Periodically sync positions from Alpaca to PostgreSQL and lake."""
|
||||
while True:
|
||||
await sync_positions(adapter, pool, account_uuid, minio_client)
|
||||
await asyncio.sleep(POSITION_SYNC_INTERVAL)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
config = load_config()
|
||||
setup_logging("broker_service", level=config.log_level, json_output=config.json_logs)
|
||||
|
||||
pool = await get_pg_pool(config)
|
||||
rds = get_redis(config)
|
||||
|
||||
# Initialize MinIO client for lake publishing
|
||||
from minio import Minio
|
||||
minio_client = Minio(
|
||||
config.minio.endpoint,
|
||||
access_key=config.minio.access_key,
|
||||
secret_key=config.minio.secret_key,
|
||||
secure=config.minio.secure,
|
||||
)
|
||||
# Ensure lakehouse bucket exists
|
||||
if not minio_client.bucket_exists(LAKEHOUSE_BUCKET):
|
||||
minio_client.make_bucket(LAKEHOUSE_BUCKET)
|
||||
|
||||
# Determine mode — default to paper for safety (Req 8.1)
|
||||
mode = TradingMode.LIVE if config.broker.mode == "live" else TradingMode.PAPER
|
||||
if mode == TradingMode.LIVE:
|
||||
logger.warning("LIVE trading mode enabled — orders will be submitted to real broker")
|
||||
|
||||
adapter = AlpacaBrokerAdapter(
|
||||
api_key=config.broker.api_key or "",
|
||||
api_secret=config.broker.api_secret or "",
|
||||
mode=mode,
|
||||
base_url=config.broker.base_url,
|
||||
)
|
||||
|
||||
# Generate a stable account UUID from the API key
|
||||
account_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, f"alpaca-{config.broker.api_key or 'default'}"))
|
||||
|
||||
# Register broker account on startup
|
||||
await register_broker_account(pool, account_uuid, adapter)
|
||||
|
||||
# Start position sync in background
|
||||
sync_task = asyncio.create_task(
|
||||
position_sync_loop(adapter, pool, account_uuid, minio_client)
|
||||
)
|
||||
|
||||
queue = queue_key(QUEUE_BROKER)
|
||||
logger.info("Broker service started (mode=%s)", mode.value)
|
||||
|
||||
try:
|
||||
while True:
|
||||
result = await rds.lpop(queue)
|
||||
raw = str(result) if result else None
|
||||
if raw:
|
||||
try:
|
||||
job = json.loads(raw)
|
||||
await process_order_job(job, adapter, pool, account_uuid, rds, minio_client)
|
||||
except Exception:
|
||||
logger.exception("Error processing broker job")
|
||||
else:
|
||||
await asyncio.sleep(2)
|
||||
finally:
|
||||
sync_task.cancel()
|
||||
await pool.close()
|
||||
await rds.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user