Celes Renata
c85c0068a2
- Replace all datetime.utcnow() with datetime.now(tz=timezone.utc) across 8 files - Fix 12 failing tests to match current implementation behavior - Fix pytest_plugins in non-top-level conftest (moved to root conftest.py) - Auto-fix 189 lint issues (import sorting, unused imports) - Add CI/CD pipeline infrastructure (ARC, ArgoCD, Kargo manifests) - Add values-beta.yaml and values-paper.yaml for staged deployments - Update GitHub Actions workflow to use self-hosted-gremlin runners - Add integration-test job to CI pipeline Result: 1596 passed, 0 failed, 0 warnings
334 lines
12 KiB
Python
334 lines
12 KiB
Python
"""Validate fail-closed behavior for broker outages and ambiguous order states.
|
|
|
|
Tests that the system rejects orders rather than risking duplicates or
|
|
ambiguous execution when the broker API is unavailable, returns errors,
|
|
times out, or returns unexpected/ambiguous responses.
|
|
|
|
Requirements: 8.4, 8.5, N5
|
|
Design: Section 10 - Reliability and Safety
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from services.adapters.broker_adapter import (
|
|
AlpacaBrokerAdapter,
|
|
OrderRequest,
|
|
OrderSide,
|
|
OrderStatus,
|
|
OrderType,
|
|
TradingMode,
|
|
)
|
|
from services.risk.engine import (
|
|
AccountRiskState,
|
|
DailyLossLimits,
|
|
PortfolioRiskConfig,
|
|
PositionLimits,
|
|
ProposedOrder,
|
|
RiskCheckResult,
|
|
evaluate_order,
|
|
)
|
|
from services.risk.engine import (
|
|
TradingMode as RiskTradingMode,
|
|
)
|
|
|
|
NOW = datetime(2026, 4, 11, 14, 0, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_adapter(base_url: str = "https://paper-api.alpaca.markets") -> AlpacaBrokerAdapter:
|
|
return AlpacaBrokerAdapter(
|
|
api_key="test-key",
|
|
api_secret="test-secret",
|
|
mode=TradingMode.PAPER,
|
|
base_url=base_url,
|
|
)
|
|
|
|
|
|
def _make_buy_order(ticker: str = "AAPL", qty: float = 10) -> OrderRequest:
|
|
return OrderRequest(
|
|
ticker=ticker,
|
|
side=OrderSide.BUY,
|
|
quantity=qty,
|
|
order_type=OrderType.MARKET,
|
|
idempotency_key=f"test-{ticker}-{qty}",
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 1. Broker network outage — submit_order fails closed
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestSubmitOrderFailsClosed:
|
|
"""submit_order must return REJECTED on any network/transport error."""
|
|
|
|
async def test_connection_error_returns_rejected(self):
|
|
adapter = _make_adapter()
|
|
order = _make_buy_order()
|
|
|
|
with patch("httpx.AsyncClient.post", side_effect=httpx.ConnectError("connection refused")):
|
|
resp = await adapter.submit_order(order)
|
|
|
|
assert resp.status == OrderStatus.REJECTED
|
|
assert resp.ok is False
|
|
assert "fail-closed" in resp.error
|
|
|
|
async def test_timeout_returns_rejected(self):
|
|
adapter = _make_adapter()
|
|
order = _make_buy_order()
|
|
|
|
with patch("httpx.AsyncClient.post", side_effect=httpx.ReadTimeout("read timed out")):
|
|
resp = await adapter.submit_order(order)
|
|
|
|
assert resp.status == OrderStatus.REJECTED
|
|
assert resp.ok is False
|
|
assert "fail-closed" in resp.error
|
|
|
|
async def test_dns_error_returns_rejected(self):
|
|
adapter = _make_adapter()
|
|
order = _make_buy_order()
|
|
|
|
with patch("httpx.AsyncClient.post", side_effect=httpx.ConnectError("DNS resolution failed")):
|
|
resp = await adapter.submit_order(order)
|
|
|
|
assert resp.status == OrderStatus.REJECTED
|
|
assert "fail-closed" in resp.error
|
|
|
|
async def test_http_500_returns_rejected(self):
|
|
"""Broker internal server error should result in rejection."""
|
|
adapter = _make_adapter()
|
|
order = _make_buy_order()
|
|
|
|
mock_resp = httpx.Response(500, text="Internal Server Error", request=httpx.Request("POST", "http://test"))
|
|
with patch("httpx.AsyncClient.post", side_effect=httpx.HTTPStatusError("500", response=mock_resp, request=mock_resp.request)):
|
|
resp = await adapter.submit_order(order)
|
|
|
|
assert resp.status == OrderStatus.REJECTED
|
|
assert resp.ok is False
|
|
assert resp.broker_order_id == ""
|
|
|
|
async def test_http_503_returns_rejected(self):
|
|
"""Broker service unavailable should result in rejection."""
|
|
adapter = _make_adapter()
|
|
order = _make_buy_order()
|
|
|
|
mock_resp = httpx.Response(503, text="Service Unavailable", request=httpx.Request("POST", "http://test"))
|
|
with patch("httpx.AsyncClient.post", side_effect=httpx.HTTPStatusError("503", response=mock_resp, request=mock_resp.request)):
|
|
resp = await adapter.submit_order(order)
|
|
|
|
assert resp.status == OrderStatus.REJECTED
|
|
assert resp.ok is False
|
|
|
|
async def test_rejected_order_has_empty_broker_id(self):
|
|
"""Fail-closed responses must not carry a broker order ID that could be confused with a real order."""
|
|
adapter = _make_adapter()
|
|
order = _make_buy_order()
|
|
|
|
with patch("httpx.AsyncClient.post", side_effect=Exception("unexpected")):
|
|
resp = await adapter.submit_order(order)
|
|
|
|
assert resp.broker_order_id == ""
|
|
assert resp.filled_quantity == 0
|
|
assert resp.filled_avg_price is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 2. Ambiguous order states — get_order_status fails closed
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestGetOrderStatusFailsClosed:
|
|
"""get_order_status must return REJECTED on errors, not an ambiguous state."""
|
|
|
|
async def test_network_error_returns_rejected(self):
|
|
adapter = _make_adapter()
|
|
|
|
with patch("httpx.AsyncClient.get", side_effect=httpx.ConnectError("refused")):
|
|
resp = await adapter.get_order_status("order-123")
|
|
|
|
assert resp.status == OrderStatus.REJECTED
|
|
assert resp.error is not None
|
|
|
|
async def test_timeout_returns_rejected(self):
|
|
adapter = _make_adapter()
|
|
|
|
with patch("httpx.AsyncClient.get", side_effect=httpx.ReadTimeout("timeout")):
|
|
resp = await adapter.get_order_status("order-123")
|
|
|
|
assert resp.status == OrderStatus.REJECTED
|
|
assert resp.error is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 3. Cancel order fails closed
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestCancelOrderFailsClosed:
|
|
"""cancel_order must return REJECTED on errors rather than leaving order in unknown state."""
|
|
|
|
async def test_network_error_returns_rejected(self):
|
|
adapter = _make_adapter()
|
|
|
|
with patch("httpx.AsyncClient.delete", side_effect=httpx.ConnectError("refused")):
|
|
resp = await adapter.cancel_order("order-456")
|
|
|
|
assert resp.status == OrderStatus.REJECTED
|
|
assert resp.error is not None
|
|
|
|
async def test_timeout_returns_rejected(self):
|
|
adapter = _make_adapter()
|
|
|
|
with patch("httpx.AsyncClient.delete", side_effect=httpx.ReadTimeout("timeout")):
|
|
resp = await adapter.cancel_order("order-456")
|
|
|
|
assert resp.status == OrderStatus.REJECTED
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 4. Position and account queries degrade safely
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestPositionAccountDegradation:
|
|
"""Position/account queries must return safe defaults on broker outage."""
|
|
|
|
async def test_get_positions_returns_empty_on_outage(self):
|
|
adapter = _make_adapter()
|
|
|
|
with patch("httpx.AsyncClient.get", side_effect=httpx.ConnectError("refused")):
|
|
positions = await adapter.get_positions()
|
|
|
|
assert positions == []
|
|
|
|
async def test_get_account_returns_zeroed_on_outage(self):
|
|
adapter = _make_adapter()
|
|
|
|
with patch("httpx.AsyncClient.get", side_effect=httpx.ConnectError("refused")):
|
|
acct = await adapter.get_account()
|
|
|
|
assert acct.buying_power == 0
|
|
assert acct.cash == 0
|
|
assert acct.portfolio_value == 0
|
|
assert acct.account_id == ""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 5. Risk engine fails closed with degraded state
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestRiskEngineFailClosed:
|
|
"""Risk engine must reject orders when account state is missing or degraded."""
|
|
|
|
def test_zero_portfolio_value_blocks_buy(self):
|
|
"""If broker is down and portfolio_value is 0, position pct → 1.0 → fail."""
|
|
config = PortfolioRiskConfig()
|
|
state = AccountRiskState(portfolio_value=0.0, cash=0.0)
|
|
order = ProposedOrder(
|
|
ticker="AAPL", sector="Technology",
|
|
estimated_value=1000, quantity=10,
|
|
)
|
|
result = evaluate_order(order, config, state)
|
|
assert not result.passed
|
|
pct_check = next(c for c in result.checks if c.check_name == "max_position_pct")
|
|
assert pct_check.result == RiskCheckResult.FAIL
|
|
assert pct_check.actual == 1.0
|
|
|
|
def test_disabled_mode_blocks_all_orders(self):
|
|
config = PortfolioRiskConfig(trading_mode=RiskTradingMode.DISABLED)
|
|
state = AccountRiskState(portfolio_value=100_000.0, cash=50_000.0)
|
|
order = ProposedOrder(
|
|
ticker="AAPL", sector="Technology",
|
|
estimated_value=1000, quantity=10,
|
|
)
|
|
result = evaluate_order(order, config, state)
|
|
assert not result.passed
|
|
assert any("disabled" in r.lower() for r in result.rejection_reasons)
|
|
|
|
def test_degraded_state_with_zero_buying_power(self):
|
|
"""When broker returns zeroed account, position value check should still block large orders."""
|
|
config = PortfolioRiskConfig(
|
|
position_limits=PositionLimits(max_position_value=5_000.0),
|
|
)
|
|
state = AccountRiskState(
|
|
portfolio_value=0.0, cash=0.0, buying_power=0.0,
|
|
)
|
|
order = ProposedOrder(
|
|
ticker="AAPL", sector="Technology",
|
|
estimated_value=10_000.0, quantity=50,
|
|
)
|
|
result = evaluate_order(order, config, state)
|
|
assert not result.passed
|
|
|
|
def test_multiple_risk_failures_all_captured_on_degraded_state(self):
|
|
"""Degraded state should trigger multiple failures, all recorded for audit."""
|
|
config = PortfolioRiskConfig(
|
|
position_limits=PositionLimits(max_position_value=500),
|
|
daily_loss=DailyLossLimits(max_daily_loss_value=0),
|
|
)
|
|
state = AccountRiskState(portfolio_value=0.0, daily_pnl=-1.0)
|
|
order = ProposedOrder(
|
|
ticker="AAPL", sector="Technology",
|
|
estimated_value=1000, quantity=10,
|
|
)
|
|
result = evaluate_order(order, config, state)
|
|
assert not result.passed
|
|
assert len(result.rejection_reasons) >= 2
|
|
# Full decision trace is preserved
|
|
assert len(result.checks) > 0
|
|
assert result.config_snapshot is not None
|
|
assert result.state_snapshot is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 6. Fetch (ingestion path) fails closed
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestFetchFailsClosed:
|
|
"""Broker fetch() for ingestion must return error result, not raise."""
|
|
|
|
async def test_fetch_connection_error_returns_error_result(self):
|
|
adapter = _make_adapter()
|
|
|
|
with patch("httpx.AsyncClient.get", side_effect=httpx.ConnectError("refused")):
|
|
result = await adapter.fetch("AAPL", {"endpoint": "positions"})
|
|
|
|
assert not result.ok
|
|
assert result.error is not None
|
|
assert result.items == []
|
|
|
|
async def test_fetch_timeout_returns_error_result(self):
|
|
adapter = _make_adapter()
|
|
|
|
with patch("httpx.AsyncClient.get", side_effect=httpx.ReadTimeout("timeout")):
|
|
result = await adapter.fetch("AAPL", {"endpoint": "orders"})
|
|
|
|
assert not result.ok
|
|
assert result.error is not None
|
|
|
|
async def test_fetch_http_429_returns_error_result(self):
|
|
adapter = _make_adapter()
|
|
|
|
mock_resp = httpx.Response(429, text="Rate limited", request=httpx.Request("GET", "http://test"))
|
|
with patch("httpx.AsyncClient.get", side_effect=httpx.HTTPStatusError("429", response=mock_resp, request=mock_resp.request)):
|
|
result = await adapter.fetch("AAPL", {"endpoint": "positions"})
|
|
|
|
assert not result.ok
|
|
assert result.http_status == 429
|