Celes Renata
7c23c044d7
- Migration 027: agent_variants table with single-active enforcement, variant_id column on agent_performance_log - API: full CRUD, clone from agent/variant, activate/deactivate, per-variant performance metrics and history endpoints - Services: extractor, event classifier, thesis rewriter all wired to AgentConfigResolver with variant override support - Frontend: variant list, comparison view, create/edit/clone forms, activate/delete actions on Agents page - Tests: API tests + 5 property-based tests (single-active invariant, clone preservation, config resolution, slug determinism, update idempotence) - Spec files for agent-variants feature
550 lines
20 KiB
Python
550 lines
20 KiB
Python
"""Unit tests for agent variant API endpoints.
|
||
|
||
Tests variant CRUD, clone, activate/deactivate, performance queries,
|
||
and edge cases (duplicate slug, non-existent resources, validation).
|
||
|
||
Requirements: 1.3, 1.4, 2.1–2.6, 3.1–3.6, 4.1–4.5, 6.3–6.5
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import uuid
|
||
from datetime import datetime, timezone
|
||
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
||
import asyncpg
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
|
||
from services.api.app import app
|
||
|
||
NOW = datetime(2026, 7, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||
AGENT_ID = str(uuid.uuid4())
|
||
VARIANT_ID = str(uuid.uuid4())
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class FakeRecord(dict):
|
||
"""Mimics asyncpg.Record for testing."""
|
||
|
||
def items(self):
|
||
return super().items()
|
||
|
||
|
||
def _variant_row(
|
||
*,
|
||
variant_id: str | None = None,
|
||
agent_id: str = AGENT_ID,
|
||
is_active: bool = False,
|
||
variant_name: str = "test-variant",
|
||
variant_slug: str = "test-variant",
|
||
model_name: str = "qwen3:8b",
|
||
**overrides,
|
||
) -> FakeRecord:
|
||
row = {
|
||
"id": variant_id or str(uuid.uuid4()),
|
||
"agent_id": agent_id,
|
||
"variant_name": variant_name,
|
||
"variant_slug": variant_slug,
|
||
"description": "",
|
||
"model_provider": "ollama",
|
||
"model_name": model_name,
|
||
"system_prompt": "You are a test agent.",
|
||
"user_prompt_template": "Analyze: {text}",
|
||
"prompt_version": "v1",
|
||
"temperature": 0.1,
|
||
"max_tokens": 16384,
|
||
"context_window": 4096,
|
||
"input_token_limit": 2048,
|
||
"token_budget": 10000,
|
||
"timeout_seconds": 120,
|
||
"max_retries": 2,
|
||
"is_active": is_active,
|
||
"created_at": NOW,
|
||
"updated_at": NOW,
|
||
}
|
||
row.update(overrides)
|
||
return FakeRecord(row)
|
||
|
||
|
||
def _agent_row(agent_id: str = AGENT_ID) -> FakeRecord:
|
||
return FakeRecord({
|
||
"id": agent_id,
|
||
"model_provider": "ollama",
|
||
"model_name": "qwen3:8b",
|
||
"system_prompt": "Base system prompt",
|
||
"user_prompt_template": "Base template: {text}",
|
||
"prompt_version": "v1",
|
||
"temperature": 0.0,
|
||
"max_tokens": 32768,
|
||
"timeout_seconds": 120,
|
||
"max_retries": 2,
|
||
})
|
||
|
||
|
||
def _perf_row() -> FakeRecord:
|
||
return FakeRecord({
|
||
"total_invocations": 100,
|
||
"successes": 90,
|
||
"failures": 10,
|
||
"avg_duration_ms": 450,
|
||
"p95_duration_ms": 900,
|
||
"avg_confidence": 0.82,
|
||
"avg_retries": 0.3,
|
||
"total_input_tokens": 50000,
|
||
"total_output_tokens": 25000,
|
||
})
|
||
|
||
|
||
def _perf_history_row(hour_offset: int = 0) -> FakeRecord:
|
||
from datetime import timedelta
|
||
|
||
return FakeRecord({
|
||
"hour": NOW - timedelta(hours=hour_offset),
|
||
"invocations": 10,
|
||
"successes": 9,
|
||
"avg_duration_ms": 400,
|
||
"avg_confidence": 0.85,
|
||
})
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Task 7.1.1 — CRUD, clone, activate/deactivate, performance
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestCreateVariant:
|
||
"""POST /api/agents/{agent_id}/variants"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_create_variant_returns_201(self):
|
||
created = _variant_row()
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(return_value=created)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(
|
||
f"/api/agents/{AGENT_ID}/variants",
|
||
json={
|
||
"variant_name": "test-variant",
|
||
"model_name": "qwen3:8b",
|
||
"context_window": 4096,
|
||
"input_token_limit": 2048,
|
||
"token_budget": 10000,
|
||
},
|
||
)
|
||
|
||
assert resp.status_code == 201
|
||
data = resp.json()
|
||
assert data["variant_name"] == "test-variant"
|
||
assert data["model_name"] == "qwen3:8b"
|
||
assert data["context_window"] == 4096
|
||
assert data["input_token_limit"] == 2048
|
||
assert data["token_budget"] == 10000
|
||
assert data["is_active"] is False
|
||
|
||
|
||
class TestCloneAgentAsVariant:
|
||
"""POST /api/agents/{agent_id}/clone"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_clone_agent_returns_201(self):
|
||
agent = _agent_row()
|
||
created = _variant_row(variant_name="cloned-from-agent", variant_slug="cloned-from-agent")
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(side_effect=[agent, created])
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(
|
||
f"/api/agents/{AGENT_ID}/clone",
|
||
json={"variant_name": "cloned-from-agent"},
|
||
)
|
||
|
||
assert resp.status_code == 201
|
||
data = resp.json()
|
||
assert data["variant_name"] == "cloned-from-agent"
|
||
assert data["agent_id"] == AGENT_ID
|
||
|
||
|
||
class TestCloneVariant:
|
||
"""POST /api/agents/{agent_id}/variants/{variant_id}/clone"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_clone_variant_returns_201(self):
|
||
source = _variant_row(variant_id=VARIANT_ID)
|
||
cloned = _variant_row(variant_name="cloned-v2", variant_slug="cloned-v2")
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(side_effect=[source, cloned])
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(
|
||
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}/clone",
|
||
json={"variant_name": "cloned-v2"},
|
||
)
|
||
|
||
assert resp.status_code == 201
|
||
data = resp.json()
|
||
assert data["variant_name"] == "cloned-v2"
|
||
|
||
|
||
class TestListVariants:
|
||
"""GET /api/agents/{agent_id}/variants"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_list_variants_returns_list(self):
|
||
rows = [
|
||
_variant_row(variant_name="v1", variant_slug="v1"),
|
||
_variant_row(variant_name="v2", variant_slug="v2", is_active=True),
|
||
]
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetch = AsyncMock(return_value=rows)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.get(f"/api/agents/{AGENT_ID}/variants")
|
||
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert len(data) == 2
|
||
assert data[0]["variant_name"] == "v1"
|
||
assert data[1]["is_active"] is True
|
||
|
||
|
||
class TestGetVariant:
|
||
"""GET /api/agents/{agent_id}/variants/{variant_id}"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_get_variant_returns_variant(self):
|
||
row = _variant_row(variant_id=VARIANT_ID)
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(return_value=row)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.get(f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}")
|
||
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["id"] == VARIANT_ID
|
||
|
||
|
||
class TestUpdateVariant:
|
||
"""PUT /api/agents/{agent_id}/variants/{variant_id}"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_update_variant_returns_updated(self):
|
||
updated = _variant_row(variant_id=VARIANT_ID, model_name="llama3.1:8b", temperature=0.5)
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(return_value=updated)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.put(
|
||
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}",
|
||
json={"model_name": "llama3.1:8b", "temperature": 0.5},
|
||
)
|
||
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["model_name"] == "llama3.1:8b"
|
||
assert data["temperature"] == 0.5
|
||
|
||
|
||
class TestDeleteVariant:
|
||
"""DELETE /api/agents/{agent_id}/variants/{variant_id}"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_delete_inactive_variant_succeeds(self):
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(return_value=FakeRecord({"is_active": False}))
|
||
mock_pool.execute = AsyncMock()
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.delete(f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}")
|
||
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["deleted"] is True
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_delete_active_variant_returns_400(self):
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(return_value=FakeRecord({"is_active": True}))
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.delete(f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}")
|
||
|
||
assert resp.status_code == 400
|
||
assert "active" in resp.json()["detail"].lower()
|
||
|
||
|
||
class TestActivateDeactivate:
|
||
"""POST .../activate and .../deactivate"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_activate_variant(self):
|
||
activated = _variant_row(variant_id=VARIANT_ID, is_active=True)
|
||
mock_conn = AsyncMock()
|
||
mock_conn.execute = AsyncMock()
|
||
mock_conn.fetchrow = AsyncMock(return_value=activated)
|
||
# transaction context manager
|
||
mock_tx = AsyncMock()
|
||
mock_tx.__aenter__ = AsyncMock(return_value=None)
|
||
mock_tx.__aexit__ = AsyncMock(return_value=False)
|
||
mock_conn.transaction = MagicMock(return_value=mock_tx)
|
||
|
||
mock_pool = AsyncMock()
|
||
mock_acquire = AsyncMock()
|
||
mock_acquire.__aenter__ = AsyncMock(return_value=mock_conn)
|
||
mock_acquire.__aexit__ = AsyncMock(return_value=False)
|
||
mock_pool.acquire = MagicMock(return_value=mock_acquire)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(
|
||
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}/activate"
|
||
)
|
||
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["is_active"] is True
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_deactivate_variants(self):
|
||
mock_pool = AsyncMock()
|
||
mock_pool.execute = AsyncMock()
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(f"/api/agents/{AGENT_ID}/variants/deactivate")
|
||
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["deactivated"] is True
|
||
|
||
|
||
class TestVariantPerformance:
|
||
"""GET .../performance and .../performance/history"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_get_variant_performance(self):
|
||
perf = _perf_row()
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(return_value=perf)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.get(
|
||
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}/performance?hours=24"
|
||
)
|
||
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert data["total_invocations"] == 100
|
||
assert data["successes"] == 90
|
||
assert data["success_rate"] == 0.9
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_get_variant_performance_history(self):
|
||
rows = [_perf_history_row(0), _perf_history_row(1)]
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetch = AsyncMock(return_value=rows)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.get(
|
||
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}/performance/history?hours=24"
|
||
)
|
||
|
||
assert resp.status_code == 200
|
||
data = resp.json()
|
||
assert len(data) == 2
|
||
assert data[0]["invocations"] == 10
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Task 7.1.2 — Edge-case tests
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class TestEdgeCases:
|
||
"""Edge-case tests: duplicate slug, non-existent resources, validation."""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_duplicate_slug_returns_409(self):
|
||
"""Creating a variant with a duplicate slug returns 409."""
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(
|
||
side_effect=asyncpg.UniqueViolationError("")
|
||
)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(
|
||
f"/api/agents/{AGENT_ID}/variants",
|
||
json={"variant_name": "dup", "model_name": "qwen3:8b"},
|
||
)
|
||
|
||
assert resp.status_code == 409
|
||
assert "already exists" in resp.json()["detail"]
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_clone_nonexistent_agent_returns_404(self):
|
||
"""Cloning from a non-existent agent returns 404."""
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(return_value=None)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(
|
||
f"/api/agents/{str(uuid.uuid4())}/clone",
|
||
json={"variant_name": "test"},
|
||
)
|
||
|
||
assert resp.status_code == 404
|
||
assert "not found" in resp.json()["detail"].lower()
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_get_nonexistent_variant_returns_404(self):
|
||
"""Getting a non-existent variant returns 404."""
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(return_value=None)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.get(
|
||
f"/api/agents/{AGENT_ID}/variants/{str(uuid.uuid4())}"
|
||
)
|
||
|
||
assert resp.status_code == 404
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_delete_nonexistent_variant_returns_404(self):
|
||
"""Deleting a non-existent variant returns 404."""
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(return_value=None)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.delete(
|
||
f"/api/agents/{AGENT_ID}/variants/{str(uuid.uuid4())}"
|
||
)
|
||
|
||
assert resp.status_code == 404
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_create_variant_empty_model_name_rejected(self):
|
||
"""Creating a variant with empty model_name is rejected by Pydantic."""
|
||
mock_pool = AsyncMock()
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(
|
||
f"/api/agents/{AGENT_ID}/variants",
|
||
json={"variant_name": "test"},
|
||
# model_name is required — omitting it should fail
|
||
)
|
||
|
||
assert resp.status_code == 422
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_update_variant_no_fields_returns_400(self):
|
||
"""Updating a variant with no fields returns 400."""
|
||
mock_pool = AsyncMock()
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.put(
|
||
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}",
|
||
json={},
|
||
)
|
||
|
||
assert resp.status_code == 400
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_clone_nonexistent_variant_returns_404(self):
|
||
"""Cloning from a non-existent variant returns 404."""
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(return_value=None)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(
|
||
f"/api/agents/{AGENT_ID}/variants/{str(uuid.uuid4())}/clone",
|
||
json={"variant_name": "test"},
|
||
)
|
||
|
||
assert resp.status_code == 404
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_activate_nonexistent_variant_returns_404(self):
|
||
"""Activating a non-existent variant returns 404."""
|
||
mock_conn = AsyncMock()
|
||
mock_conn.execute = AsyncMock()
|
||
mock_conn.fetchrow = AsyncMock(return_value=None)
|
||
mock_tx = AsyncMock()
|
||
mock_tx.__aenter__ = AsyncMock(return_value=None)
|
||
mock_tx.__aexit__ = AsyncMock(return_value=False)
|
||
mock_conn.transaction = MagicMock(return_value=mock_tx)
|
||
|
||
mock_pool = AsyncMock()
|
||
mock_acquire = AsyncMock()
|
||
mock_acquire.__aenter__ = AsyncMock(return_value=mock_conn)
|
||
mock_acquire.__aexit__ = AsyncMock(return_value=False)
|
||
mock_pool.acquire = MagicMock(return_value=mock_acquire)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(
|
||
f"/api/agents/{AGENT_ID}/variants/{str(uuid.uuid4())}/activate"
|
||
)
|
||
|
||
assert resp.status_code == 404
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_duplicate_slug_on_clone_returns_409(self):
|
||
"""Cloning an agent with a duplicate slug returns 409."""
|
||
agent = _agent_row()
|
||
mock_pool = AsyncMock()
|
||
mock_pool.fetchrow = AsyncMock(
|
||
side_effect=[agent, asyncpg.UniqueViolationError("")]
|
||
)
|
||
|
||
with patch("services.api.app.pool", mock_pool):
|
||
transport = ASGITransport(app=app)
|
||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||
resp = await client.post(
|
||
f"/api/agents/{AGENT_ID}/clone",
|
||
json={"variant_name": "dup", "variant_slug": "existing-slug"},
|
||
)
|
||
|
||
assert resp.status_code == 409
|