feat: agent variants — migration, API, service integration, frontend, tests

- Migration 027: agent_variants table with single-active enforcement,
  variant_id column on agent_performance_log
- API: full CRUD, clone from agent/variant, activate/deactivate,
  per-variant performance metrics and history endpoints
- Services: extractor, event classifier, thesis rewriter all wired
  to AgentConfigResolver with variant override support
- Frontend: variant list, comparison view, create/edit/clone forms,
  activate/delete actions on Agents page
- Tests: API tests + 5 property-based tests (single-active invariant,
  clone preservation, config resolution, slug determinism, update idempotence)
- Spec files for agent-variants feature
This commit is contained in:
Celes Renata
2026-04-17 05:15:42 +00:00
parent 734bf001a7
commit 7c23c044d7
14 changed files with 3118 additions and 120 deletions
+549
View File
@@ -0,0 +1,549 @@
"""Unit tests for agent variant API endpoints.
Tests variant CRUD, clone, activate/deactivate, performance queries,
and edge cases (duplicate slug, non-existent resources, validation).
Requirements: 1.3, 1.4, 2.12.6, 3.13.6, 4.14.5, 6.36.5
"""
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import asyncpg
import pytest
from httpx import ASGITransport, AsyncClient
from services.api.app import app
NOW = datetime(2026, 7, 1, 12, 0, 0, tzinfo=timezone.utc)
AGENT_ID = str(uuid.uuid4())
VARIANT_ID = str(uuid.uuid4())
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
class FakeRecord(dict):
"""Mimics asyncpg.Record for testing."""
def items(self):
return super().items()
def _variant_row(
*,
variant_id: str | None = None,
agent_id: str = AGENT_ID,
is_active: bool = False,
variant_name: str = "test-variant",
variant_slug: str = "test-variant",
model_name: str = "qwen3:8b",
**overrides,
) -> FakeRecord:
row = {
"id": variant_id or str(uuid.uuid4()),
"agent_id": agent_id,
"variant_name": variant_name,
"variant_slug": variant_slug,
"description": "",
"model_provider": "ollama",
"model_name": model_name,
"system_prompt": "You are a test agent.",
"user_prompt_template": "Analyze: {text}",
"prompt_version": "v1",
"temperature": 0.1,
"max_tokens": 16384,
"context_window": 4096,
"input_token_limit": 2048,
"token_budget": 10000,
"timeout_seconds": 120,
"max_retries": 2,
"is_active": is_active,
"created_at": NOW,
"updated_at": NOW,
}
row.update(overrides)
return FakeRecord(row)
def _agent_row(agent_id: str = AGENT_ID) -> FakeRecord:
return FakeRecord({
"id": agent_id,
"model_provider": "ollama",
"model_name": "qwen3:8b",
"system_prompt": "Base system prompt",
"user_prompt_template": "Base template: {text}",
"prompt_version": "v1",
"temperature": 0.0,
"max_tokens": 32768,
"timeout_seconds": 120,
"max_retries": 2,
})
def _perf_row() -> FakeRecord:
return FakeRecord({
"total_invocations": 100,
"successes": 90,
"failures": 10,
"avg_duration_ms": 450,
"p95_duration_ms": 900,
"avg_confidence": 0.82,
"avg_retries": 0.3,
"total_input_tokens": 50000,
"total_output_tokens": 25000,
})
def _perf_history_row(hour_offset: int = 0) -> FakeRecord:
from datetime import timedelta
return FakeRecord({
"hour": NOW - timedelta(hours=hour_offset),
"invocations": 10,
"successes": 9,
"avg_duration_ms": 400,
"avg_confidence": 0.85,
})
# ---------------------------------------------------------------------------
# Task 7.1.1 — CRUD, clone, activate/deactivate, performance
# ---------------------------------------------------------------------------
class TestCreateVariant:
"""POST /api/agents/{agent_id}/variants"""
@pytest.mark.asyncio
async def test_create_variant_returns_201(self):
created = _variant_row()
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(return_value=created)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(
f"/api/agents/{AGENT_ID}/variants",
json={
"variant_name": "test-variant",
"model_name": "qwen3:8b",
"context_window": 4096,
"input_token_limit": 2048,
"token_budget": 10000,
},
)
assert resp.status_code == 201
data = resp.json()
assert data["variant_name"] == "test-variant"
assert data["model_name"] == "qwen3:8b"
assert data["context_window"] == 4096
assert data["input_token_limit"] == 2048
assert data["token_budget"] == 10000
assert data["is_active"] is False
class TestCloneAgentAsVariant:
"""POST /api/agents/{agent_id}/clone"""
@pytest.mark.asyncio
async def test_clone_agent_returns_201(self):
agent = _agent_row()
created = _variant_row(variant_name="cloned-from-agent", variant_slug="cloned-from-agent")
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(side_effect=[agent, created])
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(
f"/api/agents/{AGENT_ID}/clone",
json={"variant_name": "cloned-from-agent"},
)
assert resp.status_code == 201
data = resp.json()
assert data["variant_name"] == "cloned-from-agent"
assert data["agent_id"] == AGENT_ID
class TestCloneVariant:
"""POST /api/agents/{agent_id}/variants/{variant_id}/clone"""
@pytest.mark.asyncio
async def test_clone_variant_returns_201(self):
source = _variant_row(variant_id=VARIANT_ID)
cloned = _variant_row(variant_name="cloned-v2", variant_slug="cloned-v2")
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(side_effect=[source, cloned])
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}/clone",
json={"variant_name": "cloned-v2"},
)
assert resp.status_code == 201
data = resp.json()
assert data["variant_name"] == "cloned-v2"
class TestListVariants:
"""GET /api/agents/{agent_id}/variants"""
@pytest.mark.asyncio
async def test_list_variants_returns_list(self):
rows = [
_variant_row(variant_name="v1", variant_slug="v1"),
_variant_row(variant_name="v2", variant_slug="v2", is_active=True),
]
mock_pool = AsyncMock()
mock_pool.fetch = AsyncMock(return_value=rows)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.get(f"/api/agents/{AGENT_ID}/variants")
assert resp.status_code == 200
data = resp.json()
assert len(data) == 2
assert data[0]["variant_name"] == "v1"
assert data[1]["is_active"] is True
class TestGetVariant:
"""GET /api/agents/{agent_id}/variants/{variant_id}"""
@pytest.mark.asyncio
async def test_get_variant_returns_variant(self):
row = _variant_row(variant_id=VARIANT_ID)
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(return_value=row)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.get(f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}")
assert resp.status_code == 200
data = resp.json()
assert data["id"] == VARIANT_ID
class TestUpdateVariant:
"""PUT /api/agents/{agent_id}/variants/{variant_id}"""
@pytest.mark.asyncio
async def test_update_variant_returns_updated(self):
updated = _variant_row(variant_id=VARIANT_ID, model_name="llama3.1:8b", temperature=0.5)
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(return_value=updated)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.put(
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}",
json={"model_name": "llama3.1:8b", "temperature": 0.5},
)
assert resp.status_code == 200
data = resp.json()
assert data["model_name"] == "llama3.1:8b"
assert data["temperature"] == 0.5
class TestDeleteVariant:
"""DELETE /api/agents/{agent_id}/variants/{variant_id}"""
@pytest.mark.asyncio
async def test_delete_inactive_variant_succeeds(self):
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(return_value=FakeRecord({"is_active": False}))
mock_pool.execute = AsyncMock()
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.delete(f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}")
assert resp.status_code == 200
data = resp.json()
assert data["deleted"] is True
@pytest.mark.asyncio
async def test_delete_active_variant_returns_400(self):
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(return_value=FakeRecord({"is_active": True}))
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.delete(f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}")
assert resp.status_code == 400
assert "active" in resp.json()["detail"].lower()
class TestActivateDeactivate:
"""POST .../activate and .../deactivate"""
@pytest.mark.asyncio
async def test_activate_variant(self):
activated = _variant_row(variant_id=VARIANT_ID, is_active=True)
mock_conn = AsyncMock()
mock_conn.execute = AsyncMock()
mock_conn.fetchrow = AsyncMock(return_value=activated)
# transaction context manager
mock_tx = AsyncMock()
mock_tx.__aenter__ = AsyncMock(return_value=None)
mock_tx.__aexit__ = AsyncMock(return_value=False)
mock_conn.transaction = MagicMock(return_value=mock_tx)
mock_pool = AsyncMock()
mock_acquire = AsyncMock()
mock_acquire.__aenter__ = AsyncMock(return_value=mock_conn)
mock_acquire.__aexit__ = AsyncMock(return_value=False)
mock_pool.acquire = MagicMock(return_value=mock_acquire)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}/activate"
)
assert resp.status_code == 200
data = resp.json()
assert data["is_active"] is True
@pytest.mark.asyncio
async def test_deactivate_variants(self):
mock_pool = AsyncMock()
mock_pool.execute = AsyncMock()
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(f"/api/agents/{AGENT_ID}/variants/deactivate")
assert resp.status_code == 200
data = resp.json()
assert data["deactivated"] is True
class TestVariantPerformance:
"""GET .../performance and .../performance/history"""
@pytest.mark.asyncio
async def test_get_variant_performance(self):
perf = _perf_row()
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(return_value=perf)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.get(
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}/performance?hours=24"
)
assert resp.status_code == 200
data = resp.json()
assert data["total_invocations"] == 100
assert data["successes"] == 90
assert data["success_rate"] == 0.9
@pytest.mark.asyncio
async def test_get_variant_performance_history(self):
rows = [_perf_history_row(0), _perf_history_row(1)]
mock_pool = AsyncMock()
mock_pool.fetch = AsyncMock(return_value=rows)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.get(
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}/performance/history?hours=24"
)
assert resp.status_code == 200
data = resp.json()
assert len(data) == 2
assert data[0]["invocations"] == 10
# ---------------------------------------------------------------------------
# Task 7.1.2 — Edge-case tests
# ---------------------------------------------------------------------------
class TestEdgeCases:
"""Edge-case tests: duplicate slug, non-existent resources, validation."""
@pytest.mark.asyncio
async def test_duplicate_slug_returns_409(self):
"""Creating a variant with a duplicate slug returns 409."""
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(
side_effect=asyncpg.UniqueViolationError("")
)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(
f"/api/agents/{AGENT_ID}/variants",
json={"variant_name": "dup", "model_name": "qwen3:8b"},
)
assert resp.status_code == 409
assert "already exists" in resp.json()["detail"]
@pytest.mark.asyncio
async def test_clone_nonexistent_agent_returns_404(self):
"""Cloning from a non-existent agent returns 404."""
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(return_value=None)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(
f"/api/agents/{str(uuid.uuid4())}/clone",
json={"variant_name": "test"},
)
assert resp.status_code == 404
assert "not found" in resp.json()["detail"].lower()
@pytest.mark.asyncio
async def test_get_nonexistent_variant_returns_404(self):
"""Getting a non-existent variant returns 404."""
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(return_value=None)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.get(
f"/api/agents/{AGENT_ID}/variants/{str(uuid.uuid4())}"
)
assert resp.status_code == 404
@pytest.mark.asyncio
async def test_delete_nonexistent_variant_returns_404(self):
"""Deleting a non-existent variant returns 404."""
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(return_value=None)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.delete(
f"/api/agents/{AGENT_ID}/variants/{str(uuid.uuid4())}"
)
assert resp.status_code == 404
@pytest.mark.asyncio
async def test_create_variant_empty_model_name_rejected(self):
"""Creating a variant with empty model_name is rejected by Pydantic."""
mock_pool = AsyncMock()
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(
f"/api/agents/{AGENT_ID}/variants",
json={"variant_name": "test"},
# model_name is required — omitting it should fail
)
assert resp.status_code == 422
@pytest.mark.asyncio
async def test_update_variant_no_fields_returns_400(self):
"""Updating a variant with no fields returns 400."""
mock_pool = AsyncMock()
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.put(
f"/api/agents/{AGENT_ID}/variants/{VARIANT_ID}",
json={},
)
assert resp.status_code == 400
@pytest.mark.asyncio
async def test_clone_nonexistent_variant_returns_404(self):
"""Cloning from a non-existent variant returns 404."""
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(return_value=None)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(
f"/api/agents/{AGENT_ID}/variants/{str(uuid.uuid4())}/clone",
json={"variant_name": "test"},
)
assert resp.status_code == 404
@pytest.mark.asyncio
async def test_activate_nonexistent_variant_returns_404(self):
"""Activating a non-existent variant returns 404."""
mock_conn = AsyncMock()
mock_conn.execute = AsyncMock()
mock_conn.fetchrow = AsyncMock(return_value=None)
mock_tx = AsyncMock()
mock_tx.__aenter__ = AsyncMock(return_value=None)
mock_tx.__aexit__ = AsyncMock(return_value=False)
mock_conn.transaction = MagicMock(return_value=mock_tx)
mock_pool = AsyncMock()
mock_acquire = AsyncMock()
mock_acquire.__aenter__ = AsyncMock(return_value=mock_conn)
mock_acquire.__aexit__ = AsyncMock(return_value=False)
mock_pool.acquire = MagicMock(return_value=mock_acquire)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(
f"/api/agents/{AGENT_ID}/variants/{str(uuid.uuid4())}/activate"
)
assert resp.status_code == 404
@pytest.mark.asyncio
async def test_duplicate_slug_on_clone_returns_409(self):
"""Cloning an agent with a duplicate slug returns 409."""
agent = _agent_row()
mock_pool = AsyncMock()
mock_pool.fetchrow = AsyncMock(
side_effect=[agent, asyncpg.UniqueViolationError("")]
)
with patch("services.api.app.pool", mock_pool):
transport = ASGITransport(app=app)
async with AsyncClient(transport=transport, base_url="http://test") as client:
resp = await client.post(
f"/api/agents/{AGENT_ID}/clone",
json={"variant_name": "dup", "variant_slug": "existing-slug"},
)
assert resp.status_code == 409