From 45752b9a294716cbbd66a4cad97439ba6efe145f Mon Sep 17 00:00:00 2001 From: Celes Renata Date: Fri, 17 Apr 2026 01:24:35 +0000 Subject: [PATCH] feat: AI Agents management page with per-agent performance tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New Agents tab in the sidebar (Ops group) for viewing, editing, and creating AI agent configurations: Database (migration 026): - ai_agents table: editable configs for each LLM agent (model, prompts, temperature, tokens, retries). source='system' for built-in, source='user' for custom. Seeds 3 system agents (Document Extractor, Event Classifier, Thesis Rewriter) using WHERE NOT EXISTS to never overwrite user edits across reinstalls. - agent_performance_log table: per-invocation metrics (duration, confidence, retries, tokens, errors) linked to agent config. API endpoints: - GET/POST /api/agents — list and create agents - GET/PUT/DELETE /api/agents/{id} — view, edit, delete (system agents can be edited but not deleted) - GET /api/agents/{id}/performance — aggregated metrics (success rate, avg/p95 latency, confidence, token usage) - GET /api/agents/{id}/performance/history — hourly time series Frontend: - AgentsPage with sidebar list + detail panel - Agent detail: config display, system prompt viewer, performance dashboard with metrics cards and time-series chart - Edit form: all config fields editable including system prompt, model, temperature, tokens, retries - Create form: new user-defined agents with auto-slug generation - System agents show blue badge, user agents show green badge --- frontend/src/components/AppLayout.tsx | 2 + frontend/src/pages/Agents.tsx | 459 ++++++++++++++++++++++++++ frontend/src/routes.tsx | 8 + infra/migrations/026_ai_agents.sql | 86 +++++ services/api/app.py | 187 +++++++++++ 5 files changed, 742 insertions(+) create mode 100644 frontend/src/pages/Agents.tsx create mode 100644 infra/migrations/026_ai_agents.sql diff --git a/frontend/src/components/AppLayout.tsx b/frontend/src/components/AppLayout.tsx index 5c8dfbf..77412ee 100644 --- a/frontend/src/components/AppLayout.tsx +++ b/frontend/src/components/AppLayout.tsx @@ -18,6 +18,7 @@ import { List, Globe, BarChart3, + Bot, } from 'lucide-react'; interface NavItem { @@ -42,6 +43,7 @@ const navItems: NavItem[] = [ { to: '/ops/pipeline', label: 'Pipeline', icon: , group: 'Ops' }, { to: '/ops/ingestion', label: 'Ingestion', icon: , group: 'Ops' }, { to: '/ops/model', label: 'Model Perf', icon: , group: 'Ops' }, + { to: '/agents', label: 'Agents', icon: , group: 'Ops' }, { to: '/ops/coverage', label: 'Coverage', icon: , group: 'Ops' }, { to: '/analytics/query', label: 'SQL Explorer', icon: , group: 'Analytics' }, { to: '/analytics/dashboards', label: 'Dashboards', icon: , group: 'Analytics' }, diff --git a/frontend/src/pages/Agents.tsx b/frontend/src/pages/Agents.tsx new file mode 100644 index 0000000..1b7def9 --- /dev/null +++ b/frontend/src/pages/Agents.tsx @@ -0,0 +1,459 @@ +import { useState } from 'react'; +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; +import { apiGet, apiPost, apiPut, apiDelete } from '../api/client'; +import { Card, LoadingSpinner, StatusBadge } from '../components/ui'; +import { + LineChart, Line, XAxis, YAxis, Tooltip, ResponsiveContainer, CartesianGrid, +} from 'recharts'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface Agent { + id: string; + name: string; + slug: string; + purpose: string; + model_provider: string; + model_name: string; + system_prompt: string; + user_prompt_template: string; + prompt_version: string; + schema_version: string; + temperature: number; + max_tokens: number; + timeout_seconds: number; + max_retries: number; + active: boolean; + source: string; + created_at: string; + updated_at: string; +} + +interface AgentPerformance { + total_invocations: number; + successes: number; + failures: number; + avg_duration_ms: number | null; + p95_duration_ms: number | null; + avg_confidence: number | null; + avg_retries: number | null; + total_input_tokens: number | null; + total_output_tokens: number | null; + success_rate: number | null; +} + +interface PerfHistoryPoint { + hour: string; + invocations: number; + successes: number; + avg_duration_ms: number; + avg_confidence: number; +} + +// --------------------------------------------------------------------------- +// Hooks +// --------------------------------------------------------------------------- + +function useAgents() { + return useQuery({ + queryKey: ['agents'], + queryFn: () => apiGet('query', '/api/agents'), + }); +} + +function useAgentPerformance(agentId: string | undefined, hours = 24) { + return useQuery({ + queryKey: ['agent-performance', agentId, hours], + queryFn: () => apiGet('query', `/api/agents/${agentId}/performance?hours=${hours}`), + enabled: !!agentId, + }); +} + +function useAgentPerfHistory(agentId: string | undefined, hours = 24) { + return useQuery({ + queryKey: ['agent-perf-history', agentId, hours], + queryFn: () => apiGet('query', `/api/agents/${agentId}/performance/history?hours=${hours}`), + enabled: !!agentId, + }); +} + +// --------------------------------------------------------------------------- +// Page Component +// --------------------------------------------------------------------------- + +export function AgentsPage() { + const qc = useQueryClient(); + const { data: agents, isLoading } = useAgents(); + const [selectedId, setSelectedId] = useState(undefined); + const [editing, setEditing] = useState(false); + const [creating, setCreating] = useState(false); + + const selected = agents?.find((a) => a.id === selectedId); + + if (isLoading) return ; + + return ( +
+ {/* Agent List Sidebar */} +
+
+

Agents

+ +
+ {(agents ?? []).map((agent) => ( + + ))} +
+ + {/* Detail Panel */} +
+ {creating ? ( + { qc.invalidateQueries({ queryKey: ['agents'] }); setSelectedId(id); setCreating(false); }} + onCancel={() => setCreating(false)} + /> + ) : selected ? ( + editing ? ( + { qc.invalidateQueries({ queryKey: ['agents'] }); setEditing(false); }} onCancel={() => setEditing(false)} /> + ) : ( + setEditing(true)} onDeleted={() => { qc.invalidateQueries({ queryKey: ['agents'] }); setSelectedId(undefined); }} /> + ) + ) : ( + +

Select an agent to view its configuration and performance, or create a new one.

+ + )} +
+
+ ); +} + +// --------------------------------------------------------------------------- +// Agent Detail View +// --------------------------------------------------------------------------- + +function AgentDetail({ agent, onEdit, onDeleted }: { agent: Agent; onEdit: () => void; onDeleted: () => void }) { + const qc = useQueryClient(); + const { data: perf } = useAgentPerformance(agent.id); + const { data: history } = useAgentPerfHistory(agent.id); + + const deleteMut = useMutation({ + mutationFn: () => apiDelete('query', `/api/agents/${agent.id}`), + onSuccess: () => { qc.invalidateQueries({ queryKey: ['agents'] }); onDeleted(); }, + }); + + const chartData = (history ?? []).map((pt) => ({ + hour: new Date(pt.hour).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }), + invocations: pt.invocations, + successRate: pt.invocations > 0 ? Math.round((pt.successes / pt.invocations) * 100) : 0, + avgLatency: pt.avg_duration_ms, + avgConfidence: Math.round((pt.avg_confidence ?? 0) * 100), + })); + + return ( +
+ {/* Header */} +
+
+

{agent.name}

+ + + {agent.source} + +
+
+ + {agent.source === 'user' && ( + + )} +
+
+ + {/* Config */} + +
+
Model
{agent.model_provider}/{agent.model_name}
+
Prompt Version
{agent.prompt_version || '—'}
+
Schema Version
{agent.schema_version}
+
Temperature
{agent.temperature}
+
Max Tokens
{agent.max_tokens.toLocaleString()}
+
Timeout
{agent.timeout_seconds}s
+
Max Retries
{agent.max_retries}
+
Updated
{new Date(agent.updated_at).toLocaleString()}
+
+ + + {agent.purpose && ( + +

Purpose

+

{agent.purpose}

+ + )} + + +

System Prompt

+ 
{agent.system_prompt || '(none)'}
+ + + {/* Performance Metrics */} + {perf && ( + +

Performance (24h)

+
+ + = 0.95 ? 'text-green-400' : 'text-yellow-400'} /> + + + +
+
+ Retries avg: {perf.avg_retries?.toFixed(1) ?? '—'} + Input tokens: {perf.total_input_tokens?.toLocaleString() ?? '—'} + Output tokens: {perf.total_output_tokens?.toLocaleString() ?? '—'} +
+ + )} + + {/* Performance Chart */} + {chartData.length > 1 && ( + +

Performance Over Time

+ + + + + + + + + + + + + + )} +
+ ); +} + +// --------------------------------------------------------------------------- +// Edit Form +// --------------------------------------------------------------------------- + +function EditAgentForm({ agent, onSaved, onCancel }: { agent: Agent; onSaved: () => void; onCancel: () => void }) { + const [form, setForm] = useState({ + name: agent.name, + purpose: agent.purpose, + model_provider: agent.model_provider, + model_name: agent.model_name, + system_prompt: agent.system_prompt, + user_prompt_template: agent.user_prompt_template, + prompt_version: agent.prompt_version, + schema_version: agent.schema_version, + temperature: agent.temperature, + max_tokens: agent.max_tokens, + timeout_seconds: agent.timeout_seconds, + max_retries: agent.max_retries, + active: agent.active, + }); + + const mutation = useMutation({ + mutationFn: (body: Record) => apiPut('query', `/api/agents/${agent.id}`, body), + onSuccess: onSaved, + }); + + function handleSubmit(e: React.FormEvent) { + e.preventDefault(); + mutation.mutate(form); + } + + return ( + +

Edit Agent: {agent.name}

+
+ + setForm({ ...form, name: e.target.value })} className="w-full rounded-md border border-surface-700 bg-surface-950 px-2 py-1.5 text-sm text-gray-200 focus:border-brand-500 focus:outline-none" /> + + +