feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,108 @@
|
||||
{{- if .Values.specialist }}
|
||||
{{- if .Values.specialist.enabled }}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: specialist
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app: specialist
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
stonks-oracle/tier: processing
|
||||
spec:
|
||||
replicas: {{ .Values.specialist.replicas | default 2 }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: specialist
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: specialist
|
||||
stonks-oracle/tier: processing
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
{{- include "stonks.podSecurityContext" . | nindent 8 }}
|
||||
containers:
|
||||
- name: specialist
|
||||
image: {{ .Values.image.registry }}/specialist:{{ .Values.image.tag }}
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
command: ["sh", "-c", "uvicorn services.specialist.app:app --host 0.0.0.0 --port 8000"]
|
||||
ports:
|
||||
- containerPort: 8000
|
||||
env:
|
||||
- name: SPECIALIST_MODEL
|
||||
value: {{ .Values.specialist.model | default "urchade/gliner_large-v2.1" | quote }}
|
||||
- name: SPECIALIST_MAX_BATCH_SIZE
|
||||
value: {{ .Values.specialist.maxBatchSize | default "32" | quote }}
|
||||
- name: SPECIALIST_MAX_WAIT_MS
|
||||
value: {{ .Values.specialist.maxWaitMs | default "50.0" | quote }}
|
||||
- name: SPECIALIST_MAX_QUEUE_SIZE
|
||||
value: {{ .Values.specialist.maxQueueSize | default "256" | quote }}
|
||||
- name: SPECIALIST_TEST_MODE
|
||||
value: {{ .Values.specialist.testMode | default "0" | quote }}
|
||||
securityContext:
|
||||
{{- include "stonks.containerSecurityContext" . | nindent 12 }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: stonks-config
|
||||
{{- range .Values.specialist.secrets }}
|
||||
- secretRef:
|
||||
name: {{ . }}
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
cpu: {{ .Values.specialist.resources.requests.cpu | default "2" | quote }}
|
||||
memory: {{ .Values.specialist.resources.requests.memory | default "4Gi" }}
|
||||
limits:
|
||||
cpu: {{ .Values.specialist.resources.limits.cpu | default "6" | quote }}
|
||||
memory: {{ .Values.specialist.resources.limits.memory | default "10Gi" }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: 8000
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
- name: model-cache
|
||||
mountPath: /root/.cache
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
sizeLimit: 10Mi
|
||||
- name: model-cache
|
||||
emptyDir:
|
||||
sizeLimit: 5Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: specialist
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app: specialist
|
||||
{{- include "stonks.labels" . | nindent 4 }}
|
||||
spec:
|
||||
selector:
|
||||
app: specialist
|
||||
ports:
|
||||
- port: 8000
|
||||
targetPort: 8000
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -289,6 +289,24 @@ superset:
|
||||
requests: { cpu: 200m, memory: 512Mi }
|
||||
limits: { cpu: "1", memory: 2Gi }
|
||||
|
||||
## Specialist inference service (CPU-first NER/classification)
|
||||
specialist:
|
||||
enabled: true
|
||||
replicas: 2
|
||||
model: "urchade/gliner_large-v2.1"
|
||||
maxBatchSize: "32"
|
||||
maxWaitMs: "50.0"
|
||||
maxQueueSize: "256"
|
||||
testMode: "0"
|
||||
secrets: [stonks-core-secrets]
|
||||
resources:
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: 4Gi
|
||||
limits:
|
||||
cpu: "6"
|
||||
memory: 10Gi
|
||||
|
||||
## Network policies
|
||||
networkPolicies:
|
||||
enabled: true
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
-- Migration 040: Inference Registry
|
||||
-- Creates tables for the capability-aware inference gateway:
|
||||
-- inference_endpoints, model_deployments, agent_stage_bindings
|
||||
-- Adds lineage columns to agent_performance_log for v3 provenance tracking.
|
||||
|
||||
-- ─── Helper: auto-update updated_at on row modification ───────────────────────
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- ─── inference_endpoints ──────────────────────────────────────────────────────
|
||||
-- Stores registered inference service endpoints (Ollama, vLLM, OpenAI-compat, specialist).
|
||||
CREATE TABLE IF NOT EXISTS inference_endpoints (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
protocol TEXT NOT NULL CHECK (protocol IN ('ollama_native', 'openai_chat', 'specialist_http')),
|
||||
base_url TEXT NOT NULL,
|
||||
auth_secret_ref TEXT,
|
||||
auth_scheme TEXT NOT NULL DEFAULT 'bearer',
|
||||
default_headers JSONB NOT NULL DEFAULT '{}',
|
||||
health_path TEXT,
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
revision INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_inference_endpoints_protocol
|
||||
ON inference_endpoints(protocol);
|
||||
|
||||
-- Auto-update updated_at on inference_endpoints changes
|
||||
DROP TRIGGER IF EXISTS trg_inference_endpoints_updated_at ON inference_endpoints;
|
||||
CREATE TRIGGER trg_inference_endpoints_updated_at
|
||||
BEFORE UPDATE ON inference_endpoints
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- ─── model_deployments ────────────────────────────────────────────────────────
|
||||
-- A model served by an endpoint, with declared capabilities and limits.
|
||||
CREATE TABLE IF NOT EXISTS model_deployments (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
endpoint_id UUID NOT NULL REFERENCES inference_endpoints(id) ON DELETE CASCADE,
|
||||
served_model_name TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
capabilities JSONB NOT NULL,
|
||||
context_window INTEGER,
|
||||
max_output_tokens INTEGER,
|
||||
quantization TEXT,
|
||||
runtime_metadata JSONB NOT NULL DEFAULT '{}',
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
revision INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(endpoint_id, served_model_name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_model_deployments_endpoint
|
||||
ON model_deployments(endpoint_id);
|
||||
|
||||
-- Auto-update updated_at on model_deployments changes
|
||||
DROP TRIGGER IF EXISTS trg_model_deployments_updated_at ON model_deployments;
|
||||
CREATE TRIGGER trg_model_deployments_updated_at
|
||||
BEFORE UPDATE ON model_deployments
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- ─── agent_stage_bindings ─────────────────────────────────────────────────────
|
||||
-- Maps an agent + pipeline stage to one or more ordered model deployments.
|
||||
CREATE TABLE IF NOT EXISTS agent_stage_bindings (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
agent_id UUID NOT NULL REFERENCES ai_agents(id) ON DELETE CASCADE,
|
||||
stage TEXT NOT NULL,
|
||||
model_deployment_id UUID REFERENCES model_deployments(id) ON DELETE SET NULL,
|
||||
route_order INTEGER NOT NULL DEFAULT 0,
|
||||
routing_config JSONB NOT NULL DEFAULT '{}',
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
revision INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(agent_id, stage, route_order)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_stage_bindings_agent
|
||||
ON agent_stage_bindings(agent_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_stage_bindings_deployment
|
||||
ON agent_stage_bindings(model_deployment_id);
|
||||
|
||||
-- Auto-update updated_at on agent_stage_bindings changes
|
||||
DROP TRIGGER IF EXISTS trg_agent_stage_bindings_updated_at ON agent_stage_bindings;
|
||||
CREATE TRIGGER trg_agent_stage_bindings_updated_at
|
||||
BEFORE UPDATE ON agent_stage_bindings
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- ─── Additive lineage columns on agent_performance_log ────────────────────────
|
||||
-- Tracks which endpoint/deployment/binding was used for each logged invocation.
|
||||
-- NOTE: Revision increment logic is handled at the application layer:
|
||||
-- each UPDATE to inference_endpoints, model_deployments, or agent_stage_bindings
|
||||
-- should increment the revision column (enforced by service code, not DB trigger,
|
||||
-- to allow flexible conflict resolution).
|
||||
|
||||
ALTER TABLE agent_performance_log
|
||||
ADD COLUMN IF NOT EXISTS endpoint_id UUID REFERENCES inference_endpoints(id) ON DELETE SET NULL;
|
||||
|
||||
ALTER TABLE agent_performance_log
|
||||
ADD COLUMN IF NOT EXISTS deployment_id UUID REFERENCES model_deployments(id) ON DELETE SET NULL;
|
||||
|
||||
ALTER TABLE agent_performance_log
|
||||
ADD COLUMN IF NOT EXISTS binding_revision INTEGER;
|
||||
|
||||
ALTER TABLE agent_performance_log
|
||||
ADD COLUMN IF NOT EXISTS structured_mode TEXT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_perf_endpoint
|
||||
ON agent_performance_log(endpoint_id)
|
||||
WHERE endpoint_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_perf_deployment
|
||||
ON agent_performance_log(deployment_id)
|
||||
WHERE deployment_id IS NOT NULL;
|
||||
@@ -0,0 +1,350 @@
|
||||
-- Migration 041: V3 Pipeline Persistence Tables
|
||||
-- Creates tables for the Intelligence Pipeline v3 staged evidence architecture:
|
||||
-- v3_pipeline_runs, v3_stage_runs, v3_document_chunks, v3_evidence_spans,
|
||||
-- v3_extracted_entities, v3_extracted_facts, v3_extracted_relations,
|
||||
-- v3_rejected_candidates, v3_company_signal_candidates,
|
||||
-- v3_adjudication_decisions, v3_routing_decisions, v3_stage_lineage
|
||||
-- Includes idempotency keys, immutable-revision constraints, and appropriate indexes.
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
-- 20.1: Pipeline runs and stage runs
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
-- ─── v3_pipeline_runs ─────────────────────────────────────────────────────────
|
||||
-- Top-level pipeline execution record for a document.
|
||||
CREATE TABLE IF NOT EXISTS v3_pipeline_runs (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
document_id UUID NOT NULL,
|
||||
pipeline_version TEXT NOT NULL DEFAULT 'v3.0',
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'running', 'completed', 'failed')),
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
idempotency_key TEXT NOT NULL UNIQUE,
|
||||
error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_pipeline_runs_document
|
||||
ON v3_pipeline_runs(document_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_pipeline_runs_status
|
||||
ON v3_pipeline_runs(status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_pipeline_runs_created
|
||||
ON v3_pipeline_runs(created_at DESC);
|
||||
|
||||
-- ─── v3_stage_runs ────────────────────────────────────────────────────────────
|
||||
-- Individual stage execution within a pipeline run.
|
||||
CREATE TABLE IF NOT EXISTS v3_stage_runs (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
pipeline_run_id UUID NOT NULL REFERENCES v3_pipeline_runs(id) ON DELETE CASCADE,
|
||||
stage TEXT NOT NULL CHECK (stage IN (
|
||||
'segmentation', 'extraction', 'sentiment', 'novelty',
|
||||
'routing', 'adjudication', 'impact', 'persistence'
|
||||
)),
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'running', 'completed', 'failed', 'skipped')),
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
endpoint_id UUID REFERENCES inference_endpoints(id) ON DELETE SET NULL,
|
||||
deployment_id UUID REFERENCES model_deployments(id) ON DELETE SET NULL,
|
||||
model_version TEXT,
|
||||
schema_version TEXT,
|
||||
input_refs JSONB NOT NULL DEFAULT '[]',
|
||||
output_refs JSONB NOT NULL DEFAULT '[]',
|
||||
trace_id TEXT,
|
||||
error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_stage_runs_pipeline
|
||||
ON v3_stage_runs(pipeline_run_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_stage_runs_stage
|
||||
ON v3_stage_runs(stage);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_stage_runs_status
|
||||
ON v3_stage_runs(status);
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
-- 20.2: Document chunks and evidence spans
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
-- ─── v3_document_chunks ───────────────────────────────────────────────────────
|
||||
-- Segmented document chunks with offset tracking.
|
||||
CREATE TABLE IF NOT EXISTS v3_document_chunks (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
document_id UUID NOT NULL,
|
||||
chunk_id TEXT NOT NULL,
|
||||
section_path JSONB NOT NULL DEFAULT '[]',
|
||||
speaker TEXT,
|
||||
start_char INTEGER NOT NULL,
|
||||
end_char INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
overlap_left INTEGER NOT NULL DEFAULT 0,
|
||||
overlap_right INTEGER NOT NULL DEFAULT 0,
|
||||
boilerplate_score REAL NOT NULL DEFAULT 0.0,
|
||||
document_type TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(document_id, chunk_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_document_chunks_document
|
||||
ON v3_document_chunks(document_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_document_chunks_type
|
||||
ON v3_document_chunks(document_type)
|
||||
WHERE document_type IS NOT NULL;
|
||||
|
||||
-- ─── v3_evidence_spans ────────────────────────────────────────────────────────
|
||||
-- Exact source text with character offsets for provenance.
|
||||
CREATE TABLE IF NOT EXISTS v3_evidence_spans (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
document_id UUID NOT NULL,
|
||||
chunk_id TEXT,
|
||||
start_char INTEGER NOT NULL,
|
||||
end_char INTEGER NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
checksum TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_evidence_spans_document
|
||||
ON v3_evidence_spans(document_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_evidence_spans_checksum
|
||||
ON v3_evidence_spans(checksum);
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
-- 20.3: Extracted entities, facts, relations, and rejected candidates
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
-- ─── v3_extracted_entities ────────────────────────────────────────────────────
|
||||
-- Entities discovered during extraction (companies, people, orgs, etc.).
|
||||
CREATE TABLE IF NOT EXISTS v3_extracted_entities (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
pipeline_run_id UUID NOT NULL REFERENCES v3_pipeline_runs(id) ON DELETE CASCADE,
|
||||
entity_type TEXT NOT NULL,
|
||||
literal_text TEXT NOT NULL,
|
||||
canonical_id UUID,
|
||||
evidence_span_id UUID REFERENCES v3_evidence_spans(id) ON DELETE SET NULL,
|
||||
confidence REAL NOT NULL DEFAULT 0.0,
|
||||
derivation TEXT NOT NULL DEFAULT 'specialist',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_extracted_entities_pipeline
|
||||
ON v3_extracted_entities(pipeline_run_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_extracted_entities_canonical
|
||||
ON v3_extracted_entities(canonical_id)
|
||||
WHERE canonical_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_extracted_entities_type
|
||||
ON v3_extracted_entities(entity_type);
|
||||
|
||||
-- ─── v3_extracted_facts ───────────────────────────────────────────────────────
|
||||
-- Structured facts (numeric values, dates, amounts, etc.).
|
||||
CREATE TABLE IF NOT EXISTS v3_extracted_facts (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
pipeline_run_id UUID NOT NULL REFERENCES v3_pipeline_runs(id) ON DELETE CASCADE,
|
||||
fact_type TEXT NOT NULL,
|
||||
subject_entity_id UUID REFERENCES v3_extracted_entities(id) ON DELETE SET NULL,
|
||||
predicate TEXT NOT NULL,
|
||||
literal_value TEXT NOT NULL,
|
||||
normalized_value JSONB,
|
||||
unit TEXT,
|
||||
period JSONB,
|
||||
evidence_span_ids UUID[] NOT NULL DEFAULT '{}',
|
||||
confidence REAL NOT NULL DEFAULT 0.0,
|
||||
derivation TEXT NOT NULL DEFAULT 'deterministic',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_extracted_facts_pipeline
|
||||
ON v3_extracted_facts(pipeline_run_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_extracted_facts_subject
|
||||
ON v3_extracted_facts(subject_entity_id)
|
||||
WHERE subject_entity_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_extracted_facts_type
|
||||
ON v3_extracted_facts(fact_type);
|
||||
|
||||
-- ─── v3_extracted_relations ───────────────────────────────────────────────────
|
||||
-- Relations between entities (competes_with, supplies, etc.).
|
||||
CREATE TABLE IF NOT EXISTS v3_extracted_relations (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
pipeline_run_id UUID NOT NULL REFERENCES v3_pipeline_runs(id) ON DELETE CASCADE,
|
||||
relation_type TEXT NOT NULL,
|
||||
source_entity_id UUID NOT NULL REFERENCES v3_extracted_entities(id) ON DELETE CASCADE,
|
||||
target_entity_id UUID NOT NULL REFERENCES v3_extracted_entities(id) ON DELETE CASCADE,
|
||||
evidence_span_ids UUID[] NOT NULL DEFAULT '{}',
|
||||
confidence REAL NOT NULL DEFAULT 0.0,
|
||||
derivation TEXT NOT NULL DEFAULT 'specialist',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_extracted_relations_pipeline
|
||||
ON v3_extracted_relations(pipeline_run_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_extracted_relations_source
|
||||
ON v3_extracted_relations(source_entity_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_extracted_relations_target
|
||||
ON v3_extracted_relations(target_entity_id);
|
||||
|
||||
-- ─── v3_rejected_candidates ──────────────────────────────────────────────────
|
||||
-- Candidates that failed validation or were rejected by a stage.
|
||||
CREATE TABLE IF NOT EXISTS v3_rejected_candidates (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
pipeline_run_id UUID NOT NULL REFERENCES v3_pipeline_runs(id) ON DELETE CASCADE,
|
||||
candidate_type TEXT NOT NULL,
|
||||
candidate_data JSONB NOT NULL,
|
||||
rejection_reason TEXT NOT NULL,
|
||||
stage TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_rejected_candidates_pipeline
|
||||
ON v3_rejected_candidates(pipeline_run_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_rejected_candidates_stage
|
||||
ON v3_rejected_candidates(stage);
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
-- 20.4: Company signal candidates and probability distributions
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
-- ─── v3_company_signal_candidates ─────────────────────────────────────────────
|
||||
-- Per-company signal output with full probability distributions.
|
||||
CREATE TABLE IF NOT EXISTS v3_company_signal_candidates (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
pipeline_run_id UUID NOT NULL REFERENCES v3_pipeline_runs(id) ON DELETE CASCADE,
|
||||
company_id UUID NOT NULL REFERENCES companies(id) ON DELETE CASCADE,
|
||||
relevance_probability REAL NOT NULL DEFAULT 0.0,
|
||||
event_probabilities JSONB NOT NULL DEFAULT '{}',
|
||||
sentiment_probabilities JSONB NOT NULL DEFAULT '{}',
|
||||
direction_probabilities JSONB NOT NULL DEFAULT '{}',
|
||||
horizon_probabilities JSONB NOT NULL DEFAULT '{}',
|
||||
expected_magnitude REAL,
|
||||
evidence_span_ids UUID[] NOT NULL DEFAULT '{}',
|
||||
routing_reasons TEXT[] NOT NULL DEFAULT '{}',
|
||||
adjudicated BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_signal_candidates_pipeline
|
||||
ON v3_company_signal_candidates(pipeline_run_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_signal_candidates_company
|
||||
ON v3_company_signal_candidates(company_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_signal_candidates_adjudicated
|
||||
ON v3_company_signal_candidates(adjudicated)
|
||||
WHERE adjudicated = TRUE;
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
-- 20.5: Adjudication decisions, routing reasons, calibration, and model lineage
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
-- ─── v3_stage_lineage ─────────────────────────────────────────────────────────
|
||||
-- Detailed model/endpoint lineage for each stage invocation.
|
||||
-- Created before adjudication_decisions because it is referenced as a FK.
|
||||
CREATE TABLE IF NOT EXISTS v3_stage_lineage (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
stage_run_id UUID NOT NULL REFERENCES v3_stage_runs(id) ON DELETE CASCADE,
|
||||
endpoint_id UUID REFERENCES inference_endpoints(id) ON DELETE SET NULL,
|
||||
deployment_id UUID REFERENCES model_deployments(id) ON DELETE SET NULL,
|
||||
model TEXT,
|
||||
protocol TEXT,
|
||||
structured_mode TEXT,
|
||||
request_id TEXT,
|
||||
latency_ms INTEGER,
|
||||
retries INTEGER NOT NULL DEFAULT 0,
|
||||
trace_id TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_stage_lineage_stage_run
|
||||
ON v3_stage_lineage(stage_run_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_stage_lineage_endpoint
|
||||
ON v3_stage_lineage(endpoint_id)
|
||||
WHERE endpoint_id IS NOT NULL;
|
||||
|
||||
-- ─── v3_adjudication_decisions ────────────────────────────────────────────────
|
||||
-- Decisions made by the 9B adjudicator for ambiguous documents.
|
||||
CREATE TABLE IF NOT EXISTS v3_adjudication_decisions (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
pipeline_run_id UUID NOT NULL REFERENCES v3_pipeline_runs(id) ON DELETE CASCADE,
|
||||
question_codes TEXT[] NOT NULL DEFAULT '{}',
|
||||
candidates JSONB NOT NULL DEFAULT '{}',
|
||||
decision JSONB NOT NULL DEFAULT '{}',
|
||||
evidence_span_ids UUID[] NOT NULL DEFAULT '{}',
|
||||
model_lineage_id UUID REFERENCES v3_stage_lineage(id) ON DELETE SET NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_adjudication_pipeline
|
||||
ON v3_adjudication_decisions(pipeline_run_id);
|
||||
|
||||
-- ─── v3_routing_decisions ─────────────────────────────────────────────────────
|
||||
-- Records of fast-path vs adjudication routing decisions.
|
||||
CREATE TABLE IF NOT EXISTS v3_routing_decisions (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
pipeline_run_id UUID NOT NULL REFERENCES v3_pipeline_runs(id) ON DELETE CASCADE,
|
||||
document_id UUID NOT NULL,
|
||||
route TEXT NOT NULL CHECK (route IN ('fast_path', 'adjudication')),
|
||||
reason_codes TEXT[] NOT NULL DEFAULT '{}',
|
||||
confidence_features JSONB NOT NULL DEFAULT '{}',
|
||||
decided_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_routing_pipeline
|
||||
ON v3_routing_decisions(pipeline_run_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_v3_routing_route
|
||||
ON v3_routing_decisions(route);
|
||||
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
-- 20.6: Idempotency and immutable-revision constraints
|
||||
-- ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
-- Pipeline runs: idempotency_key UNIQUE is already defined above in the table.
|
||||
-- Stage runs: unique per pipeline_run_id + stage to prevent duplicate stage execution.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_v3_stage_runs_idempotent
|
||||
ON v3_stage_runs(pipeline_run_id, stage);
|
||||
|
||||
-- Company signal candidates: unique per pipeline_run_id + company_id.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_v3_signal_candidates_idempotent
|
||||
ON v3_company_signal_candidates(pipeline_run_id, company_id);
|
||||
|
||||
-- Routing decisions: unique per pipeline_run_id (one routing decision per run).
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_v3_routing_idempotent
|
||||
ON v3_routing_decisions(pipeline_run_id);
|
||||
|
||||
-- Evidence spans: unique by document + checksum to avoid storing duplicates.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_v3_evidence_spans_idempotent
|
||||
ON v3_evidence_spans(document_id, checksum);
|
||||
|
||||
-- Immutable revision rule: pipeline_runs and stage_runs cannot be updated once completed.
|
||||
-- Enforced via trigger: reject updates to rows where status = 'completed' or 'failed'.
|
||||
CREATE OR REPLACE FUNCTION v3_immutable_completed_row()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
IF OLD.status IN ('completed', 'failed') THEN
|
||||
RAISE EXCEPTION 'Cannot modify a % record with status=%', TG_TABLE_NAME, OLD.status;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_v3_pipeline_runs_immutable ON v3_pipeline_runs;
|
||||
CREATE TRIGGER trg_v3_pipeline_runs_immutable
|
||||
BEFORE UPDATE ON v3_pipeline_runs
|
||||
FOR EACH ROW EXECUTE FUNCTION v3_immutable_completed_row();
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_v3_stage_runs_immutable ON v3_stage_runs;
|
||||
CREATE TRIGGER trg_v3_stage_runs_immutable
|
||||
BEFORE UPDATE ON v3_stage_runs
|
||||
FOR EACH ROW EXECUTE FUNCTION v3_immutable_completed_row();
|
||||
@@ -0,0 +1,74 @@
|
||||
-- Migration 042: Seed Inference Registry
|
||||
-- Populates initial endpoint profiles and model deployments for the
|
||||
-- existing Ollama and vLLM services.
|
||||
--
|
||||
-- Task 18.1: Create the current Ollama endpoint profile
|
||||
-- Task 18.2: Create the current vLLM OpenAI-compatible endpoint profile
|
||||
-- Task 18.3: Create model deployments matching actual runtime state
|
||||
--
|
||||
-- This is a DATA migration. The schema was created in 040_inference_registry.sql.
|
||||
-- Uses ON CONFLICT DO NOTHING for idempotency.
|
||||
|
||||
-- ─── 18.1: Ollama endpoint profile ───────────────────────────────────────────
|
||||
INSERT INTO inference_endpoints (id, name, protocol, base_url, auth_secret_ref, auth_scheme, default_headers, health_path, enabled)
|
||||
VALUES (
|
||||
'a0000000-0000-4000-8000-000000000001'::uuid,
|
||||
'stonks-ollama',
|
||||
'ollama_native',
|
||||
'http://ollama.ollama-service.svc.cluster.local:11434',
|
||||
NULL,
|
||||
'none',
|
||||
'{}',
|
||||
'/api/tags',
|
||||
TRUE
|
||||
)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
-- ─── 18.2: vLLM OpenAI-compatible endpoint profile ──────────────────────────
|
||||
INSERT INTO inference_endpoints (id, name, protocol, base_url, auth_secret_ref, auth_scheme, default_headers, health_path, enabled)
|
||||
VALUES (
|
||||
'a0000000-0000-4000-8000-000000000002'::uuid,
|
||||
'stonks-vllm',
|
||||
'openai_chat',
|
||||
'http://kube-vllm.stonks-oracle.svc.cluster.local:8000',
|
||||
NULL,
|
||||
'none',
|
||||
'{}',
|
||||
'/health',
|
||||
TRUE
|
||||
)
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
-- ─── 18.3: Model deployments ─────────────────────────────────────────────────
|
||||
|
||||
-- Ollama model deployment (qwen3.5:9b served via Ollama native protocol)
|
||||
INSERT INTO model_deployments (id, endpoint_id, served_model_name, display_name, capabilities, context_window, max_output_tokens, quantization, runtime_metadata, enabled)
|
||||
VALUES (
|
||||
'b0000000-0000-4000-8000-000000000001'::uuid,
|
||||
'a0000000-0000-4000-8000-000000000001'::uuid,
|
||||
'qwen3.5:9b',
|
||||
'Qwen 3.5 9B (Ollama)',
|
||||
'{"chat_completions": true, "json_schema": false, "json_object": true, "seed": false, "usage": false, "max_completion_tokens": false, "model_listing": true}',
|
||||
32768,
|
||||
32768,
|
||||
NULL,
|
||||
'{"source": "ollama_native", "notes": "Ollama-served model with native JSON mode"}',
|
||||
TRUE
|
||||
)
|
||||
ON CONFLICT (endpoint_id, served_model_name) DO NOTHING;
|
||||
|
||||
-- vLLM model deployment (AxionML/Qwen3.5-9B-NVFP4 on RTX 4070 Ti SUPER)
|
||||
INSERT INTO model_deployments (id, endpoint_id, served_model_name, display_name, capabilities, context_window, max_output_tokens, quantization, runtime_metadata, enabled)
|
||||
VALUES (
|
||||
'b0000000-0000-4000-8000-000000000002'::uuid,
|
||||
'a0000000-0000-4000-8000-000000000002'::uuid,
|
||||
'AxionML/Qwen3.5-9B-NVFP4',
|
||||
'Qwen 3.5 9B NVFP4 (vLLM)',
|
||||
'{"chat_completions": true, "json_schema": true, "json_object": true, "seed": true, "usage": true, "max_completion_tokens": true, "model_listing": true}',
|
||||
8192,
|
||||
2048,
|
||||
'NVFP4',
|
||||
'{"gpu": "RTX 4070 Ti SUPER", "gpu_memory_utilization": 0.80, "max_num_seqs": 8, "vllm_structured_outputs": true}',
|
||||
TRUE
|
||||
)
|
||||
ON CONFLICT (endpoint_id, served_model_name) DO NOTHING;
|
||||
Reference in New Issue
Block a user