"""Pydantic models for novelty and duplicate detection.""" from __future__ import annotations from pydantic import BaseModel, Field, field_validator class NearestMatch(BaseModel): """A single nearest-neighbor match from the novelty index.""" doc_id: str = Field(description="Document or event identifier of the match") similarity_score: float = Field( ge=0.0, le=1.0, description="Cosine similarity score (0=unrelated, 1=identical)" ) metadata: dict = Field(default_factory=dict, description="Optional metadata about the match") class NoveltyResult(BaseModel): """Result of novelty scoring for a document or event. Novelty is scored from 0 (exact duplicate) to 1 (completely novel). The formula version tracks which scoring algorithm produced the result. """ document_novelty: float = Field( ge=0.0, le=1.0, description="Document-level novelty (0=duplicate, 1=novel)" ) event_novelty: float = Field( ge=0.0, le=1.0, description="Event-level novelty (0=duplicate event, 1=novel event)" ) combined_novelty: float = Field( ge=0.0, le=1.0, description="Combined novelty score used downstream" ) nearest_matches: list[NearestMatch] = Field( default_factory=list, description="Nearest matches for explainability" ) formula_version: str = Field(description="Versioned identifier for the novelty formula used") is_exact_duplicate: bool = Field( default=False, description="Whether an exact content fingerprint match was found" ) is_near_duplicate: bool = Field( default=False, description="Whether a near-duplicate fingerprint match was found" ) @field_validator("nearest_matches") @classmethod def matches_sorted_descending(cls, v: list[NearestMatch]) -> list[NearestMatch]: """Ensure nearest matches are sorted by similarity descending.""" return sorted(v, key=lambda m: m.similarity_score, reverse=True) class FingerprintRecord(BaseModel): """Stored fingerprint for a document.""" doc_id: str = Field(description="Document identifier") exact_fingerprint: str = Field(description="SHA-256 of normalized text") simhash: int = Field(description="SimHash value for near-duplicate detection") metadata: dict = Field(default_factory=dict, description="Additional metadata") class EmbeddingRecord(BaseModel): """Stored embedding vector for a document or event.""" doc_id: str = Field(description="Document or event identifier") embedding: list[float] = Field(description="Dense embedding vector") record_type: str = Field(description="Type: 'document' or 'company_event'") metadata: dict = Field(default_factory=dict, description="Additional metadata") @field_validator("record_type") @classmethod def valid_record_type(cls, v: str) -> str: valid = {"document", "company_event"} if v not in valid: raise ValueError(f"record_type must be one of {valid}, got '{v}'") return v