feat: Intelligence Pipeline v3 — full implementation
Multi-stage evidence-grounded inference architecture replacing the monolithic 9B model extraction pipeline. CPU-first specialist services handle routine extraction while the 9B vLLM model is preserved for semantic adjudication of ambiguous cases. Key components: - Capability-aware inference gateway (OpenAI-compatible + Ollama) - Endpoint registry with DB migrations and REST API - Sentence-aware document segmenter (property tests) - Deterministic financial parsing with offset integrity - Symbol resolution with ambiguity detection - Specialist service (GLiNER2, dynamic batching, K8s deployment) - Company-specific sentiment (FinBERT, calibration) - Retrieval-based novelty and duplicate detection - Confidence calibration pipeline - Deterministic routing engine (property tests) - 9B adjudication layer with VRAM gating - Stock-specific impact model (features, labels, baseline, trained) - Pipeline orchestrator (state machine, queues, leases, feature flags) - Bounded parallelism (async workers, semaphore, load shedding) - Observability (tracing, metrics, alerts) - Compatibility adapter (v3→v2 golden mapping tests) - Shadow/canary promotion framework - Active learning and fine-tuning pipeline Test results: 1,161 tests pass, ruff lint clean. All 282 spec tasks completed.
This commit is contained in:
Celes Renata
@@ -0,0 +1,37 @@
|
||||
"""Retrieval-based novelty and duplicate detection.
|
||||
|
||||
Replaces model-generated novelty with deterministic fingerprinting,
|
||||
semantic embeddings, and similarity-based scoring against a recent
|
||||
history window.
|
||||
"""
|
||||
|
||||
from services.intelligence_pipeline_v3.novelty.embeddings import (
|
||||
EmbeddingBackend,
|
||||
MockEmbeddingBackend,
|
||||
SentenceTransformerBackend,
|
||||
cosine_similarity,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.novelty.fingerprints import (
|
||||
compute_exact_fingerprint,
|
||||
compute_simhash,
|
||||
hamming_distance,
|
||||
is_near_duplicate,
|
||||
)
|
||||
from services.intelligence_pipeline_v3.novelty.index import Match, NoveltyIndex
|
||||
from services.intelligence_pipeline_v3.novelty.models import NoveltyResult
|
||||
from services.intelligence_pipeline_v3.novelty.scorer import NoveltyScorer
|
||||
|
||||
__all__ = [
|
||||
"EmbeddingBackend",
|
||||
"Match",
|
||||
"MockEmbeddingBackend",
|
||||
"NoveltyIndex",
|
||||
"NoveltyResult",
|
||||
"NoveltyScorer",
|
||||
"SentenceTransformerBackend",
|
||||
"compute_exact_fingerprint",
|
||||
"compute_simhash",
|
||||
"cosine_similarity",
|
||||
"hamming_distance",
|
||||
"is_near_duplicate",
|
||||
]
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Replaceable compact embedding backend for novelty retrieval.
|
||||
|
||||
Provides:
|
||||
- EmbeddingBackend protocol for pluggable embedding models
|
||||
- MockEmbeddingBackend for deterministic testing
|
||||
- SentenceTransformerBackend stub for production (all-MiniLM-L6-v2)
|
||||
- cosine_similarity utility function
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import struct
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class EmbeddingBackend(Protocol):
|
||||
"""Protocol for embedding backends.
|
||||
|
||||
Implementations must produce fixed-dimension vectors for a batch of texts.
|
||||
The backend is designed to be replaceable: swap between mock, local model,
|
||||
and remote API backends without changing scoring logic.
|
||||
"""
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
"""Return the embedding dimension produced by this backend."""
|
||||
...
|
||||
|
||||
def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Embed a batch of texts into dense vectors.
|
||||
|
||||
Args:
|
||||
texts: List of text strings to embed.
|
||||
|
||||
Returns:
|
||||
List of embedding vectors, one per input text.
|
||||
Each vector has length == self.dimension.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class MockEmbeddingBackend:
|
||||
"""Deterministic hash-based embedding backend for testing.
|
||||
|
||||
Produces consistent embeddings using text hashing. Useful for
|
||||
unit tests and integration tests that need repeatable results
|
||||
without loading a real model.
|
||||
"""
|
||||
|
||||
def __init__(self, dimension: int = 384) -> None:
|
||||
self._dimension = dimension
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
return self._dimension
|
||||
|
||||
def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Generate deterministic embeddings from text hashes.
|
||||
|
||||
Uses SHA-256 expanded to fill the dimension. Normalizes to unit length
|
||||
for compatibility with cosine similarity.
|
||||
"""
|
||||
results = []
|
||||
for text in texts:
|
||||
raw = self._hash_to_vector(text)
|
||||
norm = math.sqrt(sum(x * x for x in raw))
|
||||
if norm > 0:
|
||||
normalized = [x / norm for x in raw]
|
||||
else:
|
||||
normalized = raw
|
||||
results.append(normalized)
|
||||
return results
|
||||
|
||||
def _hash_to_vector(self, text: str) -> list[float]:
|
||||
"""Expand text hash into a vector of the target dimension."""
|
||||
vector = []
|
||||
# Generate enough hash bytes to fill dimension
|
||||
chunk_idx = 0
|
||||
while len(vector) < self._dimension:
|
||||
data = f"{text}:{chunk_idx}".encode("utf-8")
|
||||
digest = hashlib.sha256(data).digest()
|
||||
# Convert 32 bytes to 8 floats (4 bytes each)
|
||||
for i in range(0, 32, 4):
|
||||
if len(vector) >= self._dimension:
|
||||
break
|
||||
# Unpack as float in [-1, 1] range
|
||||
raw_int = struct.unpack("<I", digest[i : i + 4])[0]
|
||||
value = (raw_int / (2**32 - 1)) * 2.0 - 1.0
|
||||
vector.append(value)
|
||||
chunk_idx += 1
|
||||
return vector[: self._dimension]
|
||||
|
||||
|
||||
class SentenceTransformerBackend:
|
||||
"""Production embedding backend using sentence-transformers.
|
||||
|
||||
Wraps all-MiniLM-L6-v2 (384-dimensional) for compact, fast embeddings.
|
||||
The model loads lazily on first call to avoid startup cost when not needed.
|
||||
|
||||
Note: Requires `sentence-transformers` package to be installed.
|
||||
This is a stub—actual model loading is deferred to production deployment.
|
||||
"""
|
||||
|
||||
MODEL_NAME = "all-MiniLM-L6-v2"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._model = None
|
||||
self._dimension = 384
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
return self._dimension
|
||||
|
||||
def embed(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Embed texts using the sentence-transformers model.
|
||||
|
||||
Lazily loads the model on first invocation.
|
||||
|
||||
Raises:
|
||||
ImportError: If sentence-transformers is not installed.
|
||||
"""
|
||||
if self._model is None:
|
||||
self._load_model()
|
||||
embeddings = self._model.encode(texts, normalize_embeddings=True)
|
||||
return [emb.tolist() for emb in embeddings]
|
||||
|
||||
def _load_model(self) -> None:
|
||||
"""Load the sentence-transformers model."""
|
||||
try:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"sentence-transformers package is required for SentenceTransformerBackend. "
|
||||
"Install with: pip install sentence-transformers"
|
||||
) from e
|
||||
self._model = SentenceTransformer(self.MODEL_NAME)
|
||||
|
||||
|
||||
def cosine_similarity(a: list[float], b: list[float]) -> float:
|
||||
"""Compute cosine similarity between two embedding vectors.
|
||||
|
||||
Args:
|
||||
a: First embedding vector.
|
||||
b: Second embedding vector (must be same dimension as a).
|
||||
|
||||
Returns:
|
||||
Cosine similarity in range [-1, 1]. Returns 0.0 for zero vectors.
|
||||
|
||||
Raises:
|
||||
ValueError: If vectors have different dimensions.
|
||||
"""
|
||||
if len(a) != len(b):
|
||||
raise ValueError(f"Vectors must have same dimension: {len(a)} != {len(b)}")
|
||||
|
||||
dot = sum(x * y for x, y in zip(a, b))
|
||||
norm_a = math.sqrt(sum(x * x for x in a))
|
||||
norm_b = math.sqrt(sum(x * x for x in b))
|
||||
|
||||
if norm_a == 0.0 or norm_b == 0.0:
|
||||
return 0.0
|
||||
|
||||
return dot / (norm_a * norm_b)
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Exact and near-duplicate fingerprinting for document deduplication.
|
||||
|
||||
Provides:
|
||||
- SHA-256 exact fingerprint on normalized text
|
||||
- SimHash for near-duplicate detection with configurable threshold
|
||||
- Hamming distance comparison between SimHash values
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import struct
|
||||
|
||||
|
||||
def _normalize_text(text: str) -> str:
|
||||
"""Normalize text for fingerprinting.
|
||||
|
||||
Lowercases, collapses whitespace, strips leading/trailing space.
|
||||
This ensures minor formatting differences don't defeat deduplication.
|
||||
"""
|
||||
text = text.lower()
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def compute_exact_fingerprint(text: str) -> str:
|
||||
"""Compute SHA-256 fingerprint of normalized text.
|
||||
|
||||
Args:
|
||||
text: Raw document text.
|
||||
|
||||
Returns:
|
||||
Hex-encoded SHA-256 digest of normalized text.
|
||||
"""
|
||||
normalized = _normalize_text(text)
|
||||
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _tokenize(text: str) -> list[str]:
|
||||
"""Split normalized text into tokens for SimHash computation."""
|
||||
normalized = _normalize_text(text)
|
||||
return normalized.split()
|
||||
|
||||
|
||||
def _hash_token(token: str) -> int:
|
||||
"""Hash a single token to a 64-bit integer using MD5 truncation."""
|
||||
digest = hashlib.md5(token.encode("utf-8")).digest() # noqa: S324
|
||||
return struct.unpack("<Q", digest[:8])[0]
|
||||
|
||||
|
||||
def compute_simhash(text: str) -> int:
|
||||
"""Compute 64-bit SimHash of text for near-duplicate detection.
|
||||
|
||||
SimHash produces locality-sensitive fingerprints: similar documents
|
||||
will have SimHash values with low Hamming distance.
|
||||
|
||||
Args:
|
||||
text: Raw document text.
|
||||
|
||||
Returns:
|
||||
64-bit SimHash integer value.
|
||||
"""
|
||||
tokens = _tokenize(text)
|
||||
if not tokens:
|
||||
return 0
|
||||
|
||||
# Accumulator for each bit position
|
||||
v = [0] * 64
|
||||
|
||||
for token in tokens:
|
||||
token_hash = _hash_token(token)
|
||||
for i in range(64):
|
||||
if token_hash & (1 << i):
|
||||
v[i] += 1
|
||||
else:
|
||||
v[i] -= 1
|
||||
|
||||
# Build final hash from accumulator signs
|
||||
fingerprint = 0
|
||||
for i in range(64):
|
||||
if v[i] > 0:
|
||||
fingerprint |= 1 << i
|
||||
|
||||
return fingerprint
|
||||
|
||||
|
||||
def hamming_distance(a: int, b: int) -> int:
|
||||
"""Compute Hamming distance between two 64-bit SimHash values.
|
||||
|
||||
Args:
|
||||
a: First SimHash value.
|
||||
b: Second SimHash value.
|
||||
|
||||
Returns:
|
||||
Number of differing bits (0-64).
|
||||
"""
|
||||
xor = a ^ b
|
||||
# Count set bits (Brian Kernighan's algorithm)
|
||||
distance = 0
|
||||
while xor:
|
||||
xor &= xor - 1
|
||||
distance += 1
|
||||
return distance
|
||||
|
||||
|
||||
def is_near_duplicate(fp1: int, fp2: int, threshold: int = 3) -> bool:
|
||||
"""Determine if two SimHash fingerprints indicate near-duplicate content.
|
||||
|
||||
Args:
|
||||
fp1: First SimHash value.
|
||||
fp2: Second SimHash value.
|
||||
threshold: Maximum Hamming distance to consider near-duplicate.
|
||||
Default of 3 is conservative for 64-bit SimHash.
|
||||
|
||||
Returns:
|
||||
True if the documents are near-duplicates.
|
||||
"""
|
||||
return hamming_distance(fp1, fp2) <= threshold
|
||||
@@ -0,0 +1,129 @@
|
||||
"""In-memory vector index for novelty retrieval.
|
||||
|
||||
Provides a simple but effective nearest-neighbor search over document
|
||||
and company-event embeddings. Designed to be replaceable with a
|
||||
production vector database (e.g., pgvector, FAISS) without changing
|
||||
the scoring interface.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from services.intelligence_pipeline_v3.novelty.embeddings import cosine_similarity
|
||||
|
||||
|
||||
@dataclass
|
||||
class Match:
|
||||
"""A nearest-neighbor match from the index."""
|
||||
|
||||
doc_id: str
|
||||
similarity_score: float
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _IndexEntry:
|
||||
"""Internal storage for an indexed document."""
|
||||
|
||||
doc_id: str
|
||||
embedding: list[float]
|
||||
metadata: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class NoveltyIndex:
|
||||
"""In-memory vector index for document and event embeddings.
|
||||
|
||||
Supports adding embeddings and searching for nearest neighbors
|
||||
by cosine similarity. Thread-safe for read-after-write but not
|
||||
for concurrent writes (use external locking if needed).
|
||||
|
||||
For production, replace with pgvector or FAISS. This implementation
|
||||
is suitable for testing, small corpora, and development.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._entries: list[_IndexEntry] = []
|
||||
self._id_set: set[str] = set()
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._entries)
|
||||
|
||||
def add(self, doc_id: str, embedding: list[float], metadata: dict | None = None) -> None:
|
||||
"""Add a document embedding to the index.
|
||||
|
||||
Args:
|
||||
doc_id: Unique document or event identifier.
|
||||
embedding: Dense embedding vector.
|
||||
metadata: Optional metadata (e.g., record_type, timestamp).
|
||||
|
||||
Note:
|
||||
If doc_id already exists, it is updated in-place.
|
||||
"""
|
||||
if metadata is None:
|
||||
metadata = {}
|
||||
|
||||
if doc_id in self._id_set:
|
||||
# Update existing entry
|
||||
for entry in self._entries:
|
||||
if entry.doc_id == doc_id:
|
||||
entry.embedding = embedding
|
||||
entry.metadata = metadata
|
||||
break
|
||||
else:
|
||||
self._entries.append(_IndexEntry(doc_id=doc_id, embedding=embedding, metadata=metadata))
|
||||
self._id_set.add(doc_id)
|
||||
|
||||
def search(self, embedding: list[float], k: int = 5) -> list[Match]:
|
||||
"""Find the k nearest neighbors to the query embedding.
|
||||
|
||||
Args:
|
||||
embedding: Query embedding vector.
|
||||
k: Maximum number of results to return.
|
||||
|
||||
Returns:
|
||||
List of Match objects sorted by similarity descending.
|
||||
Similarity scores are clamped to [0, 1] (negative cosine
|
||||
similarities are treated as 0 for novelty purposes).
|
||||
"""
|
||||
if not self._entries:
|
||||
return []
|
||||
|
||||
scored: list[tuple[float, _IndexEntry]] = []
|
||||
for entry in self._entries:
|
||||
sim = cosine_similarity(embedding, entry.embedding)
|
||||
# Clamp to [0, 1] for novelty scoring purposes
|
||||
sim = max(0.0, min(1.0, sim))
|
||||
scored.append((sim, entry))
|
||||
|
||||
# Sort descending by similarity
|
||||
scored.sort(key=lambda x: x[0], reverse=True)
|
||||
|
||||
results = []
|
||||
for sim, entry in scored[:k]:
|
||||
results.append(
|
||||
Match(doc_id=entry.doc_id, similarity_score=sim, metadata=entry.metadata)
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def remove(self, doc_id: str) -> bool:
|
||||
"""Remove a document from the index.
|
||||
|
||||
Args:
|
||||
doc_id: Document identifier to remove.
|
||||
|
||||
Returns:
|
||||
True if the document was found and removed.
|
||||
"""
|
||||
if doc_id not in self._id_set:
|
||||
return False
|
||||
|
||||
self._entries = [e for e in self._entries if e.doc_id != doc_id]
|
||||
self._id_set.discard(doc_id)
|
||||
return True
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Remove all entries from the index."""
|
||||
self._entries.clear()
|
||||
self._id_set.clear()
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Pydantic models for novelty and duplicate detection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class NearestMatch(BaseModel):
|
||||
"""A single nearest-neighbor match from the novelty index."""
|
||||
|
||||
doc_id: str = Field(description="Document or event identifier of the match")
|
||||
similarity_score: float = Field(
|
||||
ge=0.0, le=1.0, description="Cosine similarity score (0=unrelated, 1=identical)"
|
||||
)
|
||||
metadata: dict = Field(default_factory=dict, description="Optional metadata about the match")
|
||||
|
||||
|
||||
class NoveltyResult(BaseModel):
|
||||
"""Result of novelty scoring for a document or event.
|
||||
|
||||
Novelty is scored from 0 (exact duplicate) to 1 (completely novel).
|
||||
The formula version tracks which scoring algorithm produced the result.
|
||||
"""
|
||||
|
||||
document_novelty: float = Field(
|
||||
ge=0.0, le=1.0, description="Document-level novelty (0=duplicate, 1=novel)"
|
||||
)
|
||||
event_novelty: float = Field(
|
||||
ge=0.0, le=1.0, description="Event-level novelty (0=duplicate event, 1=novel event)"
|
||||
)
|
||||
combined_novelty: float = Field(
|
||||
ge=0.0, le=1.0, description="Combined novelty score used downstream"
|
||||
)
|
||||
nearest_matches: list[NearestMatch] = Field(
|
||||
default_factory=list, description="Nearest matches for explainability"
|
||||
)
|
||||
formula_version: str = Field(description="Versioned identifier for the novelty formula used")
|
||||
is_exact_duplicate: bool = Field(
|
||||
default=False, description="Whether an exact content fingerprint match was found"
|
||||
)
|
||||
is_near_duplicate: bool = Field(
|
||||
default=False, description="Whether a near-duplicate fingerprint match was found"
|
||||
)
|
||||
|
||||
@field_validator("nearest_matches")
|
||||
@classmethod
|
||||
def matches_sorted_descending(cls, v: list[NearestMatch]) -> list[NearestMatch]:
|
||||
"""Ensure nearest matches are sorted by similarity descending."""
|
||||
return sorted(v, key=lambda m: m.similarity_score, reverse=True)
|
||||
|
||||
|
||||
class FingerprintRecord(BaseModel):
|
||||
"""Stored fingerprint for a document."""
|
||||
|
||||
doc_id: str = Field(description="Document identifier")
|
||||
exact_fingerprint: str = Field(description="SHA-256 of normalized text")
|
||||
simhash: int = Field(description="SimHash value for near-duplicate detection")
|
||||
metadata: dict = Field(default_factory=dict, description="Additional metadata")
|
||||
|
||||
|
||||
class EmbeddingRecord(BaseModel):
|
||||
"""Stored embedding vector for a document or event."""
|
||||
|
||||
doc_id: str = Field(description="Document or event identifier")
|
||||
embedding: list[float] = Field(description="Dense embedding vector")
|
||||
record_type: str = Field(description="Type: 'document' or 'company_event'")
|
||||
metadata: dict = Field(default_factory=dict, description="Additional metadata")
|
||||
|
||||
@field_validator("record_type")
|
||||
@classmethod
|
||||
def valid_record_type(cls, v: str) -> str:
|
||||
valid = {"document", "company_event"}
|
||||
if v not in valid:
|
||||
raise ValueError(f"record_type must be one of {valid}, got '{v}'")
|
||||
return v
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Novelty scoring formula implementation.
|
||||
|
||||
Combines fingerprint-based duplicate detection with embedding-based
|
||||
semantic novelty to produce a versioned, deterministic novelty score.
|
||||
|
||||
Formula v1: novelty = 1 - max_similarity (clamped to [0, 1])
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from services.intelligence_pipeline_v3.novelty.index import Match, NoveltyIndex
|
||||
from services.intelligence_pipeline_v3.novelty.models import NearestMatch, NoveltyResult
|
||||
|
||||
# Current formula version — bump when the scoring algorithm changes
|
||||
FORMULA_VERSION = "v1.0"
|
||||
|
||||
|
||||
class NoveltyScorer:
|
||||
"""Computes novelty scores from embedding similarity and fingerprints.
|
||||
|
||||
The scorer queries the novelty index for nearest neighbors and applies
|
||||
a versioned formula to produce document-level, event-level, and combined
|
||||
novelty scores.
|
||||
|
||||
Formula v1.0:
|
||||
document_novelty = 1 - max_document_similarity
|
||||
event_novelty = 1 - max_event_similarity
|
||||
combined_novelty = min(document_novelty, event_novelty)
|
||||
All values clamped to [0, 1].
|
||||
"""
|
||||
|
||||
def __init__(self, k: int = 5, formula_version: str = FORMULA_VERSION) -> None:
|
||||
"""Initialize the scorer.
|
||||
|
||||
Args:
|
||||
k: Number of nearest neighbors to retrieve for scoring.
|
||||
formula_version: Version string for the scoring formula.
|
||||
"""
|
||||
self.k = k
|
||||
self.formula_version = formula_version
|
||||
|
||||
def compute_novelty(
|
||||
self,
|
||||
document_embedding: list[float],
|
||||
event_embedding: list[float],
|
||||
index: NoveltyIndex,
|
||||
is_exact_duplicate: bool = False,
|
||||
is_near_duplicate: bool = False,
|
||||
) -> NoveltyResult:
|
||||
"""Compute novelty for a document and its primary event.
|
||||
|
||||
Args:
|
||||
document_embedding: Embedding of the document content.
|
||||
event_embedding: Embedding of the canonical company-event.
|
||||
index: NoveltyIndex containing recent document/event embeddings.
|
||||
is_exact_duplicate: Whether an exact fingerprint match exists.
|
||||
is_near_duplicate: Whether a near-duplicate fingerprint match exists.
|
||||
|
||||
Returns:
|
||||
NoveltyResult with document, event, and combined novelty scores.
|
||||
"""
|
||||
# If exact duplicate, novelty is zero
|
||||
if is_exact_duplicate:
|
||||
doc_matches = index.search(document_embedding, k=self.k)
|
||||
return NoveltyResult(
|
||||
document_novelty=0.0,
|
||||
event_novelty=0.0,
|
||||
combined_novelty=0.0,
|
||||
nearest_matches=self._to_nearest_matches(doc_matches),
|
||||
formula_version=self.formula_version,
|
||||
is_exact_duplicate=True,
|
||||
is_near_duplicate=True,
|
||||
)
|
||||
|
||||
# Search for similar documents
|
||||
doc_matches = index.search(document_embedding, k=self.k)
|
||||
event_matches = index.search(event_embedding, k=self.k)
|
||||
|
||||
# Compute novelty from max similarity
|
||||
document_novelty = self._compute_novelty_from_matches(doc_matches)
|
||||
event_novelty = self._compute_novelty_from_matches(event_matches)
|
||||
|
||||
# If near-duplicate detected via fingerprint, cap document novelty
|
||||
if is_near_duplicate:
|
||||
document_novelty = min(document_novelty, 0.2)
|
||||
|
||||
# Combined novelty: conservative (take the minimum)
|
||||
combined_novelty = min(document_novelty, event_novelty)
|
||||
|
||||
# Merge matches for explainability, deduplicated by doc_id
|
||||
all_matches = self._merge_matches(doc_matches, event_matches)
|
||||
|
||||
return NoveltyResult(
|
||||
document_novelty=document_novelty,
|
||||
event_novelty=event_novelty,
|
||||
combined_novelty=combined_novelty,
|
||||
nearest_matches=all_matches,
|
||||
formula_version=self.formula_version,
|
||||
is_exact_duplicate=is_exact_duplicate,
|
||||
is_near_duplicate=is_near_duplicate,
|
||||
)
|
||||
|
||||
def _compute_novelty_from_matches(self, matches: list[Match]) -> float:
|
||||
"""Apply the v1 formula: novelty = 1 - max_similarity."""
|
||||
if not matches:
|
||||
return 1.0 # No history = fully novel
|
||||
|
||||
max_sim = max(m.similarity_score for m in matches)
|
||||
novelty = 1.0 - max_sim
|
||||
return max(0.0, min(1.0, novelty))
|
||||
|
||||
def _to_nearest_matches(self, matches: list[Match]) -> list[NearestMatch]:
|
||||
"""Convert internal Match objects to NearestMatch models."""
|
||||
return [
|
||||
NearestMatch(
|
||||
doc_id=m.doc_id,
|
||||
similarity_score=m.similarity_score,
|
||||
metadata=m.metadata,
|
||||
)
|
||||
for m in matches
|
||||
]
|
||||
|
||||
def _merge_matches(
|
||||
self, doc_matches: list[Match], event_matches: list[Match]
|
||||
) -> list[NearestMatch]:
|
||||
"""Merge document and event matches, keeping highest similarity per doc_id."""
|
||||
seen: dict[str, NearestMatch] = {}
|
||||
|
||||
for m in doc_matches + event_matches:
|
||||
if m.doc_id not in seen or m.similarity_score > seen[m.doc_id].similarity_score:
|
||||
seen[m.doc_id] = NearestMatch(
|
||||
doc_id=m.doc_id,
|
||||
similarity_score=m.similarity_score,
|
||||
metadata=m.metadata,
|
||||
)
|
||||
|
||||
return sorted(seen.values(), key=lambda x: x.similarity_score, reverse=True)
|
||||
Reference in New Issue
Block a user