"""Schema validators for v3 annotations. Validates completeness, cross-references, evidence coverage, and offset integrity for annotated documents before they enter the Gold Corpus or production pipeline. Schema version: 1.0.0 """ from __future__ import annotations from dataclasses import dataclass, field from enum import Enum from typing import TYPE_CHECKING if TYPE_CHECKING: from services.intelligence_pipeline_v3.schemas.annotations import AnnotatedDocument class ValidationSeverity(str, Enum): ERROR = "error" WARNING = "warning" @dataclass class ValidationError: """A single validation issue found in an annotation.""" severity: ValidationSeverity field_path: str message: str entity_id: str | None = None @dataclass class ValidationResult: """Complete validation result for an annotated document.""" valid: bool errors: list[ValidationError] = field(default_factory=list) warnings: list[ValidationError] = field(default_factory=list) @property def error_count(self) -> int: return len(self.errors) @property def warning_count(self) -> int: return len(self.warnings) def validate_annotation(doc: "AnnotatedDocument") -> ValidationResult: """Validate completeness and cross-references of an annotated document. Checks performed: 1. All evidence_ids referenced by entities/events/relations/facts exist in evidence_spans 2. All entity IDs referenced by events/relations/effects exist in entities 3. Evidence span offsets are within the source_text bounds 4. Evidence span text matches the source_text at the given offsets 5. Sentiment probabilities are valid 6. No orphaned evidence spans (warning only) 7. Direct effects reference valid event and entity IDs 8. Inferred exposures reference valid event and entity IDs """ errors: list[ValidationError] = [] warnings: list[ValidationError] = [] # Build lookup indexes evidence_ids = {span.id for span in doc.evidence_spans} entity_ids = {entity.id for entity in doc.entities} event_ids = {event.id for event in doc.events} all_annotation_ids = evidence_ids | entity_ids | event_ids # Track which evidence spans are referenced referenced_evidence: set[str] = set() # 1. Validate evidence spans themselves for i, span in enumerate(doc.evidence_spans): path = f"evidence_spans[{i}]" # Offset bounds check if span.start_char >= len(doc.source_text): errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.start_char", message=f"start_char {span.start_char} exceeds source_text length {len(doc.source_text)}", entity_id=span.id, ) ) elif span.end_char > len(doc.source_text): errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.end_char", message=f"end_char {span.end_char} exceeds source_text length {len(doc.source_text)}", entity_id=span.id, ) ) else: # Text match check expected_text = doc.source_text[span.start_char : span.end_char] if expected_text != span.text: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.text", message=( f"Span text does not match source_text at offsets " f"[{span.start_char}:{span.end_char}]. " f"Expected: {expected_text!r}, got: {span.text!r}" ), entity_id=span.id, ) ) # 2. Validate entity evidence references for i, entity in enumerate(doc.entities): path = f"entities[{i}]" for eid in entity.evidence_ids: referenced_evidence.add(eid) if eid not in evidence_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.evidence_ids", message=f"References non-existent evidence span: {eid}", entity_id=entity.id, ) ) # 3. Validate event evidence and entity references for i, event in enumerate(doc.events): path = f"events[{i}]" for eid in event.evidence_ids: referenced_evidence.add(eid) if eid not in evidence_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.evidence_ids", message=f"References non-existent evidence span: {eid}", entity_id=event.id, ) ) for cid in event.primary_company_ids: if cid not in entity_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.primary_company_ids", message=f"References non-existent entity: {cid}", entity_id=event.id, ) ) # 4. Validate relation references for i, rel in enumerate(doc.relations): path = f"relations[{i}]" for eid in rel.evidence_ids: referenced_evidence.add(eid) if eid not in evidence_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.evidence_ids", message=f"References non-existent evidence span: {eid}", entity_id=rel.id, ) ) if rel.source_id not in all_annotation_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.source_id", message=f"source_id references non-existent annotation: {rel.source_id}", entity_id=rel.id, ) ) if rel.target_id not in entity_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.target_id", message=f"target_id references non-existent entity: {rel.target_id}", entity_id=rel.id, ) ) # 5. Validate numeric fact references for i, fact in enumerate(doc.numeric_facts): path = f"numeric_facts[{i}]" for eid in fact.evidence_ids: referenced_evidence.add(eid) if eid not in evidence_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.evidence_ids", message=f"References non-existent evidence span: {eid}", entity_id=fact.id, ) ) if fact.subject_entity_id and fact.subject_entity_id not in entity_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.subject_entity_id", message=f"References non-existent entity: {fact.subject_entity_id}", entity_id=fact.id, ) ) # 6. Validate sentiment references for i, sent in enumerate(doc.sentiments): path = f"sentiments[{i}]" for eid in sent.evidence_ids: referenced_evidence.add(eid) if eid not in evidence_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.evidence_ids", message=f"References non-existent evidence span: {eid}", entity_id=sent.id, ) ) if sent.company_entity_id not in entity_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.company_entity_id", message=f"References non-existent entity: {sent.company_entity_id}", entity_id=sent.id, ) ) # 7. Validate direct effects for i, effect in enumerate(doc.direct_effects): path = f"direct_effects[{i}]" for eid in effect.evidence_ids: referenced_evidence.add(eid) if eid not in evidence_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.evidence_ids", message=f"References non-existent evidence span: {eid}", ) ) if effect.event_id not in event_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.event_id", message=f"References non-existent event: {effect.event_id}", ) ) if effect.company_entity_id not in entity_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.company_entity_id", message=f"References non-existent entity: {effect.company_entity_id}", ) ) # 8. Validate inferred exposures for i, exposure in enumerate(doc.inferred_exposures): path = f"inferred_exposures[{i}]" for eid in exposure.evidence_ids: referenced_evidence.add(eid) if eid not in evidence_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.evidence_ids", message=f"References non-existent evidence span: {eid}", ) ) if exposure.event_id not in event_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.event_id", message=f"References non-existent event: {exposure.event_id}", ) ) if exposure.company_entity_id not in entity_ids: errors.append( ValidationError( severity=ValidationSeverity.ERROR, field_path=f"{path}.company_entity_id", message=f"References non-existent entity: {exposure.company_entity_id}", ) ) # 9. Check for orphaned evidence spans (warning) orphaned = evidence_ids - referenced_evidence for eid in orphaned: warnings.append( ValidationError( severity=ValidationSeverity.WARNING, field_path="evidence_spans", message=f"Evidence span {eid} is not referenced by any annotation.", entity_id=eid, ) ) # 10. Check minimum annotation completeness if not doc.entities: warnings.append( ValidationError( severity=ValidationSeverity.WARNING, field_path="entities", message="Document has no entity annotations.", ) ) if not doc.events: warnings.append( ValidationError( severity=ValidationSeverity.WARNING, field_path="events", message="Document has no event annotations.", ) ) is_valid = len(errors) == 0 return ValidationResult(valid=is_valid, errors=errors, warnings=warnings)