stonks-oracle/services/intelligence_pipeline_v3/orchestrator/queues.py

"""Queue definitions and routing for the v3 intelligence pipeline.

Provides fast-path, adjudication, persistence, and review queues with
backpressure and dead-letter support.
"""

from __future__ import annotations

import enum
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any
from uuid import UUID, uuid4


class QueueName(str, enum.Enum):
    """Named queues in the v3 pipeline topology."""

    INCOMING = "intelligence.v3.incoming"
    FAST_PATH = "intelligence.v3.fast"
    ADJUDICATION = "intelligence.v3.adjudication"
    PERSISTENCE = "intelligence.v3.persist"
    REVIEW = "intelligence.v3.review"
    DEAD_LETTER = "intelligence.v3.dead_letter"


@dataclass(frozen=True)
class QueueMessage:
    """Immutable message envelope for queue transport."""

    message_id: UUID
    queue: QueueName
    run_id: UUID
    document_id: str
    payload: dict[str, Any]
    enqueued_at: datetime
    attempt: int = 0
    idempotency_key: str = ""
    priority: int = 0

    @classmethod
    def create(
        cls,
        queue: QueueName,
        run_id: UUID,
        document_id: str,
        payload: dict[str, Any] | None = None,
        priority: int = 0,
        idempotency_key: str = "",
    ) -> QueueMessage:
        return cls(
            message_id=uuid4(),
            queue=queue,
            run_id=run_id,
            document_id=document_id,
            payload=payload or {},
            enqueued_at=datetime.now(timezone.utc),
            priority=priority,
            idempotency_key=idempotency_key,
        )


@dataclass
class QueueRouter:
    """In-memory queue router with backpressure and depth tracking.

    In production, this would be backed by Redis lists or a dedicated
    message broker. This implementation provides the queue routing logic
    and depth-based backpressure for testing and single-process usage.
    """

    max_depth: int = 1000
    _queues: dict[QueueName, list[QueueMessage]] = field(default_factory=dict)
    _processed_keys: set[str] = field(default_factory=set)

    def __post_init__(self) -> None:
        for q in QueueName:
            if q not in self._queues:
                self._queues[q] = []

    def enqueue(self, message: QueueMessage) -> bool:
        """Add a message to its designated queue.

        Returns False if backpressure is triggered (queue full) or
        if the idempotency key was already processed.
        """
        if message.idempotency_key and message.idempotency_key in self._processed_keys:
            return False  # Duplicate — idempotent reject

        queue = self._queues.setdefault(message.queue, [])
        if len(queue) >= self.max_depth:
            return False  # Backpressure

        queue.append(message)
        return True

    def dequeue(self, queue: QueueName) -> QueueMessage | None:
        """Pop the next message from a queue (FIFO). Returns None if empty."""
        q = self._queues.get(queue, [])
        if not q:
            return None
        msg = q.pop(0)
        if msg.idempotency_key:
            self._processed_keys.add(msg.idempotency_key)
        return msg

    def depth(self, queue: QueueName) -> int:
        """Current depth of the given queue."""
        return len(self._queues.get(queue, []))

    def is_saturated(self, queue: QueueName) -> bool:
        """Whether the queue has reached max depth (backpressure active)."""
        return self.depth(queue) >= self.max_depth

    def move_to_dead_letter(self, message: QueueMessage) -> QueueMessage:
        """Move a failed message to the dead-letter queue."""
        dlq_msg = QueueMessage(
            message_id=uuid4(),
            queue=QueueName.DEAD_LETTER,
            run_id=message.run_id,
            document_id=message.document_id,
            payload={**message.payload, "original_queue": message.queue.value},
            enqueued_at=datetime.now(timezone.utc),
            attempt=message.attempt,
            idempotency_key="",  # DLQ messages get new identity
            priority=message.priority,
        )
        self._queues.setdefault(QueueName.DEAD_LETTER, []).append(dlq_msg)
        return dlq_msg

    def total_depth(self) -> int:
        """Sum of all queue depths."""
        return sum(len(q) for q in self._queues.values())