Celes Renata
85 lines
2.7 KiB
Python
85 lines
2.7 KiB
Python
"""Base adapter interface for all external API integrations.
|
|
|
|
All ingestion adapters follow the same contract:
|
|
1. Fetch external payloads for a given ticker/source config.
|
|
2. Return a structured result with raw bytes, parsed items, and metadata.
|
|
3. The ingestion worker handles MinIO upload, PostgreSQL metadata, and downstream job emission.
|
|
|
|
Requirements: 2.1, 2.2, 2.3, 2.4, 2.5, 3.1, 3.2, 3.3, 3.4
|
|
"""
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
|
|
@dataclass
|
|
class AdapterResult:
|
|
"""Result of a single adapter fetch operation."""
|
|
|
|
source_type: str
|
|
ticker: str
|
|
items: list[dict[str, Any]]
|
|
raw_payload: bytes
|
|
content_hash: str
|
|
fetched_at: datetime
|
|
error: str | None = None
|
|
# HTTP metadata for observability
|
|
http_status: int | None = None
|
|
response_time_ms: float | None = None
|
|
# Additional metadata the adapter wants to pass downstream
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
@property
|
|
def ok(self) -> bool:
|
|
"""True if the fetch succeeded without error."""
|
|
return self.error is None and len(self.items) > 0
|
|
|
|
@property
|
|
def item_count(self) -> int:
|
|
return len(self.items)
|
|
|
|
|
|
class BaseAdapter(ABC):
|
|
"""Interface for all ingestion adapters.
|
|
|
|
Subclasses implement fetch() for their specific API and source_type()
|
|
to identify the adapter class. The ingestion worker orchestrates
|
|
persistence and downstream job emission.
|
|
"""
|
|
|
|
@abstractmethod
|
|
async def fetch(self, ticker: str, config: dict[str, Any]) -> AdapterResult:
|
|
"""Fetch data for a given ticker using source config.
|
|
|
|
Args:
|
|
ticker: The company ticker symbol.
|
|
config: Source-specific configuration from the sources table.
|
|
|
|
Returns:
|
|
AdapterResult with raw payload, parsed items, and metadata.
|
|
"""
|
|
...
|
|
|
|
@abstractmethod
|
|
def source_type(self) -> str:
|
|
"""Return the source type identifier for this adapter (e.g. 'market_api')."""
|
|
...
|
|
|
|
def bucket_name(self) -> str:
|
|
"""Return the MinIO bucket name for raw artifact storage.
|
|
|
|
Override in subclasses if the bucket differs from the default pattern.
|
|
"""
|
|
return f"stonks-raw-{self.source_type().replace('_api', '').replace('_', '-')}"
|
|
|
|
def artifact_path(self, ticker: str, document_id: str, now: datetime) -> str:
|
|
"""Build the MinIO object path for a raw artifact.
|
|
|
|
Pattern: /{source_type}/{ticker}/{yyyy}/{mm}/{dd}/{document_id}/raw.json
|
|
"""
|
|
return (
|
|
f"{self.source_type()}/{ticker}/"
|
|
f"{now.strftime('%Y/%m/%d')}/{document_id}/raw.json"
|
|
)
|