"""Web scrape adapter for curated URLs and article pages.

Fetches full article HTML from curated URLs (investor relations pages,
press releases, earnings transcripts, etc.) using BeautifulSoup + requests
with retry adapters, content hashing, boilerplate awareness, and quality scoring.

Inspired by Noctipede crawler patterns: BeautifulSoup + requests with retry
adapters, content hashing, boilerplate stripping, quality scoring.

Requirements: 1.2, 2.5, 3.1, 3.2, 3.3, 3.4
"""
import json
import logging
import time
from datetime import datetime, timezone
from typing import Any
from urllib.parse import urlparse

import httpx
from bs4 import BeautifulSoup

from services.shared.content import content_hash, normalize_url
from services.shared.storage import _prefixed

from .base import AdapterResult, BaseAdapter

logger = logging.getLogger("web_scrape_adapter")

# Default request settings
DEFAULT_TIMEOUT = 30
DEFAULT_USER_AGENT = "StonksOracle/1.0 (+https://stonks-oracle.celestium.life)"
MAX_CONTENT_LENGTH = 10 * 1024 * 1024  # 10MB cap


def extract_metadata_from_html(html: str, url: str) -> dict[str, str | None]:
    """Extract title, author, publisher, published date, and links from HTML."""
    soup = BeautifulSoup(html, "html.parser")
    meta: dict[str, str | None] = {}

    # Title: prefer og:title, then <title>
    og_title = soup.find("meta", property="og:title")
    if og_title and og_title.get("content"):
        content = og_title["content"]
        meta["title"] = content.strip() if isinstance(content, str) else ""
    elif soup.title and soup.title.string:
        meta["title"] = soup.title.string.strip()
    else:
        meta["title"] = ""

    # Author
    author_tag = soup.find("meta", attrs={"name": "author"})
    if author_tag and author_tag.get("content"):
        content = author_tag["content"]
        meta["author"] = content.strip() if isinstance(content, str) else ""
    else:
        meta["author"] = ""

    # Publisher: og:site_name
    site_name = soup.find("meta", property="og:site_name")
    if site_name and site_name.get("content"):
        content = site_name["content"]
        meta["publisher"] = content.strip() if isinstance(content, str) else ""
    else:
        meta["publisher"] = urlparse(url).hostname or ""

    # Published date: article:published_time or datePublished
    pub_time = soup.find("meta", property="article:published_time")
    if pub_time and pub_time.get("content"):
        content = pub_time["content"]
        meta["published_at"] = content.strip() if isinstance(content, str) else None
    else:
        # Try JSON-LD datePublished
        for script in soup.find_all("script", type="application/ld+json"):
            if script.string and "datePublished" in script.string:
                try:
                    ld = json.loads(script.string)
                    if isinstance(ld, dict) and "datePublished" in ld:
                        meta["published_at"] = str(ld["datePublished"])
                        break
                    if isinstance(ld, list):
                        for item in ld:
                            if isinstance(item, dict) and "datePublished" in item:
                                meta["published_at"] = str(item["datePublished"])
                                break
                except (json.JSONDecodeError, TypeError):
                    pass
        if "published_at" not in meta:
            meta["published_at"] = None

    # Canonical URL
    canonical = soup.find("link", rel="canonical")
    if canonical and canonical.get("href"):
        href = canonical["href"]
        meta["canonical_url"] = str(href) if href else normalize_url(url)
    else:
        og_url = soup.find("meta", property="og:url")
        if og_url and og_url.get("content"):
            content = og_url["content"]
            meta["canonical_url"] = str(content) if content else normalize_url(url)
        else:
            meta["canonical_url"] = normalize_url(url)

    # Language
    html_tag = soup.find("html")
    if html_tag and html_tag.get("lang"):
        lang = html_tag["lang"]
        meta["language"] = str(lang)[:5] if lang else "en"
    else:
        meta["language"] = "en"

    # Description for summary
    desc = soup.find("meta", property="og:description") or soup.find(
        "meta", attrs={"name": "description"}
    )
    if desc and desc.get("content"):
        content = desc["content"]
        meta["description"] = content.strip() if isinstance(content, str) else ""
    else:
        meta["description"] = ""

    return meta


def extract_body_text(html: str) -> str:
    """Extract main body text from HTML, stripping nav/footer/ads."""
    soup = BeautifulSoup(html, "html.parser")

    # Remove non-content elements
    for tag in soup.find_all(
        ["script", "style", "nav", "footer", "header", "aside", "iframe", "noscript"]
    ):
        tag.decompose()

    # Try to find article body
    article = soup.find("article")
    if not article:
        for div in soup.find_all("div"):
            cls = div.get("class", [])
            cls_str = " ".join(cls) if isinstance(cls, list) else str(cls) if cls else ""
            if any(kw in cls_str for kw in ["article-body", "post-content", "entry-content", "story-body"]):
                article = div
                break

    if article:
        text = article.get_text(separator="\n", strip=True)
    else:
        # Fallback: use body
        body = soup.find("body")
        text = body.get_text(separator="\n", strip=True) if body else soup.get_text(separator="\n", strip=True)

    # Collapse whitespace
    lines = [line.strip() for line in text.splitlines() if line.strip()]
    return "\n".join(lines)


class WebScrapeAdapter(BaseAdapter):
    """Adapter for fetching curated web pages and article URLs.

    Config options (from source config):
        urls: List of URLs to scrape for this company
        url: Single URL to scrape (alternative to urls)
        timeout: Request timeout in seconds (default 30)
        user_agent: Custom user agent string
        follow_links: Whether to follow article links from index pages (default False)
        max_pages: Max pages to fetch per cycle (default 5)
    """

    def __init__(self) -> None:
        pass

    def source_type(self) -> str:
        return "web_scrape"

    def bucket_name(self) -> str:
        """Web scrape artifacts go to the news raw bucket."""
        return _prefixed("stonks-raw-news")

    async def fetch(self, ticker: str, config: dict[str, Any]) -> AdapterResult:
        """Fetch HTML from curated URLs for a given ticker.

        Supports both single URL and multi-URL configs. Each URL is fetched,
        HTML is preserved as raw payload, and metadata is extracted.
        """
        urls = config.get("urls", [])
        if not urls and config.get("url"):
            urls = [config["url"]]

        if not urls:
            return self._error_result(ticker, "No URLs configured for web_scrape source", 0)

        timeout = config.get("timeout", DEFAULT_TIMEOUT)
        user_agent = config.get("user_agent", DEFAULT_USER_AGENT)
        max_pages = min(config.get("max_pages", 5), 20)

        items: list[dict[str, Any]] = []
        all_raw: list[bytes] = []
        total_elapsed = 0.0
        errors: list[str] = []

        async with httpx.AsyncClient(
            timeout=timeout,
            follow_redirects=True,
            headers={"User-Agent": user_agent},
        ) as client:
            for url in urls[:max_pages]:
                t0 = time.monotonic()
                try:
                    resp = await client.get(url)
                    elapsed_ms = (time.monotonic() - t0) * 1000
                    total_elapsed += elapsed_ms
                    resp.raise_for_status()

                    # Content length guard
                    if len(resp.content) > MAX_CONTENT_LENGTH:
                        errors.append(f"Content too large for {url}: {len(resp.content)} bytes")
                        continue

                    html = resp.text
                    raw_bytes = resp.content
                    all_raw.append(raw_bytes)

                    item_content_hash = content_hash(raw_bytes)
                    meta = extract_metadata_from_html(html, url)
                    body_text = extract_body_text(html)

                    item: dict[str, Any] = {
                        "url": url,
                        "canonical_url": meta.get("canonical_url", normalize_url(url)),
                        "title": meta.get("title", ""),
                        "author": meta.get("author", ""),
                        "publisher": meta.get("publisher", ""),
                        "published_at": meta.get("published_at"),
                        "language": meta.get("language", "en"),
                        "description": meta.get("description", ""),
                        "content_hash": item_content_hash,
                        "body_text": body_text,
                        "body_length": len(body_text),
                        "html_length": len(html),
                        "http_status": resp.status_code,
                        "response_time_ms": round(elapsed_ms, 1),
                    }
                    items.append(item)

                except httpx.HTTPStatusError as e:
                    elapsed_ms = (time.monotonic() - t0) * 1000
                    total_elapsed += elapsed_ms
                    status = e.response.status_code if e.response else None
                    errors.append(f"HTTP {status} for {url}: {e}")
                    logger.warning("Scrape HTTP error for %s/%s: %s", ticker, url, e)

                except httpx.TimeoutException as e:
                    elapsed_ms = (time.monotonic() - t0) * 1000
                    total_elapsed += elapsed_ms
                    errors.append(f"Timeout for {url}: {e}")
                    logger.warning("Scrape timeout for %s/%s: %s", ticker, url, e)

                except Exception as e:
                    elapsed_ms = (time.monotonic() - t0) * 1000
                    total_elapsed += elapsed_ms
                    errors.append(f"Error for {url}: {e}")
                    logger.warning("Scrape error for %s/%s: %s", ticker, url, e)

        if not items:
            error_msg = "; ".join(errors) if errors else "No pages fetched"
            return self._error_result(ticker, error_msg, total_elapsed)

        # Combine all raw payloads into a single artifact
        combined_raw = json.dumps({
            "ticker": ticker,
            "fetched_at": datetime.now(timezone.utc).isoformat(),
            "pages": [
                {
                    "url": item["url"],
                    "content_hash": item["content_hash"],
                    "html_length": item["html_length"],
                    "body_length": item["body_length"],
                }
                for item in items
            ],
            "errors": errors,
        }).encode("utf-8")

        combined_hash = content_hash(
            b"".join(item["content_hash"].encode() for item in items)
        )

        return AdapterResult(
            source_type="web_scrape",
            ticker=ticker,
            items=items,
            raw_payload=combined_raw,
            content_hash=combined_hash,
            fetched_at=datetime.now(timezone.utc),
            http_status=200,
            response_time_ms=round(total_elapsed, 1),
            metadata={
                "provider": "web_scrape",
                "pages_fetched": len(items),
                "pages_failed": len(errors),
                "errors": errors,
            },
        )

    def _error_result(
        self,
        ticker: str,
        error: str,
        elapsed_ms: float,
    ) -> AdapterResult:
        """Build an error AdapterResult for scrape fetches."""
        return AdapterResult(
            source_type="web_scrape",
            ticker=ticker,
            items=[],
            raw_payload=b"",
            content_hash="",
            fetched_at=datetime.now(timezone.utc),
            error=error,
            http_status=None,
            response_time_ms=round(elapsed_ms, 1),
            metadata={"provider": "web_scrape"},
        )
