feat: competitive intelligence & historical pattern matching layer

services/api/app.py

@@ -28,7 +28,7 @@ from starlette.middleware.base import BaseHTTPMiddleware
 from starlette.responses import Response
 
 from services.extractor.metrics import get_model_performance_summary
-from services.shared.audit import get_entity_audit_trail, get_order_audit_trail
+from services.shared.audit import get_entity_audit_trail, get_order_audit_trail, record_audit_event
 from services.shared.config import load_config
 from services.shared.db import get_pg_pool
 from services.shared.logging import new_trace_id, set_trace_context, setup_logging
@@ -376,6 +376,24 @@ async def list_trends(
         ):
             d[jsonb_field] = _parse_jsonb(d.get(jsonb_field))
         results.append(d)
+
+    # Include projection data for each trend (Requirement 12.10)
+    if results:
+        trend_ids = [r["id"] for r in rows]
+        proj_rows = await pool.fetch(
+            """SELECT DISTINCT ON (trend_window_id)
+                      trend_window_id, projected_direction, projected_strength,
+                      projected_confidence, projection_horizon,
+                      macro_contribution_pct, diverges_from_current
+               FROM trend_projections
+               WHERE trend_window_id = ANY($1::uuid[])
+               ORDER BY trend_window_id, computed_at DESC""",
+            trend_ids,
+        )
+        proj_map = {str(p["trend_window_id"]): _row_to_dict(p) for p in proj_rows}
+        for d in results:
+            d["projection"] = proj_map.get(d["id"])
+
     return results
 
 
@@ -1687,3 +1705,581 @@ async def delete_saved_query(query_id: str):
     if result == "DELETE 0":
         raise HTTPException(404, "Query not found")
     return {"status": "deleted"}
+
+
+# ---------------------------------------------------------------------------
+# Admin: Macro Signal Layer Toggle  (Requirement 11.1, 11.5, 11.7)
+# ---------------------------------------------------------------------------
+
+class MacroToggleBody(BaseModel):
+    enabled: bool
+    operator: str = "operator"
+
+
+@app.get("/api/admin/macro/status")
+async def get_macro_status():
+    """Return the current macro signal layer enabled/disabled state.
+
+    Reads from the active risk_configs row's JSONB config field.
+    Requirements: 11.1, 11.5
+    """
+    row = await pool.fetchrow(
+        """SELECT config->>'macro_enabled' AS macro_enabled
+           FROM risk_configs
+           WHERE active = TRUE
+           ORDER BY updated_at DESC
+           LIMIT 1""",
+    )
+    if row is None or row["macro_enabled"] is None:
+        return {"macro_enabled": True, "source": "default"}
+    return {
+        "macro_enabled": row["macro_enabled"].lower() == "true",
+        "source": "risk_configs",
+    }
+
+
+@app.put("/api/admin/macro/toggle")
+async def toggle_macro_layer(body: MacroToggleBody):
+    """Toggle the macro signal layer on or off.
+
+    Persists the new state into the active risk_configs row's JSONB config
+    and records an audit event with previous state, new state, and operator.
+
+    The toggle state is read from PostgreSQL at the start of each aggregation
+    cycle (no caching), so changes take effect on the next cycle.
+
+    Requirements: 11.1, 11.5, 11.7
+    """
+    # Read current state
+    current_row = await pool.fetchrow(
+        """SELECT id, config->>'macro_enabled' AS macro_enabled
+           FROM risk_configs
+           WHERE active = TRUE
+           ORDER BY updated_at DESC
+           LIMIT 1""",
+    )
+
+    if current_row is None:
+        # No active config exists — create one
+        new_config = json.dumps({"macro_enabled": str(body.enabled).lower()})
+        current_row = await pool.fetchrow(
+            """INSERT INTO risk_configs (name, trading_mode, config, active)
+               VALUES ('default', 'paper', $1::jsonb, TRUE)
+               RETURNING id, config->>'macro_enabled' AS macro_enabled""",
+            new_config,
+        )
+        previous_enabled = True  # default was enabled
+    else:
+        prev_val = current_row["macro_enabled"]
+        previous_enabled = prev_val.lower() == "true" if prev_val else True
+
+    config_id = str(current_row["id"])
+
+    # Update the config JSONB to set macro_enabled
+    await pool.execute(
+        """UPDATE risk_configs
+           SET config = config || $2::jsonb, updated_at = NOW()
+           WHERE id = $1""",
+        current_row["id"],
+        json.dumps({"macro_enabled": str(body.enabled).lower()}),
+    )
+
+    # Record audit event (Requirement 11.7)
+    await record_audit_event(
+        pool,
+        event_type="macro.layer_toggled",
+        entity_type="risk_config",
+        entity_id=config_id,
+        data={
+            "previous_enabled": previous_enabled,
+            "new_enabled": body.enabled,
+        },
+        actor=body.operator,
+    )
+
+    return {
+        "macro_enabled": body.enabled,
+        "previous_enabled": previous_enabled,
+        "toggled_by": body.operator,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Macro Events and Impacts  (Requirement 8.1, 8.2, 12.10)
+# ---------------------------------------------------------------------------
+
+@app.get("/api/macro/events")
+async def list_macro_events(
+    severity: Optional[str] = None,
+    region: Optional[str] = None,
+    sector: Optional[str] = None,
+    since: Optional[str] = None,
+    until: Optional[str] = None,
+    limit: int = Query(default=50, le=200),
+    offset: int = 0,
+):
+    """List recent global events with filtering by severity, region, sector, date range.
+
+    Requirements: 8.1
+    """
+    conditions: list[str] = []
+    params: list[Any] = []
+    idx = 1
+
+    if severity:
+        conditions.append(f"ge.severity = ${idx}")
+        params.append(severity)
+        idx += 1
+    if region:
+        conditions.append(f"${idx} = ANY(ge.affected_regions)")
+        params.append(region)
+        idx += 1
+    if sector:
+        conditions.append(f"${idx} = ANY(ge.affected_sectors)")
+        params.append(sector)
+        idx += 1
+    if since:
+        conditions.append(f"ge.created_at >= ${idx}::timestamptz")
+        params.append(since)
+        idx += 1
+    if until:
+        conditions.append(f"ge.created_at <= ${idx}::timestamptz")
+        params.append(until)
+        idx += 1
+
+    where = ("WHERE " + " AND ".join(conditions)) if conditions else ""
+
+    rows = await pool.fetch(
+        f"""SELECT ge.id, ge.event_types, ge.severity, ge.affected_regions,
+                   ge.affected_sectors, ge.affected_commodities, ge.summary,
+                   ge.key_facts, ge.estimated_duration, ge.confidence,
+                   ge.source_document_id, ge.created_at
+            FROM global_events ge
+            {where}
+            ORDER BY ge.created_at DESC
+            LIMIT ${idx} OFFSET ${idx + 1}""",
+        *params, limit, offset,
+    )
+    results = []
+    for r in rows:
+        d = _row_to_dict(r)
+        d["key_facts"] = _parse_jsonb(d.get("key_facts"))
+        results.append(d)
+    return results
+
+
+@app.get("/api/macro/events/{event_id}")
+async def get_macro_event(event_id: str):
+    """Event detail with list of affected companies and their macro impact scores.
+
+    Requirements: 8.2
+    """
+    row = await pool.fetchrow(
+        """SELECT id, event_types, severity, affected_regions, affected_sectors,
+                  affected_commodities, summary, key_facts, estimated_duration,
+                  confidence, source_document_id, model_provider, model_name,
+                  prompt_version, schema_version, created_at
+           FROM global_events WHERE id = $1""",
+        event_id,
+    )
+    if not row:
+        raise HTTPException(404, "Global event not found")
+
+    result = _row_to_dict(row)
+    result["key_facts"] = _parse_jsonb(result.get("key_facts"))
+
+    # Affected companies with macro impact scores
+    impacts = await pool.fetch(
+        """SELECT mir.id, mir.company_id, mir.ticker, mir.macro_impact_score,
+                  mir.impact_direction, mir.contributing_factors, mir.confidence,
+                  mir.computed_at, c.legal_name, c.sector
+           FROM macro_impact_records mir
+           JOIN companies c ON c.id = mir.company_id
+           WHERE mir.event_id = $1
+           ORDER BY mir.macro_impact_score DESC""",
+        event_id,
+    )
+    impact_list = []
+    for imp in impacts:
+        imp_dict = _row_to_dict(imp)
+        imp_dict["contributing_factors"] = _parse_jsonb(imp_dict.get("contributing_factors"))
+        impact_list.append(imp_dict)
+    result["affected_companies"] = impact_list
+
+    return result
+
+
+@app.get("/api/macro/impacts/{ticker}")
+async def get_macro_impacts_for_ticker(
+    ticker: str,
+    since: Optional[str] = None,
+    limit: int = Query(default=50, le=200),
+    offset: int = 0,
+):
+    """Macro impacts for a specific company.
+
+    Requirements: 8.2
+    """
+    conditions = ["mir.ticker = $1"]
+    params: list[Any] = [ticker.upper()]
+    idx = 2
+
+    if since:
+        conditions.append(f"mir.computed_at >= ${idx}::timestamptz")
+        params.append(since)
+        idx += 1
+
+    where = " AND ".join(conditions)
+
+    rows = await pool.fetch(
+        f"""SELECT mir.id, mir.event_id, mir.company_id, mir.ticker,
+                   mir.macro_impact_score, mir.impact_direction,
+                   mir.contributing_factors, mir.confidence, mir.computed_at,
+                   ge.summary AS event_summary, ge.severity AS event_severity,
+                   ge.event_types AS event_types, ge.affected_regions
+            FROM macro_impact_records mir
+            JOIN global_events ge ON ge.id = mir.event_id
+            WHERE {where}
+            ORDER BY mir.computed_at DESC
+            LIMIT ${idx} OFFSET ${idx + 1}""",
+        *params, limit, offset,
+    )
+    results = []
+    for r in rows:
+        d = _row_to_dict(r)
+        d["contributing_factors"] = _parse_jsonb(d.get("contributing_factors"))
+        results.append(d)
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Trend Projections  (Requirement 12.10)
+# ---------------------------------------------------------------------------
+
+@app.get("/api/trends/{trend_id}/projection")
+async def get_trend_projection(trend_id: str):
+    """Trend projection for a specific trend window.
+
+    Requirements: 12.10
+    """
+    # Verify trend exists
+    trend_row = await pool.fetchrow(
+        "SELECT id FROM trend_windows WHERE id = $1", trend_id,
+    )
+    if not trend_row:
+        raise HTTPException(404, "Trend not found")
+
+    row = await pool.fetchrow(
+        """SELECT id, trend_window_id, projected_direction, projected_strength,
+                  projected_confidence, projection_horizon, driving_factors,
+                  macro_contribution_pct, diverges_from_current, computed_at
+           FROM trend_projections WHERE trend_window_id = $1
+           ORDER BY computed_at DESC LIMIT 1""",
+        trend_id,
+    )
+    if not row:
+        return {"trend_window_id": trend_id, "projection": None}
+
+    d = _row_to_dict(row)
+    d["driving_factors"] = _parse_jsonb(d.get("driving_factors"))
+    return d
+
+
+# ---------------------------------------------------------------------------
+# Competitive Layer Toggle  (Requirements 6.1, 6.2, 6.3, 6.4, 6.5, 6.7)
+# ---------------------------------------------------------------------------
+
+class CompetitiveToggleBody(BaseModel):
+    enabled: bool
+    operator: str = "operator"
+
+
+@app.get("/api/admin/competitive/status")
+async def get_competitive_status():
+    """Return the current competitive signal layer enabled/disabled state.
+
+    Reads from the active risk_configs row's JSONB config field.
+    Requirements: 6.1, 6.5
+    """
+    row = await pool.fetchrow(
+        """SELECT config->>'competitive_enabled' AS competitive_enabled
+           FROM risk_configs
+           WHERE active = TRUE
+           ORDER BY updated_at DESC
+           LIMIT 1""",
+    )
+    if row is None or row["competitive_enabled"] is None:
+        return {"competitive_enabled": True, "source": "default"}
+    return {
+        "competitive_enabled": row["competitive_enabled"].lower() == "true",
+        "source": "risk_configs",
+    }
+
+
+@app.put("/api/admin/competitive/toggle")
+async def toggle_competitive_layer(body: CompetitiveToggleBody):
+    """Toggle the competitive signal layer on or off.
+
+    Persists the new state into the active risk_configs row's JSONB config
+    and records an audit event with previous state, new state, and operator.
+
+    Toggle state is read from PostgreSQL at the start of each aggregation
+    cycle (no caching), so changes take effect on the next cycle.
+
+    When disabled, pattern mining remains queryable via API but signal
+    propagation is skipped during aggregation.  When re-enabled, the engine
+    resumes computing signals using latest historical data including
+    intelligence ingested while disabled.
+
+    Requirements: 6.1, 6.2, 6.3, 6.4, 6.5, 6.7
+    """
+    # Read current state
+    current_row = await pool.fetchrow(
+        """SELECT id, config->>'competitive_enabled' AS competitive_enabled
+           FROM risk_configs
+           WHERE active = TRUE
+           ORDER BY updated_at DESC
+           LIMIT 1""",
+    )
+
+    if current_row is None:
+        # No active config exists — create one
+        new_config = json.dumps({"competitive_enabled": str(body.enabled).lower()})
+        current_row = await pool.fetchrow(
+            """INSERT INTO risk_configs (name, trading_mode, config, active)
+               VALUES ('default', 'paper', $1::jsonb, TRUE)
+               RETURNING id, config->>'competitive_enabled' AS competitive_enabled""",
+            new_config,
+        )
+        previous_enabled = True  # default was enabled
+    else:
+        prev_val = current_row["competitive_enabled"]
+        previous_enabled = prev_val.lower() == "true" if prev_val else True
+
+    config_id = str(current_row["id"])
+
+    # Update the config JSONB to set competitive_enabled
+    await pool.execute(
+        """UPDATE risk_configs
+           SET config = config || $2::jsonb, updated_at = NOW()
+           WHERE id = $1""",
+        current_row["id"],
+        json.dumps({"competitive_enabled": str(body.enabled).lower()}),
+    )
+
+    # Record audit event (Requirement 6.7)
+    await record_audit_event(
+        pool,
+        event_type="competitive.layer_toggled",
+        entity_type="risk_config",
+        entity_id=config_id,
+        data={
+            "previous_enabled": previous_enabled,
+            "new_enabled": body.enabled,
+        },
+        actor=body.operator,
+    )
+
+    return {
+        "competitive_enabled": body.enabled,
+        "previous_enabled": previous_enabled,
+        "toggled_by": body.operator,
+    }
+
+
+# ---------------------------------------------------------------------------
+# Historical Pattern & Competitive Signal Query Endpoints
+# (Requirements 10.1, 10.2, 10.3, 10.4, 11.4, 11.6)
+# ---------------------------------------------------------------------------
+
+from dataclasses import asdict
+
+from services.aggregation.pattern_matcher import (
+    find_cross_company_patterns,
+    find_self_patterns,
+)
+from services.shared.schemas import MAJOR_DECISION_CATALYSTS
+
+
+def _pattern_to_dict(p) -> dict[str, Any]:
+    """Convert a HistoricalPattern dataclass to a JSON-safe dict."""
+    d = asdict(p)
+    for key, val in d.items():
+        if isinstance(val, datetime):
+            d[key] = val.isoformat()
+    return d
+
+
+@app.get("/api/patterns/{ticker}")
+async def get_patterns_for_ticker(
+    ticker: str,
+    catalyst_type: Optional[str] = None,
+    time_horizon: Optional[str] = None,
+):
+    """Historical patterns for a company.
+
+    Filterable by catalyst_type and time_horizon.
+    Returns sample_count, outcome distribution, pattern_confidence,
+    and date range for each pattern.
+
+    Requirements: 10.1, 10.3
+    """
+    horizons = [time_horizon] if time_horizon else None
+
+    if catalyst_type:
+        patterns = await find_self_patterns(pool, ticker, catalyst_type, horizons=horizons)
+    else:
+        # Query across all catalyst types present in the company's history
+        rows = await pool.fetch(
+            """SELECT DISTINCT di.catalyst_type
+               FROM document_impact_records dir
+               JOIN document_intelligence di ON di.document_id = dir.document_id
+               JOIN documents d ON d.id = dir.document_id
+               WHERE dir.ticker = $1
+                 AND di.validation_status = 'valid'
+                 AND d.status != 'rejected'
+                 AND di.catalyst_type IS NOT NULL""",
+            ticker,
+        )
+        patterns = []
+        for row in rows:
+            ct = row["catalyst_type"]
+            patterns.extend(await find_self_patterns(pool, ticker, ct, horizons=horizons))
+
+    return {
+        "ticker": ticker,
+        "patterns": [_pattern_to_dict(p) for p in patterns],
+        "count": len(patterns),
+    }
+
+
+@app.get("/api/patterns/{ticker}/competitors")
+async def get_competitor_patterns(
+    ticker: str,
+    catalyst_type: Optional[str] = None,
+    time_horizon: Optional[str] = None,
+):
+    """Cross-company patterns showing how this company's catalysts affected competitors.
+
+    Requirements: 10.2, 10.3
+    """
+    horizons = [time_horizon] if time_horizon else None
+
+    # Find active competitors for this ticker
+    comp_rows = await pool.fetch(
+        """SELECT DISTINCT
+               CASE WHEN ca.ticker = $1 THEN cb.ticker ELSE ca.ticker END AS competitor_ticker
+           FROM competitor_relationships cr
+           JOIN companies ca ON ca.id = cr.company_a_id
+           JOIN companies cb ON cb.id = cr.company_b_id
+           WHERE cr.active = TRUE
+             AND (ca.ticker = $1 OR cb.ticker = $1)""",
+        ticker,
+    )
+
+    # Determine catalyst types to query
+    if catalyst_type:
+        catalyst_types = [catalyst_type]
+    else:
+        ct_rows = await pool.fetch(
+            """SELECT DISTINCT di.catalyst_type
+               FROM document_impact_records dir
+               JOIN document_intelligence di ON di.document_id = dir.document_id
+               JOIN documents d ON d.id = dir.document_id
+               WHERE dir.ticker = $1
+                 AND di.validation_status = 'valid'
+                 AND d.status != 'rejected'
+                 AND di.catalyst_type IS NOT NULL""",
+            ticker,
+        )
+        catalyst_types = [r["catalyst_type"] for r in ct_rows]
+
+    patterns = []
+    for comp_row in comp_rows:
+        comp_ticker = comp_row["competitor_ticker"]
+        for ct in catalyst_types:
+            cross = await find_cross_company_patterns(
+                pool, ticker, comp_ticker, ct, horizons=horizons,
+            )
+            patterns.extend(cross)
+
+    return {
+        "ticker": ticker,
+        "cross_company_patterns": [_pattern_to_dict(p) for p in patterns],
+        "count": len(patterns),
+    }
+
+
+@app.get("/api/patterns/{ticker}/competitive-signals")
+async def get_competitive_signals(ticker: str):
+    """Recent competitive signals targeting this company.
+
+    Requirements: 10.4
+    """
+    rows = await pool.fetch(
+        """SELECT id, source_document_id, source_ticker, target_ticker,
+                  catalyst_type, pattern_confidence, signal_direction,
+                  signal_strength, relationship_strength, computed_at
+           FROM competitive_signal_records
+           WHERE target_ticker = $1
+           ORDER BY computed_at DESC
+           LIMIT 100""",
+        ticker,
+    )
+    return {
+        "ticker": ticker,
+        "competitive_signals": [_row_to_dict(r) for r in rows],
+        "count": len(rows),
+    }
+
+
+@app.get("/api/patterns/{ticker}/decisions")
+async def get_decision_history(
+    ticker: str,
+    time_horizon: Optional[str] = None,
+):
+    """Major corporate decision history with trend outcomes and pattern statistics.
+
+    Queries document_impact_records filtered by MAJOR_DECISION_CATALYSTS,
+    joined with trend_windows for outcome data.
+
+    Requirements: 11.4, 11.6
+    """
+    major_types = list(MAJOR_DECISION_CATALYSTS)
+    horizons = [time_horizon] if time_horizon else None
+
+    # Fetch major decision records for this ticker
+    rows = await pool.fetch(
+        """SELECT dir.id, dir.document_id, dir.ticker,
+                  di.catalyst_type, di.summary,
+                  dir.impact_score, dir.created_at,
+                  d.published_at
+           FROM document_impact_records dir
+           JOIN document_intelligence di ON di.document_id = dir.document_id
+           JOIN documents d ON d.id = dir.document_id
+           WHERE dir.ticker = $1
+             AND di.validation_status = 'valid'
+             AND d.status != 'rejected'
+             AND di.catalyst_type = ANY($2)
+           ORDER BY dir.created_at DESC
+           LIMIT 50""",
+        ticker,
+        major_types,
+    )
+
+    decisions = []
+    for row in rows:
+        decision = _row_to_dict(row)
+
+        # Fetch pattern statistics for this catalyst type
+        ct = row["catalyst_type"]
+        patterns = await find_self_patterns(pool, ticker, ct, horizons=horizons)
+        decision["pattern_statistics"] = [_pattern_to_dict(p) for p in patterns]
+
+        decisions.append(decision)
+
+    return {
+        "ticker": ticker,
+        "decisions": decisions,
+        "count": len(decisions),
+    }