Celes Renata
76 lines
4.1 KiB
JSON
76 lines
4.1 KiB
JSON
{
|
|
"dashboard_title": "Ingestion Throughput",
|
|
"description": "Operational dashboard for monitoring ingestion pipeline throughput, success rates, and item counts across source types.",
|
|
"slug": "ingestion-throughput",
|
|
"position_json": {
|
|
"HEADER_ID": {"id": "HEADER_ID", "type": "HEADER", "meta": {"text": "Ingestion Throughput"}},
|
|
"ROW-1": {
|
|
"type": "ROW",
|
|
"children": ["CHART-throughput-timeseries", "CHART-source-type-breakdown"]
|
|
},
|
|
"ROW-2": {
|
|
"type": "ROW",
|
|
"children": ["CHART-success-failure-rate", "CHART-items-fetched"]
|
|
},
|
|
"ROW-3": {
|
|
"type": "ROW",
|
|
"children": ["CHART-stale-sources", "CHART-active-companies"]
|
|
}
|
|
},
|
|
"metadata": {
|
|
"refresh_frequency": 300,
|
|
"default_filters": "{}",
|
|
"color_scheme": "supersetColors"
|
|
},
|
|
"charts": [
|
|
{
|
|
"slice_name": "Ingestion Runs Over Time",
|
|
"viz_type": "echarts_timeseries_bar",
|
|
"description": "Ingestion run counts bucketed by hour, stacked by source type",
|
|
"datasource_type": "query",
|
|
"query": "SELECT date_trunc('hour', ir.started_at) AS bucket, ir.source_type, COUNT(*) AS run_count, COUNT(*) FILTER (WHERE ir.status = 'completed') AS completed, COUNT(*) FILTER (WHERE ir.status = 'failed') AS failed FROM ingestion_runs ir WHERE ir.started_at >= NOW() - INTERVAL '24 hours' GROUP BY 1, 2 ORDER BY 1",
|
|
"params": {
|
|
"x_axis": "bucket",
|
|
"metrics": ["run_count"],
|
|
"groupby": ["source_type"],
|
|
"time_grain_sqla": "PT1H"
|
|
}
|
|
},
|
|
{
|
|
"slice_name": "Source Type Breakdown",
|
|
"viz_type": "pie",
|
|
"description": "Distribution of ingestion runs by source type in the last 24h",
|
|
"datasource_type": "query",
|
|
"query": "SELECT ir.source_type, COUNT(*) AS runs FROM ingestion_runs ir WHERE ir.started_at >= NOW() - INTERVAL '24 hours' GROUP BY ir.source_type ORDER BY runs DESC"
|
|
},
|
|
{
|
|
"slice_name": "Success vs Failure Rate",
|
|
"viz_type": "echarts_timeseries_line",
|
|
"description": "Hourly success and failure counts over time",
|
|
"datasource_type": "query",
|
|
"query": "SELECT date_trunc('hour', ir.started_at) AS bucket, COUNT(*) FILTER (WHERE ir.status = 'completed') AS completed, COUNT(*) FILTER (WHERE ir.status = 'failed') AS failed, ROUND(COUNT(*) FILTER (WHERE ir.status = 'completed')::numeric / NULLIF(COUNT(*), 0), 3) AS success_rate FROM ingestion_runs ir WHERE ir.started_at >= NOW() - INTERVAL '24 hours' GROUP BY 1 ORDER BY 1"
|
|
},
|
|
{
|
|
"slice_name": "Items Fetched Over Time",
|
|
"viz_type": "echarts_timeseries_bar",
|
|
"description": "Total items fetched and new items per hour",
|
|
"datasource_type": "query",
|
|
"query": "SELECT date_trunc('hour', ir.started_at) AS bucket, COALESCE(SUM(ir.items_fetched), 0) AS items_fetched, COALESCE(SUM(ir.items_new), 0) AS items_new FROM ingestion_runs ir WHERE ir.started_at >= NOW() - INTERVAL '24 hours' GROUP BY 1 ORDER BY 1"
|
|
},
|
|
{
|
|
"slice_name": "Stale Sources",
|
|
"viz_type": "table",
|
|
"description": "Sources with no successful run in the last 24 hours",
|
|
"datasource_type": "query",
|
|
"query": "SELECT c.ticker, s.source_type, s.source_name, MAX(ir.started_at) FILTER (WHERE ir.status = 'completed') AS last_success, COUNT(*) FILTER (WHERE ir.status = 'failed' AND ir.started_at >= NOW() - INTERVAL '24 hours') AS recent_failures FROM sources s JOIN companies c ON c.id = s.company_id LEFT JOIN ingestion_runs ir ON ir.source_id = s.id WHERE s.active = TRUE AND c.active = TRUE GROUP BY c.ticker, s.source_type, s.source_name HAVING MAX(ir.started_at) FILTER (WHERE ir.status = 'completed') < NOW() - INTERVAL '24 hours' OR MAX(ir.started_at) FILTER (WHERE ir.status = 'completed') IS NULL ORDER BY c.ticker"
|
|
},
|
|
{
|
|
"slice_name": "Active Companies Ingested",
|
|
"viz_type": "big_number_total",
|
|
"description": "Count of distinct companies with ingestion activity in the last 24h",
|
|
"datasource_type": "query",
|
|
"query": "SELECT COUNT(DISTINCT company_id) AS active_companies FROM ingestion_runs WHERE started_at >= NOW() - INTERVAL '24 hours'"
|
|
}
|
|
]
|
|
}
|