From 7ad9a93decb50b2713cdeed8d08df4a19f43d98d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Aug 2026 09:52:06 -0600 Subject: [PATCH 1/5] refactor(scheduler): extract REST DTOs into ballista-history and factor out dto_build Move the scheduler's REST response types into a new leaf crate, `ballista-history`, and pull the graph-to-DTO construction out of the axum handlers into a pure `api::dto_build` module. Behavior preserving: the same DTOs are produced from the same state, so live REST responses are byte-identical. The existing handler tests cover this, and the helper unit tests move alongside the functions they test. This is the first step toward a history server that replays completed jobs and serves the same `/api/*` responses without a live scheduler. Splitting the DTOs into a serde-only crate lets that server build the identical wire types without depending on the scheduler's live execution graph, and moving construction out of the handlers means it can run against state that did not come from a handler request. `JobResponse::job_id` becomes a `String` rather than `ballista_core::JobId` so the new crate stays serde-only. `JobId` is `#[serde(transparent)]` over `String`, so the JSON is unchanged. --- Cargo.lock | 9 + Cargo.toml | 1 + ballista/history/Cargo.toml | 30 ++ ballista/history/src/dto.rs | 114 ++++ ballista/history/src/lib.rs | 24 + ballista/scheduler/Cargo.toml | 4 +- ballista/scheduler/src/api/dto_build.rs | 581 ++++++++++++++++++++ ballista/scheduler/src/api/handlers.rs | 683 +----------------------- ballista/scheduler/src/api/mod.rs | 2 + 9 files changed, 771 insertions(+), 677 deletions(-) create mode 100644 ballista/history/Cargo.toml create mode 100644 ballista/history/src/dto.rs create mode 100644 ballista/history/src/lib.rs create mode 100644 ballista/scheduler/src/api/dto_build.rs diff --git a/Cargo.lock b/Cargo.lock index dac81a2616..498f742dfe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1200,6 +1200,13 @@ dependencies = [ "uuid", ] +[[package]] +name = "ballista-history" +version = "54.0.0" +dependencies = [ + "serde", +] + [[package]] name = "ballista-scheduler" version = "54.0.0" @@ -1208,6 +1215,7 @@ dependencies = [ "async-trait", "axum", "ballista-core", + "ballista-history", "clap 4.6.3", "dashmap", "datafusion", @@ -1230,6 +1238,7 @@ dependencies = [ "regex", "rstest", "serde", + "serde_json", "tokio", "tokio-stream", "tonic", diff --git a/Cargo.toml b/Cargo.toml index 5a6b966dd8..50ffac073d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "ballista/client", "ballista/core", "ballista/executor", + "ballista/history", "ballista/scheduler", "benchmarks", "chaos-testing", diff --git a/ballista/history/Cargo.toml b/ballista/history/Cargo.toml new file mode 100644 index 0000000000..a4e56d709c --- /dev/null +++ b/ballista/history/Cargo.toml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "ballista-history" +description = "Shared REST response types for the Ballista scheduler and history server" +license = "Apache-2.0" +version = "54.0.0" +homepage = "https://datafusion.apache.org/ballista/" +repository = "https://github.com/apache/datafusion-ballista" +authors = ["Apache DataFusion "] +edition = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +serde = { workspace = true, features = ["derive"] } diff --git a/ballista/history/src/dto.rs b/ballista/history/src/dto.rs new file mode 100644 index 0000000000..74bbb7fe89 --- /dev/null +++ b/ballista/history/src/dto.rs @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! REST response DTOs shared by the live scheduler API handlers and the history +//! server, so both serialize byte-identical JSON. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobResponse { + /// A `String` rather than `ballista_core::JobId` so this crate stays a + /// serde-only leaf. `JobId` is `#[serde(transparent)]` over `String`, so + /// the serialized JSON is unchanged. + pub job_id: String, + pub job_name: String, + pub job_status: String, + pub status: String, + pub num_stages: usize, + pub completed_stages: usize, + pub percent_complete: u8, + pub start_time: u64, + pub end_time: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub logical_plan: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub physical_plan: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stage_plan: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum TaskStatus { + Running, + Successful, + Failed { reason: String, error: String }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TaskSummary { + /// task id + pub id: usize, + /// Task status + pub status: TaskStatus, + /// Global partition ids covered by this task. For a single-partition + /// task this is a one-element list — JSON-compatible with the old + /// scalar `partition_id` for callers that read `partition_id[0]`, and + /// honestly plural for multi-partition tasks. + pub partition_id: Vec, + /// Scheduler schedule time + pub scheduled_time: u64, + /// Scheduler launch time (ms since epoch) + pub launch_time: u64, + /// The time the Executor start to run the task (ms since epoch) + pub start_exec_time: u64, + /// The time the Executor finish the task (ms since epoch) + pub end_exec_time: u64, + /// total execution time (ms) + pub exec_duration: u64, + /// Scheduler side finish time (ms since epoch) + pub finish_time: u64, + /// Number of input rows + pub input_rows: usize, + /// Number of output rows + pub output_rows: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Percentiles { + pub min: u64, + pub p25: u64, + pub median: u64, + pub p75: u64, + pub max: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryStageSummary { + pub stage_id: String, + pub stage_status: String, + pub input_rows: usize, + pub output_rows: usize, + pub elapsed_compute: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stage_plan: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub task_duration_percentiles: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub task_input_percentiles: Option, + pub tasks: Vec>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueryStagesResponse { + pub stages: Vec, +} + +/// Session config as flat key/value pairs (from `SessionConfig::to_props()`), +/// sorted for stable output. +pub type JobConfig = BTreeMap; diff --git a/ballista/history/src/lib.rs b/ballista/history/src/lib.rs new file mode 100644 index 0000000000..affddf8a53 --- /dev/null +++ b/ballista/history/src/lib.rs @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared types for Ballista's scheduler REST API. +//! +//! These response types live outside `ballista-scheduler` so that a future +//! history server can serialize byte-identical JSON from stored state without +//! depending on the scheduler's live execution graph. + +pub mod dto; diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml index b8b7127733..17ce991bd2 100644 --- a/ballista/scheduler/Cargo.toml +++ b/ballista/scheduler/Cargo.toml @@ -41,7 +41,7 @@ disable-stage-plan-cache = [] graphviz-support = ["dep:graphviz-rust"] keda-scaler = ["dep:tonic-prost-build", "dep:tonic-prost"] prometheus-metrics = ["prometheus", "once_cell"] -rest-api = [] +rest-api = ["dep:ballista-history"] spark-compat = ["ballista-core/spark-compat"] substrait = ["dep:datafusion-substrait"] @@ -50,6 +50,7 @@ arrow-flight = { workspace = true } async-trait = { workspace = true } axum = "0.8.9" ballista-core = { path = "../core", version = "54.0.0" } +ballista-history = { path = "../history", version = "54.0.0", optional = true } clap = { workspace = true, optional = true } dashmap = { workspace = true } datafusion = { workspace = true } @@ -88,6 +89,7 @@ path = "tests/tpch_plan_stability/main.rs" datafusion-functions-aggregate-common = { workspace = true } regex = "1" rstest = { workspace = true } +serde_json = "1" [build-dependencies] tonic-prost-build = { workspace = true, optional = true } diff --git a/ballista/scheduler/src/api/dto_build.rs b/ballista/scheduler/src/api/dto_build.rs new file mode 100644 index 0000000000..ce1ec45315 --- /dev/null +++ b/ballista/scheduler/src/api/dto_build.rs @@ -0,0 +1,581 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Builds REST response DTOs from scheduler execution state. +//! +//! These functions are the only place the scheduler's `ExecutionGraph` / +//! `JobOverview` shapes are translated into the wire types in +//! [`ballista_history::dto`]. Keeping the translation here (rather than inline +//! in the axum handlers) means the same DTOs can be produced from state that +//! did not come from a live handler request. +//! +//! Everything in this module is pure: state in, DTO out, no I/O. + +use crate::api::handlers::{JobQueryParams, PlanFormat}; +use crate::display::format_stage_metrics; +use crate::state::execution_graph::{ExecutionGraphBox, ExecutionStage}; +use crate::state::execution_stage::TaskInfo; +use crate::state::task_manager::JobOverview; +use ballista_core::serde::protobuf::failed_task::FailedReason::{ + ExecutionError, ExecutorLost, FetchPartitionError, IoError, ResultLost, TaskKilled, +}; +use ballista_core::serde::protobuf::job_status::Status; +use ballista_core::serde::protobuf::{FailedTask, task_status}; +use ballista_core::utils::get_current_time; +use ballista_history::dto::{ + JobResponse, Percentiles, QueryStageSummary, QueryStagesResponse, TaskStatus, + TaskSummary, +}; +use datafusion::physical_plan::display::DisplayableExecutionPlan; +use datafusion::physical_plan::displayable; +use datafusion::physical_plan::metrics::{MetricsSet, Time}; +use std::time::Duration; + +/// Build the `JobResponse` list entry served by `GET /api/jobs`. +/// +/// Plan fields are always `None` here: the job list is built from +/// [`JobOverview`] summaries, which do not carry plans. +pub fn job_overview_to_response(job: &JobOverview) -> JobResponse { + let (plain_status, job_status) = format_job_status( + &job.status.status, + job_elapsed_ms(job.start_time, job.end_time), + ); + + // calculate progress based on completed stages for now, but we could use completed + // tasks in the future to make this more accurate + let percent_complete = if job.num_stages == 0 { + 0 + } else { + ((job.completed_stages as f32 / job.num_stages as f32) * 100_f32) as u8 + }; + + JobResponse { + job_id: job.job_id.to_string(), + job_name: job.job_name.to_owned(), + job_status, + status: plain_status, + start_time: job.start_time, + end_time: job.end_time, + num_stages: job.num_stages, + completed_stages: job.completed_stages, + percent_complete, + logical_plan: None, + physical_plan: None, + stage_plan: None, + } +} + +/// Build the `JobResponse` served by `GET /api/job/{job_id}`, including the +/// rendered logical, physical, and stage plans. +pub fn graph_to_job_response( + graph: &ExecutionGraphBox, + query: &JobQueryParams, +) -> JobResponse { + let stage_plan = format!("{graph:?}"); + let job = graph.as_ref(); + + let (plain_status, job_status) = format_job_status( + &job.status().status, + job_elapsed_ms(job.start_time(), job.end_time()), + ); + + let num_stages = job.stage_count(); + let completed_stages = job.completed_stages(); + let percent_complete = + ((completed_stages as f32 / num_stages as f32) * 100_f32) as u8; + + let plan_format = query.plan_format.clone().unwrap_or_default(); + let physical_plan = match plan_format { + PlanFormat::Default | PlanFormat::Metrics => { + DisplayableExecutionPlan::new(job.physical_plan().as_ref()) + .indent(false) + .to_string() + } + PlanFormat::Tree => displayable(job.physical_plan().as_ref()) + .tree_render() + .to_string(), + }; + + JobResponse { + job_id: job.job_id().to_string(), + job_name: job.job_name().to_owned(), + job_status, + status: plain_status, + start_time: job.start_time(), + end_time: job.end_time(), + num_stages, + completed_stages, + percent_complete, + logical_plan: job.logical_plan().map(str::to_owned), + physical_plan: Some(physical_plan), + stage_plan: Some(stage_plan), + } +} + +/// Build the per-stage summaries served by `GET /api/job/{job_id}/stages`. +pub fn graph_to_query_stages( + graph: &ExecutionGraphBox, + query: &JobQueryParams, +) -> QueryStagesResponse { + let plan_format = query.plan_format.clone().unwrap_or_default(); + + let stages = graph + .as_ref() + .stages() + .iter() + .map(|(id, stage)| { + let mut summary = QueryStageSummary { + stage_id: id.to_string(), + stage_status: stage.variant_name().to_string(), + input_rows: 0, + output_rows: 0, + elapsed_compute: None, + tasks: vec![], + task_duration_percentiles: None, + task_input_percentiles: None, + stage_plan: None, + }; + + match stage { + ExecutionStage::Running(running_stage) => { + let metrics = running_stage.stage_metrics.as_deref().unwrap_or(&[]); + summary.stage_plan = Some(render_stage_plan( + running_stage.plan.as_ref(), + metrics, + &plan_format, + )); + summary.input_rows = get_combined_count(metrics, "input_rows"); + summary.output_rows = get_combined_count(metrics, "output_rows"); + summary.elapsed_compute = get_running_stage_time( + &running_stage.task_infos, + get_current_time(), + ); + summary.tasks = task_summaries(&running_stage.task_infos, metrics); + } + ExecutionStage::Successful(completed_stage) => { + let metrics = completed_stage.stage_metrics.as_slice(); + summary.stage_plan = Some(render_stage_plan( + completed_stage.plan.as_ref(), + metrics, + &plan_format, + )); + summary.input_rows = get_combined_count(metrics, "input_rows"); + summary.output_rows = get_combined_count(metrics, "output_rows"); + summary.elapsed_compute = + get_finished_stage_time(&completed_stage.task_infos); + summary.tasks = task_summaries(&completed_stage.task_infos, metrics); + } + ExecutionStage::Failed(failed_stage) => { + let metrics = failed_stage.stage_metrics.as_deref().unwrap_or(&[]); + summary.stage_plan = Some(render_stage_plan( + failed_stage.plan.as_ref(), + metrics, + &plan_format, + )); + summary.input_rows = get_combined_count(metrics, "input_rows"); + summary.output_rows = get_combined_count(metrics, "output_rows"); + summary.elapsed_compute = + get_finished_stage_time(&failed_stage.task_infos); + summary.tasks = task_summaries(&failed_stage.task_infos, metrics); + } + _ => {} + } + + summary.task_duration_percentiles = task_duration_percentiles(&summary.tasks); + summary.task_input_percentiles = task_input_percentiles(&summary.tasks); + summary + }) + .collect(); + + QueryStagesResponse { stages } +} + +/// Render one stage's plan in the requested format. `Metrics` overlays the +/// stage's aggregated metrics onto the plan; the other formats ignore them. +fn render_stage_plan( + plan: &dyn datafusion::physical_plan::ExecutionPlan, + metrics: &[MetricsSet], + plan_format: &PlanFormat, +) -> String { + match plan_format { + PlanFormat::Default => displayable(plan).indent(false).to_string(), + PlanFormat::Tree => displayable(plan).tree_render().to_string(), + PlanFormat::Metrics => format_stage_metrics(plan, metrics), + } +} + +/// Build one [`TaskSummary`] per task in a stage. Row counts are summed over +/// the global partitions each task owns. +/// +/// The `Option` wrapper on each entry is part of the wire format and is always +/// `Some` here; it predates multi-partition tasks, when a stage's task list was +/// indexed by partition and could be sparse. +fn task_summaries( + task_infos: &[TaskInfo], + metrics: &[MetricsSet], +) -> Vec> { + task_infos + .iter() + .map(|info| { + let (input_rows, output_rows) = + get_partition_counts(metrics, &info.global_input_partition_ids); + + let start_exec_time = info.start_exec_time as u64; + let end_exec_time = info.end_exec_time as u64; + + Some(TaskSummary { + id: info.task_id, + partition_id: info + .global_input_partition_ids + .iter() + .map(|&p| p as u32) + .collect(), + scheduled_time: info.scheduled_time as u64, + launch_time: info.launch_time as u64, + start_exec_time, + end_exec_time, + exec_duration: end_exec_time.saturating_sub(start_exec_time), + finish_time: info.finish_time as u64, + input_rows, + output_rows, + status: task_status_to_dto(&info.task_status), + }) + }) + .collect() +} + +/// Map a protobuf task status onto the wire enum. +/// +/// A free function rather than a `From` impl: both types are foreign to this +/// crate now that [`TaskStatus`] lives in `ballista-history`. +pub fn task_status_to_dto(value: &task_status::Status) -> TaskStatus { + match value { + task_status::Status::Running(_) => TaskStatus::Running, + task_status::Status::Failed(failed) => TaskStatus::Failed { + reason: failed_reason(failed), + error: failed.error.clone(), + }, + task_status::Status::Successful(_) => TaskStatus::Successful, + } +} + +fn percentile_duration(sorted: &[u64], pct: f64) -> u64 { + let idx = ((pct / 100.0) * (sorted.len() - 1) as f64).round() as usize; + sorted[idx.min(sorted.len() - 1)] +} + +fn percentiles_of(mut values: Vec) -> Option { + if values.is_empty() { + return None; + } + + values.sort_unstable(); + + Some(Percentiles { + min: values[0], + p25: percentile_duration(&values, 25.0), + median: percentile_duration(&values, 50.0), + p75: percentile_duration(&values, 75.0), + max: *values.last().unwrap(), + }) +} + +fn task_input_percentiles(tasks: &[Option]) -> Option { + percentiles_of( + tasks + .iter() + .flatten() + .map(|t| t.input_rows as u64) + .collect(), + ) +} + +fn task_duration_percentiles(tasks: &[Option]) -> Option { + percentiles_of(tasks.iter().flatten().map(|t| t.exec_duration).collect()) +} + +/// Returns elapsed wall time in milliseconds for API formatting. +/// +/// Uses saturating subtraction so inconsistent timestamps (e.g. failed jobs, or +/// `end_time` still zero while `start_time` is set) do not panic on subtract. +fn job_elapsed_ms(start_time: u64, end_time: u64) -> u64 { + end_time.saturating_sub(start_time) +} + +fn format_job_status(status: &Option, elapsed_ms: u64) -> (String, String) { + match status { + Some(Status::Queued(_)) => ("Queued".to_string(), "Queued".to_string()), + Some(Status::Running(_)) => ("Running".to_string(), "Running".to_string()), + Some(Status::Failed(error)) => { + ("Failed".to_string(), format!("Failed: {}", error.error)) + } + Some(Status::Successful(completed)) => { + let num_rows = completed + .partition_location + .iter() + .map(|p| p.partition_stats.as_ref().map(|s| s.num_rows).unwrap_or(0)) + .sum::(); + let num_rows_term = if num_rows == 1 { "row" } else { "rows" }; + let num_partitions = completed.partition_location.len(); + let num_partitions_term = if num_partitions == 1 { + "partition" + } else { + "partitions" + }; + ( + "Completed".to_string(), + format!( + "Completed. Produced {} {} containing {} {}. Elapsed time: {} ms.", + num_partitions, + num_partitions_term, + num_rows, + num_rows_term, + elapsed_ms + ), + ) + } + _ => ("Invalid".to_string(), "Invalid State".to_string()), + } +} + +fn get_running_stage_time(task_infos: &[TaskInfo], current_time: u128) -> Option { + let min_start = task_infos + .iter() + .map(|t| t.start_exec_time) + .filter(|t| *t > 0) + .min(); + + match (min_start, current_time) { + (Some(start), end) if end >= start => Some(format_millis(end - start)), + _ => None, + } +} + +fn get_finished_stage_time(task_infos: &[TaskInfo]) -> Option { + let min_start = task_infos + .iter() + .map(|t| t.start_exec_time) + .filter(|t| *t > 0) + .min(); + + let max_end = task_infos + .iter() + .map(|t| t.end_exec_time) + .filter(|t| *t > 0) + .max(); + + match (min_start, max_end) { + (Some(start), Some(end)) if end >= start => Some(format_millis(end - start)), + _ => None, + } +} + +/// Format a millisecond duration the way DataFusion renders elapsed-time +/// metrics, so stage timings match plan output. +fn format_millis(millis: u128) -> String { + let time = Time::new(); + time.add_duration(Duration::from_millis(millis as u64)); + time.to_string() +} + +fn failed_reason(failed: &FailedTask) -> String { + match &failed.failed_reason { + Some(ExecutionError(_)) => "ExecutionError", + Some(FetchPartitionError(_)) => "FetchPartitionError", + Some(IoError(_)) => "IoError", + Some(ExecutorLost(_)) => "ExecutorLost", + Some(ResultLost(_)) => "ResultLost", + Some(TaskKilled(_)) => "TaskKilled", + None => "Failed", + } + .to_string() +} + +/// Sum a task's `input_rows` / `output_rows` across the global partitions the +/// task owns. Metrics are keyed by global partition id — for single-partition +/// tasks `partitions` is a one-element slice; for multi-partition tasks it is +/// the task's `global_input_partition_ids`. +fn get_partition_counts(metrics: &[MetricsSet], partitions: &[usize]) -> (usize, usize) { + let input_rows = get_partition_count(metrics, partitions, "input_rows"); + let output_rows = get_partition_count(metrics, partitions, "output_rows"); + (input_rows, output_rows) +} + +fn get_partition_count( + metrics: &[MetricsSet], + partitions: &[usize], + name: &str, +) -> usize { + metrics + .iter() + .flat_map(|vec| { + vec.iter().map(|metric| { + let metric_value = metric.value(); + let owned_by_task = metric + .partition() + .map(|p| partitions.contains(&p)) + .unwrap_or(false); + if owned_by_task && metric_value.name() == name { + metric_value.as_usize() + } else { + 0 + } + }) + }) + .sum() +} + +fn get_combined_count(metrics: &[MetricsSet], name: &str) -> usize { + metrics + .iter() + .flat_map(|vec| { + vec.iter().map(|metric| { + let metric_value = metric.value(); + if metric_value.name() == name { + metric_value.as_usize() + } else { + 0 + } + }) + }) + .sum() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::state::execution_stage::TaskInfo; + use ballista_core::serde::protobuf::task_status; + + fn make_task_info(start: u128, end: u128) -> TaskInfo { + TaskInfo { + task_id: 0, + scheduled_time: 0, + launch_time: 0, + start_exec_time: start, + end_exec_time: end, + finish_time: 0, + task_status: task_status::Status::Running(Default::default()), + global_input_partition_ids: vec![], + vcores_consumed: 0, + } + } + + #[test] + fn test_job_elapsed_saturates_when_end_precedes_start() { + assert_eq!(job_elapsed_ms(900, 100), 0); + } + + // --- get_finished_stage_time --- + + #[test] + fn test_finished_empty_slice_returns_none() { + assert_eq!(get_finished_stage_time(&[]), None); + } + + #[test] + fn test_finished_all_zero_timestamps_returns_none() { + let tasks = vec![make_task_info(0, 0), make_task_info(0, 0)]; + assert_eq!(get_finished_stage_time(&tasks), None); + } + + #[test] + fn test_finished_single_task_elapsed() { + // 600 - 100 = 500 ms → "500.00ms" + let tasks = vec![make_task_info(100, 600)]; + assert_eq!( + get_finished_stage_time(&tasks), + Some("500.00ms".to_string()) + ); + } + + #[test] + fn test_finished_picks_earliest_start_and_latest_end() { + // min start = 100, max end = 900 → 800 ms + let tasks = vec![ + make_task_info(100, 500), + make_task_info(200, 900), + make_task_info(300, 700), + ]; + assert_eq!( + get_finished_stage_time(&tasks), + Some("800.00ms".to_string()) + ); + } + + #[test] + fn test_finished_end_before_start_returns_none() { + let tasks = vec![make_task_info(900, 100)]; + assert_eq!(get_finished_stage_time(&tasks), None); + } + + // --- get_running_stage_time --- + + #[test] + fn test_running_empty_slice_returns_none() { + assert_eq!(get_running_stage_time(&[], 1000), None); + } + + #[test] + fn test_running_all_zero_start_returns_none() { + let tasks: Vec = vec![make_task_info(0, 0), make_task_info(0, 0)]; + assert_eq!(get_running_stage_time(&tasks, 1000), None); + } + + #[test] + fn test_running_future_start_returns_none() { + // start_exec_time beyond current time → elapsed clamped to 0 + let tasks = vec![make_task_info(u128::MAX, 0)]; + assert_eq!(get_running_stage_time(&tasks, 1000), None); + } + + #[test] + fn test_running_past_start_returns_some() { + let now = 4_000; + let start = 1_000; + let tasks = vec![make_task_info(start, 0)]; + assert_eq!( + get_running_stage_time(&tasks, now), + Some("3.00s".to_string()) + ); + } + + #[test] + fn test_running_mixed_zero_start_uses_earliest_nonzero() { + let now = 3_000; + let earlier = 1_000; + let later = 2_000; + let tasks = vec![ + make_task_info(0, 0), + make_task_info(later, 0), + make_task_info(earlier, 0), + make_task_info(0, 0), + ]; + let result = get_running_stage_time(&tasks, now); + assert_eq!(result, Some("2.00s".to_string())); + } + + #[test] + fn test_job_elapsed_ms_normal() { + assert_eq!(super::job_elapsed_ms(100, 500), 400); + } + + #[test] + fn test_job_elapsed_ms_end_before_start_saturates_to_zero() { + assert_eq!(super::job_elapsed_ms(500, 100), 0); + } +} diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index 5409f07801..f236dd37a6 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -10,11 +10,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::display::format_stage_metrics; +use crate::api::dto_build; use crate::scheduler_server::event::QueryStageSchedulerEvent; -use crate::state::execution_graph::ExecutionStage; use crate::state::execution_graph_dot::ExecutionGraphDot; -use crate::state::execution_stage::TaskInfo; use crate::{api::SchedulerErrorResponse, scheduler_server::SchedulerServer}; use axum::extract::Query; use axum::response::Redirect; @@ -23,22 +21,14 @@ use axum::{ extract::{Path, State}, response::{IntoResponse, Response}, }; -use ballista_core::serde::protobuf::failed_task::FailedReason::{ - ExecutionError, ExecutorLost, FetchPartitionError, IoError, ResultLost, TaskKilled, -}; +use ballista_core::BALLISTA_VERSION; use ballista_core::serde::protobuf::job_status::Status; -use ballista_core::serde::protobuf::{ - ExecutorMetric, FailedTask, executor_metric::Metric, task_status, -}; +use ballista_core::serde::protobuf::{ExecutorMetric, executor_metric::Metric}; use ballista_core::serde::scheduler::{ ExecutorOperatingSystemSpecification, ExecutorSpecification, }; -use ballista_core::utils::get_current_time; -use ballista_core::{BALLISTA_VERSION, JobId}; +use ballista_history::dto::JobResponse; use datafusion::DATAFUSION_VERSION; -use datafusion::physical_plan::display::DisplayableExecutionPlan; -use datafusion::physical_plan::displayable; -use datafusion::physical_plan::metrics::{MetricsSet, Time}; use datafusion_proto::logical_plan::AsLogicalPlan; use datafusion_proto::physical_plan::AsExecutionPlan; #[cfg(feature = "graphviz-support")] @@ -48,10 +38,8 @@ use graphviz_rust::{ printer::PrinterContext, }; use http::{HeaderMap, StatusCode, header::CONTENT_TYPE}; -use serde::Serialize; use std::collections::HashMap; use std::sync::Arc; -use std::time::Duration; #[derive(Debug, serde::Serialize)] struct SchedulerStateResponse { @@ -112,27 +100,6 @@ impl ExecutorMetricResponse { } } -#[derive(Debug, serde::Serialize)] -pub struct JobResponse { - pub job_id: JobId, - pub job_name: String, - pub job_status: String, - pub status: String, - pub num_stages: usize, - pub completed_stages: usize, - pub percent_complete: u8, - /// Timestamp when the job started. - pub start_time: u64, - /// Timestamp when the job ended (0 if still running). - pub end_time: u64, - #[serde(skip_serializing_if = "Option::is_none")] - pub logical_plan: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub physical_plan: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub stage_plan: Option, -} - #[derive(Debug, serde::Serialize)] struct CancelJobResponse { pub cancelled: bool, @@ -140,80 +107,6 @@ struct CancelJobResponse { pub reason: Option, } -#[derive(Debug, serde::Serialize)] -pub struct TaskSummary { - /// task id - pub id: usize, - /// Task status - pub status: TaskStatus, - /// Global partition ids covered by this task. For a single-partition - /// task this is a one-element list — JSON-compatible with the old - /// scalar `partition_id` for callers that read `partition_id[0]`, and - /// honestly plural for multi-partition tasks. - pub partition_id: Vec, - /// Scheduler schedule time - pub scheduled_time: u64, - /// Scheduler launch time (ms since epoch) - pub launch_time: u64, - /// The time the Executor start to run the task (ms since epoch) - pub start_exec_time: u64, - /// The time the Executor finish the task (ms since epoch) - pub end_exec_time: u64, - /// total execution time (ms) - pub exec_duration: u64, - /// Scheduler side finish time (ms since epoch) - pub finish_time: u64, - /// Number of input rows - pub input_rows: usize, - /// Number of output rows - pub output_rows: usize, -} - -#[derive(Debug, Clone, Serialize)] -pub enum TaskStatus { - Running, - Successful, - Failed { reason: String, error: String }, -} - -impl From<&task_status::Status> for TaskStatus { - fn from(value: &task_status::Status) -> Self { - match value { - task_status::Status::Running(_) => TaskStatus::Running, - task_status::Status::Failed(failed) => TaskStatus::Failed { - reason: failed_reason(failed), - error: failed.error.clone(), - }, - task_status::Status::Successful(_) => TaskStatus::Successful, - } - } -} - -#[derive(Debug, serde::Serialize)] -pub struct Percentiles { - pub min: u64, - pub p25: u64, - pub median: u64, - pub p75: u64, - pub max: u64, -} - -#[derive(Debug, serde::Serialize)] -pub struct QueryStageSummary { - pub stage_id: String, - pub stage_status: String, - pub input_rows: usize, - pub output_rows: usize, - pub elapsed_compute: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub stage_plan: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub task_duration_percentiles: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub task_input_percentiles: Option, - pub tasks: Vec>, -} - #[derive(Debug, serde::Deserialize, Default)] pub struct JobQueryParams { /// Controls plan format @@ -395,34 +288,7 @@ pub async fn get_jobs< let jobs: Vec = jobs .iter() - .map(|job| { - let (plain_status, job_status) = format_job_status( - &job.status.status, - job_elapsed_ms(job.start_time, job.end_time), - ); - - // calculate progress based on completed stages for now, but we could use completed - // tasks in the future to make this more accurate - let percent_complete = if job.num_stages == 0 { - 0 - } else { - ((job.completed_stages as f32 / job.num_stages as f32) * 100_f32) as u8 - }; - JobResponse { - job_id: job.job_id.to_owned(), - job_name: job.job_name.to_owned(), - job_status, - status: plain_status, - start_time: job.start_time, - end_time: job.end_time, - num_stages: job.num_stages, - completed_stages: job.completed_stages, - percent_complete, - logical_plan: None, - physical_plan: None, - stage_plan: None, - } - }) + .map(dto_build::job_overview_to_response) .collect(); Ok(Json(jobs)) @@ -446,45 +312,8 @@ pub async fn get_job< SchedulerErrorResponse::with_error(StatusCode::INTERNAL_SERVER_ERROR, format!("Error occurred while getting the execution graph for job '{job_id}'")) })? .ok_or_else(|| SchedulerErrorResponse::new(StatusCode::NOT_FOUND))?; - let stage_plan = format!("{:?}", graph); - let job = graph.as_ref(); - let (plain_status, job_status) = format_job_status( - &job.status().status, - job_elapsed_ms(job.start_time(), job.end_time()), - ); - - let num_stages = job.stage_count(); - let completed_stages = job.completed_stages(); - let percent_complete = - ((completed_stages as f32 / num_stages as f32) * 100_f32) as u8; - - let plan_format = query.plan_format.clone().unwrap_or_default(); - - let physical_plan = match plan_format { - PlanFormat::Default | PlanFormat::Metrics => { - DisplayableExecutionPlan::new(job.physical_plan().as_ref()) - .indent(false) - .to_string() - } - PlanFormat::Tree => displayable(job.physical_plan().as_ref()) - .tree_render() - .to_string(), - }; - Ok(Json(JobResponse { - job_id: job.job_id().to_owned(), - job_name: job.job_name().to_owned(), - job_status, - status: plain_status, - start_time: job.start_time(), - end_time: job.end_time(), - num_stages, - completed_stages, - percent_complete, - logical_plan: job.logical_plan().map(str::to_owned), - physical_plan: Some(physical_plan), - stage_plan: Some(stage_plan), - })) + Ok(Json(dto_build::graph_to_job_response(&graph, &query))) } pub async fn cancel_job< @@ -554,11 +383,6 @@ pub async fn cancel_job< } } -#[derive(Debug, serde::Serialize)] -pub struct QueryStagesResponse { - pub stages: Vec, -} - pub async fn get_query_stages< T: AsLogicalPlan + Clone + Send + Sync + 'static, U: AsExecutionPlan + Send + Sync + 'static, @@ -567,8 +391,6 @@ pub async fn get_query_stages< Path(job_id): Path, query: Query, ) -> Result { - let plan_format = query.plan_format.clone().unwrap_or_default(); - if let Some(graph) = data_server .state .task_manager @@ -582,383 +404,12 @@ pub async fn get_query_stages< ) })? { - let stages = graph - .as_ref() - .stages() - .iter() - .map(|(id, stage)| { - let mut summary = QueryStageSummary { - stage_id: id.to_string(), - stage_status: stage.variant_name().to_string(), - input_rows: 0, - output_rows: 0, - elapsed_compute: None, - tasks: vec![], - task_duration_percentiles: None, - task_input_percentiles: None, - stage_plan: None, - }; - match stage { - ExecutionStage::Running(running_stage) => { - let metrics = running_stage.stage_metrics.as_deref().unwrap_or(&[]); - summary.stage_plan = Some(match plan_format { - PlanFormat::Default => displayable(running_stage.plan.as_ref()).indent(false).to_string(), - PlanFormat::Tree => displayable(running_stage.plan.as_ref()).tree_render().to_string(), - PlanFormat::Metrics => format_stage_metrics(running_stage.plan.as_ref(), metrics), - }); - summary.input_rows = running_stage - .stage_metrics - .as_ref() - .map(|m| get_combined_count(m.as_slice(), "input_rows")) - .unwrap_or(0); - summary.output_rows = running_stage - .stage_metrics - .as_ref() - .map(|m| get_combined_count(m.as_slice(), "output_rows")) - .unwrap_or(0); - summary.elapsed_compute = get_running_stage_time(&running_stage - .task_infos, get_current_time()); - summary.tasks = running_stage - .task_infos - .iter() - .map(|info| { - let (input_rows, output_rows) = running_stage - .stage_metrics - .as_deref() - .map(|metrics| { - get_partition_counts( - metrics, - &info.global_input_partition_ids, - ) - }) - .unwrap_or((0, 0)); - - let start_exec_time = info.start_exec_time as u64; - let end_exec_time = info.end_exec_time as u64; - - let task_status: TaskStatus = (&info.task_status).into(); - - Some(TaskSummary { - id: info.task_id, - partition_id: info - .global_input_partition_ids - .iter() - .map(|&p| p as u32) - .collect(), - scheduled_time: info.scheduled_time as u64, - launch_time: info.launch_time as u64, - start_exec_time, - end_exec_time, - exec_duration: end_exec_time.saturating_sub(start_exec_time), - finish_time: info.finish_time as u64, - input_rows, - output_rows, - status: task_status, - }) - }) - .collect(); - } - ExecutionStage::Successful(completed_stage) => { - summary.stage_plan = Some(match plan_format { - PlanFormat::Default => displayable(completed_stage.plan.as_ref()).indent(false).to_string(), - PlanFormat::Tree => displayable(completed_stage.plan.as_ref()).tree_render().to_string(), - PlanFormat::Metrics => format_stage_metrics(completed_stage.plan.as_ref(), &completed_stage.stage_metrics), - }); - summary.input_rows = get_combined_count( - &completed_stage.stage_metrics, - "input_rows", - ); - summary.output_rows = get_combined_count( - &completed_stage.stage_metrics, - "output_rows", - ); - summary.elapsed_compute = - get_finished_stage_time(&completed_stage.task_infos); - - summary.tasks = completed_stage - .task_infos - .iter() - .map(|task_info| { - let (input_rows, output_rows) = get_partition_counts( - &completed_stage.stage_metrics, - &task_info.global_input_partition_ids, - ); - - let start_exec_time = task_info.start_exec_time as u64; - let end_exec_time = task_info.end_exec_time as u64; - let task_status = (&task_info.task_status).into(); - Some(TaskSummary { - id: task_info.task_id, - partition_id: task_info - .global_input_partition_ids - .iter() - .map(|&p| p as u32) - .collect(), - scheduled_time: task_info.scheduled_time as u64, - launch_time: task_info.launch_time as u64, - start_exec_time, - end_exec_time, - exec_duration: end_exec_time.saturating_sub(start_exec_time), - finish_time: task_info.finish_time as u64, - input_rows, - output_rows, - status: task_status, - }) - }) - .collect(); - } - ExecutionStage::Failed(failed_stage) => { - let metrics = failed_stage.stage_metrics.as_deref().unwrap_or(&[]); - summary.stage_plan = Some(match plan_format { - PlanFormat::Default => displayable(failed_stage.plan.as_ref()).indent(false).to_string(), - PlanFormat::Tree => displayable(failed_stage.plan.as_ref()).tree_render().to_string(), - PlanFormat::Metrics => format_stage_metrics(failed_stage.plan.as_ref(), metrics), - }); - summary.input_rows = get_combined_count(metrics, "input_rows"); - summary.output_rows = get_combined_count(metrics, "output_rows"); - summary.elapsed_compute = - get_finished_stage_time(&failed_stage.task_infos); - - summary.tasks = failed_stage - .task_infos - .iter() - .map(|info| { - let (input_rows, output_rows) = get_partition_counts( - metrics, - &info.global_input_partition_ids, - ); - - let start_exec_time = info.start_exec_time as u64; - let end_exec_time = info.end_exec_time as u64; - let task_status: TaskStatus = (&info.task_status).into(); - - Some(TaskSummary { - id: info.task_id, - partition_id: info - .global_input_partition_ids - .iter() - .map(|&p| p as u32) - .collect(), - scheduled_time: info.scheduled_time as u64, - launch_time: info.launch_time as u64, - start_exec_time, - end_exec_time, - exec_duration: end_exec_time.saturating_sub(start_exec_time), - finish_time: info.finish_time as u64, - input_rows, - output_rows, - status: task_status, - }) - }) - .collect(); - } - _ => {} - } - summary.task_duration_percentiles = task_duration_percentiles(&summary.tasks); - summary.task_input_percentiles = task_input_percentiles(&summary.tasks); - summary - }) - .collect(); - - Ok(Json(QueryStagesResponse { stages })) + Ok(Json(dto_build::graph_to_query_stages(&graph, &query))) } else { Err(SchedulerErrorResponse::new(StatusCode::NOT_FOUND)) } } -fn percentile_duration(sorted: &[u64], pct: f64) -> u64 { - let idx = ((pct / 100.0) * (sorted.len() - 1) as f64).round() as usize; - sorted[idx.min(sorted.len() - 1)] -} - -fn task_input_percentiles(tasks: &[Option]) -> Option { - let mut durations: Vec = tasks - .iter() - .flatten() - .map(|t| t.input_rows as u64) - .collect(); - - if durations.is_empty() { - return None; - } - - durations.sort_unstable(); - - Some(Percentiles { - min: durations[0], - p25: percentile_duration(&durations, 25.0), - median: percentile_duration(&durations, 50.0), - p75: percentile_duration(&durations, 75.0), - max: *durations.last().unwrap(), - }) -} - -fn task_duration_percentiles(tasks: &[Option]) -> Option { - let mut durations: Vec = - tasks.iter().flatten().map(|t| t.exec_duration).collect(); - - if durations.is_empty() { - return None; - } - - durations.sort_unstable(); - - Some(Percentiles { - min: durations[0], - p25: percentile_duration(&durations, 25.0), - median: percentile_duration(&durations, 50.0), - p75: percentile_duration(&durations, 75.0), - max: *durations.last().unwrap(), - }) -} - -/// Returns elapsed wall time in milliseconds for API formatting. -/// -/// Uses saturating subtraction so inconsistent timestamps (e.g. failed jobs, or -/// `end_time` still zero while `start_time` is set) do not panic on subtract. -fn job_elapsed_ms(start_time: u64, end_time: u64) -> u64 { - end_time.saturating_sub(start_time) -} - -fn format_job_status(status: &Option, elapsed_ms: u64) -> (String, String) { - match status { - Some(Status::Queued(_)) => ("Queued".to_string(), "Queued".to_string()), - Some(Status::Running(_)) => ("Running".to_string(), "Running".to_string()), - Some(Status::Failed(error)) => { - ("Failed".to_string(), format!("Failed: {}", error.error)) - } - Some(Status::Successful(completed)) => { - let num_rows = completed - .partition_location - .iter() - .map(|p| p.partition_stats.as_ref().map(|s| s.num_rows).unwrap_or(0)) - .sum::(); - let num_rows_term = if num_rows == 1 { "row" } else { "rows" }; - let num_partitions = completed.partition_location.len(); - let num_partitions_term = if num_partitions == 1 { - "partition" - } else { - "partitions" - }; - ( - "Completed".to_string(), - format!( - "Completed. Produced {} {} containing {} {}. Elapsed time: {} ms.", - num_partitions, - num_partitions_term, - num_rows, - num_rows_term, - elapsed_ms - ), - ) - } - _ => ("Invalid".to_string(), "Invalid State".to_string()), - } -} - -fn get_running_stage_time(task_infos: &[TaskInfo], current_time: u128) -> Option { - let min_start = task_infos - .iter() - .map(|t| t.start_exec_time) - .filter(|t| *t > 0) - .min(); - - match (min_start, current_time) { - (Some(start), end) if end >= start => { - let time = Time::new(); - time.add_duration(Duration::from_millis((end - start) as u64)); - Some(time.to_string()) - } - _ => None, - } -} - -fn failed_reason(failed: &FailedTask) -> String { - match &failed.failed_reason { - Some(ExecutionError(_)) => "ExecutionError", - Some(FetchPartitionError(_)) => "FetchPartitionError", - Some(IoError(_)) => "IoError", - Some(ExecutorLost(_)) => "ExecutorLost", - Some(ResultLost(_)) => "ResultLost", - Some(TaskKilled(_)) => "TaskKilled", - None => "Failed", - } - .to_string() -} - -fn get_finished_stage_time(task_infos: &[TaskInfo]) -> Option { - let min_start = task_infos - .iter() - .map(|t| t.start_exec_time) - .filter(|t| *t > 0) - .min(); - - let max_end = task_infos - .iter() - .map(|t| t.end_exec_time) - .filter(|t| *t > 0) - .max(); - - match (min_start, max_end) { - (Some(start), Some(end)) if end >= start => { - let time = Time::new(); - time.add_duration(Duration::from_millis((end - start) as u64)); - Some(time.to_string()) - } - _ => None, - } -} - -/// Sum a task's `input_rows` / `output_rows` across the global partitions the -/// task owns. Metrics are keyed by global partition id — for single-partition -/// tasks `partitions` is a one-element slice; for multi-partition tasks it is -/// the task's `global_input_partition_ids`. -fn get_partition_counts(metrics: &[MetricsSet], partitions: &[usize]) -> (usize, usize) { - let input_rows = get_partition_count(metrics, partitions, "input_rows"); - let output_rows = get_partition_count(metrics, partitions, "output_rows"); - (input_rows, output_rows) -} - -fn get_partition_count( - metrics: &[MetricsSet], - partitions: &[usize], - name: &str, -) -> usize { - metrics - .iter() - .flat_map(|vec| { - vec.iter().map(|metric| { - let metric_value = metric.value(); - let owned_by_task = metric - .partition() - .map(|p| partitions.contains(&p)) - .unwrap_or(false); - if owned_by_task && metric_value.name() == name { - metric_value.as_usize() - } else { - 0 - } - }) - }) - .sum() -} - -fn get_combined_count(metrics: &[MetricsSet], name: &str) -> usize { - metrics - .iter() - .flat_map(|vec| { - vec.iter().map(|metric| { - let metric_value = metric.value(); - if metric_value.name() == name { - metric_value.as_usize() - } else { - 0 - } - }) - }) - .sum() -} - pub async fn get_job_dot_graph< T: AsLogicalPlan + Clone + Send + Sync + 'static, U: AsExecutionPlan + Send + Sync + 'static, @@ -1081,126 +532,6 @@ pub async fn get_job_config< #[cfg(test)] mod tests { use super::*; - use crate::state::execution_stage::TaskInfo; - use ballista_core::serde::protobuf::task_status; - - fn make_task_info(start: u128, end: u128) -> TaskInfo { - TaskInfo { - task_id: 0, - scheduled_time: 0, - launch_time: 0, - start_exec_time: start, - end_exec_time: end, - finish_time: 0, - task_status: task_status::Status::Running(Default::default()), - global_input_partition_ids: vec![], - vcores_consumed: 0, - } - } - - #[test] - fn test_job_elapsed_saturates_when_end_precedes_start() { - assert_eq!(job_elapsed_ms(900, 100), 0); - } - - // --- get_finished_stage_time --- - - #[test] - fn test_finished_empty_slice_returns_none() { - assert_eq!(get_finished_stage_time(&[]), None); - } - - #[test] - fn test_finished_all_zero_timestamps_returns_none() { - let tasks = vec![make_task_info(0, 0), make_task_info(0, 0)]; - assert_eq!(get_finished_stage_time(&tasks), None); - } - - #[test] - fn test_finished_single_task_elapsed() { - // 600 - 100 = 500 ms → "500.00ms" - let tasks = vec![make_task_info(100, 600)]; - assert_eq!( - get_finished_stage_time(&tasks), - Some("500.00ms".to_string()) - ); - } - - #[test] - fn test_finished_picks_earliest_start_and_latest_end() { - // min start = 100, max end = 900 → 800 ms - let tasks = vec![ - make_task_info(100, 500), - make_task_info(200, 900), - make_task_info(300, 700), - ]; - assert_eq!( - get_finished_stage_time(&tasks), - Some("800.00ms".to_string()) - ); - } - - #[test] - fn test_finished_end_before_start_returns_none() { - let tasks = vec![make_task_info(900, 100)]; - assert_eq!(get_finished_stage_time(&tasks), None); - } - - // --- get_running_stage_time --- - - #[test] - fn test_running_empty_slice_returns_none() { - assert_eq!(get_running_stage_time(&[], 1000), None); - } - - #[test] - fn test_running_all_zero_start_returns_none() { - let tasks: Vec = vec![make_task_info(0, 0), make_task_info(0, 0)]; - assert_eq!(get_running_stage_time(&tasks, 1000), None); - } - - #[test] - fn test_running_future_start_returns_none() { - // start_exec_time beyond current time → elapsed clamped to 0 - let tasks = vec![make_task_info(u128::MAX, 0)]; - assert_eq!(get_running_stage_time(&tasks, 1000), None); - } - - #[test] - fn test_running_past_start_returns_some() { - let now = 4_000; - let start = 1_000; - let tasks = vec![make_task_info(start, 0)]; - assert_eq!( - get_running_stage_time(&tasks, now), - Some("3.00s".to_string()) - ); - } - - #[test] - fn test_running_mixed_zero_start_uses_earliest_nonzero() { - let now = 3_000; - let earlier = 1_000; - let later = 2_000; - let tasks = vec![ - make_task_info(0, 0), - make_task_info(later, 0), - make_task_info(earlier, 0), - make_task_info(0, 0), - ]; - let result = get_running_stage_time(&tasks, now); - assert_eq!(result, Some("2.00s".to_string())); - } - - #[test] - fn test_job_elapsed_ms_normal() { - assert_eq!(super::job_elapsed_ms(100, 500), 400); - } - - #[test] - fn test_job_elapsed_ms_end_before_start_saturates_to_zero() { - assert_eq!(super::job_elapsed_ms(500, 100), 0); - } mod get_webtui { use super::*; diff --git a/ballista/scheduler/src/api/mod.rs b/ballista/scheduler/src/api/mod.rs index 81d7177cfa..527e7610b8 100644 --- a/ballista/scheduler/src/api/mod.rs +++ b/ballista/scheduler/src/api/mod.rs @@ -10,6 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(feature = "rest-api")] +mod dto_build; #[cfg(feature = "rest-api")] mod handlers; mod health; From 8c55ee35fd6b435bad713971f43d87009fe3827c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Aug 2026 10:05:30 -0600 Subject: [PATCH 2/5] refactor(scheduler): tighten the DTO extraction Follow-up cleanups on the extraction: - Collapse the three near-identical ExecutionStage arms in graph_to_query_stages into one destructuring match, dropping the mutable placeholder-zero summary. - Take PlanFormat by value instead of &JobQueryParams, and move PlanFormat into ballista-history. It is part of the wire contract, and the pure builder no longer imports an axum query-param type back out of the handler module. - Inject `now` into graph_to_query_stages rather than reading the clock, so replaying a stored log renders stable elapsed times. - Share percent_complete and min_start_time; use displayable() instead of the longhand DisplayableExecutionPlan::new(). - Drop the dead JobConfig alias and the unused serde_json dev-dependency, make task_status_to_dto private, and remove a duplicated test. - Enable #![warn(missing_docs)] on ballista-history and document the types, matching the other Ballista crates. - Register ballista-history with the release tooling: version bump script, publish order, and crate dependency graph. --- ballista/history/src/dto.rs | 62 ++++++++- ballista/history/src/lib.rs | 14 ++ ballista/scheduler/src/api/dto_build.rs | 169 ++++++++++-------------- ballista/scheduler/src/api/handlers.rs | 26 ++-- dev/release/README.md | 2 + dev/release/crate-deps.dot | 2 + dev/update_ballista_versions.py | 2 + 7 files changed, 157 insertions(+), 120 deletions(-) diff --git a/ballista/history/src/dto.rs b/ballista/history/src/dto.rs index 74bbb7fe89..272730af84 100644 --- a/ballista/history/src/dto.rs +++ b/ballista/history/src/dto.rs @@ -19,37 +19,58 @@ //! server, so both serialize byte-identical JSON. use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +/// Summary of one job, served by `GET /api/jobs` and `GET /api/job/{job_id}`. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct JobResponse { /// A `String` rather than `ballista_core::JobId` so this crate stays a /// serde-only leaf. `JobId` is `#[serde(transparent)]` over `String`, so /// the serialized JSON is unchanged. pub job_id: String, + /// Human-readable job name. pub job_name: String, + /// Verbose status, including the completion or failure detail. pub job_status: String, + /// Plain status word: `Queued`, `Running`, `Completed`, `Failed`, or `Invalid`. pub status: String, + /// Total number of stages in the job. pub num_stages: usize, + /// Number of stages that finished successfully. pub completed_stages: usize, + /// Progress as a percentage of completed stages. pub percent_complete: u8, + /// Timestamp when the job started. pub start_time: u64, + /// Timestamp when the job ended (0 if still running). pub end_time: u64, + /// Rendered logical plan. Absent in the job list. #[serde(skip_serializing_if = "Option::is_none")] pub logical_plan: Option, + /// Rendered physical plan. Absent in the job list. #[serde(skip_serializing_if = "Option::is_none")] pub physical_plan: Option, + /// Rendered stage DAG. Absent in the job list. #[serde(skip_serializing_if = "Option::is_none")] pub stage_plan: Option, } +/// Terminal or in-flight state of a single task. #[derive(Debug, Clone, Serialize, Deserialize)] pub enum TaskStatus { + /// Task is currently executing. Running, + /// Task completed successfully. Successful, - Failed { reason: String, error: String }, + /// Task failed, with the classified reason and the underlying error. + Failed { + /// Failure category, e.g. `ExecutionError` or `FetchPartitionError`. + reason: String, + /// Underlying error message. + error: String, + }, } +/// Per-task detail within a stage. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TaskSummary { /// task id @@ -79,36 +100,67 @@ pub struct TaskSummary { pub output_rows: usize, } +/// Five-number summary over a stage's tasks, used to spot skew. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Percentiles { + /// Smallest observed value. pub min: u64, + /// 25th percentile. pub p25: u64, + /// 50th percentile. pub median: u64, + /// 75th percentile. pub p75: u64, + /// Largest observed value. pub max: u64, } +/// Summary of one query stage, served by `GET /api/job/{job_id}/stages`. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QueryStageSummary { + /// Stage id, as a string. pub stage_id: String, + /// Stage state, e.g. `Running`, `Successful`, `Failed`. pub stage_status: String, + /// Rows read by the stage, summed across tasks. pub input_rows: usize, + /// Rows produced by the stage, summed across tasks. pub output_rows: usize, + /// Formatted wall time across the stage's tasks, if any have started. pub elapsed_compute: Option, + /// Rendered plan for this stage, in the requested [`PlanFormat`]. #[serde(skip_serializing_if = "Option::is_none")] pub stage_plan: Option, + /// Distribution of per-task execution time. #[serde(skip_serializing_if = "Option::is_none")] pub task_duration_percentiles: Option, + /// Distribution of per-task input row counts. #[serde(skip_serializing_if = "Option::is_none")] pub task_input_percentiles: Option, + /// One entry per task. Always `Some` today; the `Option` predates + /// multi-partition tasks, when this list was indexed by partition and + /// could be sparse. pub tasks: Vec>, } +/// Response body for `GET /api/job/{job_id}/stages`. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QueryStagesResponse { + /// One summary per stage, in stage order. pub stages: Vec, } -/// Session config as flat key/value pairs (from `SessionConfig::to_props()`), -/// sorted for stable output. -pub type JobConfig = BTreeMap; +/// How a plan should be rendered. Parsed from the `?plan_format=` query +/// parameter, and part of the REST contract, so it lives alongside the +/// response types rather than with the HTTP handlers. +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PlanFormat { + /// `?plan_format=default` => plain indent, no metrics + #[default] + Default, + /// `?plan_format=tree` => tree render, no metrics + Tree, + /// `?plan_format=metrics` => indent with aggregated metrics + Metrics, +} diff --git a/ballista/history/src/lib.rs b/ballista/history/src/lib.rs index affddf8a53..22c4d525d8 100644 --- a/ballista/history/src/lib.rs +++ b/ballista/history/src/lib.rs @@ -15,10 +15,24 @@ // specific language governing permissions and limitations // under the License. +#![warn(missing_docs)] + //! Shared types for Ballista's scheduler REST API. //! //! These response types live outside `ballista-scheduler` so that a future //! history server can serialize byte-identical JSON from stored state without //! depending on the scheduler's live execution graph. +//! +//! # What belongs here +//! +//! A type belongs in this crate when it is part of the `/api/*` wire contract +//! *and* can be reconstructed from a stored event log. Types describing live +//! scheduler state (`SchedulerStateResponse`, `CancelJobResponse`) stay in +//! `ballista-scheduler`, because there is nothing to replay. +//! +//! `ExecutorResponse` also stays behind, for a different reason: it embeds +//! `ballista-core` types, and this crate is deliberately serde-only so it +//! stays cheap to depend on. Sharing it would mean either taking a +//! `ballista-core` dependency here or duplicating those structs. pub mod dto; diff --git a/ballista/scheduler/src/api/dto_build.rs b/ballista/scheduler/src/api/dto_build.rs index ce1ec45315..70bafd10e9 100644 --- a/ballista/scheduler/src/api/dto_build.rs +++ b/ballista/scheduler/src/api/dto_build.rs @@ -23,9 +23,10 @@ //! in the axum handlers) means the same DTOs can be produced from state that //! did not come from a live handler request. //! -//! Everything in this module is pure: state in, DTO out, no I/O. +//! Everything in this module is pure: state in, DTO out, no I/O and no +//! wall-clock reads. `graph_to_query_stages` takes `now` from its caller so a +//! replay of a stored log renders the same elapsed times every time. -use crate::api::handlers::{JobQueryParams, PlanFormat}; use crate::display::format_stage_metrics; use crate::state::execution_graph::{ExecutionGraphBox, ExecutionStage}; use crate::state::execution_stage::TaskInfo; @@ -35,12 +36,11 @@ use ballista_core::serde::protobuf::failed_task::FailedReason::{ }; use ballista_core::serde::protobuf::job_status::Status; use ballista_core::serde::protobuf::{FailedTask, task_status}; -use ballista_core::utils::get_current_time; use ballista_history::dto::{ - JobResponse, Percentiles, QueryStageSummary, QueryStagesResponse, TaskStatus, - TaskSummary, + JobResponse, Percentiles, PlanFormat, QueryStageSummary, QueryStagesResponse, + TaskStatus, TaskSummary, }; -use datafusion::physical_plan::display::DisplayableExecutionPlan; +use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::displayable; use datafusion::physical_plan::metrics::{MetricsSet, Time}; use std::time::Duration; @@ -55,14 +55,6 @@ pub fn job_overview_to_response(job: &JobOverview) -> JobResponse { job_elapsed_ms(job.start_time, job.end_time), ); - // calculate progress based on completed stages for now, but we could use completed - // tasks in the future to make this more accurate - let percent_complete = if job.num_stages == 0 { - 0 - } else { - ((job.completed_stages as f32 / job.num_stages as f32) * 100_f32) as u8 - }; - JobResponse { job_id: job.job_id.to_string(), job_name: job.job_name.to_owned(), @@ -72,7 +64,7 @@ pub fn job_overview_to_response(job: &JobOverview) -> JobResponse { end_time: job.end_time, num_stages: job.num_stages, completed_stages: job.completed_stages, - percent_complete, + percent_complete: percent_complete(job.completed_stages, job.num_stages), logical_plan: None, physical_plan: None, stage_plan: None, @@ -83,7 +75,7 @@ pub fn job_overview_to_response(job: &JobOverview) -> JobResponse { /// rendered logical, physical, and stage plans. pub fn graph_to_job_response( graph: &ExecutionGraphBox, - query: &JobQueryParams, + plan_format: PlanFormat, ) -> JobResponse { let stage_plan = format!("{graph:?}"); let job = graph.as_ref(); @@ -95,13 +87,10 @@ pub fn graph_to_job_response( let num_stages = job.stage_count(); let completed_stages = job.completed_stages(); - let percent_complete = - ((completed_stages as f32 / num_stages as f32) * 100_f32) as u8; - let plan_format = query.plan_format.clone().unwrap_or_default(); let physical_plan = match plan_format { PlanFormat::Default | PlanFormat::Metrics => { - DisplayableExecutionPlan::new(job.physical_plan().as_ref()) + displayable(job.physical_plan().as_ref()) .indent(false) .to_string() } @@ -119,7 +108,7 @@ pub fn graph_to_job_response( end_time: job.end_time(), num_stages, completed_stages, - percent_complete, + percent_complete: percent_complete(completed_stages, num_stages), logical_plan: job.logical_plan().map(str::to_owned), physical_plan: Some(physical_plan), stage_plan: Some(stage_plan), @@ -129,75 +118,53 @@ pub fn graph_to_job_response( /// Build the per-stage summaries served by `GET /api/job/{job_id}/stages`. pub fn graph_to_query_stages( graph: &ExecutionGraphBox, - query: &JobQueryParams, + plan_format: PlanFormat, + now: u128, ) -> QueryStagesResponse { - let plan_format = query.plan_format.clone().unwrap_or_default(); - let stages = graph .as_ref() .stages() .iter() .map(|(id, stage)| { - let mut summary = QueryStageSummary { + // Every started stage contributes the same three things; only how + // it reaches its metrics and which elapsed-time rule applies + // differ. Stages that have not started yet contribute nothing. + let started: Option<(&[MetricsSet], &[TaskInfo], Option)> = + match stage { + ExecutionStage::Running(s) => Some(( + s.stage_metrics.as_deref().unwrap_or(&[]), + &s.task_infos, + get_running_stage_time(&s.task_infos, now), + )), + ExecutionStage::Successful(s) => Some(( + &s.stage_metrics, + &s.task_infos, + get_finished_stage_time(&s.task_infos), + )), + ExecutionStage::Failed(s) => Some(( + s.stage_metrics.as_deref().unwrap_or(&[]), + &s.task_infos, + get_finished_stage_time(&s.task_infos), + )), + _ => None, + }; + + let has_started = started.is_some(); + let (metrics, task_infos, elapsed_compute) = started.unwrap_or_default(); + let tasks = task_summaries(task_infos, metrics); + + QueryStageSummary { stage_id: id.to_string(), stage_status: stage.variant_name().to_string(), - input_rows: 0, - output_rows: 0, - elapsed_compute: None, - tasks: vec![], - task_duration_percentiles: None, - task_input_percentiles: None, - stage_plan: None, - }; - - match stage { - ExecutionStage::Running(running_stage) => { - let metrics = running_stage.stage_metrics.as_deref().unwrap_or(&[]); - summary.stage_plan = Some(render_stage_plan( - running_stage.plan.as_ref(), - metrics, - &plan_format, - )); - summary.input_rows = get_combined_count(metrics, "input_rows"); - summary.output_rows = get_combined_count(metrics, "output_rows"); - summary.elapsed_compute = get_running_stage_time( - &running_stage.task_infos, - get_current_time(), - ); - summary.tasks = task_summaries(&running_stage.task_infos, metrics); - } - ExecutionStage::Successful(completed_stage) => { - let metrics = completed_stage.stage_metrics.as_slice(); - summary.stage_plan = Some(render_stage_plan( - completed_stage.plan.as_ref(), - metrics, - &plan_format, - )); - summary.input_rows = get_combined_count(metrics, "input_rows"); - summary.output_rows = get_combined_count(metrics, "output_rows"); - summary.elapsed_compute = - get_finished_stage_time(&completed_stage.task_infos); - summary.tasks = task_summaries(&completed_stage.task_infos, metrics); - } - ExecutionStage::Failed(failed_stage) => { - let metrics = failed_stage.stage_metrics.as_deref().unwrap_or(&[]); - summary.stage_plan = Some(render_stage_plan( - failed_stage.plan.as_ref(), - metrics, - &plan_format, - )); - summary.input_rows = get_combined_count(metrics, "input_rows"); - summary.output_rows = get_combined_count(metrics, "output_rows"); - summary.elapsed_compute = - get_finished_stage_time(&failed_stage.task_infos); - summary.tasks = task_summaries(&failed_stage.task_infos, metrics); - } - _ => {} + input_rows: get_combined_count(metrics, "input_rows"), + output_rows: get_combined_count(metrics, "output_rows"), + elapsed_compute, + stage_plan: has_started + .then(|| render_stage_plan(stage.plan(), metrics, plan_format)), + task_duration_percentiles: task_duration_percentiles(&tasks), + task_input_percentiles: task_input_percentiles(&tasks), + tasks, } - - summary.task_duration_percentiles = task_duration_percentiles(&summary.tasks); - summary.task_input_percentiles = task_input_percentiles(&summary.tasks); - summary }) .collect(); @@ -207,9 +174,9 @@ pub fn graph_to_query_stages( /// Render one stage's plan in the requested format. `Metrics` overlays the /// stage's aggregated metrics onto the plan; the other formats ignore them. fn render_stage_plan( - plan: &dyn datafusion::physical_plan::ExecutionPlan, + plan: &dyn ExecutionPlan, metrics: &[MetricsSet], - plan_format: &PlanFormat, + plan_format: PlanFormat, ) -> String { match plan_format { PlanFormat::Default => displayable(plan).indent(false).to_string(), @@ -262,7 +229,7 @@ fn task_summaries( /// /// A free function rather than a `From` impl: both types are foreign to this /// crate now that [`TaskStatus`] lives in `ballista-history`. -pub fn task_status_to_dto(value: &task_status::Status) -> TaskStatus { +fn task_status_to_dto(value: &task_status::Status) -> TaskStatus { match value { task_status::Status::Running(_) => TaskStatus::Running, task_status::Status::Failed(failed) => TaskStatus::Failed { @@ -273,6 +240,15 @@ pub fn task_status_to_dto(value: &task_status::Status) -> TaskStatus { } } +/// Progress as a percentage of stages completed. Zero-stage jobs report 0 +/// rather than dividing by zero. +fn percent_complete(completed_stages: usize, num_stages: usize) -> u8 { + if num_stages == 0 { + return 0; + } + ((completed_stages as f32 / num_stages as f32) * 100_f32) as u8 +} + fn percentile_duration(sorted: &[u64], pct: f64) -> u64 { let idx = ((pct / 100.0) * (sorted.len() - 1) as f64).round() as usize; sorted[idx.min(sorted.len() - 1)] @@ -352,33 +328,31 @@ fn format_job_status(status: &Option, elapsed_ms: u64) -> (String, Strin } } -fn get_running_stage_time(task_infos: &[TaskInfo], current_time: u128) -> Option { - let min_start = task_infos +/// Earliest non-zero task start in a stage. Zero means "not started yet", so +/// those entries are ignored rather than dragging the minimum to 0. +fn min_start_time(task_infos: &[TaskInfo]) -> Option { + task_infos .iter() .map(|t| t.start_exec_time) .filter(|t| *t > 0) - .min(); + .min() +} - match (min_start, current_time) { +fn get_running_stage_time(task_infos: &[TaskInfo], current_time: u128) -> Option { + match (min_start_time(task_infos), current_time) { (Some(start), end) if end >= start => Some(format_millis(end - start)), _ => None, } } fn get_finished_stage_time(task_infos: &[TaskInfo]) -> Option { - let min_start = task_infos - .iter() - .map(|t| t.start_exec_time) - .filter(|t| *t > 0) - .min(); - let max_end = task_infos .iter() .map(|t| t.end_exec_time) .filter(|t| *t > 0) .max(); - match (min_start, max_end) { + match (min_start_time(task_infos), max_end) { (Some(start), Some(end)) if end >= start => Some(format_millis(end - start)), _ => None, } @@ -475,11 +449,6 @@ mod tests { } } - #[test] - fn test_job_elapsed_saturates_when_end_precedes_start() { - assert_eq!(job_elapsed_ms(900, 100), 0); - } - // --- get_finished_stage_time --- #[test] diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index f236dd37a6..1c79c7c49c 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -27,7 +27,8 @@ use ballista_core::serde::protobuf::{ExecutorMetric, executor_metric::Metric}; use ballista_core::serde::scheduler::{ ExecutorOperatingSystemSpecification, ExecutorSpecification, }; -use ballista_history::dto::JobResponse; +use ballista_core::utils::get_current_time; +use ballista_history::dto::{JobResponse, PlanFormat}; use datafusion::DATAFUSION_VERSION; use datafusion_proto::logical_plan::AsLogicalPlan; use datafusion_proto::physical_plan::AsExecutionPlan; @@ -113,18 +114,6 @@ pub struct JobQueryParams { pub plan_format: Option, } -#[derive(Debug, serde::Deserialize, Default, Clone)] -#[serde(rename_all = "snake_case")] -pub enum PlanFormat { - /// ?plan_format=default => plain indent, no metrics - #[default] - Default, - /// ?plan_format=tree => tree render, no metrics - Tree, - /// ?plan_format=metrics => indent with aggregated metrics - Metrics, -} - /// A handler for GET requests to the root (`/`). /// It redirects to `https://nightlies.apache.org/datafusion/ballista/tui//` /// forwarding any query parameters @@ -313,7 +302,10 @@ pub async fn get_job< })? .ok_or_else(|| SchedulerErrorResponse::new(StatusCode::NOT_FOUND))?; - Ok(Json(dto_build::graph_to_job_response(&graph, &query))) + Ok(Json(dto_build::graph_to_job_response( + &graph, + query.plan_format.unwrap_or_default(), + ))) } pub async fn cancel_job< @@ -404,7 +396,11 @@ pub async fn get_query_stages< ) })? { - Ok(Json(dto_build::graph_to_query_stages(&graph, &query))) + Ok(Json(dto_build::graph_to_query_stages( + &graph, + query.plan_format.unwrap_or_default(), + get_current_time(), + ))) } else { Err(SchedulerErrorResponse::new(StatusCode::NOT_FOUND)) } diff --git a/dev/release/README.md b/dev/release/README.md index 3f34ba10e5..cdacee9970 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -287,6 +287,7 @@ of the following crates: - [ballista-cli](https://crates.io/crates/ballista-cli) - [ballista-core](https://crates.io/crates/ballista-core) - [ballista-executor](https://crates.io/crates/ballista-executor) +- [ballista-history](https://crates.io/crates/ballista-history) - [ballista-scheduler](https://crates.io/crates/ballista-scheduler) Download and unpack the official release tarball @@ -306,6 +307,7 @@ dot -Tsvg dev/release/crate-deps.dot > dev/release/crate-deps.svg ```shell (cd ballista/core && cargo publish) (cd ballista/executor && cargo publish) +(cd ballista/history && cargo publish) (cd ballista/scheduler && cargo publish) (cd ballista/client && cargo publish) (cd ballista-cli && cargo publish) diff --git a/dev/release/crate-deps.dot b/dev/release/crate-deps.dot index c3113e6a3d..2986ee5251 100644 --- a/dev/release/crate-deps.dot +++ b/dev/release/crate-deps.dot @@ -18,12 +18,14 @@ digraph G { ballista_core + ballista_history ballista_scheduler ballista_executor ballista ballista_cli ballista_scheduler -> ballista_core + ballista_scheduler -> ballista_history ballista_executor -> ballista_core diff --git a/dev/update_ballista_versions.py b/dev/update_ballista_versions.py index 8fedb21023..9d29f65482 100755 --- a/dev/update_ballista_versions.py +++ b/dev/update_ballista_versions.py @@ -43,6 +43,7 @@ def update_cargo_toml(cargo_toml: str, new_version: str): 'ballista', 'ballista-core', 'ballista-executor', + 'ballista-history', 'ballista-scheduler', 'ballista-cli', ) @@ -80,6 +81,7 @@ def main(): for rel_path in [ 'ballista-cli', 'ballista/core', + 'ballista/history', 'ballista/scheduler', 'ballista/executor', 'ballista/client', From b6b6a714dcb2b13f57bcee60e4ac4500c63bb840 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Aug 2026 10:33:38 -0600 Subject: [PATCH 3/5] docs(history): state the write-once, replay-verbatim data flow The previous wording left it ambiguous whether the history server re-derives responses from stored execution state or replays stored DTOs. It replays them: the scheduler builds each response once against the live graph and writes it to the event log, so byte-identical output is a structural property rather than two implementations agreeing. Also records the consequence, that anything not captured at write time cannot be recovered at replay time. --- ballista/history/src/lib.rs | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/ballista/history/src/lib.rs b/ballista/history/src/lib.rs index 22c4d525d8..2f814e0fdc 100644 --- a/ballista/history/src/lib.rs +++ b/ballista/history/src/lib.rs @@ -19,15 +19,37 @@ //! Shared types for Ballista's scheduler REST API. //! -//! These response types live outside `ballista-scheduler` so that a future -//! history server can serialize byte-identical JSON from stored state without -//! depending on the scheduler's live execution graph. +//! This crate is the serialization boundary between the live scheduler and the +//! history server: the scheduler builds these types, writes them to an event +//! log, and the history server reads them back and serves them. +//! +//! # Write once, replay verbatim +//! +//! The history server does **not** re-derive responses from stored execution +//! state. When a job finishes, the scheduler runs its DTO builders once against +//! the live execution graph and writes the finished [`dto::JobResponse`] and +//! [`dto::QueryStagesResponse`] into the log's terminal record. Replay +//! deserializes those values and re-serializes them unchanged: +//! +//! ```text +//! scheduler ──> dto builders ──> DTO ──> event log ──> history server ──> JSON +//! ``` +//! +//! So byte-identical output is a structural property, not a convention two +//! implementations have to keep agreeing on. There is one definition of each +//! type and one place that populates it. That is also why the builders +//! themselves stay in `ballista-scheduler`: they need the live execution graph, +//! and nothing on the replay path calls them. +//! +//! One consequence worth knowing: anything not captured when the record is +//! written cannot be recovered later. Plans, for instance, are rendered in a +//! single format at write time, so replay cannot re-render them differently. //! //! # What belongs here //! //! A type belongs in this crate when it is part of the `/api/*` wire contract -//! *and* can be reconstructed from a stored event log. Types describing live -//! scheduler state (`SchedulerStateResponse`, `CancelJobResponse`) stay in +//! *and* is worth persisting in the event log. Types describing live scheduler +//! state (`SchedulerStateResponse`, `CancelJobResponse`) stay in //! `ballista-scheduler`, because there is nothing to replay. //! //! `ExecutorResponse` also stays behind, for a different reason: it embeds From e2e187352e3a7586f773295666ede81d7f2f9253 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Aug 2026 10:40:05 -0600 Subject: [PATCH 4/5] refactor: rename ballista-history to ballista-api-types The crate holds the /api/* wire types, and it has three parties, not one: the scheduler serves them, the web TUI deserializes them, and a future history server will serve replayed copies. Naming it after the history server made it awkward for the TUI, which parses live scheduler responses and today keeps its own duplicate declarations. Renaming it after the contract it defines removes that friction. The event-log schema, writer, and reader can then land as a separate ballista-history crate that depends on this one. --- Cargo.lock | 16 +++--- Cargo.toml | 2 +- ballista/{history => api-types}/Cargo.toml | 4 +- ballista/{history => api-types}/src/dto.rs | 0 ballista/api-types/src/lib.rs | 56 ++++++++++++++++++++ ballista/history/src/lib.rs | 60 ---------------------- ballista/scheduler/Cargo.toml | 4 +- ballista/scheduler/src/api/dto_build.rs | 12 ++--- ballista/scheduler/src/api/handlers.rs | 2 +- dev/release/README.md | 4 +- dev/release/crate-deps.dot | 4 +- dev/update_ballista_versions.py | 4 +- 12 files changed, 82 insertions(+), 86 deletions(-) rename ballista/{history => api-types}/Cargo.toml (90%) rename ballista/{history => api-types}/src/dto.rs (100%) create mode 100644 ballista/api-types/src/lib.rs delete mode 100644 ballista/history/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 498f742dfe..697df0a10f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1024,6 +1024,13 @@ dependencies = [ "uuid", ] +[[package]] +name = "ballista-api-types" +version = "54.0.0" +dependencies = [ + "serde", +] + [[package]] name = "ballista-benchmarks" version = "54.0.0" @@ -1200,13 +1207,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "ballista-history" -version = "54.0.0" -dependencies = [ - "serde", -] - [[package]] name = "ballista-scheduler" version = "54.0.0" @@ -1214,8 +1214,8 @@ dependencies = [ "arrow-flight", "async-trait", "axum", + "ballista-api-types", "ballista-core", - "ballista-history", "clap 4.6.3", "dashmap", "datafusion", diff --git a/Cargo.toml b/Cargo.toml index 50ffac073d..15817827b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ members = [ "ballista/client", "ballista/core", "ballista/executor", - "ballista/history", + "ballista/api-types", "ballista/scheduler", "benchmarks", "chaos-testing", diff --git a/ballista/history/Cargo.toml b/ballista/api-types/Cargo.toml similarity index 90% rename from ballista/history/Cargo.toml rename to ballista/api-types/Cargo.toml index a4e56d709c..9c9485e332 100644 --- a/ballista/history/Cargo.toml +++ b/ballista/api-types/Cargo.toml @@ -16,8 +16,8 @@ # under the License. [package] -name = "ballista-history" -description = "Shared REST response types for the Ballista scheduler and history server" +name = "ballista-api-types" +description = "Wire types for the Ballista scheduler REST API" license = "Apache-2.0" version = "54.0.0" homepage = "https://datafusion.apache.org/ballista/" diff --git a/ballista/history/src/dto.rs b/ballista/api-types/src/dto.rs similarity index 100% rename from ballista/history/src/dto.rs rename to ballista/api-types/src/dto.rs diff --git a/ballista/api-types/src/lib.rs b/ballista/api-types/src/lib.rs new file mode 100644 index 0000000000..08fab21c28 --- /dev/null +++ b/ballista/api-types/src/lib.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![warn(missing_docs)] + +//! Wire types for Ballista's scheduler REST API. +//! +//! This crate is the single definition of what `/api/*` puts on the wire. It +//! exists because that contract has more than one party: +//! +//! - **`ballista-scheduler`** serves these responses for live jobs. +//! - **The web TUI** deserializes them to render the cluster. +//! - **A history server** ([#1923]) will serve them for completed jobs, replayed +//! from a durable event log. +//! +//! Each of those independently re-declaring the same structs is how the shapes +//! drift apart, so they share one definition instead. +//! +//! The crate is deliberately serde-only. That keeps it cheap enough for anyone +//! to depend on, and it is what lets the TUI use it from a `wasm32` build. +//! +//! # What belongs here +//! +//! A type belongs here when it is part of the `/api/*` contract and more than +//! one party needs it. Types describing live scheduler internals +//! (`SchedulerStateResponse`, `CancelJobResponse`) stay in `ballista-scheduler`. +//! +//! `ExecutorResponse` stays behind for a different reason: it embeds +//! `ballista-core` types, so sharing it would mean either taking a +//! `ballista-core` dependency here or duplicating those structs. Neither is +//! worth it until something outside the scheduler needs it. +//! +//! Note that the types are shared but the *construction* is not: building a +//! response from a live execution graph needs scheduler internals, so those +//! builders live in `ballista-scheduler`. A history server does not re-derive +//! responses — it replays ones the scheduler already built and stored, so +//! byte-identical output is a structural property rather than two +//! implementations agreeing. +//! +//! [#1923]: https://github.com/apache/datafusion-ballista/issues/1923 + +pub mod dto; diff --git a/ballista/history/src/lib.rs b/ballista/history/src/lib.rs deleted file mode 100644 index 2f814e0fdc..0000000000 --- a/ballista/history/src/lib.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#![warn(missing_docs)] - -//! Shared types for Ballista's scheduler REST API. -//! -//! This crate is the serialization boundary between the live scheduler and the -//! history server: the scheduler builds these types, writes them to an event -//! log, and the history server reads them back and serves them. -//! -//! # Write once, replay verbatim -//! -//! The history server does **not** re-derive responses from stored execution -//! state. When a job finishes, the scheduler runs its DTO builders once against -//! the live execution graph and writes the finished [`dto::JobResponse`] and -//! [`dto::QueryStagesResponse`] into the log's terminal record. Replay -//! deserializes those values and re-serializes them unchanged: -//! -//! ```text -//! scheduler ──> dto builders ──> DTO ──> event log ──> history server ──> JSON -//! ``` -//! -//! So byte-identical output is a structural property, not a convention two -//! implementations have to keep agreeing on. There is one definition of each -//! type and one place that populates it. That is also why the builders -//! themselves stay in `ballista-scheduler`: they need the live execution graph, -//! and nothing on the replay path calls them. -//! -//! One consequence worth knowing: anything not captured when the record is -//! written cannot be recovered later. Plans, for instance, are rendered in a -//! single format at write time, so replay cannot re-render them differently. -//! -//! # What belongs here -//! -//! A type belongs in this crate when it is part of the `/api/*` wire contract -//! *and* is worth persisting in the event log. Types describing live scheduler -//! state (`SchedulerStateResponse`, `CancelJobResponse`) stay in -//! `ballista-scheduler`, because there is nothing to replay. -//! -//! `ExecutorResponse` also stays behind, for a different reason: it embeds -//! `ballista-core` types, and this crate is deliberately serde-only so it -//! stays cheap to depend on. Sharing it would mean either taking a -//! `ballista-core` dependency here or duplicating those structs. - -pub mod dto; diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml index 17ce991bd2..cd26d8d420 100644 --- a/ballista/scheduler/Cargo.toml +++ b/ballista/scheduler/Cargo.toml @@ -41,7 +41,7 @@ disable-stage-plan-cache = [] graphviz-support = ["dep:graphviz-rust"] keda-scaler = ["dep:tonic-prost-build", "dep:tonic-prost"] prometheus-metrics = ["prometheus", "once_cell"] -rest-api = ["dep:ballista-history"] +rest-api = ["dep:ballista-api-types"] spark-compat = ["ballista-core/spark-compat"] substrait = ["dep:datafusion-substrait"] @@ -49,8 +49,8 @@ substrait = ["dep:datafusion-substrait"] arrow-flight = { workspace = true } async-trait = { workspace = true } axum = "0.8.9" +ballista-api-types = { path = "../api-types", version = "54.0.0", optional = true } ballista-core = { path = "../core", version = "54.0.0" } -ballista-history = { path = "../history", version = "54.0.0", optional = true } clap = { workspace = true, optional = true } dashmap = { workspace = true } datafusion = { workspace = true } diff --git a/ballista/scheduler/src/api/dto_build.rs b/ballista/scheduler/src/api/dto_build.rs index 70bafd10e9..d1f343666b 100644 --- a/ballista/scheduler/src/api/dto_build.rs +++ b/ballista/scheduler/src/api/dto_build.rs @@ -19,7 +19,7 @@ //! //! These functions are the only place the scheduler's `ExecutionGraph` / //! `JobOverview` shapes are translated into the wire types in -//! [`ballista_history::dto`]. Keeping the translation here (rather than inline +//! [`ballista_api_types::dto`]. Keeping the translation here (rather than inline //! in the axum handlers) means the same DTOs can be produced from state that //! did not come from a live handler request. //! @@ -31,15 +31,15 @@ use crate::display::format_stage_metrics; use crate::state::execution_graph::{ExecutionGraphBox, ExecutionStage}; use crate::state::execution_stage::TaskInfo; use crate::state::task_manager::JobOverview; +use ballista_api_types::dto::{ + JobResponse, Percentiles, PlanFormat, QueryStageSummary, QueryStagesResponse, + TaskStatus, TaskSummary, +}; use ballista_core::serde::protobuf::failed_task::FailedReason::{ ExecutionError, ExecutorLost, FetchPartitionError, IoError, ResultLost, TaskKilled, }; use ballista_core::serde::protobuf::job_status::Status; use ballista_core::serde::protobuf::{FailedTask, task_status}; -use ballista_history::dto::{ - JobResponse, Percentiles, PlanFormat, QueryStageSummary, QueryStagesResponse, - TaskStatus, TaskSummary, -}; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::displayable; use datafusion::physical_plan::metrics::{MetricsSet, Time}; @@ -228,7 +228,7 @@ fn task_summaries( /// Map a protobuf task status onto the wire enum. /// /// A free function rather than a `From` impl: both types are foreign to this -/// crate now that [`TaskStatus`] lives in `ballista-history`. +/// crate now that [`TaskStatus`] lives in `ballista-api-types`. fn task_status_to_dto(value: &task_status::Status) -> TaskStatus { match value { task_status::Status::Running(_) => TaskStatus::Running, diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index 1c79c7c49c..bd6d77a501 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -21,6 +21,7 @@ use axum::{ extract::{Path, State}, response::{IntoResponse, Response}, }; +use ballista_api_types::dto::{JobResponse, PlanFormat}; use ballista_core::BALLISTA_VERSION; use ballista_core::serde::protobuf::job_status::Status; use ballista_core::serde::protobuf::{ExecutorMetric, executor_metric::Metric}; @@ -28,7 +29,6 @@ use ballista_core::serde::scheduler::{ ExecutorOperatingSystemSpecification, ExecutorSpecification, }; use ballista_core::utils::get_current_time; -use ballista_history::dto::{JobResponse, PlanFormat}; use datafusion::DATAFUSION_VERSION; use datafusion_proto::logical_plan::AsLogicalPlan; use datafusion_proto::physical_plan::AsExecutionPlan; diff --git a/dev/release/README.md b/dev/release/README.md index cdacee9970..c6b0de31c0 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -287,7 +287,7 @@ of the following crates: - [ballista-cli](https://crates.io/crates/ballista-cli) - [ballista-core](https://crates.io/crates/ballista-core) - [ballista-executor](https://crates.io/crates/ballista-executor) -- [ballista-history](https://crates.io/crates/ballista-history) +- [ballista-api-types](https://crates.io/crates/ballista-api-types) - [ballista-scheduler](https://crates.io/crates/ballista-scheduler) Download and unpack the official release tarball @@ -307,7 +307,7 @@ dot -Tsvg dev/release/crate-deps.dot > dev/release/crate-deps.svg ```shell (cd ballista/core && cargo publish) (cd ballista/executor && cargo publish) -(cd ballista/history && cargo publish) +(cd ballista/api-types && cargo publish) (cd ballista/scheduler && cargo publish) (cd ballista/client && cargo publish) (cd ballista-cli && cargo publish) diff --git a/dev/release/crate-deps.dot b/dev/release/crate-deps.dot index 2986ee5251..27bee709ba 100644 --- a/dev/release/crate-deps.dot +++ b/dev/release/crate-deps.dot @@ -18,14 +18,14 @@ digraph G { ballista_core - ballista_history + ballista_api_types ballista_scheduler ballista_executor ballista ballista_cli ballista_scheduler -> ballista_core - ballista_scheduler -> ballista_history + ballista_scheduler -> ballista_api_types ballista_executor -> ballista_core diff --git a/dev/update_ballista_versions.py b/dev/update_ballista_versions.py index 9d29f65482..ca5bef3bf8 100755 --- a/dev/update_ballista_versions.py +++ b/dev/update_ballista_versions.py @@ -43,7 +43,7 @@ def update_cargo_toml(cargo_toml: str, new_version: str): 'ballista', 'ballista-core', 'ballista-executor', - 'ballista-history', + 'ballista-api-types', 'ballista-scheduler', 'ballista-cli', ) @@ -81,7 +81,7 @@ def main(): for rel_path in [ 'ballista-cli', 'ballista/core', - 'ballista/history', + 'ballista/api-types', 'ballista/scheduler', 'ballista/executor', 'ballista/client', From efbb1a99249c7c4eee2abbcd562243a403310443 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 8 Aug 2026 11:13:22 -0600 Subject: [PATCH 5/5] feat: event log and history server Adds a Spark-History-Server equivalent: the scheduler can record a durable per-job event log during execution, and a standalone history server replays those logs and serves the same /api/* responses the live scheduler does, so the existing TUI browses completed jobs with no scheduler running. New ballista-history crate: - A versioned JSONL event schema. JobStart / StageStart / StageEnd / TaskEnd form an incremental timeline; the terminal JobEnd embeds the finished REST responses. - An async buffered EventLogWriter. Timeline events are dropped rather than allowed to block if the queue backs up, so logging cannot stall scheduling. JobEnd is the exception and waits for capacity, since a job missing it is invisible to the history server. - A reader that folds a completed log back into the served payload. Scheduler: - New --event-log-dir flag, off by default. When unset there is no channel, no task, no file, and no per-event work beyond one Option check. - An event-log tee at the top of QueryStageScheduler::on_receive maps JobSubmitted, TaskUpdating, JobFinished, JobRunningFailed and JobCancel onto history events. Cancellation is recorded there because the handler below it drops the graph. - JobEnd renders its stage snapshot as of completed_at rather than the wall clock, so replay is deterministic. History server: - ballista-history-server --event-log-dir loads completed logs and serves /api/* from the stored responses. Corrupt or partial logs are skipped rather than failing startup. Because JobEnd stores responses the scheduler already built, replay never re-derives them and there is no second implementation to drift. A test asserts the history server serves byte-identical JSON to the live scheduler for the same graph. get_job_config now returns the sorted JobConfig map so live and replayed output agree on key order. Scope for this first cut: local filesystem storage, completed jobs only. The TaskEnd timeline is captured but not yet surfaced in a UI. --- Cargo.lock | 15 + Cargo.toml | 1 + ballista/api-types/src/dto.rs | 8 + ballista/history/Cargo.toml | 38 ++ ballista/history/src/event.rs | 174 ++++++++ ballista/history/src/lib.rs | 52 +++ ballista/history/src/reader.rs | 139 ++++++ ballista/history/src/writer.rs | 313 ++++++++++++++ ballista/scheduler/Cargo.toml | 11 +- ballista/scheduler/src/api/dto_build.rs | 53 ++- ballista/scheduler/src/api/handlers.rs | 2 +- ballista/scheduler/src/api/mod.rs | 4 +- ballista/scheduler/src/bin/history_server.rs | 98 +++++ ballista/scheduler/src/config.rs | 13 + ballista/scheduler/src/history/mod.rs | 370 ++++++++++++++++ ballista/scheduler/src/lib.rs | 3 + .../src/scheduler_server/event_log.rs | 408 ++++++++++++++++++ .../scheduler/src/scheduler_server/mod.rs | 25 ++ .../scheduler_server/query_stage_scheduler.rs | 153 +++++++ .../src/state/execution_graph_dot.rs | 4 +- dev/release/README.md | 2 + dev/release/crate-deps.dot | 4 + dev/update_ballista_versions.py | 2 + docs/source/index.rst | 1 + docs/source/user-guide/history-server.md | 109 +++++ 25 files changed, 1993 insertions(+), 9 deletions(-) create mode 100644 ballista/history/Cargo.toml create mode 100644 ballista/history/src/event.rs create mode 100644 ballista/history/src/lib.rs create mode 100644 ballista/history/src/reader.rs create mode 100644 ballista/history/src/writer.rs create mode 100644 ballista/scheduler/src/bin/history_server.rs create mode 100644 ballista/scheduler/src/history/mod.rs create mode 100644 ballista/scheduler/src/scheduler_server/event_log.rs create mode 100644 docs/source/user-guide/history-server.md diff --git a/Cargo.lock b/Cargo.lock index 697df0a10f..a3da232e12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1207,6 +1207,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "ballista-history" +version = "54.0.0" +dependencies = [ + "ballista-api-types", + "log", + "serde", + "serde_json", + "tempfile", + "tokio", +] + [[package]] name = "ballista-scheduler" version = "54.0.0" @@ -1216,6 +1228,7 @@ dependencies = [ "axum", "ballista-api-types", "ballista-core", + "ballista-history", "clap 4.6.3", "dashmap", "datafusion", @@ -1239,11 +1252,13 @@ dependencies = [ "rstest", "serde", "serde_json", + "tempfile", "tokio", "tokio-stream", "tonic", "tonic-prost", "tonic-prost-build", + "tower", "tower-http 0.7.0", "tracing", "tracing-appender", diff --git a/Cargo.toml b/Cargo.toml index 15817827b1..3a7e1966cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "ballista/client", "ballista/core", "ballista/executor", + "ballista/history", "ballista/api-types", "ballista/scheduler", "benchmarks", diff --git a/ballista/api-types/src/dto.rs b/ballista/api-types/src/dto.rs index 272730af84..094d8b6ddc 100644 --- a/ballista/api-types/src/dto.rs +++ b/ballista/api-types/src/dto.rs @@ -19,6 +19,7 @@ //! server, so both serialize byte-identical JSON. use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; /// Summary of one job, served by `GET /api/jobs` and `GET /api/job/{job_id}`. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -164,3 +165,10 @@ pub enum PlanFormat { /// `?plan_format=metrics` => indent with aggregated metrics Metrics, } + +/// Session config for one job, as flat key/value pairs. Served by +/// `GET /api/job/{job_id}/config`. +/// +/// A `BTreeMap` so key order is deterministic: the same job must serialize +/// identically whether it is served live or replayed from a stored log. +pub type JobConfig = BTreeMap; diff --git a/ballista/history/Cargo.toml b/ballista/history/Cargo.toml new file mode 100644 index 0000000000..ecf4713669 --- /dev/null +++ b/ballista/history/Cargo.toml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "ballista-history" +description = "Event-log schema, writer, and reader for the Ballista history server" +license = "Apache-2.0" +version = "54.0.0" +homepage = "https://datafusion.apache.org/ballista/" +repository = "https://github.com/apache/datafusion-ballista" +authors = ["Apache DataFusion "] +edition = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +ballista-api-types = { path = "../api-types", version = "54.0.0" } +log = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = "1" +tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt", "sync"] } + +[dev-dependencies] +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/ballista/history/src/event.rs b/ballista/history/src/event.rs new file mode 100644 index 0000000000..3f848ce509 --- /dev/null +++ b/ballista/history/src/event.rs @@ -0,0 +1,174 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The on-disk event-log schema. One `HistoryEvent` is serialized per JSONL line. +//! This is a frozen public projection of the scheduler's internal events; the +//! embedded DTOs are the stable contract the history server serves. + +use ballista_api_types::dto::{JobConfig, JobResponse, QueryStagesResponse, TaskStatus}; +use serde::{Deserialize, Serialize}; + +/// Current on-disk schema version, stamped on `JobStart`/`JobEnd`. +pub const SCHEMA_VERSION: u32 = 1; + +/// How a job reached its terminal state. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum JobEndStatus { + /// Job completed successfully. + Succeeded, + /// Job terminated with the given error. + Failed(String), + /// Job was cancelled before completing. + Cancelled, +} + +/// Metrics captured per finished task on the incremental timeline. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TaskEndMetrics { + /// Rows the task read. + pub input_rows: u64, + /// Rows the task produced. + pub output_rows: u64, + /// Time the task spent computing, in nanoseconds. + pub elapsed_compute_nanos: u64, +} + +/// A single record in a job's event log. +/// +/// Stage and partition identifiers are fixed-width (`u32`) throughout rather +/// than `usize`: this is a durable, cross-machine format, so a log written by a +/// 64-bit scheduler must mean the same thing to any reader. Callers holding the +/// scheduler's own `usize` stage ids cast on the way in. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "ev")] +pub enum HistoryEvent { + /// First record in every log. Written when the scheduler accepts the job. + JobStart { + /// Schema version, see [`SCHEMA_VERSION`]. + version: u32, + /// Job identifier, matching the log's filename. + job_id: String, + /// Human-readable job name. + job_name: String, + /// When the job entered the queue. + queued_at: u64, + /// When the job was submitted for planning. + submitted_at: u64, + /// Rendered logical plan, if one was captured. + logical_plan: Option, + /// Rendered physical plan, if one was captured. + physical_plan: Option, + }, + /// A stage became runnable. + StageStart { + /// Stage identifier within the job. + stage_id: u32, + /// Number of partitions the stage will produce. + partitions: u32, + }, + /// A stage reached a terminal state. + StageEnd { + /// Stage identifier within the job. + stage_id: u32, + /// Terminal stage status. + status: String, + }, + /// A task finished, successfully or otherwise. + TaskEnd { + /// Stage the task belonged to. + stage_id: u32, + /// Task's slot within the stage. Under the multi-partition-task model a + /// task owns a slice of partitions, so this names the task rather than + /// a single partition. + task_id: u32, + /// Executor that ran the task. + executor_id: String, + /// Outcome of the task. + status: TaskStatus, + /// When the scheduler launched the task. + launch_time: u64, + /// When the executor began running it. + start_exec_time: u64, + /// When the executor finished it. + end_exec_time: u64, + /// Row counts and compute time for the task. + metrics: TaskEndMetrics, + }, + /// Terminal record, and the only one the history server serves from. It + /// carries the finished REST responses so replay never re-derives them. + JobEnd { + /// Schema version, see [`SCHEMA_VERSION`]. + version: u32, + /// How the job ended. + status: JobEndStatus, + /// When the job entered the queue. + queued_at: u64, + /// When the job started executing. + started_at: u64, + /// When the job reached its terminal state. + completed_at: u64, + /// Finished `GET /api/job/{job_id}` response. + job: Box, + /// Finished `GET /api/job/{job_id}/stages` response. + stages: Box, + /// Finished `GET /api/job/{job_id}/config` response. + config: JobConfig, + /// Rendered DOT graph of the stage DAG. + dot: String, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + use ballista_api_types::dto::{JobResponse, QueryStagesResponse}; + use std::collections::BTreeMap; + + #[test] + fn job_end_round_trips_through_jsonl() { + let job = JobResponse { + job_id: "job-1".into(), + job_name: "q1".into(), + job_status: "COMPLETED".into(), + status: "Successful".into(), + num_stages: 2, + completed_stages: 2, + percent_complete: 100, + start_time: 10, + end_time: 20, + logical_plan: Some("Projection".into()), + physical_plan: Some("ProjectionExec".into()), + stage_plan: Some("stage plan".into()), + }; + let event = HistoryEvent::JobEnd { + version: SCHEMA_VERSION, + status: JobEndStatus::Succeeded, + queued_at: 5, + started_at: 10, + completed_at: 20, + job: Box::new(job), + stages: Box::new(QueryStagesResponse { stages: vec![] }), + config: BTreeMap::from([("k".to_string(), "v".to_string())]), + dot: "digraph {}".into(), + }; + let line = serde_json::to_string(&event).unwrap(); + assert!(line.contains("\"ev\":\"JobEnd\"")); + let back: HistoryEvent = serde_json::from_str(&line).unwrap(); + // Re-serialize and compare strings (stable, discriminating). + assert_eq!(line, serde_json::to_string(&back).unwrap()); + } +} diff --git a/ballista/history/src/lib.rs b/ballista/history/src/lib.rs new file mode 100644 index 0000000000..156935e825 --- /dev/null +++ b/ballista/history/src/lib.rs @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![warn(missing_docs)] + +//! Durable event logs for completed Ballista jobs. +//! +//! The scheduler appends one JSONL record per event to +//! `/.eventlog` while a job runs. The history server +//! reads those files back and serves the same `/api/*` responses the live +//! scheduler does, long after the scheduler has forgotten the job. +//! +//! # Write once, replay verbatim +//! +//! The terminal [`event::HistoryEvent::JobEnd`] record embeds the finished +//! [`ballista_api_types::dto::JobResponse`] and +//! [`ballista_api_types::dto::QueryStagesResponse`] the scheduler built from +//! its live execution graph. Replay deserializes those values and re-serializes +//! them unchanged, so the history server never re-derives a response and there +//! is no second implementation to drift. +//! +//! The earlier records ([`event::HistoryEvent::JobStart`], `StageStart`, +//! `StageEnd`, `TaskEnd`) form an incremental timeline. Nothing reads them yet; +//! they exist so a future UI can show a job progressing rather than only its +//! final state. +//! +//! # Durability +//! +//! [`writer::EventLogWriter`] does all file I/O on a background task, so the +//! scheduler's event loop never waits on a disk write. Timeline events are +//! dropped rather than allowed to block if the queue backs up, on the grounds +//! that losing a progress record is better than stalling scheduling. The +//! terminal `JobEnd` is the exception: it waits for queue capacity, because a +//! job missing its `JobEnd` is invisible to the history server. + +pub mod event; +pub mod reader; +pub mod writer; diff --git a/ballista/history/src/reader.rs b/ballista/history/src/reader.rs new file mode 100644 index 0000000000..bf7a87d805 --- /dev/null +++ b/ballista/history/src/reader.rs @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Reads a completed `.eventlog` into the DTO bundle the history server +//! serves. A file is "completed" once it contains a `JobEnd` record. + +use crate::event::HistoryEvent; +use ballista_api_types::dto::{JobConfig, JobResponse, QueryStagesResponse}; +use std::io::BufRead; +use std::path::Path; + +/// The served payload recovered from a completed job's event log. +#[derive(Debug, Clone)] +pub struct ReplayedJob { + /// `GET /api/job/{job_id}` response. + pub job: JobResponse, + /// `GET /api/job/{job_id}/stages` response. + pub stages: QueryStagesResponse, + /// `GET /api/job/{job_id}/config` response. + pub config: JobConfig, + /// Rendered DOT graph of the stage DAG. + pub dot: String, +} + +/// Read a completed job's payload out of its event log. +/// +/// Returns `Ok(None)` when the file has no `JobEnd` record, which means the job +/// is still running or the scheduler died before finishing it. Malformed lines +/// are skipped rather than treated as fatal, so a partially-written log still +/// yields its job if the terminal record survived. +pub fn read_completed_job(path: &Path) -> std::io::Result> { + let file = std::fs::File::open(path)?; + let reader = std::io::BufReader::new(file); + for line in reader.lines() { + let line = line?; + if line.is_empty() { + continue; + } + // Only JobEnd carries the served payload; other lines are the timeline + // and are ignored here. Unknown/garbled lines are skipped, not fatal. + if let Ok(HistoryEvent::JobEnd { + job, + stages, + config, + dot, + .. + }) = serde_json::from_str::(&line) + { + return Ok(Some(ReplayedJob { + job: *job, + stages: *stages, + config, + dot, + })); + } + } + Ok(None) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event::{HistoryEvent, JobEndStatus, SCHEMA_VERSION}; + use ballista_api_types::dto::{JobResponse, QueryStagesResponse}; + use std::io::Write; + + fn job_end_line() -> String { + let event = HistoryEvent::JobEnd { + version: SCHEMA_VERSION, + status: JobEndStatus::Succeeded, + queued_at: 1, + started_at: 2, + completed_at: 3, + job: Box::new(JobResponse { + job_id: "job-1".into(), + job_name: "q1".into(), + job_status: "COMPLETED".into(), + status: "Successful".into(), + num_stages: 1, + completed_stages: 1, + percent_complete: 100, + start_time: 2, + end_time: 3, + logical_plan: Some("Projection".into()), + physical_plan: Some("ProjectionExec".into()), + stage_plan: Some("stage".into()), + }), + stages: Box::new(QueryStagesResponse { stages: vec![] }), + config: Default::default(), + dot: "digraph {}".into(), + }; + serde_json::to_string(&event).unwrap() + } + + #[test] + fn reads_job_end_and_ignores_unknown_timeline_lines() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("job-1.eventlog"); + let mut f = std::fs::File::create(&path).unwrap(); + // Unknown/other event lines before the JobEnd must be tolerated. + writeln!(f, r#"{{"ev":"StageStart","stage_id":1,"partitions":4}}"#).unwrap(); + writeln!(f, "{}", job_end_line()).unwrap(); + drop(f); + + let replayed = read_completed_job(&path).unwrap().expect("completed"); + assert_eq!(replayed.job.job_id, "job-1"); + assert_eq!( + replayed.job.physical_plan.as_deref(), + Some("ProjectionExec") + ); + assert_eq!(replayed.dot, "digraph {}"); + } + + #[test] + fn returns_none_when_no_job_end() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("job-2.eventlog"); + std::fs::write( + &path, + "{\"ev\":\"StageStart\",\"stage_id\":1,\"partitions\":4}\n", + ) + .unwrap(); + assert!(read_completed_job(&path).unwrap().is_none()); + } +} diff --git a/ballista/history/src/writer.rs b/ballista/history/src/writer.rs new file mode 100644 index 0000000000..de0e798fea --- /dev/null +++ b/ballista/history/src/writer.rs @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Async, buffered event-log writer. Each job's events append to +//! `/.eventlog` as JSONL. Appends are non-blocking; a background +//! task performs the file I/O so the scheduler hot path never waits on disk. + +use crate::event::HistoryEvent; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use tokio::io::AsyncWriteExt; +use tokio::sync::{mpsc, oneshot}; + +enum WriterMsg { + Event { + job_id: String, + event: Box, + }, + Flush { + job_id: String, + done: oneshot::Sender<()>, + }, + Finish { + job_id: String, + done: oneshot::Sender<()>, + }, +} + +/// Handle to the background event-log writer task. +/// +/// Cloning is cheap and every clone feeds the same task, so the scheduler can +/// hand one to each component that needs to record events. +#[derive(Clone)] +pub struct EventLogWriter { + tx: mpsc::Sender, +} + +impl EventLogWriter { + /// Spawn the writer task, appending logs under `log_dir`. + /// + /// `buffer` bounds the in-flight event queue; beyond it, timeline events are + /// dropped rather than allowed to stall the caller. + pub fn new(log_dir: PathBuf, buffer: usize) -> EventLogWriter { + let (tx, rx) = mpsc::channel(buffer.max(1)); + tokio::spawn(run(log_dir, rx)); + EventLogWriter { tx } + } + + /// Enqueue an event for `job_id`. Never blocks; drops (with a warning) if the + /// channel is full, so logging cannot stall scheduling. + pub fn append(&self, job_id: &str, event: HistoryEvent) { + let msg = WriterMsg::Event { + job_id: job_id.to_string(), + event: Box::new(event), + }; + if self.tx.try_send(msg).is_err() { + log::warn!( + "event-log writer: dropping event for {job_id} (channel full or closed)" + ); + } + } + + /// Await all currently-enqueued writes for `job_id` (best effort). + pub async fn flush_job(&self, job_id: &str) { + let (done, wait) = oneshot::channel(); + if self + .tx + .send(WriterMsg::Flush { + job_id: job_id.to_string(), + done, + }) + .await + .is_ok() + { + let _ = wait.await; + } + } + + /// Enqueue a terminal event (e.g. `JobEnd`) for `job_id`. Unlike `append`, this + /// awaits channel capacity instead of dropping the event when the channel is + /// full, so the terminal record is never silently lost. Still best-effort at + /// the process boundary: if the channel is closed (background task gone) this + /// logs and returns rather than panicking. + pub async fn append_final(&self, job_id: &str, event: HistoryEvent) { + let msg = WriterMsg::Event { + job_id: job_id.to_string(), + event: Box::new(event), + }; + if self.tx.send(msg).await.is_err() { + log::warn!( + "event-log writer: failed to enqueue terminal event for {job_id} (channel closed)" + ); + } + } + + /// Flush and close the per-job file handle for `job_id`. Must be called after + /// the terminal event has been enqueued (e.g. via `append_final`) so it is + /// ordered after it on the single-consumer FIFO channel. Best effort: if the + /// channel is closed this logs and returns. + pub async fn finish_job(&self, job_id: &str) { + let (done, wait) = oneshot::channel(); + if self + .tx + .send(WriterMsg::Finish { + job_id: job_id.to_string(), + done, + }) + .await + .is_ok() + { + let _ = wait.await; + } else { + log::warn!( + "event-log writer: failed to enqueue finish for {job_id} (channel closed)" + ); + } + } +} + +async fn run(log_dir: PathBuf, mut rx: mpsc::Receiver) { + if let Err(e) = tokio::fs::create_dir_all(&log_dir).await { + log::warn!("event-log writer: cannot create {}: {e}", log_dir.display()); + return; + } + // One open append handle per job for the life of the process. + let mut handles: HashMap = HashMap::new(); + + while let Some(msg) = rx.recv().await { + match msg { + WriterMsg::Event { job_id, event } => { + let file = match open_for(&log_dir, &mut handles, &job_id).await { + Some(f) => f, + None => continue, + }; + match serde_json::to_string(&*event) { + Ok(mut line) => { + line.push('\n'); + if let Err(e) = file.write_all(line.as_bytes()).await { + log::warn!( + "event-log writer: write failed for {job_id}: {e}" + ); + } + } + Err(e) => log::warn!("event-log writer: serialize failed: {e}"), + } + } + WriterMsg::Flush { job_id, done } => { + if let Some(file) = handles.get_mut(&job_id) { + let _ = file.flush().await; + } + let _ = done.send(()); + } + WriterMsg::Finish { job_id, done } => { + if let Some(mut file) = handles.remove(&job_id) { + let _ = file.flush().await; + // Dropping `file` here closes the fd. + } + let _ = done.send(()); + } + } + } +} + +async fn open_for<'a>( + log_dir: &Path, + handles: &'a mut HashMap, + job_id: &str, +) -> Option<&'a mut tokio::fs::File> { + if !handles.contains_key(job_id) { + let path = log_dir.join(format!("{job_id}.eventlog")); + match tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .await + { + Ok(f) => { + handles.insert(job_id.to_string(), f); + } + Err(e) => { + log::warn!("event-log writer: cannot open {}: {e}", path.display()); + return None; + } + } + } + handles.get_mut(job_id) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event::{HistoryEvent, JobEndStatus, SCHEMA_VERSION}; + use ballista_api_types::dto::{JobResponse, QueryStagesResponse}; + use std::collections::BTreeMap; + + #[tokio::test] + async fn terminal_job_end_is_not_dropped_on_a_saturated_channel() { + let dir = tempfile::tempdir().unwrap(); + // Tiny buffer so the non-blocking `append` path would readily drop events + // under load; `append_final` must still guarantee delivery. + let writer = EventLogWriter::new(dir.path().to_path_buf(), 1); + + writer.append( + "job-1", + HistoryEvent::JobStart { + version: SCHEMA_VERSION, + job_id: "job-1".into(), + job_name: "q1".into(), + queued_at: 1, + submitted_at: 2, + logical_plan: None, + physical_plan: None, + }, + ); + for stage_id in 0..10 { + writer.append( + "job-1", + HistoryEvent::StageStart { + stage_id, + partitions: 4, + }, + ); + } + + let job = JobResponse { + job_id: "job-1".into(), + job_name: "q1".into(), + job_status: "COMPLETED".into(), + status: "Successful".into(), + num_stages: 2, + completed_stages: 2, + percent_complete: 100, + start_time: 10, + end_time: 20, + logical_plan: Some("Projection".into()), + physical_plan: Some("ProjectionExec".into()), + stage_plan: Some("stage plan".into()), + }; + let job_end = HistoryEvent::JobEnd { + version: SCHEMA_VERSION, + status: JobEndStatus::Succeeded, + queued_at: 1, + started_at: 2, + completed_at: 20, + job: Box::new(job), + stages: Box::new(QueryStagesResponse { stages: vec![] }), + config: BTreeMap::new(), + dot: "digraph {}".into(), + }; + writer.append_final("job-1", job_end).await; + writer.finish_job("job-1").await; + + let path = dir.path().join("job-1.eventlog"); + let contents = tokio::fs::read_to_string(&path).await.unwrap(); + let lines: Vec<&str> = contents.lines().collect(); + assert!( + lines.iter().any(|l| l.contains("\"ev\":\"JobEnd\"")), + "expected a JobEnd line in the event log, got: {contents}" + ); + assert_eq!( + lines.last().map(|l| l.contains("\"ev\":\"JobEnd\"")), + Some(true), + "JobEnd should be the last line written" + ); + } + + #[tokio::test] + async fn append_writes_one_jsonl_line_per_event() { + let dir = tempfile::tempdir().unwrap(); + let writer = EventLogWriter::new(dir.path().to_path_buf(), 16); + writer.append( + "job-1", + HistoryEvent::JobStart { + version: SCHEMA_VERSION, + job_id: "job-1".into(), + job_name: "q1".into(), + queued_at: 1, + submitted_at: 2, + logical_plan: None, + physical_plan: None, + }, + ); + writer.append( + "job-1", + HistoryEvent::StageStart { + stage_id: 1, + partitions: 4, + }, + ); + writer.flush_job("job-1").await; + + let path = dir.path().join("job-1.eventlog"); + let contents = tokio::fs::read_to_string(&path).await.unwrap(); + let lines: Vec<&str> = contents.lines().collect(); + assert_eq!(lines.len(), 2); + assert!(lines[0].contains("\"ev\":\"JobStart\"")); + assert!(lines[1].contains("\"ev\":\"StageStart\"")); + } +} diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml index cd26d8d420..a44eeb5c9c 100644 --- a/ballista/scheduler/Cargo.toml +++ b/ballista/scheduler/Cargo.toml @@ -32,6 +32,11 @@ name = "ballista-scheduler" path = "src/bin/main.rs" required-features = ["build-binary"] +[[bin]] +name = "ballista-history-server" +path = "src/bin/history_server.rs" +required-features = ["build-binary", "rest-api"] + [features] build-binary = ["clap", "tracing-subscriber", "tracing-appender", "tracing", "ballista-core/build-binary"] default = ["build-binary", "rest-api"] @@ -41,7 +46,7 @@ disable-stage-plan-cache = [] graphviz-support = ["dep:graphviz-rust"] keda-scaler = ["dep:tonic-prost-build", "dep:tonic-prost"] prometheus-metrics = ["prometheus", "once_cell"] -rest-api = ["dep:ballista-api-types"] +rest-api = ["dep:ballista-api-types", "dep:ballista-history", "dep:serde_json"] spark-compat = ["ballista-core/spark-compat"] substrait = ["dep:datafusion-substrait"] @@ -51,6 +56,7 @@ async-trait = { workspace = true } axum = "0.8.9" ballista-api-types = { path = "../api-types", version = "54.0.0", optional = true } ballista-core = { path = "../core", version = "54.0.0" } +ballista-history = { path = "../history", version = "54.0.0", optional = true } clap = { workspace = true, optional = true } dashmap = { workspace = true } datafusion = { workspace = true } @@ -70,6 +76,7 @@ prost = { workspace = true } prost-types = { workspace = true } rand = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde_json = { version = "1", optional = true } tokio = { workspace = true, features = ["full"] } tokio-stream = { workspace = true, features = ["net"] } tonic = { workspace = true, features = ["router"] } @@ -90,6 +97,8 @@ datafusion-functions-aggregate-common = { workspace = true } regex = "1" rstest = { workspace = true } serde_json = "1" +tempfile = { workspace = true } +tower = "0.5" [build-dependencies] tonic-prost-build = { workspace = true, optional = true } diff --git a/ballista/scheduler/src/api/dto_build.rs b/ballista/scheduler/src/api/dto_build.rs index d1f343666b..eb6f0c6ac3 100644 --- a/ballista/scheduler/src/api/dto_build.rs +++ b/ballista/scheduler/src/api/dto_build.rs @@ -29,17 +29,19 @@ use crate::display::format_stage_metrics; use crate::state::execution_graph::{ExecutionGraphBox, ExecutionStage}; +use crate::state::execution_graph_dot::ExecutionGraphDot; use crate::state::execution_stage::TaskInfo; use crate::state::task_manager::JobOverview; use ballista_api_types::dto::{ - JobResponse, Percentiles, PlanFormat, QueryStageSummary, QueryStagesResponse, - TaskStatus, TaskSummary, + JobConfig, JobResponse, Percentiles, PlanFormat, QueryStageSummary, + QueryStagesResponse, TaskStatus, TaskSummary, }; use ballista_core::serde::protobuf::failed_task::FailedReason::{ ExecutionError, ExecutorLost, FetchPartitionError, IoError, ResultLost, TaskKilled, }; use ballista_core::serde::protobuf::job_status::Status; -use ballista_core::serde::protobuf::{FailedTask, task_status}; +use ballista_core::serde::protobuf::{FailedTask, OperatorMetricsSet, task_status}; +use datafusion::execution::context::SessionConfig; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::displayable; use datafusion::physical_plan::metrics::{MetricsSet, Time}; @@ -115,6 +117,12 @@ pub fn graph_to_job_response( } } +/// Flatten a session config into the sorted key/value map served by +/// `GET /api/job/{job_id}/config`. +pub fn session_config_to_job_config(config: &SessionConfig) -> JobConfig { + config.to_props().into_iter().collect() +} + /// Build the per-stage summaries served by `GET /api/job/{job_id}/stages`. pub fn graph_to_query_stages( graph: &ExecutionGraphBox, @@ -225,11 +233,48 @@ fn task_summaries( .collect() } +/// Render a job's stage DAG in DOT format. +pub fn build_job_dot(graph: &ExecutionGraphBox) -> Result { + ExecutionGraphDot::generate(graph.as_ref()) +} + +/// Sum one task's raw operator metrics into +/// `(input_rows, output_rows, elapsed_compute_nanos)`. +/// +/// Distinct from [`get_partition_counts`], which reads a stage's already-merged +/// [`MetricsSet`]s and filters by partition. This takes the raw protobuf +/// [`OperatorMetricsSet`]s an executor reports for a single task, so there is no +/// partition to filter on, and it also sums `elapsed_compute` for the event +/// log's per-task timeline records. +pub fn task_row_counts(metrics: &[OperatorMetricsSet]) -> (u64, u64, u64) { + let mut input_rows: u64 = 0; + let mut output_rows: u64 = 0; + let mut elapsed_compute_nanos: u64 = 0; + + for operator_metrics in metrics { + let Ok(metrics_set) = TryInto::::try_into(operator_metrics.clone()) + else { + continue; + }; + for metric in metrics_set.iter() { + let value = metric.value(); + match value.name() { + "input_rows" => input_rows += value.as_usize() as u64, + "output_rows" => output_rows += value.as_usize() as u64, + "elapsed_compute" => elapsed_compute_nanos += value.as_usize() as u64, + _ => {} + } + } + } + + (input_rows, output_rows, elapsed_compute_nanos) +} + /// Map a protobuf task status onto the wire enum. /// /// A free function rather than a `From` impl: both types are foreign to this /// crate now that [`TaskStatus`] lives in `ballista-api-types`. -fn task_status_to_dto(value: &task_status::Status) -> TaskStatus { +pub fn task_status_to_dto(value: &task_status::Status) -> TaskStatus { match value { task_status::Status::Running(_) => TaskStatus::Running, task_status::Status::Failed(failed) => TaskStatus::Failed { diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index bd6d77a501..ff24a5d861 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -521,7 +521,7 @@ pub async fn get_job_config< .task_manager .get_job_config(&job_id.clone().into()) .await - .map(|e| Json(e.to_props())) + .map(|e| Json(dto_build::session_config_to_job_config(&e))) .map_err(|_| SchedulerErrorResponse::new(StatusCode::NOT_FOUND)) } diff --git a/ballista/scheduler/src/api/mod.rs b/ballista/scheduler/src/api/mod.rs index 527e7610b8..1e865d84b2 100644 --- a/ballista/scheduler/src/api/mod.rs +++ b/ballista/scheduler/src/api/mod.rs @@ -11,7 +11,9 @@ // limitations under the License. #[cfg(feature = "rest-api")] -mod dto_build; +// `pub(crate)` so `scheduler_server::event_log` can build the same DTOs the +// live REST API serves. +pub(crate) mod dto_build; #[cfg(feature = "rest-api")] mod handlers; mod health; diff --git a/ballista/scheduler/src/bin/history_server.rs b/ballista/scheduler/src/bin/history_server.rs new file mode 100644 index 0000000000..212df58174 --- /dev/null +++ b/ballista/scheduler/src/bin/history_server.rs @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Standalone Ballista history server binary: loads completed event logs from +//! a directory and serves the same `/api/*` responses the live scheduler does, +//! so the existing TUI can connect to it unchanged. + +use ballista_core::error::{BallistaError, Result}; +use ballista_scheduler::history::{HistoryStore, history_router}; +use clap::Parser; +use std::env; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use tracing_subscriber::EnvFilter; + +#[derive(Debug, clap::Parser)] +#[command( + name = "ballista-history-server", + version, + about = "Ballista history server" +)] +struct Args { + /// Directory containing per-job event logs. + #[arg(long)] + event_log_dir: PathBuf, + /// Host to bind the HTTP server to. + #[arg(long, default_value = "0.0.0.0")] + bind_host: String, + /// Port to bind the HTTP server to. + #[arg(long, default_value_t = 50060)] + bind_port: u16, +} + +fn main() -> Result<()> { + let rust_log = env::var(EnvFilter::DEFAULT_ENV); + let log_filter = EnvFilter::new(rust_log.unwrap_or_else(|_| "info".to_string())); + tracing_subscriber::fmt() + .with_ansi(false) + .with_writer(std::io::stdout) + .with_env_filter(log_filter) + .init(); + + let args = Args::parse(); + + // `HistoryStore::load` walks the log directory with blocking file I/O, and + // how long it takes scales with the number of stored jobs. Run it here, + // before the runtime exists, rather than parking a runtime worker on it. + let store = Arc::new(HistoryStore::load(&args.event_log_dir)?); + tracing::info!( + "Loaded {} completed job(s) from {}", + store.jobs.len(), + args.event_log_dir.display() + ); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_io() + .enable_time() + .build() + .map_err(BallistaError::IoError)?; + + runtime.block_on(serve(args, store)) +} + +async fn serve(args: Args, store: Arc) -> Result<()> { + let app = history_router(store); + + let addr: SocketAddr = format!("{}:{}", args.bind_host, args.bind_port) + .parse() + .map_err(|e: std::net::AddrParseError| { + BallistaError::Configuration(e.to_string()) + })?; + + let listener = tokio::net::TcpListener::bind(&addr) + .await + .map_err(BallistaError::IoError)?; + tracing::info!("History server listening on http://{addr}"); + + axum::serve(listener, app.into_make_service()) + .await + .map_err(BallistaError::IoError)?; + + Ok(()) +} diff --git a/ballista/scheduler/src/config.rs b/ballista/scheduler/src/config.rs index c24face929..558dfb40bb 100644 --- a/ballista/scheduler/src/config.rs +++ b/ballista/scheduler/src/config.rs @@ -147,6 +147,9 @@ pub struct Config { help = "Log dir: a path to save log. This will create a new storage directory at the specified path if it does not already exist." )] pub log_dir: Option, + /// Directory to write per-job event logs to. Enables the history server. + #[arg(long)] + pub event_log_dir: Option, /// Whether to print thread IDs and names in log files. #[arg( long, @@ -365,6 +368,8 @@ pub struct SchedulerConfig { #[cfg(feature = "rest-api")] /// Comma-separated list of allowed methods for CORS pub cors_allowed_methods: String, + /// Directory to write per-job event logs to. `None` disables event logging. + pub event_log_dir: Option, #[cfg(feature = "rest-api")] /// The HTTP path that will redirect to the WebTUI app at `https://nightlies.apache.org` pub web_tui_route: String, @@ -410,6 +415,7 @@ impl Default for SchedulerConfig { cors_allowed_origins: String::default(), #[cfg(feature = "rest-api")] cors_allowed_methods: String::default(), + event_log_dir: None, #[cfg(feature = "rest-api")] web_tui_route: String::from("/"), on_work_available: None, @@ -556,6 +562,12 @@ impl SchedulerConfig { self } + /// Sets the directory to write per-job event logs to. + pub fn with_event_log_dir(mut self, event_log_dir: Option) -> Self { + self.event_log_dir = event_log_dir; + self + } + /// Sets whether TLS should be used when connecting to executors (for flight proxy). pub fn with_use_tls(mut self, use_tls: bool) -> Self { self.use_tls = use_tls; @@ -671,6 +683,7 @@ impl TryFrom for SchedulerConfig { cors_allowed_origins: opt.cors_allowed_origins, #[cfg(feature = "rest-api")] cors_allowed_methods: opt.cors_allowed_methods, + event_log_dir: opt.event_log_dir, #[cfg(feature = "rest-api")] web_tui_route: opt.web_tui_route, on_work_available: None, diff --git a/ballista/scheduler/src/history/mod.rs b/ballista/scheduler/src/history/mod.rs new file mode 100644 index 0000000000..bad65a0a7a --- /dev/null +++ b/ballista/scheduler/src/history/mod.rs @@ -0,0 +1,370 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Standalone history server: loads completed event logs and serves the same +//! `/api/*` responses the live scheduler does, from stored DTOs. + +use crate::api::SchedulerErrorResponse; +use axum::{ + Json, Router, + extract::{Path as AxumPath, State}, + routing::get, +}; +use ballista_api_types::dto::{JobConfig, JobResponse, QueryStagesResponse}; +use ballista_core::BALLISTA_VERSION; +use ballista_history::reader::{ReplayedJob, read_completed_job}; +use datafusion::DATAFUSION_VERSION; +use http::StatusCode; +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; + +/// In-memory store of completed jobs, loaded once at startup from a directory +/// of `.eventlog` files. +#[derive(Default)] +pub struct HistoryStore { + /// Completed jobs keyed by job id. + pub jobs: HashMap, +} + +impl HistoryStore { + /// Load every completed job found under `dir`. Missing directories yield + /// an empty store rather than an error. + /// + /// A single unreadable/corrupt `.eventlog` file (e.g. truncated by a + /// crash mid-write) is logged and skipped rather than failing the whole + /// load — one bad log must not hide every other completed job. Only a + /// failure to read the directory itself is propagated. + pub fn load(dir: &Path) -> std::io::Result { + let mut jobs = HashMap::new(); + if dir.exists() { + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + if path.extension().and_then(|e| e.to_str()) != Some("eventlog") { + continue; + } + match read_completed_job(&path) { + Ok(Some(replayed)) => { + jobs.insert(replayed.job.job_id.clone(), replayed); + } + Ok(None) => {} + Err(err) => { + tracing::warn!( + "skipping unreadable event log {}: {err}", + path.display() + ); + } + } + } + } + Ok(HistoryStore { jobs }) + } +} + +/// Build the axum router serving `/api/*` from a loaded [`HistoryStore`]. +pub fn history_router(store: Arc) -> Router { + Router::new() + .route("/api/jobs", get(get_jobs)) + .route("/api/job/{job_id}", get(get_job)) + .route("/api/job/{job_id}/stages", get(get_stages)) + .route("/api/job/{job_id}/config", get(get_config)) + .route("/api/job/{job_id}/dot", get(get_dot)) + .route("/api/executors", get(get_executors_empty)) + .route("/api/state", get(get_state)) + .with_state(store) +} + +async fn get_jobs(State(store): State>) -> Json> { + // The live list endpoint omits plans; null them for byte-identical output. + let mut jobs: Vec = store + .jobs + .values() + .map(|j| { + let mut r = j.job.clone(); + r.logical_plan = None; + r.physical_plan = None; + r.stage_plan = None; + r + }) + .collect(); + jobs.sort_by(|a, b| a.job_id.cmp(&b.job_id)); + Json(jobs) +} + +async fn get_job( + State(store): State>, + AxumPath(job_id): AxumPath, +) -> Result, SchedulerErrorResponse> { + store + .jobs + .get(&job_id) + .map(|j| Json(j.job.clone())) + .ok_or_else(|| SchedulerErrorResponse::new(StatusCode::NOT_FOUND)) +} + +async fn get_stages( + State(store): State>, + AxumPath(job_id): AxumPath, +) -> Result, SchedulerErrorResponse> { + store + .jobs + .get(&job_id) + .map(|j| Json(j.stages.clone())) + .ok_or_else(|| SchedulerErrorResponse::new(StatusCode::NOT_FOUND)) +} + +async fn get_config( + State(store): State>, + AxumPath(job_id): AxumPath, +) -> Result, SchedulerErrorResponse> { + store + .jobs + .get(&job_id) + .map(|j| Json(j.config.clone())) + .ok_or_else(|| SchedulerErrorResponse::new(StatusCode::NOT_FOUND)) +} + +async fn get_dot( + State(store): State>, + AxumPath(job_id): AxumPath, +) -> Result { + store + .jobs + .get(&job_id) + .map(|j| j.dot.clone()) + .ok_or_else(|| SchedulerErrorResponse::new(StatusCode::NOT_FOUND)) +} + +async fn get_executors_empty() -> Json> { + Json(vec![]) +} + +/// Static `/api/state` payload. The history server has no live scheduler +/// process behind it, so every field that would normally reflect runtime +/// state (uptime, feature flags, scheduling policy) is a fixed placeholder. +/// Field names/types match the live `/api/state` response +/// (`SchedulerStateResponse` in `api/handlers.rs`) and what the TUI +/// deserializes into (`ballista-cli/src/tui/domain/mod.rs::SchedulerState`), +/// so the TUI's startup call succeeds instead of erroring out. +async fn get_state() -> Json { + Json(serde_json::json!({ + "started": 0, + "version": BALLISTA_VERSION, + "datafusion_version": DATAFUSION_VERSION, + "substrait_support": false, + "keda_support": false, + "prometheus_support": false, + "graphviz_support": false, + "spark_support": false, + "scheduling_policy": "history-server", + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use ballista_api_types::dto::QueryStageSummary; + use ballista_history::event::{HistoryEvent, JobEndStatus, SCHEMA_VERSION}; + use std::io::Write; + use tempfile::tempdir; + use tower::ServiceExt; // oneshot + + const STAGE_ID_MARKER: &str = "stage-42"; + + fn sample_replayed_job(job_id: &str) -> ReplayedJob { + ReplayedJob { + job: JobResponse { + job_id: job_id.into(), + job_name: "q1".into(), + job_status: "COMPLETED".into(), + status: "Successful".into(), + num_stages: 1, + completed_stages: 1, + percent_complete: 100, + start_time: 2, + end_time: 3, + logical_plan: Some("Projection".into()), + physical_plan: Some("ProjectionExec".into()), + stage_plan: Some("stage".into()), + }, + stages: QueryStagesResponse { + stages: vec![QueryStageSummary { + stage_id: STAGE_ID_MARKER.into(), + stage_status: "Completed".into(), + input_rows: 10, + output_rows: 5, + elapsed_compute: Some("1ms".into()), + stage_plan: None, + task_duration_percentiles: None, + task_input_percentiles: None, + tasks: vec![], + }], + }, + config: Default::default(), + dot: "digraph {}".into(), + } + } + + fn store_with_one_job() -> Arc { + let mut jobs = HashMap::new(); + jobs.insert("job-1".to_string(), sample_replayed_job("job-1")); + Arc::new(HistoryStore { jobs }) + } + + #[tokio::test] + async fn jobs_endpoint_nulls_plan_fields() { + let app = history_router(store_with_one_job()); + let resp = app + .oneshot( + Request::builder() + .uri("/api/jobs") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body = String::from_utf8(bytes.to_vec()).unwrap(); + assert!(body.contains("\"job_id\":\"job-1\"")); + assert!(!body.contains("physical_plan")); // nulled + skip_serializing_if + } + + #[tokio::test] + async fn stages_endpoint_returns_stored_dto() { + let app = history_router(store_with_one_job()); + let resp = app + .oneshot( + Request::builder() + .uri("/api/job/job-1/stages") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body: QueryStagesResponse = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body.stages.len(), 1); + assert_eq!(body.stages[0].stage_id, STAGE_ID_MARKER); + assert_eq!(body.stages[0].input_rows, 10); + assert_eq!(body.stages[0].output_rows, 5); + } + + #[tokio::test] + async fn missing_job_returns_404_on_job_and_stages() { + let app = history_router(store_with_one_job()); + + let resp = app + .clone() + .oneshot( + Request::builder() + .uri("/api/job/does-not-exist") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + let resp = app + .oneshot( + Request::builder() + .uri("/api/job/does-not-exist/stages") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn state_endpoint_returns_static_payload() { + let app = history_router(store_with_one_job()); + let resp = app + .oneshot( + Request::builder() + .uri("/api/state") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + for field in [ + "started", + "version", + "datafusion_version", + "substrait_support", + "keda_support", + "prometheus_support", + "graphviz_support", + "spark_support", + "scheduling_policy", + ] { + assert!(value.get(field).is_some(), "missing field: {field}"); + } + } + + fn write_job_end_log(path: &Path, job_id: &str) { + let replayed = sample_replayed_job(job_id); + let event = HistoryEvent::JobEnd { + version: SCHEMA_VERSION, + status: JobEndStatus::Succeeded, + queued_at: 0, + started_at: 2, + completed_at: 3, + job: Box::new(replayed.job), + stages: Box::new(replayed.stages), + config: replayed.config, + dot: replayed.dot, + }; + let line = serde_json::to_string(&event).unwrap(); + std::fs::write(path, format!("{line}\n")).unwrap(); + } + + #[test] + fn load_skips_corrupt_eventlog_and_keeps_good_one() { + let dir = tempdir().unwrap(); + + // A good, readable event log. + write_job_end_log(&dir.path().join("job-good.eventlog"), "job-good"); + + // A corrupt file: invalid UTF-8, as if a crash truncated a write + // mid-multibyte-character. + let mut corrupt = + std::fs::File::create(dir.path().join("job-bad.eventlog")).unwrap(); + corrupt.write_all(&[0xff, 0xfe, 0xfd]).unwrap(); + drop(corrupt); + + let store = HistoryStore::load(dir.path()).unwrap(); + assert_eq!(store.jobs.len(), 1); + assert!(store.jobs.contains_key("job-good")); + assert!(!store.jobs.contains_key("job-bad")); + } +} diff --git a/ballista/scheduler/src/lib.rs b/ballista/scheduler/src/lib.rs index 533711ecb0..8338a65372 100644 --- a/ballista/scheduler/src/lib.rs +++ b/ballista/scheduler/src/lib.rs @@ -25,6 +25,9 @@ pub mod cluster; pub mod config; /// Display utilities for execution plans and state. pub mod display; +/// Standalone history server: serves `/api/*` from stored event logs. +#[cfg(feature = "rest-api")] +pub mod history; /// Metrics collection and reporting. pub mod metrics; /// Physical query plan optimizers. diff --git a/ballista/scheduler/src/scheduler_server/event_log.rs b/ballista/scheduler/src/scheduler_server/event_log.rs new file mode 100644 index 0000000000..f6ab129e8e --- /dev/null +++ b/ballista/scheduler/src/scheduler_server/event_log.rs @@ -0,0 +1,408 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Builds `HistoryEvent`s from scheduler event-loop state and appends them to +//! the [`ballista_history::writer::EventLogWriter`]. These are the pure, +//! synchronous event-builder functions; the actual emission -- deciding +//! *when* to call them -- lives at the top of +//! `query_stage_scheduler::QueryStageScheduler::on_receive`. +//! +//! The builders reuse `crate::api::dto_build`, the same DTO builders backing +//! the live REST API, so a job's `JobEnd` event and its `GET /api/job/{id}` +//! response serialize identically for the same graph state. + +use crate::api::dto_build::{ + build_job_dot, graph_to_job_response, graph_to_query_stages, + session_config_to_job_config, task_row_counts, task_status_to_dto, +}; +use crate::state::execution_graph::ExecutionGraphBox; +use ballista_api_types::dto::PlanFormat; +use ballista_core::serde::protobuf::{TaskStatus, task_status}; +use ballista_history::event::{ + HistoryEvent, JobEndStatus, SCHEMA_VERSION, TaskEndMetrics, +}; +use datafusion::physical_plan::displayable; + +/// Builds the `JobStart` event for a job that has just been submitted. +pub(crate) fn job_start_event( + graph: &ExecutionGraphBox, + queued_at: u64, + submitted_at: u64, +) -> HistoryEvent { + HistoryEvent::JobStart { + version: SCHEMA_VERSION, + job_id: graph.job_id().to_string(), + job_name: graph.job_name().to_string(), + queued_at, + submitted_at, + logical_plan: graph.logical_plan().map(|p| p.to_string()), + // Same rendering the `get_job` handler uses for `PlanFormat::Default`. + physical_plan: Some( + displayable(graph.physical_plan().as_ref()) + .indent(false) + .to_string(), + ), + } +} + +/// Builds one `TaskEnd` event per finished task in `statuses`. +/// +/// Only terminal statuses are recorded: `Running` updates are transient +/// in-flight reports, and a status-less update carries no outcome to record at +/// all, so both are skipped. +pub(crate) fn task_end_events( + executor_id: &str, + statuses: &[TaskStatus], +) -> Vec { + statuses + .iter() + .filter_map(|s| { + let status = match s.status.as_ref()? { + task_status::Status::Running(_) => return None, + terminal => task_status_to_dto(terminal), + }; + Some(HistoryEvent::TaskEnd { + stage_id: s.stage_id, + task_id: s.task_id, + executor_id: executor_id.to_string(), + status, + launch_time: s.launch_time, + start_exec_time: s.start_exec_time, + end_exec_time: s.end_exec_time, + metrics: task_end_metrics(s), + }) + }) + .collect() +} + +/// Sums a task's raw operator metrics into the timeline's `TaskEndMetrics`, +/// using the same extraction the stage-summary REST DTO performs so the +/// timeline and stage views agree. Absent metrics yield zeros. +fn task_end_metrics(status: &TaskStatus) -> TaskEndMetrics { + let (input_rows, output_rows, elapsed_compute_nanos) = + task_row_counts(&status.metrics); + TaskEndMetrics { + input_rows, + output_rows, + elapsed_compute_nanos, + } +} + +/// Builds the `JobEnd` event for a job that has just finished (successfully +/// or not), embedding the same DTOs the REST API would serve for this job at +/// this moment: the job summary, per-stage summaries, the session config, and +/// the DOT-format graph. +pub(crate) fn job_end_event( + graph: &ExecutionGraphBox, + status: JobEndStatus, + queued_at: u64, + completed_at: u64, +) -> HistoryEvent { + // `PlanFormat::Default` -- the same default the `get_job` handler falls + // back to when no `?plan_format=` query param is supplied -- so the + // stored JSON matches what a live `GET /api/job/{id}` would have + // returned at this point. + let job = Box::new(graph_to_job_response(graph, PlanFormat::Default)); + // Rendered as of `completed_at` rather than the wall clock, so the stored + // record is a snapshot of the job at the moment it ended and replaying it + // is deterministic. + let stages = Box::new(graph_to_query_stages( + graph, + PlanFormat::Default, + completed_at as u128, + )); + let dot = build_job_dot(graph).unwrap_or_default(); + let config = session_config_to_job_config(graph.session_config().as_ref()); + + HistoryEvent::JobEnd { + version: SCHEMA_VERSION, + status, + queued_at, + started_at: graph.start_time(), + completed_at, + job, + stages, + config, + dot, + } +} + +/// Builds the terminal `JobEnd` event for a cancelled job. +/// +/// Cancellation is recorded from the event-log tee, which runs *before* the +/// scheduler applies the cancel to the graph, so the graph still reports the +/// job as running here. The embedded job DTO's status fields are therefore +/// overwritten, otherwise the history server would show a cancelled job as +/// perpetually `Running`. +/// +/// `queued_at` is not carried on `JobCancel`, so it is recorded as 0. +pub(crate) fn job_cancel_event( + graph: &ExecutionGraphBox, + cancelled_at: u64, +) -> HistoryEvent { + let mut event = job_end_event( + graph, + JobEndStatus::Cancelled, + /* queued_at */ 0, + cancelled_at, + ); + if let HistoryEvent::JobEnd { job, .. } = &mut event { + job.job_status = CANCELLED_STATUS.to_string(); + job.status = CANCELLED_STATUS.to_string(); + } + event +} + +/// The status string a cancelled job is recorded (and served) with. The live +/// REST API has no equivalent -- a cancelled job's graph is dropped before it +/// can be observed -- so this is history-only. +const CANCELLED_STATUS: &str = "Cancelled"; + +#[cfg(test)] +mod tests { + use crate::scheduler_server::event_log::*; + use crate::state::execution_graph::ExecutionGraphBox; + use crate::state::execution_graph_dot::tests::test_graph; + use ballista_core::serde::protobuf::{SuccessfulTask, TaskStatus, task_status}; + use ballista_history::writer::EventLogWriter; + + #[test] + fn task_end_events_map_one_per_finished_task() { + let statuses = vec![TaskStatus { + task_id: 7, + job_id: "job-1".into(), + stage_id: 1, + stage_attempt_num: 0, + launch_time: 100, + start_exec_time: 110, + end_exec_time: 200, + metrics: vec![], + status: Some(task_status::Status::Successful(SuccessfulTask::default())), + }]; + let events = task_end_events("exec-1", &statuses); + assert_eq!(events.len(), 1); + let line = serde_json::to_string(&events[0]).unwrap(); + assert!(line.contains("\"ev\":\"TaskEnd\"")); + assert!(line.contains("\"task_id\":7")); + assert!(line.contains("\"executor_id\":\"exec-1\"")); + } + + // `task_end_events` skips `Running` statuses -- they are transient + // in-flight updates, not the terminal states the timeline records. + #[test] + fn task_end_events_skips_running_tasks() { + let statuses = vec![TaskStatus { + task_id: 0, + job_id: "job-1".into(), + stage_id: 1, + stage_attempt_num: 0, + launch_time: 100, + start_exec_time: 110, + end_exec_time: 0, + metrics: vec![], + status: Some(task_status::Status::Running(Default::default())), + }]; + assert!(task_end_events("exec-1", &statuses).is_empty()); + } + + // A status update with no status at all reports no outcome, so there is + // nothing terminal to record for it either. + #[test] + fn task_end_events_skips_tasks_with_no_status() { + let statuses = vec![TaskStatus { + task_id: 0, + job_id: "job-1".into(), + stage_id: 1, + stage_attempt_num: 0, + launch_time: 100, + start_exec_time: 110, + end_exec_time: 0, + metrics: vec![], + status: None, + }]; + assert!(task_end_events("exec-1", &statuses).is_empty()); + } + + /// A cancelled job is recorded from a graph that has not yet been marked + /// failed, so the event must carry the cancelled status itself rather than + /// the graph's in-flight `Running`. + #[tokio::test] + async fn job_cancel_event_reports_cancelled_not_running() { + let graph = test_graph().await.unwrap(); + let graph: ExecutionGraphBox = Box::new(graph); + + let event = job_cancel_event(&graph, /* cancelled_at */ 42); + let json = serde_json::to_string(&event).unwrap(); + + assert!(json.contains("\"ev\":\"JobEnd\"")); + assert!(json.contains("\"status\":\"Cancelled\"")); + assert!(json.contains("\"job_status\":\"Cancelled\"")); + assert!( + !json.contains("\"Running\""), + "cancelled job should not be recorded as running, got: {json}" + ); + } + + /// `job_end_event`'s embedded DTOs (`job`, `stages`, `dot`) are built by + /// the same `dto_build` functions backing the live REST API, so a direct + /// builder test on the serialized event is an adequate substitute for + /// wiring a full `QueryStageScheduler` through `on_receive` here (which + /// would additionally require standing up executor/task-manager state + /// just to reach a finished job). + #[tokio::test] + async fn job_end_event_embeds_job_and_stage_plan_strings() { + let graph = test_graph().await.unwrap(); + let graph: ExecutionGraphBox = Box::new(graph); + + let event = job_end_event( + &graph, + JobEndStatus::Succeeded, + /* queued_at */ 1, + /* completed_at */ 2, + ); + let json = serde_json::to_string(&event).unwrap(); + + assert!(json.contains("\"ev\":\"JobEnd\"")); + assert!(json.contains("\"status\":\"Succeeded\"")); + assert!(json.contains("\"job_id\":\"job_id\"")); + // The job's top-level physical plan, embedded via `build_job_response`. + assert!( + json.contains("DataSourceExec: (Memory)"), + "expected job physical plan in JobEnd event, got: {json}" + ); + // The per-stage summaries, embedded via `build_query_stages_response`. + assert!( + json.contains("\"stage_status\":\"Resolved\""), + "expected a resolved stage summary in JobEnd event, got: {json}" + ); + // The DOT graph, embedded via `build_job_dot`. + assert!( + json.contains("digraph"), + "expected a dot graph in JobEnd event, got: {json}" + ); + } + + #[tokio::test] + async fn job_start_event_captures_plan_and_timestamps() { + let graph = test_graph().await.unwrap(); + let graph: ExecutionGraphBox = Box::new(graph); + + let event = + job_start_event(&graph, /* queued_at */ 5, /* submitted_at */ 10); + let json = serde_json::to_string(&event).unwrap(); + + assert!(json.contains("\"ev\":\"JobStart\"")); + assert!(json.contains("\"job_id\":\"job_id\"")); + assert!(json.contains("\"job_name\":\"job_name\"")); + assert!(json.contains("\"queued_at\":5")); + assert!(json.contains("\"submitted_at\":10")); + // `job_start_event` renders the graph's pre-staging physical plan + // (`graph.physical_plan()`), unlike `job_end_event`'s embedded job + // DTO which shows the plan reconstructed from stages -- assert on the + // join at its root rather than the leaf scan string. + assert!( + json.contains("HashJoinExec"), + "expected job physical plan in JobStart event, got: {json}" + ); + } + + /// End-to-end: drive the real `EventLogWriter` over a temp dir with a + /// `JobStart` followed by a `JobEnd`, then read the `.eventlog` file back + /// and assert on line order and content -- proving the builders here + /// produce events the writer can actually persist and that a consumer + /// (the eventual history server) can read back as ordered JSONL. + #[tokio::test] + async fn writer_round_trip_orders_job_start_before_job_end() { + let graph = test_graph().await.unwrap(); + let graph: ExecutionGraphBox = Box::new(graph); + let job_id = graph.job_id().to_string(); + + let dir = tempfile::tempdir().unwrap(); + let writer = EventLogWriter::new(dir.path().to_path_buf(), 16); + + writer.append(&job_id, job_start_event(&graph, 1, 2)); + writer.append( + &job_id, + job_end_event(&graph, JobEndStatus::Succeeded, 1, 20), + ); + writer.flush_job(&job_id).await; + + let path = dir.path().join(format!("{job_id}.eventlog")); + let contents = tokio::fs::read_to_string(&path).await.unwrap(); + let lines: Vec<&str> = contents.lines().collect(); + + assert_eq!(lines.len(), 2); + assert!(lines[0].contains("\"ev\":\"JobStart\"")); + assert!(lines[1].contains("\"ev\":\"JobEnd\"")); + assert!(lines[1].contains("DataSourceExec: (Memory)")); + } + + /// End-to-end DTO parity: the history server, replaying a real event log + /// through `EventLogWriter` + `HistoryStore::load`, must serve + /// byte-identical JSON to what the live scheduler would return for the + /// same job via `dto_build::{build_job_response, build_query_stages_response}`. + /// + /// This is `job_end_event`'s only round-trip coverage through the real + /// writer + `HistoryStore` (as opposed to the direct builder-output + /// assertions above), so it lives here as a crate-internal `#[cfg(test)]` + /// module rather than an integration test under `tests/`: `dto_build`'s + /// builders, `job_end_event` itself, and + /// `execution_graph_dot::tests::test_graph` are all `pub(crate)` / + /// cfg(test)-gated and therefore unreachable from a separate `tests/` + /// integration-test crate, which only sees the scheduler crate's public API. + #[tokio::test] + async fn history_store_serves_byte_identical_json_to_live_scheduler() { + use crate::history::HistoryStore; + + let graph = test_graph().await.unwrap(); + let graph: ExecutionGraphBox = Box::new(graph); + let job_id = graph.job_id().to_string(); + + // The live DTOs a running scheduler would serve for this job right now, + // via the same builders the REST handlers call. + // `completed_at` is the snapshot instant for both sides, so the + // comparison does not race the wall clock. + const COMPLETED_AT: u64 = 2; + let live_job = graph_to_job_response(&graph, PlanFormat::Default); + let live_stages = + graph_to_query_stages(&graph, PlanFormat::Default, COMPLETED_AT as u128); + + // Produce the same `JobEnd` event the scheduler emits on completion, + // write it through the real async writer, then load it back via the + // history server's own `HistoryStore::load` -- exercising the full + // write -> read -> serve path, not just the event builder. + let event = job_end_event(&graph, JobEndStatus::Succeeded, 1, COMPLETED_AT); + let dir = tempfile::tempdir().unwrap(); + let writer = EventLogWriter::new(dir.path().to_path_buf(), 16); + writer.append(&job_id, event); + writer.flush_job(&job_id).await; + + let store = HistoryStore::load(dir.path()).unwrap(); + let replayed = store.jobs.get(&job_id).expect("job should be replayed"); + + // The core fidelity guarantee: the history server would return + // byte-identical JSON to what the live scheduler returned for this job. + assert_eq!( + serde_json::to_string(&replayed.job).unwrap(), + serde_json::to_string(&live_job).unwrap(), + ); + assert_eq!( + serde_json::to_string(&replayed.stages).unwrap(), + serde_json::to_string(&live_stages).unwrap(), + ); + } +} diff --git a/ballista/scheduler/src/scheduler_server/mod.rs b/ballista/scheduler/src/scheduler_server/mod.rs index 67c175dee8..7f56c3f441 100644 --- a/ballista/scheduler/src/scheduler_server/mod.rs +++ b/ballista/scheduler/src/scheduler_server/mod.rs @@ -53,6 +53,13 @@ pub mod externalscaler { /// Events for the scheduler event loop. pub mod event; +/// Builds `HistoryEvent`s emitted from the event loop into the event log. +/// +/// Depends on `crate::api::dto_build`, which is only compiled with the +/// `rest-api` feature (the default), so this module and the event-log wiring +/// into `QueryStageScheduler` are gated the same way. +#[cfg(feature = "rest-api")] +mod event_log; #[cfg(feature = "keda-scaler")] mod external_scaler; mod grpc; @@ -101,10 +108,19 @@ impl SchedulerServer SchedulerServer no_executor_check_pending: Arc, + /// Tees scheduler events into a durable per-job event log. `None` unless + /// `event_log_dir` is configured. + #[cfg(feature = "rest-api")] + event_log: Option, } impl QueryStageScheduler { @@ -58,12 +64,46 @@ impl QueryStageSchedul state: Arc>, metrics_collector: Arc, config: Arc, + #[cfg(feature = "rest-api")] event_log: Option< + ballista_history::writer::EventLogWriter, + >, ) -> Self { Self { state, metrics_collector, config, no_executor_check_pending: Arc::new(AtomicBool::new(false)), + #[cfg(feature = "rest-api")] + event_log, + } + } + + /// Fetches a job's execution graph for the event log, reporting rather than + /// swallowing the cases where it is unavailable. Returning `None` costs the + /// job its event, not its execution: event logging is never allowed to fail + /// scheduling. + #[cfg(feature = "rest-api")] + async fn event_log_graph( + &self, + job_id: &ballista_core::JobId, + ) -> Option { + match self + .state + .task_manager + .get_job_execution_graph(job_id) + .await + { + Ok(Some(graph)) => Some(graph), + Ok(None) => { + warn!("event log: no execution graph for job {job_id}, skipping event"); + None + } + Err(e) => { + warn!( + "event log: failed to read execution graph for job {job_id}: {e:?}" + ); + None + } } } @@ -88,6 +128,25 @@ impl QueryStageSchedul } } +/// Groups task status updates by job id, so a single `TaskUpdating` batch +/// (which can span multiple jobs) can be appended to each job's own event log. +#[cfg(feature = "rest-api")] +fn group_by_job( + statuses: &[ballista_core::serde::protobuf::TaskStatus], +) -> std::collections::HashMap> { + let mut by_job: std::collections::HashMap< + String, + Vec, + > = std::collections::HashMap::new(); + for status in statuses { + by_job + .entry(status.job_id.clone()) + .or_default() + .push(status.clone()); + } + by_job +} + #[async_trait::async_trait] impl EventAction for QueryStageScheduler @@ -106,6 +165,100 @@ impl tx_event: &mpsc::Sender, _rx_event: &mpsc::Receiver, ) -> Result<()> { + #[cfg(feature = "rest-api")] + if let Some(log) = &self.event_log { + match &event { + QueryStageSchedulerEvent::JobSubmitted { + job_id, + queued_at, + submitted_at, + } => { + if let Some(graph) = self.event_log_graph(job_id).await { + log.append( + job_id.as_str(), + event_log::job_start_event(&graph, *queued_at, *submitted_at), + ); + } + } + QueryStageSchedulerEvent::TaskUpdating(executor_id, statuses) => { + for (job_id, group) in group_by_job(statuses) { + for ev in event_log::task_end_events(executor_id, &group) { + log.append(&job_id, ev); + } + } + } + // The three terminal events below each close out the job's log. + // `finish_job` runs even when no `JobEnd` could be built, so a + // job that cannot be recorded still releases its file handle; + // such a log has no `JobEnd` line and the history server skips + // it rather than serving a half-written job. + QueryStageSchedulerEvent::JobFinished { + job_id, + queued_at, + completed_at, + } => { + if let Some(graph) = self.event_log_graph(job_id).await { + log.append_final( + job_id.as_str(), + event_log::job_end_event( + &graph, + ballista_history::event::JobEndStatus::Succeeded, + *queued_at, + *completed_at, + ), + ) + .await; + } + log.finish_job(job_id.as_str()).await; + } + QueryStageSchedulerEvent::JobRunningFailed { + job_id, + fail_message, + queued_at, + failed_at, + } => { + if let Some(graph) = self.event_log_graph(job_id).await { + log.append_final( + job_id.as_str(), + event_log::job_end_event( + &graph, + ballista_history::event::JobEndStatus::Failed( + fail_message.clone(), + ), + *queued_at, + *failed_at, + ), + ) + .await; + } + log.finish_job(job_id.as_str()).await; + } + QueryStageSchedulerEvent::JobCancel(job_id) => { + // Cancellation is terminal: the handler below drops the + // graph, so this is the last point at which the job can be + // recorded. Without this the log would never receive a + // `JobEnd`, leaving the cancelled job invisible to the + // history server and its file handle open for the life of + // the process. + if let Some(graph) = self.event_log_graph(job_id).await { + log.append_final( + job_id.as_str(), + event_log::job_cancel_event( + &graph, + ballista_core::utils::get_current_time() as u64, + ), + ) + .await; + } + log.finish_job(job_id.as_str()).await; + } + // `JobPlanningFailed` is deliberately absent: it is only posted + // when `submit_job` fails, i.e. instead of `JobSubmitted`, so + // the job has neither an execution graph nor an open log file. + _ => {} + } + } + let mut time_recorder = None; if self.config.scheduler_event_expected_processing_duration > 0 { time_recorder = Some((Instant::now(), event.clone())); diff --git a/ballista/scheduler/src/state/execution_graph_dot.rs b/ballista/scheduler/src/state/execution_graph_dot.rs index eaef4dc5e9..c9a622aaba 100644 --- a/ballista/scheduler/src/state/execution_graph_dot.rs +++ b/ballista/scheduler/src/state/execution_graph_dot.rs @@ -406,7 +406,7 @@ fn get_file_scan(scan: &FileScanConfig) -> String { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use crate::planner::DefaultDistributedPlanner; use crate::state::execution_graph::StaticExecutionGraph; use crate::state::execution_graph_dot::ExecutionGraphDot; @@ -578,7 +578,7 @@ filter_expr="] Ok(()) } - async fn test_graph() -> Result { + pub(crate) async fn test_graph() -> Result { let mut config = SessionConfig::new() .with_target_partitions(48) .with_batch_size(4096); diff --git a/dev/release/README.md b/dev/release/README.md index c6b0de31c0..502b0a71af 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -288,6 +288,7 @@ of the following crates: - [ballista-core](https://crates.io/crates/ballista-core) - [ballista-executor](https://crates.io/crates/ballista-executor) - [ballista-api-types](https://crates.io/crates/ballista-api-types) +- [ballista-history](https://crates.io/crates/ballista-history) - [ballista-scheduler](https://crates.io/crates/ballista-scheduler) Download and unpack the official release tarball @@ -308,6 +309,7 @@ dot -Tsvg dev/release/crate-deps.dot > dev/release/crate-deps.svg (cd ballista/core && cargo publish) (cd ballista/executor && cargo publish) (cd ballista/api-types && cargo publish) +(cd ballista/history && cargo publish) (cd ballista/scheduler && cargo publish) (cd ballista/client && cargo publish) (cd ballista-cli && cargo publish) diff --git a/dev/release/crate-deps.dot b/dev/release/crate-deps.dot index 27bee709ba..272da20f55 100644 --- a/dev/release/crate-deps.dot +++ b/dev/release/crate-deps.dot @@ -19,6 +19,7 @@ digraph G { ballista_core ballista_api_types + ballista_history ballista_scheduler ballista_executor ballista @@ -26,6 +27,9 @@ digraph G { ballista_scheduler -> ballista_core ballista_scheduler -> ballista_api_types + ballista_scheduler -> ballista_history + + ballista_history -> ballista_api_types ballista_executor -> ballista_core diff --git a/dev/update_ballista_versions.py b/dev/update_ballista_versions.py index ca5bef3bf8..926520f42d 100755 --- a/dev/update_ballista_versions.py +++ b/dev/update_ballista_versions.py @@ -44,6 +44,7 @@ def update_cargo_toml(cargo_toml: str, new_version: str): 'ballista-core', 'ballista-executor', 'ballista-api-types', + 'ballista-history', 'ballista-scheduler', 'ballista-cli', ) @@ -82,6 +83,7 @@ def main(): 'ballista-cli', 'ballista/core', 'ballista/api-types', + 'ballista/history', 'ballista/scheduler', 'ballista/executor', 'ballista/client', diff --git a/docs/source/index.rst b/docs/source/index.rst index ca71559d00..0ee337f54c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -37,6 +37,7 @@ Table of content Deployment Scheduler + History Server .. toctree:: :maxdepth: 1 diff --git a/docs/source/user-guide/history-server.md b/docs/source/user-guide/history-server.md new file mode 100644 index 0000000000..23b58cebbb --- /dev/null +++ b/docs/source/user-guide/history-server.md @@ -0,0 +1,109 @@ + + +# History Server + +The scheduler forgets a job shortly after it finishes. Completed jobs are cleaned +up after `finished_job_state_clean_up_interval_seconds`, and everything is gone +when the scheduler restarts, so by the time you want to look at a slow query it +is usually too late. + +The history server is Ballista's equivalent of the Spark History Server. When +event logging is enabled the scheduler writes a durable record of each job as it +runs, and the history server replays those records and serves the same `/api/*` +responses the live scheduler does. The existing TUI can point at it and browse +completed jobs with no scheduler running at all. + +## Enabling event logging + +Event logging is off by default. Start the scheduler with a directory to write +to: + +```shell +ballista-scheduler --event-log-dir /var/lib/ballista/history +``` + +The scheduler writes one file per job, `.eventlog`, in +[JSON Lines](https://jsonlines.org/) format. Files are appended as the job runs +and closed when it reaches a terminal state. + +Writes happen on a background task, so the scheduler's event loop never waits on +disk. If the queue backs up, progress records are dropped rather than allowed to +stall scheduling. The terminal record is the exception: it waits for queue +capacity, because a job missing it is invisible to the history server. + +## Running the history server + +Point it at the same directory: + +```shell +ballista-history-server \ + --event-log-dir /var/lib/ballista/history \ + --bind-host 0.0.0.0 \ + --bind-port 50060 +``` + +It loads every completed log it finds at startup and serves them over the same +paths as the live scheduler: + +| Endpoint | Serves | +| ------------------------------ | ----------------------------------- | +| `GET /api/jobs` | every completed job | +| `GET /api/job/{job_id}` | one job's summary and plans | +| `GET /api/job/{job_id}/stages` | per-stage and per-task detail | +| `GET /api/job/{job_id}/config` | the session config the job ran with | +| `GET /api/job/{job_id}/dot` | the stage DAG in DOT format | + +Because the TUI talks to that same API, you can browse history with: + +```shell +ballista-cli --tui --host localhost --port 50060 +``` + +Logs are read once at startup. Restart the history server to pick up jobs that +finished since it launched. + +## What is recorded + +Each log holds an ordered timeline: the job's submission, each stage starting +and ending, and each task finishing with its row counts and compute time. The +final record carries the finished API responses themselves. + +That last point is what makes replayed output trustworthy. The history server +does not rebuild a response from stored state; it re-serves the exact response +the scheduler built while the job was alive. There is no second implementation +that could drift from the live one. + +Only the final record is served today. The per-task timeline is recorded so a +future UI can show a job progressing rather than only its end state. + +## Operational notes + +- **Disk is not reclaimed automatically.** Logs accumulate until you remove + them. Size them against your job volume and prune with whatever you already + use for log rotation. +- **A corrupt log is skipped, not fatal.** If the scheduler dies mid-write the + affected file simply has no terminal record, so the history server ignores it + and still serves every other job. +- **Plans are rendered once, when the job ends.** The `?plan_format=` query + parameter therefore has no effect against a history server; it returns the + format captured at write time. +- **The history server has no cluster behind it.** `GET /api/executors` returns + an empty list and `GET /api/state` returns a static payload, so that TUI + screens expecting them still load.