diff --git a/ballista/core/src/config.rs b/ballista/core/src/config.rs index 9fdfec840..c3981bef7 100644 --- a/ballista/core/src/config.rs +++ b/ballista/core/src/config.rs @@ -286,6 +286,30 @@ impl parse_arg::ParseArgFromStr for TaskSchedulingPolicy { } } +// an enum used to configure the log rolling policy +// needs to be visible to code generated by configure_me +#[derive(Clone, ArgEnum, Copy, Debug, serde::Deserialize)] +pub enum LogRotationPolicy { + Minutely, + Hourly, + Daily, + Never, +} + +impl std::str::FromStr for LogRotationPolicy { + type Err = String; + + fn from_str(s: &str) -> std::result::Result { + ArgEnum::from_str(s, true) + } +} + +impl parse_arg::ParseArgFromStr for LogRotationPolicy { + fn describe_type(mut writer: W) -> fmt::Result { + write!(writer, "The log rotation policy") + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/ballista/executor/executor_config_spec.toml b/ballista/executor/executor_config_spec.toml index 7b9f11dd2..06831b251 100644 --- a/ballista/executor/executor_config_spec.toml +++ b/ballista/executor/executor_config_spec.toml @@ -125,3 +125,9 @@ name = "log_level_setting" type = "String" doc = "special log level for sub mod. link: https://docs.rs/env_logger/latest/env_logger/#enabling-logging. For example we want whole level is INFO but datafusion mode is DEBUG" default = "std::string::String::from(\"INFO,datafusion=INFO\")" + +[[param]] +name = "log_rotation_policy" +type = "ballista_core::config::LogRotationPolicy" +doc = "Tracing log rotation policy, possible values: minutely, hourly, daily, never. Default: daily" +default = "ballista_core::config::LogRotationPolicy::Daily" \ No newline at end of file diff --git a/ballista/executor/src/main.rs b/ballista/executor/src/main.rs index 5f54fa696..6d737ab37 100644 --- a/ballista/executor/src/main.rs +++ b/ballista/executor/src/main.rs @@ -33,7 +33,7 @@ use tokio::signal; use tokio::{fs, time}; use uuid::Uuid; -use ballista_core::config::TaskSchedulingPolicy; +use ballista_core::config::{LogRotationPolicy, TaskSchedulingPolicy}; use ballista_core::error::BallistaError; use ballista_core::serde::protobuf::{ executor_registration, scheduler_grpc_client::SchedulerGrpcClient, @@ -93,13 +93,32 @@ async fn main() -> Result<()> { let grpc_port = opt.bind_grpc_port; let log_dir = opt.log_dir; let print_thread_info = opt.print_thread_info; - let scheduler_name = format!("executor_{}_{}", bind_host, port); + let log_file_name_prefix = format!( + "executor_{}_{}", + external_host + .clone() + .unwrap_or_else(|| "localhost".to_string()), + port + ); let rust_log = env::var(EnvFilter::DEFAULT_ENV); let log_filter = EnvFilter::new(rust_log.unwrap_or(special_mod_log_level)); // File layer if let Some(log_dir) = log_dir { - let log_file = tracing_appender::rolling::daily(log_dir, &scheduler_name); + let log_file = match opt.log_rotation_policy { + LogRotationPolicy::Minutely => { + tracing_appender::rolling::minutely(log_dir, &log_file_name_prefix) + } + LogRotationPolicy::Hourly => { + tracing_appender::rolling::hourly(log_dir, &log_file_name_prefix) + } + LogRotationPolicy::Daily => { + tracing_appender::rolling::daily(log_dir, &log_file_name_prefix) + } + LogRotationPolicy::Never => { + tracing_appender::rolling::never(log_dir, &log_file_name_prefix) + } + }; tracing_subscriber::fmt() .with_ansi(true) .with_thread_names(print_thread_info) diff --git a/ballista/scheduler/scheduler_config_spec.toml b/ballista/scheduler/scheduler_config_spec.toml index 4a4e53896..451becf8d 100644 --- a/ballista/scheduler/scheduler_config_spec.toml +++ b/ballista/scheduler/scheduler_config_spec.toml @@ -129,3 +129,9 @@ name = "log_level_setting" type = "String" doc = "special log level for sub mod. link: https://docs.rs/env_logger/latest/env_logger/#enabling-logging. For example we want whole level is INFO but datafusion mode is DEBUG" default = "std::string::String::from(\"INFO,datafusion=INFO\")" + +[[param]] +name = "log_rotation_policy" +type = "ballista_core::config::LogRotationPolicy" +doc = "Tracing log rotation policy, possible values: minutely, hourly, daily, never. Default: daily" +default = "ballista_core::config::LogRotationPolicy::Daily" diff --git a/ballista/scheduler/src/main.rs b/ballista/scheduler/src/main.rs index 692ba14ce..47a19e42f 100644 --- a/ballista/scheduler/src/main.rs +++ b/ballista/scheduler/src/main.rs @@ -62,6 +62,7 @@ mod config { use ballista_core::utils::create_grpc_server; +use ballista_core::config::LogRotationPolicy; use ballista_scheduler::config::SchedulerConfig; #[cfg(feature = "flight-sql")] use ballista_scheduler::flight_sql::FlightSqlServiceImpl; @@ -170,7 +171,20 @@ async fn main() -> Result<()> { let log_filter = EnvFilter::new(rust_log.unwrap_or(special_mod_log_level)); // File layer if let Some(log_dir) = log_dir { - let log_file = tracing_appender::rolling::daily(log_dir, &log_file_name_prefix); + let log_file = match opt.log_rotation_policy { + LogRotationPolicy::Minutely => { + tracing_appender::rolling::minutely(log_dir, &log_file_name_prefix) + } + LogRotationPolicy::Hourly => { + tracing_appender::rolling::hourly(log_dir, &log_file_name_prefix) + } + LogRotationPolicy::Daily => { + tracing_appender::rolling::daily(log_dir, &log_file_name_prefix) + } + LogRotationPolicy::Never => { + tracing_appender::rolling::never(log_dir, &log_file_name_prefix) + } + }; tracing_subscriber::fmt() .with_ansi(true) .with_thread_names(print_thread_info)