From 134779664e70f29406a1edde3c9e7e835ea10536 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Tue, 24 May 2022 12:54:42 +0200 Subject: [PATCH] manager: hidden command to move datastore prune opts into jobs Signed-off-by: Wolfgang Bumiller --- pbs-api-types/src/datastore.rs | 46 ++------------- pbs-config/src/datastore.rs | 9 +-- pbs-config/src/prune.rs | 1 + src/api2/config/datastore.rs | 36 ++++++------ src/bin/proxmox-backup-manager.rs | 4 ++ src/bin/proxmox-backup-proxy.rs | 13 +---- src/bin/proxmox_backup_manager/prune.rs | 77 ++++++++++++++++++++++++- tests/prune.rs | 64 ++++++++++---------- 8 files changed, 142 insertions(+), 108 deletions(-) diff --git a/pbs-api-types/src/datastore.rs b/pbs-api-types/src/datastore.rs index a462c2ec..2bb47bbc 100644 --- a/pbs-api-types/src/datastore.rs +++ b/pbs-api-types/src/datastore.rs @@ -218,29 +218,8 @@ pub const DATASTORE_TUNING_STRING_SCHEMA: Schema = StringSchema::new("Datastore optional: true, schema: PRUNE_SCHEDULE_SCHEMA, }, - "keep-last": { - optional: true, - schema: PRUNE_SCHEMA_KEEP_LAST, - }, - "keep-hourly": { - optional: true, - schema: PRUNE_SCHEMA_KEEP_HOURLY, - }, - "keep-daily": { - optional: true, - schema: PRUNE_SCHEMA_KEEP_DAILY, - }, - "keep-weekly": { - optional: true, - schema: PRUNE_SCHEMA_KEEP_WEEKLY, - }, - "keep-monthly": { - optional: true, - schema: PRUNE_SCHEMA_KEEP_MONTHLY, - }, - "keep-yearly": { - optional: true, - schema: PRUNE_SCHEMA_KEEP_YEARLY, + keep: { + type: crate::KeepOptions, }, "verify-new": { description: "If enabled, all new backups will be verified right after completion.", @@ -277,18 +256,8 @@ pub struct DataStoreConfig { #[serde(skip_serializing_if = "Option::is_none")] pub prune_schedule: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub keep_last: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub keep_hourly: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub keep_daily: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub keep_weekly: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub keep_monthly: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub keep_yearly: Option, + #[serde(flatten)] + pub keep: crate::KeepOptions, /// If enabled, all backups will be verified right after completion. #[serde(skip_serializing_if = "Option::is_none")] @@ -319,12 +288,7 @@ impl DataStoreConfig { comment: None, gc_schedule: None, prune_schedule: None, - keep_last: None, - keep_hourly: None, - keep_daily: None, - keep_weekly: None, - keep_monthly: None, - keep_yearly: None, + keep: Default::default(), verify_new: None, notify_user: None, notify: None, diff --git a/pbs-config/src/datastore.rs b/pbs-config/src/datastore.rs index b2b4d2a0..161fda2f 100644 --- a/pbs-config/src/datastore.rs +++ b/pbs-config/src/datastore.rs @@ -2,7 +2,7 @@ use anyhow::Error; use lazy_static::lazy_static; use std::collections::HashMap; -use proxmox_schema::{ApiType, Schema}; +use proxmox_schema::{AllOfSchema, ApiType, Schema}; use proxmox_section_config::{SectionConfig, SectionConfigData, SectionConfigPlugin}; use pbs_api_types::{DataStoreConfig, DATASTORE_SCHEMA}; @@ -14,15 +14,12 @@ lazy_static! { } fn init() -> SectionConfig { - let obj_schema = match DataStoreConfig::API_SCHEMA { - Schema::Object(ref obj_schema) => obj_schema, - _ => unreachable!(), - }; + const OBJ_SCHEMA: &AllOfSchema = DataStoreConfig::API_SCHEMA.unwrap_all_of_schema(); let plugin = SectionConfigPlugin::new( "datastore".to_string(), Some(String::from("name")), - obj_schema, + OBJ_SCHEMA, ); let mut config = SectionConfig::new(&DATASTORE_SCHEMA); config.register_plugin(plugin); diff --git a/pbs-config/src/prune.rs b/pbs-config/src/prune.rs index 2c5964a9..e9a6ed58 100644 --- a/pbs-config/src/prune.rs +++ b/pbs-config/src/prune.rs @@ -39,6 +39,7 @@ pub fn config() -> Result<(SectionConfigData, [u8; 32]), Error> { let digest = openssl::sha::sha256(content.as_bytes()); let data = CONFIG.parse(PRUNE_CFG_FILENAME, &content)?; + Ok((data, digest)) } diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs index 28342c2c..93f0fae0 100644 --- a/src/api2/config/datastore.rs +++ b/src/api2/config/datastore.rs @@ -251,22 +251,22 @@ pub fn update_datastore( data.prune_schedule = None; } DeletableProperty::keep_last => { - data.keep_last = None; + data.keep.keep_last = None; } DeletableProperty::keep_hourly => { - data.keep_hourly = None; + data.keep.keep_hourly = None; } DeletableProperty::keep_daily => { - data.keep_daily = None; + data.keep.keep_daily = None; } DeletableProperty::keep_weekly => { - data.keep_weekly = None; + data.keep.keep_weekly = None; } DeletableProperty::keep_monthly => { - data.keep_monthly = None; + data.keep.keep_monthly = None; } DeletableProperty::keep_yearly => { - data.keep_yearly = None; + data.keep.keep_yearly = None; } DeletableProperty::verify_new => { data.verify_new = None; @@ -308,23 +308,23 @@ pub fn update_datastore( data.prune_schedule = update.prune_schedule; } - if update.keep_last.is_some() { - data.keep_last = update.keep_last; + if update.keep.keep_last.is_some() { + data.keep.keep_last = update.keep.keep_last; } - if update.keep_hourly.is_some() { - data.keep_hourly = update.keep_hourly; + if update.keep.keep_hourly.is_some() { + data.keep.keep_hourly = update.keep.keep_hourly; } - if update.keep_daily.is_some() { - data.keep_daily = update.keep_daily; + if update.keep.keep_daily.is_some() { + data.keep.keep_daily = update.keep.keep_daily; } - if update.keep_weekly.is_some() { - data.keep_weekly = update.keep_weekly; + if update.keep.keep_weekly.is_some() { + data.keep.keep_weekly = update.keep.keep_weekly; } - if update.keep_monthly.is_some() { - data.keep_monthly = update.keep_monthly; + if update.keep.keep_monthly.is_some() { + data.keep.keep_monthly = update.keep.keep_monthly; } - if update.keep_yearly.is_some() { - data.keep_yearly = update.keep_yearly; + if update.keep.keep_yearly.is_some() { + data.keep.keep_yearly = update.keep.keep_yearly; } if let Some(notify_str) = update.notify { diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs index 8436c484..89598c90 100644 --- a/src/bin/proxmox-backup-manager.rs +++ b/src/bin/proxmox-backup-manager.rs @@ -453,6 +453,9 @@ async fn run() -> Result<(), Error> { .insert("versions", CliCommand::new(&API_METHOD_GET_VERSIONS)); let args: Vec = std::env::args().take(2).collect(); + if args.len() >= 2 && args[1] == "update-to-prune-jobs-config" { + return update_to_prune_jobs_config(); + } let avoid_init = args.len() >= 2 && (args[1] == "bashcomplete" || args[1] == "printdoc"); if !avoid_init { @@ -460,6 +463,7 @@ async fn run() -> Result<(), Error> { let file_opts = CreateOptions::new() .owner(backup_user.uid) .group(backup_user.gid); + proxmox_rest_server::init_worker_tasks( pbs_buildcfg::PROXMOX_BACKUP_LOG_DIR_M!().into(), file_opts, diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index 17f4e660..731098d7 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -47,8 +47,8 @@ use pbs_buildcfg::configdir; use proxmox_time::CalendarEvent; use pbs_api_types::{ - Authid, DataStoreConfig, KeepOptions, Operation, PruneJobConfig, PruneJobOptions, - SyncJobConfig, TapeBackupJobConfig, VerificationJobConfig, + Authid, DataStoreConfig, Operation, PruneJobConfig, PruneJobOptions, SyncJobConfig, + TapeBackupJobConfig, VerificationJobConfig, }; use proxmox_rest_server::daemon; @@ -692,14 +692,7 @@ async fn schedule_datastore_prune() { }; let prune_options = PruneJobOptions { - keep: KeepOptions { - keep_last: store_config.keep_last, - keep_hourly: store_config.keep_hourly, - keep_daily: store_config.keep_daily, - keep_weekly: store_config.keep_weekly, - keep_monthly: store_config.keep_monthly, - keep_yearly: store_config.keep_yearly, - }, + keep: store_config.keep, ..Default::default() }; diff --git a/src/bin/proxmox_backup_manager/prune.rs b/src/bin/proxmox_backup_manager/prune.rs index b67acd40..8fd054ae 100644 --- a/src/bin/proxmox_backup_manager/prune.rs +++ b/src/bin/proxmox_backup_manager/prune.rs @@ -1,12 +1,13 @@ use std::collections::HashMap; use anyhow::Error; +use serde::Deserialize; use serde_json::Value; use proxmox_router::{cli::*, ApiHandler, RpcEnvironment}; use proxmox_schema::api; -use pbs_api_types::{PruneJobConfig, JOB_ID_SCHEMA}; +use pbs_api_types::{DataStoreConfig, PruneJobConfig, PruneJobOptions, JOB_ID_SCHEMA}; use pbs_config::prune; use proxmox_backup::api2; @@ -155,3 +156,77 @@ fn get_prune_job(id: &str) -> Result { config.lookup("prune", id) } + +pub(crate) fn update_to_prune_jobs_config() -> Result<(), Error> { + use pbs_config::datastore; + + let _prune_lock = prune::lock_config()?; + let _datastore_lock = datastore::lock_config()?; + + let (mut data, _digest) = prune::config()?; + let (mut storeconfig, _digest) = datastore::config()?; + + for (store, entry) in storeconfig.sections.iter_mut() { + let ty = &entry.0; + + if ty != "datastore" { + continue; + } + + let mut config = match DataStoreConfig::deserialize(&entry.1) { + Ok(c) => c, + Err(err) => { + eprintln!("failed to parse config of store {store}: {err}"); + continue; + } + }; + + let options = PruneJobOptions { + keep: std::mem::take(&mut config.keep), + ..Default::default() + }; + + let schedule = config.prune_schedule.take(); + + entry.1 = serde_json::to_value(config)?; + + let schedule = match schedule { + Some(s) => s, + None => { + eprintln!("dropping disabled prune job in datastore.cfg"); + continue; + } + }; + + let mut id = format!("storeconfig-{store}"); + id.truncate(32); + if data.sections.contains_key(&id) { + eprintln!("skipping existing converted prune job: {id}"); + continue; + } + + if !options.keeps_something() { + eprintln!("dropping empty prune job data in datastore.cfg"); + continue; + } + + let prune_config = PruneJobConfig { + id: id.clone(), + store: store.clone(), + disable: false, + comment: None, + schedule, + options, + }; + + let prune_config = serde_json::to_value(prune_config)?; + + data.sections + .insert(id, ("prune".to_string(), prune_config)); + } + + prune::save_config(&data)?; + datastore::save_config(&storeconfig)?; + + Ok(()) +} diff --git a/tests/prune.rs b/tests/prune.rs index 11f91ed8..85fd8f7d 100644 --- a/tests/prune.rs +++ b/tests/prune.rs @@ -12,7 +12,7 @@ fn get_prune_list( return_kept: bool, options: &PruneJobOptions, ) -> Vec { - let mut prune_info = compute_prune_info(list, options).unwrap(); + let mut prune_info = compute_prune_info(list, &options.keep).unwrap(); prune_info.reverse(); @@ -68,13 +68,13 @@ fn test_prune_protected() -> Result<(), Error> { eprintln!("{:?}", orig_list); let mut options = PruneJobOptions::default(); - options.keep_last = Some(1); + options.keep.keep_last = Some(1); let remove_list = get_prune_list(orig_list.clone(), false, &options); let expect: Vec = vec![PathBuf::from("host/elsa/2019-11-15T10:39:15Z")]; assert_eq!(remove_list, expect); let mut options = PruneJobOptions::default(); - options.keep_hourly = Some(1); + options.keep.keep_hourly = Some(1); let remove_list = get_prune_list(orig_list.clone(), false, &options); let expect: Vec = vec![PathBuf::from("host/elsa/2019-11-15T10:39:15Z")]; assert_eq!(remove_list, expect); @@ -94,7 +94,7 @@ fn test_prune_hourly() -> Result<(), Error> { let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_hourly = Some(3); + options.keep.keep_hourly = Some(3); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-11-15T10:49:15Z"), @@ -105,7 +105,7 @@ fn test_prune_hourly() -> Result<(), Error> { let list = orig_list; let mut options = PruneJobOptions::default(); - options.keep_hourly = Some(2); + options.keep.keep_hourly = Some(2); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-11-15T10:59:15Z"), @@ -132,15 +132,15 @@ fn test_prune_simple2() -> Result<(), Error> { let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_daily = Some(1); + options.keep.keep_daily = Some(1); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![PathBuf::from("host/elsa/2019-12-04T11:59:15Z")]; assert_eq!(remove_list, expect); let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_last = Some(1); - options.keep_daily = Some(1); + options.keep.keep_last = Some(1); + options.keep.keep_daily = Some(1); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-03T11:59:15Z"), @@ -150,8 +150,8 @@ fn test_prune_simple2() -> Result<(), Error> { let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_daily = Some(1); - options.keep_weekly = Some(1); + options.keep.keep_daily = Some(1); + options.keep.keep_weekly = Some(1); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-01T11:59:15Z"), @@ -161,9 +161,9 @@ fn test_prune_simple2() -> Result<(), Error> { let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_daily = Some(1); - options.keep_weekly = Some(1); - options.keep_monthly = Some(1); + options.keep.keep_daily = Some(1); + options.keep.keep_weekly = Some(1); + options.keep.keep_monthly = Some(1); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-11-22T11:59:15Z"), @@ -174,8 +174,8 @@ fn test_prune_simple2() -> Result<(), Error> { let list = orig_list; let mut options = PruneJobOptions::default(); - options.keep_monthly = Some(1); - options.keep_yearly = Some(1); + options.keep.keep_monthly = Some(1); + options.keep.keep_yearly = Some(1); let remove_list = get_prune_list(list, true, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2018-11-15T11:59:15Z"), @@ -199,21 +199,21 @@ fn test_prune_simple() -> Result<(), Error> { let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_last = Some(4); + options.keep.keep_last = Some(4); let remove_list = get_prune_list(list, false, &options); let expect: Vec = Vec::new(); assert_eq!(remove_list, expect); let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_last = Some(3); + options.keep.keep_last = Some(3); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![PathBuf::from("host/elsa/2019-12-02T11:59:15Z")]; assert_eq!(remove_list, expect); let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_last = Some(2); + options.keep.keep_last = Some(2); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -223,7 +223,7 @@ fn test_prune_simple() -> Result<(), Error> { let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_last = Some(1); + options.keep.keep_last = Some(1); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -234,7 +234,7 @@ fn test_prune_simple() -> Result<(), Error> { let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_last = Some(0); + options.keep.keep_last = Some(0); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -247,8 +247,8 @@ fn test_prune_simple() -> Result<(), Error> { // keep-last, keep-daily mixed let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_last = Some(2); - options.keep_daily = Some(2); + options.keep.keep_last = Some(2); + options.keep.keep_daily = Some(2); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![]; assert_eq!(remove_list, expect); @@ -256,7 +256,7 @@ fn test_prune_simple() -> Result<(), Error> { // keep-daily test let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_daily = Some(3); + options.keep.keep_daily = Some(3); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![PathBuf::from("host/elsa/2019-12-04T11:59:15Z")]; assert_eq!(remove_list, expect); @@ -264,7 +264,7 @@ fn test_prune_simple() -> Result<(), Error> { // keep-daily test let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_daily = Some(2); + options.keep.keep_daily = Some(2); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -275,7 +275,7 @@ fn test_prune_simple() -> Result<(), Error> { // keep-weekly let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_weekly = Some(5); + options.keep.keep_weekly = Some(5); let remove_list = get_prune_list(list, false, &options); // all backup are within the same week, so we only keep a single file let expect: Vec = vec![ @@ -288,8 +288,8 @@ fn test_prune_simple() -> Result<(), Error> { // keep-daily + keep-weekly let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_daily = Some(1); - options.keep_weekly = Some(5); + options.keep.keep_daily = Some(1); + options.keep.keep_weekly = Some(5); let remove_list = get_prune_list(list, false, &options); let expect: Vec = vec![ PathBuf::from("host/elsa/2019-12-02T11:59:15Z"), @@ -301,7 +301,7 @@ fn test_prune_simple() -> Result<(), Error> { // keep-monthly let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_monthly = Some(6); + options.keep.keep_monthly = Some(6); let remove_list = get_prune_list(list, false, &options); // all backup are within the same month, so we only keep a single file let expect: Vec = vec![ @@ -314,7 +314,7 @@ fn test_prune_simple() -> Result<(), Error> { // keep-yearly let list = orig_list.clone(); let mut options = PruneJobOptions::default(); - options.keep_yearly = Some(7); + options.keep.keep_yearly = Some(7); let remove_list = get_prune_list(list, false, &options); // all backup are within the same year, so we only keep a single file let expect: Vec = vec![ @@ -327,9 +327,9 @@ fn test_prune_simple() -> Result<(), Error> { // keep-weekly + keep-monthly + keep-yearly let list = orig_list; let mut options = PruneJobOptions::default(); - options.keep_weekly = Some(5); - options.keep_monthly = Some(6); - options.keep_yearly = Some(7); + options.keep.keep_weekly = Some(5); + options.keep.keep_monthly = Some(6); + options.keep.keep_yearly = Some(7); let remove_list = get_prune_list(list, false, &options); // all backup are within one week, so we only keep a single file let expect: Vec = vec![