From 90e386969062f1c08fcb773b341a529be8a9ab8a Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Sun, 24 Apr 2022 20:23:30 +0200 Subject: [PATCH] datastore: add single-level and recursive namespace iterators Signed-off-by: Thomas Lamprecht --- pbs-datastore/examples/ls-snapshots.rs | 14 +- pbs-datastore/src/datastore.rs | 212 +++++++++++++++++++++++++ pbs-datastore/src/lib.rs | 2 +- 3 files changed, 222 insertions(+), 6 deletions(-) diff --git a/pbs-datastore/examples/ls-snapshots.rs b/pbs-datastore/examples/ls-snapshots.rs index 7b4445b2..1e4587f7 100644 --- a/pbs-datastore/examples/ls-snapshots.rs +++ b/pbs-datastore/examples/ls-snapshots.rs @@ -12,12 +12,16 @@ fn run() -> Result<(), Error> { let store = unsafe { DataStore::open_path("", &base, None)? }; - for group in store.iter_backup_groups(Default::default())? { - let group = group?; - println!("found group {}", group); + for ns in store.recursive_iter_backup_ns_ok(Default::default())? { + println!("found namespace store:/{}", ns); - for snapshot in group.iter_snapshots()? { - println!("\t{}", snapshot?); + for group in store.iter_backup_groups(ns)? { + let group = group?; + println!(" found group {}", group); + + for snapshot in group.iter_snapshots()? { + println!("\t{}", snapshot?); + } } } diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index 8a9f16b8..cb2d6640 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -407,6 +407,13 @@ impl DataStore { Ok(()) } + /// Returns the absolute path for a backup namespace on this datastore + pub fn ns_path(&self, ns: &BackupNamespace) -> PathBuf { + let mut full_path = self.base_path(); + full_path.push(ns.path()); + full_path + } + /// Returns the absolute path for a backup_group pub fn group_path(&self, backup_group: &pbs_api_types::BackupGroup) -> PathBuf { let mut full_path = self.base_path(); @@ -589,6 +596,68 @@ impl DataStore { } } + /// Get a streaming iter over single-level backup namespaces of a datatstore + /// + /// The iterated item is still a Result that can contain errors from rather unexptected FS or + /// parsing errors. + pub fn iter_backup_ns( + self: &Arc, + ns: BackupNamespace, + ) -> Result { + ListNamespaces::new(Arc::clone(self), ns) + } + + /// Get a streaming iter over single-level backup namespaces of a datatstore, filtered by Ok + /// + /// The iterated item's result is already unwrapped, if it contained an error it will be + /// logged. Can be useful in iterator chain commands + pub fn iter_backup_ns_ok( + self: &Arc, + ns: BackupNamespace, + ) -> Result + 'static, Error> { + let this = Arc::clone(self); + Ok( + ListNamespaces::new(Arc::clone(&self), ns)?.filter_map(move |ns| match ns { + Ok(ns) => Some(ns), + Err(err) => { + log::error!("list groups error on datastore {} - {}", this.name(), err); + None + } + }), + ) + } + + /// Get a streaming iter over single-level backup namespaces of a datatstore + /// + /// The iterated item is still a Result that can contain errors from rather unexptected FS or + /// parsing errors. + pub fn recursive_iter_backup_ns( + self: &Arc, + ns: BackupNamespace, + ) -> Result { + ListNamespacesRecursive::new(Arc::clone(self), ns) + } + + /// Get a streaming iter over single-level backup namespaces of a datatstore, filtered by Ok + /// + /// The iterated item's result is already unwrapped, if it contained an error it will be + /// logged. Can be useful in iterator chain commands + pub fn recursive_iter_backup_ns_ok( + self: &Arc, + ns: BackupNamespace, + ) -> Result + 'static, Error> { + let this = Arc::clone(self); + Ok( + ListNamespacesRecursive::new(Arc::clone(&self), ns)?.filter_map(move |ns| match ns { + Ok(ns) => Some(ns), + Err(err) => { + log::error!("list groups error on datastore {} - {}", this.name(), err); + None + } + }), + ) + } + /// Get a streaming iter over top-level backup groups of a datatstore /// /// The iterated item is still a Result that can contain errors from rather unexptected FS or @@ -1246,3 +1315,146 @@ impl Iterator for ListGroups { } } } + +/// A iterator for a (single) level of Namespaces +pub struct ListNamespaces { + ns: BackupNamespace, + base_path: PathBuf, + ns_state: Option, +} + +impl ListNamespaces { + /// construct a new single-level namespace iterator on a datastore with an optional anchor ns + pub fn new(store: Arc, ns: BackupNamespace) -> Result { + Ok(ListNamespaces { + ns, + base_path: store.base_path(), + ns_state: None, + }) + } + + /// to allow constructing the iter directly on a path, e.g., provided by section config + /// + /// NOTE: it's recommended to use the datastore one constructor or go over the recursive iter + pub fn new_from_path(path: PathBuf, ns: Option) -> Result { + Ok(ListNamespaces { + ns: ns.unwrap_or_default(), + base_path: path, + ns_state: None, + }) + } +} + +impl Iterator for ListNamespaces { + type Item = Result; + + fn next(&mut self) -> Option { + loop { + if let Some(ref mut id_fd) = self.ns_state { + let item = id_fd.next()?; // if this returns none we are done + let entry = match item { + Ok(ref entry) => { + match entry.file_type() { + Some(nix::dir::Type::Directory) => entry, // OK + _ => continue, + } + } + Err(err) => return Some(Err(err)), + }; + if let Ok(name) = entry.file_name().to_str() { + if name != "." && name != ".." { + return Some(BackupNamespace::from_parent_ns(&self.ns, name.to_string())); + } + } + continue; // file did not match regex or isn't valid utf-8 + } else { + let mut base_path = self.base_path.to_owned(); + if !self.ns.is_root() { + base_path.push(self.ns.path()); + } + base_path.push("ns"); + + let ns_dirfd = match proxmox_sys::fs::read_subdir(libc::AT_FDCWD, &base_path) { + Ok(dirfd) => dirfd, + Err(nix::Error::Sys(nix::errno::Errno::ENOENT)) => return None, + Err(err) => return Some(Err(err.into())), + }; + // found a ns directory, descend into it to scan all it's namespaces + self.ns_state = Some(ns_dirfd); + } + } + } +} + +/// A iterator for all Namespaces below an anchor namespace, most often that will be the +/// `BackupNamespace::root()` one. +/// +/// Descends depth-first (pre-order) into the namespace hierachy yielding namespaces immediately as +/// it finds them. +/// +/// Note: The anchor namespaces passed on creating the iterator will yielded as first element, this +/// can be usefull for searching all backup groups from a certain anchor, as that can contain +/// sub-namespaces but also groups on its own level, so otherwise one would need to special case +/// the ones from the own level. +pub struct ListNamespacesRecursive { + store: Arc, + /// the starting namespace we search downward from + ns: BackupNamespace, + state: Option>, // vector to avoid code recursion +} + +impl ListNamespacesRecursive { + /// Creates an recursive namespace iterator. + pub fn new(store: Arc, ns: BackupNamespace) -> Result { + Ok(ListNamespacesRecursive { + store: store, + ns, + state: None, + }) + } +} + +impl Iterator for ListNamespacesRecursive { + type Item = Result; + + fn next(&mut self) -> Option { + loop { + if let Some(ref mut state) = self.state { + if state.is_empty() { + return None; // there's a state but it's empty -> we're all done + } + let iter = match state.last_mut() { + Some(iter) => iter, + None => return None, // unexpected, should we just unwrap? + }; + match iter.next() { + Some(Ok(ns)) => { + match ListNamespaces::new(Arc::clone(&self.store), ns.to_owned()) { + Ok(iter) => state.push(iter), + Err(err) => log::error!("failed to create child namespace iter {err}"), + } + return Some(Ok(ns)); + } + Some(ns_err) => return Some(ns_err), + None => { + let _ = state.pop(); // done at this (and belows) level, continue in parent + } + } + } else { + // first next call ever: initialize state vector and start iterating at our level + let mut state = Vec::with_capacity(pbs_api_types::MAX_NAMESPACE_DEPTH); + match ListNamespaces::new(Arc::clone(&self.store), self.ns.to_owned()) { + Ok(list_ns) => state.push(list_ns), + Err(err) => { + // yield the error but set the state to Some to avoid re-try, a future + // next() will then see the state, and the empty check yield's None + self.state = Some(state); + return Some(Err(err)); + } + } + self.state = Some(state); + return Some(Ok(self.ns.to_owned())); // return our anchor ns for convenience + } + } + } +} diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs index f08d92f8..fcf97522 100644 --- a/pbs-datastore/src/lib.rs +++ b/pbs-datastore/src/lib.rs @@ -206,7 +206,7 @@ pub use manifest::BackupManifest; pub use store_progress::StoreProgress; mod datastore; -pub use datastore::{check_backup_owner, DataStore, ListGroups, ListSnapshots}; +pub use datastore::{check_backup_owner, DataStore, ListGroups, ListNamespaces, ListSnapshots}; mod snapshot_reader; pub use snapshot_reader::SnapshotReader;