From 2eeaacb9746d785f448dd9669788778f865dca8a Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 24 Jul 2019 07:48:59 +0200 Subject: [PATCH] src/pxar/encoder.rs: allow to pass list of devices For better mount point include control... --- src/bin/proxmox-backup-client.rs | 34 +++++++++++++++++++++++--- src/bin/pxar.rs | 5 +++- src/client/pxar_backup_stream.rs | 9 ++++--- src/pxar/encoder.rs | 42 +++++++++++++++++++++++++------- tests/catar.rs | 2 +- 5 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/bin/proxmox-backup-client.rs b/src/bin/proxmox-backup-client.rs index c18bad5c..bd0ee15d 100644 --- a/src/bin/proxmox-backup-client.rs +++ b/src/bin/proxmox-backup-client.rs @@ -5,7 +5,7 @@ use failure::*; //use std::os::unix::io::AsRawFd; use chrono::{Local, Utc, TimeZone}; use std::path::{Path, PathBuf}; -use std::collections::HashMap; +use std::collections::{HashSet, HashMap}; use std::io::Write; use proxmox_backup::tools; @@ -151,12 +151,12 @@ fn backup_directory>( dir_path: P, archive_name: &str, chunk_size: Option, - all_file_systems: bool, + device_set: Option>, verbose: bool, crypt_config: Option>, ) -> Result<(), Error> { - let pxar_stream = PxarBackupStream::open(dir_path.as_ref(), all_file_systems, verbose)?; + let pxar_stream = PxarBackupStream::open(dir_path.as_ref(), device_set, verbose)?; let chunk_stream = ChunkStream::new(pxar_stream, chunk_size); let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks @@ -431,6 +431,25 @@ fn create_backup( let backup_id = param["host-id"].as_str().unwrap_or(&tools::nodename()); + let include_dev = param["include-dev"].as_array(); + + let mut devices = if all_file_systems { None } else { Some(HashSet::new()) }; + + if let Some(include_dev) = include_dev { + if all_file_systems { + bail!("option 'all-file-systems' conflicts with option 'include-dev'"); + } + + let mut set = HashSet::new(); + for path in include_dev { + let path = path.as_str().unwrap(); + let stat = nix::sys::stat::stat(path) + .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?; + set.insert(stat.st_dev); + } + devices = Some(set); + } + let mut upload_list = vec![]; enum BackupType { PXAR, IMAGE, CONFIG }; @@ -522,7 +541,7 @@ fn create_backup( &filename, &target, chunk_size_opt, - all_file_systems, + devices.clone(), verbose, crypt_config.clone(), )?; @@ -1219,6 +1238,13 @@ fn main() { ).min_length(1) ) .optional("repository", REPO_URL_SCHEMA.clone()) + .optional( + "include-dev", + ArraySchema::new( + "Include mountpoints with same st_dev number (see ``man fstat``) as specified files.", + StringSchema::new("Path to file.").into() + ) + ) .optional( "keyfile", StringSchema::new("Path to encryption key. All data will be encrypted using this key.")) diff --git a/src/bin/pxar.rs b/src/bin/pxar.rs index 360e982d..3f5aa535 100644 --- a/src/bin/pxar.rs +++ b/src/bin/pxar.rs @@ -15,6 +15,7 @@ use std::fs::OpenOptions; use std::sync::Arc; use std::os::unix::fs::OpenOptionsExt; use std::os::unix::io::AsRawFd; +use std::collections::HashSet; use proxmox_backup::pxar; @@ -155,6 +156,8 @@ fn create_archive( let no_fcaps = param["no-fcaps"].as_bool().unwrap_or(false); let no_acls = param["no-acls"].as_bool().unwrap_or(false); + let devices = if all_file_systems { None } else { Some(HashSet::new()) }; + let source = PathBuf::from(source); let mut dir = nix::dir::Dir::open( @@ -178,7 +181,7 @@ fn create_archive( feature_flags ^= pxar::CA_FORMAT_WITH_ACL; } - pxar::Encoder::encode(source, &mut dir, &mut writer, all_file_systems, verbose, feature_flags)?; + pxar::Encoder::encode(source, &mut dir, &mut writer, devices, verbose, feature_flags)?; writer.flush()?; diff --git a/src/client/pxar_backup_stream.rs b/src/client/pxar_backup_stream.rs index e25ee0b4..8e24e9bb 100644 --- a/src/client/pxar_backup_stream.rs +++ b/src/client/pxar_backup_stream.rs @@ -3,6 +3,7 @@ use failure::*; use std::thread; use std::os::unix::io::FromRawFd; use std::path::{Path, PathBuf}; +use std::collections::HashSet; use futures::Poll; use futures::stream::Stream; @@ -34,7 +35,7 @@ impl Drop for PxarBackupStream { impl PxarBackupStream { - pub fn new(mut dir: Dir, path: PathBuf, all_file_systems: bool, verbose: bool) -> Result { + pub fn new(mut dir: Dir, path: PathBuf, device_set: Option>, verbose: bool) -> Result { let (rx, tx) = nix::unistd::pipe()?; @@ -43,7 +44,7 @@ impl PxarBackupStream { let child = thread::spawn(move|| { let mut writer = unsafe { std::fs::File::from_raw_fd(tx) }; - if let Err(err) = pxar::Encoder::encode(path, &mut dir, &mut writer, all_file_systems, verbose, pxar::CA_FORMAT_DEFAULT) { + if let Err(err) = pxar::Encoder::encode(path, &mut dir, &mut writer, device_set, verbose, pxar::CA_FORMAT_DEFAULT) { eprintln!("pxar encode failed - {}", err); } }); @@ -54,12 +55,12 @@ impl PxarBackupStream { Ok(Self { stream: Some(stream), child: Some(child) }) } - pub fn open(dirname: &Path, all_file_systems: bool, verbose: bool) -> Result { + pub fn open(dirname: &Path, device_set: Option>, verbose: bool) -> Result { let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?; let path = std::path::PathBuf::from(dirname); - Self::new(dir, path, all_file_systems, verbose) + Self::new(dir, path, device_set, verbose) } } diff --git a/src/pxar/encoder.rs b/src/pxar/encoder.rs index 91b26c96..4ffe9064 100644 --- a/src/pxar/encoder.rs +++ b/src/pxar/encoder.rs @@ -4,7 +4,7 @@ use failure::*; use endian_trait::Endian; -use std::collections::HashMap; +use std::collections::{HashSet, HashMap}; use super::format_definition::*; use super::binary_search_tree::*; @@ -48,8 +48,7 @@ pub struct Encoder<'a, W: Write> { writer_pos: usize, _size: usize, file_copy_buffer: Vec, - all_file_systems: bool, - root_st_dev: u64, + device_set: Option>, verbose: bool, // Flags set by the user feature_flags: u64, @@ -65,11 +64,20 @@ impl <'a, W: Write> Encoder<'a, W> { self.base_path.join(&self.relative_path) } + /// Create archive, write result data to ``writer``. + /// + /// The ``device_set`` can be use used to limit included mount points. + /// + /// - ``None``: include all mount points + /// - ``Some(set)``: only include devices listed in this set (the + /// root path device is automathically added to this list, so + /// you can pass an empty set if you want to archive a single + /// mount point.) pub fn encode( path: PathBuf, dir: &mut nix::dir::Dir, writer: &'a mut W, - all_file_systems: bool, + device_set: Option>, verbose: bool, feature_flags: u64, ) -> Result<(), Error> { @@ -90,6 +98,11 @@ impl <'a, W: Write> Encoder<'a, W> { bail!("got unexpected file type {:?} (not a directory)", path); } + let mut device_set = device_set.clone(); + if let Some(ref mut set) = device_set { + set.insert(stat.st_dev); + } + let magic = detect_fs_type(dir_fd)?; if is_virtual_file_system(magic) { @@ -105,8 +118,7 @@ impl <'a, W: Write> Encoder<'a, W> { writer_pos: 0, _size: 0, file_copy_buffer, - all_file_systems, - root_st_dev: stat.st_dev, + device_set, verbose, feature_flags, fs_feature_flags, @@ -616,7 +628,11 @@ impl <'a, W: Write> Encoder<'a, W> { if is_virtual_file_system(magic) { include_children = false; } else { - include_children = (self.root_st_dev == dir_stat.st_dev) || self.all_file_systems; + if let Some(set) = &self.device_set { + include_children = set.contains(&dir_stat.st_dev); + } else { + include_children = true; + } } // Expand the exclude match pattern inherited from the parent by local entries, if present @@ -859,7 +875,11 @@ impl <'a, W: Write> Encoder<'a, W> { if is_virtual_file_system(magic) { include_payload = false; } else { - include_payload = (stat.st_dev == self.root_st_dev) || self.all_file_systems; + if let Some(ref set) = &self.device_set { + include_payload = set.contains(&stat.st_dev); + } else { + include_payload = true; + } } if !include_payload { @@ -986,7 +1006,11 @@ impl <'a, W: Write> Encoder<'a, W> { if is_virtual_file_system(magic) { include_payload = false; } else { - include_payload = (stat.st_dev == self.root_st_dev) || self.all_file_systems; + if let Some(set) = &self.device_set { + include_payload = set.contains(&stat.st_dev); + } else { + include_payload = true; + } } if !include_payload { diff --git a/tests/catar.rs b/tests/catar.rs index f08c18ea..8f4cc033 100644 --- a/tests/catar.rs +++ b/tests/catar.rs @@ -26,7 +26,7 @@ fn run_test(dir_name: &str) -> Result<(), Error> { let path = std::path::PathBuf::from(dir_name); - Encoder::encode(path, &mut dir, &mut writer, false, false, CA_FORMAT_DEFAULT)?; + Encoder::encode(path, &mut dir, &mut writer, None, false, CA_FORMAT_DEFAULT)?; Command::new("cmp") .arg("--verbose")