diff --git a/Cargo.toml b/Cargo.toml index 9dc5de2e..72d786c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ members = [ "pbs-fuse-loop", "pbs-runtime", "pbs-systemd", + "pbs-tape", "pbs-tools", "proxmox-backup-banner", @@ -108,6 +109,7 @@ pbs-datastore = { path = "pbs-datastore" } pbs-runtime = { path = "pbs-runtime" } pbs-systemd = { path = "pbs-systemd" } pbs-tools = { path = "pbs-tools" } +pbs-tape = { path = "pbs-tape" } # Local path overrides # NOTE: You must run `cargo update` after changing this for it to take effect! diff --git a/Makefile b/Makefile index 29792290..e6c85a2e 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,7 @@ SUBCRATES := \ pbs-fuse-loop \ pbs-runtime \ pbs-systemd \ + pbs-tape \ pbs-tools \ proxmox-backup-banner \ proxmox-backup-client \ @@ -184,9 +185,11 @@ $(COMPILED_BINS) $(COMPILEDIR)/dump-catalog-shell-cli $(COMPILEDIR)/docgen: .do- --bin proxmox-file-restore \ --package pxar-bin \ --bin pxar \ + --package pbs-tape \ + --bin pmt \ + --bin pmtx \ --package proxmox-backup \ --bin dump-catalog-shell-cli \ - --bin pmt --bin pmtx \ --bin proxmox-daily-update \ --bin proxmox-file-restore \ --bin proxmox-restore-daemon \ diff --git a/pbs-tape/Cargo.toml b/pbs-tape/Cargo.toml new file mode 100644 index 00000000..719ef01c --- /dev/null +++ b/pbs-tape/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "pbs-tape" +version = "0.1.0" +authors = ["Proxmox Support Team "] +edition = "2018" +description = "LTO tage support" + +[dependencies] +lazy_static = "1.4" +libc = "0.2" +anyhow = "1.0" +thiserror = "1.0" +endian_trait = { version = "0.6", features = ["arrays"] } +nix = "0.19.1" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +bitflags = "1.2.1" +regex = "1.2" +udev = ">= 0.3, <0.5" + +proxmox = { version = "0.13.0", default-features = false, features = [] } + +pbs-api-types = { path = "../pbs-api-types" } +pbs-tools = { path = "../pbs-tools" } +pbs-config = { path = "../pbs-config" } diff --git a/src/bin/pmt.rs b/pbs-tape/src/bin/pmt.rs similarity index 95% rename from src/bin/pmt.rs rename to pbs-tape/src/bin/pmt.rs index 011e8c2a..d36de114 100644 --- a/src/bin/pmt.rs +++ b/pbs-tape/src/bin/pmt.rs @@ -13,6 +13,8 @@ /// - support volume statistics /// - read cartridge memory +use std::convert::TryInto; + use anyhow::{bail, Error}; use serde_json::Value; @@ -34,17 +36,9 @@ use pbs_api_types::{ LTO_DRIVE_PATH_SCHEMA, DRIVE_NAME_SCHEMA, LtoTapeDrive, }; use pbs_config::drive::complete_drive_name; - -use proxmox_backup::{ - tape::{ - complete_drive_path, - lto_tape_device_list, - drive::{ - TapeDriver, - LtoTapeHandle, - open_lto_tape_device, - }, - }, +use pbs_tape::{ + sg_tape::SgTape, + linux_list_drives::{complete_drive_path, lto_tape_device_list, open_lto_tape_device}, }; pub const FILE_MARK_COUNT_SCHEMA: Schema = @@ -74,30 +68,30 @@ pub const DRIVE_OPTION_LIST_SCHEMA: Schema = .min_length(1) .schema(); -fn get_tape_handle(param: &Value) -> Result { +fn get_tape_handle(param: &Value) -> Result { if let Some(name) = param["drive"].as_str() { let (config, _digest) = pbs_config::drive::config()?; let drive: LtoTapeDrive = config.lookup("lto", &name)?; eprintln!("using device {}", drive.path); - return LtoTapeHandle::new(open_lto_tape_device(&drive.path)?); + return SgTape::new(open_lto_tape_device(&drive.path)?); } if let Some(device) = param["device"].as_str() { eprintln!("using device {}", device); - return LtoTapeHandle::new(open_lto_tape_device(&device)?); + return SgTape::new(open_lto_tape_device(&device)?); } if let Ok(name) = std::env::var("PROXMOX_TAPE_DRIVE") { let (config, _digest) = pbs_config::drive::config()?; let drive: LtoTapeDrive = config.lookup("lto", &name)?; eprintln!("using device {}", drive.path); - return LtoTapeHandle::new(open_lto_tape_device(&drive.path)?); + return SgTape::new(open_lto_tape_device(&drive.path)?); } if let Ok(device) = std::env::var("TAPE") { eprintln!("using device {}", device); - return LtoTapeHandle::new(open_lto_tape_device(&device)?); + return SgTape::new(open_lto_tape_device(&device)?); } let (config, _digest) = pbs_config::drive::config()?; @@ -112,7 +106,7 @@ fn get_tape_handle(param: &Value) -> Result { let name = drive_names[0]; let drive: LtoTapeDrive = config.lookup("lto", &name)?; eprintln!("using device {}", drive.path); - return LtoTapeHandle::new(open_lto_tape_device(&drive.path)?); + return SgTape::new(open_lto_tape_device(&drive.path)?); } bail!("no drive/device specified"); @@ -171,7 +165,7 @@ fn bsf(count: usize, param: Value) -> Result<(), Error> { let mut handle = get_tape_handle(¶m)?; - handle.backward_space_count_files(count)?; + handle.space_filemarks(-count.try_into()?)?; Ok(()) } @@ -202,8 +196,8 @@ fn bsfm(count: usize, param: Value) -> Result<(), Error> { let mut handle = get_tape_handle(¶m)?; - handle.backward_space_count_files(count)?; - handle.forward_space_count_files(1)?; + handle.space_filemarks(-count.try_into()?)?; + handle.space_filemarks(1)?; Ok(()) } @@ -231,7 +225,7 @@ fn bsr(count: usize, param: Value) -> Result<(), Error> { let mut handle = get_tape_handle(¶m)?; - handle.backward_space_count_records(count)?; + handle.space_blocks(-count.try_into()?)?; Ok(()) } @@ -355,7 +349,7 @@ fn tape_alert_flags(param: Value) -> Result<(), Error> { fn eject(param: Value) -> Result<(), Error> { let mut handle = get_tape_handle(¶m)?; - handle.eject_media()?; + handle.eject()?; Ok(()) } @@ -467,7 +461,7 @@ fn fsf(count: usize, param: Value) -> Result<(), Error> { let mut handle = get_tape_handle(¶m)?; - handle.forward_space_count_files(count)?; + handle.space_filemarks(count.try_into()?)?; Ok(()) } @@ -497,8 +491,8 @@ fn fsfm(count: usize, param: Value) -> Result<(), Error> { let mut handle = get_tape_handle(¶m)?; - handle.forward_space_count_files(count)?; - handle.backward_space_count_files(1)?; + handle.space_filemarks(count.try_into()?)?; + handle.space_filemarks(-1)?; Ok(()) } @@ -526,7 +520,7 @@ fn fsr(count: usize, param: Value) -> Result<(), Error> { let mut handle = get_tape_handle(¶m)?; - handle.forward_space_count_records(count)?; + handle.space_blocks(count.try_into()?)?; Ok(()) } @@ -575,7 +569,7 @@ fn lock(param: Value) -> Result<(), Error> { let mut handle = get_tape_handle(¶m)?; - handle.lock()?; + handle.set_medium_removal(false)?; Ok(()) } @@ -667,6 +661,7 @@ fn status(param: Value) -> Result<(), Error> { let output_format = get_output_format(¶m); let mut handle = get_tape_handle(¶m)?; + let result = handle.get_drive_and_media_status(); if output_format == "json-pretty" { @@ -712,7 +707,7 @@ fn unlock(param: Value) -> Result<(), Error> { let mut handle = get_tape_handle(¶m)?; - handle.unlock()?; + handle.set_medium_removal(true)?; Ok(()) } @@ -792,7 +787,7 @@ fn weof(count: Option, param: Value) -> Result<(), Error> { let mut handle = get_tape_handle(¶m)?; - handle.write_filemarks(count)?; + handle.write_filemarks(count, false)?; Ok(()) } diff --git a/src/bin/pmtx.rs b/pbs-tape/src/bin/pmtx.rs similarity index 98% rename from src/bin/pmtx.rs rename to pbs-tape/src/bin/pmtx.rs index 606144c5..ea0478c7 100644 --- a/src/bin/pmtx.rs +++ b/pbs-tape/src/bin/pmtx.rs @@ -29,17 +29,11 @@ use pbs_config::drive::complete_changer_name; use pbs_api_types::{ SCSI_CHANGER_PATH_SCHEMA, CHANGER_NAME_SCHEMA, ScsiTapeChanger, LtoTapeDrive, }; - -use proxmox_backup::{ - tools::sgutils2::scsi_inquiry, - tape::{ - linux_tape_changer_list, - complete_changer_path, - changer::{ - ElementStatus, - sg_pt_changer, - }, - }, +use pbs_tape::{ + sgutils2::scsi_inquiry, + ElementStatus, + sg_pt_changer, + linux_list_drives::{complete_changer_path, linux_tape_changer_list}, }; fn get_changer_handle(param: &Value) -> Result { diff --git a/src/tape/file_formats/blocked_reader.rs b/pbs-tape/src/blocked_reader.rs similarity index 96% rename from src/tape/file_formats/blocked_reader.rs rename to pbs-tape/src/blocked_reader.rs index ce2fc9aa..78595e53 100644 --- a/src/tape/file_formats/blocked_reader.rs +++ b/pbs-tape/src/blocked_reader.rs @@ -1,14 +1,12 @@ use std::io::Read; -use crate::tape::{ +use crate::{ TapeRead, BlockRead, BlockReadError, - file_formats::{ - PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0, - BlockHeader, - BlockHeaderFlags, - }, + PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0, + BlockHeader, + BlockHeaderFlags, }; /// Read a block stream generated by 'BlockWriter'. @@ -246,15 +244,14 @@ impl Read for BlockedReader { mod test { use std::io::Read; use anyhow::{bail, Error}; - use crate::tape::{ + use crate::{ TapeWrite, BlockReadError, - helpers::{EmulateTapeReader, EmulateTapeWriter}, - file_formats::{ - PROXMOX_TAPE_BLOCK_SIZE, - BlockedReader, - BlockedWriter, - }, + EmulateTapeReader, + EmulateTapeWriter, + PROXMOX_TAPE_BLOCK_SIZE, + BlockedReader, + BlockedWriter, }; fn write_and_verify(data: &[u8]) -> Result<(), Error> { diff --git a/src/tape/file_formats/blocked_writer.rs b/pbs-tape/src/blocked_writer.rs similarity index 97% rename from src/tape/file_formats/blocked_writer.rs rename to pbs-tape/src/blocked_writer.rs index 33fa2955..8220d3eb 100644 --- a/src/tape/file_formats/blocked_writer.rs +++ b/pbs-tape/src/blocked_writer.rs @@ -1,12 +1,10 @@ use proxmox::tools::vec; -use crate::tape::{ +use crate::{ TapeWrite, BlockWrite, - file_formats::{ - BlockHeader, - BlockHeaderFlags, - }, + BlockHeader, + BlockHeaderFlags, }; /// Assemble and write blocks of data diff --git a/src/tape/helpers/emulate_tape_reader.rs b/pbs-tape/src/emulate_tape_reader.rs similarity index 93% rename from src/tape/helpers/emulate_tape_reader.rs rename to pbs-tape/src/emulate_tape_reader.rs index 92c975f9..34d6606c 100644 --- a/src/tape/helpers/emulate_tape_reader.rs +++ b/pbs-tape/src/emulate_tape_reader.rs @@ -2,11 +2,7 @@ use std::io::Read; use proxmox::tools::io::ReadExt; -use crate::tape::{ - BlockRead, - BlockReadError, - file_formats::PROXMOX_TAPE_BLOCK_SIZE, -}; +use crate::{BlockRead, BlockReadError, PROXMOX_TAPE_BLOCK_SIZE}; /// Emulate tape read behavior on a normal Reader /// diff --git a/src/tape/helpers/emulate_tape_writer.rs b/pbs-tape/src/emulate_tape_writer.rs similarity index 95% rename from src/tape/helpers/emulate_tape_writer.rs rename to pbs-tape/src/emulate_tape_writer.rs index eb4f29d7..c8f1cbef 100644 --- a/src/tape/helpers/emulate_tape_writer.rs +++ b/pbs-tape/src/emulate_tape_writer.rs @@ -1,9 +1,6 @@ use std::io::{self, Write}; -use crate::tape::{ - BlockWrite, - file_formats::PROXMOX_TAPE_BLOCK_SIZE, -}; +use crate::{BlockWrite, PROXMOX_TAPE_BLOCK_SIZE}; /// Emulate tape write behavior on a normal Writer /// diff --git a/pbs-tape/src/lib.rs b/pbs-tape/src/lib.rs new file mode 100644 index 00000000..27f9b44e --- /dev/null +++ b/pbs-tape/src/lib.rs @@ -0,0 +1,334 @@ +use std::collections::HashSet; + +use anyhow::{bail, Error}; +use bitflags::bitflags; +use endian_trait::Endian; +use serde::{Serialize, Deserialize}; +use serde_json::Value; + +use proxmox::tools::Uuid; +use proxmox::api::schema::parse_property_string; + +use pbs_api_types::{ScsiTapeChanger, SLOT_ARRAY_SCHEMA}; + +pub mod linux_list_drives; + +pub mod sgutils2; + +mod blocked_reader; +pub use blocked_reader::BlockedReader; + +mod blocked_writer; +pub use blocked_writer::BlockedWriter; + +mod tape_write; +pub use tape_write::*; + +mod tape_read; +pub use tape_read::*; + +mod emulate_tape_reader; +pub use emulate_tape_reader::EmulateTapeReader; + +mod emulate_tape_writer; +pub use emulate_tape_writer::EmulateTapeWriter; + +pub mod sg_tape; + +pub mod sg_pt_changer; + +/// We use 256KB blocksize (always) +pub const PROXMOX_TAPE_BLOCK_SIZE: usize = 256*1024; + +// openssl::sha::sha256(b"Proxmox Tape Block Header v1.0")[0..8] +pub const PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0: [u8; 8] = [220, 189, 175, 202, 235, 160, 165, 40]; + +// openssl::sha::sha256(b"Proxmox Backup Content Header v1.0")[0..8]; +pub const PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0: [u8; 8] = [99, 238, 20, 159, 205, 242, 155, 12]; +// openssl::sha::sha256(b"Proxmox Backup Tape Label v1.0")[0..8]; +pub const PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0: [u8; 8] = [42, 5, 191, 60, 176, 48, 170, 57]; +// openssl::sha::sha256(b"Proxmox Backup MediaSet Label v1.0") +pub const PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0: [u8; 8] = [8, 96, 99, 249, 47, 151, 83, 216]; + +/// Tape Block Header with data payload +/// +/// All tape files are written as sequence of blocks. +/// +/// Note: this struct is large, never put this on the stack! +/// so we use an unsized type to avoid that. +/// +/// Tape data block are always read/written with a fixed size +/// (`PROXMOX_TAPE_BLOCK_SIZE`). But they may contain less data, so the +/// header has an additional size field. For streams of blocks, there +/// is a sequence number (`seq_nr`) which may be use for additional +/// error checking. +#[repr(C,packed)] +pub struct BlockHeader { + /// fixed value `PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0` + pub magic: [u8; 8], + pub flags: BlockHeaderFlags, + /// size as 3 bytes unsigned, little endian + pub size: [u8; 3], + /// block sequence number + pub seq_nr: u32, + pub payload: [u8], +} + +bitflags! { + /// Header flags (e.g. `END_OF_STREAM` or `INCOMPLETE`) + pub struct BlockHeaderFlags: u8 { + /// Marks the last block in a stream. + const END_OF_STREAM = 0b00000001; + /// Mark multivolume streams (when set in the last block) + const INCOMPLETE = 0b00000010; + } +} + +#[derive(Endian, Copy, Clone, Debug)] +#[repr(C,packed)] +/// Media Content Header +/// +/// All tape files start with this header. The header may contain some +/// informational data indicated by `size`. +/// +/// `| MediaContentHeader | header data (size) | stream data |` +/// +/// Note: The stream data following may be of any size. +pub struct MediaContentHeader { + /// fixed value `PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0` + pub magic: [u8; 8], + /// magic number for the content following + pub content_magic: [u8; 8], + /// unique ID to identify this data stream + pub uuid: [u8; 16], + /// stream creation time + pub ctime: i64, + /// Size of header data + pub size: u32, + /// Part number for multipart archives. + pub part_number: u8, + /// Reserved for future use + pub reserved_0: u8, + /// Reserved for future use + pub reserved_1: u8, + /// Reserved for future use + pub reserved_2: u8, +} + +impl MediaContentHeader { + + /// Create a new instance with autogenerated Uuid + pub fn new(content_magic: [u8; 8], size: u32) -> Self { + let uuid = *proxmox::tools::uuid::Uuid::generate() + .into_inner(); + Self { + magic: PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0, + content_magic, + uuid, + ctime: proxmox::tools::time::epoch_i64(), + size, + part_number: 0, + reserved_0: 0, + reserved_1: 0, + reserved_2: 0, + } + } + + /// Helper to check magic numbers and size constraints + pub fn check(&self, content_magic: [u8; 8], min_size: u32, max_size: u32) -> Result<(), Error> { + if self.magic != PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0 { + bail!("MediaContentHeader: wrong magic"); + } + if self.content_magic != content_magic { + bail!("MediaContentHeader: wrong content magic"); + } + if self.size < min_size || self.size > max_size { + bail!("MediaContentHeader: got unexpected size"); + } + Ok(()) + } + + /// Returns the content Uuid + pub fn content_uuid(&self) -> Uuid { + Uuid::from(self.uuid) + } +} + + +impl BlockHeader { + + pub const SIZE: usize = PROXMOX_TAPE_BLOCK_SIZE; + + /// Allocates a new instance on the heap + pub fn new() -> Box { + use std::alloc::{alloc_zeroed, Layout}; + + // align to PAGESIZE, so that we can use it with SG_IO + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize; + + let mut buffer = unsafe { + let ptr = alloc_zeroed( + Layout::from_size_align(Self::SIZE, page_size) + .unwrap(), + ); + Box::from_raw( + std::slice::from_raw_parts_mut(ptr, Self::SIZE - 16) + as *mut [u8] as *mut Self + ) + }; + buffer.magic = PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0; + buffer + } + + /// Set the `size` field + pub fn set_size(&mut self, size: usize) { + let size = size.to_le_bytes(); + self.size.copy_from_slice(&size[..3]); + } + + /// Returns the `size` field + pub fn size(&self) -> usize { + (self.size[0] as usize) + ((self.size[1] as usize)<<8) + ((self.size[2] as usize)<<16) + } + + /// Set the `seq_nr` field + pub fn set_seq_nr(&mut self, seq_nr: u32) { + self.seq_nr = seq_nr.to_le(); + } + + /// Returns the `seq_nr` field + pub fn seq_nr(&self) -> u32 { + u32::from_le(self.seq_nr) + } +} + +/// Changer element status. +/// +/// Drive and slots may be `Empty`, or contain some media, either +/// with known volume tag `VolumeTag(String)`, or without (`Full`). +#[derive(Serialize, Deserialize, Debug)] +pub enum ElementStatus { + Empty, + Full, + VolumeTag(String), +} + +/// Changer drive status. +#[derive(Serialize, Deserialize)] +pub struct DriveStatus { + /// The slot the element was loaded from (if known). + pub loaded_slot: Option, + /// The status. + pub status: ElementStatus, + /// Drive Identifier (Serial number) + pub drive_serial_number: Option, + /// Drive Vendor + pub vendor: Option, + /// Drive Model + pub model: Option, + /// Element Address + pub element_address: u16, +} + +/// Storage element status. +#[derive(Serialize, Deserialize)] +pub struct StorageElementStatus { + /// Flag for Import/Export slots + pub import_export: bool, + /// The status. + pub status: ElementStatus, + /// Element Address + pub element_address: u16, +} + +/// Transport element status. +#[derive(Serialize, Deserialize)] +pub struct TransportElementStatus { + /// The status. + pub status: ElementStatus, + /// Element Address + pub element_address: u16, +} + +/// Changer status - show drive/slot usage +#[derive(Serialize, Deserialize)] +pub struct MtxStatus { + /// List of known drives + pub drives: Vec, + /// List of known storage slots + pub slots: Vec, + /// Transport elements + /// + /// Note: Some libraries do not report transport elements. + pub transports: Vec, +} + +impl MtxStatus { + + pub fn slot_address(&self, slot: u64) -> Result { + if slot == 0 { + bail!("invalid slot number '{}' (slots numbers starts at 1)", slot); + } + if slot > (self.slots.len() as u64) { + bail!("invalid slot number '{}' (max {} slots)", slot, self.slots.len()); + } + + Ok(self.slots[(slot -1) as usize].element_address) + } + + pub fn drive_address(&self, drivenum: u64) -> Result { + if drivenum >= (self.drives.len() as u64) { + bail!("invalid drive number '{}'", drivenum); + } + + Ok(self.drives[drivenum as usize].element_address) + } + + pub fn transport_address(&self) -> u16 { + // simply use first transport + // (are there changers exposing more than one?) + // defaults to 0 for changer that do not report transports + self + .transports + .get(0) + .map(|t| t.element_address) + .unwrap_or(0u16) + } + + pub fn find_free_slot(&self, import_export: bool) -> Option { + let mut free_slot = None; + for (i, slot_info) in self.slots.iter().enumerate() { + if slot_info.import_export != import_export { + continue; // skip slots of wrong type + } + if let ElementStatus::Empty = slot_info.status { + free_slot = Some((i+1) as u64); + break; + } + } + free_slot + } + + pub fn mark_import_export_slots(&mut self, config: &ScsiTapeChanger) -> Result<(), Error>{ + let mut export_slots: HashSet = HashSet::new(); + + if let Some(slots) = &config.export_slots { + let slots: Value = parse_property_string(&slots, &SLOT_ARRAY_SCHEMA)?; + export_slots = slots + .as_array() + .unwrap() + .iter() + .filter_map(|v| v.as_u64()) + .collect(); + } + + for (i, entry) in self.slots.iter_mut().enumerate() { + let slot = i as u64 + 1; + if export_slots.contains(&slot) { + entry.import_export = true; // mark as IMPORT/EXPORT + } + } + + Ok(()) + } +} diff --git a/src/tape/linux_list_drives.rs b/pbs-tape/src/linux_list_drives.rs similarity index 82% rename from src/tape/linux_list_drives.rs rename to pbs-tape/src/linux_list_drives.rs index 29aaae4a..b8a40d8e 100644 --- a/src/tape/linux_list_drives.rs +++ b/pbs-tape/src/linux_list_drives.rs @@ -1,7 +1,13 @@ use std::path::{Path, PathBuf}; use std::collections::HashMap; +use std::fs::{OpenOptions, File}; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::io::AsRawFd; -use anyhow::{bail, Error}; +use anyhow::{bail, format_err, Error}; +use nix::fcntl::{fcntl, FcntlArg, OFlag}; + +use proxmox::sys::error::SysResult; use pbs_tools::fs::scan_subdir; use pbs_api_types::{DeviceKind, OptionalDeviceIdentification, TapeDeviceInfo}; @@ -248,6 +254,61 @@ pub fn check_drive_path( Ok(()) } + +/// Check for correct Major/Minor numbers +pub fn check_tape_is_lto_tape_device(file: &File) -> Result<(), Error> { + + let stat = nix::sys::stat::fstat(file.as_raw_fd())?; + + let devnum = stat.st_rdev; + + let major = unsafe { libc::major(devnum) }; + let _minor = unsafe { libc::minor(devnum) }; + + if major == 9 { + bail!("not a scsi-generic tape device (cannot use linux tape devices)"); + } + + if major != 21 { + bail!("not a scsi-generic tape device"); + } + + Ok(()) +} + +/// Opens a Lto tape device +/// +/// The open call use O_NONBLOCK, but that flag is cleard after open +/// succeeded. This also checks if the device is a non-rewinding tape +/// device. +pub fn open_lto_tape_device( + path: &str, +) -> Result { + + let file = OpenOptions::new() + .read(true) + .write(true) + .custom_flags(libc::O_NONBLOCK) + .open(path)?; + + // clear O_NONBLOCK from now on. + + let flags = fcntl(file.as_raw_fd(), FcntlArg::F_GETFL) + .into_io_result()?; + + let mut flags = OFlag::from_bits_truncate(flags); + flags.remove(OFlag::O_NONBLOCK); + + fcntl(file.as_raw_fd(), FcntlArg::F_SETFL(flags)) + .into_io_result()?; + + check_tape_is_lto_tape_device(&file) + .map_err(|err| format_err!("device type check {:?} failed - {}", path, err))?; + + Ok(file) +} + + // shell completion helper /// List changer device paths diff --git a/src/tape/changer/sg_pt_changer.rs b/pbs-tape/src/sg_pt_changer.rs similarity index 99% rename from src/tape/changer/sg_pt_changer.rs rename to pbs-tape/src/sg_pt_changer.rs index 0f0c0913..223079d2 100644 --- a/src/tape/changer/sg_pt_changer.rs +++ b/pbs-tape/src/sg_pt_changer.rs @@ -1,5 +1,4 @@ //! SCSI changer implementation using libsgutil2 - use std::os::unix::prelude::AsRawFd; use std::io::Read; use std::collections::HashMap; @@ -14,16 +13,8 @@ use proxmox::tools::io::ReadExt; use pbs_api_types::ScsiTapeChanger; use crate::{ - tape::{ - changer::{ - DriveStatus, - ElementStatus, - StorageElementStatus, - TransportElementStatus, - MtxStatus, - }, - }, - tools::sgutils2::{ + ElementStatus,MtxStatus,TransportElementStatus,DriveStatus,StorageElementStatus, + sgutils2::{ SgRaw, SENSE_KEY_NOT_READY, ScsiError, diff --git a/src/tape/drive/lto/sg_tape.rs b/pbs-tape/src/sg_tape.rs similarity index 90% rename from src/tape/drive/lto/sg_tape.rs rename to pbs-tape/src/sg_tape.rs index 6b570e69..c41de85f 100644 --- a/src/tape/drive/lto/sg_tape.rs +++ b/pbs-tape/src/sg_tape.rs @@ -4,6 +4,7 @@ use std::os::unix::fs::OpenOptionsExt; use std::os::unix::io::AsRawFd; use std::path::Path; use std::convert::TryFrom; +use std::convert::TryInto; use anyhow::{bail, format_err, Error}; use endian_trait::Endian; @@ -29,19 +30,15 @@ use proxmox::{ tools::io::{ReadExt, WriteExt}, }; -use pbs_api_types::{MamAttribute, Lp17VolumeStatistics}; +use pbs_api_types::{MamAttribute, Lp17VolumeStatistics, LtoDriveAndMediaStatus}; use crate::{ - tape::{ - BlockRead, - BlockReadError, - BlockWrite, - file_formats::{ - BlockedWriter, - BlockedReader, - }, - }, - tools::sgutils2::{ + BlockRead, + BlockReadError, + BlockWrite, + BlockedWriter, + BlockedReader, + sgutils2::{ SgRaw, SenseInfo, ScsiError, @@ -722,6 +719,18 @@ impl SgTape { BlockedReader::open(reader) } + /// Set all options we need/want + pub fn set_default_options(&mut self) -> Result<(), Error> { + + let compression = Some(true); + let block_length = Some(0); // variable length mode + let buffer_mode = Some(true); // Always use drive buffer + + self.set_drive_options(compression, block_length, buffer_mode)?; + + Ok(()) + } + /// Set important drive options pub fn set_drive_options( &mut self, @@ -845,6 +854,77 @@ impl SgTape { density_code: block_descriptor.density_code, }) } + + /// Get Tape and Media status + pub fn get_drive_and_media_status(&mut self) -> Result { + + let drive_status = self.read_drive_status()?; + + let alert_flags = self.tape_alert_flags() + .map(|flags| format!("{:?}", flags)) + .ok(); + + let mut status = LtoDriveAndMediaStatus { + vendor: self.info().vendor.clone(), + product: self.info().product.clone(), + revision: self.info().revision.clone(), + blocksize: drive_status.block_length, + compression: drive_status.compression, + buffer_mode: drive_status.buffer_mode, + density: drive_status.density_code.try_into()?, + alert_flags, + write_protect: None, + file_number: None, + block_number: None, + manufactured: None, + bytes_read: None, + bytes_written: None, + medium_passes: None, + medium_wearout: None, + volume_mounts: None, + }; + + if self.test_unit_ready().is_ok() { + + if drive_status.write_protect { + status.write_protect = Some(drive_status.write_protect); + } + + let position = self.position()?; + + status.file_number = Some(position.logical_file_id); + status.block_number = Some(position.logical_object_number); + + if let Ok(mam) = self.cartridge_memory() { + + let usage = mam_extract_media_usage(&mam)?; + + status.manufactured = Some(usage.manufactured); + status.bytes_read = Some(usage.bytes_read); + status.bytes_written = Some(usage.bytes_written); + + if let Ok(volume_stats) = self.volume_statistics() { + + let passes = std::cmp::max( + volume_stats.beginning_of_medium_passes, + volume_stats.middle_of_tape_passes, + ); + + // assume max. 16000 medium passes + // see: https://en.wikipedia.org/wiki/Linear_Tape-Open + let wearout: f64 = (passes as f64)/(16000.0 as f64); + + status.medium_passes = Some(passes); + status.medium_wearout = Some(wearout); + + status.volume_mounts = Some(volume_stats.volume_mounts); + } + } + } + + Ok(status) + } + } impl Drop for SgTape { diff --git a/src/tape/drive/lto/sg_tape/encryption.rs b/pbs-tape/src/sg_tape/encryption.rs similarity index 99% rename from src/tape/drive/lto/sg_tape/encryption.rs rename to pbs-tape/src/sg_tape/encryption.rs index e86ecbfb..6d99d317 100644 --- a/src/tape/drive/lto/sg_tape/encryption.rs +++ b/pbs-tape/src/sg_tape/encryption.rs @@ -6,10 +6,7 @@ use endian_trait::Endian; use proxmox::tools::io::{ReadExt, WriteExt}; -use crate::tools::sgutils2::{ - SgRaw, - alloc_page_aligned_buffer, -}; +use crate::sgutils2::{SgRaw, alloc_page_aligned_buffer}; /// Test if drive supports hardware encryption /// diff --git a/src/tape/drive/lto/sg_tape/mam.rs b/pbs-tape/src/sg_tape/mam.rs similarity index 99% rename from src/tape/drive/lto/sg_tape/mam.rs rename to pbs-tape/src/sg_tape/mam.rs index 69f21184..b9796940 100644 --- a/src/tape/drive/lto/sg_tape/mam.rs +++ b/pbs-tape/src/sg_tape/mam.rs @@ -9,10 +9,9 @@ use proxmox::tools::io::ReadExt; use pbs_api_types::MamAttribute; -use crate::{ - tools::sgutils2::SgRaw, - tape::drive::lto::TapeAlertFlags, -}; +use crate::sgutils2::SgRaw; + +use super::TapeAlertFlags; // Read Medium auxiliary memory attributes (MAM) // see IBM SCSI reference: https://www-01.ibm.com/support/docview.wss?uid=ssg1S7003556&aid=1 diff --git a/src/tape/drive/lto/sg_tape/report_density.rs b/pbs-tape/src/sg_tape/report_density.rs similarity index 98% rename from src/tape/drive/lto/sg_tape/report_density.rs rename to pbs-tape/src/sg_tape/report_density.rs index 9b0a233f..6baddfe1 100644 --- a/src/tape/drive/lto/sg_tape/report_density.rs +++ b/pbs-tape/src/sg_tape/report_density.rs @@ -5,7 +5,7 @@ use std::os::unix::io::AsRawFd; use proxmox::tools::io::ReadExt; -use crate::tools::sgutils2::SgRaw; +use crate::sgutils2::SgRaw; #[repr(C, packed)] #[derive(Endian)] diff --git a/src/tape/drive/lto/sg_tape/tape_alert_flags.rs b/pbs-tape/src/sg_tape/tape_alert_flags.rs similarity index 99% rename from src/tape/drive/lto/sg_tape/tape_alert_flags.rs rename to pbs-tape/src/sg_tape/tape_alert_flags.rs index 63381559..481618f4 100644 --- a/src/tape/drive/lto/sg_tape/tape_alert_flags.rs +++ b/pbs-tape/src/sg_tape/tape_alert_flags.rs @@ -5,7 +5,7 @@ use anyhow::{bail, format_err, Error}; use proxmox::tools::io::ReadExt; -use crate::tools::sgutils2::SgRaw; +use crate::sgutils2::SgRaw; bitflags::bitflags!{ diff --git a/src/tape/drive/lto/sg_tape/volume_statistics.rs b/pbs-tape/src/sg_tape/volume_statistics.rs similarity index 99% rename from src/tape/drive/lto/sg_tape/volume_statistics.rs rename to pbs-tape/src/sg_tape/volume_statistics.rs index b3878784..cc60da10 100644 --- a/src/tape/drive/lto/sg_tape/volume_statistics.rs +++ b/pbs-tape/src/sg_tape/volume_statistics.rs @@ -8,7 +8,7 @@ use proxmox::tools::io::ReadExt; use pbs_api_types::Lp17VolumeStatistics; -use crate::tools::sgutils2::SgRaw; +use crate::sgutils2::SgRaw; /// SCSI command to query volume statistics /// diff --git a/src/tools/sgutils2.rs b/pbs-tape/src/sgutils2.rs similarity index 100% rename from src/tools/sgutils2.rs rename to pbs-tape/src/sgutils2.rs diff --git a/src/tape/tape_read.rs b/pbs-tape/src/tape_read.rs similarity index 100% rename from src/tape/tape_read.rs rename to pbs-tape/src/tape_read.rs diff --git a/src/tape/tape_write.rs b/pbs-tape/src/tape_write.rs similarity index 97% rename from src/tape/tape_write.rs rename to pbs-tape/src/tape_write.rs index 593d1a29..d3d6aaa2 100644 --- a/src/tape/tape_write.rs +++ b/pbs-tape/src/tape_write.rs @@ -1,6 +1,6 @@ use endian_trait::Endian; -use crate::tape::file_formats::MediaContentHeader; +use crate::MediaContentHeader; /// Write trait for tape devices /// diff --git a/src/api2/config/changer.rs b/src/api2/config/changer.rs index 5ef974a0..a9f7622f 100644 --- a/src/api2/config/changer.rs +++ b/src/api2/config/changer.rs @@ -16,13 +16,7 @@ use pbs_api_types::{ PRIV_TAPE_AUDIT, PRIV_TAPE_MODIFY, }; use pbs_config::CachedUserInfo; - -use crate::{ - tape::{ - linux_tape_changer_list, - check_drive_path, - }, -}; +use pbs_tape::linux_list_drives::{linux_tape_changer_list, check_drive_path}; #[api( protected: true, diff --git a/src/api2/config/drive.rs b/src/api2/config/drive.rs index 703bf00a..26b4ddd5 100644 --- a/src/api2/config/drive.rs +++ b/src/api2/config/drive.rs @@ -10,12 +10,7 @@ use pbs_api_types::{ }; use pbs_config::CachedUserInfo; -use crate::{ - tape::{ - lto_tape_device_list, - check_drive_path, - }, -}; +use pbs_tape::linux_list_drives::{lto_tape_device_list, check_drive_path}; #[api( protected: true, diff --git a/src/api2/tape/changer.rs b/src/api2/tape/changer.rs index 36250788..3d9dcb70 100644 --- a/src/api2/tape/changer.rs +++ b/src/api2/tape/changer.rs @@ -12,20 +12,21 @@ use pbs_api_types::{ CHANGER_NAME_SCHEMA, PRIV_TAPE_AUDIT, PRIV_TAPE_READ, }; use pbs_config::CachedUserInfo; +use pbs_tape::{ + ElementStatus, + linux_list_drives::{lookup_device_identification, linux_tape_changer_list}, +}; use crate::{ tape::{ TAPE_STATUS_DIR, Inventory, - linux_tape_changer_list, changer::{ OnlineStatusMap, - ElementStatus, ScsiMediaChange, mtx_status_to_online_set, }, drive::get_tape_device_state, - lookup_device_identification, }, }; diff --git a/src/api2/tape/drive.rs b/src/api2/tape/drive.rs index 9c8bacc3..58f49f43 100644 --- a/src/api2/tape/drive.rs +++ b/src/api2/tape/drive.rs @@ -31,6 +31,11 @@ use pbs_api_types::{ use pbs_datastore::task_log; use pbs_api_types::{PRIV_TAPE_AUDIT, PRIV_TAPE_READ, PRIV_TAPE_WRITE}; use pbs_config::CachedUserInfo; +use pbs_tape::{ + BlockReadError, + sg_tape::tape_alert_flags_critical, + linux_list_drives::{lto_tape_device_list, lookup_device_identification, open_lto_tape_device}, +}; use crate::{ api2::tape::restore::{ @@ -43,12 +48,9 @@ use crate::{ Inventory, MediaCatalog, MediaId, - BlockReadError, lock_media_set, lock_media_pool, lock_unassigned_media_pool, - lto_tape_device_list, - lookup_device_identification, file_formats::{ MediaLabel, MediaSetLabel, @@ -56,7 +58,6 @@ use crate::{ drive::{ TapeDriver, LtoTapeHandle, - open_lto_tape_device, open_lto_tape_drive, media_changer, required_media_changer, @@ -64,7 +65,6 @@ use crate::{ lock_tape_device, set_tape_device_state, get_tape_device_state, - tape_alert_flags_critical, }, changer::update_changer_online_status, }, diff --git a/src/api2/tape/mod.rs b/src/api2/tape/mod.rs index 1e7731a9..049bb936 100644 --- a/src/api2/tape/mod.rs +++ b/src/api2/tape/mod.rs @@ -13,13 +13,7 @@ use proxmox::{ }; use pbs_api_types::TapeDeviceInfo; - -use crate::{ - tape::{ - lto_tape_device_list, - linux_tape_changer_list, - }, -}; +use pbs_tape::linux_list_drives::{lto_tape_device_list, linux_tape_changer_list}; pub mod drive; pub mod changer; diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs index 24e8765f..4ab60e8f 100644 --- a/src/api2/tape/restore.rs +++ b/src/api2/tape/restore.rs @@ -42,6 +42,10 @@ use pbs_datastore::index::IndexFile; use pbs_datastore::manifest::{archive_type, ArchiveType, BackupManifest, MANIFEST_BLOB_NAME}; use pbs_datastore::task::TaskState; use pbs_config::CachedUserInfo; +use pbs_tape::{ + TapeRead, BlockReadError, MediaContentHeader, + PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0, +}; use crate::{ tools::ParallelHandler, @@ -52,8 +56,6 @@ use crate::{ }, tape::{ TAPE_STATUS_DIR, - TapeRead, - BlockReadError, MediaId, MediaSet, MediaCatalog, @@ -65,11 +67,9 @@ use crate::{ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1, PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, - PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0, PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0, PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1, PROXMOX_BACKUP_CATALOG_ARCHIVE_MAGIC_1_0, - MediaContentHeader, ChunkArchiveHeader, ChunkArchiveDecoder, SnapshotArchiveHeader, diff --git a/src/bin/proxmox-tape.rs b/src/bin/proxmox-tape.rs index 276ec1e2..615c8a91 100644 --- a/src/bin/proxmox-tape.rs +++ b/src/bin/proxmox-tape.rs @@ -30,11 +30,13 @@ use pbs_api_types::{ DRIVE_NAME_SCHEMA, MEDIA_LABEL_SCHEMA, MEDIA_POOL_NAME_SCHEMA, TAPE_RESTORE_SNAPSHOT_SCHEMA, }; +use pbs_tape::{ + PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0, BlockReadError, MediaContentHeader, +}; use proxmox_backup::{ api2, tape::{ - BlockReadError, drive::{ open_drive, lock_tape_device, @@ -44,8 +46,6 @@ use proxmox_backup::{ complete_media_set_uuid, complete_media_set_snapshots, file_formats::{ - PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0, - MediaContentHeader, proxmox_tape_magic_to_text, }, }, diff --git a/src/bin/proxmox_tape/changer.rs b/src/bin/proxmox_tape/changer.rs index 12d0952c..4c62218e 100644 --- a/src/bin/proxmox_tape/changer.rs +++ b/src/bin/proxmox_tape/changer.rs @@ -18,13 +18,9 @@ use pbs_config::drive::{ use pbs_api_types::CHANGER_NAME_SCHEMA; -use proxmox_backup::{ - api2, - tape::{ - complete_changer_path, - drive::media_changer, - }, -}; +use pbs_tape::linux_list_drives::{complete_changer_path}; + +use proxmox_backup::{api2, tape::drive::media_changer}; pub fn lookup_changer_name( param: &Value, diff --git a/src/bin/proxmox_tape/drive.rs b/src/bin/proxmox_tape/drive.rs index b33de823..7972ed2a 100644 --- a/src/bin/proxmox_tape/drive.rs +++ b/src/bin/proxmox_tape/drive.rs @@ -18,7 +18,9 @@ use pbs_config::drive::{ complete_lto_drive_name, }; -use proxmox_backup::{api2, tape::complete_drive_path}; +use pbs_tape::linux_list_drives::{complete_drive_path}; + +use proxmox_backup::api2; pub fn drive_commands() -> CommandLineInterface { diff --git a/src/bin/sg-tape-cmd.rs b/src/bin/sg-tape-cmd.rs index f8508176..ee4c32c2 100644 --- a/src/bin/sg-tape-cmd.rs +++ b/src/bin/sg-tape-cmd.rs @@ -23,14 +23,14 @@ use pbs_api_types::{ MEDIA_SET_UUID_SCHEMA, LtoTapeDrive, }; +use pbs_tape::linux_list_drives::{open_lto_tape_device, check_tape_is_lto_tape_device}; + use proxmox_backup::{ tape::{ drive::{ TapeDriver, LtoTapeHandle, - open_lto_tape_device, open_lto_tape_drive, - check_tape_is_lto_tape_device, }, }, }; diff --git a/src/tape/changer/mod.rs b/src/tape/changer/mod.rs index 4e6d6bb6..cc9ee734 100644 --- a/src/tape/changer/mod.rs +++ b/src/tape/changer/mod.rs @@ -1,160 +1,19 @@ //! Media changer implementation (SCSI media changer) -pub mod sg_pt_changer; - pub mod mtx; mod online_status_map; pub use online_status_map::*; -use std::collections::HashSet; use std::path::PathBuf; use anyhow::{bail, Error}; -use serde::{Serialize, Deserialize}; -use serde_json::Value; -use proxmox::{ - api::schema::parse_property_string, - tools::fs::{ - CreateOptions, - replace_file, - file_read_optional_string, - }, -}; +use proxmox::tools::fs::{CreateOptions, replace_file, file_read_optional_string}; -use pbs_api_types::{SLOT_ARRAY_SCHEMA, ScsiTapeChanger, LtoTapeDrive}; +use pbs_api_types::{ScsiTapeChanger, LtoTapeDrive}; -/// Changer element status. -/// -/// Drive and slots may be `Empty`, or contain some media, either -/// with known volume tag `VolumeTag(String)`, or without (`Full`). -#[derive(Serialize, Deserialize, Debug)] -pub enum ElementStatus { - Empty, - Full, - VolumeTag(String), -} - -/// Changer drive status. -#[derive(Serialize, Deserialize)] -pub struct DriveStatus { - /// The slot the element was loaded from (if known). - pub loaded_slot: Option, - /// The status. - pub status: ElementStatus, - /// Drive Identifier (Serial number) - pub drive_serial_number: Option, - /// Drive Vendor - pub vendor: Option, - /// Drive Model - pub model: Option, - /// Element Address - pub element_address: u16, -} - -/// Storage element status. -#[derive(Serialize, Deserialize)] -pub struct StorageElementStatus { - /// Flag for Import/Export slots - pub import_export: bool, - /// The status. - pub status: ElementStatus, - /// Element Address - pub element_address: u16, -} - -/// Transport element status. -#[derive(Serialize, Deserialize)] -pub struct TransportElementStatus { - /// The status. - pub status: ElementStatus, - /// Element Address - pub element_address: u16, -} - -/// Changer status - show drive/slot usage -#[derive(Serialize, Deserialize)] -pub struct MtxStatus { - /// List of known drives - pub drives: Vec, - /// List of known storage slots - pub slots: Vec, - /// Transport elements - /// - /// Note: Some libraries do not report transport elements. - pub transports: Vec, -} - -impl MtxStatus { - - pub fn slot_address(&self, slot: u64) -> Result { - if slot == 0 { - bail!("invalid slot number '{}' (slots numbers starts at 1)", slot); - } - if slot > (self.slots.len() as u64) { - bail!("invalid slot number '{}' (max {} slots)", slot, self.slots.len()); - } - - Ok(self.slots[(slot -1) as usize].element_address) - } - - pub fn drive_address(&self, drivenum: u64) -> Result { - if drivenum >= (self.drives.len() as u64) { - bail!("invalid drive number '{}'", drivenum); - } - - Ok(self.drives[drivenum as usize].element_address) - } - - pub fn transport_address(&self) -> u16 { - // simply use first transport - // (are there changers exposing more than one?) - // defaults to 0 for changer that do not report transports - self - .transports - .get(0) - .map(|t| t.element_address) - .unwrap_or(0u16) - } - - pub fn find_free_slot(&self, import_export: bool) -> Option { - let mut free_slot = None; - for (i, slot_info) in self.slots.iter().enumerate() { - if slot_info.import_export != import_export { - continue; // skip slots of wrong type - } - if let ElementStatus::Empty = slot_info.status { - free_slot = Some((i+1) as u64); - break; - } - } - free_slot - } - - pub fn mark_import_export_slots(&mut self, config: &ScsiTapeChanger) -> Result<(), Error>{ - let mut export_slots: HashSet = HashSet::new(); - - if let Some(slots) = &config.export_slots { - let slots: Value = parse_property_string(&slots, &SLOT_ARRAY_SCHEMA)?; - export_slots = slots - .as_array() - .unwrap() - .iter() - .filter_map(|v| v.as_u64()) - .collect(); - } - - for (i, entry) in self.slots.iter_mut().enumerate() { - let slot = i as u64 + 1; - if export_slots.contains(&slot) { - entry.import_export = true; // mark as IMPORT/EXPORT - } - } - - Ok(()) - } -} +use pbs_tape::{sg_pt_changer, MtxStatus, ElementStatus}; /// Interface to SCSI changer devices pub trait ScsiMediaChange { diff --git a/src/tape/changer/mtx/mtx_wrapper.rs b/src/tape/changer/mtx/mtx_wrapper.rs index e3901773..7fcf4ad4 100644 --- a/src/tape/changer/mtx/mtx_wrapper.rs +++ b/src/tape/changer/mtx/mtx_wrapper.rs @@ -2,11 +2,11 @@ use anyhow::Error; use pbs_tools::run_command; use pbs_api_types::ScsiTapeChanger; +use pbs_tape::MtxStatus; use crate::{ tape::changer::{ - MtxStatus, - mtx::parse_mtx_status, + mtx::parse_mtx_status, }, }; diff --git a/src/tape/changer/mtx/parse_mtx_status.rs b/src/tape/changer/mtx/parse_mtx_status.rs index b2f5c5d7..c22c7f4a 100644 --- a/src/tape/changer/mtx/parse_mtx_status.rs +++ b/src/tape/changer/mtx/parse_mtx_status.rs @@ -1,20 +1,12 @@ use anyhow::Error; -use nom::{ - bytes::complete::{take_while, tag}, -}; +use nom::bytes::complete::{take_while, tag}; -use crate::{ - tools::nom::{ - parse_complete, multispace0, multispace1, parse_u64, - parse_failure, parse_error, IResult, - }, - tape::changer::{ - ElementStatus, - MtxStatus, - DriveStatus, - StorageElementStatus, - }, +use pbs_tape::{ElementStatus, MtxStatus, DriveStatus, StorageElementStatus}; + +use crate::tools::nom::{ + parse_complete, multispace0, multispace1, parse_u64, + parse_failure, parse_error, IResult, }; diff --git a/src/tape/changer/online_status_map.rs b/src/tape/changer/online_status_map.rs index 2a3aaf11..43760663 100644 --- a/src/tape/changer/online_status_map.rs +++ b/src/tape/changer/online_status_map.rs @@ -7,18 +7,10 @@ use proxmox::tools::Uuid; use proxmox::api::section_config::SectionConfigData; use pbs_api_types::{VirtualTapeDrive, ScsiTapeChanger}; +use pbs_tape::{ElementStatus, MtxStatus}; -use crate::{ - tape::{ - Inventory, - changer::{ - MediaChange, - MtxStatus, - ElementStatus, - ScsiMediaChange, - }, - }, -}; +use crate::tape::Inventory; +use crate::tape::changer::{MediaChange, ScsiMediaChange}; /// Helper to update media online status /// diff --git a/src/tape/drive/lto/mod.rs b/src/tape/drive/lto/mod.rs index dd9c9aac..7dcd2809 100644 --- a/src/tape/drive/lto/mod.rs +++ b/src/tape/drive/lto/mod.rs @@ -11,41 +11,29 @@ //! //! - unability to detect EOT (you just get EIO) -mod sg_tape; -pub use sg_tape::*; - -use std::fs::{OpenOptions, File}; -use std::os::unix::fs::OpenOptionsExt; +use std::fs::File; use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; use std::convert::TryInto; use anyhow::{bail, format_err, Error}; -use nix::fcntl::{fcntl, FcntlArg, OFlag}; -use proxmox::{ - tools::Uuid, - sys::error::SysResult, -}; +use proxmox::tools::Uuid; use pbs_api_types::{ Fingerprint, MamAttribute, LtoDriveAndMediaStatus, LtoTapeDrive, Lp17VolumeStatistics, }; use pbs_config::key_config::KeyConfig; use pbs_tools::run_command; +use pbs_tape::{ + TapeWrite, TapeRead, BlockReadError, MediaContentHeader, + sg_tape::{SgTape, TapeAlertFlags}, + linux_list_drives::open_lto_tape_device, +}; use crate::{ tape::{ - TapeRead, - TapeWrite, - BlockReadError, - drive::{ - TapeDriver, - }, - file_formats::{ - PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, - MediaSetLabel, - MediaContentHeader, - }, + drive::TapeDriver, + file_formats::{PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, MediaSetLabel}, }, }; @@ -94,13 +82,7 @@ impl LtoTapeHandle { /// Set all options we need/want pub fn set_default_options(&mut self) -> Result<(), Error> { - - let compression = Some(true); - let block_length = Some(0); // variable length mode - let buffer_mode = Some(true); // Always use drive buffer - - self.set_drive_options(compression, block_length, buffer_mode)?; - + self.sg_tape.set_default_options()?; Ok(()) } @@ -121,72 +103,7 @@ impl LtoTapeHandle { /// Get Tape and Media status pub fn get_drive_and_media_status(&mut self) -> Result { - - let drive_status = self.sg_tape.read_drive_status()?; - - let alert_flags = self.tape_alert_flags() - .map(|flags| format!("{:?}", flags)) - .ok(); - - let mut status = LtoDriveAndMediaStatus { - vendor: self.sg_tape.info().vendor.clone(), - product: self.sg_tape.info().product.clone(), - revision: self.sg_tape.info().revision.clone(), - blocksize: drive_status.block_length, - compression: drive_status.compression, - buffer_mode: drive_status.buffer_mode, - density: drive_status.density_code.try_into()?, - alert_flags, - write_protect: None, - file_number: None, - block_number: None, - manufactured: None, - bytes_read: None, - bytes_written: None, - medium_passes: None, - medium_wearout: None, - volume_mounts: None, - }; - - if self.sg_tape.test_unit_ready().is_ok() { - - if drive_status.write_protect { - status.write_protect = Some(drive_status.write_protect); - } - - let position = self.sg_tape.position()?; - - status.file_number = Some(position.logical_file_id); - status.block_number = Some(position.logical_object_number); - - if let Ok(mam) = self.cartridge_memory() { - - let usage = mam_extract_media_usage(&mam)?; - - status.manufactured = Some(usage.manufactured); - status.bytes_read = Some(usage.bytes_read); - status.bytes_written = Some(usage.bytes_written); - - if let Ok(volume_stats) = self.volume_statistics() { - - let passes = std::cmp::max( - volume_stats.beginning_of_medium_passes, - volume_stats.middle_of_tape_passes, - ); - - // assume max. 16000 medium passes - // see: https://en.wikipedia.org/wiki/Linear_Tape-Open - let wearout: f64 = (passes as f64)/(16000.0 as f64); - - status.medium_passes = Some(passes); - status.medium_wearout = Some(wearout); - - status.volume_mounts = Some(volume_stats.volume_mounts); - } - } - } - - Ok(status) + self.sg_tape.get_drive_and_media_status() } pub fn forward_space_count_files(&mut self, count: usize) -> Result<(), Error> { @@ -413,59 +330,6 @@ impl TapeDriver for LtoTapeHandle { } } -/// Check for correct Major/Minor numbers -pub fn check_tape_is_lto_tape_device(file: &File) -> Result<(), Error> { - - let stat = nix::sys::stat::fstat(file.as_raw_fd())?; - - let devnum = stat.st_rdev; - - let major = unsafe { libc::major(devnum) }; - let _minor = unsafe { libc::minor(devnum) }; - - if major == 9 { - bail!("not a scsi-generic tape device (cannot use linux tape devices)"); - } - - if major != 21 { - bail!("not a scsi-generic tape device"); - } - - Ok(()) -} - -/// Opens a Lto tape device -/// -/// The open call use O_NONBLOCK, but that flag is cleard after open -/// succeeded. This also checks if the device is a non-rewinding tape -/// device. -pub fn open_lto_tape_device( - path: &str, -) -> Result { - - let file = OpenOptions::new() - .read(true) - .write(true) - .custom_flags(libc::O_NONBLOCK) - .open(path)?; - - // clear O_NONBLOCK from now on. - - let flags = fcntl(file.as_raw_fd(), FcntlArg::F_GETFL) - .into_io_result()?; - - let mut flags = OFlag::from_bits_truncate(flags); - flags.remove(OFlag::O_NONBLOCK); - - fcntl(file.as_raw_fd(), FcntlArg::F_SETFL(flags)) - .into_io_result()?; - - check_tape_is_lto_tape_device(&file) - .map_err(|err| format_err!("device type check {:?} failed - {}", path, err))?; - - Ok(file) -} - fn run_sg_tape_cmd(subcmd: &str, args: &[&str], fd: RawFd) -> Result { let mut command = std::process::Command::new( "/usr/lib/x86_64-linux-gnu/proxmox-backup/sg-tape-cmd"); diff --git a/src/tape/drive/mod.rs b/src/tape/drive/mod.rs index a42ebafe..afb91b64 100644 --- a/src/tape/drive/mod.rs +++ b/src/tape/drive/mod.rs @@ -33,26 +33,26 @@ use pbs_config::key_config::KeyConfig; use pbs_datastore::task::TaskState; use pbs_datastore::task_log; +use pbs_tape::{ + TapeWrite, TapeRead, BlockReadError, MediaContentHeader, + sg_tape::TapeAlertFlags, +}; + use crate::{ server::{ send_load_media_email, WorkerTask, }, tape::{ - TapeWrite, - TapeRead, - BlockReadError, MediaId, drive::{ virtual_tape::open_virtual_tape_drive, - lto::TapeAlertFlags, }, file_formats::{ PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0, PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, MediaLabel, MediaSetLabel, - MediaContentHeader, }, changer::{ MediaChange, diff --git a/src/tape/drive/virtual_tape.rs b/src/tape/drive/virtual_tape.rs index 0dadeede..8038231c 100644 --- a/src/tape/drive/virtual_tape.rs +++ b/src/tape/drive/virtual_tape.rs @@ -11,33 +11,31 @@ use proxmox::tools::{ }; use pbs_config::key_config::KeyConfig; +use pbs_tape::{ + TapeWrite, + TapeRead, + BlockedReader, + BlockedWriter, + BlockReadError, + MtxStatus, + DriveStatus, + ElementStatus, + StorageElementStatus, + MediaContentHeader, + EmulateTapeReader, + EmulateTapeWriter, +}; use crate::{ tape::{ - TapeWrite, - TapeRead, - BlockReadError, - changer::{ - MediaChange, - MtxStatus, - DriveStatus, - ElementStatus, - StorageElementStatus, - }, - drive::{ - VirtualTapeDrive, - TapeDriver, + drive::{ + VirtualTapeDrive, + TapeDriver, + MediaChange, }, file_formats::{ MediaSetLabel, - MediaContentHeader, PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, - BlockedReader, - BlockedWriter, - }, - helpers::{ - EmulateTapeReader, - EmulateTapeWriter, }, }, }; diff --git a/src/tape/file_formats/catalog_archive.rs b/src/tape/file_formats/catalog_archive.rs index d48a2ca5..0b58958f 100644 --- a/src/tape/file_formats/catalog_archive.rs +++ b/src/tape/file_formats/catalog_archive.rs @@ -6,13 +6,15 @@ use proxmox::{ tools::Uuid, }; +use pbs_tape::{ + PROXMOX_TAPE_BLOCK_SIZE, + TapeWrite, MediaContentHeader, +}; + use crate::{ tape::{ - TapeWrite, file_formats::{ - PROXMOX_TAPE_BLOCK_SIZE, PROXMOX_BACKUP_CATALOG_ARCHIVE_MAGIC_1_0, - MediaContentHeader, CatalogArchiveHeader, }, }, diff --git a/src/tape/file_formats/chunk_archive.rs b/src/tape/file_formats/chunk_archive.rs index 94bada9c..c9d8524b 100644 --- a/src/tape/file_formats/chunk_archive.rs +++ b/src/tape/file_formats/chunk_archive.rs @@ -9,17 +9,16 @@ use proxmox::tools::{ }; use pbs_datastore::DataBlob; +use pbs_tape::{ + PROXMOX_TAPE_BLOCK_SIZE, + TapeWrite, MediaContentHeader, +}; -use crate::tape::{ - TapeWrite, - file_formats::{ - PROXMOX_TAPE_BLOCK_SIZE, - PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1, - PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0, - MediaContentHeader, - ChunkArchiveHeader, - ChunkArchiveEntryHeader, - }, +use crate::tape::file_formats::{ + PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1, + PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0, + ChunkArchiveHeader, + ChunkArchiveEntryHeader, }; /// Writes chunk archives to tape. diff --git a/src/tape/file_formats/mod.rs b/src/tape/file_formats/mod.rs index b75b7727..de43d6bc 100644 --- a/src/tape/file_formats/mod.rs +++ b/src/tape/file_formats/mod.rs @@ -3,8 +3,6 @@ use std::collections::HashMap; -use anyhow::{bail, Error}; -use bitflags::bitflags; use endian_trait::Endian; use serde::{Deserialize, Serialize}; @@ -12,12 +10,6 @@ use proxmox::tools::Uuid; use pbs_api_types::Fingerprint; -mod blocked_reader; -pub use blocked_reader::*; - -mod blocked_writer; -pub use blocked_writer::*; - mod chunk_archive; pub use chunk_archive::*; @@ -33,14 +25,6 @@ pub use multi_volume_writer::*; mod multi_volume_reader; pub use multi_volume_reader::*; -/// We use 256KB blocksize (always) -pub const PROXMOX_TAPE_BLOCK_SIZE: usize = 256*1024; - -// openssl::sha::sha256(b"Proxmox Tape Block Header v1.0")[0..8] -pub const PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0: [u8; 8] = [220, 189, 175, 202, 235, 160, 165, 40]; - -// openssl::sha::sha256(b"Proxmox Backup Content Header v1.0")[0..8]; -pub const PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0: [u8; 8] = [99, 238, 20, 159, 205, 242, 155, 12]; // openssl::sha::sha256(b"Proxmox Backup Tape Label v1.0")[0..8]; pub const PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0: [u8; 8] = [42, 5, 191, 60, 176, 48, 170, 57]; // openssl::sha::sha256(b"Proxmox Backup MediaSet Label v1.0") @@ -84,109 +68,6 @@ pub fn proxmox_tape_magic_to_text(magic: &[u8; 8]) -> Option { PROXMOX_TAPE_CONTENT_NAME.get(magic).map(|s| String::from(*s)) } -/// Tape Block Header with data payload -/// -/// All tape files are written as sequence of blocks. -/// -/// Note: this struct is large, never put this on the stack! -/// so we use an unsized type to avoid that. -/// -/// Tape data block are always read/written with a fixed size -/// (`PROXMOX_TAPE_BLOCK_SIZE`). But they may contain less data, so the -/// header has an additional size field. For streams of blocks, there -/// is a sequence number (`seq_nr`) which may be use for additional -/// error checking. -#[repr(C,packed)] -pub struct BlockHeader { - /// fixed value `PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0` - pub magic: [u8; 8], - pub flags: BlockHeaderFlags, - /// size as 3 bytes unsigned, little endian - pub size: [u8; 3], - /// block sequence number - pub seq_nr: u32, - pub payload: [u8], -} - -bitflags! { - /// Header flags (e.g. `END_OF_STREAM` or `INCOMPLETE`) - pub struct BlockHeaderFlags: u8 { - /// Marks the last block in a stream. - const END_OF_STREAM = 0b00000001; - /// Mark multivolume streams (when set in the last block) - const INCOMPLETE = 0b00000010; - } -} - -#[derive(Endian, Copy, Clone, Debug)] -#[repr(C,packed)] -/// Media Content Header -/// -/// All tape files start with this header. The header may contain some -/// informational data indicated by `size`. -/// -/// `| MediaContentHeader | header data (size) | stream data |` -/// -/// Note: The stream data following may be of any size. -pub struct MediaContentHeader { - /// fixed value `PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0` - pub magic: [u8; 8], - /// magic number for the content following - pub content_magic: [u8; 8], - /// unique ID to identify this data stream - pub uuid: [u8; 16], - /// stream creation time - pub ctime: i64, - /// Size of header data - pub size: u32, - /// Part number for multipart archives. - pub part_number: u8, - /// Reserved for future use - pub reserved_0: u8, - /// Reserved for future use - pub reserved_1: u8, - /// Reserved for future use - pub reserved_2: u8, -} - -impl MediaContentHeader { - - /// Create a new instance with autogenerated Uuid - pub fn new(content_magic: [u8; 8], size: u32) -> Self { - let uuid = *proxmox::tools::uuid::Uuid::generate() - .into_inner(); - Self { - magic: PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0, - content_magic, - uuid, - ctime: proxmox::tools::time::epoch_i64(), - size, - part_number: 0, - reserved_0: 0, - reserved_1: 0, - reserved_2: 0, - } - } - - /// Helper to check magic numbers and size constraints - pub fn check(&self, content_magic: [u8; 8], min_size: u32, max_size: u32) -> Result<(), Error> { - if self.magic != PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0 { - bail!("MediaContentHeader: wrong magic"); - } - if self.content_magic != content_magic { - bail!("MediaContentHeader: wrong content magic"); - } - if self.size < min_size || self.size > max_size { - bail!("MediaContentHeader: got unexpected size"); - } - Ok(()) - } - - /// Returns the content Uuid - pub fn content_uuid(&self) -> Uuid { - Uuid::from(self.uuid) - } -} #[derive(Deserialize, Serialize)] /// Header for chunk archives @@ -280,49 +161,3 @@ impl MediaSetLabel { } } -impl BlockHeader { - - pub const SIZE: usize = PROXMOX_TAPE_BLOCK_SIZE; - - /// Allocates a new instance on the heap - pub fn new() -> Box { - use std::alloc::{alloc_zeroed, Layout}; - - // align to PAGESIZE, so that we can use it with SG_IO - let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize; - - let mut buffer = unsafe { - let ptr = alloc_zeroed( - Layout::from_size_align(Self::SIZE, page_size) - .unwrap(), - ); - Box::from_raw( - std::slice::from_raw_parts_mut(ptr, Self::SIZE - 16) - as *mut [u8] as *mut Self - ) - }; - buffer.magic = PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0; - buffer - } - - /// Set the `size` field - pub fn set_size(&mut self, size: usize) { - let size = size.to_le_bytes(); - self.size.copy_from_slice(&size[..3]); - } - - /// Returns the `size` field - pub fn size(&self) -> usize { - (self.size[0] as usize) + ((self.size[1] as usize)<<8) + ((self.size[2] as usize)<<16) - } - - /// Set the `seq_nr` field - pub fn set_seq_nr(&mut self, seq_nr: u32) { - self.seq_nr = seq_nr.to_le(); - } - - /// Returns the `seq_nr` field - pub fn seq_nr(&self) -> u32 { - u32::from_le(self.seq_nr) - } -} diff --git a/src/tape/file_formats/multi_volume_reader.rs b/src/tape/file_formats/multi_volume_reader.rs index 801a7eab..84f861b5 100644 --- a/src/tape/file_formats/multi_volume_reader.rs +++ b/src/tape/file_formats/multi_volume_reader.rs @@ -4,10 +4,7 @@ use anyhow::{bail, Error}; use proxmox::tools::io::ReadExt; -use crate::tape::{ - TapeRead, - file_formats::MediaContentHeader, -}; +use pbs_tape::{TapeRead, MediaContentHeader}; /// Read multi volume data streams written by `MultiVolumeWriter` /// diff --git a/src/tape/file_formats/multi_volume_writer.rs b/src/tape/file_formats/multi_volume_writer.rs index d58285e9..6f2de9f0 100644 --- a/src/tape/file_formats/multi_volume_writer.rs +++ b/src/tape/file_formats/multi_volume_writer.rs @@ -2,10 +2,7 @@ use anyhow::Error; use proxmox::tools::Uuid; -use crate::tape::{ - TapeWrite, - file_formats::MediaContentHeader, -}; +use pbs_tape::{TapeWrite, MediaContentHeader}; /// Writes data streams using multiple volumes /// diff --git a/src/tape/file_formats/snapshot_archive.rs b/src/tape/file_formats/snapshot_archive.rs index 1c709b4b..56c3e9c1 100644 --- a/src/tape/file_formats/snapshot_archive.rs +++ b/src/tape/file_formats/snapshot_archive.rs @@ -7,13 +7,15 @@ use proxmox::{ tools::Uuid, }; +use pbs_tape::{ + PROXMOX_TAPE_BLOCK_SIZE, + TapeWrite, MediaContentHeader, +}; + use crate::tape::{ - TapeWrite, SnapshotReader, file_formats::{ - PROXMOX_TAPE_BLOCK_SIZE, PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1, - MediaContentHeader, SnapshotArchiveHeader, }, }; diff --git a/src/tape/helpers/mod.rs b/src/tape/helpers/mod.rs index 4adc6e85..3fd8f4b0 100644 --- a/src/tape/helpers/mod.rs +++ b/src/tape/helpers/mod.rs @@ -1,8 +1,2 @@ -mod emulate_tape_writer; -pub use emulate_tape_writer::*; - -mod emulate_tape_reader; -pub use emulate_tape_reader::*; - mod snapshot_reader; pub use snapshot_reader::*; diff --git a/src/tape/mod.rs b/src/tape/mod.rs index d8799041..0acc8748 100644 --- a/src/tape/mod.rs +++ b/src/tape/mod.rs @@ -14,12 +14,6 @@ mod test; pub mod file_formats; -mod tape_write; -pub use tape_write::*; - -mod tape_read; -pub use tape_read::*; - mod helpers; pub use helpers::*; @@ -29,9 +23,6 @@ pub use media_set::*; mod inventory; pub use inventory::*; -mod linux_list_drives; -pub use linux_list_drives::*; - pub mod changer; pub mod drive; diff --git a/src/tape/pool_writer/mod.rs b/src/tape/pool_writer/mod.rs index 8a728f1d..b2663543 100644 --- a/src/tape/pool_writer/mod.rs +++ b/src/tape/pool_writer/mod.rs @@ -15,6 +15,10 @@ use proxmox::tools::Uuid; use pbs_datastore::task_log; use pbs_config::tape_encryption_keys::load_key_configs; +use pbs_tape::{ + TapeWrite, + sg_tape::tape_alert_flags_critical, +}; use crate::{ backup::{ @@ -25,7 +29,6 @@ use crate::{ TAPE_STATUS_DIR, MAX_CHUNK_ARCHIVE_SIZE, COMMIT_BLOCK_SIZE, - TapeWrite, SnapshotReader, MediaPool, MediaId, @@ -39,7 +42,6 @@ use crate::{ drive::{ TapeDriver, request_and_load_media, - tape_alert_flags_critical, media_changer, }, }, diff --git a/src/tools/mod.rs b/src/tools/mod.rs index c2bcd215..2201191e 100644 --- a/src/tools/mod.rs +++ b/src/tools/mod.rs @@ -33,7 +33,6 @@ pub mod statistics; pub mod subscription; pub mod systemd; pub mod ticket; -pub mod sgutils2; pub mod parallel_handler; pub use parallel_handler::ParallelHandler;