From 991abfa8b4ea268ec9f634b81983931067737748 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 22 Jun 2019 09:12:25 +0200 Subject: [PATCH] src/backup/file_formats.rs: split out file format data --- src/backup.rs | 33 ++------------------- src/backup/data_blob.rs | 41 ++++++++++++++++---------- src/backup/data_chunk.rs | 60 ++++++++++++++++++++++++-------------- src/backup/file_formats.rs | 57 ++++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 68 deletions(-) create mode 100644 src/backup/file_formats.rs diff --git a/src/backup.rs b/src/backup.rs index 6e781e2a..19869ff1 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -107,37 +107,8 @@ macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 { () => { "proxmox-backup-protocol-v1" } } -// WARNING: PLEASE DO NOT MODIFY THOSE MAGIC VALUES - -// openssl::sha::sha256(b"Proxmox Backup uncompressed chunk v1.0")[0..8] -pub static UNCOMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [79, 127, 200, 4, 121, 74, 135, 239]; - -// openssl::sha::sha256(b"Proxmox Backup encrypted chunk v1.0")[0..8] -pub static ENCRYPTED_CHUNK_MAGIC_1_0: [u8; 8] = [8, 54, 114, 153, 70, 156, 26, 151]; - -// openssl::sha::sha256(b"Proxmox Backup zstd compressed chunk v1.0")[0..8] -pub static COMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [191, 237, 46, 195, 108, 17, 228, 235]; - -// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted chunk v1.0")[0..8] -pub static ENCR_COMPR_CHUNK_MAGIC_1_0: [u8; 8] = [9, 40, 53, 200, 37, 150, 90, 196]; - -// openssl::sha::sha256(b"Proxmox Backup uncompressed blob v1.0")[0..8] -pub static UNCOMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [66, 171, 56, 7, 190, 131, 112, 161]; - -//openssl::sha::sha256(b"Proxmox Backup zstd compressed blob v1.0")[0..8] -pub static COMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [49, 185, 88, 66, 111, 182, 163, 127]; - -// openssl::sha::sha256(b"Proxmox Backup encrypted blob v1.0")[0..8] -pub static ENCRYPTED_BLOB_MAGIC_1_0: [u8; 8] = [123, 103, 133, 190, 34, 45, 76, 240]; - -// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted blob v1.0")[0..8] -pub static ENCR_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [230, 89, 27, 191, 11, 191, 216, 11]; - -// openssl::sha::sha256(b"Proxmox Backup fixed sized chunk index v1.0")[0..8] -pub static FIXED_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [47, 127, 65, 237, 145, 253, 15, 205]; - -// openssl::sha::sha256(b"Proxmox Backup dynamic sized chunk index v1.0")[0..8] -pub static DYNAMIC_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [28, 145, 78, 165, 25, 186, 179, 205]; +mod file_formats; +pub use file_formats::*; mod crypt_config; pub use crypt_config::*; diff --git a/src/backup/data_blob.rs b/src/backup/data_blob.rs index ecd8639e..6dd0be66 100644 --- a/src/backup/data_blob.rs +++ b/src/backup/data_blob.rs @@ -1,7 +1,7 @@ use failure::*; use std::convert::TryInto; -use std::io::Write; +use crate::tools::write::*; use super::*; /// Data blob binary storage format @@ -28,6 +28,7 @@ use super::*; /// /// This is basically the same format we use for ``DataChunk``, but /// with other magic numbers so that we can distinguish them. + pub struct DataBlob { raw_data: Vec, // tagged, compressed, encryped data } @@ -46,18 +47,21 @@ impl DataBlob { /// accessor to crc32 checksum pub fn crc(&self) -> u32 { - u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap()) + let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc); + u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap()) } // set the CRC checksum field pub fn set_crc(&mut self, crc: u32) { - self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes()); + let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc); + self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes()); } /// compute the CRC32 checksum pub fn compute_crc(&mut self) -> u32 { let mut hasher = crc32fast::Hasher::new(); - hasher.update(&self.raw_data[12..]); + let start = std::mem::size_of::(); // start after HEAD + hasher.update(&self.raw_data[start..]); hasher.finalize() } @@ -82,23 +86,30 @@ impl DataBlob { return Ok(DataBlob { raw_data: enc_data }); } else { + let max_data_len = data.len() + std::mem::size_of::(); if compress { - let mut comp_data = Vec::with_capacity(data.len() + 8 + 4); + let mut comp_data = Vec::with_capacity(max_data_len); - comp_data.write_all(&COMPRESSED_BLOB_MAGIC_1_0)?; - comp_data.write_all(&[0u8, 4])?; // CRC set to 0 + let head = DataBlobHeader { + magic: COMPRESSED_BLOB_MAGIC_1_0, + crc: [0; 4], + }; + comp_data.write_value(&head)?; zstd::stream::copy_encode(data, &mut comp_data, 1)?; - if comp_data.len() < (data.len() + 8 + 4) { + if comp_data.len() < max_data_len { return Ok(DataBlob { raw_data: comp_data }); } } - let mut raw_data = Vec::with_capacity(data.len() + 8 + 4); + let mut raw_data = Vec::with_capacity(max_data_len); - raw_data.write_all(&UNCOMPRESSED_BLOB_MAGIC_1_0)?; - raw_data.write_all(&[0u8; 4])?; + let head = DataBlobHeader { + magic: UNCOMPRESSED_BLOB_MAGIC_1_0, + crc: [0; 4], + }; + raw_data.write_value(&head)?; raw_data.extend_from_slice(data); return Ok(DataBlob { raw_data }); @@ -111,12 +122,12 @@ impl DataBlob { let magic = self.magic(); if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 { - return Ok(self.raw_data[12..].to_vec()); + let data_start = std::mem::size_of::(); + return Ok(self.raw_data[data_start..].to_vec()); } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 { - - let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?; + let data_start = std::mem::size_of::(); + let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?; return Ok(data); - } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 { if let Some(config) = config { let data = if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 { diff --git a/src/backup/data_chunk.rs b/src/backup/data_chunk.rs index e5de3b0a..48feeab7 100644 --- a/src/backup/data_chunk.rs +++ b/src/backup/data_chunk.rs @@ -1,6 +1,6 @@ use failure::*; use std::convert::TryInto; -use std::io::Write; +use crate::tools::write::*; use super::*; @@ -57,22 +57,25 @@ impl DataChunk { /// accessor to crc32 checksum pub fn crc(&self) -> u32 { - u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap()) + let crc_o = proxmox::tools::offsetof!(DataChunkHeader, crc); + u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap()) } // set the CRC checksum field pub fn set_crc(&mut self, crc: u32) { - self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes()); + let crc_o = proxmox::tools::offsetof!(DataChunkHeader, crc); + self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes()); } /// compute the CRC32 checksum pub fn compute_crc(&mut self) -> u32 { let mut hasher = crc32fast::Hasher::new(); - hasher.update(&self.raw_data[12..]); + let start = std::mem::size_of::(); // start after HEAD + hasher.update(&self.raw_data[start..]); hasher.finalize() } - fn new( + fn encode( data: &[u8], config: Option<&CryptConfig>, digest: [u8;32], @@ -92,23 +95,31 @@ impl DataChunk { Ok(chunk) } else { + let max_data_len = data.len() + std::mem::size_of::(); if compress { - let mut comp_data = Vec::with_capacity(data.len() + 8 + 4); + let mut comp_data = Vec::with_capacity(max_data_len); + + let head = DataChunkHeader { + magic: COMPRESSED_CHUNK_MAGIC_1_0, + crc: [0; 4], + }; + comp_data.write_value(&head)?; - comp_data.write_all(&COMPRESSED_CHUNK_MAGIC_1_0)?; - comp_data.write_all(&[0u8, 4])?; // CRC set to 0 zstd::stream::copy_encode(data, &mut comp_data, 1)?; - if comp_data.len() < (data.len() + 8 + 4) { + if comp_data.len() < max_data_len { let chunk = DataChunk { digest, raw_data: comp_data }; return Ok(chunk); } } - let mut raw_data = Vec::with_capacity(data.len() + 8 + 4); + let mut raw_data = Vec::with_capacity(max_data_len); - raw_data.write_all(&UNCOMPRESSED_CHUNK_MAGIC_1_0)?; - raw_data.write_all(&[0u8, 4])?; // CRC set to 0 + let head = DataChunkHeader { + magic: UNCOMPRESSED_CHUNK_MAGIC_1_0, + crc: [0; 4], + }; + raw_data.write_value(&head)?; raw_data.extend_from_slice(data); let chunk = DataChunk { digest, raw_data }; @@ -122,12 +133,12 @@ impl DataChunk { let magic = self.magic(); if magic == &UNCOMPRESSED_CHUNK_MAGIC_1_0 { - return Ok(self.raw_data[12..].to_vec()); + let data_start = std::mem::size_of::(); + return Ok(self.raw_data[data_start..].to_vec()); } else if magic == &COMPRESSED_CHUNK_MAGIC_1_0 { - - let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?; + let data_start = std::mem::size_of::(); + let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?; return Ok(data); - } else if magic == &ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == &ENCRYPTED_CHUNK_MAGIC_1_0 { if let Some(config) = config { let data = if magic == &ENCR_COMPR_CHUNK_MAGIC_1_0 { @@ -160,7 +171,7 @@ impl DataChunk { /// Create Instance from raw data pub fn from_raw(data: Vec, digest: [u8;32]) -> Result { - if data.len() < 12 { + if data.len() < std::mem::size_of::() { bail!("chunk too small ({} bytes).", data.len()); } @@ -168,7 +179,7 @@ impl DataChunk { if magic == ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == ENCRYPTED_CHUNK_MAGIC_1_0 { - if data.len() < 44 { + if data.len() < std::mem::size_of::() { bail!("encrypted chunk too small ({} bytes).", data.len()); } @@ -191,7 +202,7 @@ impl DataChunk { /// this is noth possible for encrypted chunks. pub fn verify_unencrypted(&self, expected_chunk_size: usize) -> Result<(), Error> { - let magic = &self.raw_data[0..8]; + let magic = self.magic(); let verify_raw_data = |data: &[u8]| { if expected_chunk_size != data.len() { @@ -204,10 +215,10 @@ impl DataChunk { Ok(()) }; - if magic == COMPRESSED_CHUNK_MAGIC_1_0 { + if magic == &COMPRESSED_CHUNK_MAGIC_1_0 { let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?; verify_raw_data(&data)?; - } else if magic == UNCOMPRESSED_CHUNK_MAGIC_1_0 { + } else if magic == &UNCOMPRESSED_CHUNK_MAGIC_1_0 { verify_raw_data(&self.raw_data[12..])?; } @@ -288,7 +299,12 @@ impl <'a, 'b> DataChunkBuilder<'a, 'b> { self.compute_digest(); } - let chunk = DataChunk::new(self.orig_data, self.config, self.digest, self.compress)?; + let chunk = DataChunk::encode( + self.orig_data, + self.config, + self.digest, + self.compress, + )?; Ok(chunk) } diff --git a/src/backup/file_formats.rs b/src/backup/file_formats.rs new file mode 100644 index 00000000..c9ecb4ac --- /dev/null +++ b/src/backup/file_formats.rs @@ -0,0 +1,57 @@ +// WARNING: PLEASE DO NOT MODIFY THOSE MAGIC VALUES + +// openssl::sha::sha256(b"Proxmox Backup uncompressed chunk v1.0")[0..8] +pub static UNCOMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [79, 127, 200, 4, 121, 74, 135, 239]; + +// openssl::sha::sha256(b"Proxmox Backup encrypted chunk v1.0")[0..8] +pub static ENCRYPTED_CHUNK_MAGIC_1_0: [u8; 8] = [8, 54, 114, 153, 70, 156, 26, 151]; + +// openssl::sha::sha256(b"Proxmox Backup zstd compressed chunk v1.0")[0..8] +pub static COMPRESSED_CHUNK_MAGIC_1_0: [u8; 8] = [191, 237, 46, 195, 108, 17, 228, 235]; + +// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted chunk v1.0")[0..8] +pub static ENCR_COMPR_CHUNK_MAGIC_1_0: [u8; 8] = [9, 40, 53, 200, 37, 150, 90, 196]; + +// openssl::sha::sha256(b"Proxmox Backup uncompressed blob v1.0")[0..8] +pub static UNCOMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [66, 171, 56, 7, 190, 131, 112, 161]; + +//openssl::sha::sha256(b"Proxmox Backup zstd compressed blob v1.0")[0..8] +pub static COMPRESSED_BLOB_MAGIC_1_0: [u8; 8] = [49, 185, 88, 66, 111, 182, 163, 127]; + +// openssl::sha::sha256(b"Proxmox Backup encrypted blob v1.0")[0..8] +pub static ENCRYPTED_BLOB_MAGIC_1_0: [u8; 8] = [123, 103, 133, 190, 34, 45, 76, 240]; + +// openssl::sha::sha256(b"Proxmox Backup zstd compressed encrypted blob v1.0")[0..8] +pub static ENCR_COMPR_BLOB_MAGIC_1_0: [u8; 8] = [230, 89, 27, 191, 11, 191, 216, 11]; + +// openssl::sha::sha256(b"Proxmox Backup fixed sized chunk index v1.0")[0..8] +pub static FIXED_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [47, 127, 65, 237, 145, 253, 15, 205]; + +// openssl::sha::sha256(b"Proxmox Backup dynamic sized chunk index v1.0")[0..8] +pub static DYNAMIC_SIZED_CHUNK_INDEX_1_0: [u8; 8] = [28, 145, 78, 165, 25, 186, 179, 205]; + +#[repr(C,packed)] +pub struct DataBlobHeader { + pub magic: [u8; 8], + pub crc: [u8; 4], +} + +#[repr(C,packed)] +pub struct EncryptedDataBlobHeader { + pub head: DataBlobHeader, + pub iv: [u8; 16], + pub tag: [u8; 16], +} + +#[repr(C,packed)] +pub struct DataChunkHeader { + pub magic: [u8; 8], + pub crc: [u8; 4], +} + +#[repr(C,packed)] +pub struct EncryptedDataChunkHeader { + pub head: DataChunkHeader, + pub iv: [u8; 16], + pub tag: [u8; 16], +}