From b7f4f27d6c14d20d9500d1c37a611caa0de2d2df Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 21 Jun 2019 17:24:21 +0200 Subject: [PATCH] add crc field for binary blobs formats --- src/backup/crypt_config.rs | 50 +++++++++++++++++-------------- src/backup/data_blob.rs | 48 +++++++++++++++++++++++------- src/backup/data_chunk.rs | 61 ++++++++++++++++++++++++-------------- 3 files changed, 104 insertions(+), 55 deletions(-) diff --git a/src/backup/crypt_config.rs b/src/backup/crypt_config.rs index 11a48326..727fda31 100644 --- a/src/backup/crypt_config.rs +++ b/src/backup/crypt_config.rs @@ -82,34 +82,36 @@ impl CryptConfig { c.aad_update(b"")?; //?? if compress { - let compr_data = zstd::block::compress(data, 1)?; + let compr_data = zstd::block::compress(data, 1)?; // Note: We only use compression if result is shorter if compr_data.len() < data.len() { - let mut enc = vec![0; compr_data.len()+40+self.cipher.block_size()]; + let mut enc = vec![0; compr_data.len()+44+self.cipher.block_size()]; enc[0..8].copy_from_slice(comp_magic); - enc[8..24].copy_from_slice(&iv); + enc[8..12].copy_from_slice(&[0u8; 4]); + enc[12..28].copy_from_slice(&iv); - let count = c.update(&compr_data, &mut enc[40..])?; - let rest = c.finalize(&mut enc[(40+count)..])?; - enc.truncate(40 + count + rest); + let count = c.update(&compr_data, &mut enc[44..])?; + let rest = c.finalize(&mut enc[(44+count)..])?; + enc.truncate(44 + count + rest); - c.get_tag(&mut enc[24..40])?; + c.get_tag(&mut enc[28..44])?; return Ok(enc) } } - let mut enc = vec![0; data.len()+40+self.cipher.block_size()]; + let mut enc = vec![0; data.len()+44+self.cipher.block_size()]; enc[0..8].copy_from_slice(uncomp_magic); - enc[8..24].copy_from_slice(&iv); + enc[8..12].copy_from_slice(&[0u8; 4]); + enc[12..28].copy_from_slice(&iv); - let count = c.update(data, &mut enc[40..])?; - let rest = c.finalize(&mut enc[(40+count)..])?; - enc.truncate(40 + count + rest); + let count = c.update(data, &mut enc[44..])?; + let rest = c.finalize(&mut enc[(44+count)..])?; + enc.truncate(44 + count + rest); - c.get_tag(&mut enc[24..40])?; + c.get_tag(&mut enc[28..44])?; Ok(enc) } @@ -120,13 +122,14 @@ impl CryptConfig { /// is not used here. pub fn decode_compressed_chunk(&self, data: &[u8]) -> Result, Error> { - if data.len() < 40 { - bail!("Invalid chunk len (<40)"); + if data.len() < 44 { + bail!("Invalid chunk len (<44)"); } // let magic = &data[0..8]; - let iv = &data[8..24]; - let mac = &data[24..40]; + // let crc = &data[8..12]; + let iv = &data[12..28]; + let mac = &data[28..44]; let dec = Vec::with_capacity(1024*1024); @@ -140,7 +143,7 @@ impl CryptConfig { let mut decr_buf = [0u8; BUFFER_SIZE]; let max_decoder_input = BUFFER_SIZE - self.cipher.block_size(); - let mut start = 40; + let mut start = 44; loop { let mut end = start + max_decoder_input; if end > data.len() { end = data.len(); } @@ -168,20 +171,21 @@ impl CryptConfig { /// is not used here. pub fn decode_uncompressed_chunk(&self, data: &[u8]) -> Result, Error> { - if data.len() < 40 { - bail!("Invalid chunk len (<40)"); + if data.len() < 44 { + bail!("Invalid chunk len (<44)"); } // let magic = &data[0..8]; - let iv = &data[8..24]; - let mac = &data[24..40]; + // let crc = &data[8..12]; + let iv = &data[12..28]; + let mac = &data[28..44]; let decr_data = decrypt_aead( self.cipher, &self.enc_key, Some(iv), b"", //?? - &data[40..], + &data[44..], mac, )?; diff --git a/src/backup/data_blob.rs b/src/backup/data_blob.rs index 9a279b19..ecd8639e 100644 --- a/src/backup/data_blob.rs +++ b/src/backup/data_blob.rs @@ -11,12 +11,20 @@ use super::*; /// them on disk or transfer them over the network. Please use index /// files to store large data files (".fidx" of ".didx"). /// -/// The format start with a 8 byte magic number to identify the type. -/// Encrypted blobs contain a 16 byte IV, followed by a 18 byte AD -/// tag, followed by the encrypted data (MAGIC || IV || TAG || -/// EncryptedData). +/// The format start with a 8 byte magic number to identify the type, +/// followed by a 4 byte CRC. This CRC is used on the server side to +/// detect file corruption (computed when upload data), so there is +/// usually no need to compute it on the client side. /// -/// Unencrypted blobs simply contain the (compressed) data. +/// Encrypted blobs contain a 16 byte IV, followed by a 16 byte AD +/// tag, followed by the encrypted data: +/// +/// (MAGIC || CRC32 || IV || TAG || EncryptedData). +/// +/// Unencrypted blobs simply contain the CRC, followed by the +/// (compressed) data. +/// +/// (MAGIC || CRC32 || Data) /// /// This is basically the same format we use for ``DataChunk``, but /// with other magic numbers so that we can distinguish them. @@ -36,6 +44,23 @@ impl DataBlob { self.raw_data[0..8].try_into().unwrap() } + /// accessor to crc32 checksum + pub fn crc(&self) -> u32 { + u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap()) + } + + // set the CRC checksum field + pub fn set_crc(&mut self, crc: u32) { + self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes()); + } + + /// compute the CRC32 checksum + pub fn compute_crc(&mut self) -> u32 { + let mut hasher = crc32fast::Hasher::new(); + hasher.update(&self.raw_data[12..]); + hasher.finalize() + } + pub fn encode( data: &[u8], config: Option<&CryptConfig>, @@ -58,19 +83,22 @@ impl DataBlob { } else { if compress { - let mut comp_data = Vec::with_capacity(data.len() + 8); + let mut comp_data = Vec::with_capacity(data.len() + 8 + 4); comp_data.write_all(&COMPRESSED_BLOB_MAGIC_1_0)?; + comp_data.write_all(&[0u8, 4])?; // CRC set to 0 + zstd::stream::copy_encode(data, &mut comp_data, 1)?; - if comp_data.len() < (data.len() + 8) { + if comp_data.len() < (data.len() + 8 + 4) { return Ok(DataBlob { raw_data: comp_data }); } } - let mut raw_data = Vec::with_capacity(data.len() + 8); + let mut raw_data = Vec::with_capacity(data.len() + 8 + 4); raw_data.write_all(&UNCOMPRESSED_BLOB_MAGIC_1_0)?; + raw_data.write_all(&[0u8; 4])?; raw_data.extend_from_slice(data); return Ok(DataBlob { raw_data }); @@ -83,10 +111,10 @@ impl DataBlob { let magic = self.magic(); if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 { - return Ok(self.raw_data); + return Ok(self.raw_data[12..].to_vec()); } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 { - let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?; + let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?; return Ok(data); } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 { diff --git a/src/backup/data_chunk.rs b/src/backup/data_chunk.rs index df0e2578..e5de3b0a 100644 --- a/src/backup/data_chunk.rs +++ b/src/backup/data_chunk.rs @@ -17,12 +17,20 @@ pub struct ChunkInfo { /// compressed and encrypted. A simply binary format is used to store /// them on disk or transfer them over the network. /// -/// The format start with a 8 byte magic number to identify the type. -/// Encrypted chunks contain a 16 byte IV, followed by a 18 byte AD -/// tag, followed by the encrypted data (MAGIC || IV || TAG || -/// EncryptedData). +/// The format start with a 8 byte magic number to identify the type, +/// followed by a 4 byte CRC. This CRC is used on the server side to +/// detect file corruption (computed when upload data), so there is +/// usually no need to compute it on the client side. /// -/// Unecrypted chunks simply contain the (compressed) data. +/// Encrypted chunks contain a 16 byte IV, followed by a 16 byte AD +/// tag, followed by the encrypted data: +/// +/// (MAGIC || CRC32 || IV || TAG || EncryptedData). +/// +/// Unencrypted blobs simply contain the CRC, followed by the +/// (compressed) data. +/// +/// (MAGIC || CRC32 || Data) /// /// Please use the ``DataChunkBuilder`` to create new instances. pub struct DataChunk { @@ -47,15 +55,22 @@ impl DataChunk { self.raw_data[0..8].try_into().unwrap() } - // only valid for enrypted data - //pub fn iv(&self) -> &[u8; 16] { - // self.raw_data[8..24].try_into().unwrap() - //} + /// accessor to crc32 checksum + pub fn crc(&self) -> u32 { + u32::from_le_bytes(self.raw_data[8..12].try_into().unwrap()) + } - // only valid for enrypted data - //pub fn mac(&self) -> &[u8; 16] { - // self.raw_data[24..40].try_into().unwrap() - //} + // set the CRC checksum field + pub fn set_crc(&mut self, crc: u32) { + self.raw_data[8..12].copy_from_slice(&crc.to_le_bytes()); + } + + /// compute the CRC32 checksum + pub fn compute_crc(&mut self) -> u32 { + let mut hasher = crc32fast::Hasher::new(); + hasher.update(&self.raw_data[12..]); + hasher.finalize() + } fn new( data: &[u8], @@ -78,20 +93,22 @@ impl DataChunk { } else { if compress { - let mut comp_data = Vec::with_capacity(data.len() + 8); + let mut comp_data = Vec::with_capacity(data.len() + 8 + 4); comp_data.write_all(&COMPRESSED_CHUNK_MAGIC_1_0)?; + comp_data.write_all(&[0u8, 4])?; // CRC set to 0 zstd::stream::copy_encode(data, &mut comp_data, 1)?; - if comp_data.len() < (data.len() + 8) { + if comp_data.len() < (data.len() + 8 + 4) { let chunk = DataChunk { digest, raw_data: comp_data }; return Ok(chunk); } } - let mut raw_data = Vec::with_capacity(data.len() + 8); + let mut raw_data = Vec::with_capacity(data.len() + 8 + 4); raw_data.write_all(&UNCOMPRESSED_CHUNK_MAGIC_1_0)?; + raw_data.write_all(&[0u8, 4])?; // CRC set to 0 raw_data.extend_from_slice(data); let chunk = DataChunk { digest, raw_data }; @@ -105,10 +122,10 @@ impl DataChunk { let magic = self.magic(); if magic == &UNCOMPRESSED_CHUNK_MAGIC_1_0 { - return Ok(self.raw_data); + return Ok(self.raw_data[12..].to_vec()); } else if magic == &COMPRESSED_CHUNK_MAGIC_1_0 { - let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?; + let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?; return Ok(data); } else if magic == &ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == &ENCRYPTED_CHUNK_MAGIC_1_0 { @@ -143,7 +160,7 @@ impl DataChunk { /// Create Instance from raw data pub fn from_raw(data: Vec, digest: [u8;32]) -> Result { - if data.len() < 8 { + if data.len() < 12 { bail!("chunk too small ({} bytes).", data.len()); } @@ -151,7 +168,7 @@ impl DataChunk { if magic == ENCR_COMPR_CHUNK_MAGIC_1_0 || magic == ENCRYPTED_CHUNK_MAGIC_1_0 { - if data.len() < 40 { + if data.len() < 44 { bail!("encrypted chunk too small ({} bytes).", data.len()); } @@ -188,10 +205,10 @@ impl DataChunk { }; if magic == COMPRESSED_CHUNK_MAGIC_1_0 { - let data = zstd::block::decompress(&self.raw_data[8..], 16*1024*1024)?; + let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?; verify_raw_data(&data)?; } else if magic == UNCOMPRESSED_CHUNK_MAGIC_1_0 { - verify_raw_data(&self.raw_data[8..])?; + verify_raw_data(&self.raw_data[12..])?; } Ok(())