Add compression to the client

This should help overall file sizes on the server

bfsp/Cargo.toml

@@ -13,7 +13,7 @@ tokio = { version = "1", features = ["fs", "io-util"] }
 sqlx = { version = "0.7", default-features = false }
 uuid = { version = "1", features = ["v4"] }
 chacha20poly1305 = { version = "0.10", features = ["std"] }
-num = "0.4.1"
+zstd = "0.13.0"
 
 [dev-dependencies]
 tokio = { version = "1", features = ["fs", "io-util", "macros", "rt-multi-thread"] }

bfsp/src/crypto.rs

@@ -1,6 +1,5 @@
 use anyhow::{anyhow, Result};
 use chacha20poly1305::{aead::OsRng, AeadInPlace, Key, KeyInit, XChaCha20Poly1305};
-use num::PrimInt;
 use rkyv::{Archive, Deserialize, Serialize};
 use sqlx::Sqlite;
 
@@ -120,7 +119,3 @@ impl EncryptionNonce {
 pub async fn init_key() -> Result<()> {
     todo!()
 }
-
-pub fn size_to_encrypted_size<S: PrimInt + std::iter::Sum + num::One>(size: S) -> S {
-    size + (0..16).into_iter().map(|_| S::one()).sum()
-}

bfsp/src/lib.rs

@@ -442,7 +442,7 @@ impl FileHeader {
 }
 
 //TODO: can this be a slice?
-pub async fn encrypted_chunk_from_file(
+pub async fn compressed_encrypted_chunk_from_file(
     file_header: &FileHeader,
     file: &mut File,
     chunk_id: ChunkID,
@@ -467,6 +467,12 @@ pub async fn encrypted_chunk_from_file(
         .read_to_end(&mut buf)
         .await?;
 
+
+    println!("Size before compression: {}KB", buf.len());
+
+    let mut buf = zstd::bulk::compress(&buf, 15)?;
+    println!("Size after compression: {}KB", buf.len());
+
     file.rewind().await?;
 
     key.encrypt_chunk_in_place(&mut buf, &chunk_meta)?;

cli/src/main.rs

@@ -9,7 +9,7 @@ use std::path::Path;
 
 use anyhow::Result;
 use bfsp::{
-    encrypted_chunk_from_file, hash_chunk, hash_file, parallel_hash_chunk, size_to_encrypted_size,
+    compressed_encrypted_chunk_from_file, hash_chunk, hash_file, parallel_hash_chunk,
     use_parallel_hasher, Action, ChunkID, ChunkMetadata, ChunksUploaded, ChunksUploadedQuery,
     DownloadChunkReq, EncryptionKey, FileHash, FileHeader,
 };
@@ -363,8 +363,10 @@ pub async fn upload_file(
         sock.write_u16(action).await?;
         sock.write_all(chunk_meta.to_bytes()?.as_slice()).await?;
 
-        let chunk = encrypted_chunk_from_file(file_header, &mut file, chunk_meta.id, key).await?;
-        debug_assert_eq!(chunk.len() as u32, size_to_encrypted_size(chunk_meta.size));
+        let chunk =
+            compressed_encrypted_chunk_from_file(file_header, &mut file, chunk_meta.id, key)
+                .await?;
+        sock.write_u32(chunk.len() as u32).await?;
         sock.write_all(chunk.as_slice()).await?;
     }
 
@@ -429,7 +431,9 @@ pub async fn download_file<P: AsRef<Path> + Display>(
 
         trace!("Reading chunk of size {}", chunk_metadata.size);
 
-        let mut chunk_buf = vec![0; size_to_encrypted_size(chunk_metadata.size) as usize];
+        let chunk_size = sock.read_u32().await?;
+
+        let mut chunk_buf = vec![0; chunk_size as usize];
         sock.read_exact(&mut chunk_buf)
             .await
             .with_context(|| "Error reading raw chunk data")?;