diff --git a/libraries/droplet/.gitignore b/libraries/droplet/.gitignore index 423b4790..d80b04ff 100644 --- a/libraries/droplet/.gitignore +++ b/libraries/droplet/.gitignore @@ -19,3 +19,7 @@ target/ # Added by cargo /target + +perf.data +flamegraph.svg +*.json \ No newline at end of file diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 61bf6bab..803b4f4c 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.12.3" +version = "0.13.0" dependencies = [ "anyhow", "async-trait", diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 55503bbe..18ef4aab 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.12.3" +version = "0.13.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 4ee35e0c..29c56dd1 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -55,26 +55,25 @@ pub async fn generate_manifest_rusty( let required_single_file = backend.require_whole_files(); - let files = backend.list_files().await?; + let mut files = backend.list_files().await?; + files.sort_by(|a, b| a.size.cmp(&b.size)); // Filepath to chunk data let mut chunks: Vec> = Vec::new(); let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new(); log_sfn("organizing files into chunks...".to_string()); - for version_file in files { - // If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on - if required_single_file && version_file.size >= CHUNK_SIZE { - let size = version_file.size; - chunks.push(vec![(version_file, 0, size)]); + if required_single_file { + for version_file in files { + if version_file.size >= CHUNK_SIZE { + let size = version_file.size; + chunks.push(vec![(version_file, 0, size)]); - continue; - } + continue; + } - let mut current_size = current_chunk.iter().map(|v| v.2 - v.1).sum::(); + let mut current_size = current_chunk.iter().map(|v| v.2).sum::(); - // If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be - if required_single_file { let size = version_file.size; current_chunk.push((version_file, 0, size)); @@ -88,43 +87,36 @@ pub async fn generate_manifest_rusty( continue; } + } else { + for version_file in files { + let current_size = current_chunk.iter().map(|v| v.2).sum::(); - // Otherwise we calculate how much of the file we need, then use that much - let remaining_budget = (CHUNK_SIZE + WIGGLE) - current_size; - if version_file.size >= remaining_budget { - let remaining_budget = CHUNK_SIZE - current_size; - current_chunk.push((version_file.clone(), 0, remaining_budget)); + if version_file.size + current_size < CHUNK_SIZE + WIGGLE { + let size = version_file.size; + current_chunk.push((version_file, 0, size)); - let new_chunk = std::mem::take(&mut current_chunk); - chunks.push(new_chunk); - - let remaining_size = version_file.size - remaining_budget; - let mut running_offset = remaining_budget; - // Do everything but the last one - while running_offset < remaining_size { - let chunk_size = CHUNK_SIZE.min(remaining_size); - let chunk = vec![(version_file.clone(), running_offset, chunk_size)]; - if chunk_size == CHUNK_SIZE { - chunks.push(chunk); - } else { - current_chunk = chunk; - } - running_offset += chunk_size; + continue; } - continue; - } else { - let size = version_file.size; - current_chunk.push((version_file, 0, size)); - current_size += size; - } + // Fill up current chunk + let remaining = CHUNK_SIZE - current_size; + current_chunk.push((version_file.clone(), 0, remaining)); + chunks.push(std::mem::take(&mut current_chunk)); - if current_size >= CHUNK_SIZE { - // Pop current and add, then reset - let new_chunk = std::mem::take(&mut current_chunk); - chunks.push(new_chunk); + // This is our offset in our current file + let mut offset = remaining; + while offset < version_file.size { + let length = CHUNK_SIZE.min(version_file.size - offset); + offset += length; + if length == CHUNK_SIZE { + chunks.push(vec![(version_file.clone(), offset, length)]); + } else { + current_chunk.push((version_file.clone(), offset, length)); + } + } } } + if !current_chunk.is_empty() { chunks.push(current_chunk); } @@ -191,8 +183,9 @@ pub async fn generate_manifest_rusty( send_log .send(format!( - "created chunk of size {} from {} files (index {})", + "created chunk of size {} ({}b) from {} files (index {})", format_size(chunk_length, BINARY), + chunk_length, chunk_data.files.len(), index ))