diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index d0f55ea4..2845c587 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -98,6 +98,21 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + [[package]] name = "bytes" version = "1.11.0" @@ -119,6 +134,25 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "data-encoding" version = "2.8.0" @@ -162,6 +196,16 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -181,11 +225,16 @@ dependencies = [ "async-trait", "dyn-clone", "hex", + "humansize", "rcgen", "ring", + "serde", + "serde_json", + "sha2", "time", "time-macros", "tokio", + "uuid", "webpki", "x509-parser 0.17.0", ] @@ -196,6 +245,16 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -207,18 +266,49 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "itoa" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "js-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -231,6 +321,12 @@ version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "memchr" version = "2.7.4" @@ -316,6 +412,12 @@ dependencies = [ "asn1-rs 0.7.1", ] +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + [[package]] name = "pem" version = "3.0.5" @@ -356,6 +458,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rcgen" version = "0.13.2" @@ -378,7 +486,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "untrusted", "windows-sys 0.52.0", @@ -400,25 +508,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" [[package]] -name = "serde" -version = "1.0.219" +name = "rustversion" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -541,6 +695,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + [[package]] name = "unicode-ident" version = "1.0.18" @@ -553,12 +713,83 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "uuid" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] + [[package]] name = "webpki" version = "0.22.4" @@ -657,6 +888,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + [[package]] name = "x509-parser" version = "0.16.0" diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 07e1dadc..30442436 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -16,6 +16,11 @@ dyn-clone = "1.0.20" tokio = { version = "^1.48.0", features = ["process", "fs", "io-util"] } anyhow = "1.0.100" async-trait = "0.1.89" +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.145" +humansize = "2.1.3" +uuid = { version = "1.19.0", features = ["v4"] } +sha2 = "0.10.9" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs index ec1ad139..99364c0d 100644 --- a/libraries/droplet/src/lib.rs +++ b/libraries/droplet/src/lib.rs @@ -3,3 +3,4 @@ pub mod file_utils; pub mod ssl; pub mod versions; +pub mod manifest; \ No newline at end of file diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs new file mode 100644 index 00000000..57d952c1 --- /dev/null +++ b/libraries/droplet/src/manifest.rs @@ -0,0 +1,193 @@ +use std::{collections::HashMap, path::Path}; + +use anyhow::anyhow; +use hex::ToHex as _; +use humansize::{BINARY, format_size}; +use serde::Serialize; +use serde_json::json; +use sha2::{Digest as _, Sha256}; +use tokio::io::AsyncReadExt as _; + +#[derive(Serialize)] +struct FileEntry { + filename: String, + start: usize, + length: usize, + permissions: u32, +} + +#[derive(Serialize)] +struct ChunkData { + files: Vec, + checksum: String, +} + +#[derive(Serialize)] +struct Manifest { + version: String, + chunks: HashMap, + size: u64, +} + +const CHUNK_SIZE: u64 = 1024 * 1024 * 64; +const WIGGLE: u64 = 1024 * 1024 * 1; + +use crate::versions::{create_backend_constructor, types::VersionFile}; + +pub async fn generate_manifest_rusty (), V: Fn(f32) -> ()>( + dir: &Path, + progress_sfn: V, + log_sfn: T, +) -> anyhow::Result { + let mut backend = + create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?; + + let required_single_file = true; //backend.require_whole_files(); + + let files = backend.list_files().await?; + // Filepath to chunk data + let mut chunks: Vec> = Vec::new(); + let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new(); + + log_sfn(format!("organizing files into chunks...",)); + + for version_file in files { + // If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on + if required_single_file && version_file.size >= CHUNK_SIZE { + let size = version_file.size; + chunks.push(vec![(version_file, 0, size)]); + + continue; + } + + let mut current_size = current_chunk.iter().map(|v| v.2 - v.1).sum::(); + + // If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be + if required_single_file { + let size = version_file.size.try_into().unwrap(); + current_chunk.push((version_file, 0, size)); + + current_size += size; + + if current_size >= CHUNK_SIZE { + // Pop current and add, then reset + let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); + chunks.push(new_chunk); + } + + continue; + } + + // Otherwise we calculate how much of the file we need, then use that much + let remaining_budget = (CHUNK_SIZE + WIGGLE) - current_size; + if version_file.size >= remaining_budget { + let remaining_budget = CHUNK_SIZE - current_size; + current_chunk.push((version_file.clone(), 0, remaining_budget)); + + let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); + chunks.push(new_chunk); + + let remaining_size = version_file.size - remaining_budget; + let mut running_offset = remaining_budget; + // Do everything but the last one + while running_offset < remaining_size { + let chunk_size = CHUNK_SIZE.min(remaining_size); + let chunk = vec![(version_file.clone(), running_offset, chunk_size)]; + if chunk_size == CHUNK_SIZE { + chunks.push(chunk); + } else { + current_chunk = chunk; + } + running_offset += chunk_size; + } + + continue; + } else { + let size = version_file.size; + current_chunk.push((version_file, 0, size)); + current_size += size; + } + + if current_size >= CHUNK_SIZE { + // Pop current and add, then reset + let new_chunk = std::mem::replace(&mut current_chunk, Vec::new()); + chunks.push(new_chunk); + } + } + if current_chunk.len() > 0 { + chunks.push(current_chunk); + } + + log_sfn(format!( + "organized into {} chunks, generating checksums...", + chunks.len() + )); + + let mut manifest: HashMap = HashMap::new(); + let mut total_manifest_length = 0; + + let mut read_buf = vec![0; 1024 * 1024 * 64]; + + let chunk_len = chunks.len(); + for (index, chunk) in chunks.into_iter().enumerate() { + let uuid = uuid::Uuid::new_v4().to_string(); + let mut hasher = Sha256::new(); + + let mut chunk_data = ChunkData { + files: Vec::new(), + checksum: String::new(), + }; + + let mut chunk_length = 0; + + for (file, start, length) in chunk { + log_sfn(format!( + "reading {} from {} to {}, {}", + file.relative_filename, + start, + start + length, + format_size(length, BINARY) + )); + let mut reader = backend.reader(&file, start, start + length).await?; + + loop { + let amount = reader.read(&mut read_buf).await?; + if amount == 0 { + break; + } + hasher.update(&read_buf[0..amount]); + } + + chunk_length += length; + + chunk_data.files.push(FileEntry { + filename: file.relative_filename, + start: start.try_into().unwrap(), + length: length.try_into().unwrap(), + permissions: file.permission, + }); + } + + log_sfn(format!( + "created chunk of size {} ({}/{})", + format_size(chunk_length, BINARY), + index, + chunk_len + )); + total_manifest_length += chunk_length; + + let hash: String = hasher.finalize().encode_hex(); + chunk_data.checksum = hash; + manifest.insert(uuid, chunk_data); + + let progress: f32 = (index as f32 / chunk_len as f32) * 100.0f32; + progress_sfn(progress); + } + + Ok(json!(Manifest { + version: "2".to_string(), + chunks: manifest, + size: total_manifest_length + }) + .to_string()) +}