jessikitty/drop
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: manifest generator

Browse Source
This commit is contained in:
DecDuck
2025-12-13 15:27:03 +11:00
parent 4dfb02b0e7
commit a3ee266f9d
4 changed files with 442 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/droplet/Cargo.lock
Generated
+243 -6
View File
@@ -98,6 +98,21 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "bumpalo"
version = "3.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
[[package]]
name = "bytes"
version = "1.11.0"
@@ -119,6 +134,25 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
[[package]]
name = "crypto-common"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "data-encoding"
version = "2.8.0"
@@ -162,6 +196,16 @@ dependencies = [
"powerfmt",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]]
name = "displaydoc"
version = "0.2.5"
@@ -181,11 +225,16 @@ dependencies = [
"async-trait",
"dyn-clone",
"hex",
"humansize",
"rcgen",
"ring",
"serde",
"serde_json",
"sha2",
"time",
"time-macros",
"tokio",
"uuid",
"webpki",
"x509-parser 0.17.0",
]
@@ -196,6 +245,16 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.15"
@@ -207,18 +266,49 @@ dependencies = [
"wasi",
]
[[package]]
name = "getrandom"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasip2",
]
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "humansize"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7"
dependencies = [
"libm",
]
[[package]]
name = "itoa"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
name = "js-sys"
version = "0.3.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8"
dependencies = [
"once_cell",
"wasm-bindgen",
]
[[package]]
name = "lazy_static"
version = "1.5.0"
@@ -231,6 +321,12 @@ version = "0.2.171"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
[[package]]
name = "libm"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
[[package]]
name = "memchr"
version = "2.7.4"
@@ -316,6 +412,12 @@ dependencies = [
"asn1-rs 0.7.1",
]
[[package]]
name = "once_cell"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "pem"
version = "3.0.5"
@@ -356,6 +458,12 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "rcgen"
version = "0.13.2"
@@ -378,7 +486,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
dependencies = [
"cc",
"cfg-if",
"getrandom",
"getrandom 0.2.15",
"libc",
"untrusted",
"windows-sys 0.52.0",
@@ -400,25 +508,71 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
[[package]]
name = "serde"
version = "1.0.219"
name = "rustversion"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "ryu"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_core"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.219"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.145"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
"serde_core",
]
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "shlex"
version = "1.3.0"
@@ -541,6 +695,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "typenum"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
[[package]]
name = "unicode-ident"
version = "1.0.18"
@@ -553,12 +713,83 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "uuid"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a"
dependencies = [
"getrandom 0.3.4",
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasip2"
version = "1.0.1+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
dependencies = [
"wit-bindgen",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40"
dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4"
dependencies = [
"unicode-ident",
]
[[package]]
name = "webpki"
version = "0.22.4"
@@ -657,6 +888,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "wit-bindgen"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
[[package]]
name = "x509-parser"
version = "0.16.0"
libraries/droplet/Cargo.toml
+5
View File
@@ -16,6 +16,11 @@ dyn-clone = "1.0.20"
tokio = { version = "^1.48.0", features = ["process", "fs", "io-util"] }
anyhow = "1.0.100"
async-trait = "0.1.89"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.145"
humansize = "2.1.3"
uuid = { version = "1.19.0", features = ["v4"] }
sha2 = "0.10.9"
[dependencies.x509-parser]
version = "0.17.0"
libraries/droplet/src/lib.rs
+1
View File
@@ -3,3 +3,4 @@
pub mod file_utils;
pub mod ssl;
pub mod versions;
pub mod manifest;
libraries/droplet/src/manifest.rs
+193
View File
@@ -0,0 +1,193 @@
use std::{collections::HashMap, path::Path};
use anyhow::anyhow;
use hex::ToHex as _;
use humansize::{BINARY, format_size};
use serde::Serialize;
use serde_json::json;
use sha2::{Digest as _, Sha256};
use tokio::io::AsyncReadExt as _;
#[derive(Serialize)]
struct FileEntry {
filename: String,
start: usize,
length: usize,
permissions: u32,
}
#[derive(Serialize)]
struct ChunkData {
files: Vec<FileEntry>,
checksum: String,
}
#[derive(Serialize)]
struct Manifest {
version: String,
chunks: HashMap<String, ChunkData>,
size: u64,
}
const CHUNK_SIZE: u64 = 1024 * 1024 * 64;
const WIGGLE: u64 = 1024 * 1024 * 1;
use crate::versions::{create_backend_constructor, types::VersionFile};
pub async fn generate_manifest_rusty<T: Fn(String) -> (), V: Fn(f32) -> ()>(
dir: &Path,
progress_sfn: V,
log_sfn: T,
) -> anyhow::Result<String> {
let mut backend =
create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?;
let required_single_file = true; //backend.require_whole_files();
let files = backend.list_files().await?;
// Filepath to chunk data
let mut chunks: Vec<Vec<(VersionFile, u64, u64)>> = Vec::new();
let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new();
log_sfn(format!("organizing files into chunks...",));
for version_file in files {
// If we need the whole file, and this file would take up a whole chunk, add it to it's own chunk and move on
if required_single_file && version_file.size >= CHUNK_SIZE {
let size = version_file.size;
chunks.push(vec![(version_file, 0, size)]);
continue;
}
let mut current_size = current_chunk.iter().map(|v| v.2 - v.1).sum::<u64>();
// If we need the whole file, add this current file and move on, potentially adding and creating new chunk if need be
if required_single_file {
let size = version_file.size.try_into().unwrap();
current_chunk.push((version_file, 0, size));
current_size += size;
if current_size >= CHUNK_SIZE {
// Pop current and add, then reset
let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
chunks.push(new_chunk);
}
continue;
}
// Otherwise we calculate how much of the file we need, then use that much
let remaining_budget = (CHUNK_SIZE + WIGGLE) - current_size;
if version_file.size >= remaining_budget {
let remaining_budget = CHUNK_SIZE - current_size;
current_chunk.push((version_file.clone(), 0, remaining_budget));
let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
chunks.push(new_chunk);
let remaining_size = version_file.size - remaining_budget;
let mut running_offset = remaining_budget;
// Do everything but the last one
while running_offset < remaining_size {
let chunk_size = CHUNK_SIZE.min(remaining_size);
let chunk = vec![(version_file.clone(), running_offset, chunk_size)];
if chunk_size == CHUNK_SIZE {
chunks.push(chunk);
} else {
current_chunk = chunk;
}
running_offset += chunk_size;
}
continue;
} else {
let size = version_file.size;
current_chunk.push((version_file, 0, size));
current_size += size;
}
if current_size >= CHUNK_SIZE {
// Pop current and add, then reset
let new_chunk = std::mem::replace(&mut current_chunk, Vec::new());
chunks.push(new_chunk);
}
}
if current_chunk.len() > 0 {
chunks.push(current_chunk);
}
log_sfn(format!(
"organized into {} chunks, generating checksums...",
chunks.len()
));
let mut manifest: HashMap<String, ChunkData> = HashMap::new();
let mut total_manifest_length = 0;
let mut read_buf = vec![0; 1024 * 1024 * 64];
let chunk_len = chunks.len();
for (index, chunk) in chunks.into_iter().enumerate() {
let uuid = uuid::Uuid::new_v4().to_string();
let mut hasher = Sha256::new();
let mut chunk_data = ChunkData {
files: Vec::new(),
checksum: String::new(),
};
let mut chunk_length = 0;
for (file, start, length) in chunk {
log_sfn(format!(
"reading {} from {} to {}, {}",
file.relative_filename,
start,
start + length,
format_size(length, BINARY)
));
let mut reader = backend.reader(&file, start, start + length).await?;
loop {
let amount = reader.read(&mut read_buf).await?;
if amount == 0 {
break;
}
hasher.update(&read_buf[0..amount]);
}
chunk_length += length;
chunk_data.files.push(FileEntry {
filename: file.relative_filename,
start: start.try_into().unwrap(),
length: length.try_into().unwrap(),
permissions: file.permission,
});
}
log_sfn(format!(
"created chunk of size {} ({}/{})",
format_size(chunk_length, BINARY),
index,
chunk_len
));
total_manifest_length += chunk_length;
let hash: String = hasher.finalize().encode_hex();
chunk_data.checksum = hash;
manifest.insert(uuid, chunk_data);
let progress: f32 = (index as f32 / chunk_len as f32) * 100.0f32;
progress_sfn(progress);
}
Ok(json!(Manifest {
version: "2".to_string(),
chunks: manifest,
size: total_manifest_length
})
.to_string())
}