diff --git a/libraries/droplet/.gitmodules b/libraries/droplet/.gitmodules new file mode 100644 index 00000000..8dfab4fd --- /dev/null +++ b/libraries/droplet/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libarchive-rust"] + path = libarchive-rust + url = git@github.com:Drop-OSS/libarchive-rust.git diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index 7f99250a..157e5988 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "anyhow" version = "1.0.100" @@ -48,8 +63,8 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 2.0.117", + "synstructure 0.13.1", ] [[package]] @@ -60,8 +75,8 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 2.0.117", + "synstructure 0.13.1", ] [[package]] @@ -72,7 +87,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -83,7 +98,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -92,6 +107,21 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base64" version = "0.22.1" @@ -214,7 +244,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -228,11 +258,13 @@ dependencies = [ "getrandom 0.3.4", "hex", "humansize", + "libarchive", "rcgen", "ring", "serde", "serde_json", "sha2", + "speedometer", "time", "tokio", "uuid", @@ -245,6 +277,28 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + [[package]] name = "futures" version = "0.3.31" @@ -301,7 +355,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -367,6 +421,12 @@ dependencies = [ "wasip2", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "hex" version = "0.4.3" @@ -405,10 +465,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] -name = "libc" -version = "0.2.171" +name = "libarchive" +version = "0.1.1" +dependencies = [ + "libarchive3-sys", + "libc", +] + +[[package]] +name = "libarchive3-sys" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "3cd3beae8f59a4c7a806523269b5392037577c150446e88d684dfa6de6031ca7" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] +name = "libc" +version = "0.2.182" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" [[package]] name = "libm" @@ -428,6 +506,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.1.0" @@ -483,6 +570,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "oid-registry" version = "0.7.1" @@ -529,6 +625,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "powerfmt" version = "0.2.0" @@ -587,6 +689,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -641,7 +749,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -690,16 +798,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] -name = "syn" -version = "2.0.100" +name = "speedometer" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "2789736092fa21b44baf8590acb4b360cb91f0f597bd6c1f1741ca9644c95c1e" +dependencies = [ + "failure", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "synstructure" version = "0.13.1" @@ -708,7 +848,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -737,7 +877,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -748,7 +888,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -805,7 +945,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -820,6 +960,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -890,7 +1036,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 27dab856..eaaddaf9 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -11,7 +11,15 @@ hex = "0.4.3" time = "0.3.41" ring = "0.17.14" dyn-clone = "1.0.20" -tokio = { version = "^1.48.0", features = ["process", "fs", "io-util", "rt", "rt-multi-thread", "macros", "sync"] } +tokio = { version = "^1.48.0", features = [ + "process", + "fs", + "io-util", + "rt", + "rt-multi-thread", + "macros", + "sync", +] } anyhow = "1.0.100" async-trait = "0.1.89" serde = { version = "1.0.228", features = ["derive"] } @@ -21,6 +29,8 @@ uuid = { version = "1.19.0", features = ["v4"] } sha2 = "0.10.9" futures = "0.3.31" getrandom = "0.3.4" +libarchive = { version = "*", path = "./libarchive-rust" } +speedometer = "0.2.2" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/README.md b/libraries/droplet/README.md index b321a75e..266e5152 100644 --- a/libraries/droplet/README.md +++ b/libraries/droplet/README.md @@ -1,2 +1,8 @@ # droplet-rs -Core rust functionality for droplet + +A Rust-based library for utilities and functionality required both by the server, client, and other tools. + +## manifest generation +`droplet-rs` contains the manifest generation code, held in `manifest.rs`. + +It also includes the version backends to provide a unified read-write interface to files. \ No newline at end of file diff --git a/libraries/droplet/libarchive-rust b/libraries/droplet/libarchive-rust new file mode 160000 index 00000000..279fd5f7 --- /dev/null +++ b/libraries/droplet/libarchive-rust @@ -0,0 +1 @@ +Subproject commit 279fd5f727d5af50771cdfd660d657dff52c42f7 diff --git a/libraries/droplet/src/lib.rs b/libraries/droplet/src/lib.rs index 6781cc7a..8ad90277 100644 --- a/libraries/droplet/src/lib.rs +++ b/libraries/droplet/src/lib.rs @@ -5,6 +5,9 @@ pub mod file_utils; pub mod ssl; pub mod versions; pub mod manifest; +pub mod vm; + +extern crate libarchive; #[cfg(test)] pub mod tests; diff --git a/libraries/droplet/src/main.rs b/libraries/droplet/src/main.rs index a9289652..c512fe89 100644 --- a/libraries/droplet/src/main.rs +++ b/libraries/droplet/src/main.rs @@ -22,6 +22,8 @@ pub async fn main() { .await .unwrap(); + return; + // Sanity checks for (_, chunk_data) in manifest.chunks { for file in chunk_data.files { diff --git a/libraries/droplet/src/manifest.rs b/libraries/droplet/src/manifest.rs index 8f7f9e50..02294aa0 100644 --- a/libraries/droplet/src/manifest.rs +++ b/libraries/droplet/src/manifest.rs @@ -1,11 +1,15 @@ use std::{ - collections::HashMap, future::Future, path::Path, sync::{ - Arc, atomic::{AtomicU64, Ordering} - } + collections::HashMap, + future::Future, + mem, + path::Path, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; -use anyhow::{anyhow, Error}; -use futures::{stream::FuturesUnordered, StreamExt}; +use anyhow::anyhow; use hex::ToHex as _; use humansize::{format_size, BINARY}; use serde::{Deserialize, Serialize}; @@ -14,6 +18,7 @@ use tokio::{ io::AsyncReadExt as _, join, sync::{Mutex, Semaphore}, + task::JoinSet, }; #[derive(Serialize, Deserialize, Clone)] @@ -39,18 +44,21 @@ pub struct Manifest { pub key: [u8; 16], } -const CHUNK_SIZE: u64 = 1024 * 1024 * 64; +const CHUNK_SIZE: u64 = 1024 * 1024; //* 64; const MAX_FILE_COUNT: usize = 512; -use crate::versions::{create_backend_constructor, types::VersionFile}; +use crate::versions::{ + create_backend_constructor, + types::{VersionBackend, VersionFile}, +}; pub async fn generate_manifest_rusty( dir: &Path, progress_sfn: V, log_sfn: T, - reader_semaphore: Option<&Semaphore>, + reader_semaphore: Option>, ) -> anyhow::Result { - let mut backend = + let backend = create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?; let required_single_file = backend.require_whole_files(); @@ -137,18 +145,19 @@ pub async fn generate_manifest_rusty( let manifest: Arc>> = Arc::new(Mutex::new(HashMap::new())); let total_manifest_length = Arc::new(AtomicU64::new(0)); - let backend = Arc::new(Mutex::new(backend)); + // SAFETY: we .join_all() the futures using this + let backend: &'static Box = + unsafe { mem::transmute(&backend) }; - let futures: FuturesUnordered>> = - FuturesUnordered::new(); + let mut futures: JoinSet> = JoinSet::new(); let (send_log, mut recieve_log) = tokio::sync::mpsc::channel(16); let chunks_length = chunks.len(); for (index, chunk) in chunks.into_iter().enumerate() { let send_log = send_log.clone(); - let backend = backend.clone(); let total_manifest_length = total_manifest_length.clone(); let manifest = manifest.clone(); - futures.push(async move { + let reader_semaphore = reader_semaphore.clone(); + futures.spawn(async move { let mut read_buf = vec![0; 1024 * 1024 * 64]; let uuid = uuid::Uuid::new_v4().to_string(); @@ -164,6 +173,8 @@ pub async fn generate_manifest_rusty( let mut chunk_length = 0; + println!("starting chunk {}", index); + for (file, start, length) in chunk { let permit = if let Some(reader_semaphore) = &reader_semaphore { Some(reader_semaphore.acquire().await?) @@ -171,18 +182,20 @@ pub async fn generate_manifest_rusty( None }; - let mut reader = { - let mut backend_lock = backend.lock().await; - let reader = backend_lock.reader(&file, start, start + length).await?; - reader - }; + let mut reader = backend.reader(&file, start, start + length).await?; + + let mut total = 0; loop { let amount = reader.read(&mut read_buf).await?; if amount == 0 { break; } + total += amount; hasher.update(&read_buf[0..amount]); + if total as u64 > length { + panic!("read too much: target {}, got {}", length, total); + } } chunk_length += length; @@ -230,7 +243,7 @@ pub async fn generate_manifest_rusty( progress_sfn((current_progress / total_progress) * 100.0f32) } }, - futures.collect::>>() + futures.join_all() ); let manifest = manifest.lock().await; diff --git a/libraries/droplet/src/versions/archive_backend.rs b/libraries/droplet/src/versions/archive_backend.rs new file mode 100644 index 00000000..2f8633ad --- /dev/null +++ b/libraries/droplet/src/versions/archive_backend.rs @@ -0,0 +1,115 @@ +use std::{path::PathBuf, task::Poll}; + +use anyhow::{anyhow}; +use async_trait::async_trait; +use libarchive::{ + archive::{Entry, FileType, ReadCompression, ReadFormat}, + reader::{Builder, FileReader, Reader}, +}; +use tokio::io::AsyncRead; + +use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; + +pub struct ZipVersionBackend { + path: PathBuf, +} +impl ZipVersionBackend { + pub fn new(path: PathBuf) -> anyhow::Result { + Ok(Self { path }) + } + + fn open_archive(&self) -> Result { + let mut archive = Builder::new(); + archive.support_format(ReadFormat::All)?; + archive.support_compression(ReadCompression::All)?; + let archive = archive.open_file(&self.path)?; + + Ok(archive) + } +} + +struct ArchiveReader { + archive: FileReader, +} + +impl AsyncRead for ArchiveReader { + fn poll_read( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let block = match self.archive.read_block() { + Ok(v) => v, + Err(err) => return Poll::Ready(Err(std::io::Error::other(err.to_string()))), + }; + + let block = match block { + Some(v) => v, + None => return Poll::Ready(Ok(())), + }; + + buf.put_slice(block); + return Poll::Ready(Ok(())); + } +} + +#[async_trait] +impl VersionBackend for ZipVersionBackend { + async fn list_files(&self) -> anyhow::Result> { + let mut archive = self.open_archive()?; + let mut results = Vec::new(); + + while let Some(header) = archive.next_header() { + match header.filetype() { + FileType::RegularFile => (), + _ => { + continue; + } + } + results.push(VersionFile { + relative_filename: header.pathname().to_string(), + permission: 0o744, + size: header.size().try_into()?, + }); + println!("storing file {}", header.pathname()); + } + + Ok(results) + } + + async fn reader( + &self, + file: &VersionFile, + _start: u64, + _end: u64, + ) -> anyhow::Result> { + let mut archive = self.open_archive()?; + + // Find entry in archive + loop { + let entry = match archive.next_header() { + Some(v) => v, + None => return Err(anyhow!("entry not found:{}", file.relative_filename)), + }; + if entry.pathname() == file.relative_filename { + break; + } + } + + Ok(Box::new(ArchiveReader { archive })) + } + + async fn peek_file(&self, sub_path: String) -> anyhow::Result { + let files = self.list_files().await?; + let file = files + .iter() + .find(|v| v.relative_filename == sub_path) + .expect("file not found"); + + Ok(file.clone()) + } + + fn require_whole_files(&self) -> bool { + true + } +} diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs deleted file mode 100644 index ecd6876b..00000000 --- a/libraries/droplet/src/versions/backends.rs +++ /dev/null @@ -1,230 +0,0 @@ -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt; -use std::{ - fs::{metadata, read_dir}, - io::SeekFrom, - path::{Path, PathBuf}, - process::Stdio, - sync::LazyLock, -}; - -use anyhow::{anyhow, Result}; -use async_trait::async_trait; -use tokio::{ - fs::File, - io::{AsyncReadExt as _, AsyncSeekExt as _, BufReader}, - process::Command, -}; - -use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; - -pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { - if metadata(path)?.is_dir() { - let paths = read_dir(path)?; - for path_result in paths { - let full_path = path_result?.path(); - if metadata(&full_path)?.is_dir() { - _list_files(vec, &full_path)?; - } else { - vec.push(full_path); - } - } - }; - - Ok(()) -} - -#[derive(Clone)] -pub struct PathVersionBackend { - pub base_dir: PathBuf, -} - -#[async_trait] -impl VersionBackend for PathVersionBackend { - async fn list_files(&mut self) -> anyhow::Result> { - let mut vec = Vec::new(); - _list_files(&mut vec, &self.base_dir)?; - - let mut results = Vec::new(); - - for pathbuf in vec.iter() { - let relative = pathbuf.strip_prefix(self.base_dir.clone())?; - - results.push( - self.peek_file( - relative - .to_str() - .ok_or(anyhow!("Could not parse path: {}", relative.to_string_lossy()))? - .to_owned(), - ) - .await?, - ); - } - - Ok(results) - } - - async fn reader( - &mut self, - file: &VersionFile, - start: u64, - end: u64, - ) -> anyhow::Result> { - let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?; - - if start != 0 { - file.seek(SeekFrom::Start(start)).await?; - } - - if end != 0 { - return Ok(Box::new(file.take(end - start))); - } - - Ok(Box::new(file)) - } - - async fn peek_file(&mut self, sub_path: String) -> anyhow::Result { - let pathbuf = self.base_dir.join(sub_path.clone()); - if !pathbuf.exists() { - return Err(anyhow!("Path doesn't exist: {}", pathbuf.to_string_lossy())); - }; - - let file = File::open(pathbuf.clone()).await?; - let metadata = file.try_clone().await?.metadata().await?; - let permission_object = metadata.permissions(); - let permissions = { - let perm: u32; - #[cfg(target_family = "unix")] - { - perm = permission_object.mode(); - } - #[cfg(not(target_family = "unix"))] - { - perm = 0 - } - perm - }; - - Ok(VersionFile { - relative_filename: sub_path, - permission: permissions, - size: metadata.len(), - }) - } - - fn require_whole_files(&self) -> bool { - false - } -} - -pub static SEVEN_ZIP_INSTALLED: LazyLock = - LazyLock::new(|| std::process::Command::new("7z").output().is_ok()); -// https://7-zip.opensource.jp/chm/general/formats.htm -// Intentionally repeated some because it's a trivial cost and it's easier to directly copy from the docs above -pub const SUPPORTED_FILE_EXTENSIONS: [&str; 89] = [ - "7z", "bz2", "bzip2", "tbz2", "tbz", "gz", "gzip", "tgz", "tar", "wim", "swm", "esd", "xz", - "txz", "zip", "zipx", "jar", "xpi", "odt", "ods", "docx", "xlsx", "epub", "apm", "ar", "a", - "deb", "lib", "arj", "cab", "chm", "chw", "chi", "chq", "msi", "msp", "doc", "xls", "ppt", - "cpio", "cramfs", "dmg", "ext", "ext2", "ext3", "ext4", "img", "fat", "img", "hfs", "hfsx", - "hxs", "hxr", "hxq", "hxw", "lit", "ihex", "iso", "img", "lzh", "lha", "lzma", "mbr", "mslz", - "mub", "nsis", "ntfs", "img", "mbr", "rar", "r00", "rpm", "ppmd", "qcow", "qcow2", "qcow2c", - "squashfs", "udf", "iso", "img", "scap", "uefif", "vdi", "vhd", "vmdk", "xar", "pkg", "z", - "taz", -]; - -#[derive(Clone)] -pub struct ZipVersionBackend { - path: String, -} -impl ZipVersionBackend { - pub fn new(path: PathBuf) -> anyhow::Result { - Ok(Self { - path: path.to_str().expect("invalid utf path").to_owned(), - }) - } -} - -#[async_trait] -impl VersionBackend for ZipVersionBackend { - async fn list_files(&mut self) -> anyhow::Result> { - let mut list_command = Command::new("7z"); - list_command.args(vec!["l", &self.path]); - let result = list_command.output().await?; - if !result.status.success() { - return Err(anyhow!( - "failed to list files: code {:?}", - result.status.code() - )); - } - let raw_result = String::from_utf8(result.stdout)?; - let mut lines = raw_result.split("\n").skip(11); - - let mut column_lines = lines - .find(|v| v.starts_with('-')) - .ok_or(anyhow!("invalid 7z output"))? - .chars(); - let file_lines = lines.take_while(|v| !v.starts_with("-")); - - /* - Date Time Attr Size Compressed Name - ------------------- ----- ------------ ------------ ------------------------ - */ - let datetime_pos = 0usize; - let attrs_pos = datetime_pos + column_lines.position(|v| v == ' ').unwrap() + 1; - let size_pos = attrs_pos + column_lines.position(|v| v == ' ').unwrap() + 1; - let compressed_size_pos = size_pos + column_lines.position(|v| v == ' ').unwrap() + 1; - let name_pos = compressed_size_pos + column_lines.position(|v| v == ' ').unwrap() + 1; - - let mut results = Vec::new(); - for file in file_lines { - let name = file[name_pos..].trim(); - let size = file[size_pos..compressed_size_pos].trim(); - - let size = str::parse::(size)?; - - let version_file = VersionFile { - relative_filename: name.to_string(), - permission: 0o744, - size, - }; - - results.push(version_file); - } - - Ok(results) - } - - async fn reader( - &mut self, - file: &VersionFile, - _start: u64, - _end: u64, - ) -> anyhow::Result> { - let mut read_command = Command::new("7z"); - read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]); - let mut output = read_command - .stdout(Stdio::piped()) - .spawn() - .expect("failed to spawn 7z"); - let stdout = output - .stdout - .take() - .ok_or(anyhow!("no stdout on 7zip process"))?; - let reader = BufReader::new(stdout); - Ok(Box::new(reader)) - } - - async fn peek_file(&mut self, sub_path: String) -> anyhow::Result { - let files = self.list_files().await?; - let file = files - .iter() - .find(|v| v.relative_filename == sub_path) - .expect("file not found"); - - Ok(file.clone()) - } - - fn require_whole_files(&self) -> bool { - true - } -} diff --git a/libraries/droplet/src/versions/mod.rs b/libraries/droplet/src/versions/mod.rs index 4127d9d2..5549ff7c 100644 --- a/libraries/droplet/src/versions/mod.rs +++ b/libraries/droplet/src/versions/mod.rs @@ -1,15 +1,37 @@ -use std::path::Path; +use std::{ + fs::{metadata, read_dir}, + path::{Path, PathBuf}, +}; use anyhow::Result; use crate::versions::{ - backends::{ - PathVersionBackend, ZipVersionBackend, SEVEN_ZIP_INSTALLED, SUPPORTED_FILE_EXTENSIONS, - }, - types::VersionBackend, + archive_backend::ZipVersionBackend, path_backend::PathVersionBackend, types::VersionBackend, }; -pub mod backends; +pub mod archive_backend; +pub mod path_backend; + +pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { + if metadata(path)?.is_dir() { + let paths = read_dir(path)?; + for path_result in paths { + let full_path = path_result?.path(); + if metadata(&full_path)?.is_dir() { + _list_files(vec, &full_path)?; + } else { + vec.push(full_path); + } + } + }; + + Ok(()) +} + +const SUPPORTED_FILE_EXTENSIONS: [&'static str; 11] = [ + "tar", "pax", "cpio", "zip", "jar", "ar", "xar", "rar", "rpm", "7z", "iso", +]; + pub mod types; pub fn create_backend_constructor<'a>( path: &Path, @@ -26,27 +48,17 @@ pub fn create_backend_constructor<'a>( })); }; - if *SEVEN_ZIP_INSTALLED { - /* - Slow 7zip integrity test - let mut test = Command::new("7z"); - test.args(vec!["t", path.to_str().expect("invalid utf path")]); - let status = test.status().ok()?; - if status.code().unwrap_or(1) == 0 { - let buf = path.to_path_buf(); - return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); - } - */ - // Fast filename-based test - if let Some(extension) = path.extension().and_then(|v| v.to_str()) { - let supported = SUPPORTED_FILE_EXTENSIONS - .iter() - .any(|v| **v == *extension); - if supported { - let buf = path.to_path_buf(); - return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); - } - } + let file_extension = path + .extension() + .map(|v| v.to_str()) + .flatten()?; + + if SUPPORTED_FILE_EXTENSIONS + .iter() + .any(|v| *v == file_extension) + { + let buf = path.to_path_buf(); + return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?)))); } None diff --git a/libraries/droplet/src/versions/path_backend.rs b/libraries/droplet/src/versions/path_backend.rs new file mode 100644 index 00000000..5a0247fa --- /dev/null +++ b/libraries/droplet/src/versions/path_backend.rs @@ -0,0 +1,98 @@ +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::{ + io::SeekFrom, + path::PathBuf, +}; + +use anyhow::anyhow; +use async_trait::async_trait; +use tokio::{ + fs::File, + io::{AsyncReadExt as _, AsyncSeekExt as _}, +}; + +#[derive(Clone)] +pub struct PathVersionBackend { + pub base_dir: PathBuf, +} + +use crate::versions::{_list_files, types::{MinimumFileObject, VersionBackend, VersionFile}}; + +#[async_trait] +impl VersionBackend for PathVersionBackend { + async fn list_files(&self) -> anyhow::Result> { + let mut vec = Vec::new(); + _list_files(&mut vec, &self.base_dir)?; + + let mut results = Vec::new(); + + for pathbuf in vec.iter() { + let relative = pathbuf.strip_prefix(self.base_dir.clone())?; + + results.push( + self.peek_file( + relative + .to_str() + .ok_or(anyhow!("Could not parse path: {}", relative.to_string_lossy()))? + .to_owned(), + ) + .await?, + ); + } + + Ok(results) + } + + async fn reader( + &self, + file: &VersionFile, + start: u64, + end: u64, + ) -> anyhow::Result> { + let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?; + + if start != 0 { + file.seek(SeekFrom::Start(start)).await?; + } + + if end != 0 { + return Ok(Box::new(file.take(end - start))); + } + + Ok(Box::new(file)) + } + + async fn peek_file(&self, sub_path: String) -> anyhow::Result { + let pathbuf = self.base_dir.join(sub_path.clone()); + if !pathbuf.exists() { + return Err(anyhow!("Path doesn't exist: {}", pathbuf.to_string_lossy())); + }; + + let file = File::open(pathbuf.clone()).await?; + let metadata = file.try_clone().await?.metadata().await?; + let permission_object = metadata.permissions(); + let permissions = { + let perm: u32; + #[cfg(target_family = "unix")] + { + perm = permission_object.mode(); + } + #[cfg(not(target_family = "unix"))] + { + perm = 0 + } + perm + }; + + Ok(VersionFile { + relative_filename: sub_path, + permission: permissions, + size: metadata.len(), + }) + } + + fn require_whole_files(&self) -> bool { + false + } +} diff --git a/libraries/droplet/src/versions/types.rs b/libraries/droplet/src/versions/types.rs index 5a7cfd95..317408da 100644 --- a/libraries/droplet/src/versions/types.rs +++ b/libraries/droplet/src/versions/types.rs @@ -1,7 +1,6 @@ use std::fmt::Debug; use async_trait::async_trait; -use dyn_clone::DynClone; use tokio::io::AsyncRead; #[derive(Debug, Clone)] @@ -16,16 +15,14 @@ impl MinimumFileObject for T {} #[async_trait] -pub trait VersionBackend: DynClone { +pub trait VersionBackend { fn require_whole_files(&self) -> bool; - async fn list_files(&mut self) -> anyhow::Result>; - async fn peek_file(&mut self, sub_path: String) -> anyhow::Result; + async fn list_files(&self) -> anyhow::Result>; + async fn peek_file(&self, sub_path: String) -> anyhow::Result; async fn reader( - &mut self, + &self, file: &VersionFile, start: u64, end: u64, ) -> anyhow::Result>; } - -dyn_clone::clone_trait_object!(VersionBackend); diff --git a/libraries/droplet/src/vm/mod.rs b/libraries/droplet/src/vm/mod.rs new file mode 100644 index 00000000..e69de29b