From 217079cdf790d4b388b61e8090959b2dc1bca7f5 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 14:29:41 +1100 Subject: [PATCH] feat: async update --- libraries/droplet/Cargo.lock | 62 ++++++++++++++- libraries/droplet/Cargo.toml | 5 +- libraries/droplet/src/versions/backends.rs | 92 +++++++++------------- libraries/droplet/src/versions/types.rs | 62 +++++++-------- 4 files changed, 131 insertions(+), 90 deletions(-) diff --git a/libraries/droplet/Cargo.lock b/libraries/droplet/Cargo.lock index a88d8f7e..17f4ffc1 100644 --- a/libraries/droplet/Cargo.lock +++ b/libraries/droplet/Cargo.lock @@ -75,6 +75,17 @@ dependencies = [ "syn", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -87,6 +98,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + [[package]] name = "cc" version = "1.2.17" @@ -158,9 +175,10 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.8.1" +version = "0.9.0" dependencies = [ "anyhow", + "async-trait", "dyn-clone", "hex", "rcgen", @@ -225,6 +243,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "mio" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "nom" version = "7.1.3" @@ -352,7 +381,7 @@ dependencies = [ "getrandom", "libc", "untrusted", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -396,6 +425,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +dependencies = [ + "libc", +] + [[package]] name = "syn" version = "2.0.100" @@ -495,7 +533,12 @@ version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ + "bytes", + "libc", + "mio", "pin-project-lite", + "signal-hook-registry", + "windows-sys 0.61.2", ] [[package]] @@ -526,6 +569,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" version = "0.52.0" @@ -535,6 +584,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.52.6" diff --git a/libraries/droplet/Cargo.toml b/libraries/droplet/Cargo.toml index 0ec263d7..12d2d69d 100644 --- a/libraries/droplet/Cargo.toml +++ b/libraries/droplet/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" authors = ["Drop-OSS"] name = "droplet-rs" -version = "0.8.1" +version = "0.9.0" license = "AGPL-3.0-only" description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop" @@ -13,8 +13,9 @@ time = "0.3.41" webpki = "0.22.4" ring = "0.17.14" dyn-clone = "1.0.20" -tokio = "1.48.0" +tokio = { version = "^1.48.0", features = ["process", "fs", "io-util"] } anyhow = "1.0.100" +async-trait = "0.1.89" [dependencies.x509-parser] version = "0.17.0" diff --git a/libraries/droplet/src/versions/backends.rs b/libraries/droplet/src/versions/backends.rs index 96fd6ad8..c42eb80a 100644 --- a/libraries/droplet/src/versions/backends.rs +++ b/libraries/droplet/src/versions/backends.rs @@ -1,21 +1,26 @@ #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::{ - cell::LazyCell, - fs::{self, metadata, File}, - io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Sink}, + fs::{metadata, read_dir}, + io::SeekFrom, path::{Path, PathBuf}, - process::{Child, ChildStdout, Command, Stdio}, - sync::{Arc, LazyLock}, + process::Stdio, + sync::LazyLock, }; use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use tokio::{ + fs::File, + io::{AsyncReadExt as _, AsyncSeekExt as _, BufReader}, + process::Command, +}; use crate::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { if metadata(path)?.is_dir() { - let paths = fs::read_dir(path)?; + let paths = read_dir(path)?; for path_result in paths { let full_path = path_result?.path(); if metadata(&full_path)?.is_dir() { @@ -33,8 +38,10 @@ pub fn _list_files(vec: &mut Vec, path: &Path) -> Result<()> { pub struct PathVersionBackend { pub base_dir: PathBuf, } + +#[async_trait] impl VersionBackend for PathVersionBackend { - fn list_files(&mut self) -> anyhow::Result> { + async fn list_files(&mut self) -> anyhow::Result> { let mut vec = Vec::new(); _list_files(&mut vec, &self.base_dir)?; @@ -49,23 +56,24 @@ impl VersionBackend for PathVersionBackend { .to_str() .ok_or(anyhow!("Could not parse path"))? .to_owned(), - )?, + ) + .await?, ); } Ok(results) } - fn reader( + async fn reader( &mut self, file: &VersionFile, start: u64, end: u64, - ) -> anyhow::Result> { - let mut file = File::open(self.base_dir.join(file.relative_filename.clone()))?; + ) -> anyhow::Result> { + let mut file = File::open(self.base_dir.join(file.relative_filename.clone())).await?; if start != 0 { - file.seek(SeekFrom::Start(start))?; + file.seek(SeekFrom::Start(start)).await?; } if end != 0 { @@ -75,14 +83,14 @@ impl VersionBackend for PathVersionBackend { Ok(Box::new(file)) } - fn peek_file(&mut self, sub_path: String) -> anyhow::Result { + async fn peek_file(&mut self, sub_path: String) -> anyhow::Result { let pathbuf = self.base_dir.join(sub_path.clone()); if !pathbuf.exists() { return Err(anyhow!("Path doesn't exist.")); }; - let file = File::open(pathbuf.clone())?; - let metadata = file.try_clone()?.metadata()?; + let file = File::open(pathbuf.clone()).await?; + let metadata = file.try_clone().await?.metadata().await?; let permission_object = metadata.permissions(); let permissions = { let perm: u32; @@ -110,7 +118,7 @@ impl VersionBackend for PathVersionBackend { } pub static SEVEN_ZIP_INSTALLED: LazyLock = - LazyLock::new(|| Command::new("7z").output().is_ok()); + LazyLock::new(|| std::process::Command::new("7z").output().is_ok()); // https://7-zip.opensource.jp/chm/general/formats.htm // Intentionally repeated some because it's a trivial cost and it's easier to directly copy from the docs above pub const SUPPORTED_FILE_EXTENSIONS: [&str; 89] = [ @@ -136,43 +144,12 @@ impl ZipVersionBackend { } } -pub struct ZipFileWrapper { - command: Child, - reader: BufReader, -} - -impl ZipFileWrapper { - pub fn new(mut command: Child) -> Self { - let stdout = command - .stdout - .take() - .expect("failed to access stdout of 7z"); - let reader = BufReader::new(stdout); - ZipFileWrapper { command, reader } - } -} - -/** - * This read implemention is a result of debugging hell - * It should probably be replaced with a .take() call. - */ -impl Read for ZipFileWrapper { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.reader.read(buf) - } -} - -impl Drop for ZipFileWrapper { - fn drop(&mut self) { - self.command.wait().expect("failed to wait for 7z exit"); - } -} - +#[async_trait] impl VersionBackend for ZipVersionBackend { - fn list_files(&mut self) -> anyhow::Result> { + async fn list_files(&mut self) -> anyhow::Result> { let mut list_command = Command::new("7z"); list_command.args(vec!["l", "-ba", &self.path]); - let result = list_command.output()?; + let result = list_command.output().await?; if !result.status.success() { return Err(anyhow!( "failed to list files: code {:?}", @@ -215,7 +192,7 @@ impl VersionBackend for ZipVersionBackend { Ok(results) } - fn reader( + async fn reader( &mut self, file: &VersionFile, start: u64, @@ -223,15 +200,20 @@ impl VersionBackend for ZipVersionBackend { ) -> anyhow::Result> { let mut read_command = Command::new("7z"); read_command.args(vec!["e", "-so", &self.path, &file.relative_filename]); - let output = read_command + let mut output = read_command .stdout(Stdio::piped()) .spawn() .expect("failed to spawn 7z"); - Ok(Box::new(ZipFileWrapper::new(output))) + let stdout = output + .stdout + .take() + .ok_or(anyhow!("no stdout on 7zip process"))?; + let reader = BufReader::new(stdout); + Ok(Box::new(reader)) } - fn peek_file(&mut self, sub_path: String) -> anyhow::Result { - let files = self.list_files()?; + async fn peek_file(&mut self, sub_path: String) -> anyhow::Result { + let files = self.list_files().await?; let file = files .iter() .find(|v| v.relative_filename == sub_path) diff --git a/libraries/droplet/src/versions/types.rs b/libraries/droplet/src/versions/types.rs index 91c234d5..4fe2503c 100644 --- a/libraries/droplet/src/versions/types.rs +++ b/libraries/droplet/src/versions/types.rs @@ -1,55 +1,55 @@ use std::{fmt::Debug, io::Read}; +use async_trait::async_trait; use dyn_clone::DynClone; use tokio::io::{self, AsyncRead}; #[derive(Debug, Clone)] pub struct VersionFile { - pub relative_filename: String, - pub permission: u32, - pub size: u64, + pub relative_filename: String, + pub permission: u32, + pub size: u64, } -pub trait MinimumFileObject: Read + Send {} -impl MinimumFileObject for T {} +pub trait MinimumFileObject: AsyncRead + Send {} +impl MinimumFileObject for T {} // Intentionally not a generic, because of types in read_file pub struct ReadToAsyncRead<'a> { - pub inner: Box, + pub inner: Box, } const ASYNC_READ_BUFFER_SIZE: usize = 8128; impl<'a> AsyncRead for ReadToAsyncRead<'a> { - fn poll_read( - mut self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - buf: &mut tokio::io::ReadBuf<'_>, - ) -> std::task::Poll> { - let mut read_buf = [0u8; ASYNC_READ_BUFFER_SIZE]; - let read_size = ASYNC_READ_BUFFER_SIZE.min(buf.remaining()); - match self.inner.read(&mut read_buf[0..read_size]) { - Ok(read) => { - buf.put_slice(&read_buf[0..read]); - std::task::Poll::Ready(Ok(())) - } - Err(err) => { - std::task::Poll::Ready(Err(err)) - }, + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let mut read_buf = [0u8; ASYNC_READ_BUFFER_SIZE]; + let read_size = ASYNC_READ_BUFFER_SIZE.min(buf.remaining()); + match self.inner.read(&mut read_buf[0..read_size]) { + Ok(read) => { + buf.put_slice(&read_buf[0..read]); + std::task::Poll::Ready(Ok(())) + } + Err(err) => std::task::Poll::Ready(Err(err)), + } } - } } +#[async_trait] pub trait VersionBackend: DynClone { - fn require_whole_files(&self) -> bool; - fn list_files(&mut self) -> anyhow::Result>; - fn peek_file(&mut self, sub_path: String) -> anyhow::Result; - fn reader( - &mut self, - file: &VersionFile, - start: u64, - end: u64, - ) -> anyhow::Result>; + fn require_whole_files(&self) -> bool; + async fn list_files(&mut self) -> anyhow::Result>; + async fn peek_file(&mut self, sub_path: String) -> anyhow::Result; + async fn reader( + &mut self, + file: &VersionFile, + start: u64, + end: u64, + ) -> anyhow::Result>; } dyn_clone::clone_trait_object!(VersionBackend);