From 1db9e6264b12c7e3175115bf29978704e846decb Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 21 Jan 2026 20:35:39 +1100 Subject: [PATCH] feat: S3 chunk uploading --- cli/.envrc | 1 + cli/.gitignore | 3 +- cli/Cargo.lock | 157 +----------------------- cli/Cargo.toml | 4 +- cli/flake.nix | 2 +- cli/src/commands/upload/chunk_reader.rs | 95 ++++++++++++++ cli/src/commands/upload/mod.rs | 1 + cli/src/commands/upload/s3.rs | 15 ++- 8 files changed, 110 insertions(+), 168 deletions(-) create mode 100644 cli/.envrc create mode 100644 cli/src/commands/upload/chunk_reader.rs diff --git a/cli/.envrc b/cli/.envrc new file mode 100644 index 00000000..3550a30f --- /dev/null +++ b/cli/.envrc @@ -0,0 +1 @@ +use flake diff --git a/cli/.gitignore b/cli/.gitignore index 9c242044..83807067 100644 --- a/cli/.gitignore +++ b/cli/.gitignore @@ -1,3 +1,4 @@ /target logs/ -.vscode \ No newline at end of file +.vscode +.direnv \ No newline at end of file diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 2b5d10b0..a1ca3993 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -2,21 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "android_system_properties" version = "0.1.5" @@ -235,21 +220,6 @@ dependencies = [ "thiserror 2.0.17", ] -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link 0.2.1", -] - [[package]] name = "base64" version = "0.22.1" @@ -384,33 +354,6 @@ dependencies = [ "cc", ] -[[package]] -name = "color-eyre" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d" -dependencies = [ - "backtrace", - "color-spantrace", - "eyre", - "indenter", - "once_cell", - "owo-colors", - "tracing-error", -] - -[[package]] -name = "color-spantrace" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427" -dependencies = [ - "once_cell", - "owo-colors", - "tracing-core", - "tracing-error", -] - [[package]] name = "colorchoice" version = "1.0.4" @@ -650,12 +593,12 @@ dependencies = [ "async-trait", "chrono", "clap", - "color-eyre", "console", "dialoguer", "dirs", "droplet-rs", "fern", + "futures", "indicatif", "log", "rand", @@ -735,16 +678,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "eyre" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" -dependencies = [ - "indenter", - "once_cell", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -929,12 +862,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "h2" version = "0.4.13" @@ -1250,12 +1177,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "indenter" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" - [[package]] name = "indexmap" version = "2.12.1" @@ -1445,15 +1366,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - [[package]] name = "mio" version = "1.1.1" @@ -1585,15 +1497,6 @@ dependencies = [ "objc2-core-foundation", ] -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "oid-registry" version = "0.7.1" @@ -1690,12 +1593,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "owo-colors" -version = "4.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" - [[package]] name = "pem" version = "3.0.6" @@ -2042,12 +1939,6 @@ dependencies = [ "url", ] -[[package]] -name = "rustc-demangle" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -2302,15 +2193,6 @@ dependencies = [ "digest", ] -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - [[package]] name = "shell-words" version = "1.1.1" @@ -2497,15 +2379,6 @@ dependencies = [ "syn", ] -[[package]] -name = "thread_local" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" -dependencies = [ - "cfg-if", -] - [[package]] name = "time" version = "0.3.44" @@ -2704,28 +2577,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", - "valuable", -] - -[[package]] -name = "tracing-error" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" -dependencies = [ - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" -dependencies = [ - "sharded-slab", - "thread_local", - "tracing-core", ] [[package]] @@ -2799,12 +2650,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - [[package]] name = "vcpkg" version = "0.2.15" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 80b1d19f..f3202a91 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -13,6 +13,7 @@ dialoguer = "0.12.0" dirs = "6.0.0" droplet-rs = { git = "https://github.com/Drop-OSS/droplet-rs.git", version = "0.14" } fern = { version = "0.7.1", features = ["colored"] } +futures = "0.3.31" indicatif = "0.18.3" log = "0.4.29" rand = "0.9.2" @@ -20,8 +21,7 @@ reqwest = { version = "0.13.1", features = ["json"] } rust-s3 = "0.37.1" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" -tokio = { version = "1.48.0", features = ["macros"] } +tokio = { version = "1.48.0", features = ["fs", "macros"] } tokio-util = "0.7.18" url = "2.5.8" webbrowser = "1.0.6" -color-eyre = "0.6.5" diff --git a/cli/flake.nix b/cli/flake.nix index e98989c5..7c806fd3 100644 --- a/cli/flake.nix +++ b/cli/flake.nix @@ -41,10 +41,10 @@ buildInputs = libraries; shellHook = '' - echo "Drop-OSS app development environment loaded." export LD_LIBRARY_PATH="${ pkgs.lib.makeLibraryPath libraries }:$LD_LIBRARY_PATH" + echo "Downpour development environment loaded" ''; }; } diff --git a/cli/src/commands/upload/chunk_reader.rs b/cli/src/commands/upload/chunk_reader.rs new file mode 100644 index 00000000..52035f19 --- /dev/null +++ b/cli/src/commands/upload/chunk_reader.rs @@ -0,0 +1,95 @@ +use droplet_rs::manifest::ChunkData; +use std::{ + cmp::min, + fs::File, + io::{Read, Seek, SeekFrom}, + task::Poll, vec::IntoIter, +}; +use tokio::io::AsyncRead; + +pub struct ChunkReader { + files: IntoIter, + active: Option, +} + +pub struct LimitedFileReader { + file: File, + to_read_remaining: usize, +} + +impl Read for LimitedFileReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let to_read = min(self.to_read_remaining, buf.len()); + let (to_read, _remaining) = buf.split_at_mut(to_read); + let read = self.file.read(to_read)?; + self.to_read_remaining -= read; + Ok(read) + } +} + +impl ChunkReader { + pub fn new(chunk: &ChunkData) -> Self { + let files = chunk.files.iter().map(|f| { + let mut file = File::open(&f.filename).unwrap(); + file.seek(SeekFrom::Start(f.start as u64)).unwrap(); // TODO: Fix unwraps + LimitedFileReader { + file, + to_read_remaining: f.length, + } + }).collect::>().into_iter(); + Self { + files: files, + active: None, + } + } +} +impl AsyncRead for ChunkReader { + fn poll_read( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let mut s = self; + loop { + if let Some(active) = &mut s.active { + match active.read(buf.initialize_unfilled()) { + Ok(0) => { + s.active = None; + continue; + } + Ok(_) => return Poll::Ready(Ok(())), + Err(e) => return Poll::Ready(Err(e)), + } + } else { + if let Some(next) = s.files.next() { + s.active = Some(next); + } else { + return Poll::Ready(Ok(())); + } + } + } + } +} + +impl Read for ChunkReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + loop { + if let Some(active) = &mut self.active { + match active.read(buf) { + Ok(0) => { + self.active = None; + continue; + } + Ok(n) => return Ok(n), + Err(e) => return Err(e), + } + } else { + if let Some(next) = self.files.next() { + self.active = Some(next); + } else { + return Ok(0); + } + } + } + } +} diff --git a/cli/src/commands/upload/mod.rs b/cli/src/commands/upload/mod.rs index 54a7fc1b..3628fe41 100644 --- a/cli/src/commands/upload/mod.rs +++ b/cli/src/commands/upload/mod.rs @@ -2,3 +2,4 @@ pub mod interface; pub mod s3; pub mod speedtest; pub mod uploadable; +pub mod chunk_reader; diff --git a/cli/src/commands/upload/s3.rs b/cli/src/commands/upload/s3.rs index 5c348b34..ba852600 100644 --- a/cli/src/commands/upload/s3.rs +++ b/cli/src/commands/upload/s3.rs @@ -1,10 +1,6 @@ -use crate::{ - commands::configure::s3::S3Config, - commands::upload::{ - speedtest::{SPEEDTEST_PATH, Speedtest}, - uploadable::Uploadable, - }, -}; +use crate::commands::{configure::s3::S3Config, upload::{ + chunk_reader::ChunkReader, speedtest::{SPEEDTEST_PATH, Speedtest}, uploadable::Uploadable + }}; use async_trait::async_trait; use droplet_rs::manifest::{ChunkData, Manifest}; use s3::Bucket; @@ -30,7 +26,10 @@ impl Uploadable for S3 { chunk_id: &String, chunk: &ChunkData, ) -> anyhow::Result<()> { - todo!() + let path = &PathBuf::from(id).join(version).join(chunk_id).to_string_lossy().to_string(); + let mut reader = ChunkReader::new(chunk); + self.put_object_stream(&mut reader, path).await?; + Ok(()) } async fn upload_speedtest(&mut self) -> anyhow::Result<()> { if self.object_exists(SPEEDTEST_PATH).await? {