jessikitty/drop
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: S3 chunk uploading

Browse Source
This commit is contained in:
quexeky
2026-01-21 20:35:39 +11:00
parent 69bef2b785
commit 1db9e6264b
8 changed files with 110 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/.envrc
+1
View File
@@ -0,0 +1 @@
use flake
cli/.gitignore
+1
View File
@@ -1,3 +1,4 @@
/target
logs/
.vscode
.direnv
cli/Cargo.lock
Generated
+1 -156
View File
@@ -2,21 +2,6 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "addr2line"
version = "0.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
dependencies = [
"gimli",
]
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "android_system_properties"
version = "0.1.5"
@@ -235,21 +220,6 @@ dependencies = [
"thiserror 2.0.17",
]
[[package]]
name = "backtrace"
version = "0.3.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
dependencies = [
"addr2line",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
"windows-link 0.2.1",
]
[[package]]
name = "base64"
version = "0.22.1"
@@ -384,33 +354,6 @@ dependencies = [
"cc",
]
[[package]]
name = "color-eyre"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d"
dependencies = [
"backtrace",
"color-spantrace",
"eyre",
"indenter",
"once_cell",
"owo-colors",
"tracing-error",
]
[[package]]
name = "color-spantrace"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427"
dependencies = [
"once_cell",
"owo-colors",
"tracing-core",
"tracing-error",
]
[[package]]
name = "colorchoice"
version = "1.0.4"
@@ -650,12 +593,12 @@ dependencies = [
"async-trait",
"chrono",
"clap",
"color-eyre",
"console",
"dialoguer",
"dirs",
"droplet-rs",
"fern",
"futures",
"indicatif",
"log",
"rand",
@@ -735,16 +678,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "eyre"
version = "0.6.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec"
dependencies = [
"indenter",
"once_cell",
]
[[package]]
name = "fastrand"
version = "2.3.0"
@@ -929,12 +862,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "gimli"
version = "0.32.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
[[package]]
name = "h2"
version = "0.4.13"
@@ -1250,12 +1177,6 @@ dependencies = [
"icu_properties",
]
[[package]]
name = "indenter"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5"
[[package]]
name = "indexmap"
version = "2.12.1"
@@ -1445,15 +1366,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
]
[[package]]
name = "mio"
version = "1.1.1"
@@ -1585,15 +1497,6 @@ dependencies = [
"objc2-core-foundation",
]
[[package]]
name = "object"
version = "0.37.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe"
dependencies = [
"memchr",
]
[[package]]
name = "oid-registry"
version = "0.7.1"
@@ -1690,12 +1593,6 @@ dependencies = [
"hashbrown 0.14.5",
]
[[package]]
name = "owo-colors"
version = "4.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52"
[[package]]
name = "pem"
version = "3.0.6"
@@ -2042,12 +1939,6 @@ dependencies = [
"url",
]
[[package]]
name = "rustc-demangle"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
[[package]]
name = "rustc-hash"
version = "2.1.1"
@@ -2302,15 +2193,6 @@ dependencies = [
"digest",
]
[[package]]
name = "sharded-slab"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
dependencies = [
"lazy_static",
]
[[package]]
name = "shell-words"
version = "1.1.1"
@@ -2497,15 +2379,6 @@ dependencies = [
"syn",
]
[[package]]
name = "thread_local"
version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
dependencies = [
"cfg-if",
]
[[package]]
name = "time"
version = "0.3.44"
@@ -2704,28 +2577,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
dependencies = [
"once_cell",
"valuable",
]
[[package]]
name = "tracing-error"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db"
dependencies = [
"tracing",
"tracing-subscriber",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e"
dependencies = [
"sharded-slab",
"thread_local",
"tracing-core",
]
[[package]]
@@ -2799,12 +2650,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "valuable"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "vcpkg"
version = "0.2.15"
cli/Cargo.toml
+2 -2
View File
@@ -13,6 +13,7 @@ dialoguer = "0.12.0"
dirs = "6.0.0"
droplet-rs = { git = "https://github.com/Drop-OSS/droplet-rs.git", version = "0.14" }
fern = { version = "0.7.1", features = ["colored"] }
futures = "0.3.31"
indicatif = "0.18.3"
log = "0.4.29"
rand = "0.9.2"
@@ -20,8 +21,7 @@ reqwest = { version = "0.13.1", features = ["json"] }
rust-s3 = "0.37.1"
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.148"
tokio = { version = "1.48.0", features = ["macros"] }
tokio = { version = "1.48.0", features = ["fs", "macros"] }
tokio-util = "0.7.18"
url = "2.5.8"
webbrowser = "1.0.6"
color-eyre = "0.6.5"
cli/flake.nix
+1 -1
View File
@@ -41,10 +41,10 @@
buildInputs = libraries;
shellHook = ''
echo "Drop-OSS app development environment loaded."
export LD_LIBRARY_PATH="${
pkgs.lib.makeLibraryPath libraries
}:$LD_LIBRARY_PATH"
echo "Downpour development environment loaded"
'';
};
}
cli/src/commands/upload/chunk_reader.rs
+95
View File
@@ -0,0 +1,95 @@
use droplet_rs::manifest::ChunkData;
use std::{
cmp::min,
fs::File,
io::{Read, Seek, SeekFrom},
task::Poll, vec::IntoIter,
};
use tokio::io::AsyncRead;
pub struct ChunkReader {
files: IntoIter<LimitedFileReader>,
active: Option<LimitedFileReader>,
}
pub struct LimitedFileReader {
file: File,
to_read_remaining: usize,
}
impl Read for LimitedFileReader {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let to_read = min(self.to_read_remaining, buf.len());
let (to_read, _remaining) = buf.split_at_mut(to_read);
let read = self.file.read(to_read)?;
self.to_read_remaining -= read;
Ok(read)
}
}
impl ChunkReader {
pub fn new(chunk: &ChunkData) -> Self {
let files = chunk.files.iter().map(|f| {
let mut file = File::open(&f.filename).unwrap();
file.seek(SeekFrom::Start(f.start as u64)).unwrap(); // TODO: Fix unwraps
LimitedFileReader {
file,
to_read_remaining: f.length,
}
}).collect::<Vec<LimitedFileReader>>().into_iter();
Self {
files: files,
active: None,
}
}
}
impl AsyncRead for ChunkReader {
fn poll_read(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
let mut s = self;
loop {
if let Some(active) = &mut s.active {
match active.read(buf.initialize_unfilled()) {
Ok(0) => {
s.active = None;
continue;
}
Ok(_) => return Poll::Ready(Ok(())),
Err(e) => return Poll::Ready(Err(e)),
}
} else {
if let Some(next) = s.files.next() {
s.active = Some(next);
} else {
return Poll::Ready(Ok(()));
}
}
}
}
}
impl Read for ChunkReader {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
loop {
if let Some(active) = &mut self.active {
match active.read(buf) {
Ok(0) => {
self.active = None;
continue;
}
Ok(n) => return Ok(n),
Err(e) => return Err(e),
}
} else {
if let Some(next) = self.files.next() {
self.active = Some(next);
} else {
return Ok(0);
}
}
}
}
}
cli/src/commands/upload/mod.rs
+1
View File
@@ -2,3 +2,4 @@ pub mod interface;
pub mod s3;
pub mod speedtest;
pub mod uploadable;
pub mod chunk_reader;
cli/src/commands/upload/s3.rs
+7 -8
View File
@@ -1,10 +1,6 @@
use crate::{
commands::configure::s3::S3Config,
commands::upload::{
speedtest::{SPEEDTEST_PATH, Speedtest},
uploadable::Uploadable,
},
};
use crate::commands::{configure::s3::S3Config, upload::{
chunk_reader::ChunkReader, speedtest::{SPEEDTEST_PATH, Speedtest}, uploadable::Uploadable
}};
use async_trait::async_trait;
use droplet_rs::manifest::{ChunkData, Manifest};
use s3::Bucket;
@@ -30,7 +26,10 @@ impl Uploadable for S3 {
chunk_id: &String,
chunk: &ChunkData,
) -> anyhow::Result<()> {
todo!()
let path = &PathBuf::from(id).join(version).join(chunk_id).to_string_lossy().to_string();
let mut reader = ChunkReader::new(chunk);
self.put_object_stream(&mut reader, path).await?;
Ok(())
}
async fn upload_speedtest(&mut self) -> anyhow::Result<()> {
if self.object_exists(SPEEDTEST_PATH).await? {