From 31e794f637edba0e58af7e3d3aeef007d12a7df9 Mon Sep 17 00:00:00 2001 From: quexeky Date: Fri, 18 Oct 2024 07:45:09 +1100 Subject: [PATCH 01/37] Progress on downloads. Currently working on parsing functions to be run asynchronously --- desktop/src-tauri/Cargo.lock | 13 ++++++ desktop/src-tauri/Cargo.toml | 1 + desktop/src-tauri/src/downloads/downloads.rs | 22 ++++++++++ desktop/src-tauri/src/downloads/manifest.rs | 3 ++ desktop/src-tauri/src/downloads/mod.rs | 2 + desktop/src-tauri/src/lib.rs | 4 ++ desktop/src-tauri/src/utils.rs | 44 ++++++++++++++++++++ 7 files changed, 89 insertions(+) create mode 100644 desktop/src-tauri/src/downloads/downloads.rs create mode 100644 desktop/src-tauri/src/downloads/manifest.rs create mode 100644 desktop/src-tauri/src/downloads/mod.rs create mode 100644 desktop/src-tauri/src/utils.rs diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index c28ec7ed..bc2f4d70 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1037,6 +1037,7 @@ dependencies = [ "tauri-plugin-dialog", "tauri-plugin-shell", "tauri-plugin-single-instance", + "tokio", "url", "uuid", "webbrowser", @@ -4500,10 +4501,22 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "socket2", + "tokio-macros", "tracing", "windows-sys 0.52.0", ] +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "tokio-native-tls" version = "0.3.1" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 00f4a04b..edac0cd8 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -37,6 +37,7 @@ hex = "0.4.3" tauri-plugin-dialog = "2" env_logger = "0.11.5" http = "1.1.0" +tokio = { version = "1.40.0", features = ["rt", "tokio-macros"] } [dependencies.uuid] version = "1.10.0" diff --git a/desktop/src-tauri/src/downloads/downloads.rs b/desktop/src-tauri/src/downloads/downloads.rs new file mode 100644 index 00000000..9b53f45d --- /dev/null +++ b/desktop/src-tauri/src/downloads/downloads.rs @@ -0,0 +1,22 @@ +/* GENERAL OUTLINE +When downloading any game, the following details must be provided to the server: + - Game ID + - User token + - TBC + +The steps to then download a game are as follows: + 1. User requests + */ +use tauri::AppHandle; +use crate::auth::generate_authorization_header; +use crate::DB; +use crate::db::DatabaseImpls; +use crate::downloads::manifest::Manifest; + +#[tauri::command] +fn download_game(app: AppHandle, game_id: String) -> Result{ + todo!() +} + + + diff --git a/desktop/src-tauri/src/downloads/manifest.rs b/desktop/src-tauri/src/downloads/manifest.rs new file mode 100644 index 00000000..7df77646 --- /dev/null +++ b/desktop/src-tauri/src/downloads/manifest.rs @@ -0,0 +1,3 @@ +pub(crate) struct Manifest { + +} \ No newline at end of file diff --git a/desktop/src-tauri/src/downloads/mod.rs b/desktop/src-tauri/src/downloads/mod.rs new file mode 100644 index 00000000..c7dc3df4 --- /dev/null +++ b/desktop/src-tauri/src/downloads/mod.rs @@ -0,0 +1,2 @@ +mod downloads; +mod manifest; \ No newline at end of file diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 0aefa927..e91a19e3 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -3,6 +3,8 @@ mod db; mod library; mod remote; mod unpacker; +mod downloads; +mod utils; use auth::{auth_initiate, generate_authorization_header, recieve_handshake}; use db::{DatabaseInterface, DATA_ROOT_DIR}; @@ -101,6 +103,8 @@ pub fn run() { // Library fetch_library, fetch_game, + // Downloads + download_game ]) .plugin(tauri_plugin_shell::init()) .setup(|app| { diff --git a/desktop/src-tauri/src/utils.rs b/desktop/src-tauri/src/utils.rs new file mode 100644 index 00000000..aeb4f376 --- /dev/null +++ b/desktop/src-tauri/src/utils.rs @@ -0,0 +1,44 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::Ordering::Relaxed; +use crate::utils::ProgressChecker::Complete; + +#[derive(Eq, PartialEq)] +pub enum ProgressChecker { + Complete, + Incomplete +} + +// This function is designed to take in any function which does not regularly return a value, +// and instead loops over it until it returns "Complete". The current number of iterations +// is counted by "progress" +pub async fn progress_updater(function: Box ProgressChecker>, progress: AtomicUsize) { + loop { + if function() == ProgressChecker::Complete { break } + progress.fetch_add(1, Relaxed); + } +} + +pub async fn threaded_progress_updater(f: F, progress: AtomicUsize, max_threads: usize, instances: usize) -> ProgressChecker +where F: Fn() -> ProgressChecker + Send + Clone + Copy + 'static +{ + let mut threads = Vec::new(); + for instance in 0..instances { + let func = tokio::spawn(async move { + let res = f(); + return res + }); + threads.push(func); + } + let mut completed = ProgressChecker::Incomplete; + for thread in threads { + if thread.await.unwrap() == Complete { + completed = Complete + } + progress.fetch_add(1, Ordering::Relaxed); + } + completed +} + +fn test() -> ProgressChecker { + ProgressChecker::Incomplete +} \ No newline at end of file From f8b96d06e703cd033c16166b765b72ae353bcbc4 Mon Sep 17 00:00:00 2001 From: quexeky Date: Fri, 18 Oct 2024 20:42:26 +1100 Subject: [PATCH 02/37] Progress checker works --- desktop/src-tauri/src/downloads/mod.rs | 3 +- desktop/src-tauri/src/downloads/progress.rs | 66 +++++++++++++++++++ desktop/src-tauri/src/lib.rs | 3 +- desktop/src-tauri/src/tests/mod.rs | 1 + desktop/src-tauri/src/tests/progress_tests.rs | 17 +++++ desktop/src-tauri/src/utils.rs | 19 +++--- 6 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 desktop/src-tauri/src/downloads/progress.rs create mode 100644 desktop/src-tauri/src/tests/mod.rs create mode 100644 desktop/src-tauri/src/tests/progress_tests.rs diff --git a/desktop/src-tauri/src/downloads/mod.rs b/desktop/src-tauri/src/downloads/mod.rs index c7dc3df4..9124aaad 100644 --- a/desktop/src-tauri/src/downloads/mod.rs +++ b/desktop/src-tauri/src/downloads/mod.rs @@ -1,2 +1,3 @@ mod downloads; -mod manifest; \ No newline at end of file +mod manifest; +pub mod progress; \ No newline at end of file diff --git a/desktop/src-tauri/src/downloads/progress.rs b/desktop/src-tauri/src/downloads/progress.rs new file mode 100644 index 00000000..dfcf2d9a --- /dev/null +++ b/desktop/src-tauri/src/downloads/progress.rs @@ -0,0 +1,66 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use rayon::ThreadPoolBuilder; + +pub struct ProgressChecker +where T: 'static + Send + Sync +{ + counter: AtomicUsize, + f: Arc>, +} + +impl ProgressChecker +where T: Send + Sync +{ + pub fn new(f: Box) -> Self { + Self { + f: f.into(), + counter: AtomicUsize::new(0) + } + } + pub async fn run_contexts_sequentially_async(&self, contexts: Vec) { + for context in contexts { + (self.f)(context); + self.counter.fetch_add(1, Ordering::Relaxed); + } + } + pub fn run_contexts_sequentially(&self, contexts: Vec) { + for context in contexts { + (self.f)(context); + self.counter.fetch_add(1, Ordering::Relaxed); + } + } + pub async fn run_contexts_parallel_async(&self, contexts: Vec, max_threads: usize) { + let threads = ThreadPoolBuilder::new() + // If max_threads == 0, then the limit will be determined + // by Rayon's internal RAYON_NUM_THREADS + .num_threads(max_threads) + .build() + .unwrap(); + + for context in contexts { + let f = self.f.clone(); + threads.spawn(move || f(context)); + } + } + pub fn run_contexts_parallel(&self, contexts: Vec, max_threads: usize) { + let threads = ThreadPoolBuilder::new() + // If max_threads == 0, then the limit will be determined + // by Rayon's internal RAYON_NUM_THREADS + .num_threads(max_threads) + .build() + .unwrap(); + + for context in contexts { + let f = self.f.clone(); + threads.spawn(move || f(context)); + } + } + pub fn get_progress(&self) -> usize { + self.counter.load(Ordering::Relaxed) + } + // I strongly dislike type casting in my own code, so I've shovelled it into here + pub fn get_progress_percentage>(&self, capacity: C) -> f64 { + (self.get_progress() as f64) / (capacity.into()) + } +} \ No newline at end of file diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index e91a19e3..4291c3fe 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -5,6 +5,8 @@ mod remote; mod unpacker; mod downloads; mod utils; +#[cfg(test)] +mod tests; use auth::{auth_initiate, generate_authorization_header, recieve_handshake}; use db::{DatabaseInterface, DATA_ROOT_DIR}; @@ -104,7 +106,6 @@ pub fn run() { fetch_library, fetch_game, // Downloads - download_game ]) .plugin(tauri_plugin_shell::init()) .setup(|app| { diff --git a/desktop/src-tauri/src/tests/mod.rs b/desktop/src-tauri/src/tests/mod.rs new file mode 100644 index 00000000..9dff719b --- /dev/null +++ b/desktop/src-tauri/src/tests/mod.rs @@ -0,0 +1 @@ +mod progress_tests; \ No newline at end of file diff --git a/desktop/src-tauri/src/tests/progress_tests.rs b/desktop/src-tauri/src/tests/progress_tests.rs new file mode 100644 index 00000000..4ebeedf4 --- /dev/null +++ b/desktop/src-tauri/src/tests/progress_tests.rs @@ -0,0 +1,17 @@ +use crate::downloads::progress::ProgressChecker; + +#[test] +fn test_progress_sequentially() { + let p = ProgressChecker::new(Box::new(test_fn)); + p.run_contexts_sequentially((1..100).collect()); + println!("Progress: {}", p.get_progress_percentage(100)); +} +#[test] +fn test_progress_parallel() { + let p = ProgressChecker::new(Box::new(test_fn)); + p.run_contexts_parallel((1..100).collect(), 10); +} + +fn test_fn(int: usize) { + println!("{}", int); +} \ No newline at end of file diff --git a/desktop/src-tauri/src/utils.rs b/desktop/src-tauri/src/utils.rs index aeb4f376..aa06dfde 100644 --- a/desktop/src-tauri/src/utils.rs +++ b/desktop/src-tauri/src/utils.rs @@ -1,13 +1,9 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::Ordering::Relaxed; -use crate::utils::ProgressChecker::Complete; +use rayon::{ThreadPool, ThreadPoolBuilder}; -#[derive(Eq, PartialEq)] -pub enum ProgressChecker { - Complete, - Incomplete -} +/* // This function is designed to take in any function which does not regularly return a value, // and instead loops over it until it returns "Complete". The current number of iterations // is counted by "progress" @@ -18,16 +14,20 @@ pub async fn progress_updater(function: Box ProgressChecker>, progre } } +pub async fn new_progress_updater(function: Box D>, contexts: T, progress: AtomicUsize) { + +} + pub async fn threaded_progress_updater(f: F, progress: AtomicUsize, max_threads: usize, instances: usize) -> ProgressChecker where F: Fn() -> ProgressChecker + Send + Clone + Copy + 'static { let mut threads = Vec::new(); + let pool = ThreadPoolBuilder::new().num_threads(max_threads).build().unwrap(); for instance in 0..instances { - let func = tokio::spawn(async move { + pool.spawn(move || -> ProgressChecker { let res = f(); return res }); - threads.push(func); } let mut completed = ProgressChecker::Incomplete; for thread in threads { @@ -41,4 +41,5 @@ where F: Fn() -> ProgressChecker + Send + Clone + Copy + 'static fn test() -> ProgressChecker { ProgressChecker::Incomplete -} \ No newline at end of file +} + */ \ No newline at end of file From deae3875f7d3c961e1fe0625ec35a4fc8bfe554d Mon Sep 17 00:00:00 2001 From: quexeky Date: Fri, 18 Oct 2024 22:35:03 +1100 Subject: [PATCH 03/37] Update on GameDownload --- desktop/src-tauri/Cargo.lock | 37 +++++++++++++++++++ desktop/src-tauri/Cargo.toml | 3 +- .../src-tauri/src/downloads/game_download.rs | 30 +++++++++++++++ desktop/src-tauri/src/downloads/mod.rs | 3 +- desktop/src-tauri/src/downloads/progress.rs | 6 +-- desktop/src-tauri/src/tests/progress_tests.rs | 8 +++- 6 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 desktop/src-tauri/src/downloads/game_download.rs diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index bc2f4d70..d4d4a6a2 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1040,6 +1040,7 @@ dependencies = [ "tokio", "url", "uuid", + "versions", "webbrowser", ] @@ -2031,6 +2032,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.8" @@ -2282,6 +2292,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.0" @@ -2395,6 +2411,16 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -4890,6 +4916,17 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "versions" +version = "6.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f25d498b63d1fdb376b4250f39ab3a5ee8d103957346abacd911e2d8b612c139" +dependencies = [ + "itertools", + "nom", + "serde", +] + [[package]] name = "vswhom" version = "0.1.0" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index edac0cd8..46dd52f3 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -23,7 +23,7 @@ tauri-build = { version = "2.0.0", features = [] } [dependencies] tauri = { version = "2.0.0", features = [] } tauri-plugin-shell = "2.0.0" -serde = { version = "1", features = ["derive"] } +serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" ciborium = "0.2.2" rayon = "1.10.0" @@ -38,6 +38,7 @@ tauri-plugin-dialog = "2" env_logger = "0.11.5" http = "1.1.0" tokio = { version = "1.40.0", features = ["rt", "tokio-macros"] } +versions = { version = "6.3.2", features = ["serde"] } [dependencies.uuid] version = "1.10.0" diff --git a/desktop/src-tauri/src/downloads/game_download.rs b/desktop/src-tauri/src/downloads/game_download.rs new file mode 100644 index 00000000..94c03eae --- /dev/null +++ b/desktop/src-tauri/src/downloads/game_download.rs @@ -0,0 +1,30 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use versions::Version; +use crate::downloads::progress::ProgressChecker; + +pub struct GameDownload { + id: String, + version: Version, + progress: Arc +} +pub struct GameChunkCtx { + +} + +impl GameDownload { + pub fn new(id: String, version: Version) -> Self { + Self { + id, + version, + progress: Arc::new(AtomicUsize::new(0)) + } + } + pub async fn download(&self, max_threads: usize, contexts: Vec) { + let progress = ProgressChecker::new(Box::new(download_game_chunk), self.progress.clone()); + progress.run_contexts_sequentially_async(contexts).await; + } +} +fn download_game_chunk(ctx: GameChunkCtx) { + todo!() +} \ No newline at end of file diff --git a/desktop/src-tauri/src/downloads/mod.rs b/desktop/src-tauri/src/downloads/mod.rs index 9124aaad..ecf9742f 100644 --- a/desktop/src-tauri/src/downloads/mod.rs +++ b/desktop/src-tauri/src/downloads/mod.rs @@ -1,3 +1,4 @@ mod downloads; mod manifest; -pub mod progress; \ No newline at end of file +pub mod progress; +mod game_download; \ No newline at end of file diff --git a/desktop/src-tauri/src/downloads/progress.rs b/desktop/src-tauri/src/downloads/progress.rs index dfcf2d9a..e8176f97 100644 --- a/desktop/src-tauri/src/downloads/progress.rs +++ b/desktop/src-tauri/src/downloads/progress.rs @@ -5,17 +5,17 @@ use rayon::ThreadPoolBuilder; pub struct ProgressChecker where T: 'static + Send + Sync { - counter: AtomicUsize, + counter: Arc, f: Arc>, } impl ProgressChecker where T: Send + Sync { - pub fn new(f: Box) -> Self { + pub fn new(f: Box, counter_reference: Arc) -> Self { Self { f: f.into(), - counter: AtomicUsize::new(0) + counter: counter_reference } } pub async fn run_contexts_sequentially_async(&self, contexts: Vec) { diff --git a/desktop/src-tauri/src/tests/progress_tests.rs b/desktop/src-tauri/src/tests/progress_tests.rs index 4ebeedf4..1a0adc8b 100644 --- a/desktop/src-tauri/src/tests/progress_tests.rs +++ b/desktop/src-tauri/src/tests/progress_tests.rs @@ -1,14 +1,18 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; use crate::downloads::progress::ProgressChecker; #[test] fn test_progress_sequentially() { - let p = ProgressChecker::new(Box::new(test_fn)); + let counter = Arc::new(AtomicUsize::new(0)); + let p = ProgressChecker::new(Box::new(test_fn), counter.clone()); p.run_contexts_sequentially((1..100).collect()); println!("Progress: {}", p.get_progress_percentage(100)); } #[test] fn test_progress_parallel() { - let p = ProgressChecker::new(Box::new(test_fn)); + let counter = Arc::new(AtomicUsize::new(0)); + let p = ProgressChecker::new(Box::new(test_fn), counter.clone()); p.run_contexts_parallel((1..100).collect(), 10); } From f5b5d164ab4a44ae5f5a8dfa10313f4d9bc9b673 Mon Sep 17 00:00:00 2001 From: quexeky Date: Sat, 19 Oct 2024 14:54:29 +1100 Subject: [PATCH 04/37] Included in AppStatus (Also trying to link to Issue #1) --- desktop/src-tauri/src/downloads/game_download.rs | 12 +++++++++--- desktop/src-tauri/src/downloads/mod.rs | 2 +- desktop/src-tauri/src/lib.rs | 4 ++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/desktop/src-tauri/src/downloads/game_download.rs b/desktop/src-tauri/src/downloads/game_download.rs index 94c03eae..22d54f89 100644 --- a/desktop/src-tauri/src/downloads/game_download.rs +++ b/desktop/src-tauri/src/downloads/game_download.rs @@ -1,15 +1,20 @@ use std::sync::Arc; use std::sync::atomic::AtomicUsize; +use serde::{Deserialize, Serialize}; use versions::Version; use crate::downloads::progress::ProgressChecker; +#[derive(Serialize, Deserialize, Clone)] +#[serde(rename_all="camelCase")] pub struct GameDownload { id: String, version: Version, progress: Arc } +#[derive(Serialize, Deserialize)] +#[serde(rename_all="camelCase")] pub struct GameChunkCtx { - + chunk_id: usize, } impl GameDownload { @@ -22,9 +27,10 @@ impl GameDownload { } pub async fn download(&self, max_threads: usize, contexts: Vec) { let progress = ProgressChecker::new(Box::new(download_game_chunk), self.progress.clone()); - progress.run_contexts_sequentially_async(contexts).await; + progress.run_contexts_parallel_async(contexts, max_threads).await; } } fn download_game_chunk(ctx: GameChunkCtx) { - todo!() + todo!(); + // Need to implement actual download logic } \ No newline at end of file diff --git a/desktop/src-tauri/src/downloads/mod.rs b/desktop/src-tauri/src/downloads/mod.rs index ecf9742f..44fba89f 100644 --- a/desktop/src-tauri/src/downloads/mod.rs +++ b/desktop/src-tauri/src/downloads/mod.rs @@ -1,4 +1,4 @@ mod downloads; mod manifest; pub mod progress; -mod game_download; \ No newline at end of file +pub mod game_download; \ No newline at end of file diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 4291c3fe..1277d5f2 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -21,6 +21,7 @@ use std::{ }; use tauri_plugin_deep_link::DeepLinkExt; use crate::db::DatabaseImpls; +use crate::downloads::game_download::GameDownload; #[derive(Clone, Copy, Serialize)] pub enum AppStatus { @@ -45,6 +46,7 @@ pub struct AppState { status: AppStatus, user: Option, games: HashMap, + game_downloads: Vec } #[tauri::command] @@ -64,6 +66,7 @@ fn setup() -> AppState { status: AppStatus::NotConfigured, user: None, games: HashMap::new(), + game_downloads: vec![], }; } @@ -72,6 +75,7 @@ fn setup() -> AppState { status: auth_result.0, user: auth_result.1, games: HashMap::new(), + game_downloads: vec![], } } From 38eee872fe5c9c0218349d953321a584b5c70e29 Mon Sep 17 00:00:00 2001 From: quexeky Date: Sat, 19 Oct 2024 17:35:26 +1100 Subject: [PATCH 05/37] More fleshing out on how specifically game downloads will work --- .../src-tauri/src/downloads/game_download.rs | 62 +++++++++++++++++-- desktop/src-tauri/src/lib.rs | 4 +- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/desktop/src-tauri/src/downloads/game_download.rs b/desktop/src-tauri/src/downloads/game_download.rs index 22d54f89..8f4cd354 100644 --- a/desktop/src-tauri/src/downloads/game_download.rs +++ b/desktop/src-tauri/src/downloads/game_download.rs @@ -1,7 +1,8 @@ -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::sync::atomic::AtomicUsize; use serde::{Deserialize, Serialize}; use versions::Version; +use crate::AppState; use crate::downloads::progress::ProgressChecker; #[derive(Serialize, Deserialize, Clone)] @@ -9,7 +10,22 @@ use crate::downloads::progress::ProgressChecker; pub struct GameDownload { id: String, version: Version, - progress: Arc + progress: GameDownloadState +} +#[derive(Serialize, Deserialize, Clone)] +pub enum GameDownloadState { + Uninitialised, + Manifest, + Downloading(Arc), + Finished, + Stalled, + Failed, + Cancelled +} + +#[derive(Serialize, Deserialize, Clone)] +pub enum GameDownloadError { + } #[derive(Serialize, Deserialize)] #[serde(rename_all="camelCase")] @@ -17,20 +33,56 @@ pub struct GameChunkCtx { chunk_id: usize, } +#[derive(Serialize, Deserialize, Clone)] +pub struct GameDownloadManifest { + // TODO: Implement game manifest +} + impl GameDownload { pub fn new(id: String, version: Version) -> Self { Self { id, version, - progress: Arc::new(AtomicUsize::new(0)) + progress: GameDownloadState::Uninitialised } } - pub async fn download(&self, max_threads: usize, contexts: Vec) { - let progress = ProgressChecker::new(Box::new(download_game_chunk), self.progress.clone()); + pub async fn download(&mut self, max_threads: usize, contexts: Vec) -> Result<(), GameDownloadError> { + let progress = Arc::new(AtomicUsize::new(0)); + self.progress = GameDownloadState::Downloading(progress.clone()); + let progress = ProgressChecker::new(Box::new(download_game_chunk), progress); progress.run_contexts_parallel_async(contexts, max_threads).await; + Ok(()) + } + pub async fn download_manifest(&mut self) -> Result { + todo!() } } fn download_game_chunk(ctx: GameChunkCtx) { todo!(); // Need to implement actual download logic +} + +#[tauri::command] +pub async fn start_game_download( + game_id: String, + game_version: Version, + max_threads: usize, + state: tauri::State<'_, Mutex>, +) -> Result<(), GameDownloadError> { + let mut download = Arc::new(GameDownload::new(game_id, game_version)); + let mut app_state = state.lock().unwrap(); + app_state.game_downloads.push(download.clone()); + + let manifest = match download.download_manifest().await { + Ok(manifest) => { manifest } + Err(e) => { return Err(e) } + }; + + download.download(max_threads, manifest.parse_to_chunks()).await +} + +impl GameDownloadManifest { + fn parse_to_chunks(self) -> Vec { + todo!() + } } \ No newline at end of file diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 1277d5f2..3adba1a9 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -19,6 +19,7 @@ use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, sync::{LazyLock, Mutex} }; +use std::sync::Arc; use tauri_plugin_deep_link::DeepLinkExt; use crate::db::DatabaseImpls; use crate::downloads::game_download::GameDownload; @@ -46,7 +47,7 @@ pub struct AppState { status: AppStatus, user: Option, games: HashMap, - game_downloads: Vec + game_downloads: Vec> } #[tauri::command] @@ -109,7 +110,6 @@ pub fn run() { // Library fetch_library, fetch_game, - // Downloads ]) .plugin(tauri_plugin_shell::init()) .setup(|app| { From 34d2b978e5ad69ce7c39414aa78c4b3b8ad3d81e Mon Sep 17 00:00:00 2001 From: quexeky Date: Sat, 19 Oct 2024 17:35:26 +1100 Subject: [PATCH 06/37] More fleshing out on how specifically game downloads will work (#1) --- .../src-tauri/src/downloads/game_download.rs | 62 +++++++++++++++++-- desktop/src-tauri/src/lib.rs | 4 +- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/desktop/src-tauri/src/downloads/game_download.rs b/desktop/src-tauri/src/downloads/game_download.rs index 22d54f89..8f4cd354 100644 --- a/desktop/src-tauri/src/downloads/game_download.rs +++ b/desktop/src-tauri/src/downloads/game_download.rs @@ -1,7 +1,8 @@ -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::sync::atomic::AtomicUsize; use serde::{Deserialize, Serialize}; use versions::Version; +use crate::AppState; use crate::downloads::progress::ProgressChecker; #[derive(Serialize, Deserialize, Clone)] @@ -9,7 +10,22 @@ use crate::downloads::progress::ProgressChecker; pub struct GameDownload { id: String, version: Version, - progress: Arc + progress: GameDownloadState +} +#[derive(Serialize, Deserialize, Clone)] +pub enum GameDownloadState { + Uninitialised, + Manifest, + Downloading(Arc), + Finished, + Stalled, + Failed, + Cancelled +} + +#[derive(Serialize, Deserialize, Clone)] +pub enum GameDownloadError { + } #[derive(Serialize, Deserialize)] #[serde(rename_all="camelCase")] @@ -17,20 +33,56 @@ pub struct GameChunkCtx { chunk_id: usize, } +#[derive(Serialize, Deserialize, Clone)] +pub struct GameDownloadManifest { + // TODO: Implement game manifest +} + impl GameDownload { pub fn new(id: String, version: Version) -> Self { Self { id, version, - progress: Arc::new(AtomicUsize::new(0)) + progress: GameDownloadState::Uninitialised } } - pub async fn download(&self, max_threads: usize, contexts: Vec) { - let progress = ProgressChecker::new(Box::new(download_game_chunk), self.progress.clone()); + pub async fn download(&mut self, max_threads: usize, contexts: Vec) -> Result<(), GameDownloadError> { + let progress = Arc::new(AtomicUsize::new(0)); + self.progress = GameDownloadState::Downloading(progress.clone()); + let progress = ProgressChecker::new(Box::new(download_game_chunk), progress); progress.run_contexts_parallel_async(contexts, max_threads).await; + Ok(()) + } + pub async fn download_manifest(&mut self) -> Result { + todo!() } } fn download_game_chunk(ctx: GameChunkCtx) { todo!(); // Need to implement actual download logic +} + +#[tauri::command] +pub async fn start_game_download( + game_id: String, + game_version: Version, + max_threads: usize, + state: tauri::State<'_, Mutex>, +) -> Result<(), GameDownloadError> { + let mut download = Arc::new(GameDownload::new(game_id, game_version)); + let mut app_state = state.lock().unwrap(); + app_state.game_downloads.push(download.clone()); + + let manifest = match download.download_manifest().await { + Ok(manifest) => { manifest } + Err(e) => { return Err(e) } + }; + + download.download(max_threads, manifest.parse_to_chunks()).await +} + +impl GameDownloadManifest { + fn parse_to_chunks(self) -> Vec { + todo!() + } } \ No newline at end of file diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 1277d5f2..3adba1a9 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -19,6 +19,7 @@ use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, sync::{LazyLock, Mutex} }; +use std::sync::Arc; use tauri_plugin_deep_link::DeepLinkExt; use crate::db::DatabaseImpls; use crate::downloads::game_download::GameDownload; @@ -46,7 +47,7 @@ pub struct AppState { status: AppStatus, user: Option, games: HashMap, - game_downloads: Vec + game_downloads: Vec> } #[tauri::command] @@ -109,7 +110,6 @@ pub fn run() { // Library fetch_library, fetch_game, - // Downloads ]) .plugin(tauri_plugin_shell::init()) .setup(|app| { From 495bebe4a39f17620d31675dc7fb0a5b1e175afd Mon Sep 17 00:00:00 2001 From: quexeky Date: Sat, 19 Oct 2024 19:31:43 +1100 Subject: [PATCH 07/37] Theoretically adding queue support and optimistic manifest downloading (#1). Needs tests when actual functions are implemented --- .../src-tauri/src/downloads/game_download.rs | 79 +++++++++++++------ 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/desktop/src-tauri/src/downloads/game_download.rs b/desktop/src-tauri/src/downloads/game_download.rs index 8f4cd354..371af575 100644 --- a/desktop/src-tauri/src/downloads/game_download.rs +++ b/desktop/src-tauri/src/downloads/game_download.rs @@ -1,3 +1,4 @@ +use std::future::Future; use std::sync::{Arc, Mutex}; use std::sync::atomic::AtomicUsize; use serde::{Deserialize, Serialize}; @@ -5,35 +6,40 @@ use versions::Version; use crate::AppState; use crate::downloads::progress::ProgressChecker; -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize)] #[serde(rename_all="camelCase")] pub struct GameDownload { id: String, version: Version, - progress: GameDownloadState + progress: Arc, + state: Mutex, + manifest: Option> } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq)] pub enum GameDownloadState { Uninitialised, + Queued, Manifest, - Downloading(Arc), + Downloading, Finished, Stalled, Failed, Cancelled } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq)] pub enum GameDownloadError { - + ManifestAlreadyExists, + ManifestDoesNotExist, + ManifestDownloadError, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Ord, PartialOrd)] #[serde(rename_all="camelCase")] pub struct GameChunkCtx { chunk_id: usize, } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Eq, PartialEq)] pub struct GameDownloadManifest { // TODO: Implement game manifest } @@ -43,17 +49,38 @@ impl GameDownload { Self { id, version, - progress: GameDownloadState::Uninitialised + progress: Arc::new(AtomicUsize::new(0)), + state: Mutex::from(GameDownloadState::Uninitialised), + manifest: None } } - pub async fn download(&mut self, max_threads: usize, contexts: Vec) -> Result<(), GameDownloadError> { + pub async fn queue(&self) -> Result<(), GameDownloadError> { + self.change_state(GameDownloadState::Queued); + if self.manifest.is_none() { + return Ok(()) + } + self.download_manifest().await + } + pub async fn download(&self, max_threads: usize, contexts: Vec) -> Result<(), GameDownloadError> { let progress = Arc::new(AtomicUsize::new(0)); - self.progress = GameDownloadState::Downloading(progress.clone()); + self.change_state(GameDownloadState::Downloading); let progress = ProgressChecker::new(Box::new(download_game_chunk), progress); progress.run_contexts_parallel_async(contexts, max_threads).await; Ok(()) } - pub async fn download_manifest(&mut self) -> Result { + pub async fn download_manifest(&self) -> Result<(), GameDownloadError> { + if self.manifest.is_some() { + return Err(GameDownloadError::ManifestAlreadyExists); + } + todo!() // Need to actually download the manifest + } + pub fn change_state(&self, state: GameDownloadState) { + let mut lock = self.state.lock().unwrap(); + *lock = state; + } +} +impl GameDownloadManifest { + fn parse_to_chunks(&self) -> Vec { todo!() } } @@ -71,18 +98,26 @@ pub async fn start_game_download( ) -> Result<(), GameDownloadError> { let mut download = Arc::new(GameDownload::new(game_id, game_version)); let mut app_state = state.lock().unwrap(); - app_state.game_downloads.push(download.clone()); - let manifest = match download.download_manifest().await { - Ok(manifest) => { manifest } - Err(e) => { return Err(e) } + let tmp = download.clone(); + let manifest = &tmp.manifest; + + let Some(unlocked) = manifest else { return Err(GameDownloadError::ManifestDoesNotExist) }; + let lock = unlocked.lock().unwrap(); + + let chunks = lock.parse_to_chunks(); + + /* + let manifest = match d.manifest { + Some(lock) => { + let lock = lock.lock().unwrap(); + lock.parse_to_chunks() + }, + None => { return Err(GameDownloadError::ManifestDoesNotExist) } }; + */ - download.download(max_threads, manifest.parse_to_chunks()).await + app_state.game_downloads.push(download.clone()); + download.download(max_threads, chunks).await } -impl GameDownloadManifest { - fn parse_to_chunks(self) -> Vec { - todo!() - } -} \ No newline at end of file From eb37b2464aced040de2b483e648ec554923a5b5a Mon Sep 17 00:00:00 2001 From: quexeky Date: Sat, 19 Oct 2024 22:17:43 +1100 Subject: [PATCH 08/37] SLowly integrating game_download into the FE. Started with using the manifest minimal example in the server (#1) --- desktop/pages/store/index.vue | 27 ++++++++- .../src-tauri/src/downloads/game_download.rs | 58 +++++++++++++++++-- desktop/src-tauri/src/lib.rs | 4 +- 3 files changed, 80 insertions(+), 9 deletions(-) diff --git a/desktop/pages/store/index.vue b/desktop/pages/store/index.vue index 27e0f695..02446b26 100644 --- a/desktop/pages/store/index.vue +++ b/desktop/pages/store/index.vue @@ -1,3 +1,26 @@ \ No newline at end of file + + + + \ No newline at end of file diff --git a/desktop/src-tauri/src/downloads/game_download.rs b/desktop/src-tauri/src/downloads/game_download.rs index 371af575..159f38a7 100644 --- a/desktop/src-tauri/src/downloads/game_download.rs +++ b/desktop/src-tauri/src/downloads/game_download.rs @@ -1,9 +1,13 @@ use std::future::Future; +use std::str::FromStr; use std::sync::{Arc, Mutex}; use std::sync::atomic::AtomicUsize; +use log::info; use serde::{Deserialize, Serialize}; use versions::Version; -use crate::AppState; +use crate::{AppState, DB}; +use crate::auth::generate_authorization_header; +use crate::db::DatabaseImpls; use crate::downloads::progress::ProgressChecker; #[derive(Serialize, Deserialize)] @@ -32,6 +36,7 @@ pub enum GameDownloadError { ManifestAlreadyExists, ManifestDoesNotExist, ManifestDownloadError, + StatusError(u16) } #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Ord, PartialOrd)] #[serde(rename_all="camelCase")] @@ -72,7 +77,39 @@ impl GameDownload { if self.manifest.is_some() { return Err(GameDownloadError::ManifestAlreadyExists); } - todo!() // Need to actually download the manifest + + info!("Getting url components"); + let base_url = DB.fetch_base_url(); + let manifest_url = base_url + .join( + format!( + "/api/v1/client/metadata/manifest?id={}&version={}", + self.id, + self.version.to_string() + ) + .as_str() + ) + .unwrap(); + + info!("Generating authorization header"); + let header = generate_authorization_header(); + + info!("Generating & sending client"); + let client = reqwest::blocking::Client::new(); + let response = client + .get(manifest_url.to_string()) + .header("Authorization", header) + .send() + .unwrap(); + + info!("Got status"); + if response.status() != 200 { + return Err(GameDownloadError::StatusError(response.status().as_u16())); + } + + info!("{:?}", response.text()); + + Ok(()) } pub fn change_state(&self, state: GameDownloadState) { let mut lock = self.state.lock().unwrap(); @@ -92,16 +129,23 @@ fn download_game_chunk(ctx: GameChunkCtx) { #[tauri::command] pub async fn start_game_download( game_id: String, - game_version: Version, + game_version: String, max_threads: usize, state: tauri::State<'_, Mutex>, ) -> Result<(), GameDownloadError> { - let mut download = Arc::new(GameDownload::new(game_id, game_version)); - let mut app_state = state.lock().unwrap(); + + info!("Triggered Game Download"); + + let mut download = Arc::new(GameDownload::new(game_id, Version::from_str(&*game_version).unwrap())); + //let mut app_state = state.lock().unwrap(); let tmp = download.clone(); - let manifest = &tmp.manifest; + //let manifest = &tmp.manifest; + let res = download.download_manifest().await; + + res + /* let Some(unlocked) = manifest else { return Err(GameDownloadError::ManifestDoesNotExist) }; let lock = unlocked.lock().unwrap(); @@ -119,5 +163,7 @@ pub async fn start_game_download( app_state.game_downloads.push(download.clone()); download.download(max_threads, chunks).await + + */ } diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 3adba1a9..b4043c96 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -22,7 +22,7 @@ use std::{ use std::sync::Arc; use tauri_plugin_deep_link::DeepLinkExt; use crate::db::DatabaseImpls; -use crate::downloads::game_download::GameDownload; +use crate::downloads::game_download::{start_game_download, GameDownload}; #[derive(Clone, Copy, Serialize)] pub enum AppStatus { @@ -110,6 +110,8 @@ pub fn run() { // Library fetch_library, fetch_game, + // Downloads + start_game_download ]) .plugin(tauri_plugin_shell::init()) .setup(|app| { From 749b650b159b10f372ebbc47e9cfb67314e8cfc3 Mon Sep 17 00:00:00 2001 From: quexeky Date: Sun, 20 Oct 2024 20:45:02 +1100 Subject: [PATCH 09/37] Validated that loading data works --- desktop/pages/store/index.vue | 4 ++-- desktop/src-tauri/src/downloads/game_download.rs | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/desktop/pages/store/index.vue b/desktop/pages/store/index.vue index 02446b26..4369b873 100644 --- a/desktop/pages/store/index.vue +++ b/desktop/pages/store/index.vue @@ -13,7 +13,7 @@ import { invoke } from "@tauri-apps/api/core"; async function requestGame() { console.log("Requested game from FE"); - await invoke("start_game_download", { gameId: "123", gameVersion: "1.2.3", maxThreads: 4 }); + await invoke("start_game_download", { gameId: "94b8ac10-a6fc-4a94-b519-e6df78018e26", gameVersion: "1.11.2", maxThreads: 4 }); } function requestGameWrapper() { console.log("Wrapper started"); @@ -23,4 +23,4 @@ function requestGameWrapper() { console.log(e); }) } - \ No newline at end of file + diff --git a/desktop/src-tauri/src/downloads/game_download.rs b/desktop/src-tauri/src/downloads/game_download.rs index 159f38a7..6c2e3d72 100644 --- a/desktop/src-tauri/src/downloads/game_download.rs +++ b/desktop/src-tauri/src/downloads/game_download.rs @@ -95,19 +95,21 @@ impl GameDownload { let header = generate_authorization_header(); info!("Generating & sending client"); - let client = reqwest::blocking::Client::new(); + let client = reqwest::Client::new(); let response = client .get(manifest_url.to_string()) .header("Authorization", header) .send() + .await .unwrap(); info!("Got status"); if response.status() != 200 { + info!("Error status: {}", response.status()); return Err(GameDownloadError::StatusError(response.status().as_u16())); } - info!("{:?}", response.text()); + info!("{:?}", response.text().await.unwrap()); Ok(()) } @@ -142,9 +144,8 @@ pub async fn start_game_download( let tmp = download.clone(); //let manifest = &tmp.manifest; - let res = download.download_manifest().await; + download.download_manifest().await - res /* let Some(unlocked) = manifest else { return Err(GameDownloadError::ManifestDoesNotExist) }; let lock = unlocked.lock().unwrap(); From 258325fc11ed1c863f320cdf2c7f529f82ce81e0 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sun, 20 Oct 2024 20:55:37 +1100 Subject: [PATCH 10/37] fixed some of quexeky's BASED design decisions --- desktop/app.vue | 2 +- desktop/components/Header.vue | 8 ++++---- desktop/pages/store/index.vue | 4 ---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/desktop/app.vue b/desktop/app.vue index a146950f..4f900fba 100644 --- a/desktop/app.vue +++ b/desktop/app.vue @@ -44,7 +44,7 @@ listen("auth/failed", () => { }); listen("auth/finished", () => { - router.push("/"); + router.push("/store"); }); useHead({ diff --git a/desktop/components/Header.vue b/desktop/components/Header.vue index 731582d3..33ee5062 100644 --- a/desktop/components/Header.vue +++ b/desktop/components/Header.vue @@ -1,11 +1,10 @@ diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index d4d4a6a2..622e924d 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1039,6 +1039,7 @@ dependencies = [ "tauri-plugin-single-instance", "tokio", "url", + "urlencoding", "uuid", "versions", "webbrowser", @@ -4815,6 +4816,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "urlpattern" version = "0.3.0" diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 46dd52f3..edbcbbb5 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -39,6 +39,7 @@ env_logger = "0.11.5" http = "1.1.0" tokio = { version = "1.40.0", features = ["rt", "tokio-macros"] } versions = { version = "6.3.2", features = ["serde"] } +urlencoding = "2.1.3" [dependencies.uuid] version = "1.10.0" diff --git a/desktop/src-tauri/src/downloads/download_files.rs b/desktop/src-tauri/src/downloads/download_files.rs deleted file mode 100644 index 97e3aeae..00000000 --- a/desktop/src-tauri/src/downloads/download_files.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::fs::File; -use std::io::{Seek, SeekFrom, Write}; -use std::os::unix::fs::MetadataExt; -use std::sync::{Arc, Mutex}; -use log::info; -use uuid::Bytes; -use crate::auth::generate_authorization_header; -use crate::DB; -use crate::db::DatabaseImpls; -use crate::downloads::manifest::DropDownloadContext; - -const CHUNK_SIZE: u64 = 1024 * 1024 * 64; -pub fn download_game_chunk(ctx: DropDownloadContext) { - let base_url = DB.fetch_base_url(); - - - let client = reqwest::blocking::Client::new(); - let chunk_url = base_url.join( - &format!( - "/api/v1/client/chunk?id={}&version={}&name={}&chunk={}", - ctx.game_id, - ctx.version, - ctx.file_name, - ctx.index - )).unwrap(); - - let header = generate_authorization_header(); - - let response = client - .get(chunk_url) - .header("Authorization", header) - .send() - .unwrap(); - let response_data = response.bytes().unwrap(); - - write_to_file(ctx.file, ctx.index as u64, response_data.to_vec()); -} - -fn write_to_file(file: Arc>, index: u64, data: Vec) { - let mut lock = file.lock().unwrap(); - - if index != 0 { - lock.seek(SeekFrom::Start(index * CHUNK_SIZE)).expect("Failed to seek to file offset"); - } - - lock.write_all(&data).unwrap(); -} \ No newline at end of file diff --git a/desktop/src-tauri/src/downloads/download_logic.rs b/desktop/src-tauri/src/downloads/download_logic.rs new file mode 100644 index 00000000..bbd1c648 --- /dev/null +++ b/desktop/src-tauri/src/downloads/download_logic.rs @@ -0,0 +1,40 @@ +use crate::auth::generate_authorization_header; +use crate::db::DatabaseImpls; +use crate::downloads::manifest::DropDownloadContext; +use crate::DB; +use log::info; +use urlencoding::encode; +use std::io::{BufWriter, Seek, SeekFrom, Write}; + +pub fn download_game_chunk(ctx: DropDownloadContext) { + let base_url = DB.fetch_base_url(); + + let client = reqwest::blocking::Client::new(); + let chunk_url = base_url + .join(&format!( + "/api/v1/client/chunk?id={}&version={}&name={}&chunk={}", + // Encode the parts we don't trust + ctx.game_id, encode(&ctx.version), encode(&ctx.file_name), ctx.index + )) + .unwrap(); + + let header = generate_authorization_header(); + + let mut response = client + .get(chunk_url) + .header("Authorization", header) + .send() + .unwrap(); + + let mut file_lock = ctx.file.lock().unwrap(); + + if ctx.offset != 0 { + file_lock + .seek(SeekFrom::Start(ctx.offset)) + .expect("Failed to seek to file offset"); + } + + let mut stream = BufWriter::with_capacity(1024, file_lock.try_clone().unwrap()); + + response.copy_to(&mut stream).unwrap(); +} diff --git a/desktop/src-tauri/src/downloads/game_download.rs b/desktop/src-tauri/src/downloads/download_manager.rs similarity index 67% rename from desktop/src-tauri/src/downloads/game_download.rs rename to desktop/src-tauri/src/downloads/download_manager.rs index 331c34d0..be0be957 100644 --- a/desktop/src-tauri/src/downloads/game_download.rs +++ b/desktop/src-tauri/src/downloads/download_manager.rs @@ -1,19 +1,19 @@ -use std::fs::File; -use std::path::Path; -use std::sync::{Arc, Mutex}; -use std::sync::atomic::AtomicUsize; -use log::info; -use serde::{Deserialize, Serialize}; -use crate::{AppState, DB}; use crate::auth::generate_authorization_header; use crate::db::{DatabaseImpls, DATA_ROOT_DIR}; -use crate::downloads::download_files; +use crate::downloads::download_logic; use crate::downloads::manifest::{DropDownloadContext, DropManifest}; use crate::downloads::progress::ProgressChecker; +use crate::{AppState, DB}; +use log::info; +use serde::{Deserialize, Serialize}; +use std::fs::{create_dir_all, File}; +use std::path::Path; +use std::sync::atomic::AtomicUsize; +use std::sync::{Arc, Mutex}; #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct GameDownload { +pub struct GameDownloadManager { id: String, version: String, progress: Arc, @@ -40,14 +40,14 @@ pub enum GameDownloadError { } #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Debug)] pub enum SystemError { - MutexLockFailed + MutexLockFailed, } #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Ord, PartialOrd)] #[serde(rename_all = "camelCase")] pub struct GameChunkCtx { chunk_id: usize, } -impl GameDownload { +impl GameDownloadManager { pub fn new(id: String, version: String) -> Self { Self { id, @@ -64,11 +64,16 @@ impl GameDownload { } self.ensure_manifest_exists().await } - pub async fn download(&self, max_threads: usize, contexts: Vec) -> Result<(), GameDownloadError> { + pub fn begin_download( + &self, + max_threads: usize, + contexts: Vec, + ) -> Result<(), GameDownloadError> { let progress = Arc::new(AtomicUsize::new(0)); self.change_state(GameDownloadState::Downloading); - let progress = ProgressChecker::new(Box::new(download_files::download_game_chunk), progress); - progress.run_contexts_parallel_async(contexts, max_threads).await; + let progress = + ProgressChecker::new(Box::new(download_logic::download_game_chunk), progress); + progress.run_contexts_parallel(contexts, max_threads); Ok(()) } pub async fn ensure_manifest_exists(&self) -> Result<(), GameDownloadError> { @@ -85,10 +90,9 @@ impl GameDownload { .join( format!( "/api/v1/client/metadata/manifest?id={}&version={}", - self.id, - self.version + self.id, self.version ) - .as_str() + .as_str(), ) .unwrap(); @@ -109,10 +113,11 @@ impl GameDownload { } let manifest_download = response.json::().await.unwrap(); - info!("Manifest: {:?}", manifest_download); if let Ok(mut manifest) = self.manifest.lock() { *manifest = Some(manifest_download) - } else { return Err(GameDownloadError::System(SystemError::MutexLockFailed)); } + } else { + return Err(GameDownloadError::System(SystemError::MutexLockFailed)); + } Ok(()) } @@ -122,26 +127,38 @@ impl GameDownload { *lock = state; } } -pub fn to_contexts(manifest: &DropManifest, version: String, game_id: String) -> Vec { +pub fn generate_job_contexts( + manifest: &DropManifest, + version: String, + game_id: String, +) -> Vec { let mut contexts = Vec::new(); - let base_path = DATA_ROOT_DIR.clone(); - for key in manifest { - let path = base_path.join(Path::new(key.0)); - let file = Arc::new(Mutex::new(File::create(path).unwrap())); - for i in 0..key.1.ids.len() { - contexts.push(DropDownloadContext { - file_chunk: Arc::new(key.1.clone()), + let base_path = DATA_ROOT_DIR.join("games").join(game_id.clone()).clone(); + create_dir_all(base_path.clone()).unwrap(); + for (raw_path, chunk) in manifest { + let path = base_path.join(Path::new(raw_path)); - file_name: key.0.clone(), + let container = path.parent().unwrap(); + create_dir_all(container).unwrap(); + + let file = Arc::new(Mutex::new(File::create(path).unwrap())); + let mut running_offset = 0; + + for i in 0..chunk.ids.len() { + if i == 1 { + info!("woah a chunk bigger than 1") + } + contexts.push(DropDownloadContext { + file_name: raw_path.to_string(), version: version.to_string(), + offset: running_offset, index: i, game_id: game_id.to_string(), file: file.clone(), }); - + running_offset += chunk.lengths[i] as u64; } } - info!("Contexts: {:?}", contexts); contexts } @@ -154,40 +171,21 @@ pub async fn start_game_download( ) -> Result<(), GameDownloadError> { info!("Triggered Game Download"); - let download = Arc::new(GameDownload::new(game_id.clone(), game_version.clone())); + let download_manager = Arc::new(GameDownloadManager::new( + game_id.clone(), + game_version.clone(), + )); - download.ensure_manifest_exists().await?; + download_manager.ensure_manifest_exists().await?; let local_manifest = { - let manifest = download.manifest.lock().unwrap(); + let manifest = download_manager.manifest.lock().unwrap(); (*manifest).clone().unwrap() }; - let contexts = to_contexts(&local_manifest, game_version.clone(), game_id); + let contexts = generate_job_contexts(&local_manifest, game_version.clone(), game_id); - let _ = download.download(max_threads, contexts).await; + let _ = download_manager.begin_download(max_threads, contexts); Ok(()) - - /* - let Some(unlocked) = manifest else { return Err(GameDownloadError::ManifestDoesNotExist) }; - let lock = unlocked.lock().unwrap(); - - let chunks = lock.parse_to_chunks(); - - /* - let manifest = match d.manifest { - Some(lock) => { - let lock = lock.lock().unwrap(); - lock.parse_to_chunks() - }, - None => { return Err(GameDownloadError::ManifestDoesNotExist) } - }; - */ - - app_state.game_downloads.push(download.clone()); - download.download(max_threads, chunks).await - - */ } - diff --git a/desktop/src-tauri/src/downloads/downloads.rs b/desktop/src-tauri/src/downloads/downloads.rs deleted file mode 100644 index 8850ac6e..00000000 --- a/desktop/src-tauri/src/downloads/downloads.rs +++ /dev/null @@ -1,11 +0,0 @@ -/* GENERAL OUTLINE -When downloading any game, the following details must be provided to the server: - - Game ID - - User token - - TBC - -The steps to then download a game are as follows: - 1. User requests - */ - - diff --git a/desktop/src-tauri/src/downloads/manifest.rs b/desktop/src-tauri/src/downloads/manifest.rs index a646179a..7feb45cc 100644 --- a/desktop/src-tauri/src/downloads/manifest.rs +++ b/desktop/src-tauri/src/downloads/manifest.rs @@ -14,10 +14,10 @@ pub struct DropChunk { #[derive(Debug, Clone)] pub struct DropDownloadContext { - pub file_chunk: Arc, pub file_name: String, pub version: String, pub index: usize, + pub offset: u64, pub game_id: String, pub file: Arc> } \ No newline at end of file diff --git a/desktop/src-tauri/src/downloads/mod.rs b/desktop/src-tauri/src/downloads/mod.rs index 907902b2..a3129e7e 100644 --- a/desktop/src-tauri/src/downloads/mod.rs +++ b/desktop/src-tauri/src/downloads/mod.rs @@ -1,5 +1,4 @@ -mod downloads; mod manifest; pub mod progress; -pub mod game_download; -mod download_files; \ No newline at end of file +pub mod download_manager; +mod download_logic; \ No newline at end of file diff --git a/desktop/src-tauri/src/downloads/progress.rs b/desktop/src-tauri/src/downloads/progress.rs index e8176f97..6194a90b 100644 --- a/desktop/src-tauri/src/downloads/progress.rs +++ b/desktop/src-tauri/src/downloads/progress.rs @@ -30,19 +30,6 @@ where T: Send + Sync self.counter.fetch_add(1, Ordering::Relaxed); } } - pub async fn run_contexts_parallel_async(&self, contexts: Vec, max_threads: usize) { - let threads = ThreadPoolBuilder::new() - // If max_threads == 0, then the limit will be determined - // by Rayon's internal RAYON_NUM_THREADS - .num_threads(max_threads) - .build() - .unwrap(); - - for context in contexts { - let f = self.f.clone(); - threads.spawn(move || f(context)); - } - } pub fn run_contexts_parallel(&self, contexts: Vec, max_threads: usize) { let threads = ThreadPoolBuilder::new() // If max_threads == 0, then the limit will be determined diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index b4043c96..a793c0f6 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -22,7 +22,7 @@ use std::{ use std::sync::Arc; use tauri_plugin_deep_link::DeepLinkExt; use crate::db::DatabaseImpls; -use crate::downloads::game_download::{start_game_download, GameDownload}; +use crate::downloads::download_manager::{start_game_download, GameDownloadManager}; #[derive(Clone, Copy, Serialize)] pub enum AppStatus { @@ -47,7 +47,7 @@ pub struct AppState { status: AppStatus, user: Option, games: HashMap, - game_downloads: Vec> + game_downloads: Vec> } #[tauri::command] From 6971a8d29e376e9c0f6945af8a2628af23b4ce53 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 24 Oct 2024 22:17:50 +1100 Subject: [PATCH 20/37] better download defaults --- desktop/src-tauri/src/downloads/download_logic.rs | 3 ++- desktop/src-tauri/src/downloads/download_manager.rs | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/desktop/src-tauri/src/downloads/download_logic.rs b/desktop/src-tauri/src/downloads/download_logic.rs index bbd1c648..fa64f1c1 100644 --- a/desktop/src-tauri/src/downloads/download_logic.rs +++ b/desktop/src-tauri/src/downloads/download_logic.rs @@ -34,7 +34,8 @@ pub fn download_game_chunk(ctx: DropDownloadContext) { .expect("Failed to seek to file offset"); } - let mut stream = BufWriter::with_capacity(1024, file_lock.try_clone().unwrap()); + let mut stream = BufWriter::with_capacity(1024 * 1024, file_lock.try_clone().unwrap()); + drop(file_lock); response.copy_to(&mut stream).unwrap(); } diff --git a/desktop/src-tauri/src/downloads/download_manager.rs b/desktop/src-tauri/src/downloads/download_manager.rs index be0be957..50af71df 100644 --- a/desktop/src-tauri/src/downloads/download_manager.rs +++ b/desktop/src-tauri/src/downloads/download_manager.rs @@ -64,6 +64,7 @@ impl GameDownloadManager { } self.ensure_manifest_exists().await } + pub fn begin_download( &self, max_threads: usize, @@ -76,6 +77,7 @@ impl GameDownloadManager { progress.run_contexts_parallel(contexts, max_threads); Ok(()) } + pub async fn ensure_manifest_exists(&self) -> Result<(), GameDownloadError> { if self.manifest.lock().unwrap().is_some() { return Ok(()); @@ -145,9 +147,6 @@ pub fn generate_job_contexts( let mut running_offset = 0; for i in 0..chunk.ids.len() { - if i == 1 { - info!("woah a chunk bigger than 1") - } contexts.push(DropDownloadContext { file_name: raw_path.to_string(), version: version.to_string(), From 45c22a40e1554a2de5994117b8a3d4abc89ee832 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Fri, 25 Oct 2024 10:28:58 +1100 Subject: [PATCH 21/37] fixed multi-chunk downloads --- desktop/src-tauri/Cargo.lock | 1 + desktop/src-tauri/Cargo.toml | 1 + desktop/src-tauri/src/downloads/download_logic.rs | 14 ++++++++------ .../src-tauri/src/downloads/download_manager.rs | 13 +++++++++---- desktop/src-tauri/src/downloads/manifest.rs | 3 ++- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 622e924d..8e498add 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1028,6 +1028,7 @@ dependencies = [ "rayon", "reqwest", "rustbreak", + "rustix", "serde", "serde_json", "structured-logger", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index edbcbbb5..21ada3fd 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -40,6 +40,7 @@ http = "1.1.0" tokio = { version = "1.40.0", features = ["rt", "tokio-macros"] } versions = { version = "6.3.2", features = ["serde"] } urlencoding = "2.1.3" +rustix = "0.38.37" [dependencies.uuid] version = "1.10.0" diff --git a/desktop/src-tauri/src/downloads/download_logic.rs b/desktop/src-tauri/src/downloads/download_logic.rs index fa64f1c1..cf999bf4 100644 --- a/desktop/src-tauri/src/downloads/download_logic.rs +++ b/desktop/src-tauri/src/downloads/download_logic.rs @@ -3,8 +3,8 @@ use crate::db::DatabaseImpls; use crate::downloads::manifest::DropDownloadContext; use crate::DB; use log::info; +use std::{fs::OpenOptions, io::{BufWriter, Seek, SeekFrom, Write}}; use urlencoding::encode; -use std::io::{BufWriter, Seek, SeekFrom, Write}; pub fn download_game_chunk(ctx: DropDownloadContext) { let base_url = DB.fetch_base_url(); @@ -14,7 +14,10 @@ pub fn download_game_chunk(ctx: DropDownloadContext) { .join(&format!( "/api/v1/client/chunk?id={}&version={}&name={}&chunk={}", // Encode the parts we don't trust - ctx.game_id, encode(&ctx.version), encode(&ctx.file_name), ctx.index + ctx.game_id, + encode(&ctx.version), + encode(&ctx.file_name), + ctx.index )) .unwrap(); @@ -26,16 +29,15 @@ pub fn download_game_chunk(ctx: DropDownloadContext) { .send() .unwrap(); - let mut file_lock = ctx.file.lock().unwrap(); + let mut file = OpenOptions::new().write(true).open(ctx.path).unwrap(); if ctx.offset != 0 { - file_lock + file .seek(SeekFrom::Start(ctx.offset)) .expect("Failed to seek to file offset"); } - let mut stream = BufWriter::with_capacity(1024 * 1024, file_lock.try_clone().unwrap()); - drop(file_lock); + let mut stream = BufWriter::with_capacity(1024 * 1024, file); response.copy_to(&mut stream).unwrap(); } diff --git a/desktop/src-tauri/src/downloads/download_manager.rs b/desktop/src-tauri/src/downloads/download_manager.rs index 50af71df..0d0662ac 100644 --- a/desktop/src-tauri/src/downloads/download_manager.rs +++ b/desktop/src-tauri/src/downloads/download_manager.rs @@ -5,6 +5,7 @@ use crate::downloads::manifest::{DropDownloadContext, DropManifest}; use crate::downloads::progress::ProgressChecker; use crate::{AppState, DB}; use log::info; +use rustix::fs::{fallocate, FallocateFlags}; use serde::{Deserialize, Serialize}; use std::fs::{create_dir_all, File}; use std::path::Path; @@ -64,7 +65,7 @@ impl GameDownloadManager { } self.ensure_manifest_exists().await } - + pub fn begin_download( &self, max_threads: usize, @@ -143,7 +144,7 @@ pub fn generate_job_contexts( let container = path.parent().unwrap(); create_dir_all(container).unwrap(); - let file = Arc::new(Mutex::new(File::create(path).unwrap())); + let file = File::create(path.clone()).unwrap(); let mut running_offset = 0; for i in 0..chunk.ids.len() { @@ -153,10 +154,12 @@ pub fn generate_job_contexts( offset: running_offset, index: i, game_id: game_id.to_string(), - file: file.clone(), + path: path.clone(), }); running_offset += chunk.lengths[i] as u64; } + + fallocate(file, FallocateFlags::empty(), 0, running_offset); } contexts } @@ -184,7 +187,9 @@ pub async fn start_game_download( let contexts = generate_job_contexts(&local_manifest, game_version.clone(), game_id); - let _ = download_manager.begin_download(max_threads, contexts); + download_manager + .begin_download(max_threads, contexts) + .unwrap(); Ok(()) } diff --git a/desktop/src-tauri/src/downloads/manifest.rs b/desktop/src-tauri/src/downloads/manifest.rs index 7feb45cc..f03a7065 100644 --- a/desktop/src-tauri/src/downloads/manifest.rs +++ b/desktop/src-tauri/src/downloads/manifest.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; use std::fs::File; +use std::path::PathBuf; use std::sync::{Arc, Mutex}; use serde::{Deserialize, Serialize}; @@ -19,5 +20,5 @@ pub struct DropDownloadContext { pub index: usize, pub offset: u64, pub game_id: String, - pub file: Arc> + pub path: PathBuf } \ No newline at end of file From 638fbdfb58e7c47dfcc3ac92f59e1b1015021319 Mon Sep 17 00:00:00 2001 From: quexeky Date: Fri, 25 Oct 2024 14:56:49 +1100 Subject: [PATCH 22/37] copy direct to disk --- desktop/pages/store/index.vue | 5 +++-- desktop/src-tauri/src/downloads/download_logic.rs | 6 ++++-- desktop/src-tauri/src/downloads/download_manager.rs | 12 +++++++----- desktop/src-tauri/src/downloads/progress.rs | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/desktop/pages/store/index.vue b/desktop/pages/store/index.vue index 486b796d..bb2bedb9 100644 --- a/desktop/pages/store/index.vue +++ b/desktop/pages/store/index.vue @@ -6,7 +6,7 @@ Load Data - + + diff --git a/desktop/src-tauri/src/downloads/download_agent.rs b/desktop/src-tauri/src/downloads/download_agent.rs index b8fc0d51..093f1cc7 100644 --- a/desktop/src-tauri/src/downloads/download_agent.rs +++ b/desktop/src-tauri/src/downloads/download_agent.rs @@ -68,30 +68,30 @@ impl GameDownloadAgent { if self.manifest.lock().unwrap().is_none() { return Ok(()); } - self.ensure_manifest_exists().await + self.ensure_manifest_exists() } - pub async fn begin_download(&self, max_threads: usize) -> Result<(), GameDownloadError> { + pub fn begin_download(&self, max_threads: usize) -> Result<(), GameDownloadError> { self.change_state(GameDownloadState::Downloading); // TODO we're coping the whole context thing // It's not necessary, I just can't figure out to make the borrow checker happy { let lock = self.contexts.lock().unwrap().to_vec(); self.progress - .run_context_parallel(lock, max_threads).await; + .run_context_parallel(lock, max_threads); } Ok(()) } - pub async fn ensure_manifest_exists(&self) -> Result<(), GameDownloadError> { + pub fn ensure_manifest_exists(&self) -> Result<(), GameDownloadError> { if self.manifest.lock().unwrap().is_some() { return Ok(()); } - self.download_manifest().await + self.download_manifest() } - async fn download_manifest(&self) -> Result<(), GameDownloadError> { + fn download_manifest(&self) -> Result<(), GameDownloadError> { let base_url = DB.fetch_base_url(); let manifest_url = base_url .join( @@ -106,12 +106,11 @@ impl GameDownloadAgent { let header = generate_authorization_header(); info!("Generating & sending client"); - let client = reqwest::Client::new(); + let client = reqwest::blocking::Client::new(); let response = client .get(manifest_url.to_string()) .header("Authorization", header) .send() - .await .unwrap(); if response.status() != 200 { @@ -119,7 +118,7 @@ impl GameDownloadAgent { return Err(GameDownloadError::Status(response.status().as_u16())); } - let manifest_download = response.json::().await.unwrap(); + let manifest_download = response.json::().unwrap(); if let Ok(mut manifest) = self.manifest.lock() { *manifest = Some(manifest_download) } else { diff --git a/desktop/src-tauri/src/downloads/download_commands.rs b/desktop/src-tauri/src/downloads/download_commands.rs index a4f0f014..550d8375 100644 --- a/desktop/src-tauri/src/downloads/download_commands.rs +++ b/desktop/src-tauri/src/downloads/download_commands.rs @@ -1,4 +1,4 @@ -use std::sync::{atomic::Ordering, Arc, Mutex}; +use std::{sync::{atomic::Ordering, Arc, Mutex}, thread}; use log::info; @@ -27,41 +27,46 @@ pub async fn start_game_downloads( state: tauri::State<'_, Mutex>, ) -> Result<(), GameDownloadError> { info!("Downloading Games"); - loop { - let mut current_id = String::new(); - let mut download_agent = None; - { - let lock = state.lock().unwrap(); - for (id, agent) in &lock.game_downloads { - if agent.get_state() == GameDownloadState::Queued { - download_agent = Some(agent.clone()); - current_id = id.clone(); - info!("Got queued game to download"); - break; + let lock = state.lock().unwrap(); + let mut game_downloads = lock.game_downloads.clone(); + drop(lock); + thread::spawn(move || { + loop { + let mut current_id = String::new(); + let mut download_agent = None; + { + for (id, agent) in &game_downloads { + if agent.get_state() == GameDownloadState::Queued { + download_agent = Some(agent.clone()); + current_id = id.clone(); + info!("Got queued game to download"); + break; + } } + if download_agent.is_none() { + info!("No more games left to download"); + return; + } + }; + info!("Downloading game"); + { + start_game_download(max_threads, download_agent.unwrap()).unwrap(); + game_downloads.remove_entry(¤t_id); } - if download_agent.is_none() { - info!("No more games left to download"); - return Ok(()) - } - }; - info!("Downloading game"); - { - start_game_download(max_threads, download_agent.unwrap()).await?; - let mut lock = state.lock().unwrap(); - lock.game_downloads.remove_entry(¤t_id); - } - } + } + }); + info!("Spawned download"); + return Ok(()) } -pub async fn start_game_download( +pub fn start_game_download( max_threads: usize, download_agent: Arc ) -> Result<(), GameDownloadError> { info!("Triggered Game Download"); - download_agent.ensure_manifest_exists().await?; + download_agent.ensure_manifest_exists()?; let local_manifest = { let manifest = download_agent.manifest.lock().unwrap(); @@ -70,19 +75,21 @@ pub async fn start_game_download( download_agent.generate_job_contexts(&local_manifest, download_agent.version.clone(), download_agent.id.clone()).unwrap(); - download_agent.begin_download(max_threads).await?; + download_agent.begin_download(max_threads).unwrap(); Ok(()) } #[tauri::command] pub async fn stop_specific_game_download(state: tauri::State<'_, Mutex>, game_id: String) -> Result<(), String> { + info!("called stop_specific_game_download"); let lock = state.lock().unwrap(); let download_agent = lock.game_downloads.get(&game_id).unwrap(); let callback = download_agent.callback.clone(); drop(lock); + info!("Stopping callback"); callback.store(true, Ordering::Release); return Ok(()) diff --git a/desktop/src-tauri/src/downloads/download_logic.rs b/desktop/src-tauri/src/downloads/download_logic.rs index 8825c684..d4327803 100644 --- a/desktop/src-tauri/src/downloads/download_logic.rs +++ b/desktop/src-tauri/src/downloads/download_logic.rs @@ -5,7 +5,8 @@ use crate::DB; use gxhash::{gxhash128, GxHasher}; use log::info; use md5::{Context, Digest}; -use std::{fs::{File, OpenOptions}, hash::Hasher, io::{self, BufWriter, Error, ErrorKind, Seek, SeekFrom, Write}, path::PathBuf, sync::{atomic::{AtomicBool, Ordering}, Arc}}; +use reqwest::blocking::Response; +use std::{fs::{File, OpenOptions}, hash::Hasher, io::{self, BufReader, BufWriter, Error, ErrorKind, Read, Seek, SeekFrom, Write}, path::PathBuf, sync::{atomic::{AtomicBool, Ordering}, Arc}}; use urlencoding::encode; pub struct DropFileWriter { @@ -30,8 +31,10 @@ impl DropFileWriter { impl Write for DropFileWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { if self.callback.load(Ordering::Acquire) { - return Err(Error::new(ErrorKind::Interrupted, "Interrupt command recieved")); + return Err(Error::new(ErrorKind::ConnectionAborted, "Interrupt command recieved")); } + + //info!("Writing data to writer"); self.hasher.write_all(buf).unwrap(); self.file.write(buf) } @@ -48,6 +51,7 @@ impl Seek for DropFileWriter { } pub fn download_game_chunk(ctx: DropDownloadContext, callback: Arc) { if callback.load(Ordering::Acquire) { + info!("Callback stopped download at start"); return; } let base_url = DB.fetch_base_url(); @@ -83,16 +87,14 @@ pub fn download_game_chunk(ctx: DropDownloadContext, callback: Arc) // Writing everything to disk directly is probably slightly faster because it balances out the writes, // but this is better than the performance loss from constantly reading the callbacks - let mut writer = BufWriter::with_capacity(1024 * 1024, file); + //let mut writer = BufWriter::with_capacity(1024 * 1024, file); - match response.copy_to(&mut writer) { + //copy_to_drop_file_writer(&mut response, &mut file); + match io::copy(&mut response, &mut file) { Ok(_) => {}, - Err(_) => { println!("Stopped printing chunk {}", ctx.file_name); return; } - }; - let file = match writer.into_inner() { - Ok(inner) => inner, - Err(_) => panic!("Failed to get BufWriter inner"), - }; + Err(e) => { info!("Copy errored with error {}", e)}, + } + let res = hex::encode(file.finish().unwrap().0); if res != ctx.checksum { info!("Checksum failed. Original: {}, Calculated: {} for {}", ctx.checksum, res, ctx.file_name); @@ -100,3 +102,23 @@ pub fn download_game_chunk(ctx: DropDownloadContext, callback: Arc) // stream.flush().unwrap(); } + +pub fn copy_to_drop_file_writer(response: &mut Response, writer: &mut DropFileWriter) { + loop { + info!("Writing to file writer"); + let mut buf = [0u8; 1024]; + response.read(&mut buf).unwrap(); + match writer.write_all(&buf) { + Ok(_) => {}, + Err(e) => { + match e.kind() { + ErrorKind::Interrupted => { + info!("Interrupted"); + return; + } + _ => { println!("{}", e); return;} + } + }, + } + } +} \ No newline at end of file diff --git a/desktop/src-tauri/src/downloads/progress.rs b/desktop/src-tauri/src/downloads/progress.rs index 78d56ed6..671ba34e 100644 --- a/desktop/src-tauri/src/downloads/progress.rs +++ b/desktop/src-tauri/src/downloads/progress.rs @@ -1,6 +1,5 @@ +use log::info; use rayon::ThreadPoolBuilder; -use uuid::timestamp::context; -use std::os::unix::thread; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; @@ -28,12 +27,6 @@ where callback } } - pub async fn run_contexts_sequentially_async(&self, contexts: Vec) { - for context in contexts { - (self.f)(context, self.callback.clone()); - self.counter.fetch_add(1, Ordering::Release); - } - } pub fn run_contexts_sequentially(&self, contexts: Vec) { for context in contexts { (self.f)(context, self.callback.clone()); @@ -54,7 +47,7 @@ where threads.spawn(move || f(context, callback)); } } - pub async fn run_context_parallel(&self, contexts: Vec, max_threads: usize) { + pub fn run_context_parallel(&self, contexts: Vec, max_threads: usize) { let threads = ThreadPoolBuilder::new() .num_threads(max_threads) .build() @@ -64,9 +57,10 @@ where for context in contexts { let callback = self.callback.clone(); let f = self.f.clone(); - s.spawn(move |_| f(context, callback)); + s.spawn(move |_| {info!("Running thread"); f(context, callback)}); } }); + info!("Concluded scope"); } pub fn get_progress(&self) -> usize { From b4a5e501a8a65c90f55673c13a2b241a328ca534 Mon Sep 17 00:00:00 2001 From: quexeky Date: Thu, 31 Oct 2024 23:06:14 +1100 Subject: [PATCH 36/37] Removed unpacker.rs Signed-off-by: quexeky --- desktop/src-tauri/src/unpacker.rs | 71 ------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 desktop/src-tauri/src/unpacker.rs diff --git a/desktop/src-tauri/src/unpacker.rs b/desktop/src-tauri/src/unpacker.rs deleted file mode 100644 index 1d66a868..00000000 --- a/desktop/src-tauri/src/unpacker.rs +++ /dev/null @@ -1,71 +0,0 @@ -use ciborium::from_reader; -use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use serde::Deserialize; -use std::{ - collections::HashMap, - fs::{create_dir_all, File}, - io::{self, BufReader, Error, Seek, Write}, - path::Path, -}; - -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt; - -#[derive(Deserialize)] -#[serde(rename_all="camelCase")] -struct ManifestChunk { - uuid: String, - index: i64, -} - -#[derive(Deserialize)] -#[serde(rename_all="camelCase")] -struct ManifestRecord { - chunks: Vec, - permissions: u32, -} - -#[derive(Deserialize)] -#[serde(rename_all="camelCase")] -struct Manifest { - record: HashMap, -} - -pub async fn unpack() -> Result<(), Error> { - let chunk_size: u64 = 1024 * 1024 * 16; - - let input = Path::new("/home/decduck/Dev/droplet-output"); - let output = Path::new("/home/decduck/Dev/droplet-rebuilt"); - - let manifest_path = input.join("manifest.drop"); - let manifest_file_handle = File::open(manifest_path).unwrap(); - - let manifest: Manifest = from_reader(manifest_file_handle).unwrap(); - manifest.record.into_par_iter().for_each(|(key, value)| { - let file = output.join(key.clone()); - create_dir_all(file.parent().unwrap()).unwrap(); - let mut file_handle = File::create(file).unwrap(); - - #[cfg(unix)] - { - let mut file_permissions = file_handle.metadata().unwrap().permissions(); - file_permissions.set_mode(value.permissions); - file_handle.set_permissions(file_permissions).unwrap(); - } - - for chunk in value.chunks { - let chunk_path = input.join(chunk.uuid + ".bin"); - let chunk_handle = File::open(chunk_path).unwrap(); - - let mut chunk_reader = BufReader::new(chunk_handle); - - let offset = u64::try_from(chunk.index).unwrap() * chunk_size; - file_handle.seek(io::SeekFrom::Start(offset)).unwrap(); - - io::copy(&mut chunk_reader, &mut file_handle).unwrap(); - file_handle.flush().unwrap(); - } - }); - - Ok(()) -} From f3df6ee6877aa71e7db244abcd5b981376af1fb5 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 31 Oct 2024 23:42:16 +1100 Subject: [PATCH 37/37] remove unpacker mod statement --- desktop/src-tauri/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index d9d78b69..44198c13 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -2,7 +2,6 @@ mod auth; mod db; mod library; mod remote; -mod unpacker; mod downloads; #[cfg(test)] mod tests;