jessikitty/drop
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

In-app store, delta version support (#179)

Browse Source 
* fix: windows launch

* feat: add necessary client fixes for store

* fix: keyring fix

* feat: delta version support

* feat: dl/disk progress

* feat: move to jwt auth

* fix: lint
This commit is contained in:
DecDuck
2026-02-06 00:30:27 +11:00
committed by GitHub
parent fc69ae30ab
commit 1f74d35bdc
23 changed files with 808 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
desktop/src-tauri/games/src/downloads/download_agent.rs
+159 -86
View File
@@ -9,13 +9,15 @@ use download_manager::error::ApplicationDownloadError;
use download_manager::util::download_thread_control_flag::{
DownloadThreadControl, DownloadThreadControlFlag,
};
use download_manager::util::progress_object::{ProgressHandle, ProgressObject};
use droplet_rs::manifest::Manifest;
use download_manager::util::progress_object::{ProgressHandle, ProgressObject, ProgressType};
use droplet_rs::manifest::{ChunkData, Manifest};
use log::{debug, error, info, warn};
use remote::auth::generate_authorization_header;
use remote::error::RemoteAccessError;
use remote::requests::generate_url;
use remote::utils::DROP_CLIENT_ASYNC;
use serde::Deserialize;
use std::collections::HashMap;
use std::fmt::Debug;
use std::mem;
use std::path::{Path, PathBuf};
@@ -34,11 +36,21 @@ use super::drop_data::DropData;
static RETRY_COUNT: usize = 3;
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DownloadInformation {
file_list: HashMap<String, String>,
manifests: HashMap<String, Manifest>,
install_size: u64,
download_size: u64,
}
pub struct GameDownloadAgent {
pub metadata: DownloadableMetadata,
pub control_flag: DownloadThreadControl,
pub manifest: Mutex<Option<Manifest>>,
pub progress: Arc<ProgressObject>,
pub dl_info: Mutex<Option<DownloadInformation>>,
pub download_progress: Arc<ProgressObject>,
pub disk_progress: Arc<ProgressObject>,
depot_manager: Arc<DepotManager>,
sender: Sender<DownloadManagerSignal>,
pub dropdata: DropData,
@@ -86,12 +98,23 @@ impl GameDownloadAgent {
metadata.target_platform,
data_base_dir_path.clone(),
);
let result = Self {
metadata,
control_flag,
manifest: Mutex::new(None),
progress: Arc::new(ProgressObject::new(0, 0, sender.clone())),
dl_info: Mutex::new(None),
download_progress: Arc::new(ProgressObject::new(
0,
0,
sender.clone(),
ProgressType::Download,
)),
disk_progress: Arc::new(ProgressObject::new(
0,
0,
sender.clone(),
ProgressType::Disk,
)),
sender,
dropdata: stored_manifest,
status: Mutex::new(DownloadStatus::Queued),
@@ -100,7 +123,7 @@ impl GameDownloadAgent {
result.ensure_manifest_exists().await?;
let required_space = lock!(result.manifest).as_ref().unwrap().size;
let required_space = lock!(result.dl_info).as_ref().unwrap().install_size;
let available_space = get_disk_available(data_base_dir_path)? as u64;
@@ -157,11 +180,11 @@ impl GameDownloadAgent {
}
pub fn check_manifest_exists(&self) -> bool {
lock!(self.manifest).is_some()
lock!(self.dl_info).is_some()
}
pub async fn ensure_manifest_exists(&self) -> Result<(), ApplicationDownloadError> {
if lock!(self.manifest).is_some() {
if lock!(self.dl_info).is_some() {
return Ok(());
}
@@ -195,12 +218,12 @@ impl GameDownloadAgent {
));
}
let manifest_download: Manifest = response
let manifest_download: DownloadInformation = response
.json()
.await
.map_err(|e| ApplicationDownloadError::Communication(e.into()))?;
if let Ok(mut manifest) = self.manifest.lock() {
if let Ok(mut manifest) = self.dl_info.lock() {
*manifest = Some(manifest_download);
return Ok(());
}
@@ -208,34 +231,62 @@ impl GameDownloadAgent {
Err(ApplicationDownloadError::Lock)
}
// Sets it up for both download and validate
// Sets up progress for download writes
fn setup_progress(&self) {
let manifest = lock!(self.manifest);
let manifest = manifest.as_ref().unwrap();
let dl_info = lock!(self.dl_info);
let dl_info = dl_info.as_ref().unwrap();
self.progress.set_max(manifest.size.try_into().unwrap());
self.progress.set_size(manifest.chunks.len());
self.progress.reset();
let total_chunks = dl_info
.manifests
.iter()
.map(|v| v.1.chunks.len())
.sum::<usize>();
self.download_progress
.set_max(dl_info.download_size.try_into().unwrap());
self.download_progress
.set_size(total_chunks);
self.download_progress.reset();
self.disk_progress.set_max(dl_info.install_size.try_into().unwrap());
self.disk_progress
.set_size(total_chunks);
self.disk_progress.reset();
}
async fn run(&self) -> Result<bool, RemoteAccessError> {
self.depot_manager.sync_depots().await?;
info!("synced depots");
self.setup_progress();
let (chunks, key) = {
let manifest = lock!(self.manifest);
let manifest = manifest.as_ref().unwrap();
(manifest.chunks.clone(), manifest.key)
info!("setup progress objects");
let manifests_chunks: Vec<(String, HashMap<String, ChunkData>, [u8; 16])> = {
let dl_info = lock!(self.dl_info);
dl_info
.as_ref()
.unwrap()
.manifests
.iter()
.map(|v| (v.0.clone(), v.1.chunks.clone(), v.1.key))
.collect()
};
let file_list = {
let dl_info = lock!(self.dl_info);
dl_info.as_ref().unwrap().file_list.clone()
};
let chunk_len = chunks.len();
let mut completed_chunks = {
let completed_chunks = lock!(self.dropdata.contexts);
completed_chunks.clone()
};
let chunk_len = manifests_chunks.iter().map(|v| v.1.len()).sum::<usize>();
let max_download_threads = borrow_db_checked().settings.max_download_threads;
let (sender, recv) = crossbeam_channel::bounded(16);
// SAFETY: I pinky-promise
// (the scope keeps these in scope)
let unsafe_self: &'static GameDownloadAgent = unsafe { mem::transmute(self) };
let file_list: &'static HashMap<String, String> = unsafe { mem::transmute(&file_list) };
let local_completed_chunks = completed_chunks.clone();
let download_join_handle = tauri::async_runtime::spawn_blocking(move || {
@@ -244,77 +295,95 @@ impl GameDownloadAgent {
.build()
.unwrap();
thread_pool.scope(move |s| {
for (index, (chunk_id, chunk_data)) in chunks.into_iter().enumerate() {
let local_sender = sender.clone();
let progress = unsafe_self.progress.get(index);
let progress_handle =
ProgressHandle::new(progress, unsafe_self.progress.clone());
let mut index = 0;
for (version_id, chunks, key) in manifests_chunks.into_iter() {
let version_id = &version_id;
for (chunk_id, chunk_data) in chunks.into_iter() {
let local_sender = sender.clone();
let download_progress_handle = ProgressHandle::new(
unsafe_self.download_progress.get(index),
unsafe_self.download_progress.clone(),
);
let disk_progress_handle = ProgressHandle::new(
unsafe_self.disk_progress.get(index),
unsafe_self.disk_progress.clone(),
);
index += 1;
let chunk_length = chunk_data.files.iter().map(|v| v.length).sum();
let chunk_length = chunk_data.files.iter().map(|v| v.length).sum();
if *local_completed_chunks.get(&chunk_id).unwrap_or(&false) {
progress_handle.skip(chunk_length);
continue;
}
let sender = unsafe_self.sender.clone();
let (depot, permit) = match unsafe_self
.depot_manager
.next_depot(&unsafe_self.metadata.id, &unsafe_self.metadata.version)
{
Ok(v) => v,
Err(err) => {
tauri::async_runtime::spawn(async move {
send!(sender, DownloadManagerSignal::Error(ApplicationDownloadError::Communication(err)));
});
return;
if *local_completed_chunks.get(&chunk_id).unwrap_or(&false) {
download_progress_handle.skip(chunk_length);
continue;
}
};
s.spawn(move |_| {
for i in 0..RETRY_COUNT {
let loop_progress_handle = progress_handle.clone();
let base_path = unsafe_self.dropdata.base_path.clone();
match download_game_chunk(
&unsafe_self.metadata.id,
&unsafe_self.metadata.version,
&chunk_id,
&depot,
&key,
&chunk_data,
base_path,
&unsafe_self.control_flag,
loop_progress_handle,
) {
Ok(true) => {
local_sender.send(chunk_id.clone()).unwrap();
drop(permit); // Take ownership
return;
}
Ok(false) => return,
Err(e) => {
warn!("got error for chunk id {}: {e:?}", chunk_id);
let sender = unsafe_self.sender.clone();
let (depot, permit) = match unsafe_self
.depot_manager
.next_depot(&unsafe_self.metadata.id, &unsafe_self.metadata.version)
{
Ok(v) => v,
Err(err) => {
tauri::async_runtime::spawn(async move {
send!(
sender,
DownloadManagerSignal::Error(
ApplicationDownloadError::Communication(err)
)
);
});
return;
}
};
let retry = true; /*matches!(
&e,
ApplicationDownloadError::Communication(_)
| ApplicationDownloadError::Checksum
| ApplicationDownloadError::Lock
| ApplicationDownloadError::IoError(_)
);*/
if i == RETRY_COUNT - 1 || !retry {
warn!("retry logic failed, not re-attempting.");
tauri::async_runtime::spawn(async move {
send!(sender, DownloadManagerSignal::Error(e));
});
let local_version_id = version_id.clone();
s.spawn(move |_| {
for i in 0..RETRY_COUNT {
let base_path = unsafe_self.dropdata.base_path.clone();
match download_game_chunk(
&unsafe_self.metadata.id,
&local_version_id,
&chunk_id,
&depot,
&key,
&chunk_data,
file_list,
base_path,
&unsafe_self.control_flag,
&download_progress_handle,
&disk_progress_handle,
) {
Ok(true) => {
local_sender.send(chunk_id.clone()).unwrap();
drop(permit); // Take ownership
return;
}
Ok(false) => return,
Err(e) => {
warn!("got error for chunk id {}: {e:?}", chunk_id);
let retry = true; /*matches!(
&e,
ApplicationDownloadError::Communication(_)
| ApplicationDownloadError::Checksum
| ApplicationDownloadError::Lock
| ApplicationDownloadError::IoError(_)
);*/
if i == RETRY_COUNT - 1 || !retry {
warn!("retry logic failed, not re-attempting.");
tauri::async_runtime::spawn(async move {
send!(sender, DownloadManagerSignal::Error(e));
});
return;
}
}
}
}
}
});
});
}
}
drop(sender);
});
});
@@ -457,8 +526,12 @@ impl Downloadable for GameDownloadAgent {
self.validate(app_handle)
}
fn progress(&self) -> Arc<ProgressObject> {
self.progress.clone()
fn dl_progress(&self) -> &Arc<ProgressObject> {
&self.download_progress
}
fn disk_progress(&self) -> &Arc<ProgressObject> {
&self.disk_progress
}
fn control_flag(&self) -> DownloadThreadControl {
desktop/src-tauri/games/src/downloads/download_logic.rs
+35 -19
View File
@@ -1,3 +1,4 @@
use std::collections::HashMap;
use std::fs::{Permissions, set_permissions};
use std::io::{Read, Seek as _, SeekFrom, Write as _};
#[cfg(unix)]
@@ -32,13 +33,18 @@ pub fn download_game_chunk(
depot: &str,
key: &[u8; 16],
chunk_data: &ChunkData,
file_list: &HashMap<String, String>,
base_path: PathBuf,
control_flag: &DownloadThreadControl,
progress: ProgressHandle,
// How much we're downloading
download_progress: &ProgressHandle,
// How much we're writing to disk
disk_progress: &ProgressHandle,
) -> Result<bool, ApplicationDownloadError> {
// If we're paused
if control_flag.get() == DownloadThreadControlFlag::Stop {
progress.set(0);
download_progress.set(0);
disk_progress.set(0);
return Ok(false);
}
@@ -48,10 +54,7 @@ pub fn download_game_chunk(
let url = Url::parse(depot)
.map_err(|v| ApplicationDownloadError::DownloadError(v.into()))?
.join(&format!(
"content/{}/{}/{}",
game_id, version_id, chunk_id
))
.join(&format!("content/{}/{}/{}", game_id, version_id, chunk_id))
.map_err(|v| ApplicationDownloadError::DownloadError(v.into()))?;
let response = DROP_CLIENT_SYNC
@@ -77,7 +80,8 @@ pub fn download_game_chunk(
}
if control_flag.get() == DownloadThreadControlFlag::Stop {
progress.set(0);
download_progress.set(0);
disk_progress.set(0);
return Ok(false);
}
@@ -95,27 +99,39 @@ pub fn download_game_chunk(
let mut cipher = Aes128Ctr64LE::new(key.into(), &chunk_data.iv.into());
let mut read_buf = vec![0u8; READ_BUF_LEN];
for file in &chunk_data.files {
let should_write = file_list
.get(&file.filename)
.map(|v| v == version_id)
.unwrap_or(false);
let path = base_path.join(file.filename.clone());
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let mut file_handle = std::fs::OpenOptions::new()
.truncate(false)
.write(true)
.append(false)
.create(true)
.open(&path)?;
file_handle.seek(SeekFrom::Start(file.start.try_into().unwrap()))?;
let mut file_handle = if should_write {
let mut file_handle = std::fs::OpenOptions::new()
.truncate(false)
.write(true)
.append(false)
.create(true)
.open(&path)?;
file_handle.seek(SeekFrom::Start(file.start.try_into().unwrap()))?;
Some(file_handle)
} else {
None
};
let mut remaining = file.length;
while remaining > 0 {
let amount = stream_reader.read(&mut read_buf[0..remaining.min(READ_BUF_LEN)])?;
progress.add(amount);
download_progress.add(amount);
remaining -= amount;
cipher.apply_keystream(&mut read_buf[0..amount]);
hasher.update(&read_buf[0..amount]);
file_handle.write_all(&read_buf[0..amount])?;
//hasher.update(&read_buf[0..amount]);
if let Some(file_handle) = &mut file_handle {
file_handle.write_all(&read_buf[0..amount])?;
disk_progress.add(amount);
}
}
#[cfg(unix)]
@@ -132,14 +148,14 @@ pub fn download_game_chunk(
}
if control_flag.get() == DownloadThreadControlFlag::Stop {
progress.set(0);
download_progress.set(0);
return Ok(false);
}
}
let digest = hex::encode(hasher.finalize());
if digest != chunk_data.checksum {
return Err(ApplicationDownloadError::Checksum);
//return Err(ApplicationDownloadError::Checksum);
}
Ok(true)
desktop/src-tauri/games/src/downloads/manifest.rs
-46
View File
@@ -1,46 +0,0 @@
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
#[derive(Debug, Clone, Serialize)]
// Drops go in buckets
pub struct DownloadDrop {
pub index: usize,
pub filename: String,
pub path: PathBuf,
pub start: usize,
pub length: usize,
pub checksum: String,
pub permissions: u32,
}
#[derive(Debug, Clone, Serialize)]
pub struct DownloadBucket {
pub game_id: String,
pub version: String,
pub drops: Vec<DownloadDrop>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct DropValidateContext {
pub index: usize,
pub offset: usize,
pub path: PathBuf,
pub checksum: String,
pub length: usize,
}
impl From<DownloadBucket> for Vec<DropValidateContext> {
fn from(value: DownloadBucket) -> Self {
value
.drops
.into_iter()
.map(|e| DropValidateContext {
index: e.index,
offset: e.start,
path: e.path,
checksum: e.checksum,
length: e.length,
})
.collect()
}
}
desktop/src-tauri/games/src/downloads/mod.rs
-1
View File
@@ -2,5 +2,4 @@ pub mod download_agent;
mod download_logic;
pub mod drop_data;
pub mod error;
mod manifest;
pub mod utils;
desktop/src-tauri/games/src/library.rs
+23 -26
View File
@@ -5,10 +5,8 @@ use database::{
};
use log::{debug, error, warn};
use remote::{
auth::generate_authorization_header,
error::RemoteAccessError,
requests::generate_url,
utils::DROP_CLIENT_ASYNC
auth::generate_authorization_header, error::RemoteAccessError, requests::generate_url,
utils::DROP_CLIENT_ASYNC,
};
use serde::{Deserialize, Serialize};
use std::fs::remove_dir_all;
@@ -160,29 +158,28 @@ pub fn uninstall_game_logic(meta: DownloadableMetadata, app_handle: &AppHandle)
spawn(move || {
if let Err(e) = remove_dir_all(install_dir) {
error!("{e}");
} else {
let mut db_handle = borrow_db_mut_checked();
db_handle.applications.transient_statuses.remove(&meta);
db_handle
.applications
.installed_game_version
.remove(&meta.id);
db_handle
.applications
.game_statuses
.insert(meta.id.clone(), GameDownloadStatus::Remote {});
let _ = db_handle.applications.transient_statuses.remove(&meta);
push_game_update(
&app_handle,
&meta.id,
None,
GameStatusManager::fetch_state(&meta.id, &db_handle),
);
debug!("uninstalled game id {}", &meta.id);
app_emit!(&app_handle, "update_library", ());
}
let mut db_handle = borrow_db_mut_checked();
db_handle.applications.transient_statuses.remove(&meta);
db_handle
.applications
.installed_game_version
.remove(&meta.id);
db_handle
.applications
.game_statuses
.insert(meta.id.clone(), GameDownloadStatus::Remote {});
let _ = db_handle.applications.transient_statuses.remove(&meta);
push_game_update(
&app_handle,
&meta.id,
None,
GameStatusManager::fetch_state(&meta.id, &db_handle),
);
debug!("uninstalled game id {}", &meta.id);
app_emit!(&app_handle, "update_library", ());
});
} else {
warn!("invalid previous state for uninstall, failing silently.");