From bd2e44d4f0af09a5ac5322954078e1bcfb256ff5 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 16:25:29 +1100 Subject: [PATCH 01/21] feat: initial commit --- torrential/.gitignore | 1 + torrential/Cargo.lock | 2121 ++++++++++++++++++++++++++++++++++++ torrential/Cargo.toml | 20 + torrential/README.md | 4 + torrential/src/download.rs | 52 + torrential/src/main.rs | 224 ++++ torrential/src/manifest.rs | 13 + torrential/src/remote.rs | 116 ++ torrential/src/util.rs | 37 + 9 files changed, 2588 insertions(+) create mode 100644 torrential/.gitignore create mode 100644 torrential/Cargo.lock create mode 100644 torrential/Cargo.toml create mode 100644 torrential/README.md create mode 100644 torrential/src/download.rs create mode 100644 torrential/src/main.rs create mode 100644 torrential/src/manifest.rs create mode 100644 torrential/src/remote.rs create mode 100644 torrential/src/util.rs diff --git a/torrential/.gitignore b/torrential/.gitignore new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/torrential/.gitignore @@ -0,0 +1 @@ +/target diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock new file mode 100644 index 00000000..a5095aa3 --- /dev/null +++ b/torrential/Cargo.lock @@ -0,0 +1,2121 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive 0.5.1", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +dependencies = [ + "asn1-rs-derive 0.6.0", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.17", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "cc" +version = "1.2.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "colored" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs 0.7.1", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "droplet-rs" +version = "0.9.2" +source = "git+https://github.com/Drop-OSS/droplet-rs.git#d8f37886d479d8d9fec7e51523628e863306dddb" +dependencies = [ + "anyhow", + "async-trait", + "dyn-clone", + "hex", + "rcgen", + "ring", + "time", + "time-macros", + "tokio", + "webpki", + "x509-parser 0.17.0", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs 0.6.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs 0.7.1", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "x509-parser 0.16.0", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reqwest" +version = "0.12.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +dependencies = [ + "libc", +] + +[[package]] +name = "simple_logger" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291bee647ce7310b0ea721bfd7e0525517b4468eb7c7e15eb8bd774343179702" +dependencies = [ + "colored", + "log", + "windows-sys 0.61.2", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl 2.0.17", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + +[[package]] +name = "time-macros" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "torrential" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "dashmap", + "droplet-rs", + "log", + "reqwest", + "serde", + "serde_json", + "simple_logger", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf146f99d442e8e68e585f5d798ccd3cad9a7835b917e09728880a862706456" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" +dependencies = [ + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.2", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.1", + "ring", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "x509-parser" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4569f339c0c402346d4a75a9e39cf8dad310e287eef1ff56d4c68e5067f53460" +dependencies = [ + "asn1-rs 0.7.1", + "data-encoding", + "der-parser 10.0.0", + "lazy_static", + "nom", + "oid-registry 0.8.1", + "ring", + "rusticata-macros", + "thiserror 2.0.17", + "time", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml new file mode 100644 index 00000000..0da9313b --- /dev/null +++ b/torrential/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "torrential" +version = "0.1.0" +edition = "2024" + +[dependencies] +axum = "0.8.7" +log = "0.4.28" +reqwest = { version = "0.12.24", features = ["json"] } +serde = { version = "1.0.228", features = ["derive"] } +simple_logger = { version = "5.1.0", default-features = false, features = [ + "colors", +] } +tokio = { version = "*", features = ["rt-multi-thread", "sync"] } +droplet-rs = { git = "https://github.com/Drop-OSS/droplet-rs.git" } +dashmap = "6.1.0" +anyhow = "1.0.100" +serde_json = "1.0.145" +url = { version = "2.5.7", default-features = false } +tokio-util = { version = "0.7.17", features = ["io"] } diff --git a/torrential/README.md b/torrential/README.md new file mode 100644 index 00000000..43640439 --- /dev/null +++ b/torrential/README.md @@ -0,0 +1,4 @@ +# Torrential + +A Rust webserver designed to hook into the Drop server and serve the content files, but at a lightning fast speed. + diff --git a/torrential/src/download.rs b/torrential/src/download.rs new file mode 100644 index 00000000..8529c031 --- /dev/null +++ b/torrential/src/download.rs @@ -0,0 +1,52 @@ +use std::{collections::HashMap, time::Instant}; + +use droplet_rs::versions::create_backend_constructor; +use log::info; +use reqwest::StatusCode; + +use crate::{AppInitData, DownloadContext, remote::{LibraryBackend, fetch_download_context}, util::ErrorOption}; + + +pub async fn create_download_context<'a>( + init_data: &AppInitData, + game_id: String, + version_name: String, +) -> Result, ErrorOption> { + let context = + fetch_download_context(init_data.token.clone(), game_id, version_name.clone()).await?; + + let (version_path, backend) = init_data + .libraries + .get(&context.library_id) + .ok_or(StatusCode::NOT_FOUND)?; + let version_path = version_path.join(context.library_path.clone()); + let version_path = match backend { + LibraryBackend::Filesystem => version_path.join(version_name), + LibraryBackend::FlatFilesystem => version_path, + }; + + let backend = + create_backend_constructor(&version_path).ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + let backend = backend()?; + + let mut chunk_lookup_table = + HashMap::with_capacity(context.manifest.values().map(|v| v.ids.len()).sum()); + + for (path, file_chunks) in context.manifest { + let mut start = 0; + for (chunk, length) in file_chunks.ids.into_iter().zip(file_chunks.lengths) { + chunk_lookup_table.insert(chunk, (path.clone(), start, start + length)); + start += length; + } + } + + let download_context = DownloadContext { + library_id: context.library_id, + library_path: context.library_path, + chunk_lookup_table, + backend, + last_access: Instant::now(), + }; + + Ok(download_context) +} diff --git a/torrential/src/main.rs b/torrential/src/main.rs new file mode 100644 index 00000000..5c37207b --- /dev/null +++ b/torrential/src/main.rs @@ -0,0 +1,224 @@ +use anyhow::{Result, anyhow}; +use dashmap::DashMap; +use droplet_rs::versions::types::{VersionBackend, VersionFile}; +use reqwest::header; +use simple_logger::SimpleLogger; +use std::{ + collections::HashMap, + env::set_current_dir, + path::PathBuf, + str::FromStr, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio_util::io::ReaderStream; + +use axum::{ + Json, Router, + body::Body, + extract::{Path, State}, + http::StatusCode, + response::{AppendHeaders, IntoResponse}, + routing::{get, post}, +}; +use log::{error, info, warn}; +use serde::Deserialize; +use tokio::{ + sync::{OnceCell, Semaphore}, + time::sleep, +}; + +use crate::{ + download::create_download_context, + remote::{LibraryBackend, fetch_library_sources}, +}; + +mod download; +mod manifest; +mod remote; +mod util; + +static GLOBAL_CONTEXT_SEMAPHORE: Semaphore = Semaphore::const_new(1); + +struct DownloadContext<'a> { + library_id: String, + library_path: String, + chunk_lookup_table: HashMap, + backend: Box, + last_access: Instant, +} + +#[derive(Debug)] +struct AppInitData { + token: String, + libraries: HashMap, +} + +struct AppState<'a> { + token: OnceCell, + context_cache: DashMap<(String, String), DownloadContext<'a>>, + working_context_cache: DashMap<(String, String), Semaphore>, +} + +async fn serve_file( + State(state): State>>, + Path((game_id, version_name, chunk_id)): Path<(String, String, String)>, +) -> Result { + let init_data = state.token.get().ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + let key = (game_id.clone(), version_name.clone()); + + let mut context = if let Some(context) = state.context_cache.get_mut(&key) { + context + } else { + let permit = GLOBAL_CONTEXT_SEMAPHORE + .acquire() + .await + .expect("failed to acquire semaphore"); + + // Check if it's been done while we've been sitting here + if let Some(already_done) = state.context_cache.get_mut(&key) { + already_done + } else { + info!("generating context..."); + let context_result = + create_download_context(init_data, game_id.clone(), version_name.clone()).await; + info!("cleaned up semaphore"); + + let new_context = context_result.inspect_err(|v| warn!("{:?}", v))?; + state.context_cache.insert(key.clone(), new_context); + + info!("continuing download"); + + drop(permit); + + state.context_cache.get_mut(&key).unwrap() + } + }; + + context.last_access = Instant::now(); + + let (relative_filename, start, end) = context + .chunk_lookup_table + .get(&chunk_id) + .cloned() + .ok_or(StatusCode::NOT_FOUND)?; + let reader = context + .backend + .reader( + &VersionFile { + relative_filename: relative_filename.to_string(), + permission: 0, + size: 0, + }, + start.try_into().unwrap(), + end.try_into().unwrap(), + ) + .await + .map_err(|v| { + error!("reader error: {v:?}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let stream = ReaderStream::new(reader); + let body = Body::from_stream(stream); + + let headers: AppendHeaders<[(header::HeaderName, String); 2]> = AppendHeaders([ + (header::CONTENT_TYPE, "application/octet-stream".to_owned()), + (header::CONTENT_LENGTH, (end - start).to_string()), + ]); + + Ok((headers, body)) +} + +#[derive(Deserialize)] +struct TokenPayload { + token: String, +} + +async fn set_token( + State(state): State>>, + Json(payload): Json, +) -> Result { + if let Some(existing_data) = state.token.get() { + if existing_data.token != payload.token { + panic!("already set up but provided with a different token"); + } + return Ok(StatusCode::OK); + } + + let token = payload.token; + + let library_sources = fetch_library_sources(token.clone()).await.map_err(|v| { + error!("{v:?}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let valid_library_sources = library_sources + .into_iter() + .filter(|v| match v.backend { + remote::LibraryBackend::Filesystem | remote::LibraryBackend::FlatFilesystem => true, + #[allow(unreachable_patterns)] + _ => false, + }) + .map(|v| { + let path = PathBuf::from_str( + v.options + .as_object() + .unwrap() + .get("baseDir") + .unwrap() + .as_str() + .unwrap(), + ) + .unwrap(); + + (v.id, (path, v.backend)) + }) + .collect::>(); + + state + .token + .set(AppInitData { + token, + libraries: valid_library_sources, + }) + .map_err(|err| { + error!("failed to set token: {err:?}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + info!("connected to drop server successfully"); + + Ok(StatusCode::OK) +} + +#[tokio::main] +async fn main() { + SimpleLogger::new() + .with_level(log::LevelFilter::Info) + .init() + .unwrap(); + + if let Ok(working_directory) = std::env::var("WORKING_DIRECTORY") { + set_current_dir(working_directory).expect("failed to change working directory"); + } + + let shared_state = Arc::new(AppState { + token: OnceCell::new(), + context_cache: DashMap::new(), + working_context_cache: DashMap::new(), + }); + + let app = Router::new() + .route( + "/api/v1/depot/{game_id}/{version_name}/{chunk_id}", + get(serve_file), + ) + .route("/token", post(set_token)) + .with_state(shared_state); + + // run our app with hyper, listening globally on port 3000 + let listener = tokio::net::TcpListener::bind("0.0.0.0:5000").await.unwrap(); + info!("started depot server"); + axum::serve(listener, app).await.unwrap(); +} diff --git a/torrential/src/manifest.rs b/torrential/src/manifest.rs new file mode 100644 index 00000000..fabe92d5 --- /dev/null +++ b/torrential/src/manifest.rs @@ -0,0 +1,13 @@ +use std::collections::HashMap; + +use serde::Deserialize; + +#[derive(Deserialize)] +pub struct DropChunk { + pub permissions: usize, + pub ids: Vec, + pub checksums: Vec, + pub lengths: Vec, +} + +pub type DropletManifest = HashMap; diff --git a/torrential/src/remote.rs b/torrential/src/remote.rs new file mode 100644 index 00000000..ab92bf9c --- /dev/null +++ b/torrential/src/remote.rs @@ -0,0 +1,116 @@ +use std::{env, sync::LazyLock}; + +use anyhow::{Result, anyhow}; +use log::info; +use reqwest::{Client, ClientBuilder, StatusCode, Url}; +use serde::{Deserialize, Serialize}; + +use crate::{manifest::DropletManifest, util::ErrorOption}; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ContextResponseBody { + timeout: String, + pub manifest: DropletManifest, + version_name: String, + pub library_id: String, + pub library_path: String, +} + +#[derive(Serialize)] +pub struct ContextQuery { + game: String, + version: String, +} + +static CLIENT: LazyLock = LazyLock::new(|| { + ClientBuilder::new() + .build() + .expect("failed to build client") +}); + +static REMOTE_URL: LazyLock = LazyLock::new(|| { + let user_provided = env::var("DROP_SERVER_URL"); + let url = Url::parse( + user_provided + .as_ref() + .map_or("http://localhost:3000", |v| v), + ) + .expect("failed to parse URL"); + info!("using Drop server url {}", url); + url +}); + +pub async fn fetch_download_context( + token: String, + game_id: String, + version_name: String, +) -> Result { + let context_response = CLIENT + .get(REMOTE_URL.join("/api/v1/admin/depot/context")?) + .query(&ContextQuery { + game: game_id, + version: version_name, + }) + .header("Authorization", format!("Bearer {}", token)) + .send() + .await?; + + if !context_response.status().is_success() { + if context_response.status() == StatusCode::BAD_REQUEST { + return Err(StatusCode::NOT_FOUND.into()); + } + + return Err(anyhow!( + "Fetching context failed with non-success code: {}, {}", + context_response.status(), + context_response + .text() + .await + .unwrap_or("(failed to read body)".to_string()) + ).into()); + } + + let context: ContextResponseBody = context_response.json().await?; + + Ok(context) +} + + +#[derive(Deserialize, Debug)] +#[non_exhaustive] +pub enum LibraryBackend { + Filesystem, + FlatFilesystem +} + +#[derive(Deserialize)] +pub struct LibrarySource { + pub options: serde_json::Value, + pub id: String, + name: String, + pub backend: LibraryBackend +} + +pub async fn fetch_library_sources(token: String) -> Result> { + let source_response = CLIENT + .get(REMOTE_URL.join("/api/v1/admin/library/sources")?) + .header("Authorization", format!("Bearer {}", token)) + .send() + .await?; + + if !source_response.status().is_success() { + return Err(anyhow!( + "Fetching library sources failed with non-success code: {}, {}", + source_response.status(), + source_response + .text() + .await + .unwrap_or("(failed to read body)".to_string()) + )); + } + + let library_sources: Vec = source_response.json().await?; + + Ok(library_sources) +} \ No newline at end of file diff --git a/torrential/src/util.rs b/torrential/src/util.rs new file mode 100644 index 00000000..f88b71d5 --- /dev/null +++ b/torrential/src/util.rs @@ -0,0 +1,37 @@ +use log::error; +use reqwest::{StatusCode, Url}; + +#[derive(Debug)] +pub struct ErrorOption(Result); +impl From for ErrorOption { + fn from(value: anyhow::Error) -> Self { + Self(Err(value)) + } +} +impl From for ErrorOption { + fn from(value: reqwest::Error) -> Self { + Self(Err(value.into())) + } +} +impl From for ErrorOption { + fn from(value: url::ParseError) -> Self { + Self(Err(value.into())) + } +} +impl From for ErrorOption { + fn from(value: StatusCode) -> Self { + Self(Ok(value)) + } +} + +impl From for StatusCode { + fn from(value: ErrorOption) -> Self { + match value.0 { + Ok(status) => status, + Err(err) => { + error!("{err:?}"); + StatusCode::INTERNAL_SERVER_ERROR + } + } + } +} From fd65a0653f4c8f5a362ffc2927aa13e73360b179 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 16:27:08 +1100 Subject: [PATCH 02/21] fix: clippy lints --- torrential/src/download.rs | 3 --- torrential/src/main.rs | 28 +++++++++------------------- torrential/src/manifest.rs | 2 -- torrential/src/remote.rs | 3 --- torrential/src/util.rs | 2 +- 5 files changed, 10 insertions(+), 28 deletions(-) diff --git a/torrential/src/download.rs b/torrential/src/download.rs index 8529c031..21827cb1 100644 --- a/torrential/src/download.rs +++ b/torrential/src/download.rs @@ -1,7 +1,6 @@ use std::{collections::HashMap, time::Instant}; use droplet_rs::versions::create_backend_constructor; -use log::info; use reqwest::StatusCode; use crate::{AppInitData, DownloadContext, remote::{LibraryBackend, fetch_download_context}, util::ErrorOption}; @@ -41,8 +40,6 @@ pub async fn create_download_context<'a>( } let download_context = DownloadContext { - library_id: context.library_id, - library_path: context.library_path, chunk_lookup_table, backend, last_access: Instant::now(), diff --git a/torrential/src/main.rs b/torrential/src/main.rs index 5c37207b..2e989045 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -1,15 +1,11 @@ -use anyhow::{Result, anyhow}; +use anyhow::Result; use dashmap::DashMap; use droplet_rs::versions::types::{VersionBackend, VersionFile}; use reqwest::header; use simple_logger::SimpleLogger; use std::{ - collections::HashMap, - env::set_current_dir, - path::PathBuf, - str::FromStr, - sync::Arc, - time::{Duration, Instant}, + collections::HashMap, env::set_current_dir, path::PathBuf, str::FromStr, sync::Arc, + time::Instant, }; use tokio_util::io::ReaderStream; @@ -23,10 +19,7 @@ use axum::{ }; use log::{error, info, warn}; use serde::Deserialize; -use tokio::{ - sync::{OnceCell, Semaphore}, - time::sleep, -}; +use tokio::sync::{OnceCell, Semaphore}; use crate::{ download::create_download_context, @@ -41,8 +34,6 @@ mod util; static GLOBAL_CONTEXT_SEMAPHORE: Semaphore = Semaphore::const_new(1); struct DownloadContext<'a> { - library_id: String, - library_path: String, chunk_lookup_table: HashMap, backend: Box, last_access: Instant, @@ -57,7 +48,6 @@ struct AppInitData { struct AppState<'a> { token: OnceCell, context_cache: DashMap<(String, String), DownloadContext<'a>>, - working_context_cache: DashMap<(String, String), Semaphore>, } async fn serve_file( @@ -155,10 +145,11 @@ async fn set_token( let valid_library_sources = library_sources .into_iter() - .filter(|v| match v.backend { - remote::LibraryBackend::Filesystem | remote::LibraryBackend::FlatFilesystem => true, - #[allow(unreachable_patterns)] - _ => false, + .filter(|v| { + matches!( + v.backend, + remote::LibraryBackend::Filesystem | remote::LibraryBackend::FlatFilesystem + ) }) .map(|v| { let path = PathBuf::from_str( @@ -206,7 +197,6 @@ async fn main() { let shared_state = Arc::new(AppState { token: OnceCell::new(), context_cache: DashMap::new(), - working_context_cache: DashMap::new(), }); let app = Router::new() diff --git a/torrential/src/manifest.rs b/torrential/src/manifest.rs index fabe92d5..f928f23c 100644 --- a/torrential/src/manifest.rs +++ b/torrential/src/manifest.rs @@ -4,9 +4,7 @@ use serde::Deserialize; #[derive(Deserialize)] pub struct DropChunk { - pub permissions: usize, pub ids: Vec, - pub checksums: Vec, pub lengths: Vec, } diff --git a/torrential/src/remote.rs b/torrential/src/remote.rs index ab92bf9c..e40f4cc0 100644 --- a/torrential/src/remote.rs +++ b/torrential/src/remote.rs @@ -10,9 +10,7 @@ use crate::{manifest::DropletManifest, util::ErrorOption}; #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct ContextResponseBody { - timeout: String, pub manifest: DropletManifest, - version_name: String, pub library_id: String, pub library_path: String, } @@ -88,7 +86,6 @@ pub enum LibraryBackend { pub struct LibrarySource { pub options: serde_json::Value, pub id: String, - name: String, pub backend: LibraryBackend } diff --git a/torrential/src/util.rs b/torrential/src/util.rs index f88b71d5..07e319cb 100644 --- a/torrential/src/util.rs +++ b/torrential/src/util.rs @@ -1,5 +1,5 @@ use log::error; -use reqwest::{StatusCode, Url}; +use reqwest::StatusCode; #[derive(Debug)] pub struct ErrorOption(Result); From 8965b05dda8e6ee3c715376bf62748096a8ecc35 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 16:28:09 +1100 Subject: [PATCH 03/21] add license --- torrential/LICENSE.md | 660 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 660 insertions(+) create mode 100644 torrential/LICENSE.md diff --git a/torrential/LICENSE.md b/torrential/LICENSE.md new file mode 100644 index 00000000..c6f01c63 --- /dev/null +++ b/torrential/LICENSE.md @@ -0,0 +1,660 @@ +# GNU AFFERO GENERAL PUBLIC LICENSE + +Version 3, 19 November 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. + + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + +## Preamble + +The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + +The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains +free software for all its users. + +When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + +Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + +A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + +The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + +An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing +under this license. + +The precise terms and conditions for copying, distribution and +modification follow. + +## TERMS AND CONDITIONS + +### 0. Definitions. + +"This License" refers to version 3 of the GNU Affero General Public +License. + +"Copyright" also means copyright-like laws that apply to other kinds +of works, such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + +To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of +an exact copy. The resulting work is called a "modified version" of +the earlier work or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based +on the Program. + +To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + +To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user +through a computer network, with no transfer of a copy, is not +conveying. + +An interactive user interface displays "Appropriate Legal Notices" to +the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + +### 1. Source Code. + +The "source code" for a work means the preferred form of the work for +making modifications to it. "Object code" means any non-source form of +a work. + +A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + +The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can +regenerate automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same +work. + +### 2. Basic Permissions. + +All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, +without conditions so long as your license otherwise remains in force. +You may convey covered works to others for the sole purpose of having +them make modifications exclusively for you, or provide you with +facilities for running those works, provided that you comply with the +terms of this License in conveying all material for which you do not +control copyright. Those thus making or running the covered works for +you must do so exclusively on your behalf, under your direction and +control, on terms that prohibit them from making any copies of your +copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under the +conditions stated below. Sublicensing is not allowed; section 10 makes +it unnecessary. + +### 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + +No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + +When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such +circumvention is effected by exercising rights under this License with +respect to the covered work, and you disclaim any intention to limit +operation or modification of the work as a means of enforcing, against +the work's users, your or third parties' legal rights to forbid +circumvention of technological measures. + +### 4. Conveying Verbatim Copies. + +You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + +### 5. Conveying Modified Source Versions. + +You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these +conditions: + +- a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. +- b) The work must carry prominent notices stating that it is + released under this License and any conditions added under + section 7. This requirement modifies the requirement in section 4 + to "keep intact all notices". +- c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. +- d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + +A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + +### 6. Conveying Non-Source Forms. + +You may convey a covered work in object code form under the terms of +sections 4 and 5, provided that you also convey the machine-readable +Corresponding Source under the terms of this License, in one of these +ways: + +- a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. +- b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the Corresponding + Source from a network server at no charge. +- c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. +- d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. +- e) Convey the object code using peer-to-peer transmission, + provided you inform other peers where the object code and + Corresponding Source of the work are being offered to the general + public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + +A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, +family, or household purposes, or (2) anything designed or sold for +incorporation into a dwelling. In determining whether a product is a +consumer product, doubtful cases shall be resolved in favor of +coverage. For a particular product received by a particular user, +"normally used" refers to a typical or common use of that class of +product, regardless of the status of the particular user or of the way +in which the particular user actually uses, or expects or is expected +to use, the product. A product is a consumer product regardless of +whether the product has substantial commercial, industrial or +non-consumer uses, unless such uses represent the only significant +mode of use of the product. + +"Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to +install and execute modified versions of a covered work in that User +Product from a modified version of its Corresponding Source. The +information must suffice to ensure that the continued functioning of +the modified object code is in no case prevented or interfered with +solely because modification has been made. + +If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + +The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or +updates for a work that has been modified or installed by the +recipient, or for the User Product in which it has been modified or +installed. Access to a network may be denied when the modification +itself materially and adversely affects the operation of the network +or violates the rules and protocols for communication across the +network. + +Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + +### 7. Additional Terms. + +"Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders +of that material) supplement the terms of this License with terms: + +- a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or +- b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or +- c) Prohibiting misrepresentation of the origin of that material, + or requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or +- d) Limiting the use for publicity purposes of names of licensors + or authors of the material; or +- e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or +- f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions + of it) with contractual assumptions of liability to the recipient, + for any liability that these contractual assumptions directly + impose on those licensors and authors. + +All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; the +above requirements apply either way. + +### 8. Termination. + +You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + +However, if you cease all violation of this License, then your license +from a particular copyright holder is reinstated (a) provisionally, +unless and until the copyright holder explicitly and finally +terminates your license, and (b) permanently, if the copyright holder +fails to notify you of the violation by some reasonable means prior to +60 days after the cessation. + +Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + +Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + +### 9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or run +a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + +### 10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + +An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + +### 11. Patents. + +A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims owned +or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + +In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + +If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + +A patent license is "discriminatory" if it does not include within the +scope of its coverage, prohibits the exercise of, or is conditioned on +the non-exercise of one or more of the rights that are specifically +granted under this License. You may not convey a covered work if you +are a party to an arrangement with a third party that is in the +business of distributing software, under which you make payment to the +third party based on the extent of your activity of conveying the +work, and under which the third party grants, to any of the parties +who would receive the covered work from you, a discriminatory patent +license (a) in connection with copies of the covered work conveyed by +you (or copies made from those copies), or (b) primarily for and in +connection with specific products or compilations that contain the +covered work, unless you entered into that arrangement, or that patent +license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + +### 12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under +this License and any other pertinent obligations, then as a +consequence you may not convey it at all. For example, if you agree to +terms that obligate you to collect a royalty for further conveying +from those to whom you convey the Program, the only way you could +satisfy both those terms and this License would be to refrain entirely +from conveying the Program. + +### 13. Remote Network Interaction; Use with the GNU General Public License. + +Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your +version supports such interaction) an opportunity to receive the +Corresponding Source of your version by providing access to the +Corresponding Source from a network server at no charge, through some +standard or customary means of facilitating copying of software. This +Corresponding Source shall include the Corresponding Source for any +work covered by version 3 of the GNU General Public License that is +incorporated pursuant to the following paragraph. + +Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + +### 14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions +of the GNU Affero General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever +published by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future versions +of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + +Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + +### 15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT +WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND +PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE +DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR +CORRECTION. + +### 16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR +CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT +NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR +LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM +TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER +PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +### 17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS + +## How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these +terms. + +To do so, attach the following notices to the program. It is safest to +attach them to the start of each source file to most effectively state +the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper +mail. + +If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for +the specific requirements. + +You should also get your employer (if you work as a programmer) or +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. For more information on this, and how to apply and follow +the GNU AGPL, see . From 39d4fea255a3a4e4fb241e19412a8033512c9646 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 16:29:23 +1100 Subject: [PATCH 04/21] update readme --- torrential/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torrential/README.md b/torrential/README.md index 43640439..daddd94f 100644 --- a/torrential/README.md +++ b/torrential/README.md @@ -1,4 +1,4 @@ -# Torrential +# torrential -A Rust webserver designed to hook into the Drop server and serve the content files, but at a lightning fast speed. +A Rust webserver designed to hook into the Drop server and serve the content files, but at a lightning fast speed. Designed to be built into the Drop Docker container, and proxied automatically via NGINX at `/api/v1/depot`. From 9e604bf61bd151389754885423fd946c35becee2 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 17:45:55 +1100 Subject: [PATCH 05/21] feat: add healthcheck --- torrential/src/download.rs | 2 +- torrential/src/main.rs | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/torrential/src/download.rs b/torrential/src/download.rs index 21827cb1..d65b545f 100644 --- a/torrential/src/download.rs +++ b/torrential/src/download.rs @@ -29,7 +29,7 @@ pub async fn create_download_context<'a>( let backend = backend()?; let mut chunk_lookup_table = - HashMap::with_capacity(context.manifest.values().map(|v| v.ids.len()).sum()); + HashMap::with_capacity_and_hasher(context.manifest.values().map(|v| v.ids.len()).sum(), Default::default()); for (path, file_chunks) in context.manifest { let mut start = 0; diff --git a/torrential/src/main.rs b/torrential/src/main.rs index 2e989045..3a9ae247 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -17,7 +17,7 @@ use axum::{ response::{AppendHeaders, IntoResponse}, routing::{get, post}, }; -use log::{error, info, warn}; +use log::{error, info}; use serde::Deserialize; use tokio::sync::{OnceCell, Semaphore}; @@ -71,11 +71,8 @@ async fn serve_file( } else { info!("generating context..."); let context_result = - create_download_context(init_data, game_id.clone(), version_name.clone()).await; - info!("cleaned up semaphore"); - - let new_context = context_result.inspect_err(|v| warn!("{:?}", v))?; - state.context_cache.insert(key.clone(), new_context); + create_download_context(init_data, game_id.clone(), version_name.clone()).await?; + state.context_cache.insert(key.clone(), context_result); info!("continuing download"); @@ -125,6 +122,14 @@ struct TokenPayload { token: String, } +async fn healthcheck(State(state): State>>) -> StatusCode { + let inited = state.token.initialized(); + if !inited { + return StatusCode::SERVICE_UNAVAILABLE; + } + return StatusCode::OK; +} + async fn set_token( State(state): State>>, Json(payload): Json, @@ -205,6 +210,7 @@ async fn main() { get(serve_file), ) .route("/token", post(set_token)) + .route("/healthcheck", get(healthcheck)) .with_state(shared_state); // run our app with hyper, listening globally on port 3000 From d10b4967fd3328b3204dddbdf353e06ba68d35d1 Mon Sep 17 00:00:00 2001 From: quexeky Date: Mon, 1 Dec 2025 20:21:40 +1100 Subject: [PATCH 06/21] refactor: Focus on maintainability Signed-off-by: quexeky --- torrential/src/download.rs | 56 +++++--- torrential/src/main.rs | 268 ++++++++++++++++++++++--------------- torrential/src/remote.rs | 12 +- 3 files changed, 204 insertions(+), 132 deletions(-) diff --git a/torrential/src/download.rs b/torrential/src/download.rs index d65b545f..ebcf63de 100644 --- a/torrential/src/download.rs +++ b/torrential/src/download.rs @@ -1,35 +1,28 @@ -use std::{collections::HashMap, time::Instant}; +use std::{collections::HashMap, path::PathBuf, time::Instant}; -use droplet_rs::versions::create_backend_constructor; +use droplet_rs::versions::{create_backend_constructor, types::VersionBackend}; use reqwest::StatusCode; -use crate::{AppInitData, DownloadContext, remote::{LibraryBackend, fetch_download_context}, util::ErrorOption}; - +use crate::{ + AppInitData, DownloadContext, + remote::{ContextResponseBody, LibraryBackend, fetch_download_context}, + util::ErrorOption, +}; pub async fn create_download_context<'a>( init_data: &AppInitData, game_id: String, version_name: String, -) -> Result, ErrorOption> { +) -> Result { let context = fetch_download_context(init_data.token.clone(), game_id, version_name.clone()).await?; - let (version_path, backend) = init_data - .libraries - .get(&context.library_id) - .ok_or(StatusCode::NOT_FOUND)?; - let version_path = version_path.join(context.library_path.clone()); - let version_path = match backend { - LibraryBackend::Filesystem => version_path.join(version_name), - LibraryBackend::FlatFilesystem => version_path, - }; + let backend = generate_backend(init_data, &context, &version_name)??; - let backend = - create_backend_constructor(&version_path).ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; - let backend = backend()?; - - let mut chunk_lookup_table = - HashMap::with_capacity_and_hasher(context.manifest.values().map(|v| v.ids.len()).sum(), Default::default()); + let mut chunk_lookup_table = HashMap::with_capacity_and_hasher( + context.manifest.values().map(|v| v.ids.len()).sum(), + Default::default(), + ); for (path, file_chunks) in context.manifest { let mut start = 0; @@ -47,3 +40,26 @@ pub async fn create_download_context<'a>( Ok(download_context) } + +fn generate_backend( + init_data: &AppInitData, + context: &ContextResponseBody, + version_name: &String, +) -> Result, anyhow::Error>, StatusCode> { + let (version_path, backend) = init_data + .libraries + .get(&context.library_id) + .ok_or(StatusCode::NOT_FOUND)?; + + let version_path = version_path.join(&context.library_path); + let version_path = match backend { + LibraryBackend::Filesystem => version_path.join(version_name), + LibraryBackend::FlatFilesystem => version_path, + }; + + let backend = + create_backend_constructor(&version_path).ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + + let backend = backend(); + Ok(backend) +} diff --git a/torrential/src/main.rs b/torrential/src/main.rs index 3a9ae247..b9310b67 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -1,6 +1,6 @@ use anyhow::Result; -use dashmap::DashMap; -use droplet_rs::versions::types::{VersionBackend, VersionFile}; +use dashmap::{DashMap, mapref::one::RefMut}; +use droplet_rs::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; use reqwest::header; use simple_logger::SimpleLogger; use std::{ @@ -19,11 +19,11 @@ use axum::{ }; use log::{error, info}; use serde::Deserialize; -use tokio::sync::{OnceCell, Semaphore}; +use tokio::sync::{OnceCell, Semaphore, SemaphorePermit}; use crate::{ download::create_download_context, - remote::{LibraryBackend, fetch_library_sources}, + remote::{LibraryBackend, LibrarySource, fetch_library_sources}, }; mod download; @@ -33,9 +33,9 @@ mod util; static GLOBAL_CONTEXT_SEMAPHORE: Semaphore = Semaphore::const_new(1); -struct DownloadContext<'a> { +struct DownloadContext { chunk_lookup_table: HashMap, - backend: Box, + backend: Box, last_access: Instant, } @@ -45,51 +45,122 @@ struct AppInitData { libraries: HashMap, } -struct AppState<'a> { +struct AppState { token: OnceCell, - context_cache: DashMap<(String, String), DownloadContext<'a>>, + context_cache: DashMap<(String, String), DownloadContext>, +} + +#[tokio::main] +async fn main() { + initialise_logger(); + + if let Ok(working_directory) = std::env::var("WORKING_DIRECTORY") { + set_current_dir(working_directory).expect("failed to change working directory"); + } + + let shared_state = Arc::new(AppState { + token: OnceCell::new(), + context_cache: DashMap::new(), + }); + + let app = setup_app(shared_state); + + serve(app).await.unwrap(); +} + +fn setup_app(shared_state: Arc) -> Router { + Router::new() + .route( + "/api/v1/depot/{game_id}/{version_name}/{chunk_id}", + get(serve_file), + ) + .route("/token", post(set_token)) + .route("/healthcheck", get(healthcheck)) + .with_state(shared_state) +} +async fn serve(app: Router) -> Result<(), std::io::Error> { + let listener = tokio::net::TcpListener::bind("0.0.0.0:5000").await.unwrap(); + info!("started depot server"); + axum::serve(listener, app).await +} + +async fn set_token( + State(state): State>, + Json(payload): Json, +) -> Result { + if check_token_exists(&state, &payload) { + return Ok(StatusCode::OK); + } + + let token = payload.token; + + let library_sources = fetch_library_sources(&token).await.map_err(|v| { + error!("{v:?}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let valid_library_sources = filter_library_sources(library_sources); + + set_generated_token(state, token, valid_library_sources)?; + + info!("connected to drop server successfully"); + + Ok(StatusCode::OK) } async fn serve_file( - State(state): State>>, + State(state): State>, Path((game_id, version_name, chunk_id)): Path<(String, String, String)>, ) -> Result { - let init_data = state.token.get().ok_or(StatusCode::SERVICE_UNAVAILABLE)?; - let key = (game_id.clone(), version_name.clone()); - - let mut context = if let Some(context) = state.context_cache.get_mut(&key) { - context - } else { - let permit = GLOBAL_CONTEXT_SEMAPHORE - .acquire() - .await - .expect("failed to acquire semaphore"); - - // Check if it's been done while we've been sitting here - if let Some(already_done) = state.context_cache.get_mut(&key) { - already_done - } else { - info!("generating context..."); - let context_result = - create_download_context(init_data, game_id.clone(), version_name.clone()).await?; - state.context_cache.insert(key.clone(), context_result); - - info!("continuing download"); - - drop(permit); - - state.context_cache.get_mut(&key).unwrap() - } - }; + let context_cache = &state.context_cache; + let mut context = get_or_generate_context(&state, context_cache, game_id, version_name).await?; context.last_access = Instant::now(); - let (relative_filename, start, end) = context + let (relative_filename, start, end) = lookup_chunk(&chunk_id, &context)?; + let reader = get_file_reader(&mut context, relative_filename, start, end).await?; + + let stream = ReaderStream::new(reader); + let body: Body = Body::from_stream(stream); + + let headers: AppendHeaders<[(header::HeaderName, String); 2]> = AppendHeaders([ + (header::CONTENT_TYPE, "application/octet-stream".to_owned()), + (header::CONTENT_LENGTH, (end - start).to_string()), + ]); + + Ok((headers, body)) +} + +fn initialise_logger() { + SimpleLogger::new() + .with_level(log::LevelFilter::Info) + .init() + .unwrap(); +} + +async fn acquire_permit<'a>() -> SemaphorePermit<'a> { + GLOBAL_CONTEXT_SEMAPHORE + .acquire() + .await + .expect("failed to acquire semaphore") +} +fn lookup_chunk( + chunk_id: &String, + context: &RefMut<'_, (String, String), DownloadContext>, +) -> Result<(String, usize, usize), StatusCode> { + context .chunk_lookup_table - .get(&chunk_id) + .get(chunk_id) .cloned() - .ok_or(StatusCode::NOT_FOUND)?; - let reader = context + .ok_or(StatusCode::NOT_FOUND) +} +async fn get_file_reader( + context: &mut RefMut<'_, (String, String), DownloadContext>, + relative_filename: String, + start: usize, + end: usize, +) -> Result, StatusCode> { + context .backend .reader( &VersionFile { @@ -104,17 +175,40 @@ async fn serve_file( .map_err(|v| { error!("reader error: {v:?}"); StatusCode::INTERNAL_SERVER_ERROR - })?; + }) +} +async fn get_or_generate_context<'a>( + state: &Arc, + context_cache: &'a DashMap<(String, String), DownloadContext>, + game_id: String, + version_name: String, +) -> Result, StatusCode> { + let initialisation_data = state.token.get().ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + let key = (game_id.clone(), version_name.clone()); - let stream = ReaderStream::new(reader); - let body = Body::from_stream(stream); + if let Some(context) = context_cache.get_mut(&key) { + Ok(context) + } else { + let permit = acquire_permit().await; - let headers: AppendHeaders<[(header::HeaderName, String); 2]> = AppendHeaders([ - (header::CONTENT_TYPE, "application/octet-stream".to_owned()), - (header::CONTENT_LENGTH, (end - start).to_string()), - ]); + // Check if it's been done while we've been sitting here + if let Some(already_done) = context_cache.get_mut(&key) { + Ok(already_done) + } else { + info!("generating context..."); + let context_result = + create_download_context(initialisation_data, game_id.clone(), version_name.clone()) + .await?; - Ok((headers, body)) + state.context_cache.insert(key.clone(), context_result); + + info!("continuing download"); + + drop(permit); + + Ok(context_cache.get_mut(&key).unwrap()) + } + } } #[derive(Deserialize)] @@ -122,33 +216,27 @@ struct TokenPayload { token: String, } -async fn healthcheck(State(state): State>>) -> StatusCode { - let inited = state.token.initialized(); - if !inited { +async fn healthcheck(State(state): State>) -> StatusCode { + let initialised = state.token.initialized(); + if !initialised { return StatusCode::SERVICE_UNAVAILABLE; } return StatusCode::OK; } -async fn set_token( - State(state): State>>, - Json(payload): Json, -) -> Result { +fn check_token_exists(state: &Arc, payload: &TokenPayload) -> bool { if let Some(existing_data) = state.token.get() { if existing_data.token != payload.token { panic!("already set up but provided with a different token"); } - return Ok(StatusCode::OK); + return true; } - - let token = payload.token; - - let library_sources = fetch_library_sources(token.clone()).await.map_err(|v| { - error!("{v:?}"); - StatusCode::INTERNAL_SERVER_ERROR - })?; - - let valid_library_sources = library_sources + false +} +fn filter_library_sources( + library_sources: Vec, +) -> HashMap { + library_sources .into_iter() .filter(|v| { matches!( @@ -170,51 +258,19 @@ async fn set_token( (v.id, (path, v.backend)) }) - .collect::>(); - + .collect() +} +fn set_generated_token( + state: Arc, + token: String, + libraries: HashMap, +) -> Result<(), StatusCode> { state .token - .set(AppInitData { - token, - libraries: valid_library_sources, - }) + .set(AppInitData { token, libraries }) .map_err(|err| { error!("failed to set token: {err:?}"); StatusCode::INTERNAL_SERVER_ERROR })?; - - info!("connected to drop server successfully"); - - Ok(StatusCode::OK) -} - -#[tokio::main] -async fn main() { - SimpleLogger::new() - .with_level(log::LevelFilter::Info) - .init() - .unwrap(); - - if let Ok(working_directory) = std::env::var("WORKING_DIRECTORY") { - set_current_dir(working_directory).expect("failed to change working directory"); - } - - let shared_state = Arc::new(AppState { - token: OnceCell::new(), - context_cache: DashMap::new(), - }); - - let app = Router::new() - .route( - "/api/v1/depot/{game_id}/{version_name}/{chunk_id}", - get(serve_file), - ) - .route("/token", post(set_token)) - .route("/healthcheck", get(healthcheck)) - .with_state(shared_state); - - // run our app with hyper, listening globally on port 3000 - let listener = tokio::net::TcpListener::bind("0.0.0.0:5000").await.unwrap(); - info!("started depot server"); - axum::serve(listener, app).await.unwrap(); + Ok(()) } diff --git a/torrential/src/remote.rs b/torrential/src/remote.rs index e40f4cc0..d242cbc9 100644 --- a/torrential/src/remote.rs +++ b/torrential/src/remote.rs @@ -66,7 +66,8 @@ pub async fn fetch_download_context( .text() .await .unwrap_or("(failed to read body)".to_string()) - ).into()); + ) + .into()); } let context: ContextResponseBody = context_response.json().await?; @@ -74,22 +75,21 @@ pub async fn fetch_download_context( Ok(context) } - #[derive(Deserialize, Debug)] #[non_exhaustive] pub enum LibraryBackend { Filesystem, - FlatFilesystem + FlatFilesystem, } #[derive(Deserialize)] pub struct LibrarySource { pub options: serde_json::Value, pub id: String, - pub backend: LibraryBackend + pub backend: LibraryBackend, } -pub async fn fetch_library_sources(token: String) -> Result> { +pub async fn fetch_library_sources(token: &String) -> Result> { let source_response = CLIENT .get(REMOTE_URL.join("/api/v1/admin/library/sources")?) .header("Authorization", format!("Bearer {}", token)) @@ -110,4 +110,4 @@ pub async fn fetch_library_sources(token: String) -> Result> let library_sources: Vec = source_response.json().await?; Ok(library_sources) -} \ No newline at end of file +} From f7735fa88a8c0e30a551a2135bf4d319138c8e4b Mon Sep 17 00:00:00 2001 From: DecDuck Date: Mon, 1 Dec 2025 21:29:19 +1100 Subject: [PATCH 07/21] fix: remove openssl --- torrential/Cargo.lock | 503 +++++++++++++++--------------------------- torrential/Cargo.toml | 5 +- 2 files changed, 181 insertions(+), 327 deletions(-) diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock index a5095aa3..19f4222c 100644 --- a/torrential/Cargo.lock +++ b/torrential/Cargo.lock @@ -190,6 +190,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "colored" version = "3.0.0" @@ -199,22 +205,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -229,7 +219,7 @@ checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ "cfg-if", "crossbeam-utils", - "hashbrown 0.14.5", + "hashbrown", "lock_api", "once_cell", "parking_lot_core", @@ -313,64 +303,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - [[package]] name = "find-msvc-tools" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -426,8 +364,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -437,28 +377,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasip2", -] - -[[package]] -name = "h2" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", + "wasm-bindgen", ] [[package]] @@ -467,12 +390,6 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" - [[package]] name = "hex" version = "0.4.3" @@ -534,7 +451,6 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", "http", "http-body", "httparse", @@ -561,22 +477,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", -] - -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", + "webpki-roots", ] [[package]] @@ -598,11 +499,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", - "system-configuration", "tokio", "tower-service", "tracing", - "windows-registry", ] [[package]] @@ -707,16 +606,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "indexmap" -version = "2.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" -dependencies = [ - "equivalent", - "hashbrown 0.16.1", -] - [[package]] name = "ipnet" version = "2.11.0" @@ -761,12 +650,6 @@ version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" -[[package]] -name = "linux-raw-sys" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" - [[package]] name = "litemap" version = "0.8.1" @@ -788,6 +671,12 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matchit" version = "0.8.4" @@ -823,23 +712,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nom" version = "7.1.3" @@ -908,50 +780,6 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -[[package]] -name = "openssl" -version = "0.10.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-sys" -version = "0.9.111" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "parking_lot_core" version = "0.9.12" @@ -993,12 +821,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - [[package]] name = "potential_utf" version = "0.1.4" @@ -1014,6 +836,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.103" @@ -1023,6 +854,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.17", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.17", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.42" @@ -1038,6 +924,35 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "rcgen" version = "0.13.2" @@ -1069,29 +984,26 @@ checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ "base64", "bytes", - "encoding_rs", "futures-core", - "h2", "http", "http-body", "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", - "mime", - "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-native-tls", + "tokio-rustls", "tower", "tower-http", "tower-service", @@ -1099,6 +1011,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", ] [[package]] @@ -1115,6 +1028,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -1124,19 +1043,6 @@ dependencies = [ "nom", ] -[[package]] -name = "rustix" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.61.2", -] - [[package]] name = "rustls" version = "0.23.35" @@ -1144,6 +1050,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" dependencies = [ "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -1156,6 +1063,7 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" dependencies = [ + "web-time", "zeroize", ] @@ -1182,44 +1090,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" -[[package]] -name = "schannel" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "serde" version = "1.0.228" @@ -1377,40 +1253,6 @@ dependencies = [ "syn", ] -[[package]] -name = "system-configuration" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" -dependencies = [ - "bitflags", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" -dependencies = [ - "fastrand", - "getrandom 0.3.4", - "once_cell", - "rustix", - "windows-sys 0.61.2", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -1492,6 +1334,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.48.0" @@ -1519,16 +1376,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" @@ -1672,12 +1519,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "want" version = "0.3.1" @@ -1770,6 +1611,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki" version = "0.22.4" @@ -1780,41 +1631,21 @@ dependencies = [ "untrusted", ] +[[package]] +name = "webpki-roots" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -2060,6 +1891,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.6" diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml index 0da9313b..07c059bc 100644 --- a/torrential/Cargo.toml +++ b/torrential/Cargo.toml @@ -6,7 +6,10 @@ edition = "2024" [dependencies] axum = "0.8.7" log = "0.4.28" -reqwest = { version = "0.12.24", features = ["json"] } +reqwest = { version = "0.12.24", default-features = false, features = [ + "json", + "rustls-tls", +] } serde = { version = "1.0.228", features = ["derive"] } simple_logger = { version = "5.1.0", default-features = false, features = [ "colors", From 51b469962a3e6c88069dfb0e47f8d9b57ee96e63 Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 3 Dec 2025 07:33:32 +1100 Subject: [PATCH 08/21] refactor: Add lints to use in future and fix some Signed-off-by: quexeky --- torrential/Cargo.lock | 334 ++++++++++++++++++++++++++++++- torrential/Cargo.toml | 21 ++ torrential/benches/torrential.rs | 14 ++ torrential/src/download.rs | 6 +- torrential/src/main.rs | 23 ++- torrential/src/remote.rs | 10 +- torrential/src/util.rs | 2 +- 7 files changed, 387 insertions(+), 23 deletions(-) create mode 100644 torrential/benches/torrential.rs diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock index 19f4222c..317c8538 100644 --- a/torrential/Cargo.lock +++ b/torrential/Cargo.lock @@ -2,6 +2,36 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + [[package]] name = "anyhow" version = "1.0.100" @@ -174,6 +204,12 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.48" @@ -196,6 +232,58 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" + [[package]] name = "colored" version = "3.0.0" @@ -205,12 +293,72 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "criterion" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0dfe5e9e71bdcf4e4954f7d14da74d1cdb92a3a07686452d1509652684b1aab" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de36c2bee19fba779808f92bf5d9b0fa5a40095c277aba10c458a12b35d21d6" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "dashmap" version = "6.1.0" @@ -303,6 +451,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "find-msvc-tools" version = "0.1.5" @@ -384,6 +538,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -622,6 +787,15 @@ dependencies = [ "serde", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -646,9 +820,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.177" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "litemap" @@ -780,6 +954,22 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot_core" version = "0.9.12" @@ -821,6 +1011,34 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -906,7 +1124,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -953,6 +1171,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rcgen" version = "0.13.2" @@ -976,6 +1214,35 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + [[package]] name = "reqwest" version = "0.12.24" @@ -1090,6 +1357,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1334,6 +1610,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.10.0" @@ -1405,6 +1691,7 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", + "criterion", "dashmap", "droplet-rs", "log", @@ -1519,6 +1806,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -1640,6 +1937,37 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml index 07c059bc..9e354b85 100644 --- a/torrential/Cargo.toml +++ b/torrential/Cargo.toml @@ -21,3 +21,24 @@ anyhow = "1.0.100" serde_json = "1.0.145" url = { version = "2.5.7", default-features = false } tokio-util = { version = "0.7.17", features = ["io"] } + +[lints.clippy] +pedantic = { level = "warn", priority = -1 } + +almost_swapped = "deny" +unwrap_used = "warn" +expect_used = "warn" +dbg_macro = "deny" +needless_match = "deny" + +cast_possible_truncation = "warn" +cast_possible_wrap = "warn" +single_match_else = "warn" +inconsistent_digit_grouping = "warn" + +[[bench]] +name = "torrential" +harness = false + +[dev-dependencies] +criterion = "0.8.0" diff --git a/torrential/benches/torrential.rs b/torrential/benches/torrential.rs new file mode 100644 index 00000000..ad7fd90a --- /dev/null +++ b/torrential/benches/torrential.rs @@ -0,0 +1,14 @@ +use criterion::{criterion_group, criterion_main, Criterion}; + +fn torrential() { + +} + +// The benchmark function setup +fn benchmark(c: &mut Criterion) { + c.bench_function("fibonacci 20", |b| b.iter(|| torrential())); +} + +// Grouping your benchmarks +criterion_group!(benches, benchmark); +criterion_main!(benches); \ No newline at end of file diff --git a/torrential/src/download.rs b/torrential/src/download.rs index ebcf63de..32272beb 100644 --- a/torrential/src/download.rs +++ b/torrential/src/download.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, path::PathBuf, time::Instant}; +use std::{collections::HashMap, hash::RandomState, time::Instant}; use droplet_rs::versions::{create_backend_constructor, types::VersionBackend}; use reqwest::StatusCode; @@ -9,7 +9,7 @@ use crate::{ util::ErrorOption, }; -pub async fn create_download_context<'a>( +pub async fn create_download_context( init_data: &AppInitData, game_id: String, version_name: String, @@ -21,7 +21,7 @@ pub async fn create_download_context<'a>( let mut chunk_lookup_table = HashMap::with_capacity_and_hasher( context.manifest.values().map(|v| v.ids.len()).sum(), - Default::default(), + RandomState::default(), ); for (path, file_chunks) in context.manifest { diff --git a/torrential/src/main.rs b/torrential/src/main.rs index b9310b67..afa4a7f2 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -4,7 +4,7 @@ use droplet_rs::versions::types::{MinimumFileObject, VersionBackend, VersionFile use reqwest::header; use simple_logger::SimpleLogger; use std::{ - collections::HashMap, env::set_current_dir, path::PathBuf, str::FromStr, sync::Arc, + collections::HashMap, env::set_current_dir, path::PathBuf, str::FromStr as _, sync::Arc, time::Instant, }; use tokio_util::io::ReaderStream; @@ -101,7 +101,7 @@ async fn set_token( let valid_library_sources = filter_library_sources(library_sources); - set_generated_token(state, token, valid_library_sources)?; + set_generated_token(&state, token, valid_library_sources)?; info!("connected to drop server successfully"); @@ -139,7 +139,7 @@ fn initialise_logger() { } async fn acquire_permit<'a>() -> SemaphorePermit<'a> { - GLOBAL_CONTEXT_SEMAPHORE + return GLOBAL_CONTEXT_SEMAPHORE .acquire() .await .expect("failed to acquire semaphore") @@ -164,12 +164,12 @@ async fn get_file_reader( .backend .reader( &VersionFile { - relative_filename: relative_filename.to_string(), + relative_filename: relative_filename.clone(), permission: 0, size: 0, }, - start.try_into().unwrap(), - end.try_into().unwrap(), + start as u64, + end as u64, ) .await .map_err(|v| { @@ -221,14 +221,15 @@ async fn healthcheck(State(state): State>) -> StatusCode { if !initialised { return StatusCode::SERVICE_UNAVAILABLE; } - return StatusCode::OK; + StatusCode::OK } fn check_token_exists(state: &Arc, payload: &TokenPayload) -> bool { if let Some(existing_data) = state.token.get() { - if existing_data.token != payload.token { - panic!("already set up but provided with a different token"); - } + assert!( + existing_data.token == payload.token, + "already set up but provided with a different token" + ); return true; } false @@ -261,7 +262,7 @@ fn filter_library_sources( .collect() } fn set_generated_token( - state: Arc, + state: &Arc, token: String, libraries: HashMap, ) -> Result<(), StatusCode> { diff --git a/torrential/src/remote.rs b/torrential/src/remote.rs index d242cbc9..a8bf3da3 100644 --- a/torrential/src/remote.rs +++ b/torrential/src/remote.rs @@ -35,7 +35,7 @@ static REMOTE_URL: LazyLock = LazyLock::new(|| { .map_or("http://localhost:3000", |v| v), ) .expect("failed to parse URL"); - info!("using Drop server url {}", url); + info!("using Drop server url {url}"); url }); @@ -50,7 +50,7 @@ pub async fn fetch_download_context( game: game_id, version: version_name, }) - .header("Authorization", format!("Bearer {}", token)) + .header("Authorization", format!("Bearer {token}")) .send() .await?; @@ -65,7 +65,7 @@ pub async fn fetch_download_context( context_response .text() .await - .unwrap_or("(failed to read body)".to_string()) + .unwrap_or("(failed to read body)".to_owned()) ) .into()); } @@ -92,7 +92,7 @@ pub struct LibrarySource { pub async fn fetch_library_sources(token: &String) -> Result> { let source_response = CLIENT .get(REMOTE_URL.join("/api/v1/admin/library/sources")?) - .header("Authorization", format!("Bearer {}", token)) + .header("Authorization", format!("Bearer {token}")) .send() .await?; @@ -103,7 +103,7 @@ pub async fn fetch_library_sources(token: &String) -> Result> source_response .text() .await - .unwrap_or("(failed to read body)".to_string()) + .unwrap_or("(failed to read body)".to_owned()) )); } diff --git a/torrential/src/util.rs b/torrential/src/util.rs index 07e319cb..be1b8579 100644 --- a/torrential/src/util.rs +++ b/torrential/src/util.rs @@ -30,7 +30,7 @@ impl From for StatusCode { Ok(status) => status, Err(err) => { error!("{err:?}"); - StatusCode::INTERNAL_SERVER_ERROR + Self::INTERNAL_SERVER_ERROR } } } From 872a5421bf25622fb9664da7a9ff45586fe899fa Mon Sep 17 00:00:00 2001 From: quexeky Date: Wed, 3 Dec 2025 10:14:31 +1100 Subject: [PATCH 09/21] refactor: Convert to a dependency injection system for library sources, contexts, and backends Signed-off-by: quexeky --- torrential/Cargo.lock | 52 +++++++ torrential/Cargo.toml | 13 +- torrential/benches/torrential.rs | 44 +++++- torrential/src/download.rs | 83 ++++++---- torrential/src/handlers.rs | 126 +++++++++++++++ torrential/src/lib.rs | 17 ++ torrential/src/main.rs | 259 ++++--------------------------- torrential/src/remote.rs | 188 ++++++++++++---------- torrential/src/state.rs | 38 +++++ torrential/src/token.rs | 94 +++++++++++ 10 files changed, 573 insertions(+), 341 deletions(-) create mode 100644 torrential/src/handlers.rs create mode 100644 torrential/src/lib.rs create mode 100644 torrential/src/state.rs create mode 100644 torrential/src/token.rs diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock index 317c8538..b3e798b7 100644 --- a/torrential/Cargo.lock +++ b/torrential/Cargo.lock @@ -315,6 +315,7 @@ dependencies = [ "serde", "serde_json", "tinytemplate", + "tokio", "walkdir", ] @@ -457,6 +458,22 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "find-msvc-tools" version = "0.1.5" @@ -824,6 +841,12 @@ version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.1" @@ -1310,6 +1333,19 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.35" @@ -1529,6 +1565,19 @@ dependencies = [ "syn", ] +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -1690,15 +1739,18 @@ name = "torrential" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "axum", "criterion", "dashmap", "droplet-rs", "log", + "rand", "reqwest", "serde", "serde_json", "simple_logger", + "tempfile", "tokio", "tokio-util", "url", diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml index 9e354b85..a4a8e02d 100644 --- a/torrential/Cargo.toml +++ b/torrential/Cargo.toml @@ -3,6 +3,14 @@ name = "torrential" version = "0.1.0" edition = "2024" +[lib] +name = "torrential" +path = "src/lib.rs" + +[[bin]] +name = "torrential" +path = "src/main.rs" + [dependencies] axum = "0.8.7" log = "0.4.28" @@ -21,6 +29,7 @@ anyhow = "1.0.100" serde_json = "1.0.145" url = { version = "2.5.7", default-features = false } tokio-util = { version = "0.7.17", features = ["io"] } +async-trait = "0.1.89" [lints.clippy] pedantic = { level = "warn", priority = -1 } @@ -41,4 +50,6 @@ name = "torrential" harness = false [dev-dependencies] -criterion = "0.8.0" +criterion = { version = "0.8.0", features = ["async", "async_tokio"] } +rand = "0.9.2" +tempfile = "3.23.0" diff --git a/torrential/benches/torrential.rs b/torrential/benches/torrential.rs index ad7fd90a..863c3314 100644 --- a/torrential/benches/torrential.rs +++ b/torrential/benches/torrential.rs @@ -1,14 +1,46 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use std::{ + cmp, + fs::File, + io::{BufWriter, Write}, +}; -fn torrential() { - +use criterion::{Criterion, criterion_group, criterion_main}; +use rand::{Rng, rng}; +use tempfile::tempfile; +use tokio::runtime::Runtime; + +async fn torrential() {} + +fn generate_file() -> File { + let total_bytes = 312 * 1024 * 1024; + let tempfile = tempfile().unwrap(); + let mut writer = BufWriter::new(tempfile); + + let mut rng = rng(); + let mut buffer = [0; 1024]; + let mut remaining_size = total_bytes; + + while remaining_size > 0 { + let to_write = cmp::min(remaining_size, buffer.len()); + let buffer = &mut buffer[..to_write]; + rng.fill(buffer); + writer.write(buffer).unwrap(); + + remaining_size -= to_write; + } + writer.into_inner().unwrap() } - // The benchmark function setup fn benchmark(c: &mut Criterion) { - c.bench_function("fibonacci 20", |b| b.iter(|| torrential())); + let rt = Runtime::new().unwrap(); + + let file = generate_file(); + + c.bench_function("torrential download", |b| { + b.to_async(&rt).iter(|| torrential()) + }); } // Grouping your benchmarks criterion_group!(benches, benchmark); -criterion_main!(benches); \ No newline at end of file +criterion_main!(benches); diff --git a/torrential/src/download.rs b/torrential/src/download.rs index 32272beb..0f3f8b72 100644 --- a/torrential/src/download.rs +++ b/torrential/src/download.rs @@ -4,20 +4,37 @@ use droplet_rs::versions::{create_backend_constructor, types::VersionBackend}; use reqwest::StatusCode; use crate::{ - AppInitData, DownloadContext, - remote::{ContextResponseBody, LibraryBackend, fetch_download_context}, + remote::{ContextResponseBody, LibraryBackend, ContextProvider}, + state::AppInitData, util::ErrorOption, }; +pub struct DownloadContext { + pub(crate) chunk_lookup_table: HashMap, + pub(crate) backend: Box, + last_access: Instant, +} +impl DownloadContext { + pub fn last_access(&self) -> Instant { + self.last_access + } + pub fn reset_last_access(&mut self) { + self.last_access = Instant::now() + } +} + pub async fn create_download_context( + metadata_provider: &dyn ContextProvider, + backend_factory: &dyn BackendFactory, init_data: &AppInitData, game_id: String, version_name: String, ) -> Result { - let context = - fetch_download_context(init_data.token.clone(), game_id, version_name.clone()).await?; + let context = metadata_provider + .fetch_context(init_data.token(), game_id, version_name.clone()) + .await?; - let backend = generate_backend(init_data, &context, &version_name)??; + let backend = backend_factory.create_backend(init_data, &context, &version_name)?; let mut chunk_lookup_table = HashMap::with_capacity_and_hasher( context.manifest.values().map(|v| v.ids.len()).sum(), @@ -41,25 +58,39 @@ pub async fn create_download_context( Ok(download_context) } -fn generate_backend( - init_data: &AppInitData, - context: &ContextResponseBody, - version_name: &String, -) -> Result, anyhow::Error>, StatusCode> { - let (version_path, backend) = init_data - .libraries - .get(&context.library_id) - .ok_or(StatusCode::NOT_FOUND)?; - - let version_path = version_path.join(&context.library_path); - let version_path = match backend { - LibraryBackend::Filesystem => version_path.join(version_name), - LibraryBackend::FlatFilesystem => version_path, - }; - - let backend = - create_backend_constructor(&version_path).ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; - - let backend = backend(); - Ok(backend) +pub trait BackendFactory: Send + Sync { + fn create_backend( + &self, + init_data: &AppInitData, + context: &ContextResponseBody, + version_name: &String, + ) -> Result, StatusCode>; +} + +pub struct DropBackendFactory; +impl BackendFactory for DropBackendFactory { + fn create_backend( + &self, + init_data: &AppInitData, + context: &ContextResponseBody, + version_name: &String, + ) -> Result, StatusCode> { + let (version_path, backend) = init_data + .libraries() + .get(&context.library_id) + .ok_or(StatusCode::NOT_FOUND)?; + + let version_path = version_path.join(&context.library_path); + let version_path = match backend { + LibraryBackend::Filesystem => version_path.join(version_name), + LibraryBackend::FlatFilesystem => version_path, + }; + + let backend = + create_backend_constructor(&version_path).ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + + // TODO: Not eat this error + let backend = backend().map_err(|_e| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(backend) + } } diff --git a/torrential/src/handlers.rs b/torrential/src/handlers.rs new file mode 100644 index 00000000..b54bee6d --- /dev/null +++ b/torrential/src/handlers.rs @@ -0,0 +1,126 @@ +use std::sync::Arc; + +use axum::{ + body::Body, + extract::{Path, State}, + response::{AppendHeaders, IntoResponse}, +}; +use dashmap::{DashMap, mapref::one::RefMut}; +use droplet_rs::versions::types::{MinimumFileObject, VersionFile}; +use log::{error, info}; +use reqwest::{StatusCode, header}; +use tokio::sync::SemaphorePermit; +use tokio_util::io::ReaderStream; + +use crate::{ + DownloadContext, GLOBAL_CONTEXT_SEMAPHORE, download::create_download_context, state::AppState, +}; + +pub async fn serve_file( + State(state): State>, + Path((game_id, version_name, chunk_id)): Path<(String, String, String)>, +) -> Result { + let context_cache = &state.context_cache; + + let mut context = get_or_generate_context(&state, context_cache, game_id, version_name).await?; + context.reset_last_access(); + + let (relative_filename, start, end) = lookup_chunk(&chunk_id, &context)?; + let reader = get_file_reader(&mut context, relative_filename, start, end).await?; + + let stream = ReaderStream::new(reader); + let body: Body = Body::from_stream(stream); + + let headers: AppendHeaders<[(header::HeaderName, String); 2]> = AppendHeaders([ + (header::CONTENT_TYPE, "application/octet-stream".to_owned()), + (header::CONTENT_LENGTH, (end - start).to_string()), + ]); + + Ok((headers, body)) +} + +pub async fn healthcheck(State(state): State>) -> StatusCode { + let initialised = state.token.initialized(); + if !initialised { + return StatusCode::SERVICE_UNAVAILABLE; + } + StatusCode::OK +} + +async fn acquire_permit<'a>() -> SemaphorePermit<'a> { + return GLOBAL_CONTEXT_SEMAPHORE + .acquire() + .await + .expect("failed to acquire semaphore"); +} +fn lookup_chunk( + chunk_id: &String, + context: &RefMut<'_, (String, String), DownloadContext>, +) -> Result<(String, usize, usize), StatusCode> { + context + .chunk_lookup_table + .get(chunk_id) + .cloned() + .ok_or(StatusCode::NOT_FOUND) +} +async fn get_file_reader( + context: &mut RefMut<'_, (String, String), DownloadContext>, + relative_filename: String, + start: usize, + end: usize, +) -> Result, StatusCode> { + context + .backend + .reader( + &VersionFile { + relative_filename: relative_filename.clone(), + permission: 0, + size: 0, + }, + start as u64, + end as u64, + ) + .await + .map_err(|v| { + error!("reader error: {v:?}"); + StatusCode::INTERNAL_SERVER_ERROR + }) +} +async fn get_or_generate_context<'a>( + state: &Arc, + context_cache: &'a DashMap<(String, String), DownloadContext>, + game_id: String, + version_name: String, +) -> Result, StatusCode> { + let initialisation_data = state.token.get().ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + let key = (game_id.clone(), version_name.clone()); + + if let Some(context) = context_cache.get_mut(&key) { + Ok(context) + } else { + let permit = acquire_permit().await; + + // Check if it's been done while we've been sitting here + if let Some(already_done) = context_cache.get_mut(&key) { + Ok(already_done) + } else { + info!("generating context..."); + let context_result = create_download_context( + &*state.metadata_provider, + &*state.backend_factory, + initialisation_data, + game_id.clone(), + version_name.clone(), + ) + .await?; + + state.context_cache.insert(key.clone(), context_result); + + info!("continuing download"); + + drop(permit); + + Ok(context_cache.get_mut(&key).unwrap()) + } + } +} diff --git a/torrential/src/lib.rs b/torrential/src/lib.rs new file mode 100644 index 00000000..4c70554f --- /dev/null +++ b/torrential/src/lib.rs @@ -0,0 +1,17 @@ +use tokio::sync::Semaphore; +mod download; +pub mod handlers; +mod manifest; +mod remote; +pub mod state; +mod token; +mod util; + +pub use download::DownloadContext; +pub use download::{BackendFactory, DropBackendFactory}; +pub use remote::{ + DropLibraryProvider, DropContextProvider, LibraryConfigurationProvider, ContextProvider, +}; +pub use token::set_token; + +static GLOBAL_CONTEXT_SEMAPHORE: Semaphore = Semaphore::const_new(1); diff --git a/torrential/src/main.rs b/torrential/src/main.rs index afa4a7f2..b9fbb195 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -1,54 +1,21 @@ -use anyhow::Result; -use dashmap::{DashMap, mapref::one::RefMut}; -use droplet_rs::versions::types::{MinimumFileObject, VersionBackend, VersionFile}; -use reqwest::header; -use simple_logger::SimpleLogger; use std::{ - collections::HashMap, env::set_current_dir, path::PathBuf, str::FromStr as _, sync::Arc, - time::Instant, + env::{self, set_current_dir}, + sync::Arc, }; -use tokio_util::io::ReaderStream; use axum::{ - Json, Router, - body::Body, - extract::{Path, State}, - http::StatusCode, - response::{AppendHeaders, IntoResponse}, + Router, routing::{get, post}, }; -use log::{error, info}; -use serde::Deserialize; -use tokio::sync::{OnceCell, Semaphore, SemaphorePermit}; - -use crate::{ - download::create_download_context, - remote::{LibraryBackend, LibrarySource, fetch_library_sources}, +use dashmap::DashMap; +use log::info; +use simple_logger::SimpleLogger; +use tokio::sync::OnceCell; +use torrential::{ + DropBackendFactory, DropLibraryProvider, DropContextProvider, handlers, set_token, + state::AppState, }; - -mod download; -mod manifest; -mod remote; -mod util; - -static GLOBAL_CONTEXT_SEMAPHORE: Semaphore = Semaphore::const_new(1); - -struct DownloadContext { - chunk_lookup_table: HashMap, - backend: Box, - last_access: Instant, -} - -#[derive(Debug)] -struct AppInitData { - token: String, - libraries: HashMap, -} - -struct AppState { - token: OnceCell, - context_cache: DashMap<(String, String), DownloadContext>, -} +use url::Url; #[tokio::main] async fn main() { @@ -58,9 +25,15 @@ async fn main() { set_current_dir(working_directory).expect("failed to change working directory"); } + let remote_url = get_remote_url(); + let shared_state = Arc::new(AppState { token: OnceCell::new(), context_cache: DashMap::new(), + + metadata_provider: Arc::new(DropContextProvider::new(remote_url.clone())), + backend_factory: Arc::new(DropBackendFactory), + library_provider: Arc::new(DropLibraryProvider::new(remote_url)), }); let app = setup_app(shared_state); @@ -72,65 +45,19 @@ fn setup_app(shared_state: Arc) -> Router { Router::new() .route( "/api/v1/depot/{game_id}/{version_name}/{chunk_id}", - get(serve_file), + get(handlers::serve_file), ) .route("/token", post(set_token)) - .route("/healthcheck", get(healthcheck)) + .route("/healthcheck", get(handlers::healthcheck)) .with_state(shared_state) } + async fn serve(app: Router) -> Result<(), std::io::Error> { let listener = tokio::net::TcpListener::bind("0.0.0.0:5000").await.unwrap(); info!("started depot server"); axum::serve(listener, app).await } -async fn set_token( - State(state): State>, - Json(payload): Json, -) -> Result { - if check_token_exists(&state, &payload) { - return Ok(StatusCode::OK); - } - - let token = payload.token; - - let library_sources = fetch_library_sources(&token).await.map_err(|v| { - error!("{v:?}"); - StatusCode::INTERNAL_SERVER_ERROR - })?; - - let valid_library_sources = filter_library_sources(library_sources); - - set_generated_token(&state, token, valid_library_sources)?; - - info!("connected to drop server successfully"); - - Ok(StatusCode::OK) -} - -async fn serve_file( - State(state): State>, - Path((game_id, version_name, chunk_id)): Path<(String, String, String)>, -) -> Result { - let context_cache = &state.context_cache; - - let mut context = get_or_generate_context(&state, context_cache, game_id, version_name).await?; - context.last_access = Instant::now(); - - let (relative_filename, start, end) = lookup_chunk(&chunk_id, &context)?; - let reader = get_file_reader(&mut context, relative_filename, start, end).await?; - - let stream = ReaderStream::new(reader); - let body: Body = Body::from_stream(stream); - - let headers: AppendHeaders<[(header::HeaderName, String); 2]> = AppendHeaders([ - (header::CONTENT_TYPE, "application/octet-stream".to_owned()), - (header::CONTENT_LENGTH, (end - start).to_string()), - ]); - - Ok((headers, body)) -} - fn initialise_logger() { SimpleLogger::new() .with_level(log::LevelFilter::Info) @@ -138,140 +65,14 @@ fn initialise_logger() { .unwrap(); } -async fn acquire_permit<'a>() -> SemaphorePermit<'a> { - return GLOBAL_CONTEXT_SEMAPHORE - .acquire() - .await - .expect("failed to acquire semaphore") -} -fn lookup_chunk( - chunk_id: &String, - context: &RefMut<'_, (String, String), DownloadContext>, -) -> Result<(String, usize, usize), StatusCode> { - context - .chunk_lookup_table - .get(chunk_id) - .cloned() - .ok_or(StatusCode::NOT_FOUND) -} -async fn get_file_reader( - context: &mut RefMut<'_, (String, String), DownloadContext>, - relative_filename: String, - start: usize, - end: usize, -) -> Result, StatusCode> { - context - .backend - .reader( - &VersionFile { - relative_filename: relative_filename.clone(), - permission: 0, - size: 0, - }, - start as u64, - end as u64, - ) - .await - .map_err(|v| { - error!("reader error: {v:?}"); - StatusCode::INTERNAL_SERVER_ERROR - }) -} -async fn get_or_generate_context<'a>( - state: &Arc, - context_cache: &'a DashMap<(String, String), DownloadContext>, - game_id: String, - version_name: String, -) -> Result, StatusCode> { - let initialisation_data = state.token.get().ok_or(StatusCode::SERVICE_UNAVAILABLE)?; - let key = (game_id.clone(), version_name.clone()); - - if let Some(context) = context_cache.get_mut(&key) { - Ok(context) - } else { - let permit = acquire_permit().await; - - // Check if it's been done while we've been sitting here - if let Some(already_done) = context_cache.get_mut(&key) { - Ok(already_done) - } else { - info!("generating context..."); - let context_result = - create_download_context(initialisation_data, game_id.clone(), version_name.clone()) - .await?; - - state.context_cache.insert(key.clone(), context_result); - - info!("continuing download"); - - drop(permit); - - Ok(context_cache.get_mut(&key).unwrap()) - } - } -} - -#[derive(Deserialize)] -struct TokenPayload { - token: String, -} - -async fn healthcheck(State(state): State>) -> StatusCode { - let initialised = state.token.initialized(); - if !initialised { - return StatusCode::SERVICE_UNAVAILABLE; - } - StatusCode::OK -} - -fn check_token_exists(state: &Arc, payload: &TokenPayload) -> bool { - if let Some(existing_data) = state.token.get() { - assert!( - existing_data.token == payload.token, - "already set up but provided with a different token" - ); - return true; - } - false -} -fn filter_library_sources( - library_sources: Vec, -) -> HashMap { - library_sources - .into_iter() - .filter(|v| { - matches!( - v.backend, - remote::LibraryBackend::Filesystem | remote::LibraryBackend::FlatFilesystem - ) - }) - .map(|v| { - let path = PathBuf::from_str( - v.options - .as_object() - .unwrap() - .get("baseDir") - .unwrap() - .as_str() - .unwrap(), - ) - .unwrap(); - - (v.id, (path, v.backend)) - }) - .collect() -} -fn set_generated_token( - state: &Arc, - token: String, - libraries: HashMap, -) -> Result<(), StatusCode> { - state - .token - .set(AppInitData { token, libraries }) - .map_err(|err| { - error!("failed to set token: {err:?}"); - StatusCode::INTERNAL_SERVER_ERROR - })?; - Ok(()) +fn get_remote_url() -> Url { + let user_provided = env::var("DROP_SERVER_URL"); + let url = Url::parse( + user_provided + .as_ref() + .map_or("http://localhost:3000", |v| v), + ) + .expect("failed to parse URL"); + info!("using Drop server url {url}"); + url } diff --git a/torrential/src/remote.rs b/torrential/src/remote.rs index a8bf3da3..0b3a0beb 100644 --- a/torrential/src/remote.rs +++ b/torrential/src/remote.rs @@ -1,9 +1,8 @@ -use std::{env, sync::LazyLock}; - use anyhow::{Result, anyhow}; -use log::info; -use reqwest::{Client, ClientBuilder, StatusCode, Url}; +use async_trait::async_trait; +use reqwest::StatusCode; use serde::{Deserialize, Serialize}; +use url::Url; use crate::{manifest::DropletManifest, util::ErrorOption}; @@ -21,60 +20,6 @@ pub struct ContextQuery { version: String, } -static CLIENT: LazyLock = LazyLock::new(|| { - ClientBuilder::new() - .build() - .expect("failed to build client") -}); - -static REMOTE_URL: LazyLock = LazyLock::new(|| { - let user_provided = env::var("DROP_SERVER_URL"); - let url = Url::parse( - user_provided - .as_ref() - .map_or("http://localhost:3000", |v| v), - ) - .expect("failed to parse URL"); - info!("using Drop server url {url}"); - url -}); - -pub async fn fetch_download_context( - token: String, - game_id: String, - version_name: String, -) -> Result { - let context_response = CLIENT - .get(REMOTE_URL.join("/api/v1/admin/depot/context")?) - .query(&ContextQuery { - game: game_id, - version: version_name, - }) - .header("Authorization", format!("Bearer {token}")) - .send() - .await?; - - if !context_response.status().is_success() { - if context_response.status() == StatusCode::BAD_REQUEST { - return Err(StatusCode::NOT_FOUND.into()); - } - - return Err(anyhow!( - "Fetching context failed with non-success code: {}, {}", - context_response.status(), - context_response - .text() - .await - .unwrap_or("(failed to read body)".to_owned()) - ) - .into()); - } - - let context: ContextResponseBody = context_response.json().await?; - - Ok(context) -} - #[derive(Deserialize, Debug)] #[non_exhaustive] pub enum LibraryBackend { @@ -89,25 +34,110 @@ pub struct LibrarySource { pub backend: LibraryBackend, } -pub async fn fetch_library_sources(token: &String) -> Result> { - let source_response = CLIENT - .get(REMOTE_URL.join("/api/v1/admin/library/sources")?) - .header("Authorization", format!("Bearer {token}")) - .send() - .await?; - - if !source_response.status().is_success() { - return Err(anyhow!( - "Fetching library sources failed with non-success code: {}, {}", - source_response.status(), - source_response - .text() - .await - .unwrap_or("(failed to read body)".to_owned()) - )); - } - - let library_sources: Vec = source_response.json().await?; - - Ok(library_sources) +pub struct DropContextProvider { + client: reqwest::Client, + base_url: Url, +} +impl DropContextProvider { + pub fn new(url: Url) -> Self { + Self { + client: reqwest::Client::new(), + base_url: url, + } + } +} +#[async_trait] +impl ContextProvider for DropContextProvider { + async fn fetch_context( + &self, + token: String, + game_id: String, + version_name: String, + ) -> Result { + let context_response = self + .client + .get(self.base_url.join("/api/v1/admin/depot/context")?) + .query(&ContextQuery { + game: game_id, + version: version_name, + }) + .header("Authorization", format!("Bearer {token}")) + .send() + .await?; + + if !context_response.status().is_success() { + if context_response.status() == StatusCode::BAD_REQUEST { + return Err(StatusCode::NOT_FOUND.into()); + } + + return Err(anyhow!( + "Fetching context failed with non-success code: {}, {}", + context_response.status(), + context_response + .text() + .await + .unwrap_or("(failed to read body)".to_owned()) + ) + .into()); + } + + let context: ContextResponseBody = context_response.json().await?; + + Ok(context) + } +} + +#[async_trait] +pub trait ContextProvider: Send + Sync { + /// Fetches the manifest for a specific game version. + async fn fetch_context( + &self, + token: String, + game_id: String, + version_name: String, + ) -> Result; +} + +#[async_trait] +pub trait LibraryConfigurationProvider: Send + Sync { + async fn fetch_sources(&self, token: &String) -> anyhow::Result>; +} +pub struct DropLibraryProvider { + client: reqwest::Client, + base_url: Url, +} +impl DropLibraryProvider { + pub fn new(url: Url) -> Self { + Self { + client: reqwest::Client::new(), + base_url: url, + } + } +} + +#[async_trait] +impl LibraryConfigurationProvider for DropLibraryProvider { + async fn fetch_sources(&self, token: &String) -> anyhow::Result> { + let source_response = self + .client + .get(self.base_url.join("/api/v1/admin/library/sources")?) + .header("Authorization", format!("Bearer {token}")) + .send() + .await?; + + if !source_response.status().is_success() { + return Err(anyhow!( + "Fetching library sources failed with non-success code: {}, {}", + source_response.status(), + source_response + .text() + .await + .unwrap_or("(failed to read body)".to_owned()) + )); + } + + let library_sources: Vec = source_response.json().await?; + + Ok(library_sources) + } } diff --git a/torrential/src/state.rs b/torrential/src/state.rs new file mode 100644 index 00000000..e6c93947 --- /dev/null +++ b/torrential/src/state.rs @@ -0,0 +1,38 @@ +use std::{collections::HashMap, path::PathBuf, sync::Arc}; + +use dashmap::DashMap; +use tokio::sync::OnceCell; + +use crate::{ + BackendFactory, DownloadContext, LibraryConfigurationProvider, ContextProvider, + remote::LibraryBackend, +}; + +pub struct AppState { + pub token: OnceCell, + pub context_cache: DashMap<(String, String), DownloadContext>, + + pub metadata_provider: Arc, + pub backend_factory: Arc, + pub library_provider: Arc, +} + +#[derive(Debug)] +pub struct AppInitData { + token: String, + libraries: HashMap, +} +impl AppInitData { + pub fn new(token: String, libraries: HashMap) -> Self { + Self { token, libraries } + } + pub fn token(&self) -> String { + self.token.clone() + } + pub fn set_token(&mut self, token: String) { + self.token = token + } + pub fn libraries(&self) -> &HashMap { + &self.libraries + } +} diff --git a/torrential/src/token.rs b/torrential/src/token.rs new file mode 100644 index 00000000..971d73ae --- /dev/null +++ b/torrential/src/token.rs @@ -0,0 +1,94 @@ +use std::{collections::HashMap, path::PathBuf, str::FromStr, sync::Arc}; + +use axum::{Json, extract::State}; +use log::{error, info}; +use reqwest::StatusCode; +use serde::Deserialize; + +use crate::remote::{self, LibraryBackend, LibrarySource}; +use crate::state::{AppInitData, AppState}; + +#[derive(Deserialize)] +pub struct TokenPayload { + token: String, +} + +pub async fn set_token( + State(state): State>, + Json(payload): Json, +) -> Result { + if check_token_exists(&state, &payload) { + return Ok(StatusCode::OK); + } + + let token = payload.token; + + let library_sources = state + .library_provider + .fetch_sources(&token) + .await + .map_err(|v| { + error!("{v:?}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let valid_library_sources = filter_library_sources(library_sources); + + set_generated_token(&state, token, valid_library_sources)?; + + info!("connected to drop server successfully"); + + Ok(StatusCode::OK) +} + +fn check_token_exists(state: &Arc, payload: &TokenPayload) -> bool { + if let Some(existing_data) = state.token.get() { + assert!( + *existing_data.token() == payload.token, + "already set up but provided with a different token" + ); + return true; + } + false +} +fn filter_library_sources( + library_sources: Vec, +) -> HashMap { + library_sources + .into_iter() + .filter(|v| { + matches!( + v.backend, + remote::LibraryBackend::Filesystem | remote::LibraryBackend::FlatFilesystem + ) + }) + .map(|v| { + let path = PathBuf::from_str( + v.options + .as_object() + .unwrap() + .get("baseDir") + .unwrap() + .as_str() + .unwrap(), + ) + .unwrap(); + + (v.id, (path, v.backend)) + }) + .collect() +} +fn set_generated_token( + state: &Arc, + token: String, + libraries: HashMap, +) -> Result<(), StatusCode> { + state + .token + .set(AppInitData::new(token, libraries)) + .map_err(|err| { + error!("failed to set token: {err:?}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + Ok(()) +} From 89af632d846973abc0971d1ad1c1c84b9277c471 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Wed, 3 Dec 2025 13:56:29 +1100 Subject: [PATCH 10/21] feat: multipart downloads --- torrential/Cargo.lock | 14 ++++ torrential/Cargo.toml | 1 + torrential/src/handlers.rs | 125 ++++------------------------------ torrential/src/lib.rs | 1 + torrential/src/main.rs | 13 ++-- torrential/src/serve.rs | 135 +++++++++++++++++++++++++++++++++++++ 6 files changed, 173 insertions(+), 116 deletions(-) create mode 100644 torrential/src/serve.rs diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock index b3e798b7..24eadec9 100644 --- a/torrential/Cargo.lock +++ b/torrential/Cargo.lock @@ -504,6 +504,17 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -523,9 +534,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -1744,6 +1757,7 @@ dependencies = [ "criterion", "dashmap", "droplet-rs", + "futures-util", "log", "rand", "reqwest", diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml index a4a8e02d..88c95fe9 100644 --- a/torrential/Cargo.toml +++ b/torrential/Cargo.toml @@ -30,6 +30,7 @@ serde_json = "1.0.145" url = { version = "2.5.7", default-features = false } tokio-util = { version = "0.7.17", features = ["io"] } async-trait = "0.1.89" +futures-util = "0.3.31" [lints.clippy] pedantic = { level = "warn", priority = -1 } diff --git a/torrential/src/handlers.rs b/torrential/src/handlers.rs index b54bee6d..f7b0fe18 100644 --- a/torrential/src/handlers.rs +++ b/torrential/src/handlers.rs @@ -1,43 +1,10 @@ use std::sync::Arc; -use axum::{ - body::Body, - extract::{Path, State}, - response::{AppendHeaders, IntoResponse}, -}; -use dashmap::{DashMap, mapref::one::RefMut}; -use droplet_rs::versions::types::{MinimumFileObject, VersionFile}; -use log::{error, info}; -use reqwest::{StatusCode, header}; -use tokio::sync::SemaphorePermit; -use tokio_util::io::ReaderStream; +use axum::{Json, extract::State}; +use reqwest::StatusCode; +use serde::Deserialize; -use crate::{ - DownloadContext, GLOBAL_CONTEXT_SEMAPHORE, download::create_download_context, state::AppState, -}; - -pub async fn serve_file( - State(state): State>, - Path((game_id, version_name, chunk_id)): Path<(String, String, String)>, -) -> Result { - let context_cache = &state.context_cache; - - let mut context = get_or_generate_context(&state, context_cache, game_id, version_name).await?; - context.reset_last_access(); - - let (relative_filename, start, end) = lookup_chunk(&chunk_id, &context)?; - let reader = get_file_reader(&mut context, relative_filename, start, end).await?; - - let stream = ReaderStream::new(reader); - let body: Body = Body::from_stream(stream); - - let headers: AppendHeaders<[(header::HeaderName, String); 2]> = AppendHeaders([ - (header::CONTENT_TYPE, "application/octet-stream".to_owned()), - (header::CONTENT_LENGTH, (end - start).to_string()), - ]); - - Ok((headers, body)) -} +use crate::state::AppState; pub async fn healthcheck(State(state): State>) -> StatusCode { let initialised = state.token.initialized(); @@ -47,80 +14,16 @@ pub async fn healthcheck(State(state): State>) -> StatusCode { StatusCode::OK } -async fn acquire_permit<'a>() -> SemaphorePermit<'a> { - return GLOBAL_CONTEXT_SEMAPHORE - .acquire() - .await - .expect("failed to acquire semaphore"); -} -fn lookup_chunk( - chunk_id: &String, - context: &RefMut<'_, (String, String), DownloadContext>, -) -> Result<(String, usize, usize), StatusCode> { - context - .chunk_lookup_table - .get(chunk_id) - .cloned() - .ok_or(StatusCode::NOT_FOUND) -} -async fn get_file_reader( - context: &mut RefMut<'_, (String, String), DownloadContext>, - relative_filename: String, - start: usize, - end: usize, -) -> Result, StatusCode> { - context - .backend - .reader( - &VersionFile { - relative_filename: relative_filename.clone(), - permission: 0, - size: 0, - }, - start as u64, - end as u64, - ) - .await - .map_err(|v| { - error!("reader error: {v:?}"); - StatusCode::INTERNAL_SERVER_ERROR - }) -} -async fn get_or_generate_context<'a>( - state: &Arc, - context_cache: &'a DashMap<(String, String), DownloadContext>, +#[derive(Deserialize)] +pub struct InvalidateBody { game_id: String, version_name: String, -) -> Result, StatusCode> { - let initialisation_data = state.token.get().ok_or(StatusCode::SERVICE_UNAVAILABLE)?; - let key = (game_id.clone(), version_name.clone()); - - if let Some(context) = context_cache.get_mut(&key) { - Ok(context) - } else { - let permit = acquire_permit().await; - - // Check if it's been done while we've been sitting here - if let Some(already_done) = context_cache.get_mut(&key) { - Ok(already_done) - } else { - info!("generating context..."); - let context_result = create_download_context( - &*state.metadata_provider, - &*state.backend_factory, - initialisation_data, - game_id.clone(), - version_name.clone(), - ) - .await?; - - state.context_cache.insert(key.clone(), context_result); - - info!("continuing download"); - - drop(permit); - - Ok(context_cache.get_mut(&key).unwrap()) - } - } +} + +pub async fn invalidate( + State(state): State>, + Json(payload): Json, +) -> StatusCode { + state.context_cache.remove(&(payload.game_id, payload.version_name)); + StatusCode::OK } diff --git a/torrential/src/lib.rs b/torrential/src/lib.rs index 4c70554f..4a28441b 100644 --- a/torrential/src/lib.rs +++ b/torrential/src/lib.rs @@ -1,5 +1,6 @@ use tokio::sync::Semaphore; mod download; +pub mod serve; pub mod handlers; mod manifest; mod remote; diff --git a/torrential/src/main.rs b/torrential/src/main.rs index b9fbb195..c1ac1b63 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -10,10 +10,9 @@ use axum::{ use dashmap::DashMap; use log::info; use simple_logger::SimpleLogger; -use tokio::sync::OnceCell; +use tokio::{runtime::Handle, sync::OnceCell}; use torrential::{ - DropBackendFactory, DropLibraryProvider, DropContextProvider, handlers, set_token, - state::AppState, + DropBackendFactory, DropContextProvider, DropLibraryProvider, handlers, serve, set_token, state::AppState }; use url::Url; @@ -25,6 +24,9 @@ async fn main() { set_current_dir(working_directory).expect("failed to change working directory"); } + let metrics = Handle::current().metrics(); + info!("using {} threads", metrics.num_workers()); + let remote_url = get_remote_url(); let shared_state = Arc::new(AppState { @@ -44,11 +46,12 @@ async fn main() { fn setup_app(shared_state: Arc) -> Router { Router::new() .route( - "/api/v1/depot/{game_id}/{version_name}/{chunk_id}", - get(handlers::serve_file), + "/api/v1/depot/{game_id}/{version_name}/{*chunk_ids}", + get(serve::serve_file), ) .route("/token", post(set_token)) .route("/healthcheck", get(handlers::healthcheck)) + .route("/invalid", post(handlers::invalidate)) .with_state(shared_state) } diff --git a/torrential/src/serve.rs b/torrential/src/serve.rs new file mode 100644 index 00000000..b16c9619 --- /dev/null +++ b/torrential/src/serve.rs @@ -0,0 +1,135 @@ +use std::sync::Arc; + +use axum::{ + body::Body, + extract::{Path, State}, + http::HeaderMap, + response::{AppendHeaders, IntoResponse}, +}; +use dashmap::{DashMap, mapref::one::RefMut}; +use droplet_rs::versions::types::{MinimumFileObject, VersionFile}; +use log::{error, info}; +use reqwest::{StatusCode, header}; +use tokio::sync::SemaphorePermit; +use tokio_util::io::ReaderStream; +use futures_util::{StreamExt as _, stream}; + + +use crate::{ + DownloadContext, GLOBAL_CONTEXT_SEMAPHORE, download::create_download_context, state::AppState, +}; + +pub async fn serve_file( + State(state): State>, + Path((game_id, version_name, chunk_ids)): Path<(String, String, String)>, +) -> Result { + let context_cache = &state.context_cache; + + let mut context = get_or_generate_context(&state, context_cache, game_id, version_name).await?; + context.reset_last_access(); + + let chunk_ids = chunk_ids.split("/").collect::>(); + let mut streams = Vec::with_capacity(chunk_ids.len()); + let mut content_lengths = Vec::with_capacity(chunk_ids.len()); + let mut total_size = 0; + for chunk_id in chunk_ids { + let (relative_filename, start, end) = lookup_chunk(chunk_id, &context)?; + let reader = get_file_reader(&mut context, relative_filename, start, end).await?; + + let stream = ReaderStream::new(reader); + streams.push(stream); + content_lengths.push((end - start).to_string()); + + total_size += end - start; + } + + let stream = stream::iter(streams).flatten(); + let body: Body = Body::from_stream(stream); + + let mut headers = HeaderMap::new(); + headers.insert("Content-Type", "application/octet-stream".parse().unwrap()); + headers.insert("Content-Length", total_size.to_string().parse().unwrap()); + headers.insert( + "Content-Lengths", + content_lengths.join(",").parse().unwrap(), + ); + + Ok((headers, body)) +} +async fn acquire_permit<'a>() -> SemaphorePermit<'a> { + return GLOBAL_CONTEXT_SEMAPHORE + .acquire() + .await + .expect("failed to acquire semaphore"); +} +fn lookup_chunk( + chunk_id: &str, + context: &RefMut<'_, (String, String), DownloadContext>, +) -> Result<(String, usize, usize), StatusCode> { + context + .chunk_lookup_table + .get(chunk_id) + .cloned() + .ok_or(StatusCode::NOT_FOUND) +} +async fn get_file_reader( + context: &mut RefMut<'_, (String, String), DownloadContext>, + relative_filename: String, + start: usize, + end: usize, +) -> Result, StatusCode> { + context + .backend + .reader( + &VersionFile { + relative_filename: relative_filename.clone(), + permission: 0, + size: 0, + }, + start as u64, + end as u64, + ) + .await + .map_err(|v| { + error!("reader error: {v:?}"); + StatusCode::INTERNAL_SERVER_ERROR + }) +} +async fn get_or_generate_context<'a>( + state: &Arc, + context_cache: &'a DashMap<(String, String), DownloadContext>, + game_id: String, + version_name: String, +) -> Result, StatusCode> { + let initialisation_data = state.token.get().ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + let key = (game_id.clone(), version_name.clone()); + + if let Some(context) = context_cache.get_mut(&key) { + Ok(context) + } else { + let permit = acquire_permit().await; + + // Check if it's been done while we've been sitting here + if let Some(already_done) = context_cache.get_mut(&key) { + Ok(already_done) + } else { + info!("generating context for {}...", game_id); + let context_result = create_download_context( + &*state.metadata_provider, + &*state.backend_factory, + initialisation_data, + game_id.clone(), + version_name.clone(), + ) + .await?; + + state.context_cache.insert(key.clone(), context_result); + + info!("continuing download for {}", game_id); + + drop(permit); + + Ok(context_cache.get_mut(&key).unwrap()) + } + } +} From 0d4c1e516fe7930d93bc2699bf2e353bbd75d3e2 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 20 Dec 2025 01:11:05 +1100 Subject: [PATCH 11/21] feat: depot API --- torrential/Cargo.lock | 199 ++++++++++++++++++++++++++++++++--- torrential/Cargo.toml | 5 +- torrential/docs/protocol.md | 8 ++ torrential/docs/structure.md | 6 ++ torrential/src/download.rs | 95 +++++++---------- torrential/src/handlers.rs | 118 +++++++++++++++++++-- torrential/src/lib.rs | 5 - torrential/src/main.rs | 33 ++---- torrential/src/manifest.rs | 11 -- torrential/src/remote.rs | 195 ++++++++++++++++------------------ torrential/src/serve.rs | 81 ++++++++------ torrential/src/state.rs | 32 ++---- torrential/src/token.rs | 55 ++-------- 13 files changed, 514 insertions(+), 329 deletions(-) create mode 100644 torrential/docs/protocol.md create mode 100644 torrential/docs/structure.md delete mode 100644 torrential/src/manifest.rs diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock index 24eadec9..dab0e33b 100644 --- a/torrential/Cargo.lock +++ b/torrential/Cargo.lock @@ -2,6 +2,17 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -192,6 +203,15 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -259,6 +279,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.53" @@ -293,6 +323,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "criterion" version = "0.8.0" @@ -360,6 +399,25 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -417,6 +475,16 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -430,19 +498,24 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.9.2" -source = "git+https://github.com/Drop-OSS/droplet-rs.git#d8f37886d479d8d9fec7e51523628e863306dddb" +version = "0.12.2" +source = "git+https://github.com/Drop-OSS/droplet-rs.git#05f7027b362fcc72b7cc621df8b5b0850b6cf082" dependencies = [ "anyhow", "async-trait", "dyn-clone", + "futures", + "getrandom 0.3.4", "hex", + "humansize", "rcgen", "ring", + "serde", + "serde_json", + "sha2", "time", - "time-macros", "tokio", - "webpki", + "uuid", "x509-parser 0.17.0", ] @@ -489,6 +562,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -496,6 +584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -504,6 +593,23 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + [[package]] name = "futures-macro" version = "0.3.31" @@ -533,14 +639,28 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", + "futures-io", "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.16" @@ -636,6 +756,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "hyper" version = "1.8.1" @@ -801,6 +930,15 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -854,6 +992,12 @@ version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -1487,6 +1631,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1751,10 +1906,13 @@ dependencies = [ name = "torrential" version = "0.1.0" dependencies = [ + "aes", "anyhow", "async-trait", "axum", + "bytes", "criterion", + "ctr", "dashmap", "droplet-rs", "futures-util", @@ -1842,6 +2000,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + [[package]] name = "unicode-ident" version = "1.0.22" @@ -1872,6 +2036,23 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -1984,16 +2165,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki-roots" version = "1.0.4" diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml index 88c95fe9..8ae8aba4 100644 --- a/torrential/Cargo.toml +++ b/torrential/Cargo.toml @@ -23,7 +23,7 @@ simple_logger = { version = "5.1.0", default-features = false, features = [ "colors", ] } tokio = { version = "*", features = ["rt-multi-thread", "sync"] } -droplet-rs = { git = "https://github.com/Drop-OSS/droplet-rs.git" } +droplet-rs = { git="https://github.com/Drop-OSS/droplet-rs.git" } dashmap = "6.1.0" anyhow = "1.0.100" serde_json = "1.0.145" @@ -31,6 +31,9 @@ url = { version = "2.5.7", default-features = false } tokio-util = { version = "0.7.17", features = ["io"] } async-trait = "0.1.89" futures-util = "0.3.31" +ctr = "0.9.2" +aes = "0.8.4" +bytes = "*" [lints.clippy] pedantic = { level = "warn", priority = -1 } diff --git a/torrential/docs/protocol.md b/torrential/docs/protocol.md new file mode 100644 index 00000000..f2e2bfa1 --- /dev/null +++ b/torrential/docs/protocol.md @@ -0,0 +1,8 @@ +# Protocol + +`torrential` implements the Depot API as defined by https://developer.droposs.org/web/depot, and it prefixed with `/api/v1/depot` for NGINX proxying. + +It also has the following endpoints, only accessible by the Drop server for security reasons: + - `/key` for sharing the authentication key from the Drop server to torrential + - `/invalidate` for pre-emptivel clearing the download context cache. Contexts are automatically cleared regardless, so this endpoint failing is not a hard error on the Drop side + - `/healthcheck`. Does healthcheck. \ No newline at end of file diff --git a/torrential/docs/structure.md b/torrential/docs/structure.md new file mode 100644 index 00000000..0985d4f3 --- /dev/null +++ b/torrential/docs/structure.md @@ -0,0 +1,6 @@ +# Structure +Torrential is a typical Rust project. Source files are in `src/`. + +`handlers.rs` contains most non-download endpoint handlers. `serve.rs` contains the download endpoint handler. + +`remote.rs` handles communciating with the Drop server. \ No newline at end of file diff --git a/torrential/src/download.rs b/torrential/src/download.rs index 0f3f8b72..7c62ef95 100644 --- a/torrential/src/download.rs +++ b/torrential/src/download.rs @@ -1,16 +1,21 @@ -use std::{collections::HashMap, hash::RandomState, time::Instant}; +use std::{path::PathBuf, time::Instant}; -use droplet_rs::versions::{create_backend_constructor, types::VersionBackend}; +use anyhow::anyhow; +use droplet_rs::{ + manifest::Manifest, + versions::{create_backend_constructor, types::VersionBackend}, +}; +use log::{info, warn}; use reqwest::StatusCode; use crate::{ - remote::{ContextResponseBody, LibraryBackend, ContextProvider}, + remote::{LibraryBackend, VersionResponseBody, fetch_version_data}, state::AppInitData, util::ErrorOption, }; pub struct DownloadContext { - pub(crate) chunk_lookup_table: HashMap, + pub(crate) manifest: Manifest, pub(crate) backend: Box, last_access: Instant, } @@ -24,33 +29,16 @@ impl DownloadContext { } pub async fn create_download_context( - metadata_provider: &dyn ContextProvider, - backend_factory: &dyn BackendFactory, init_data: &AppInitData, game_id: String, version_name: String, ) -> Result { - let context = metadata_provider - .fetch_context(init_data.token(), game_id, version_name.clone()) - .await?; + let version_data = fetch_version_data(init_data, game_id, version_name.clone()).await?; - let backend = backend_factory.create_backend(init_data, &context, &version_name)?; - - let mut chunk_lookup_table = HashMap::with_capacity_and_hasher( - context.manifest.values().map(|v| v.ids.len()).sum(), - RandomState::default(), - ); - - for (path, file_chunks) in context.manifest { - let mut start = 0; - for (chunk, length) in file_chunks.ids.into_iter().zip(file_chunks.lengths) { - chunk_lookup_table.insert(chunk, (path.clone(), start, start + length)); - start += length; - } - } + let backend = create_backend(&version_data)?; let download_context = DownloadContext { - chunk_lookup_table, + manifest: version_data.manifest, backend, last_access: Instant::now(), }; @@ -58,39 +46,34 @@ pub async fn create_download_context( Ok(download_context) } -pub trait BackendFactory: Send + Sync { - fn create_backend( - &self, - init_data: &AppInitData, - context: &ContextResponseBody, - version_name: &String, - ) -> Result, StatusCode>; -} +fn create_backend( + version_data: &VersionResponseBody, +) -> Result, StatusCode> { + let base_path = version_data + .library + .options + .get("baseDir") + .unwrap() + .as_str() + .unwrap(); -pub struct DropBackendFactory; -impl BackendFactory for DropBackendFactory { - fn create_backend( - &self, - init_data: &AppInitData, - context: &ContextResponseBody, - version_name: &String, - ) -> Result, StatusCode> { - let (version_path, backend) = init_data - .libraries() - .get(&context.library_id) - .ok_or(StatusCode::NOT_FOUND)?; + let version_path = PathBuf::from(base_path); + let version_path = version_path.join(version_data.library_path.clone()); + let version_path = match version_data.library.backend { + LibraryBackend::Filesystem => version_path.join(version_data.version_path.clone()), + LibraryBackend::FlatFilesystem => version_path, + }; - let version_path = version_path.join(&context.library_path); - let version_path = match backend { - LibraryBackend::Filesystem => version_path.join(version_name), - LibraryBackend::FlatFilesystem => version_path, - }; - - let backend = - create_backend_constructor(&version_path).ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; - - // TODO: Not eat this error - let backend = backend().map_err(|_e| StatusCode::INTERNAL_SERVER_ERROR)?; - Ok(backend) + if !version_path.exists() { + warn!("{} path doesn't exist for version", version_path.display()); + return Err(StatusCode::INTERNAL_SERVER_ERROR); } + + let backend = + create_backend_constructor(&version_path).ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + + let backend = backend() + .inspect_err(|err| warn!("failed to create version backend: {:?}", err)) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(backend) } diff --git a/torrential/src/handlers.rs b/torrential/src/handlers.rs index f7b0fe18..636a8de0 100644 --- a/torrential/src/handlers.rs +++ b/torrential/src/handlers.rs @@ -1,10 +1,22 @@ -use std::sync::Arc; +use std::{ + collections::HashMap, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + task::Poll, +}; -use axum::{Json, extract::State}; -use reqwest::StatusCode; -use serde::Deserialize; +use axum::{Json, body::Body, extract::State, http::{HeaderMap, HeaderValue}, response::IntoResponse}; +use bytes::BufMut; +use reqwest::{StatusCode, header::CONTENT_TYPE}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use std::io::Write; +use tokio::io::AsyncRead; +use tokio_util::io::ReaderStream; -use crate::state::AppState; +use crate::{remote::fetch_instance_games, state::AppState}; pub async fn healthcheck(State(state): State>) -> StatusCode { let initialised = state.token.initialized(); @@ -16,14 +28,104 @@ pub async fn healthcheck(State(state): State>) -> StatusCode { #[derive(Deserialize)] pub struct InvalidateBody { - game_id: String, - version_name: String, + game: String, + version: String, } pub async fn invalidate( State(state): State>, Json(payload): Json, ) -> StatusCode { - state.context_cache.remove(&(payload.game_id, payload.version_name)); + state.context_cache.remove(&(payload.game, payload.version)); StatusCode::OK } + +struct SpeedtestStream { + remaining: usize, +} + +impl SpeedtestStream { + pub fn new() -> Self { + SpeedtestStream { + remaining: 1024 * 1024 * 50, + } + } + fn content_length(&self) -> usize { + self.remaining + } +} +const ZERO: [u8; 1024] = [0u8; _]; +impl AsyncRead for SpeedtestStream { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + if self.remaining > 0 { + let mut writer = buf.writer(); + + let amount = writer.write(&ZERO); + match amount { + Ok(amount) => self.remaining -= amount, + Err(err) => return Poll::Ready(Err(err)), + }; + }; + return Poll::Ready(Ok(())); + } +} + +pub async fn speedtest() -> Result { + let speedtest = SpeedtestStream::new(); + let ct = speedtest.content_length(); + let speedtest_stream = ReaderStream::new(speedtest); + let body = Body::from_stream(speedtest_stream); + + let mut headers = HeaderMap::new(); + headers.insert("Content-Type", "application/octet-stream".parse().unwrap()); + headers.insert("Content-Length", ct.into()); + + Ok((headers, body)) +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct GameData { + version_id: String, + compression: String, +} + +#[derive(Serialize)] +struct Manifest { + content: HashMap>, +} + +pub async fn manifest( + State(state): State>, +) -> Result { + let games = fetch_instance_games( + state + .token + .get() + .ok_or(StatusCode::from_u16(503).unwrap())?, + ) + .await?; + + let mut content = HashMap::new(); + for game in games { + content.insert( + game.id, + game.versions + .into_iter() + .map(|v| GameData { + version_id: v.version_id, + compression: "none".to_owned(), + }) + .collect::>(), + ); + } + + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + + Ok((headers, json!(Manifest { content }).to_string())) +} diff --git a/torrential/src/lib.rs b/torrential/src/lib.rs index 4a28441b..e97b94f1 100644 --- a/torrential/src/lib.rs +++ b/torrential/src/lib.rs @@ -2,17 +2,12 @@ use tokio::sync::Semaphore; mod download; pub mod serve; pub mod handlers; -mod manifest; mod remote; pub mod state; mod token; mod util; pub use download::DownloadContext; -pub use download::{BackendFactory, DropBackendFactory}; -pub use remote::{ - DropLibraryProvider, DropContextProvider, LibraryConfigurationProvider, ContextProvider, -}; pub use token::set_token; static GLOBAL_CONTEXT_SEMAPHORE: Semaphore = Semaphore::const_new(1); diff --git a/torrential/src/main.rs b/torrential/src/main.rs index c1ac1b63..14fd39c3 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -4,16 +4,14 @@ use std::{ }; use axum::{ - Router, + Router, handler, routing::{get, post}, }; use dashmap::DashMap; use log::info; use simple_logger::SimpleLogger; use tokio::{runtime::Handle, sync::OnceCell}; -use torrential::{ - DropBackendFactory, DropContextProvider, DropLibraryProvider, handlers, serve, set_token, state::AppState -}; +use torrential::{handlers, serve, set_token, state::AppState}; use url::Url; #[tokio::main] @@ -21,21 +19,16 @@ async fn main() { initialise_logger(); if let Ok(working_directory) = std::env::var("WORKING_DIRECTORY") { + info!("moving to working directory {}", working_directory); set_current_dir(working_directory).expect("failed to change working directory"); } let metrics = Handle::current().metrics(); info!("using {} threads", metrics.num_workers()); - let remote_url = get_remote_url(); - let shared_state = Arc::new(AppState { token: OnceCell::new(), context_cache: DashMap::new(), - - metadata_provider: Arc::new(DropContextProvider::new(remote_url.clone())), - backend_factory: Arc::new(DropBackendFactory), - library_provider: Arc::new(DropLibraryProvider::new(remote_url)), }); let app = setup_app(shared_state); @@ -46,12 +39,14 @@ async fn main() { fn setup_app(shared_state: Arc) -> Router { Router::new() .route( - "/api/v1/depot/{game_id}/{version_name}/{*chunk_ids}", + "/api/v1/depot/content/{game_id}/{version_name}/{chunk_id}", get(serve::serve_file), ) - .route("/token", post(set_token)) + .route("/api/v1/depot/manifest.json", get(handlers::manifest)) + .route("/api/v1/depot/speedtest", get(handlers::speedtest)) + .route("/key", post(set_token)) .route("/healthcheck", get(handlers::healthcheck)) - .route("/invalid", post(handlers::invalidate)) + .route("/invalidate", post(handlers::invalidate)) .with_state(shared_state) } @@ -67,15 +62,3 @@ fn initialise_logger() { .init() .unwrap(); } - -fn get_remote_url() -> Url { - let user_provided = env::var("DROP_SERVER_URL"); - let url = Url::parse( - user_provided - .as_ref() - .map_or("http://localhost:3000", |v| v), - ) - .expect("failed to parse URL"); - info!("using Drop server url {url}"); - url -} diff --git a/torrential/src/manifest.rs b/torrential/src/manifest.rs deleted file mode 100644 index f928f23c..00000000 --- a/torrential/src/manifest.rs +++ /dev/null @@ -1,11 +0,0 @@ -use std::collections::HashMap; - -use serde::Deserialize; - -#[derive(Deserialize)] -pub struct DropChunk { - pub ids: Vec, - pub lengths: Vec, -} - -pub type DropletManifest = HashMap; diff --git a/torrential/src/remote.rs b/torrential/src/remote.rs index 0b3a0beb..61838bae 100644 --- a/torrential/src/remote.rs +++ b/torrential/src/remote.rs @@ -1,21 +1,44 @@ +use std::{env, sync::LazyLock}; + use anyhow::{Result, anyhow}; use async_trait::async_trait; -use reqwest::StatusCode; +use droplet_rs::manifest::Manifest; +use log::info; +use reqwest::{Client, ClientBuilder, StatusCode}; use serde::{Deserialize, Serialize}; use url::Url; -use crate::{manifest::DropletManifest, util::ErrorOption}; +use crate::{state::AppInitData, util::ErrorOption}; + +static CLIENT: LazyLock = LazyLock::new(|| { + ClientBuilder::new() + .build() + .expect("failed to build client") +}); + +static REMOTE_URL: LazyLock = LazyLock::new(|| { + let user_provided = env::var("DROP_SERVER_URL"); + let url = Url::parse( + user_provided + .as_ref() + .map_or("http://localhost:3000", |v| v), + ) + .expect("failed to parse URL"); + info!("using Drop server url {}", url); + url +}); #[derive(Deserialize)] #[serde(rename_all = "camelCase")] -pub struct ContextResponseBody { - pub manifest: DropletManifest, - pub library_id: String, +pub struct VersionResponseBody { + pub manifest: Manifest, + pub library: LibrarySource, pub library_path: String, + pub version_path: String, } #[derive(Serialize)] -pub struct ContextQuery { +pub struct VersionQuery { game: String, version: String, } @@ -34,110 +57,78 @@ pub struct LibrarySource { pub backend: LibraryBackend, } -pub struct DropContextProvider { - client: reqwest::Client, - base_url: Url, -} -impl DropContextProvider { - pub fn new(url: Url) -> Self { - Self { - client: reqwest::Client::new(), - base_url: url, - } - } -} -#[async_trait] -impl ContextProvider for DropContextProvider { - async fn fetch_context( - &self, - token: String, - game_id: String, - version_name: String, - ) -> Result { - let context_response = self - .client - .get(self.base_url.join("/api/v1/admin/depot/context")?) - .query(&ContextQuery { - game: game_id, - version: version_name, - }) - .header("Authorization", format!("Bearer {token}")) - .send() - .await?; +pub async fn fetch_version_data( + init_data: &AppInitData, + game_id: String, + version_id: String, +) -> Result { + let version_data_response = CLIENT + .get(REMOTE_URL.join("/api/v1/admin/depot/manifest")?) + .query(&VersionQuery { + game: game_id, + version: version_id, + }) + .header("Authorization", format!("Bearer {}", init_data.key)) + .send() + .await?; - if !context_response.status().is_success() { - if context_response.status() == StatusCode::BAD_REQUEST { - return Err(StatusCode::NOT_FOUND.into()); - } - - return Err(anyhow!( - "Fetching context failed with non-success code: {}, {}", - context_response.status(), - context_response - .text() - .await - .unwrap_or("(failed to read body)".to_owned()) - ) - .into()); + if !version_data_response.status().is_success() { + if version_data_response.status() == StatusCode::BAD_REQUEST { + return Err(StatusCode::NOT_FOUND.into()); } - let context: ContextResponseBody = context_response.json().await?; - - Ok(context) + return Err(anyhow!( + "Fetching context failed with non-success code: {}, {}", + version_data_response.status(), + version_data_response + .text() + .await + .unwrap_or("(failed to read body)".to_owned()) + ) + .into()); } + + let version_data: VersionResponseBody = version_data_response.json().await?; + + Ok(version_data) } -#[async_trait] -pub trait ContextProvider: Send + Sync { - /// Fetches the manifest for a specific game version. - async fn fetch_context( - &self, - token: String, - game_id: String, - version_name: String, - ) -> Result; +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SkeletonVersion { + pub version_id: String, } -#[async_trait] -pub trait LibraryConfigurationProvider: Send + Sync { - async fn fetch_sources(&self, token: &String) -> anyhow::Result>; +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SkeletonGame { + pub id: String, + pub versions: Vec, } -pub struct DropLibraryProvider { - client: reqwest::Client, - base_url: Url, -} -impl DropLibraryProvider { - pub fn new(url: Url) -> Self { - Self { - client: reqwest::Client::new(), - base_url: url, - } - } -} - -#[async_trait] -impl LibraryConfigurationProvider for DropLibraryProvider { - async fn fetch_sources(&self, token: &String) -> anyhow::Result> { - let source_response = self - .client - .get(self.base_url.join("/api/v1/admin/library/sources")?) - .header("Authorization", format!("Bearer {token}")) - .send() - .await?; - - if !source_response.status().is_success() { - return Err(anyhow!( - "Fetching library sources failed with non-success code: {}, {}", - source_response.status(), - source_response - .text() - .await - .unwrap_or("(failed to read body)".to_owned()) - )); - } - - let library_sources: Vec = source_response.json().await?; - - Ok(library_sources) + +pub async fn fetch_instance_games( + init_data: &AppInitData, +) -> Result, ErrorOption> { + let context_response = CLIENT + .get(REMOTE_URL.join("/api/v1/admin/depot/versions")?) + .header("Authorization", format!("Bearer {}", init_data.key)) + .send() + .await?; + + if !context_response.status().is_success() { + + return Err(anyhow!( + "Fetching instance games failed with non-success code: {}, {}", + context_response.status(), + context_response + .text() + .await + .unwrap_or("(failed to read body)".to_owned()) + ) + .into()); } + + let games: Vec = context_response.json().await?; + + Ok(games) } diff --git a/torrential/src/serve.rs b/torrential/src/serve.rs index b16c9619..20bf3b5a 100644 --- a/torrential/src/serve.rs +++ b/torrential/src/serve.rs @@ -1,58 +1,72 @@ -use std::sync::Arc; +use std::{io::Error, rc::Rc, sync::Arc}; +use aes::cipher::{KeyIvInit, StreamCipher}; use axum::{ body::Body, extract::{Path, State}, http::HeaderMap, response::{AppendHeaders, IntoResponse}, }; +use bytes::Bytes; use dashmap::{DashMap, mapref::one::RefMut}; -use droplet_rs::versions::types::{MinimumFileObject, VersionFile}; +use droplet_rs::{ + manifest::ChunkData, + versions::types::{MinimumFileObject, VersionFile}, +}; +use futures_util::{StreamExt, stream}; use log::{error, info}; use reqwest::{StatusCode, header}; use tokio::sync::SemaphorePermit; use tokio_util::io::ReaderStream; -use futures_util::{StreamExt as _, stream}; - use crate::{ DownloadContext, GLOBAL_CONTEXT_SEMAPHORE, download::create_download_context, state::AppState, }; +type Aes128Ctr64LE = ctr::Ctr64LE; + pub async fn serve_file( State(state): State>, - Path((game_id, version_name, chunk_ids)): Path<(String, String, String)>, + Path((game_id, version_name, chunk_id)): Path<(String, String, String)>, ) -> Result { let context_cache = &state.context_cache; - let mut context = get_or_generate_context(&state, context_cache, game_id, version_name).await?; + let mut context = get_or_create_context(&state, context_cache, game_id, version_name).await?; context.reset_last_access(); - let chunk_ids = chunk_ids.split("/").collect::>(); - let mut streams = Vec::with_capacity(chunk_ids.len()); - let mut content_lengths = Vec::with_capacity(chunk_ids.len()); - let mut total_size = 0; - for chunk_id in chunk_ids { - let (relative_filename, start, end) = lookup_chunk(chunk_id, &context)?; - let reader = get_file_reader(&mut context, relative_filename, start, end).await?; + let chunk_data = lookup_chunk(&chunk_id, &context)?; + let mut streams = Vec::with_capacity(chunk_data.files.len()); + let mut content_length = 0; + + for file_entry in &chunk_data.files { + let reader = get_file_reader( + &mut context, + file_entry.filename.clone(), + file_entry.start, + file_entry.start + file_entry.length, + ) + .await?; let stream = ReaderStream::new(reader); streams.push(stream); - content_lengths.push((end - start).to_string()); - - total_size += end - start; + content_length += file_entry.length; } let stream = stream::iter(streams).flatten(); - let body: Body = Body::from_stream(stream); + let mut cipher = Aes128Ctr64LE::new(&context.manifest.key.into(), &chunk_data.iv.into()); + let encrypted_stream = stream.chunks(3).map(move |raw| -> Result { + let data: Result, Error> = raw.into_iter().collect(); + let mut data = data?.concat(); + + cipher.apply_keystream(&mut data); + + Ok(data.into()) + }); + let body: Body = Body::from_stream(encrypted_stream); let mut headers = HeaderMap::new(); headers.insert("Content-Type", "application/octet-stream".parse().unwrap()); - headers.insert("Content-Length", total_size.to_string().parse().unwrap()); - headers.insert( - "Content-Lengths", - content_lengths.join(",").parse().unwrap(), - ); + headers.insert("Content-Length", content_length.into()); Ok((headers, body)) } @@ -62,12 +76,16 @@ async fn acquire_permit<'a>() -> SemaphorePermit<'a> { .await .expect("failed to acquire semaphore"); } +/** + * Needs to be cloned for reference reasons + */ fn lookup_chunk( chunk_id: &str, context: &RefMut<'_, (String, String), DownloadContext>, -) -> Result<(String, usize, usize), StatusCode> { +) -> Result { context - .chunk_lookup_table + .manifest + .chunks .get(chunk_id) .cloned() .ok_or(StatusCode::NOT_FOUND) @@ -91,11 +109,11 @@ async fn get_file_reader( ) .await .map_err(|v| { - error!("reader error: {v:?}"); + error!("reader error for '{}': {v:?}", relative_filename); StatusCode::INTERNAL_SERVER_ERROR }) } -async fn get_or_generate_context<'a>( +async fn get_or_create_context<'a>( state: &Arc, context_cache: &'a DashMap<(String, String), DownloadContext>, game_id: String, @@ -114,14 +132,9 @@ async fn get_or_generate_context<'a>( Ok(already_done) } else { info!("generating context for {}...", game_id); - let context_result = create_download_context( - &*state.metadata_provider, - &*state.backend_factory, - initialisation_data, - game_id.clone(), - version_name.clone(), - ) - .await?; + let context_result = + create_download_context(initialisation_data, game_id.clone(), version_name.clone()) + .await?; state.context_cache.insert(key.clone(), context_result); diff --git a/torrential/src/state.rs b/torrential/src/state.rs index e6c93947..3cfd35c7 100644 --- a/torrential/src/state.rs +++ b/torrential/src/state.rs @@ -1,38 +1,18 @@ -use std::{collections::HashMap, path::PathBuf, sync::Arc}; +use std::{collections::HashMap, path::PathBuf}; use dashmap::DashMap; use tokio::sync::OnceCell; -use crate::{ - BackendFactory, DownloadContext, LibraryConfigurationProvider, ContextProvider, - remote::LibraryBackend, -}; +use crate:: + DownloadContext +; pub struct AppState { pub token: OnceCell, pub context_cache: DashMap<(String, String), DownloadContext>, - - pub metadata_provider: Arc, - pub backend_factory: Arc, - pub library_provider: Arc, } #[derive(Debug)] pub struct AppInitData { - token: String, - libraries: HashMap, -} -impl AppInitData { - pub fn new(token: String, libraries: HashMap) -> Self { - Self { token, libraries } - } - pub fn token(&self) -> String { - self.token.clone() - } - pub fn set_token(&mut self, token: String) { - self.token = token - } - pub fn libraries(&self) -> &HashMap { - &self.libraries - } -} + pub key: String, +} \ No newline at end of file diff --git a/torrential/src/token.rs b/torrential/src/token.rs index 971d73ae..2b46ccfc 100644 --- a/torrential/src/token.rs +++ b/torrential/src/token.rs @@ -1,16 +1,15 @@ -use std::{collections::HashMap, path::PathBuf, str::FromStr, sync::Arc}; +use std::sync::Arc; use axum::{Json, extract::State}; use log::{error, info}; use reqwest::StatusCode; use serde::Deserialize; -use crate::remote::{self, LibraryBackend, LibrarySource}; use crate::state::{AppInitData, AppState}; #[derive(Deserialize)] pub struct TokenPayload { - token: String, + key: String, } pub async fn set_token( @@ -21,20 +20,9 @@ pub async fn set_token( return Ok(StatusCode::OK); } - let token = payload.token; + let key = payload.key; - let library_sources = state - .library_provider - .fetch_sources(&token) - .await - .map_err(|v| { - error!("{v:?}"); - StatusCode::INTERNAL_SERVER_ERROR - })?; - - let valid_library_sources = filter_library_sources(library_sources); - - set_generated_token(&state, token, valid_library_sources)?; + set_depot_key(&state, key)?; info!("connected to drop server successfully"); @@ -44,48 +32,21 @@ pub async fn set_token( fn check_token_exists(state: &Arc, payload: &TokenPayload) -> bool { if let Some(existing_data) = state.token.get() { assert!( - *existing_data.token() == payload.token, + *existing_data.key == payload.key, "already set up but provided with a different token" ); return true; } false } -fn filter_library_sources( - library_sources: Vec, -) -> HashMap { - library_sources - .into_iter() - .filter(|v| { - matches!( - v.backend, - remote::LibraryBackend::Filesystem | remote::LibraryBackend::FlatFilesystem - ) - }) - .map(|v| { - let path = PathBuf::from_str( - v.options - .as_object() - .unwrap() - .get("baseDir") - .unwrap() - .as_str() - .unwrap(), - ) - .unwrap(); - (v.id, (path, v.backend)) - }) - .collect() -} -fn set_generated_token( +fn set_depot_key( state: &Arc, - token: String, - libraries: HashMap, + key: String ) -> Result<(), StatusCode> { state .token - .set(AppInitData::new(token, libraries)) + .set(AppInitData { key }) .map_err(|err| { error!("failed to set token: {err:?}"); StatusCode::INTERNAL_SERVER_ERROR From b812543a4c3d2305aab85dffd34d1c7b6803df1c Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 20 Dec 2025 20:09:53 +1100 Subject: [PATCH 12/21] feat: open file semaphore --- torrential/Cargo.lock | 20 +++++++++++++ torrential/Cargo.toml | 2 ++ torrential/src/main.rs | 32 ++++++++++++++++++++- torrential/src/serve.rs | 64 +++++++++++++++++++++++++++++++++++++---- torrential/src/state.rs | 4 +-- 5 files changed, 113 insertions(+), 9 deletions(-) diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock index dab0e33b..b51b484a 100644 --- a/torrential/Cargo.lock +++ b/torrential/Cargo.lock @@ -547,6 +547,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "file_open_limit" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a1c2ca7813e68cf8e8aa5fbfe64016332355b70495d57e69df4058fd08cdbb" +dependencies = [ + "rlimit", +] + [[package]] name = "find-msvc-tools" version = "0.1.5" @@ -1475,6 +1484,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rlimit" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7278a1ec8bfd4a4e07515c589f5ff7b309a373f987393aef44813d9dcf87aa3" +dependencies = [ + "libc", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -1915,8 +1933,10 @@ dependencies = [ "ctr", "dashmap", "droplet-rs", + "file_open_limit", "futures-util", "log", + "pin-project-lite", "rand", "reqwest", "serde", diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml index 8ae8aba4..ad4ea957 100644 --- a/torrential/Cargo.toml +++ b/torrential/Cargo.toml @@ -34,6 +34,8 @@ futures-util = "0.3.31" ctr = "0.9.2" aes = "0.8.4" bytes = "*" +file_open_limit = "0.0.5" +pin-project-lite = "0.2.16" [lints.clippy] pedantic = { level = "warn", priority = -1 } diff --git a/torrential/src/main.rs b/torrential/src/main.rs index 14fd39c3..4fd95e86 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -1,6 +1,7 @@ use std::{ env::{self, set_current_dir}, sync::Arc, + time::{Duration, Instant}, }; use axum::{ @@ -10,10 +11,12 @@ use axum::{ use dashmap::DashMap; use log::info; use simple_logger::SimpleLogger; -use tokio::{runtime::Handle, sync::OnceCell}; +use tokio::{runtime::Handle, spawn, sync::OnceCell, time}; use torrential::{handlers, serve, set_token, state::AppState}; use url::Url; +const CONTEXT_TTL: u64 = 10 * 60; + #[tokio::main] async fn main() { initialise_logger(); @@ -31,6 +34,33 @@ async fn main() { context_cache: DashMap::new(), }); + let interval_shared_state = shared_state.clone(); + + spawn(async move { + let shared_state = interval_shared_state; + let mut interval = time::interval(Duration::from_mins(1)); + + loop { + interval.tick().await; + let keys = shared_state + .context_cache + .iter() + .map(|v| v.key().clone()) + .collect::>(); + for key in keys { + let last_access = if let Some(context) = shared_state.context_cache.get(&key) { + context.last_access() + } else { + Instant::now() + }; + if last_access.elapsed().as_secs() >= CONTEXT_TTL { + shared_state.context_cache.remove(&key); + info!("cleaned context: {:?}", key); + } + } + } + }); + let app = setup_app(shared_state); serve(app).await.unwrap(); diff --git a/torrential/src/serve.rs b/torrential/src/serve.rs index 20bf3b5a..348038e5 100644 --- a/torrential/src/serve.rs +++ b/torrential/src/serve.rs @@ -1,4 +1,9 @@ -use std::{io::Error, rc::Rc, sync::Arc}; +use std::{ + cell::{LazyCell, OnceCell}, + io::Error, + rc::Rc, + sync::{Arc, LazyLock}, +}; use aes::cipher::{KeyIvInit, StreamCipher}; use axum::{ @@ -13,10 +18,11 @@ use droplet_rs::{ manifest::ChunkData, versions::types::{MinimumFileObject, VersionFile}, }; -use futures_util::{StreamExt, stream}; +use futures_util::{Stream, StreamExt, stream}; use log::{error, info}; -use reqwest::{StatusCode, header}; -use tokio::sync::SemaphorePermit; +use pin_project_lite::pin_project; +use reqwest::StatusCode; +use tokio::sync::{Semaphore, SemaphorePermit}; use tokio_util::io::ReaderStream; use crate::{ @@ -25,6 +31,44 @@ use crate::{ type Aes128Ctr64LE = ctr::Ctr64LE; +pin_project! { + struct SemaphoreStream<'a, T> + where T: Stream + { + #[pin] + stream: T, + semaphore: SemaphorePermit<'a>, + } +} + +impl<'a, T: Stream> SemaphoreStream<'a, T> { + fn new(stream: T, permit: SemaphorePermit<'a>) -> Self { + Self { + stream, + semaphore: permit, + } + } +} + +impl<'a, T: Stream> Stream for SemaphoreStream<'a, T> +where + T: Stream, +{ + type Item = T::Item; + + fn poll_next( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let this = self.project(); + this.stream.poll_next(cx) + } +} + +static SEMPAHORE_COUNT: LazyLock = + LazyLock::new(|| file_open_limit::get().expect("failed to count max open files")); +static FILE_SEMAPHORE: LazyLock = LazyLock::new(|| Semaphore::new(*SEMPAHORE_COUNT)); + pub async fn serve_file( State(state): State>, Path((game_id, version_name, chunk_id)): Path<(String, String, String)>, @@ -35,6 +79,13 @@ pub async fn serve_file( context.reset_last_access(); let chunk_data = lookup_chunk(&chunk_id, &context)?; + if chunk_data.files.len() >= *SEMPAHORE_COUNT { + return Err(StatusCode::INSUFFICIENT_STORAGE); + } + let permit = FILE_SEMAPHORE + .acquire_many(chunk_data.files.len().try_into().unwrap()) + .await + .map_err(|_| StatusCode::INSUFFICIENT_STORAGE)?; let mut streams = Vec::with_capacity(chunk_data.files.len()); let mut content_length = 0; @@ -54,7 +105,7 @@ pub async fn serve_file( let stream = stream::iter(streams).flatten(); let mut cipher = Aes128Ctr64LE::new(&context.manifest.key.into(), &chunk_data.iv.into()); - let encrypted_stream = stream.chunks(3).map(move |raw| -> Result { + let encrypted_stream = stream.chunks(16).map(move |raw| -> Result { let data: Result, Error> = raw.into_iter().collect(); let mut data = data?.concat(); @@ -62,7 +113,8 @@ pub async fn serve_file( Ok(data.into()) }); - let body: Body = Body::from_stream(encrypted_stream); + let permit_stream = SemaphoreStream::new(encrypted_stream, permit); + let body: Body = Body::from_stream(permit_stream); let mut headers = HeaderMap::new(); headers.insert("Content-Type", "application/octet-stream".parse().unwrap()); diff --git a/torrential/src/state.rs b/torrential/src/state.rs index 3cfd35c7..131239f5 100644 --- a/torrential/src/state.rs +++ b/torrential/src/state.rs @@ -1,7 +1,7 @@ -use std::{collections::HashMap, path::PathBuf}; +use std::{collections::HashMap, path::PathBuf, sync::Arc}; use dashmap::DashMap; -use tokio::sync::OnceCell; +use tokio::sync::{OnceCell, Semaphore}; use crate:: DownloadContext From 34f2a92785f71825ea538f8b27caee6492f20e2c Mon Sep 17 00:00:00 2001 From: DecDuck Date: Wed, 21 Jan 2026 16:52:49 +1100 Subject: [PATCH 13/21] feat: use new depot endpoints in game specialisation --- torrential/src/remote.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torrential/src/remote.rs b/torrential/src/remote.rs index 61838bae..7d97e159 100644 --- a/torrential/src/remote.rs +++ b/torrential/src/remote.rs @@ -63,7 +63,7 @@ pub async fn fetch_version_data( version_id: String, ) -> Result { let version_data_response = CLIENT - .get(REMOTE_URL.join("/api/v1/admin/depot/manifest")?) + .get(REMOTE_URL.join("/api/v1/admin/depot/torrential/manifest")?) .query(&VersionQuery { game: game_id, version: version_id, @@ -110,7 +110,7 @@ pub async fn fetch_instance_games( init_data: &AppInitData, ) -> Result, ErrorOption> { let context_response = CLIENT - .get(REMOTE_URL.join("/api/v1/admin/depot/versions")?) + .get(REMOTE_URL.join("/api/v1/admin/depot/torrential/versions")?) .header("Authorization", format!("Bearer {}", init_data.key)) .send() .await?; From aa46a889574215662adbf301704af8dc15c3d670 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Wed, 4 Feb 2026 13:44:07 +1100 Subject: [PATCH 14/21] feat: protobuf communication --- torrential/.gitignore | 2 + torrential/Cargo.lock | 225 ++++++++++++++++++++++++++++-- torrential/Cargo.toml | 5 + torrential/build.rs | 25 ++++ torrential/proto/core.proto | 24 ++++ torrential/proto/manifest.proto | 14 ++ torrential/proto/version.proto | 39 ++++++ torrential/src/conversions.rs | 37 +++++ torrential/src/download.rs | 34 +++-- torrential/src/handlers.rs | 26 ++-- torrential/src/lib.rs | 10 +- torrential/src/main.rs | 14 +- torrential/src/proto/.gitkeep | 0 torrential/src/remote.rs | 134 ------------------ torrential/src/serve.rs | 4 +- torrential/src/server/download.rs | 15 ++ torrential/src/server/mod.rs | 107 ++++++++++++++ torrential/src/state.rs | 11 +- torrential/src/token.rs | 55 -------- 19 files changed, 520 insertions(+), 261 deletions(-) create mode 100644 torrential/build.rs create mode 100644 torrential/proto/core.proto create mode 100644 torrential/proto/manifest.proto create mode 100644 torrential/proto/version.proto create mode 100644 torrential/src/conversions.rs create mode 100644 torrential/src/proto/.gitkeep delete mode 100644 torrential/src/remote.rs create mode 100644 torrential/src/server/download.rs create mode 100644 torrential/src/server/mod.rs delete mode 100644 torrential/src/token.rs diff --git a/torrential/.gitignore b/torrential/.gitignore index ea8c4bf7..4ee2e9b6 100644 --- a/torrential/.gitignore +++ b/torrential/.gitignore @@ -1 +1,3 @@ /target +/src/proto/* +!/src/proto/.gitkeep diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock index b51b484a..fa905fd5 100644 --- a/torrential/Cargo.lock +++ b/torrential/Cargo.lock @@ -8,11 +8,20 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cipher", "cpufeatures", ] +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" +dependencies = [ + "const-random", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -240,6 +249,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.4" @@ -323,6 +338,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "tiny-keccak", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -418,15 +453,26 @@ dependencies = [ "cipher", ] +[[package]] +name = "dashmap" +version = "3.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5" +dependencies = [ + "ahash", + "cfg-if 0.1.10", + "num_cpus", +] + [[package]] name = "dashmap" version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "crossbeam-utils", - "hashbrown", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -531,6 +577,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -676,7 +728,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", "wasi", @@ -689,7 +741,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", "r-efi", @@ -703,7 +755,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "crunchy", "zerocopy", ] @@ -714,12 +766,33 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "1.4.0" @@ -939,6 +1012,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", +] + [[package]] name = "inout" version = "0.1.4" @@ -1007,6 +1090,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -1119,6 +1208,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "oid-registry" version = "0.7.1" @@ -1165,7 +1264,7 @@ version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", "redox_syscall", "smallvec", @@ -1261,6 +1360,57 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-codegen" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3976825c0014bbd2f3b34f0001876604fe87e0c86cd8fa54251530f1544ace" +dependencies = [ + "anyhow", + "once_cell", + "protobuf", + "protobuf-parse", + "regex", + "tempfile", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-parse" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973" +dependencies = [ + "anyhow", + "indexmap", + "log", + "protobuf", + "protobuf-support", + "tempfile", + "thiserror 1.0.69", + "which", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "quinn" version = "0.11.9" @@ -1477,7 +1627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "getrandom 0.2.16", "libc", "untrusted", @@ -1508,6 +1658,19 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.2" @@ -1517,7 +1680,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.11.0", "windows-sys 0.61.2", ] @@ -1655,7 +1818,7 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures", "digest", ] @@ -1760,7 +1923,7 @@ dependencies = [ "fastrand", "getrandom 0.3.4", "once_cell", - "rustix", + "rustix 1.1.2", "windows-sys 0.61.2", ] @@ -1835,6 +1998,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -1931,12 +2103,14 @@ dependencies = [ "bytes", "criterion", "ctr", - "dashmap", + "dashmap 6.1.0", "droplet-rs", "file_open_limit", "futures-util", "log", "pin-project-lite", + "protobuf", + "protobuf-codegen", "rand", "reqwest", "serde", @@ -1946,6 +2120,7 @@ dependencies = [ "tokio", "tokio-util", "url", + "waitmap", ] [[package]] @@ -2073,6 +2248,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "waitmap" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28491611b6b9a0b9f027be139a4be792b13a20780100dd8b054d44dbf596d52b" +dependencies = [ + "dashmap 3.11.10", + "smallvec", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -2113,7 +2298,7 @@ version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "once_cell", "rustversion", "wasm-bindgen-macro", @@ -2126,7 +2311,7 @@ version = "0.4.56" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "once_cell", "wasm-bindgen", @@ -2194,6 +2379,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml index ad4ea957..51b90a5f 100644 --- a/torrential/Cargo.toml +++ b/torrential/Cargo.toml @@ -36,6 +36,8 @@ aes = "0.8.4" bytes = "*" file_open_limit = "0.0.5" pin-project-lite = "0.2.16" +protobuf = "3.7.2" +waitmap = "1.1.0" [lints.clippy] pedantic = { level = "warn", priority = -1 } @@ -59,3 +61,6 @@ harness = false criterion = { version = "0.8.0", features = ["async", "async_tokio"] } rand = "0.9.2" tempfile = "3.23.0" + +[build-dependencies] +protobuf-codegen = "3.7.2" diff --git a/torrential/build.rs b/torrential/build.rs new file mode 100644 index 00000000..23828b9f --- /dev/null +++ b/torrential/build.rs @@ -0,0 +1,25 @@ +use std::fs::{self, read_dir}; + +use protobuf_codegen::Codegen; + +const OUT_DIR: &'static str = "./src/proto/"; + +fn main() { + let files = read_dir("./proto").unwrap(); + let files = files.map(|v| format!("proto/{}", v.unwrap().file_name().into_string().unwrap())); + + read_dir(OUT_DIR).unwrap().into_iter().for_each(|v| { + if let Ok(entry) = v { + if entry.file_name().to_str().unwrap().ends_with(".rs") { + fs::remove_file(entry.path()).unwrap(); + } + } + }); + + Codegen::new() + .inputs(files) + .include("proto") + .out_dir(OUT_DIR) + .run() + .unwrap(); +} diff --git a/torrential/proto/core.proto b/torrential/proto/core.proto new file mode 100644 index 00000000..b2348a2c --- /dev/null +++ b/torrential/proto/core.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +enum ResponseType { + ERROR = 0; + SERVER_GAMES_RESPONSE = 1; + VERSION_RESPONSE = 2; +} + +message Response { + string message_id = 1; + ResponseType type = 2; + bytes data = 3; +} + +enum QueryType { + SERVER_GAMES_QUERY = 0; + VERSION_QUERY = 1; +} + +message Query { + string message_id = 1; + QueryType type = 2; + bytes data = 3; +} diff --git a/torrential/proto/manifest.proto b/torrential/proto/manifest.proto new file mode 100644 index 00000000..84415b7e --- /dev/null +++ b/torrential/proto/manifest.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +message ServerGamesQuery {} + +message ServerGamesResponse { + message SkeletonGame { + message SkeletonVersion { + string version_id = 1; + } + string id = 1; + repeated SkeletonVersion versions = 2; + } + repeated SkeletonGame games = 1; +} diff --git a/torrential/proto/version.proto b/torrential/proto/version.proto new file mode 100644 index 00000000..faf9cc6b --- /dev/null +++ b/torrential/proto/version.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +message VersionQuery { + string version_id = 1; +} + +message VersionResponse { + message Manifest { + string version = 1; + message ChunkData { + message FileEntry { + string filename = 1; + uint64 start = 2; + uint64 length = 3; + fixed32 permissions = 4; + } + repeated FileEntry files = 1; + string checksum = 2; + bytes iv = 3; + } + map chunks = 2; + uint64 size = 3; + bytes key = 4; + } + Manifest manifest = 1; + + message LibrarySource { + enum LibraryBackend { + FILESYSTEM = 0; + FLAT_FILESYSTEM = 1; + } + string options = 1; /// JSON + string id = 2; + LibraryBackend backend = 3; + } + LibrarySource source = 2; + string library_path = 3; + string version_path = 4; +} diff --git a/torrential/src/conversions.rs b/torrential/src/conversions.rs new file mode 100644 index 00000000..b606b18e --- /dev/null +++ b/torrential/src/conversions.rs @@ -0,0 +1,37 @@ +use crate::proto::{self, version::version_response::Manifest}; + +fn fixed_length(v: Vec) -> [T; N] { + v.try_into() + .unwrap_or_else(|v: Vec| panic!("Expected a Vec of length {} but it was {}", N, v.len())) +} + +pub fn convert_protobuf_manifest(source: Manifest) -> droplet_rs::manifest::Manifest { + droplet_rs::manifest::Manifest { + version: source.version, + chunks: source + .chunks + .into_iter() + .map(|(id, chunk_data)| { + ( + id, + droplet_rs::manifest::ChunkData { + files: chunk_data + .files + .into_iter() + .map(|file_entry| droplet_rs::manifest::FileEntry { + filename: file_entry.filename, + start: file_entry.start.try_into().unwrap(), + length: file_entry.length.try_into().unwrap(), + permissions: file_entry.permissions, + }) + .collect(), + checksum: chunk_data.checksum, + iv: fixed_length(chunk_data.iv), + }, + ) + }) + .collect(), + size: source.size, + key: fixed_length(source.key), + } +} diff --git a/torrential/src/download.rs b/torrential/src/download.rs index 7c62ef95..91134f68 100644 --- a/torrential/src/download.rs +++ b/torrential/src/download.rs @@ -1,16 +1,18 @@ use std::{path::PathBuf, time::Instant}; -use anyhow::anyhow; use droplet_rs::{ manifest::Manifest, versions::{create_backend_constructor, types::VersionBackend}, }; -use log::{info, warn}; +use log::warn; use reqwest::StatusCode; +use serde_json::Value; use crate::{ - remote::{LibraryBackend, VersionResponseBody, fetch_version_data}, - state::AppInitData, + conversions::convert_protobuf_manifest, + proto::version::{VersionResponse, version_response::library_source::LibraryBackend}, + server::download::fetch_version_data, + state::AppState, util::ErrorOption, }; @@ -29,16 +31,16 @@ impl DownloadContext { } pub async fn create_download_context( - init_data: &AppInitData, + app_state: &AppState, game_id: String, version_name: String, ) -> Result { - let version_data = fetch_version_data(init_data, game_id, version_name.clone()).await?; + let version_data = fetch_version_data(app_state, game_id, version_name.clone()).await?; let backend = create_backend(&version_data)?; let download_context = DownloadContext { - manifest: version_data.manifest, + manifest: convert_protobuf_manifest(version_data.manifest.unwrap()), backend, last_access: Instant::now(), }; @@ -47,21 +49,17 @@ pub async fn create_download_context( } fn create_backend( - version_data: &VersionResponseBody, + version_data: &VersionResponse, ) -> Result, StatusCode> { - let base_path = version_data - .library - .options - .get("baseDir") - .unwrap() - .as_str() - .unwrap(); + let base_path = serde_json::from_str::(&version_data.source.options) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let base_path = base_path.get("baseDir").unwrap().as_str().unwrap(); let version_path = PathBuf::from(base_path); let version_path = version_path.join(version_data.library_path.clone()); - let version_path = match version_data.library.backend { - LibraryBackend::Filesystem => version_path.join(version_data.version_path.clone()), - LibraryBackend::FlatFilesystem => version_path, + let version_path = match version_data.source.backend.unwrap() { + LibraryBackend::FILESYSTEM => version_path.join(version_data.version_path.clone()), + LibraryBackend::FLAT_FILESYSTEM => version_path, }; if !version_path.exists() { diff --git a/torrential/src/handlers.rs b/torrential/src/handlers.rs index 636a8de0..c342f416 100644 --- a/torrential/src/handlers.rs +++ b/torrential/src/handlers.rs @@ -7,7 +7,13 @@ use std::{ task::Poll, }; -use axum::{Json, body::Body, extract::State, http::{HeaderMap, HeaderValue}, response::IntoResponse}; +use axum::{ + Json, + body::Body, + extract::State, + http::{HeaderMap, HeaderValue}, + response::IntoResponse, +}; use bytes::BufMut; use reqwest::{StatusCode, header::CONTENT_TYPE}; use serde::{Deserialize, Serialize}; @@ -16,13 +22,9 @@ use std::io::Write; use tokio::io::AsyncRead; use tokio_util::io::ReaderStream; -use crate::{remote::fetch_instance_games, state::AppState}; +use crate::{server::download::fetch_instance_games, state::AppState}; pub async fn healthcheck(State(state): State>) -> StatusCode { - let initialised = state.token.initialized(); - if !initialised { - return StatusCode::SERVICE_UNAVAILABLE; - } StatusCode::OK } @@ -99,16 +101,8 @@ struct Manifest { content: HashMap>, } -pub async fn manifest( - State(state): State>, -) -> Result { - let games = fetch_instance_games( - state - .token - .get() - .ok_or(StatusCode::from_u16(503).unwrap())?, - ) - .await?; +pub async fn manifest(State(state): State>) -> Result { + let games = fetch_instance_games(&state).await?; let mut content = HashMap::new(); for game in games { diff --git a/torrential/src/lib.rs b/torrential/src/lib.rs index e97b94f1..8cc3309f 100644 --- a/torrential/src/lib.rs +++ b/torrential/src/lib.rs @@ -1,13 +1,13 @@ use tokio::sync::Semaphore; -mod download; +pub mod download; pub mod serve; pub mod handlers; -mod remote; pub mod state; -mod token; -mod util; +pub mod util; +pub mod proto; +pub mod conversions; +pub mod server; pub use download::DownloadContext; -pub use token::set_token; static GLOBAL_CONTEXT_SEMAPHORE: Semaphore = Semaphore::const_new(1); diff --git a/torrential/src/main.rs b/torrential/src/main.rs index 4fd95e86..025d6ee4 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -1,19 +1,18 @@ use std::{ - env::{self, set_current_dir}, + env::set_current_dir, sync::Arc, time::{Duration, Instant}, }; use axum::{ - Router, handler, + Router, routing::{get, post}, }; use dashmap::DashMap; use log::info; use simple_logger::SimpleLogger; -use tokio::{runtime::Handle, spawn, sync::OnceCell, time}; -use torrential::{handlers, serve, set_token, state::AppState}; -use url::Url; +use tokio::{runtime::Handle, spawn, time}; +use torrential::{handlers, serve, server::create_drop_server, state::AppState}; const CONTEXT_TTL: u64 = 10 * 60; @@ -29,9 +28,11 @@ async fn main() { let metrics = Handle::current().metrics(); info!("using {} threads", metrics.num_workers()); + let server = create_drop_server().await.expect("failed to connect to drop server"); + let shared_state = Arc::new(AppState { - token: OnceCell::new(), context_cache: DashMap::new(), + server: server, }); let interval_shared_state = shared_state.clone(); @@ -74,7 +75,6 @@ fn setup_app(shared_state: Arc) -> Router { ) .route("/api/v1/depot/manifest.json", get(handlers::manifest)) .route("/api/v1/depot/speedtest", get(handlers::speedtest)) - .route("/key", post(set_token)) .route("/healthcheck", get(handlers::healthcheck)) .route("/invalidate", post(handlers::invalidate)) .with_state(shared_state) diff --git a/torrential/src/proto/.gitkeep b/torrential/src/proto/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/torrential/src/remote.rs b/torrential/src/remote.rs deleted file mode 100644 index 7d97e159..00000000 --- a/torrential/src/remote.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::{env, sync::LazyLock}; - -use anyhow::{Result, anyhow}; -use async_trait::async_trait; -use droplet_rs::manifest::Manifest; -use log::info; -use reqwest::{Client, ClientBuilder, StatusCode}; -use serde::{Deserialize, Serialize}; -use url::Url; - -use crate::{state::AppInitData, util::ErrorOption}; - -static CLIENT: LazyLock = LazyLock::new(|| { - ClientBuilder::new() - .build() - .expect("failed to build client") -}); - -static REMOTE_URL: LazyLock = LazyLock::new(|| { - let user_provided = env::var("DROP_SERVER_URL"); - let url = Url::parse( - user_provided - .as_ref() - .map_or("http://localhost:3000", |v| v), - ) - .expect("failed to parse URL"); - info!("using Drop server url {}", url); - url -}); - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct VersionResponseBody { - pub manifest: Manifest, - pub library: LibrarySource, - pub library_path: String, - pub version_path: String, -} - -#[derive(Serialize)] -pub struct VersionQuery { - game: String, - version: String, -} - -#[derive(Deserialize, Debug)] -#[non_exhaustive] -pub enum LibraryBackend { - Filesystem, - FlatFilesystem, -} - -#[derive(Deserialize)] -pub struct LibrarySource { - pub options: serde_json::Value, - pub id: String, - pub backend: LibraryBackend, -} - -pub async fn fetch_version_data( - init_data: &AppInitData, - game_id: String, - version_id: String, -) -> Result { - let version_data_response = CLIENT - .get(REMOTE_URL.join("/api/v1/admin/depot/torrential/manifest")?) - .query(&VersionQuery { - game: game_id, - version: version_id, - }) - .header("Authorization", format!("Bearer {}", init_data.key)) - .send() - .await?; - - if !version_data_response.status().is_success() { - if version_data_response.status() == StatusCode::BAD_REQUEST { - return Err(StatusCode::NOT_FOUND.into()); - } - - return Err(anyhow!( - "Fetching context failed with non-success code: {}, {}", - version_data_response.status(), - version_data_response - .text() - .await - .unwrap_or("(failed to read body)".to_owned()) - ) - .into()); - } - - let version_data: VersionResponseBody = version_data_response.json().await?; - - Ok(version_data) -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SkeletonVersion { - pub version_id: String, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SkeletonGame { - pub id: String, - pub versions: Vec, -} - -pub async fn fetch_instance_games( - init_data: &AppInitData, -) -> Result, ErrorOption> { - let context_response = CLIENT - .get(REMOTE_URL.join("/api/v1/admin/depot/torrential/versions")?) - .header("Authorization", format!("Bearer {}", init_data.key)) - .send() - .await?; - - if !context_response.status().is_success() { - - return Err(anyhow!( - "Fetching instance games failed with non-success code: {}, {}", - context_response.status(), - context_response - .text() - .await - .unwrap_or("(failed to read body)".to_owned()) - ) - .into()); - } - - let games: Vec = context_response.json().await?; - - Ok(games) -} diff --git a/torrential/src/serve.rs b/torrential/src/serve.rs index 348038e5..3a22b370 100644 --- a/torrential/src/serve.rs +++ b/torrential/src/serve.rs @@ -171,7 +171,6 @@ async fn get_or_create_context<'a>( game_id: String, version_name: String, ) -> Result, StatusCode> { - let initialisation_data = state.token.get().ok_or(StatusCode::SERVICE_UNAVAILABLE)?; let key = (game_id.clone(), version_name.clone()); if let Some(context) = context_cache.get_mut(&key) { @@ -185,8 +184,7 @@ async fn get_or_create_context<'a>( } else { info!("generating context for {}...", game_id); let context_result = - create_download_context(initialisation_data, game_id.clone(), version_name.clone()) - .await?; + create_download_context(state, game_id.clone(), version_name.clone()).await?; state.context_cache.insert(key.clone(), context_result); diff --git a/torrential/src/server/download.rs b/torrential/src/server/download.rs new file mode 100644 index 00000000..5a366455 --- /dev/null +++ b/torrential/src/server/download.rs @@ -0,0 +1,15 @@ +use crate::{proto::{manifest::server_games_response::SkeletonGame, version::VersionResponse}, state::{AppState}, util::ErrorOption}; + +pub async fn fetch_version_data( + app_state: &AppState, + game_id: String, + version_id: String, +) -> Result { + unreachable!(); +} + +pub async fn fetch_instance_games( + app_state: &AppState, +) -> Result, ErrorOption> { + unreachable!() +} diff --git a/torrential/src/server/mod.rs b/torrential/src/server/mod.rs new file mode 100644 index 00000000..d59bc8c0 --- /dev/null +++ b/torrential/src/server/mod.rs @@ -0,0 +1,107 @@ +use std::sync::{Arc, Mutex}; + +use anyhow::anyhow; +use protobuf::Message; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt as _, BufReader}, + net::{ + TcpListener, + tcp::{OwnedReadHalf, OwnedWriteHalf}, + }, + spawn, +}; +use waitmap::WaitMap; + +use crate::proto::core::Response; + +pub mod download; + +pub struct DropServer { + write_stream: Mutex, + waitmap: WaitMap, +} + +impl DropServer { + async fn recieve_subroutine(myself: Arc, read_stream: OwnedReadHalf) -> ! { + let mut buffered_reader = BufReader::new(read_stream); + + loop { + let mut length_buffer: [u8; 8] = [0; 8]; + buffered_reader + .read_exact(&mut length_buffer) + .await + .expect("failed to read from internal pipe"); + + let length = usize::from_le_bytes(length_buffer); + let mut buffer = Vec::with_capacity(length); + + buffered_reader + .read_exact(&mut buffer) + .await + .expect("failed to read from internal pipe"); + + let message = + Response::parse_from_bytes(&buffer).expect("response didn't deserialize correctly"); + myself.waitmap.insert(message.message_id.clone(), message); + } + } + + async fn wait_for_message_id(&self, message_id: &str) -> Result + where + T: protobuf::Message, + { + let message = self + .waitmap + .wait(message_id.clone()) + .await + .ok_or(anyhow!("no response returned for value"))?; + + let message = message.value(); + + match message.type_.unwrap() { + crate::proto::core::ResponseType::ERROR => { + return Err(anyhow!(String::from_utf8(message.data.clone()).unwrap())); + } + _ => { + let response = T::parse_from_bytes(&message.data)?; + return Ok(response); + } + } + } + + async fn send_message(&self, message: T) -> Result<(), anyhow::Error> + where + T: protobuf::Message, + { + let mut buf = Vec::new(); + message.write_to_vec(&mut buf)?; + + { + let mut mutex_lock = self + .write_stream + .lock() + .expect("failed to lock send stream"); + mutex_lock.write(&buf.len().to_le_bytes()).await?; + mutex_lock.write_all(&buf).await?; + }; + + Ok(()) + } +} + +pub async fn create_drop_server() -> Result, anyhow::Error> { + let server = TcpListener::bind("127.0.0.1:33148").await?; + + let (drop_stream, _) = server.accept().await?; + + let (read, write) = drop_stream.into_split(); + + let client = Arc::new(DropServer { + write_stream: Mutex::new(write), + waitmap: WaitMap::new(), + }); + + spawn(DropServer::recieve_subroutine(client.clone(), read)); + + Ok(client) +} diff --git a/torrential/src/state.rs b/torrential/src/state.rs index 131239f5..f55cdc7d 100644 --- a/torrential/src/state.rs +++ b/torrential/src/state.rs @@ -3,16 +3,9 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc}; use dashmap::DashMap; use tokio::sync::{OnceCell, Semaphore}; -use crate:: - DownloadContext -; +use crate::{DownloadContext, server::DropServer}; pub struct AppState { - pub token: OnceCell, pub context_cache: DashMap<(String, String), DownloadContext>, + pub server: Arc, } - -#[derive(Debug)] -pub struct AppInitData { - pub key: String, -} \ No newline at end of file diff --git a/torrential/src/token.rs b/torrential/src/token.rs deleted file mode 100644 index 2b46ccfc..00000000 --- a/torrential/src/token.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::sync::Arc; - -use axum::{Json, extract::State}; -use log::{error, info}; -use reqwest::StatusCode; -use serde::Deserialize; - -use crate::state::{AppInitData, AppState}; - -#[derive(Deserialize)] -pub struct TokenPayload { - key: String, -} - -pub async fn set_token( - State(state): State>, - Json(payload): Json, -) -> Result { - if check_token_exists(&state, &payload) { - return Ok(StatusCode::OK); - } - - let key = payload.key; - - set_depot_key(&state, key)?; - - info!("connected to drop server successfully"); - - Ok(StatusCode::OK) -} - -fn check_token_exists(state: &Arc, payload: &TokenPayload) -> bool { - if let Some(existing_data) = state.token.get() { - assert!( - *existing_data.key == payload.key, - "already set up but provided with a different token" - ); - return true; - } - false -} - -fn set_depot_key( - state: &Arc, - key: String -) -> Result<(), StatusCode> { - state - .token - .set(AppInitData { key }) - .map_err(|err| { - error!("failed to set token: {err:?}"); - StatusCode::INTERNAL_SERVER_ERROR - })?; - Ok(()) -} From 55b76a65297b4357d3713fbc46b8fcdc8ab967b7 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 5 Feb 2026 12:00:25 +1100 Subject: [PATCH 15/21] feat: manifest generation --- torrential/Cargo.lock | 23 +++- torrential/Cargo.toml | 6 +- torrential/proto/core.proto | 18 ++-- torrential/proto/droplet.proto | 37 +++++++ torrential/src/conversions.rs | 2 +- torrential/src/{ => downloads}/download.rs | 0 torrential/src/{ => downloads}/handlers.rs | 9 +- torrential/src/downloads/mod.rs | 3 + torrential/src/{ => downloads}/serve.rs | 7 +- torrential/src/droplet/cert.rs | 0 torrential/src/droplet/manifest.rs | 105 ++++++++++++++++++ torrential/src/droplet/mod.rs | 2 + torrential/src/lib.rs | 7 +- torrential/src/main.rs | 10 +- torrential/src/server/download.rs | 36 +++++-- torrential/src/server/mod.rs | 120 ++++++++++++++++----- 16 files changed, 321 insertions(+), 64 deletions(-) create mode 100644 torrential/proto/droplet.proto rename torrential/src/{ => downloads}/download.rs (100%) rename torrential/src/{ => downloads}/handlers.rs (93%) create mode 100644 torrential/src/downloads/mod.rs rename torrential/src/{ => downloads}/serve.rs (96%) create mode 100644 torrential/src/droplet/cert.rs create mode 100644 torrential/src/droplet/manifest.rs create mode 100644 torrential/src/droplet/mod.rs diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock index fa905fd5..c9de6188 100644 --- a/torrential/Cargo.lock +++ b/torrential/Cargo.lock @@ -155,6 +155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" dependencies = [ "axum-core", + "axum-macros", "bytes", "form_urlencoded", "futures-util", @@ -200,6 +201,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "base64" version = "0.22.1" @@ -544,8 +556,9 @@ dependencies = [ [[package]] name = "droplet-rs" -version = "0.12.2" -source = "git+https://github.com/Drop-OSS/droplet-rs.git#05f7027b362fcc72b7cc621df8b5b0850b6cf082" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef23d187f7153bc3ce1c6a5440be1206e2aaf42b153825ddf2f8151f278539c5" dependencies = [ "anyhow", "async-trait", @@ -2108,6 +2121,7 @@ dependencies = [ "file_open_limit", "futures-util", "log", + "num_cpus", "pin-project-lite", "protobuf", "protobuf-codegen", @@ -2120,6 +2134,7 @@ dependencies = [ "tokio", "tokio-util", "url", + "uuid", "waitmap", ] @@ -2233,9 +2248,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" dependencies = [ "getrandom 0.3.4", "js-sys", diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml index 51b90a5f..d2603d75 100644 --- a/torrential/Cargo.toml +++ b/torrential/Cargo.toml @@ -12,7 +12,7 @@ name = "torrential" path = "src/main.rs" [dependencies] -axum = "0.8.7" +axum = { version = "0.8.7", features = ["macros"] } log = "0.4.28" reqwest = { version = "0.12.24", default-features = false, features = [ "json", @@ -23,7 +23,7 @@ simple_logger = { version = "5.1.0", default-features = false, features = [ "colors", ] } tokio = { version = "*", features = ["rt-multi-thread", "sync"] } -droplet-rs = { git="https://github.com/Drop-OSS/droplet-rs.git" } +droplet-rs = "0.15.1" dashmap = "6.1.0" anyhow = "1.0.100" serde_json = "1.0.145" @@ -38,6 +38,8 @@ file_open_limit = "0.0.5" pin-project-lite = "0.2.16" protobuf = "3.7.2" waitmap = "1.1.0" +uuid = { version = "1.20.0", features = ["v4"] } +num_cpus = "1.17.0" [lints.clippy] pedantic = { level = "warn", priority = -1 } diff --git a/torrential/proto/core.proto b/torrential/proto/core.proto index b2348a2c..2277396f 100644 --- a/torrential/proto/core.proto +++ b/torrential/proto/core.proto @@ -1,24 +1,30 @@ syntax = "proto3"; -enum ResponseType { +enum TorrentialBoundType { ERROR = 0; SERVER_GAMES_RESPONSE = 1; VERSION_RESPONSE = 2; + GENERATE_MANIFEST = 3; } -message Response { +message TorrentialBound { string message_id = 1; - ResponseType type = 2; + TorrentialBoundType type = 2; bytes data = 3; } -enum QueryType { +enum DropBoundType { SERVER_GAMES_QUERY = 0; VERSION_QUERY = 1; + + MANIFEST_PROGRESS = 2; + MANIFEST_LOG = 3; + MANIFEST_COMPLETE = 4; + MANIFEST_ERROR = 5; } -message Query { +message DropBound { string message_id = 1; - QueryType type = 2; + DropBoundType type = 2; bytes data = 3; } diff --git a/torrential/proto/droplet.proto b/torrential/proto/droplet.proto new file mode 100644 index 00000000..ddb4da99 --- /dev/null +++ b/torrential/proto/droplet.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; + +/// Certificates +message RootCertQuery {} +message RootCertResponse { + string cert = 1; + string priv = 2; +} + +message ClientCertQuery { + string client_id = 1; + string client_name = 2; + string root_cert = 3; + string root_priv = 4; +} +message ClientCertResponse { + string cert = 1; + string priv = 2; +} + +/// Manifest generation +message GenerateManifest { + string version_dir = 1; +} + +message ManifestProgress { + float progress = 1; +} +message ManifestLog { + string log_line = 1; +} +message ManifestComplete { + string manifest = 1; +} +message ManifestError { + string error = 1; +} diff --git a/torrential/src/conversions.rs b/torrential/src/conversions.rs index b606b18e..622168da 100644 --- a/torrential/src/conversions.rs +++ b/torrential/src/conversions.rs @@ -1,4 +1,4 @@ -use crate::proto::{self, version::version_response::Manifest}; +use crate::proto::version::version_response::Manifest; fn fixed_length(v: Vec) -> [T; N] { v.try_into() diff --git a/torrential/src/download.rs b/torrential/src/downloads/download.rs similarity index 100% rename from torrential/src/download.rs rename to torrential/src/downloads/download.rs diff --git a/torrential/src/handlers.rs b/torrential/src/downloads/handlers.rs similarity index 93% rename from torrential/src/handlers.rs rename to torrential/src/downloads/handlers.rs index c342f416..ad597bf1 100644 --- a/torrential/src/handlers.rs +++ b/torrential/src/downloads/handlers.rs @@ -1,9 +1,6 @@ use std::{ collections::HashMap, - sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }, + sync::Arc, task::Poll, }; @@ -24,7 +21,7 @@ use tokio_util::io::ReaderStream; use crate::{server::download::fetch_instance_games, state::AppState}; -pub async fn healthcheck(State(state): State>) -> StatusCode { +pub async fn healthcheck() -> StatusCode { StatusCode::OK } @@ -60,7 +57,7 @@ const ZERO: [u8; 1024] = [0u8; _]; impl AsyncRead for SpeedtestStream { fn poll_read( mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, + _cx: &mut std::task::Context<'_>, buf: &mut tokio::io::ReadBuf<'_>, ) -> Poll> { if self.remaining > 0 { diff --git a/torrential/src/downloads/mod.rs b/torrential/src/downloads/mod.rs new file mode 100644 index 00000000..b9959744 --- /dev/null +++ b/torrential/src/downloads/mod.rs @@ -0,0 +1,3 @@ +pub mod handlers; +pub mod serve; +pub mod download; diff --git a/torrential/src/serve.rs b/torrential/src/downloads/serve.rs similarity index 96% rename from torrential/src/serve.rs rename to torrential/src/downloads/serve.rs index 3a22b370..d700a75f 100644 --- a/torrential/src/serve.rs +++ b/torrential/src/downloads/serve.rs @@ -1,7 +1,5 @@ use std::{ - cell::{LazyCell, OnceCell}, io::Error, - rc::Rc, sync::{Arc, LazyLock}, }; @@ -10,7 +8,7 @@ use axum::{ body::Body, extract::{Path, State}, http::HeaderMap, - response::{AppendHeaders, IntoResponse}, + response::IntoResponse, }; use bytes::Bytes; use dashmap::{DashMap, mapref::one::RefMut}; @@ -26,7 +24,8 @@ use tokio::sync::{Semaphore, SemaphorePermit}; use tokio_util::io::ReaderStream; use crate::{ - DownloadContext, GLOBAL_CONTEXT_SEMAPHORE, download::create_download_context, state::AppState, + DownloadContext, GLOBAL_CONTEXT_SEMAPHORE, downloads::download::create_download_context, + state::AppState, }; type Aes128Ctr64LE = ctr::Ctr64LE; diff --git a/torrential/src/droplet/cert.rs b/torrential/src/droplet/cert.rs new file mode 100644 index 00000000..e69de29b diff --git a/torrential/src/droplet/manifest.rs b/torrential/src/droplet/manifest.rs new file mode 100644 index 00000000..6af4f0a4 --- /dev/null +++ b/torrential/src/droplet/manifest.rs @@ -0,0 +1,105 @@ +use std::{ + path::PathBuf, + sync::{Arc, LazyLock}, +}; + +use log::{info, warn}; +use protobuf::Message; +use serde_json::json; +use tokio::{spawn, sync::Semaphore}; + +use crate::{ + droplet, + proto::{ + core::{DropBoundType, TorrentialBound}, + droplet::{ + GenerateManifest, ManifestComplete, ManifestError, ManifestLog, ManifestProgress, + }, + }, + server::DropServer, +}; + +static READER_SEMAPHORE: LazyLock = LazyLock::new(|| { + let cores = num_cpus::get(); + Semaphore::new(cores) +}); + +async fn generate_manifest_raw( + server: Arc, + message: TorrentialBound, +) -> Result<(), anyhow::Error> { + let manifest_message = GenerateManifest::parse_from_bytes(&message.data)?; + info!( + "seven zip install: {}", + *droplet_rs::versions::backends::SEVEN_ZIP_INSTALLED + ); + + let manifest = droplet_rs::manifest::generate_manifest_rusty( + &PathBuf::from(manifest_message.version_dir), + |progress| { + let mut progress_message = ManifestProgress::new(); + progress_message.progress = progress; + + let server = server.clone(); + let message_id = message.message_id.clone(); + spawn(async move { + let _ = server + .send_message( + DropBoundType::MANIFEST_PROGRESS, + progress_message, + Some(message_id), + ) + .await; + }); + }, + |log_line| { + let mut progress_log = ManifestLog::new(); + progress_log.log_line = log_line; + + let server = server.clone(); + let message_id = message.message_id.clone(); + spawn(async move { + let _ = server + .send_message(DropBoundType::MANIFEST_LOG, progress_log, Some(message_id)) + .await; + }); + }, + Some(&READER_SEMAPHORE), + ) + .await?; + + let mut manifest_complete = ManifestComplete::new(); + manifest_complete.manifest = json!(manifest).to_string(); + + server + .send_message( + DropBoundType::MANIFEST_COMPLETE, + manifest_complete, + Some(message.message_id), + ) + .await?; + + Ok(()) +} + +pub async fn generate_manifest(server: Arc, message: TorrentialBound) { + let message_id = message.message_id.clone(); + warn!("generating manifest..."); + let result = generate_manifest_raw(server.clone(), message).await; + info!("manifest generation exited"); + if let Err(err) = result { + warn!("manifest generation failed with err: {:?}", err); + let mut manifest_err = ManifestError::new(); + manifest_err.error = err.to_string(); + let _ = server + .send_message( + DropBoundType::MANIFEST_ERROR, + manifest_err, + Some(message_id), + ) + .await + .inspect_err(|err| { + warn!("failed to send manifest err: {err:?}"); + }); + } +} diff --git a/torrential/src/droplet/mod.rs b/torrential/src/droplet/mod.rs new file mode 100644 index 00000000..e07a6a54 --- /dev/null +++ b/torrential/src/droplet/mod.rs @@ -0,0 +1,2 @@ +pub mod manifest; +pub mod cert; diff --git a/torrential/src/lib.rs b/torrential/src/lib.rs index 8cc3309f..3fa089f6 100644 --- a/torrential/src/lib.rs +++ b/torrential/src/lib.rs @@ -1,13 +1,12 @@ use tokio::sync::Semaphore; -pub mod download; -pub mod serve; -pub mod handlers; +pub mod downloads; pub mod state; pub mod util; pub mod proto; pub mod conversions; pub mod server; +pub mod droplet; -pub use download::DownloadContext; +pub use downloads::download::DownloadContext; static GLOBAL_CONTEXT_SEMAPHORE: Semaphore = Semaphore::const_new(1); diff --git a/torrential/src/main.rs b/torrential/src/main.rs index 025d6ee4..bfdf92e4 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -12,7 +12,11 @@ use dashmap::DashMap; use log::info; use simple_logger::SimpleLogger; use tokio::{runtime::Handle, spawn, time}; -use torrential::{handlers, serve, server::create_drop_server, state::AppState}; +use torrential::{ + downloads::{handlers, serve}, + server::create_drop_server, + state::AppState, +}; const CONTEXT_TTL: u64 = 10 * 60; @@ -28,7 +32,9 @@ async fn main() { let metrics = Handle::current().metrics(); info!("using {} threads", metrics.num_workers()); - let server = create_drop_server().await.expect("failed to connect to drop server"); + let server = create_drop_server() + .await + .expect("failed to connect to drop server"); let shared_state = Arc::new(AppState { context_cache: DashMap::new(), diff --git a/torrential/src/server/download.rs b/torrential/src/server/download.rs index 5a366455..6c71aaea 100644 --- a/torrential/src/server/download.rs +++ b/torrential/src/server/download.rs @@ -1,15 +1,37 @@ -use crate::{proto::{manifest::server_games_response::SkeletonGame, version::VersionResponse}, state::{AppState}, util::ErrorOption}; +use crate::{ + proto::{ + core::DropBoundType, + manifest::{ServerGamesQuery, ServerGamesResponse, server_games_response::SkeletonGame}, + version::{VersionQuery, VersionResponse}, + }, + state::AppState, + util::ErrorOption, +}; pub async fn fetch_version_data( app_state: &AppState, - game_id: String, + _game_id: String, version_id: String, ) -> Result { - unreachable!(); + let mut query = VersionQuery::new(); + query.version_id = version_id; + let message_id = app_state + .server + .send_message(DropBoundType::VERSION_QUERY, query, None) + .await?; + + let response: VersionResponse = app_state.server.wait_for_message_id(&message_id).await?; + + Ok(response) } -pub async fn fetch_instance_games( - app_state: &AppState, -) -> Result, ErrorOption> { - unreachable!() +pub async fn fetch_instance_games(app_state: &AppState) -> Result, ErrorOption> { + let message_id = app_state + .server + .send_message(DropBoundType::SERVER_GAMES_QUERY, ServerGamesQuery::new(), None) + .await?; + + let response: ServerGamesResponse = app_state.server.wait_for_message_id(&message_id).await?; + + return Ok(response.games); } diff --git a/torrential/src/server/mod.rs b/torrential/src/server/mod.rs index d59bc8c0..ca583c51 100644 --- a/torrential/src/server/mod.rs +++ b/torrential/src/server/mod.rs @@ -1,7 +1,8 @@ -use std::sync::{Arc, Mutex}; +use std::{mem, sync::Arc}; use anyhow::anyhow; -use protobuf::Message; +use log::{info, warn}; +use protobuf::{EnumOrUnknown, Message}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt as _, BufReader}, net::{ @@ -9,57 +10,102 @@ use tokio::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, }, spawn, + sync::Mutex, }; use waitmap::WaitMap; -use crate::proto::core::Response; +use crate::{ + droplet::manifest::generate_manifest, + proto::core::{DropBound, DropBoundType, TorrentialBound, TorrentialBoundType}, +}; pub mod download; pub struct DropServer { + server: TcpListener, write_stream: Mutex, - waitmap: WaitMap, + waitmap: WaitMap, } impl DropServer { + /** + Reads from the socket, and tries to parse it into a message, + and then updates the waitmap with the corresponding message ID + and content + */ + async fn recieve_loop( + myself: Arc, + buffered_reader: &mut BufReader, + ) -> Result<(), anyhow::Error> { + let mut length_buffer: [u8; 8] = [0; 8]; + buffered_reader.read_exact(&mut length_buffer).await?; + + let length = usize::from_le_bytes(length_buffer); + let mut buffer = vec![0; length]; + + buffered_reader.read_exact(&mut buffer).await?; + + let message = TorrentialBound::parse_from_bytes(&buffer) + .expect("response didn't deserialize correctly"); + + match message.type_.unwrap() { + TorrentialBoundType::GENERATE_MANIFEST => { + spawn(async move { generate_manifest(myself.clone(), message).await }); + } + _ => { + myself.waitmap.insert(message.message_id.clone(), message); + } + } + + Ok(()) + } + + /** + Long-lived subroutine that never returns, runs the recieve_loop and reconnects + as necessary + */ async fn recieve_subroutine(myself: Arc, read_stream: OwnedReadHalf) -> ! { let mut buffered_reader = BufReader::new(read_stream); loop { - let mut length_buffer: [u8; 8] = [0; 8]; - buffered_reader - .read_exact(&mut length_buffer) - .await - .expect("failed to read from internal pipe"); + if let Err(err) = Self::recieve_loop(myself.clone(), &mut buffered_reader).await { + warn!("server disconnected with error: {:?}", err); - let length = usize::from_le_bytes(length_buffer); - let mut buffer = Vec::with_capacity(length); + let (drop_stream, _) = myself + .server + .accept() + .await + .expect("failed to accept new listener"); + let (read, mut write) = drop_stream.into_split(); - buffered_reader - .read_exact(&mut buffer) - .await - .expect("failed to read from internal pipe"); + info!("reconnected to drop server"); - let message = - Response::parse_from_bytes(&buffer).expect("response didn't deserialize correctly"); - myself.waitmap.insert(message.message_id.clone(), message); + let mut lock = myself.write_stream.lock().await; + mem::swap(&mut *lock, &mut write); + + let mut new_reader = BufReader::new(read); + mem::swap(&mut buffered_reader, &mut new_reader); + } } } - async fn wait_for_message_id(&self, message_id: &str) -> Result + /** + Uses the waitmap to wait for a response from a query + */ + pub async fn wait_for_message_id(&self, message_id: &str) -> Result where T: protobuf::Message, { let message = self .waitmap - .wait(message_id.clone()) + .wait(message_id) .await .ok_or(anyhow!("no response returned for value"))?; let message = message.value(); match message.type_.unwrap() { - crate::proto::core::ResponseType::ERROR => { + crate::proto::core::TorrentialBoundType::ERROR => { return Err(anyhow!(String::from_utf8(message.data.clone()).unwrap())); } _ => { @@ -69,26 +115,41 @@ impl DropServer { } } - async fn send_message(&self, message: T) -> Result<(), anyhow::Error> + /** + Sends a message, returning the message ID + */ + pub async fn send_message( + &self, + message_type: DropBoundType, + message: T, + message_id: Option, + ) -> Result where T: protobuf::Message, { + let mut query = DropBound::new(); + query.message_id = message_id.unwrap_or(uuid::Uuid::new_v4().to_string()); + query.type_ = EnumOrUnknown::new(message_type); + query.data = Vec::new(); + message.write_to_vec(&mut query.data)?; + let mut buf = Vec::new(); - message.write_to_vec(&mut buf)?; + query.write_to_vec(&mut buf)?; { - let mut mutex_lock = self - .write_stream - .lock() - .expect("failed to lock send stream"); + let mut mutex_lock = self.write_stream.lock().await; mutex_lock.write(&buf.len().to_le_bytes()).await?; mutex_lock.write_all(&buf).await?; }; - Ok(()) + Ok(query.message_id) } } +/** +Spins up the TCP listener, and waits for the first client to connect +Also starts the recieve subroutine +*/ pub async fn create_drop_server() -> Result, anyhow::Error> { let server = TcpListener::bind("127.0.0.1:33148").await?; @@ -97,11 +158,14 @@ pub async fn create_drop_server() -> Result, anyhow::Error> { let (read, write) = drop_stream.into_split(); let client = Arc::new(DropServer { + server, write_stream: Mutex::new(write), waitmap: WaitMap::new(), }); spawn(DropServer::recieve_subroutine(client.clone(), read)); + info!("created client subroutine"); + Ok(client) } From 006ed0a1f83bf61806294962fba09fe58574239d Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 5 Feb 2026 19:17:24 +1100 Subject: [PATCH 16/21] fix: remove log line --- torrential/src/droplet/manifest.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/torrential/src/droplet/manifest.rs b/torrential/src/droplet/manifest.rs index 6af4f0a4..b91fb959 100644 --- a/torrential/src/droplet/manifest.rs +++ b/torrential/src/droplet/manifest.rs @@ -29,11 +29,7 @@ async fn generate_manifest_raw( message: TorrentialBound, ) -> Result<(), anyhow::Error> { let manifest_message = GenerateManifest::parse_from_bytes(&message.data)?; - info!( - "seven zip install: {}", - *droplet_rs::versions::backends::SEVEN_ZIP_INSTALLED - ); - + let manifest = droplet_rs::manifest::generate_manifest_rusty( &PathBuf::from(manifest_message.version_dir), |progress| { From 454fb19941bb03cc91dbe805e5289b68285a3ee4 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 5 Feb 2026 22:36:50 +1100 Subject: [PATCH 17/21] feat: rest of droplet calls --- torrential/proto/core.proto | 47 ++++++++----- torrential/proto/droplet.proto | 28 +++++++- torrential/src/droplet/backend.rs | 102 +++++++++++++++++++++++++++++ torrential/src/droplet/cert.rs | 71 ++++++++++++++++++++ torrential/src/droplet/manifest.rs | 40 +++-------- torrential/src/droplet/mod.rs | 32 ++++++++- torrential/src/server/mod.rs | 30 ++++++++- 7 files changed, 299 insertions(+), 51 deletions(-) create mode 100644 torrential/src/droplet/backend.rs diff --git a/torrential/proto/core.proto b/torrential/proto/core.proto index 2277396f..0b1da663 100644 --- a/torrential/proto/core.proto +++ b/torrential/proto/core.proto @@ -1,30 +1,45 @@ syntax = "proto3"; enum TorrentialBoundType { - ERROR = 0; - SERVER_GAMES_RESPONSE = 1; - VERSION_RESPONSE = 2; - GENERATE_MANIFEST = 3; + ERROR = 0; + SERVER_GAMES_RESPONSE = 1; + VERSION_RESPONSE = 2; + + GENERATE_MANIFEST = 3; + GENERATE_ROOT_CA = 4; + GENERATE_CLIENT_CERT = 5; + + HAS_BACKEND_QUERY = 6; + LIST_FILES_QUERY = 7; + PEEK_FILE_QUERY = 8; } message TorrentialBound { - string message_id = 1; - TorrentialBoundType type = 2; - bytes data = 3; + string message_id = 1; + TorrentialBoundType type = 2; + bytes data = 3; } enum DropBoundType { - SERVER_GAMES_QUERY = 0; - VERSION_QUERY = 1; + SERVER_GAMES_QUERY = 0; + VERSION_QUERY = 1; - MANIFEST_PROGRESS = 2; - MANIFEST_LOG = 3; - MANIFEST_COMPLETE = 4; - MANIFEST_ERROR = 5; + RPC_ERROR = 2; + + MANIFEST_PROGRESS = 3; + MANIFEST_LOG = 4; + MANIFEST_COMPLETE = 5; + + ROOT_CA_COMPLETE = 6; + CLIENT_CERT_COMPLETE = 7; + + HAS_BACKEND_COMPLETE = 8; + LIST_FILES_COMPLETE = 9; + PEEK_FILE_COMPLETE = 10; } message DropBound { - string message_id = 1; - DropBoundType type = 2; - bytes data = 3; + string message_id = 1; + DropBoundType type = 2; + bytes data = 3; } diff --git a/torrential/proto/droplet.proto b/torrential/proto/droplet.proto index ddb4da99..357a31f8 100644 --- a/torrential/proto/droplet.proto +++ b/torrential/proto/droplet.proto @@ -1,5 +1,9 @@ syntax = "proto3"; +message RpcError { + string error = 1; +} + /// Certificates message RootCertQuery {} message RootCertResponse { @@ -32,6 +36,26 @@ message ManifestLog { message ManifestComplete { string manifest = 1; } -message ManifestError { - string error = 1; + +/// Backend tools +message HasBackendQuery { + string path = 1; +} +message HasBackendResponse { + bool result = 1; +} + +message ListFilesQuery { + string path = 1; +} +message ListFilesResponse { + repeated string files = 1; +} + +message PeekFileQuery { + string path = 1; + string filename = 2; +} +message PeekFileResponse { + uint64 size = 1; } diff --git a/torrential/src/droplet/backend.rs b/torrential/src/droplet/backend.rs new file mode 100644 index 00000000..d91ba401 --- /dev/null +++ b/torrential/src/droplet/backend.rs @@ -0,0 +1,102 @@ +use std::{ + path::Path, + sync::{Arc, LazyLock}, +}; + +use anyhow::anyhow; +use dashmap::DashMap; +use droplet_rs::versions::types::VersionBackend; +use protobuf::Message; + +use crate::{ + proto::{ + core::{DropBoundType, TorrentialBound}, + droplet::{ + HasBackendQuery, HasBackendResponse, ListFilesQuery, ListFilesResponse, PeekFileQuery, + PeekFileResponse, + }, + }, + server::DropServer, +}; + +pub async fn has_backend_rpc( + server: Arc, + message: TorrentialBound, +) -> Result<(), anyhow::Error> { + let has_backend = HasBackendQuery::parse_from_bytes(&message.data)?; + + let has_backend = { + let path = Path::new(&has_backend.path); + let backend_constructor = droplet_rs::versions::create_backend_constructor(path); + + backend_constructor.is_some() + }; + + let mut response = HasBackendResponse::new(); + response.result = has_backend; + + server + .send_message( + DropBoundType::HAS_BACKEND_COMPLETE, + response, + Some(message.message_id), + ) + .await?; + + Ok(()) +} + +fn create_backend(path: &String) -> Result, anyhow::Error> { + let backend_constructor = droplet_rs::versions::create_backend_constructor(Path::new(path)) + .ok_or(anyhow!("backend doesn't exist at path {}", path))?; + let backend = backend_constructor()?; + + Ok(backend) +} + +pub async fn list_files_rpc( + server: Arc, + message: TorrentialBound, +) -> Result<(), anyhow::Error> { + let query = ListFilesQuery::parse_from_bytes(&message.data)?; + + let mut backend = create_backend(&query.path)?; + + let files = backend.list_files().await?; + + let mut response = ListFilesResponse::new(); + response.files = files.into_iter().map(|v| v.relative_filename).collect(); + + server + .send_message( + DropBoundType::LIST_FILES_COMPLETE, + response, + Some(message.message_id), + ) + .await?; + + Ok(()) +} + +pub async fn peek_file_rpc( + server: Arc, + message: TorrentialBound, +) -> Result<(), anyhow::Error> { + let query = PeekFileQuery::parse_from_bytes(&message.data)?; + + let mut backend = create_backend(&query.path)?; + let file_peek = backend.peek_file(query.filename).await?; + + let mut response = PeekFileResponse::new(); + response.size = file_peek.size; + + server + .send_message( + DropBoundType::PEEK_FILE_COMPLETE, + response, + Some(message.message_id), + ) + .await?; + + Ok(()) +} diff --git a/torrential/src/droplet/cert.rs b/torrential/src/droplet/cert.rs index e69de29b..80db5f4b 100644 --- a/torrential/src/droplet/cert.rs +++ b/torrential/src/droplet/cert.rs @@ -0,0 +1,71 @@ +use std::sync::Arc; + +use anyhow::anyhow; +use protobuf::Message as _; + +use crate::{ + proto::{ + core::{DropBoundType, TorrentialBound}, + droplet::{ClientCertQuery, ClientCertResponse, RootCertResponse}, + }, + server::DropServer, +}; + +pub async fn generate_root_ca_rpc( + server: Arc, + message: TorrentialBound, +) -> Result<(), anyhow::Error> { + let manifest = droplet_rs::ssl::generate_root_ca()?; + let mut manifest = manifest.into_iter(); + + let mut root_ca = RootCertResponse::new(); + root_ca.cert = manifest + .next() + .ok_or(anyhow!("root ca generation missing cert"))?; + root_ca.priv_ = manifest + .next() + .ok_or(anyhow!("root ca generation missing priv"))?; + + server + .send_message( + DropBoundType::ROOT_CA_COMPLETE, + root_ca, + Some(message.message_id), + ) + .await?; + + Ok(()) +} + +pub async fn generate_client_cert_rpc( + server: Arc, + message: TorrentialBound, +) -> Result<(), anyhow::Error> { + let generate_message = ClientCertQuery::parse_from_bytes(&message.data)?; + + let cert = droplet_rs::ssl::generate_client_certificate( + generate_message.client_id, + generate_message.client_name, + generate_message.root_cert, + generate_message.root_priv, + )?; + let mut cert = cert.into_iter(); + + let mut client_cert = ClientCertResponse::new(); + client_cert.cert = cert + .next() + .ok_or(anyhow!("client cert generation missing cert"))?; + client_cert.priv_ = cert + .next() + .ok_or(anyhow!("client cert generation missing priv"))?; + + server + .send_message( + DropBoundType::CLIENT_CERT_COMPLETE, + client_cert, + Some(message.message_id), + ) + .await?; + + Ok(()) +} diff --git a/torrential/src/droplet/manifest.rs b/torrential/src/droplet/manifest.rs index b91fb959..688a9ce4 100644 --- a/torrential/src/droplet/manifest.rs +++ b/torrential/src/droplet/manifest.rs @@ -3,33 +3,35 @@ use std::{ sync::{Arc, LazyLock}, }; -use log::{info, warn}; +use log::info; use protobuf::Message; use serde_json::json; use tokio::{spawn, sync::Semaphore}; use crate::{ - droplet, proto::{ core::{DropBoundType, TorrentialBound}, - droplet::{ - GenerateManifest, ManifestComplete, ManifestError, ManifestLog, ManifestProgress, - }, + droplet::{GenerateManifest, ManifestComplete, ManifestLog, ManifestProgress}, }, server::DropServer, }; static READER_SEMAPHORE: LazyLock = LazyLock::new(|| { - let cores = num_cpus::get(); + let cores = std::env::var("READER_THREADS") + .ok() + .map(|v| str::parse::(&v).ok()) + .flatten() + .unwrap_or(num_cpus::get() / 2); + info!("using {} import threads", cores); Semaphore::new(cores) }); -async fn generate_manifest_raw( +pub async fn generate_manifest_rpc( server: Arc, message: TorrentialBound, ) -> Result<(), anyhow::Error> { let manifest_message = GenerateManifest::parse_from_bytes(&message.data)?; - + let manifest = droplet_rs::manifest::generate_manifest_rusty( &PathBuf::from(manifest_message.version_dir), |progress| { @@ -77,25 +79,3 @@ async fn generate_manifest_raw( Ok(()) } - -pub async fn generate_manifest(server: Arc, message: TorrentialBound) { - let message_id = message.message_id.clone(); - warn!("generating manifest..."); - let result = generate_manifest_raw(server.clone(), message).await; - info!("manifest generation exited"); - if let Err(err) = result { - warn!("manifest generation failed with err: {:?}", err); - let mut manifest_err = ManifestError::new(); - manifest_err.error = err.to_string(); - let _ = server - .send_message( - DropBoundType::MANIFEST_ERROR, - manifest_err, - Some(message_id), - ) - .await - .inspect_err(|err| { - warn!("failed to send manifest err: {err:?}"); - }); - } -} diff --git a/torrential/src/droplet/mod.rs b/torrential/src/droplet/mod.rs index e07a6a54..9b636aac 100644 --- a/torrential/src/droplet/mod.rs +++ b/torrential/src/droplet/mod.rs @@ -1,2 +1,32 @@ -pub mod manifest; +use std::sync::Arc; + +use log::{info, warn}; + +use crate::{proto::{core::{DropBoundType, TorrentialBound}, droplet::RpcError}, server::DropServer}; + pub mod cert; +pub mod manifest; +pub mod backend; + +pub async fn call_rpc(server: Arc, message: TorrentialBound, rpc: T) +where + T: AsyncFn(Arc, TorrentialBound) -> Result<(), anyhow::Error>, +{ + let message_id = message.message_id.clone(); + let result = rpc(server.clone(), message).await; + if let Err(err) = result { + warn!("manifest generation failed with err: {:?}", err); + let mut manifest_err = RpcError::new(); + manifest_err.error = err.to_string(); + let _ = server + .send_message( + DropBoundType::RPC_ERROR, + manifest_err, + Some(message_id), + ) + .await + .inspect_err(|err| { + warn!("failed to send manifest err: {err:?}"); + }); + } +} diff --git a/torrential/src/server/mod.rs b/torrential/src/server/mod.rs index ca583c51..fbe602ae 100644 --- a/torrential/src/server/mod.rs +++ b/torrential/src/server/mod.rs @@ -15,12 +15,23 @@ use tokio::{ use waitmap::WaitMap; use crate::{ - droplet::manifest::generate_manifest, + droplet::{ + backend::{has_backend_rpc, list_files_rpc, peek_file_rpc}, + call_rpc, + cert::generate_client_cert_rpc, + manifest::generate_manifest_rpc, + }, proto::core::{DropBound, DropBoundType, TorrentialBound, TorrentialBoundType}, }; pub mod download; +macro_rules! spawn_rpc { + ($myself:ident, $message:ident, $func_name:ident) => { + spawn(async move { call_rpc($myself.clone(), $message, $func_name).await }); + }; +} + pub struct DropServer { server: TcpListener, write_stream: Mutex, @@ -50,7 +61,22 @@ impl DropServer { match message.type_.unwrap() { TorrentialBoundType::GENERATE_MANIFEST => { - spawn(async move { generate_manifest(myself.clone(), message).await }); + spawn_rpc!(myself, message, generate_manifest_rpc); + } + TorrentialBoundType::GENERATE_ROOT_CA => { + spawn_rpc!(myself, message, generate_manifest_rpc); + } + TorrentialBoundType::GENERATE_CLIENT_CERT => { + spawn_rpc!(myself, message, generate_client_cert_rpc); + } + TorrentialBoundType::LIST_FILES_QUERY => { + spawn_rpc!(myself, message, list_files_rpc); + } + TorrentialBoundType::HAS_BACKEND_QUERY => { + spawn_rpc!(myself, message, has_backend_rpc); + } + TorrentialBoundType::PEEK_FILE_QUERY => { + spawn_rpc!(myself, message, peek_file_rpc); } _ => { myself.waitmap.insert(message.message_id.clone(), message); From c84ead096987fa7014664f6c73129d795ca812ba Mon Sep 17 00:00:00 2001 From: DecDuck Date: Thu, 5 Feb 2026 22:40:00 +1100 Subject: [PATCH 18/21] feat: formatting --- torrential/build.rs | 9 ++++----- torrential/src/downloads/download.rs | 5 +++-- torrential/src/downloads/handlers.rs | 6 +++--- torrential/src/downloads/serve.rs | 8 ++++---- torrential/src/droplet/backend.rs | 5 ++--- torrential/src/droplet/manifest.rs | 5 ++--- torrential/src/droplet/mod.rs | 4 ++-- torrential/src/main.rs | 14 ++++++++------ torrential/src/server/download.rs | 2 +- torrential/src/server/mod.rs | 17 +++++++---------- torrential/src/state.rs | 3 +-- 11 files changed, 37 insertions(+), 41 deletions(-) diff --git a/torrential/build.rs b/torrential/build.rs index 23828b9f..2481ed17 100644 --- a/torrential/build.rs +++ b/torrential/build.rs @@ -2,18 +2,17 @@ use std::fs::{self, read_dir}; use protobuf_codegen::Codegen; -const OUT_DIR: &'static str = "./src/proto/"; +const OUT_DIR: &str = "./src/proto/"; fn main() { let files = read_dir("./proto").unwrap(); let files = files.map(|v| format!("proto/{}", v.unwrap().file_name().into_string().unwrap())); - read_dir(OUT_DIR).unwrap().into_iter().for_each(|v| { - if let Ok(entry) = v { - if entry.file_name().to_str().unwrap().ends_with(".rs") { + read_dir(OUT_DIR).unwrap().for_each(|v| { + if let Ok(entry) = v + && entry.file_name().to_str().unwrap().ends_with(".rs") { fs::remove_file(entry.path()).unwrap(); } - } }); Codegen::new() diff --git a/torrential/src/downloads/download.rs b/torrential/src/downloads/download.rs index 91134f68..d8cd4551 100644 --- a/torrential/src/downloads/download.rs +++ b/torrential/src/downloads/download.rs @@ -22,11 +22,12 @@ pub struct DownloadContext { last_access: Instant, } impl DownloadContext { + #[must_use] pub fn last_access(&self) -> Instant { self.last_access } pub fn reset_last_access(&mut self) { - self.last_access = Instant::now() + self.last_access = Instant::now(); } } @@ -71,7 +72,7 @@ fn create_backend( create_backend_constructor(&version_path).ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; let backend = backend() - .inspect_err(|err| warn!("failed to create version backend: {:?}", err)) + .inspect_err(|err| warn!("failed to create version backend: {err:?}")) .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; Ok(backend) } diff --git a/torrential/src/downloads/handlers.rs b/torrential/src/downloads/handlers.rs index ad597bf1..f94064de 100644 --- a/torrential/src/downloads/handlers.rs +++ b/torrential/src/downloads/handlers.rs @@ -67,9 +67,9 @@ impl AsyncRead for SpeedtestStream { match amount { Ok(amount) => self.remaining -= amount, Err(err) => return Poll::Ready(Err(err)), - }; - }; - return Poll::Ready(Ok(())); + } + } + Poll::Ready(Ok(())) } } diff --git a/torrential/src/downloads/serve.rs b/torrential/src/downloads/serve.rs index d700a75f..5847826c 100644 --- a/torrential/src/downloads/serve.rs +++ b/torrential/src/downloads/serve.rs @@ -49,7 +49,7 @@ impl<'a, T: Stream> SemaphoreStream<'a, T> { } } -impl<'a, T: Stream> Stream for SemaphoreStream<'a, T> +impl Stream for SemaphoreStream<'_, T> where T: Stream, { @@ -160,7 +160,7 @@ async fn get_file_reader( ) .await .map_err(|v| { - error!("reader error for '{}': {v:?}", relative_filename); + error!("reader error for '{relative_filename}': {v:?}"); StatusCode::INTERNAL_SERVER_ERROR }) } @@ -181,13 +181,13 @@ async fn get_or_create_context<'a>( if let Some(already_done) = context_cache.get_mut(&key) { Ok(already_done) } else { - info!("generating context for {}...", game_id); + info!("generating context for {game_id}..."); let context_result = create_download_context(state, game_id.clone(), version_name.clone()).await?; state.context_cache.insert(key.clone(), context_result); - info!("continuing download for {}", game_id); + info!("continuing download for {game_id}"); drop(permit); diff --git a/torrential/src/droplet/backend.rs b/torrential/src/droplet/backend.rs index d91ba401..5cda2ce2 100644 --- a/torrential/src/droplet/backend.rs +++ b/torrential/src/droplet/backend.rs @@ -1,10 +1,9 @@ use std::{ path::Path, - sync::{Arc, LazyLock}, + sync::Arc, }; use anyhow::anyhow; -use dashmap::DashMap; use droplet_rs::versions::types::VersionBackend; use protobuf::Message; @@ -48,7 +47,7 @@ pub async fn has_backend_rpc( fn create_backend(path: &String) -> Result, anyhow::Error> { let backend_constructor = droplet_rs::versions::create_backend_constructor(Path::new(path)) - .ok_or(anyhow!("backend doesn't exist at path {}", path))?; + .ok_or(anyhow!("backend doesn't exist at path {path}"))?; let backend = backend_constructor()?; Ok(backend) diff --git a/torrential/src/droplet/manifest.rs b/torrential/src/droplet/manifest.rs index 688a9ce4..b2f11e8c 100644 --- a/torrential/src/droplet/manifest.rs +++ b/torrential/src/droplet/manifest.rs @@ -19,10 +19,9 @@ use crate::{ static READER_SEMAPHORE: LazyLock = LazyLock::new(|| { let cores = std::env::var("READER_THREADS") .ok() - .map(|v| str::parse::(&v).ok()) - .flatten() + .and_then(|v| str::parse::(&v).ok()) .unwrap_or(num_cpus::get() / 2); - info!("using {} import threads", cores); + info!("using {cores} import threads"); Semaphore::new(cores) }); diff --git a/torrential/src/droplet/mod.rs b/torrential/src/droplet/mod.rs index 9b636aac..5909f874 100644 --- a/torrential/src/droplet/mod.rs +++ b/torrential/src/droplet/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use log::{info, warn}; +use log::warn; use crate::{proto::{core::{DropBoundType, TorrentialBound}, droplet::RpcError}, server::DropServer}; @@ -15,7 +15,7 @@ where let message_id = message.message_id.clone(); let result = rpc(server.clone(), message).await; if let Err(err) = result { - warn!("manifest generation failed with err: {:?}", err); + warn!("manifest generation failed with err: {err:?}"); let mut manifest_err = RpcError::new(); manifest_err.error = err.to_string(); let _ = server diff --git a/torrential/src/main.rs b/torrential/src/main.rs index bfdf92e4..45ea7e90 100644 --- a/torrential/src/main.rs +++ b/torrential/src/main.rs @@ -25,7 +25,7 @@ async fn main() { initialise_logger(); if let Ok(working_directory) = std::env::var("WORKING_DIRECTORY") { - info!("moving to working directory {}", working_directory); + info!("moving to working directory {working_directory}"); set_current_dir(working_directory).expect("failed to change working directory"); } @@ -38,7 +38,7 @@ async fn main() { let shared_state = Arc::new(AppState { context_cache: DashMap::new(), - server: server, + server, }); let interval_shared_state = shared_state.clone(); @@ -62,7 +62,7 @@ async fn main() { }; if last_access.elapsed().as_secs() >= CONTEXT_TTL { shared_state.context_cache.remove(&key); - info!("cleaned context: {:?}", key); + info!("cleaned context: {key:?}"); } } } @@ -70,7 +70,7 @@ async fn main() { let app = setup_app(shared_state); - serve(app).await.unwrap(); + serve(app).await.expect("failed to serve app"); } fn setup_app(shared_state: Arc) -> Router { @@ -87,7 +87,9 @@ fn setup_app(shared_state: Arc) -> Router { } async fn serve(app: Router) -> Result<(), std::io::Error> { - let listener = tokio::net::TcpListener::bind("0.0.0.0:5000").await.unwrap(); + let listener = tokio::net::TcpListener::bind("0.0.0.0:5000") + .await + .expect("failed to bind tcp server"); info!("started depot server"); axum::serve(listener, app).await } @@ -96,5 +98,5 @@ fn initialise_logger() { SimpleLogger::new() .with_level(log::LevelFilter::Info) .init() - .unwrap(); + .expect("failed to init logger"); } diff --git a/torrential/src/server/download.rs b/torrential/src/server/download.rs index 6c71aaea..51c1cb2a 100644 --- a/torrential/src/server/download.rs +++ b/torrential/src/server/download.rs @@ -33,5 +33,5 @@ pub async fn fetch_instance_games(app_state: &AppState) -> Result, read_stream: OwnedReadHalf) -> ! { @@ -95,7 +95,7 @@ impl DropServer { loop { if let Err(err) = Self::recieve_loop(myself.clone(), &mut buffered_reader).await { - warn!("server disconnected with error: {:?}", err); + warn!("server disconnected with error: {err:?}"); let (drop_stream, _) = myself .server @@ -130,14 +130,11 @@ impl DropServer { let message = message.value(); - match message.type_.unwrap() { - crate::proto::core::TorrentialBoundType::ERROR => { - return Err(anyhow!(String::from_utf8(message.data.clone()).unwrap())); - } - _ => { - let response = T::parse_from_bytes(&message.data)?; - return Ok(response); - } + if message.type_.unwrap() == crate::proto::core::TorrentialBoundType::ERROR { + Err(anyhow!(String::from_utf8(message.data.clone()).unwrap())) + } else { + let response = T::parse_from_bytes(&message.data)?; + Ok(response) } } diff --git a/torrential/src/state.rs b/torrential/src/state.rs index f55cdc7d..9d2ca0d6 100644 --- a/torrential/src/state.rs +++ b/torrential/src/state.rs @@ -1,7 +1,6 @@ -use std::{collections::HashMap, path::PathBuf, sync::Arc}; +use std::sync::Arc; use dashmap::DashMap; -use tokio::sync::{OnceCell, Semaphore}; use crate::{DownloadContext, server::DropServer}; From 838c39e849ba3332486f6566f8f9dd14ef13151f Mon Sep 17 00:00:00 2001 From: Huskydog9988 <39809509+Huskydog9988@users.noreply.github.com> Date: Sun, 8 Feb 2026 11:54:08 -0500 Subject: [PATCH 19/21] fix build failing due to missing command --- torrential/Cargo.lock | 65 +++++++++++++++++++++++++++++++++++++++++++ torrential/Cargo.toml | 1 + torrential/build.rs | 1 + 3 files changed, 67 insertions(+) diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock index c9de6188..8e8eb25d 100644 --- a/torrential/Cargo.lock +++ b/torrential/Cargo.lock @@ -1424,6 +1424,70 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "protoc-bin-vendored" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa" +dependencies = [ + "protoc-bin-vendored-linux-aarch_64", + "protoc-bin-vendored-linux-ppcle_64", + "protoc-bin-vendored-linux-s390_64", + "protoc-bin-vendored-linux-x86_32", + "protoc-bin-vendored-linux-x86_64", + "protoc-bin-vendored-macos-aarch_64", + "protoc-bin-vendored-macos-x86_64", + "protoc-bin-vendored-win32", +] + +[[package]] +name = "protoc-bin-vendored-linux-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c" + +[[package]] +name = "protoc-bin-vendored-linux-ppcle_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c" + +[[package]] +name = "protoc-bin-vendored-linux-s390_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0" + +[[package]] +name = "protoc-bin-vendored-linux-x86_32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5" + +[[package]] +name = "protoc-bin-vendored-linux-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78" + +[[package]] +name = "protoc-bin-vendored-macos-aarch_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092" + +[[package]] +name = "protoc-bin-vendored-macos-x86_64" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756" + +[[package]] +name = "protoc-bin-vendored-win32" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" + [[package]] name = "quinn" version = "0.11.9" @@ -2125,6 +2189,7 @@ dependencies = [ "pin-project-lite", "protobuf", "protobuf-codegen", + "protoc-bin-vendored", "rand", "reqwest", "serde", diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml index d2603d75..db3fe6dc 100644 --- a/torrential/Cargo.toml +++ b/torrential/Cargo.toml @@ -66,3 +66,4 @@ tempfile = "3.23.0" [build-dependencies] protobuf-codegen = "3.7.2" +protoc-bin-vendored = "3.2.0" diff --git a/torrential/build.rs b/torrential/build.rs index 2481ed17..5f4b1477 100644 --- a/torrential/build.rs +++ b/torrential/build.rs @@ -16,6 +16,7 @@ fn main() { }); Codegen::new() + .protoc_path(&protoc_bin_vendored::protoc_bin_path().unwrap()) .inputs(files) .include("proto") .out_dir(OUT_DIR) From 116b2183bb6ec74f159dd8265a25c8c2401b8212 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Wed, 25 Feb 2026 02:14:18 +1100 Subject: [PATCH 20/21] fix: root ca gen and warning message --- torrential/src/droplet/mod.rs | 3 ++- torrential/src/server/mod.rs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/torrential/src/droplet/mod.rs b/torrential/src/droplet/mod.rs index 5909f874..a80abaa3 100644 --- a/torrential/src/droplet/mod.rs +++ b/torrential/src/droplet/mod.rs @@ -13,9 +13,10 @@ where T: AsyncFn(Arc, TorrentialBound) -> Result<(), anyhow::Error>, { let message_id = message.message_id.clone(); + let message_type = message.type_.enum_value().unwrap(); let result = rpc(server.clone(), message).await; if let Err(err) = result { - warn!("manifest generation failed with err: {err:?}"); + warn!("rpc call for {message_type:?} failed with error: {err:?}"); let mut manifest_err = RpcError::new(); manifest_err.error = err.to_string(); let _ = server diff --git a/torrential/src/server/mod.rs b/torrential/src/server/mod.rs index a73f9612..e98ce058 100644 --- a/torrential/src/server/mod.rs +++ b/torrential/src/server/mod.rs @@ -18,7 +18,7 @@ use crate::{ droplet::{ backend::{has_backend_rpc, list_files_rpc, peek_file_rpc}, call_rpc, - cert::generate_client_cert_rpc, + cert::{generate_client_cert_rpc, generate_root_ca_rpc}, manifest::generate_manifest_rpc, }, proto::core::{DropBound, DropBoundType, TorrentialBound, TorrentialBoundType}, @@ -64,7 +64,7 @@ impl DropServer { spawn_rpc!(myself, message, generate_manifest_rpc); } TorrentialBoundType::GENERATE_ROOT_CA => { - spawn_rpc!(myself, message, generate_manifest_rpc); + spawn_rpc!(myself, message, generate_root_ca_rpc); } TorrentialBoundType::GENERATE_CLIENT_CERT => { spawn_rpc!(myself, message, generate_client_cert_rpc); From ff17d4256a885af94072b4b12e6b0485ae0a5602 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sun, 1 Mar 2026 22:56:52 +1100 Subject: [PATCH 21/21] feat: switch to libarchive backend --- torrential/Cargo.lock | 204 +++++++++++++++++++++++++---- torrential/Cargo.toml | 2 +- torrential/src/droplet/manifest.rs | 7 +- 3 files changed, 180 insertions(+), 33 deletions(-) diff --git a/torrential/Cargo.lock b/torrential/Cargo.lock index 8e8eb25d..55e8bbab 100644 --- a/torrential/Cargo.lock +++ b/torrential/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aes" version = "0.8.4" @@ -98,8 +113,8 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 2.0.111", + "synstructure 0.13.2", ] [[package]] @@ -110,8 +125,8 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 2.0.111", + "synstructure 0.13.2", ] [[package]] @@ -122,7 +137,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -133,7 +148,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -209,7 +224,22 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", +] + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if 1.0.4", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", ] [[package]] @@ -551,14 +581,13 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] name = "droplet-rs" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef23d187f7153bc3ce1c6a5440be1206e2aaf42b153825ddf2f8151f278539c5" +version = "0.16.2" +source = "git+https://github.com/Drop-OSS/droplet-rs.git#dfda1781ac39937d8f3773631b3f855466e19a2c" dependencies = [ "anyhow", "async-trait", @@ -567,11 +596,13 @@ dependencies = [ "getrandom 0.3.4", "hex", "humansize", + "libarchive-drop", "rcgen", "ring", "serde", "serde_json", "sha2", + "speedometer", "time", "tokio", "uuid", @@ -603,7 +634,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", ] [[package]] @@ -692,7 +745,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -762,6 +815,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "half" version = "2.7.1" @@ -1091,6 +1150,25 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "libarchive-drop" +version = "0.1.1" +source = "git+https://github.com/Drop-OSS/droplet-rs.git#dfda1781ac39937d8f3773631b3f855466e19a2c" +dependencies = [ + "libarchive3-sys", + "libc", +] + +[[package]] +name = "libarchive3-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd3beae8f59a4c7a806523269b5392037577c150446e88d684dfa6de6031ca7" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "libc" version = "0.2.178" @@ -1166,6 +1244,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.1.0" @@ -1231,6 +1318,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "oid-registry" version = "0.7.1" @@ -1312,6 +1408,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "plotters" version = "0.3.7" @@ -1540,7 +1642,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -1720,6 +1822,12 @@ dependencies = [ "libc", ] +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -1758,7 +1866,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1850,7 +1958,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1948,6 +2056,15 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "speedometer" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2789736092fa21b44baf8590acb4b360cb91f0f597bd6c1f1741ca9644c95c1e" +dependencies = [ + "failure", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1960,6 +2077,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.111" @@ -1980,6 +2108,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1988,7 +2128,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2001,7 +2141,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2030,7 +2170,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2041,7 +2181,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2143,7 +2283,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2287,6 +2427,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -2417,7 +2563,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.111", "wasm-bindgen-shared", ] @@ -2493,7 +2639,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2749,8 +2895,8 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 2.0.111", + "synstructure 0.13.2", ] [[package]] @@ -2770,7 +2916,7 @@ checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -2790,8 +2936,8 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 2.0.111", + "synstructure 0.13.2", ] [[package]] @@ -2830,5 +2976,5 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] diff --git a/torrential/Cargo.toml b/torrential/Cargo.toml index db3fe6dc..e73f387e 100644 --- a/torrential/Cargo.toml +++ b/torrential/Cargo.toml @@ -23,7 +23,7 @@ simple_logger = { version = "5.1.0", default-features = false, features = [ "colors", ] } tokio = { version = "*", features = ["rt-multi-thread", "sync"] } -droplet-rs = "0.15.1" +droplet-rs = { version = "0.16.2", git = "https://github.com/Drop-OSS/droplet-rs.git" } dashmap = "6.1.0" anyhow = "1.0.100" serde_json = "1.0.145" diff --git a/torrential/src/droplet/manifest.rs b/torrential/src/droplet/manifest.rs index b2f11e8c..a27ed612 100644 --- a/torrential/src/droplet/manifest.rs +++ b/torrential/src/droplet/manifest.rs @@ -16,13 +16,14 @@ use crate::{ server::DropServer, }; -static READER_SEMAPHORE: LazyLock = LazyLock::new(|| { +// TODO: re-write the droplet interface so it takes a 'static reference instead of an arc +static READER_SEMAPHORE: LazyLock> = LazyLock::new(|| { let cores = std::env::var("READER_THREADS") .ok() .and_then(|v| str::parse::(&v).ok()) .unwrap_or(num_cpus::get() / 2); info!("using {cores} import threads"); - Semaphore::new(cores) + Arc::new(Semaphore::new(cores)) }); pub async fn generate_manifest_rpc( @@ -61,7 +62,7 @@ pub async fn generate_manifest_rpc( .await; }); }, - Some(&READER_SEMAPHORE), + Some(READER_SEMAPHORE.clone()), ) .await?;