From 8c8e9ad4c9ccce16091d4bedc42f23f5369c9ce0 Mon Sep 17 00:00:00 2001 From: quexeky Date: Sun, 25 Jan 2026 21:04:03 +1100 Subject: [PATCH] feat: Migrate to Apache opendal --- cli/Cargo.lock | 687 ++++++------------ cli/Cargo.toml | 4 +- cli/src/cli.rs | 4 +- cli/src/commands/configure/config_option.rs | 18 - cli/src/commands/configure/configurable.rs | 5 - cli/src/commands/configure/s3.rs | 65 -- cli/src/commands/configure/server.rs | 92 --- .../commands/{configure => connect}/config.rs | 65 +- cli/src/commands/connect/config_option.rs | 26 + cli/src/commands/connect/configurable.rs | 5 + .../{configure => connect}/interactive.rs | 4 +- .../commands/{configure => connect}/mod.rs | 1 - cli/src/commands/connect/s3.rs | 66 ++ cli/src/commands/mod.rs | 2 +- cli/src/commands/upload/interface.rs | 26 +- cli/src/commands/upload/mod.rs | 1 - cli/src/commands/upload/s3.rs | 84 --- cli/src/commands/upload/uploadable.rs | 19 +- cli/src/main.rs | 6 +- cli/src/manifest.rs | 35 +- 20 files changed, 443 insertions(+), 772 deletions(-) delete mode 100644 cli/src/commands/configure/config_option.rs delete mode 100644 cli/src/commands/configure/configurable.rs delete mode 100644 cli/src/commands/configure/s3.rs delete mode 100644 cli/src/commands/configure/server.rs rename cli/src/commands/{configure => connect}/config.rs (60%) create mode 100644 cli/src/commands/connect/config_option.rs create mode 100644 cli/src/commands/connect/configurable.rs rename cli/src/commands/{configure => connect}/interactive.rs (86%) rename cli/src/commands/{configure => connect}/mod.rs (86%) create mode 100644 cli/src/commands/connect/s3.rs delete mode 100644 cli/src/commands/upload/s3.rs diff --git a/cli/Cargo.lock b/cli/Cargo.lock index a1ca3993..50755aba 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -151,44 +151,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "attohttpc" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" -dependencies = [ - "base64", - "http", - "log", - "native-tls", - "serde", - "serde_json", - "url", -] - [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "aws-creds" -version = "0.39.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3b85155d265df828f84e53886ed9e427aed979dd8a39f5b8b2162c77e142d7" -dependencies = [ - "attohttpc", - "home", - "log", - "quick-xml", - "rust-ini", - "serde", - "thiserror 2.0.17", - "time", - "url", -] - [[package]] name = "aws-lc-rs" version = "1.15.2" @@ -212,12 +180,14 @@ dependencies = [ ] [[package]] -name = "aws-region" -version = "0.28.1" +name = "backon" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "838b36c8dc927b6db1b6c6b8f5d05865f2213550b9e83bf92fa99ed6525472c0" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ - "thiserror 2.0.17", + "fastrand", + "gloo-timers", + "tokio", ] [[package]] @@ -253,15 +223,6 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" -[[package]] -name = "castaway" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" -dependencies = [ - "rustversion", -] - [[package]] name = "cc" version = "1.2.49" @@ -302,7 +263,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -380,19 +341,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "compact_str" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f86b9c4c00838774a6d902ef931eff7470720c51d90c2e32cfe15dc304737b3f" -dependencies = [ - "castaway", - "cfg-if", - "itoa", - "ryu", - "static_assertions", -] - [[package]] name = "console" version = "0.16.2" @@ -406,6 +354,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-random" version = "0.1.18" @@ -461,6 +415,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -518,7 +481,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", - "serde_core", ] [[package]] @@ -540,6 +502,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] @@ -601,9 +564,9 @@ dependencies = [ "futures", "indicatif", "log", - "rand", + "opendal", + "rand 0.9.2", "reqwest 0.13.1", - "rust-s3", "serde", "serde_json", "tokio", @@ -706,21 +669,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -862,6 +810,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.13" @@ -1007,22 +967,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", -] - -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", + "webpki-roots", ] [[package]] @@ -1063,7 +1008,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core", ] [[package]] @@ -1228,6 +1173,47 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jiff" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys 0.61.2", +] + +[[package]] +name = "jiff-static" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.21.1" @@ -1323,22 +1309,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] -name = "maybe-async" -version = "0.2.10" +name = "md-5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "proc-macro2", - "quote", - "syn", + "cfg-if", + "digest", ] -[[package]] -name = "md5" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" - [[package]] name = "memchr" version = "2.7.6" @@ -1351,15 +1330,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minidom" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e394a0e3c7ccc2daea3dffabe82f09857b6b510cb25af87d54bf3e910ac1642d" -dependencies = [ - "rxml", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1377,23 +1347,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe 0.1.6", - "openssl-sys", - "schannel", - "security-framework 2.11.1", - "security-framework-sys", - "tempfile", -] - [[package]] name = "ndk-context" version = "0.1.1" @@ -1410,15 +1363,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "ntapi" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081" -dependencies = [ - "winapi", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -1462,15 +1406,6 @@ dependencies = [ "objc2-encode", ] -[[package]] -name = "objc2-core-foundation" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" -dependencies = [ - "bitflags", -] - [[package]] name = "objc2-encode" version = "4.1.0" @@ -1487,16 +1422,6 @@ dependencies = [ "objc2", ] -[[package]] -name = "objc2-io-kit" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" -dependencies = [ - "libc", - "objc2-core-foundation", -] - [[package]] name = "oid-registry" version = "0.7.1" @@ -1528,55 +1453,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] -name = "openssl" -version = "0.10.75" +name = "opendal" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", + "anyhow", + "backon", + "base64", + "bytes", + "crc32c", + "futures", + "getrandom 0.2.16", + "http", + "http-body", + "jiff", + "log", + "md-5", + "percent-encoding", + "quick-xml 0.38.4", + "reqsign", + "reqwest 0.12.28", + "serde", + "serde_json", + "tokio", + "url", + "uuid", ] -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - [[package]] name = "openssl-probe" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391" -[[package]] -name = "openssl-sys" -version = "0.9.111" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "option-ext" version = "0.2.0" @@ -1621,18 +1531,21 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - [[package]] name = "portable-atomic" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -1666,6 +1579,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick-xml" version = "0.38.4" @@ -1706,7 +1629,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand", + "rand 0.9.2", "ring", "rustc-hash", "rustls", @@ -1747,14 +1670,35 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -1764,7 +1708,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", ] [[package]] @@ -1801,6 +1754,35 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "reqsign" +version = "0.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +dependencies = [ + "anyhow", + "async-trait", + "base64", + "chrono", + "form_urlencoded", + "getrandom 0.2.16", + "hex", + "hmac", + "home", + "http", + "log", + "percent-encoding", + "quick-xml 0.37.5", + "rand 0.8.5", + "reqwest 0.12.28", + "rust-ini", + "serde", + "serde_json", + "sha1", + "sha2", + "tokio", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -1815,20 +1797,21 @@ dependencies = [ "http-body", "http-body-util", "hyper", - "hyper-tls", + "hyper-rustls", "hyper-util", "js-sys", "log", - "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-native-tls", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -1838,6 +1821,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots", ] [[package]] @@ -1904,47 +1888,21 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-s3" -version = "0.37.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4af74047374528b627109d579ce86b23ccf6ffba7ff363c807126c1aff69e1bb" -dependencies = [ - "async-trait", - "aws-creds", - "aws-region", - "base64", - "bytes", - "cfg-if", - "futures-util", - "hex", - "hmac", - "http", - "log", - "maybe-async", - "md5", - "minidom", - "percent-encoding", - "quick-xml", - "reqwest 0.12.28", - "serde", - "serde_derive", - "serde_json", - "sha2", - "sysinfo", - "thiserror 2.0.17", - "time", - "tokio", - "tokio-stream", - "url", -] - [[package]] name = "rustc-hash" version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -1975,6 +1933,7 @@ checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -1987,10 +1946,10 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "openssl-probe 0.2.0", + "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework", ] [[package]] @@ -2018,7 +1977,7 @@ dependencies = [ "rustls-native-certs", "rustls-platform-verifier-android", "rustls-webpki", - "security-framework 3.5.1", + "security-framework", "security-framework-sys", "webpki-root-certs", "windows-sys 0.61.2", @@ -2048,25 +2007,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" -[[package]] -name = "rxml" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc94b580d0f5a6b7a2d604e597513d3c673154b52ddeccd1d5c32360d945ee" -dependencies = [ - "bytes", - "rxml_validation", -] - -[[package]] -name = "rxml_validation" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e80413b9a35e9d33217b3dcac04cf95f6559d15944b93887a08be5496c4a4" -dependencies = [ - "compact_str", -] - [[package]] name = "ryu" version = "1.0.22" @@ -2091,19 +2031,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - [[package]] name = "security-framework" version = "3.5.1" @@ -2127,6 +2054,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + [[package]] name = "serde" version = "1.0.228" @@ -2182,6 +2115,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2242,12 +2186,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "strsim" version = "0.11.1" @@ -2291,20 +2229,6 @@ dependencies = [ "syn", ] -[[package]] -name = "sysinfo" -version = "0.37.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" -dependencies = [ - "libc", - "memchr", - "ntapi", - "objc2-core-foundation", - "objc2-io-kit", - "windows", -] - [[package]] name = "system-configuration" version = "0.6.1" @@ -2471,16 +2395,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" @@ -2491,17 +2405,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.18" @@ -2510,6 +2413,7 @@ checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -2647,15 +2551,10 @@ checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ "getrandom 0.3.4", "js-sys", + "serde_core", "wasm-bindgen", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.5" @@ -2813,21 +2712,14 @@ dependencies = [ ] [[package]] -name = "winapi" -version = "0.3.9" +name = "webpki-roots" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "rustls-pki-types", ] -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.11" @@ -2837,47 +2729,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" -dependencies = [ - "windows-collections", - "windows-core 0.61.2", - "windows-future", - "windows-link 0.1.3", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" -dependencies = [ - "windows-core 0.61.2", -] - -[[package]] -name = "windows-core" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", -] - [[package]] name = "windows-core" version = "0.62.2" @@ -2886,20 +2737,9 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-future" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", - "windows-threading", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] @@ -2924,46 +2764,21 @@ dependencies = [ "syn", ] -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-numerics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" -dependencies = [ - "windows-core 0.61.2", - "windows-link 0.1.3", -] - [[package]] name = "windows-registry" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "windows-link 0.2.1", - "windows-result 0.4.1", - "windows-strings 0.5.1", -] - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link 0.1.3", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] @@ -2972,16 +2787,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link 0.2.1", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link 0.1.3", + "windows-link", ] [[package]] @@ -2990,7 +2796,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -3026,7 +2832,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -3066,7 +2872,7 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link 0.2.1", + "windows-link", "windows_aarch64_gnullvm 0.53.1", "windows_aarch64_msvc 0.53.1", "windows_i686_gnu 0.53.1", @@ -3077,15 +2883,6 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] -[[package]] -name = "windows-threading" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" -dependencies = [ - "windows-link 0.1.3", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index f3202a91..cdfbd0fa 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -16,12 +16,12 @@ fern = { version = "0.7.1", features = ["colored"] } futures = "0.3.31" indicatif = "0.18.3" log = "0.4.29" +opendal = { version = "0.55.0", features = ["services-s3"] } rand = "0.9.2" reqwest = { version = "0.13.1", features = ["json"] } -rust-s3 = "0.37.1" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" tokio = { version = "1.48.0", features = ["fs", "macros"] } -tokio-util = "0.7.18" +tokio-util = { version = "0.7.18", features = ["compat"] } url = "2.5.8" webbrowser = "1.0.6" diff --git a/cli/src/cli.rs b/cli/src/cli.rs index af391b0d..1ebd41e1 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -1,6 +1,6 @@ use clap::{Args, Parser, Subcommand, ValueEnum}; -use crate::commands::configure::config_option::ConfigOptionCli; +use crate::commands::connect::config_option::ConfigOptionCli; #[derive(Parser)] #[command(version, about, long_about = None)] @@ -16,7 +16,7 @@ pub struct Cli { #[derive(Subcommand)] pub enum Commands { /// Configures downpour endpoints - Configure { + Connect { #[arg(short, long)] name: String, #[command(subcommand)] diff --git a/cli/src/commands/configure/config_option.rs b/cli/src/commands/configure/config_option.rs deleted file mode 100644 index 638b2a8d..00000000 --- a/cli/src/commands/configure/config_option.rs +++ /dev/null @@ -1,18 +0,0 @@ -use clap::Subcommand; -use serde::{Deserialize, Serialize}; - -use crate::commands::configure::{ - s3::{S3Config, S3ConfigCli}, - server::{ServerConfig, ServerConfigCli}, -}; - -#[derive(Subcommand, Clone)] -pub enum ConfigOptionCli { - Server(ServerConfigCli), - S3(S3ConfigCli), -} -#[derive(Serialize, Deserialize, Clone)] -pub enum ConfigOption { - Server(ServerConfig), - S3(S3Config), -} diff --git a/cli/src/commands/configure/configurable.rs b/cli/src/commands/configure/configurable.rs deleted file mode 100644 index f8240083..00000000 --- a/cli/src/commands/configure/configurable.rs +++ /dev/null @@ -1,5 +0,0 @@ -use crate::commands::configure::config_option::ConfigOption; - -pub trait Configurable { - async fn configure(self) -> anyhow::Result; -} diff --git a/cli/src/commands/configure/s3.rs b/cli/src/commands/configure/s3.rs deleted file mode 100644 index 92dc3502..00000000 --- a/cli/src/commands/configure/s3.rs +++ /dev/null @@ -1,65 +0,0 @@ -use std::str::FromStr; - -use clap::Args; -use s3::{Bucket, Region, creds::Credentials}; -use serde::{Deserialize, Serialize}; - -use crate::{ - commands::configure::{config_option::ConfigOption, configurable::Configurable}, - interactive_optional_variable, interactive_variable, -}; - -#[derive(Args, Clone)] -pub struct S3ConfigCli { - secret_key: Option, - key_id: Option, - region: Option, - bucket_name: Option, - endpoint: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct S3Config { - secret_key: String, - key_id: String, - region: String, - bucket_name: String, - endpoint: Option, -} - -impl Configurable for S3ConfigCli { - async fn configure(self) -> anyhow::Result { - interactive_variable!(self, secret_key, "S3 Secret Key"); - interactive_variable!(self, key_id, "S3 Key ID"); - interactive_variable!(self, region, "S3 Region"); - interactive_variable!(self, bucket_name, "S3 Bucket Name"); - interactive_optional_variable!(self, endpoint, "S3 Endpoint (leave blank for none"); - Ok(ConfigOption::S3(S3Config { - secret_key, - key_id, - region, - bucket_name, - endpoint, - })) - } -} - -impl S3Config { - pub fn generate_bucket(&self) -> anyhow::Result { - let credentials = - Credentials::new(Some(&self.key_id), Some(&self.secret_key), None, None, None)?; - - let region = if let Some(endpoint) = &self.endpoint { - Region::Custom { - region: self.region.clone(), - endpoint: endpoint.clone(), - } - } else { - Region::from_str(&self.region)? - }; - - let bucket = Bucket::new(&self.bucket_name, region, credentials)?; - - Ok(*bucket) - } -} diff --git a/cli/src/commands/configure/server.rs b/cli/src/commands/configure/server.rs deleted file mode 100644 index a9be5c21..00000000 --- a/cli/src/commands/configure/server.rs +++ /dev/null @@ -1,92 +0,0 @@ -use clap::Args; -use serde::{Deserialize, Serialize}; -use std::sync::LazyLock; - -use anyhow::{Result, anyhow}; -use dialoguer::{Confirm, Input, theme::ColorfulTheme}; -use reqwest::Client; -use url::Url; - -use crate::commands::configure::{config_option::ConfigOption, configurable::Configurable}; - -#[derive(Serialize, Deserialize, Clone)] -pub struct ServerConfig { - url: String, - token: String, -} -#[derive(Args, Clone)] -pub struct ServerConfigCli { - /// Endpoint of the Drop server - url: String, - #[arg(short, long)] - token: Option, -} - -const TOKEN_CREATE_PAYLOAD: &str = - "eyJuYW1lIjoiZG93bnBvdXIgKGNsaSkiLCJhY2xzIjpbImRlcG90Om5ldyJdfQ=="; - -impl Configurable for ServerConfigCli { - async fn configure(self) -> anyhow::Result { - let base_url = Url::parse(&self.url)?; - let mut token_create_url = base_url.join("/admin/settings/tokens")?; - { - let mut query = token_create_url.query_pairs_mut(); - query.append_pair("payload", TOKEN_CREATE_PAYLOAD); - }; - - let confirm = Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt(format!( - "Open \"{}\" in your default browser?", - token_create_url.as_str() - )) - .interact()?; - - if !confirm { - return Err(anyhow!("User cancelled action")); - } - - webbrowser::open(token_create_url.as_str())?; - - let token: String = Input::with_theme(&ColorfulTheme::default()) - .with_prompt("API token") - .interact_text()?; - - validate_configuration(&self.url, &token).await?; - - Ok(ConfigOption::Server(ServerConfig { - url: self.url, - token, - })) - } -} - -static CLIENT: LazyLock = LazyLock::new(|| reqwest::Client::new()); -const REQUIRED_ACLS: [&str; 1] = ["depot:new"]; - -pub async fn validate_configuration(url: &str, token: &str) -> Result<()> { - let base_url = Url::parse(&url)?; - let token_check_url = base_url.join("/api/v1/token")?; - - let acl_check = CLIENT - .get(token_check_url) - .bearer_auth(token) - .send() - .await?; - - if !acl_check.status().is_success() { - return Err(anyhow!( - "ACL check failed with response code: {}", - acl_check.status() - )); - } - - let acls: Vec = acl_check.json().await?; - - for acl in REQUIRED_ACLS { - if !acls.contains(&acl.to_string()) { - return Err(anyhow!("Token missing {} acl", acl)); - } - } - - Ok(()) -} diff --git a/cli/src/commands/configure/config.rs b/cli/src/commands/connect/config.rs similarity index 60% rename from cli/src/commands/configure/config.rs rename to cli/src/commands/connect/config.rs index a5697496..8ac23e81 100644 --- a/cli/src/commands/configure/config.rs +++ b/cli/src/commands/connect/config.rs @@ -1,29 +1,32 @@ -use crate::commands::configure::{ +use crate::{commands::{connect::{ config_option::{ConfigOption, ConfigOptionCli}, - configurable::Configurable, + configurable::Configure, s3::S3Config, -}; +}, upload::speedtest::Speedtest}, manifest::DepotManifest}; use dialoguer::{Confirm, theme::ColorfulTheme}; -use log::warn; +use futures::AsyncWriteExt; +use log::{debug, info, warn}; +use opendal::Operator; use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, fs}; +use tokio_util::compat::FuturesAsyncWriteCompatExt; +use std::{collections::HashMap, fs, ops::Not}; const CONFIG_DIR: &str = "downpour/config.json"; #[derive(Serialize, Deserialize)] pub struct Config { - items: HashMap, + configurations: HashMap, active_s3: Option, } impl Config { pub fn new() -> Self { Self { - items: HashMap::new(), + configurations: HashMap::new(), active_s3: None, } } pub fn exists(&self, name: &String) -> bool { - self.items.contains_key(name) + self.configurations.contains_key(name) } pub fn save(&self) -> anyhow::Result<()> { let json = serde_json::to_string(self)?; @@ -48,13 +51,13 @@ impl Config { if matches!(object, ConfigOption::S3(..)) { self.active_s3 = Some(name.clone()) } - self.items.insert(name, object); + self.configurations.insert(name, object); self.save().expect("Failed to save config"); } pub fn get_active_s3(&self) -> Option { if let Some(active_s3) = &self.active_s3 { - self.items + self.configurations .iter() .filter_map(|(name, option)| { if *name == *active_s3 { @@ -76,6 +79,9 @@ impl Config { None } } + pub fn get>(&self, name: T) -> Option<&ConfigOption> { + self.configurations.get(name.as_ref()) + } } pub async fn manage_configuration( @@ -94,12 +100,37 @@ pub async fn manage_configuration( return Err(anyhow::anyhow!("User cancelled action")); } } - config.add_item( - name.clone(), - match option { - ConfigOptionCli::Server(server_config) => server_config.clone().configure().await?, - ConfigOptionCli::S3(s3_config_cli) => s3_config_cli.clone().configure().await?, - }, - ); + let config_option = match option { + ConfigOptionCli::S3(s3_config_cli) => s3_config_cli.clone().configure().await?, + }; + config.add_item(name.clone(), config_option.clone()); + let operator = config_option.build()?; + + generate_speedtest(&operator).await?; + generate_manifest(&operator).await?; + Ok(()) } + +async fn generate_speedtest(operator: &Operator) -> anyhow::Result<()> { + if operator.exists("speedtest").await?.not() { + info!("Speedtest already exists on Depot. Skipping speedtest upload..."); + return Ok(()) + } + let mut writer = operator.writer("speedtest").await?.into_futures_async_write().compat_write(); + let mut reader = Speedtest::new(); + let written = tokio::io::copy(&mut reader, &mut writer).await?; + debug!("Wrote {} bytes to {:?}", written, operator.info()); + writer.into_inner().close().await?; + Ok(()) +} +async fn generate_manifest(operator: &Operator) -> anyhow::Result<()> { + info!("Manifest already exists on Depot. Skipping manifest upload..."); + if operator.exists("manifest.json").await?.not() { + return Ok(()) + } + let data = DepotManifest::new(); + operator.write("manifest.json", serde_json::to_string(&data)?).await?; + + Ok(()) +} \ No newline at end of file diff --git a/cli/src/commands/connect/config_option.rs b/cli/src/commands/connect/config_option.rs new file mode 100644 index 00000000..fd90453e --- /dev/null +++ b/cli/src/commands/connect/config_option.rs @@ -0,0 +1,26 @@ +use clap::Subcommand; +use opendal::{Operator, layers::LoggingLayer}; +use serde::{Deserialize, Serialize}; + +use crate::commands::{ + connect::s3::{S3Config, S3ConfigCli}, + upload::uploadable::OperatorBuilder, +}; + +#[derive(Subcommand, Clone)] +pub enum ConfigOptionCli { + S3(S3ConfigCli), +} +#[derive(Serialize, Deserialize, Clone)] +pub enum ConfigOption { + S3(S3Config), +} + +impl ConfigOption { + pub fn build(&self) -> anyhow::Result { + Ok(match self { + ConfigOption::S3(s3_config) => s3_config.build()?, + } + .layer(LoggingLayer::default())) + } +} diff --git a/cli/src/commands/connect/configurable.rs b/cli/src/commands/connect/configurable.rs new file mode 100644 index 00000000..ad7b4a61 --- /dev/null +++ b/cli/src/commands/connect/configurable.rs @@ -0,0 +1,5 @@ +use crate::commands::connect::config_option::ConfigOption; + +pub trait Configure { + async fn configure(self) -> anyhow::Result; +} diff --git a/cli/src/commands/configure/interactive.rs b/cli/src/commands/connect/interactive.rs similarity index 86% rename from cli/src/commands/configure/interactive.rs rename to cli/src/commands/connect/interactive.rs index d567f3f5..8c53012d 100644 --- a/cli/src/commands/configure/interactive.rs +++ b/cli/src/commands/connect/interactive.rs @@ -8,7 +8,7 @@ macro_rules! interactive_variable { let $var = if let Some($var) = $value.$var { $var } else { - crate::commands::configure::interactive::query_variable($prompt).unwrap() + crate::commands::connect::interactive::query_variable($prompt).unwrap() }; }; } @@ -18,7 +18,7 @@ macro_rules! interactive_optional_variable { let $var = if let Some($var) = $value.$var { Some($var) } else { - crate::commands::configure::interactive::query_optional_variable($prompt).unwrap() + crate::commands::connect::interactive::query_optional_variable($prompt).unwrap() }; }; } diff --git a/cli/src/commands/configure/mod.rs b/cli/src/commands/connect/mod.rs similarity index 86% rename from cli/src/commands/configure/mod.rs rename to cli/src/commands/connect/mod.rs index 843cc4c5..dcb64e3e 100644 --- a/cli/src/commands/configure/mod.rs +++ b/cli/src/commands/connect/mod.rs @@ -1,7 +1,6 @@ pub mod config; pub mod configurable; pub mod s3; -pub mod server; #[macro_use] pub mod interactive; pub mod config_option; diff --git a/cli/src/commands/connect/s3.rs b/cli/src/commands/connect/s3.rs new file mode 100644 index 00000000..6a7608d4 --- /dev/null +++ b/cli/src/commands/connect/s3.rs @@ -0,0 +1,66 @@ +use clap::Args; +use opendal::Operator; +use serde::{Deserialize, Serialize}; + +use crate::{ + commands::{ + connect::{config_option::ConfigOption, configurable::Configure}, + upload::uploadable::OperatorBuilder, + }, + interactive_variable, +}; + +#[derive(Args, Clone)] +pub struct S3ConfigCli { + key_id: Option, + secret_key: Option, + endpoint: Option, + region: Option, + bucket_name: Option, + root: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct S3Config { + key_id: String, + secret_key: String, + endpoint: String, + region: String, + bucket_name: String, + root: Option, +} + +impl Configure for S3ConfigCli { + async fn configure(self) -> anyhow::Result { + interactive_variable!(self, key_id, "S3 Key ID"); + interactive_variable!(self, secret_key, "S3 Secret Key"); + interactive_variable!(self, region, "S3 Region"); + interactive_variable!(self, bucket_name, "S3 Bucket Name"); + interactive_variable!(self, endpoint, "S3 Endpoint"); + Ok(ConfigOption::S3(S3Config { + secret_key, + key_id, + region, + bucket_name, + endpoint, + root: self.root, + })) + } +} + +impl OperatorBuilder for S3Config { + fn build(&self) -> anyhow::Result { + let builder = opendal::services::S3::default() + .access_key_id(&self.key_id) + .secret_access_key(&self.secret_key) + .region(&self.region) + .endpoint(&self.endpoint) + .root(self.root.as_ref().map(|s| s.as_str()).unwrap_or("/")) + .bucket(&self.bucket_name) + .disable_config_load(); + + let op: Operator = Operator::new(builder)?.finish(); + + Ok(op) + } +} diff --git a/cli/src/commands/mod.rs b/cli/src/commands/mod.rs index db8ad986..0a9a925a 100644 --- a/cli/src/commands/mod.rs +++ b/cli/src/commands/mod.rs @@ -1,2 +1,2 @@ -pub mod configure; +pub mod connect; pub mod upload; diff --git a/cli/src/commands/upload/interface.rs b/cli/src/commands/upload/interface.rs index d87c0817..b2244142 100644 --- a/cli/src/commands/upload/interface.rs +++ b/cli/src/commands/upload/interface.rs @@ -2,11 +2,11 @@ use std::path::{Path, PathBuf}; use crate::{ cli::UploadInfo, - commands::configure::config::Config, - commands::upload::{s3::S3, uploadable::Uploadable}, + commands::{connect::config::Config, upload::{chunk_reader::ChunkReader, uploadable::OperatorBuilder}}, manifest::generate_manifest, }; use log::info; +use tokio_util::compat::FuturesAsyncWriteCompatExt; pub async fn upload(info: &UploadInfo, config: Config) -> anyhow::Result<()> { let game_id = &info.game_id; @@ -14,27 +14,19 @@ pub async fn upload(info: &UploadInfo, config: Config) -> anyhow::Result<()> { let version_id = &info.version_id; let manifest = generate_manifest(&Path::new(path)).await?; - let mut uploader: Box = match info.upload_style { - crate::cli::UploadStyle::S3 => Box::new(S3::new( - &config + let operator = match info.upload_style { + crate::cli::UploadStyle::S3 => config .get_active_s3() - .ok_or(anyhow::Error::msg("Could not get active S3 value"))?, - )?), + .ok_or(anyhow::Error::msg("Could not get active S3 value"))?.build()?, }; info!("Uploading chunks"); for (id, data) in &manifest.chunks { info!("Uploading chunk id {id}"); - uploader.upload_chunk(PathBuf::from(path), game_id, version_id, id, data).await?; + let mut reader = ChunkReader::new(&path, data); + let mut writer = operator.writer(&format!("{game_id}/{version_id}/{id}")).await?.into_futures_async_write().compat_write(); + tokio::io::copy(&mut reader, &mut writer); } info!("Finished uploading chunks"); - - info!("Uploading manifest"); - uploader - .upload_manifest(manifest, game_id, version_id) - .await?; - - info!("Uploading speedtest"); - uploader.upload_speedtest().await?; - Ok(()) } + diff --git a/cli/src/commands/upload/mod.rs b/cli/src/commands/upload/mod.rs index 3628fe41..76fe819d 100644 --- a/cli/src/commands/upload/mod.rs +++ b/cli/src/commands/upload/mod.rs @@ -1,5 +1,4 @@ pub mod interface; -pub mod s3; pub mod speedtest; pub mod uploadable; pub mod chunk_reader; diff --git a/cli/src/commands/upload/s3.rs b/cli/src/commands/upload/s3.rs deleted file mode 100644 index 387b6bcc..00000000 --- a/cli/src/commands/upload/s3.rs +++ /dev/null @@ -1,84 +0,0 @@ -use crate::commands::{ - configure::s3::S3Config, - upload::{ - chunk_reader::ChunkReader, - speedtest::{SPEEDTEST_PATH, Speedtest}, - uploadable::Uploadable, - }, -}; -use async_trait::async_trait; -use droplet_rs::{ - manifest::{ChunkData, Manifest}, -}; -use s3::Bucket; -use serde_json::json; -use std::{ops::Deref, path::PathBuf}; - -pub struct S3 { - bucket: s3::Bucket, -} -impl S3 { - pub fn new(config: &S3Config) -> anyhow::Result { - Ok(Self { - bucket: config.generate_bucket()?, - }) - } -} -#[async_trait] -impl Uploadable for S3 { - async fn upload_chunk( - &mut self, - base_path: PathBuf, - id: &String, - version: &String, - chunk_id: &String, - chunk: &ChunkData, - ) -> anyhow::Result<()> { - let path = &PathBuf::from(id) - .join(version) - .join(chunk_id) - .to_string_lossy() - .to_string(); - let mut reader = ChunkReader::new(&base_path, chunk); - self.put_object_stream(&mut reader, &path).await?; - Ok(()) - } - async fn upload_speedtest(&mut self) -> anyhow::Result<()> { - if self.object_exists(SPEEDTEST_PATH).await? { - return Ok(()); - } - println!("Uploading speedtest"); - let mut speedtest = Speedtest::new(); - self.put_object_stream(&mut speedtest, SPEEDTEST_PATH) - .await?; - Ok(()) - } - - async fn upload_manifest( - &mut self, - manifest: Manifest, - game_id: &String, - version_id: &String, - ) -> anyhow::Result<()> { - self.put_object_builder( - PathBuf::from(game_id) - .join(version_id) - .join("manifest.json") - .to_string_lossy() - .to_string(), - json!(manifest).to_string().as_bytes(), - ) - .with_content_type("application/json") - .execute() - .await?; - Ok(()) - } -} - -impl Deref for S3 { - type Target = Bucket; - - fn deref(&self) -> &Self::Target { - &self.bucket - } -} diff --git a/cli/src/commands/upload/uploadable.rs b/cli/src/commands/upload/uploadable.rs index e6dbec3d..268a5d0b 100644 --- a/cli/src/commands/upload/uploadable.rs +++ b/cli/src/commands/upload/uploadable.rs @@ -2,22 +2,9 @@ use std::path::PathBuf; use async_trait::async_trait; use droplet_rs::manifest::{ChunkData, Manifest}; +use opendal::Operator; #[async_trait] -pub trait Uploadable { - async fn upload_chunk( - &mut self, - base_path: PathBuf, - id: &String, - version: &String, - chunk_id: &String, - chunk: &ChunkData, - ) -> anyhow::Result<()>; - async fn upload_speedtest(&mut self) -> anyhow::Result<()>; - async fn upload_manifest( - &mut self, - manifest: Manifest, - game_id: &String, - version_id: &String, - ) -> anyhow::Result<()>; +pub trait OperatorBuilder { + fn build(&self) -> anyhow::Result; } diff --git a/cli/src/main.rs b/cli/src/main.rs index 6c9527a3..e87e1969 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,7 +1,7 @@ -use crate::commands::configure::config::manage_configuration; +use crate::commands::connect::config::manage_configuration; use crate::{ cli::{Cli, Commands}, - commands::configure::config::Config, + commands::connect::config::Config, commands::upload, }; use clap::Parser; @@ -18,7 +18,7 @@ async fn main() -> anyhow::Result<()> { let mut config = Config::read(); match &cli.command { - Commands::Configure { name, option } => { + Commands::Connect { name, option } => { manage_configuration(&mut config, name, option).await? } Commands::Upload(info) => { diff --git a/cli/src/manifest.rs b/cli/src/manifest.rs index ebc539e4..0ef40c0f 100644 --- a/cli/src/manifest.rs +++ b/cli/src/manifest.rs @@ -1,7 +1,40 @@ -use std::path::Path; +use std::{collections::HashMap, path::Path}; use droplet_rs::manifest::{Manifest, generate_manifest_rusty}; use indicatif::{ProgressBar, ProgressStyle}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct DepotManifest { + content: HashMap, +} +#[derive(Serialize, Deserialize)] +struct DepotManifestGameData { + version_id: String, + compression: CompressionOption, +} +#[derive(Serialize, Deserialize)] +pub enum CompressionOption { + None, + Gzip, + Zstd, +} +impl DepotManifest { + pub fn new() -> Self { + Self { + content: HashMap::new(), + } + } + pub fn add(&mut self, game_id: String, version_id: String, compression: CompressionOption) { + self.content.insert( + game_id, + DepotManifestGameData { + version_id, + compression, + }, + ); + } +} pub async fn generate_manifest(dir: &Path) -> anyhow::Result { let progress_bar = ProgressBar::new(100_00).with_style(