Compare commits

..

6 Commits

Author SHA1 Message Date
DecDuck 107ee40f43 Merge branch 'develop' into v0.4.0 2026-03-30 22:22:56 +11:00
DecDuck ee2e3e5e32 Merge remote-tracking branch 'server/libarchive-backend' into v0.4.0 2026-03-30 19:19:54 +11:00
DecDuck 9ce40297ea Merge remote-tracking branch 'promo/v0.4.0' into v0.4.0 2026-03-30 19:15:58 +11:00
DecDuck 6a9d74ef84 feat: move to libarchive backend 2026-03-01 22:57:22 +11:00
DecDuck 581198f9e6 fix: vulns 2026-03-01 22:11:57 +11:00
DecDuck 1a6b11df68 feat: dark mode 2026-02-18 15:23:24 +11:00
50 changed files with 1087 additions and 1583 deletions
-1
View File
@@ -1 +0,0 @@
/bin
-19
View File
@@ -1,19 +0,0 @@
package core
import (
"context"
"fmt"
"os"
"github.com/jackc/pgx/v5"
)
func connect() {
conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL"))
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err)
os.Exit(1)
}
defer conn.Close(context.Background())
}
-10
View File
@@ -1,10 +0,0 @@
module drop/core
go 1.26.1
require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.9.1 // indirect
golang.org/x/text v0.29.0 // indirect
)
-15
View File
@@ -1,15 +0,0 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc=
github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-5
View File
@@ -1,5 +0,0 @@
module drop
go 1.26.1
require github.com/gorilla/mux v1.8.1
-2
View File
@@ -1,2 +0,0 @@
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
-3
View File
@@ -1,3 +0,0 @@
go 1.26.1
use ./core
-9
View File
@@ -1,9 +0,0 @@
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-37
View File
@@ -1,37 +0,0 @@
package main
import (
"fmt"
"log"
"net/http"
"strings"
"github.com/gorilla/mux"
)
func handler(res http.ResponseWriter, req *http.Request) {
fmt.Fprintf(res, "G'day there mate")
}
func routingMiddleware(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
url := *r.URL
url.Path = strings.TrimSuffix(r.URL.Path, "/")
r.URL = &url
h.ServeHTTP(w, r)
})
}
func main() {
r := mux.NewRouter().StrictSlash(true)
r.Use(routingMiddleware)
r.HandleFunc("/api/v1", handler)
srv := &http.Server{
Addr: ":3433",
Handler: r,
}
log.Printf("starting drop server on :3433")
srv.ListenAndServe()
}
+73 -281
View File
@@ -2,21 +2,6 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "addr2line"
version = "0.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
dependencies = [
"gimli",
]
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "android_system_properties"
version = "0.1.5"
@@ -120,10 +105,10 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"synstructure 0.13.2",
"proc-macro2",
"quote",
"syn",
"synstructure",
]
[[package]]
@@ -132,10 +117,10 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"synstructure 0.13.2",
"proc-macro2",
"quote",
"syn",
"synstructure",
]
[[package]]
@@ -144,9 +129,9 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -155,9 +140,9 @@ version = "0.1.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -205,21 +190,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "backtrace"
version = "0.3.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
dependencies = [
"addr2line",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
"windows-link",
]
[[package]]
name = "base64"
version = "0.22.1"
@@ -325,9 +295,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671"
dependencies = [
"heck",
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -564,9 +534,9 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -595,7 +565,7 @@ dependencies = [
"indicatif",
"log",
"opendal",
"rand 0.9.3",
"rand 0.9.2",
"reqwest 0.13.1",
"serde",
"serde_json",
@@ -607,37 +577,26 @@ dependencies = [
[[package]]
name = "droplet-rs"
version = "0.16.3"
version = "0.14.1"
dependencies = [
"anyhow",
"async-trait",
"droplet_types",
"dyn-clone",
"futures",
"getrandom 0.3.4",
"hex",
"humansize",
"libarchive-drop",
"rcgen",
"ring",
"serde",
"serde_json",
"sha2",
"speedometer",
"test-generator",
"time",
"tokio",
"uuid",
"x509-parser 0.17.0",
]
[[package]]
name = "droplet_types"
version = "0.1.0"
dependencies = [
"serde",
]
[[package]]
name = "dunce"
version = "1.0.5"
@@ -681,28 +640,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "failure"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
dependencies = [
"backtrace",
"failure_derive",
]
[[package]]
name = "failure_derive"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 1.0.109",
"synstructure 0.12.6",
]
[[package]]
name = "fastrand"
version = "2.3.0"
@@ -800,9 +737,9 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -872,18 +809,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "gimli"
version = "0.32.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
[[package]]
name = "glob"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
[[package]]
name = "gloo-timers"
version = "0.3.0"
@@ -1268,9 +1193,9 @@ version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -1336,24 +1261,6 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libarchive-drop"
version = "0.1.1"
dependencies = [
"libarchive3-sys",
"libc",
]
[[package]]
name = "libarchive3-sys"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cd3beae8f59a4c7a806523269b5392037577c150446e88d684dfa6de6031ca7"
dependencies = [
"libc",
"pkg-config",
]
[[package]]
name = "libc"
version = "0.2.178"
@@ -1428,15 +1335,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
]
[[package]]
name = "mio"
version = "1.1.1"
@@ -1523,15 +1421,6 @@ dependencies = [
"objc2",
]
[[package]]
name = "object"
version = "0.37.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe"
dependencies = [
"memchr",
]
[[package]]
name = "oid-registry"
version = "0.7.1"
@@ -1641,12 +1530,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "portable-atomic"
version = "1.11.1"
@@ -1686,15 +1569,6 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "proc-macro2"
version = "0.4.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
dependencies = [
"unicode-xid 0.1.0",
]
[[package]]
name = "proc-macro2"
version = "1.0.103"
@@ -1754,7 +1628,7 @@ dependencies = [
"bytes",
"getrandom 0.3.4",
"lru-slab",
"rand 0.9.3",
"rand 0.9.2",
"ring",
"rustc-hash",
"rustls",
@@ -1780,22 +1654,13 @@ dependencies = [
"windows-sys 0.60.2",
]
[[package]]
name = "quote"
version = "0.6.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
dependencies = [
"proc-macro2 0.4.30",
]
[[package]]
name = "quote"
version = "1.0.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a"
dependencies = [
"proc-macro2 1.0.103",
"proc-macro2",
]
[[package]]
@@ -1817,9 +1682,9 @@ dependencies = [
[[package]]
name = "rand"
version = "0.9.3"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166"
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
dependencies = [
"rand_chacha 0.9.0",
"rand_core 0.9.3",
@@ -2022,12 +1887,6 @@ dependencies = [
"ordered-multimap",
]
[[package]]
name = "rustc-demangle"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
[[package]]
name = "rustc-hash"
version = "2.1.1"
@@ -2225,9 +2084,9 @@ version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -2321,15 +2180,6 @@ dependencies = [
"windows-sys 0.60.2",
]
[[package]]
name = "speedometer"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2789736092fa21b44baf8590acb4b360cb91f0f597bd6c1f1741ca9644c95c1e"
dependencies = [
"failure",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
@@ -2348,36 +2198,14 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "0.15.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5"
dependencies = [
"proc-macro2 0.4.30",
"quote 0.6.13",
"unicode-xid 0.1.0",
]
[[package]]
name = "syn"
version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"proc-macro2",
"quote",
"unicode-ident",
]
@@ -2390,27 +2218,15 @@ dependencies = [
"futures-core",
]
[[package]]
name = "synstructure"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 1.0.109",
"unicode-xid 0.2.6",
]
[[package]]
name = "synstructure"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -2447,18 +2263,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "test-generator"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b23be2add79223226e1cb6446cb3e37506a5927089870687a0f1149bb7a073a"
dependencies = [
"glob",
"proc-macro2 0.4.30",
"quote 0.6.13",
"syn 0.15.44",
]
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -2483,9 +2287,9 @@ version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -2494,9 +2298,9 @@ version = "2.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -2586,9 +2390,9 @@ version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -2703,18 +2507,6 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
[[package]]
name = "unicode-xid"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
[[package]]
name = "unicode-xid"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "unit-prefix"
version = "0.5.2"
@@ -2835,7 +2627,7 @@ version = "0.2.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3"
dependencies = [
"quote 1.0.43",
"quote",
"wasm-bindgen-macro-support",
]
@@ -2846,9 +2638,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40"
dependencies = [
"bumpalo",
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
@@ -2956,9 +2748,9 @@ version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -2967,9 +2759,9 @@ version = "0.59.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -3303,10 +3095,10 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"synstructure 0.13.2",
"proc-macro2",
"quote",
"syn",
"synstructure",
]
[[package]]
@@ -3324,9 +3116,9 @@ version = "0.8.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -3344,10 +3136,10 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"synstructure 0.13.2",
"proc-macro2",
"quote",
"syn",
"synstructure",
]
[[package]]
@@ -3384,9 +3176,9 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
dependencies = [
"proc-macro2 1.0.103",
"quote 1.0.43",
"syn 2.0.114",
"proc-macro2",
"quote",
"syn",
]
[[package]]
+2 -2
View File
@@ -11,13 +11,13 @@ clap = { version = "4.5.54", features = ["derive"] }
console = "0.16.2"
dialoguer = "0.12.0"
dirs = "6.0.0"
droplet-rs = { path = "../libraries/droplet" }
droplet-rs = { path = "../droplet-rs", version = "0.14" }
fern = { version = "0.7.1", features = ["colored"] }
futures = "0.3.31"
indicatif = "0.18.3"
log = "0.4.29"
opendal = { version = "0.55.0", features = ["services-s3"] }
rand = "0.9.3"
rand = "0.9.2"
reqwest = { version = "0.13.1", features = ["json"] }
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.148"
+21 -23
View File
@@ -3,7 +3,7 @@ use std::path::Path;
use crate::{
cli::UploadInfo,
commands::connect::{config::Config, config_option::ConfigOption},
manifest::{ClosureFactory, CompressionOption, DepotManifest, generate_v2_manifest},
manifest::{CompressionOption, DepotManifest, generate_v2_manifest},
operator_builder::OperatorBuilder,
};
use futures::AsyncWriteExt;
@@ -12,13 +12,13 @@ use opendal::{FuturesAsyncWriter, Operator};
use tokio_util::compat::{Compat, FuturesAsyncWriteCompatExt};
pub async fn upload(
upload_info: &UploadInfo,
info: &UploadInfo,
config: Config,
name: &Option<String>,
) -> anyhow::Result<()> {
let game_id = upload_info.game_id.clone();
let path = upload_info.path.clone();
let version_id = upload_info.version_id.clone();
let game_id = &info.game_id;
let path = &info.path;
let version_id = &info.version_id;
let operator = get_operator(config, name)?;
@@ -27,30 +27,28 @@ pub async fn upload(
info!("Uploading chunks");
let v2_manifest = generate_v2_manifest(
Path::new(&path),
ClosureFactory::new(
async move |id: String| {
info!("Uploading chunk id {id}");
let writer = operator
.writer(&format!("{game_id}/{version_id}/{id}"))
.await
.unwrap()
.into_futures_async_write()
.compat_write();
writer
},
|writer: Compat<FuturesAsyncWriter>| async {
writer.into_inner().close().await.unwrap();
},
),
Path::new(path),
async |id: String| {
info!("Uploading chunk id {id}");
let writer = operator
.writer(&format!("{game_id}/{version_id}/{id}"))
.await
.unwrap()
.into_futures_async_write()
.compat_write();
writer
},
|writer: Compat<FuturesAsyncWriter>| async {
writer.into_inner().close().await.unwrap();
},
)
.await?;
info!("Finished uploading chunks");
existing_depot_manifest.append(
upload_info.game_id.to_string(),
upload_info.version_id.to_string(),
game_id.to_string(),
version_id.to_string(),
CompressionOption::None,
);
Ok(())
-2
View File
@@ -1,5 +1,3 @@
#![feature(async_fn_traits)]
use crate::commands::connect::config::manage_configuration;
use crate::{
cli::{Cli, Commands},
+10 -58
View File
@@ -1,7 +1,8 @@
use std::{collections::HashMap, path::Path};
use async_trait::async_trait;
use droplet_rs::manifest::{Manifest, ManifestWriterFactory, generate_manifest_rusty};
use droplet_rs::manifest::{
Manifest, generate_manifest_rusty, generate_manifest_rusty_v2,
};
use indicatif::{ProgressBar, ProgressStyle};
use log::info;
use serde::{Deserialize, Serialize};
@@ -39,60 +40,11 @@ impl DepotManifest {
}
}
pub struct ClosureFactory<Writer, Factory, Closer>
pub async fn generate_v2_manifest<W, F, CloseF>(dir: &Path, factory: F, closer: CloseF) -> anyhow::Result<Manifest>
where
Writer: AsyncWrite + Unpin,
Factory: AsyncFn(String) -> Writer,
Closer: AsyncFn(Writer),
{
writer: Factory,
closer: Closer,
}
#[async_trait]
impl<
W: AsyncWrite + Unpin + Send + Sync,
F: AsyncFn(String) -> W + Send + Sync + 'static,
C: AsyncFn(W) + Send + Sync,
> ManifestWriterFactory for ClosureFactory<W, F, C>
where
for<'a> F::CallRefFuture<'a>: Send,
for<'b> C::CallRefFuture<'b>: Send,
{
type Writer = W;
async fn create(&self, id: String) -> anyhow::Result<Self::Writer> {
let func = &self.writer;
let output = func(id).await;
Ok(output)
}
async fn close(&self, writer: Self::Writer) -> anyhow::Result<()> {
let func = &self.closer;
func(writer).await;
Ok(())
}
}
impl<
W: AsyncWrite + Unpin + Send + Sync,
F: AsyncFn(String) -> W + Send + Sync + 'static,
C: AsyncFn(W) + Sync,
> ClosureFactory<W, F, C>
where
for<'a> F::CallRefFuture<'a>: Send,
for<'b> C::CallRefFuture<'b>: Send,
{
pub fn new(f: F, c: C) -> Self {
Self {
writer: f,
closer: c,
}
}
}
pub async fn generate_v2_manifest<Factory>(dir: &Path, factory: Factory) -> anyhow::Result<Manifest>
where
Factory: ManifestWriterFactory,
W: AsyncWrite + Unpin,
F: AsyncFn(String) -> W,
CloseF: AsyncFn(W)
{
let progress_bar = ProgressBar::new(10_000).with_style(
ProgressStyle::default_bar()
@@ -100,15 +52,15 @@ where
.unwrap(),
);
generate_manifest_rusty(
generate_manifest_rusty_v2(
dir,
|progress| {
let progress_int = (progress * 100f32).round() as u64;
progress_bar.set_position(progress_int);
},
|log| progress_bar.suspend(|| info!("{}", log)),
Some(&factory),
None,
factory,
closer
)
.await
}
+6
View File
@@ -0,0 +1,6 @@
[submodule "src-tauri/tailscale/libtailscale"]
path = src-tauri/tailscale/libtailscale
url = https://github.com/tailscale/libtailscale.git
[submodule "libs/drop-base"]
path = libs/drop-base
url = https://github.com/drop-oss/drop-base.git
+7
View File
@@ -21,6 +21,13 @@ async function spawn(exec, opts) {
});
}
const expectedLibs = ["drop-base/package.json"];
for (const lib of expectedLibs) {
const path = `./libs/${lib}`;
if (!fs.existsSync(path)) throw `Missing "${expectedLibs}". Run "git submodule update --init --recursive"`;
}
const views = fs.readdirSync(".").filter((view) => {
const expectedPath = `./${view}/package.json`;
return fs.existsSync(expectedPath);
+1 -1
View File
@@ -36,7 +36,7 @@
as="div"
v-for="(nav, navIndex) in filteredNavigation"
:key="nav.id"
:class="['first:pt-0 last:pb-0', nav.tools && !filteredNavigation[navIndex - 1].tools ? 'mt-auto' : '']"
:class="['first:pt-0 last:pb-0', nav.tools ? 'mt-auto' : '']"
v-slot="{ open }"
:default-open="nav.deft"
>
+1 -5
View File
@@ -1,7 +1,3 @@
const path = require('path');
const dropbase = path.join(__dirname, "../../libraries/base")
/** @type {import('tailwindcss').Config} */
export default {
content: [
@@ -11,7 +7,7 @@ export default {
"./plugins/**/*.{js,ts}",
"./app.vue",
"./error.vue",
`${dropbase}/components/**/*.{js,vue,ts}`,
"../libs/drop-base/**/*.{js,vue,ts}",
],
theme: {
extend: {
+528 -133
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -41,6 +41,7 @@ database = { path = "./database" } # database
deranged = "=0.4.0"
dirs = "6.0.0"
download_manager = { path = "./download_manager", version = "0.1.0" } # download manager
droplet-rs = "0.7.3"
filetime = "0.2.25"
futures-core = "0.3.31"
futures-lite = "2.6.0"
@@ -1,9 +1,7 @@
use std::{
collections::HashMap,
env,
sync::RwLock,
time::{Duration, Instant},
usize,
time::{Duration, Instant}, usize,
};
use futures_util::StreamExt;
@@ -36,7 +34,7 @@ struct Depot {
manifest: Option<DepotManifest>,
latest_speed: Option<usize>, // bytes per second
current_downloads: SyncSemaphore,
enabled: bool,
enabled: bool
}
pub struct DepotManager {
@@ -115,16 +113,10 @@ impl DepotManager {
for depot in &mut new_depots {
if let Err(sync_error) = self.sync_depot(depot).await {
warn!("failed to sync depot {}: {:?}", depot.endpoint, sync_error);
if env::var("FORCE_ENABLE_DEPOTS")
.map(|v| !v.is_empty())
.unwrap_or(false)
{
} else {
depot.enabled = false;
}
depot.enabled = false;
}
}
let enabled = new_depots.iter().filter(|v| v.enabled).count();
if enabled == 0 {
return Err(RemoteAccessError::NoDepots);
+1 -1
View File
@@ -14,7 +14,7 @@ crossbeam-channel = "0.5.15"
ctr = "0.9.2"
database = { path = "../database", version = "0.1.0" }
download_manager = { path = "../download_manager", version = "0.1.0" }
droplet_types = { path = "../../../libraries/droplet_types" }
droplet-rs = { path = "../../../libraries/droplet" }
futures-util = "*"
hex = "0.4.3"
log = "0.4.28"
@@ -11,7 +11,7 @@ use download_manager::util::download_thread_control_flag::{
DownloadThreadControl, DownloadThreadControlFlag,
};
use download_manager::util::progress_object::{ProgressHandle, ProgressObject, ProgressType};
use droplet_types::{ChunkData, Manifest};
use droplet_rs::manifest::{ChunkData, Manifest};
use futures_util::StreamExt;
use futures_util::stream::FuturesUnordered;
use log::{debug, error, info, warn};
@@ -13,7 +13,7 @@ use download_manager::util::download_thread_control_flag::{
DownloadThreadControl, DownloadThreadControlFlag,
};
use download_manager::util::progress_object::ProgressHandle;
use droplet_types::ChunkData;
use droplet_rs::manifest::ChunkData;
use futures_util::StreamExt as _;
use log::{debug, info};
use remote::auth::generate_authorization_header;
+1
View File
@@ -10,6 +10,7 @@ bytes = "1.11.0"
chrono = "0.4.42"
client = { path = "../client", version = "0.1.0" }
database = { path = "../database", version = "0.1.0" }
droplet-rs = "0.7.3"
gethostname = "1.0.2"
hex = "0.4.3"
http = "1.3.1"
+1 -1
View File
@@ -27,7 +27,7 @@
},
"bundle": {
"active": true,
"targets": ["nsis", "deb", "rpm", "dmg"],
"targets": ["nsis", "deb", "rpm", "dmg", "appimage"],
"windows": {
"nsis": {
"installMode": "both"
-1
View File
@@ -21,7 +21,6 @@ target/
/target
perf.data
perf.data.old
flamegraph.svg
*.json
+3
View File
@@ -0,0 +1,3 @@
[submodule "libarchive-rust"]
path = libarchive-rust
url = https://github.com/Drop-OSS/libarchive-rust.git
+157 -414
View File
File diff suppressed because it is too large Load Diff
-1
View File
@@ -7,7 +7,6 @@ license = "AGPL-3.0-only"
description = "Droplet is a `napi.rs` Rust/Node.js package full of high-performance and low-level utils for Drop"
[dependencies]
droplet_types = { path = "../droplet_types" }
hex = "0.4.3"
time = "0.3.41"
ring = "0.17.14"
Submodule libraries/droplet/libarchive-rust added at fdb73ef2de
+2 -3
View File
@@ -1,14 +1,13 @@
#![deny(clippy::all)]
#![feature(impl_trait_in_bindings)]
#![feature(nonpoison_mutex)]
#![feature(sync_nonpoison)]
pub mod file_utils;
pub mod manifest;
pub mod ssl;
pub mod versions;
pub mod vm;
pub use manifest::{CHUNK_SIZE, MAX_FILE_COUNT};
extern crate libarchive_drop;
#[cfg(test)]
pub mod tests;
+1 -15
View File
@@ -1,21 +1,8 @@
use std::{env, path::PathBuf};
use droplet_rs::manifest::{ManifestWriterFactory, generate_manifest_rusty};
use droplet_rs::manifest::generate_manifest_rusty;
use tokio::runtime::Handle;
struct SinkFactory {}
#[async_trait::async_trait]
impl ManifestWriterFactory for SinkFactory {
type Writer = tokio::io::Sink;
async fn create(&self, _id: String) -> anyhow::Result<Self::Writer> {
Ok(tokio::io::sink())
}
async fn close(&self, _writer: Self::Writer) -> anyhow::Result<()> {
Ok(())
}
}
#[tokio::main]
pub async fn main() {
let mut args = env::args();
@@ -30,7 +17,6 @@ pub async fn main() {
|message| {
println!("{}", message);
},
Some(&SinkFactory {}),
None,
)
.await
+190 -201
View File
@@ -1,172 +1,162 @@
use std::{collections::HashMap, ops::Not, path::Path};
use std::{
collections::HashMap,
mem,
path::Path,
sync::{
atomic::{AtomicU64, Ordering},
Arc,
},
};
use anyhow::anyhow;
use async_trait::async_trait;
use futures::StreamExt;
use hex::ToHex as _;
use humansize::{format_size, BINARY};
use serde::{Deserialize, Serialize};
use sha2::{Digest as _, Sha256};
use tokio::io::AsyncWriteExt;
use tokio::io::{AsyncReadExt as _, AsyncWrite};
use tokio::sync::Semaphore;
pub use droplet_types::{ChunkData, FileEntry, Manifest};
use tokio::{
io::AsyncReadExt as _,
join,
sync::{Mutex, Semaphore},
task::JoinSet,
};
#[derive(Serialize, Deserialize, Clone)]
pub struct FileEntry {
pub filename: String,
pub start: usize,
pub length: usize,
pub permissions: u32,
}
pub const CHUNK_SIZE: u64 = 1024 * 1024 * 64;
pub const MAX_FILE_COUNT: usize = 512;
#[derive(Serialize, Deserialize, Clone)]
pub struct ChunkData {
pub files: Vec<FileEntry>,
pub checksum: String,
pub iv: [u8; 16],
}
#[derive(Serialize, Deserialize)]
pub struct Manifest {
pub version: String,
pub chunks: HashMap<String, ChunkData>,
pub size: u64,
pub key: [u8; 16],
}
const CHUNK_SIZE: u64 = 1024 * 1024 * 64;
const MAX_FILE_COUNT: usize = 512;
use crate::versions::{
create_backend_constructor,
types::{VersionBackend, VersionFile},
};
#[async_trait]
pub trait ManifestWriterFactory: Send + Sync {
type Writer: AsyncWrite + Unpin;
async fn create(&self, id: String) -> anyhow::Result<Self::Writer>;
async fn close(&self, writer: Self::Writer) -> anyhow::Result<()>;
}
pub async fn generate_manifest_rusty<T: Fn(String), V: Fn(f32)>(
dir: &Path,
progress_sfn: V,
log_sfn: T,
reader_semaphore: Option<Arc<Semaphore>>,
) -> anyhow::Result<Manifest> {
let backend =
create_backend_constructor(dir).ok_or(anyhow!("Could not create backend for path."))?()?;
let required_single_file = backend.require_whole_files();
pub async fn generate_manifest_rusty<P, LogFn, ProgFn, Writer>(
dir: P,
progress_sfn: ProgFn,
log_sfn: LogFn,
factory: Option<&dyn ManifestWriterFactory<Writer = Writer>>,
semaphore: Option<&Semaphore>,
) -> anyhow::Result<Manifest>
where
P: AsRef<Path>,
LogFn: Fn(String) + Clone,
ProgFn: Fn(f32),
Writer: AsyncWrite + Unpin,
{
let backend = create_backend_constructor(dir).ok_or(anyhow!(
"Could not create backend for path. Is this structure supported?"
))?()?;
let mut files = backend.list_files().await?;
files.sort_by(|a, b| b.size.cmp(&a.size));
files.sort_by_key(|b| std::cmp::Reverse(b.size));
// Filepath to chunk data
let mut chunks: Vec<Vec<(VersionFile, u64, u64)>> = Vec::new();
let mut current_chunk: Vec<(VersionFile, u64, u64)> = Vec::new();
log_sfn("organising files into chunks...".to_string());
log_sfn("organizing files into chunks...".to_string());
let chunks = organise_files(files, backend.require_whole_files());
if required_single_file {
for version_file in files {
if version_file.size >= CHUNK_SIZE {
let size = version_file.size;
chunks.push(vec![(version_file, 0, size)]);
log_sfn(format!(
"organized into {} chunks, generating checksums...",
chunks.len()
));
let manifest = read_chunks_and_generate_manifest(
backend.as_ref(),
chunks,
progress_sfn,
&log_sfn,
factory,
semaphore,
)
.await?;
let mut key = [0u8; 16];
getrandom::fill(&mut key).map_err(|err| anyhow!("failed to generate key: {:?}", err))?;
let total_manifest_length = manifest
.values()
.map(|value| value.files.iter().map(|f| f.length as u64).sum::<u64>())
.sum::<u64>();
Ok(Manifest {
version: "2".to_string(),
chunks: manifest,
size: total_manifest_length,
key,
})
}
fn organise_files(
files: Vec<VersionFile>,
require_whole_files: bool,
) -> Vec<Vec<(VersionFile, u64, u64)>> {
let mut chunks = Vec::new();
let mut current_chunk = Vec::new();
for version_file in files {
if current_chunk.len() >= MAX_FILE_COUNT {
// Pop current chunk
chunks.push(std::mem::take(&mut current_chunk));
println!("Chunks: {}", chunks.len());
}
let current_chunk_size = current_chunk
.iter()
.map(|(_, _, length)| *length)
.sum::<u64>();
let version_file_size = version_file.size;
if require_whole_files {
// If the current chunk is larger than chunk size, there's no point adding
// it to the current_chunk. Just push it by itself
if version_file_size >= CHUNK_SIZE {
chunks.push(vec![(version_file, 0, version_file_size)]);
println!("Chunks: {}", chunks.len());
continue;
}
current_chunk.push((version_file, 0, version_file_size));
if current_chunk_size + version_file_size >= CHUNK_SIZE {
// Pop current chunk
let mut current_size = current_chunk.iter().map(|v| v.2).sum::<u64>();
let size = version_file.size;
current_chunk.push((version_file, 0, size));
current_size += size;
if current_size >= CHUNK_SIZE {
// Pop current and add, then reset
let new_chunk = std::mem::take(&mut current_chunk);
chunks.push(new_chunk);
}
if current_chunk.len() >= MAX_FILE_COUNT {
chunks.push(std::mem::take(&mut current_chunk));
println!("Chunks: {}", chunks.len());
}
} else {
// Enough space for it to be put in immediately
if version_file_size + current_chunk_size < CHUNK_SIZE {
current_chunk.push((version_file, 0, version_file_size));
continue;
}
} else {
for version_file in files {
if current_chunk.len() >= MAX_FILE_COUNT {
chunks.push(std::mem::take(&mut current_chunk));
}
let current_size = current_chunk.iter().map(|v| v.2).sum::<u64>();
if version_file.size + current_size < CHUNK_SIZE {
let size = version_file.size;
current_chunk.push((version_file, 0, size));
continue;
}
let bytes_free_in_existing_chunk = CHUNK_SIZE - current_chunk_size;
current_chunk.push((version_file.clone(), 0, bytes_free_in_existing_chunk));
// Fill up current chunk
let remaining = CHUNK_SIZE - current_size;
current_chunk.push((version_file.clone(), 0, remaining));
chunks.push(std::mem::take(&mut current_chunk));
// Loop over remaining data and create sufficient chunks to use it
let mut offset = bytes_free_in_existing_chunk;
while offset < version_file_size {
let length = CHUNK_SIZE.min(version_file_size - offset);
// This is our offset in our current file
let mut offset = remaining;
while offset < version_file.size {
let length = CHUNK_SIZE.min(version_file.size - offset);
if length == CHUNK_SIZE {
chunks.push(vec![(version_file.clone(), offset, length)]);
println!("Chunks: {}", chunks.len());
} else {
current_chunk.push((version_file.clone(), offset, length));
println!("Chunks: {}", chunks.len());
}
offset += length;
}
}
}
if current_chunk.is_empty().not() {
if !current_chunk.is_empty() {
chunks.push(current_chunk);
println!("Pushed final chunk: {}", chunks.len());
}
println!("Chunks: {}", chunks.len());
chunks
}
async fn read_chunks_and_generate_manifest<LogFn, ProgFn, Writer>(
backend: &(dyn VersionBackend + Send + Sync),
chunks: Vec<Vec<(VersionFile, u64, u64)>>,
progress_sfn: ProgFn,
log_sfn: &LogFn,
factory: Option<&dyn ManifestWriterFactory<Writer = Writer>>,
semaphore: Option<&Semaphore>,
) -> anyhow::Result<HashMap<String, ChunkData>>
where
LogFn: Fn(String),
ProgFn: Fn(f32),
Writer: AsyncWrite + Unpin,
{
let total_chunk_count = chunks.len();
log_sfn(format!(
"organized into {} chunks, generating checksums...",
chunks.len()
));
let futures = chunks.into_iter().enumerate().map(|(index, chunk)| {
// To make the borrow checker happy
async move {
let mut read_buf = vec![0; 1024 * 1024 * 64];
let manifest: Arc<Mutex<HashMap<String, ChunkData>>> = Arc::new(Mutex::new(HashMap::new()));
let total_manifest_length = Arc::new(AtomicU64::new(0));
// SAFETY: we .join_all() the futures using this
let backend: &'static (dyn VersionBackend + Send + Sync) = unsafe { mem::transmute(&*backend) };
let mut futures: JoinSet<Result<(), anyhow::Error>> = JoinSet::new();
let (send_log, mut recieve_log) = tokio::sync::mpsc::channel(16);
let chunks_length = chunks.len();
for (index, chunk) in chunks.into_iter().enumerate() {
let send_log = send_log.clone();
let total_manifest_length = total_manifest_length.clone();
let manifest = manifest.clone();
let reader_semaphore = reader_semaphore.clone();
futures.spawn(async move {
let mut read_buf = vec![0u8; 1024 * 1024 * 8];
let uuid = uuid::Uuid::new_v4().to_string();
let mut hasher = Sha256::new();
@@ -178,92 +168,91 @@ where
checksum: String::new(),
iv,
};
let mut chunk_length = 0u64;
let mut writer = match factory {
Some(factory) => Some(factory.create(uuid.clone()).await?),
None => None,
};
let mut chunk_length = 0;
for (file, start, length) in chunk {
let permit = if let Some(semaphore) = &semaphore {
Some(semaphore.acquire().await?)
let permit = if let Some(reader_semaphore) = &reader_semaphore {
Some(reader_semaphore.acquire().await?)
} else {
None
};
chunk_data.files.push(
read_and_generate_chunk_file_data(
backend,
&file,
start,
length,
&mut hasher,
&mut read_buf,
&mut writer,
)
.await?,
);
let mut reader = backend.reader(&file, start, start + length).await?;
let mut total = 0;
loop {
let amount = reader.read(&mut read_buf).await?;
if amount == 0 {
break;
}
total += amount;
hasher.update(&read_buf[0..amount]);
}
if total as u64 > length {
panic!("read too much: target {}, got {}", length, total);
}
chunk_length += length;
chunk_data.files.push(FileEntry {
filename: file.relative_filename,
start: start.try_into().unwrap(),
length: length.try_into().unwrap(),
permissions: file.permission,
});
drop(permit);
}
if let Some(factory) = factory {
factory.close(writer.expect("Failed to get writer")).await?;
}
send_log
.send(format!(
"created chunk of size {} ({}b) from {} files (index {})",
format_size(chunk_length, BINARY),
chunk_length,
chunk_data.files.len(),
index
))
.await?;
total_manifest_length.fetch_add(chunk_length, Ordering::Relaxed);
let hash: String = hasher.finalize().encode_hex();
chunk_data.checksum = hash;
{
let mut manifest_lock = manifest.lock().await;
manifest_lock.insert(uuid, chunk_data);
};
log_sfn(format!(
"created chunk of size {} ({}b) from {} files (index {})",
format_size(chunk_length, BINARY),
chunk_length,
chunk_data.files.len(),
index
));
Ok::<_, anyhow::Error>((uuid, chunk_data))
}
});
let mut stream = futures::stream::iter(futures)
.buffer_unordered(semaphore.map(|s| s.available_permits()).unwrap_or(4))
.enumerate();
let mut results = HashMap::new();
let mut current_progress = 0f32;
while let Some((_, res)) = stream.next().await {
let (id, data) = res?;
current_progress += 1.0;
progress_sfn((current_progress / total_chunk_count as f32) * 100.0f32);
results.insert(id, data);
Ok(())
});
}
Ok(results)
}
async fn read_and_generate_chunk_file_data<Writer>(
backend: &(dyn VersionBackend + Sync + Send),
file: &VersionFile,
start: u64,
length: u64,
hasher: &mut Sha256,
read_buf: &mut [u8],
writer: &mut Option<Writer>,
) -> anyhow::Result<FileEntry>
where
Writer: AsyncWrite + Unpin,
{
let mut reader = backend.reader(file, start, start + length).await?;
drop(send_log);
join!(
async move {
let mut current_progress = 0f32;
let total_progress = chunks_length as f32;
while let Some(message) = recieve_log.recv().await {
log_sfn(message);
current_progress += 1.0f32;
progress_sfn((current_progress / total_progress) * 100.0f32)
}
},
futures.join_all()
);
loop {
let amount = reader.read(read_buf).await?;
let manifest = manifest.lock().await;
let manifest = manifest.clone();
if amount == 0 {
break;
}
if let Some(writer) = writer.as_mut() {
writer.write_all(&read_buf[0..amount]).await?;
}
hasher.update(&read_buf[0..amount]);
}
let mut key = [0u8; 16];
getrandom::fill(&mut key).map_err(|err| anyhow!("failed to generate key: {:?}", err))?;
Ok(FileEntry {
filename: file.relative_filename.clone(),
start: start.try_into().unwrap(),
length: length.try_into().unwrap(),
permissions: file.permission,
Ok(Manifest {
version: "2".to_string(),
chunks: manifest,
size: total_manifest_length.fetch_add(0, Ordering::Relaxed),
key,
})
}
+1 -3
View File
@@ -4,9 +4,8 @@ extern crate test_generator;
use std::path::Path;
use test_generator::test_resources;
use tokio::io::SimplexStream;
use crate::manifest::{generate_manifest_rusty, ManifestWriterFactory};
use crate::manifest::generate_manifest_rusty;
#[test_resources("testfiles/**/*.7z")]
fn manifest_gen(resource: &str) {
@@ -23,7 +22,6 @@ fn manifest_gen(resource: &str) {
|message| {
println!("({}) {}", filepath.display(), message);
},
None::<&dyn ManifestWriterFactory<Writer = SimplexStream>>, // Dummy type signature, not actually used
None,
)
.await
@@ -28,12 +28,12 @@ impl ZipVersionBackend {
}
}
struct ArchiveReader {
struct ArchiveReader<'a> {
archive: FileReader,
prev_block: Option<Vec<u8>>,
prev_block: Option<&'a [u8]>,
}
impl AsyncRead for ArchiveReader {
impl<'a> AsyncRead for ArchiveReader<'a> {
fn poll_read(
mut self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
@@ -41,8 +41,9 @@ impl AsyncRead for ArchiveReader {
) -> std::task::Poll<std::io::Result<()>> {
if let Some(block) = &mut self.prev_block {
let to_read = buf.remaining().min(block.len());
let result = block.split_off(to_read);
buf.put_slice(&result);
let result = block.split_off(..to_read);
let result = result.unwrap(); // SAFETY: above .min statement
buf.put_slice(result);
// If the block is empty, we can read more
if block.is_empty() {
@@ -51,31 +52,27 @@ impl AsyncRead for ArchiveReader {
return Poll::Ready(Ok(()));
}
}
let prev_block_update = {
let block = match self.archive.read_block() {
Ok(v) => v,
Err(err) => return Poll::Ready(Err(std::io::Error::other(err.to_string()))),
};
let mut block = match block {
Some(v) => v,
None => return Poll::Ready(Ok(())),
};
let write_amount = buf.remaining().min(block.len());
let to_write = block.split_off(..write_amount);
let to_write = to_write.unwrap(); // SAFETY: above .min statement
buf.put_slice(to_write);
if !block.is_empty() {
Some(block[buf.remaining()..].to_vec())
} else {
None
}
let block = match self.archive.read_block() {
Ok(v) => v,
Err(err) => return Poll::Ready(Err(std::io::Error::other(err.to_string()))),
};
if let Some(prev_block) = prev_block_update {
self.prev_block.replace(prev_block);
let mut block = match block {
Some(v) => v,
None => return Poll::Ready(Ok(())),
};
let write_amount = buf.remaining().min(block.len());
let to_write = block.split_off(..write_amount);
let to_write = to_write.unwrap(); // SAFETY: above .min statement
buf.put_slice(to_write);
if !block.is_empty() {
#[cfg(debug_assertions)]
if self.prev_block.is_some() {
panic!("replacing prev_block while it contains data")
}
self.prev_block.replace(&block[buf.remaining()..]);
}
Poll::Ready(Ok(()))
+9 -12
View File
@@ -5,11 +5,10 @@ use std::{
use anyhow::Result;
use crate::versions::{path_backend::PathVersionBackend, types::VersionBackend};
use crate::versions::{
archive_backend::ZipVersionBackend, path_backend::PathVersionBackend, types::VersionBackend,
};
use crate::versions::archive_backend::ZipVersionBackend;
// libarchive backend is Linux-only for now
pub mod archive_backend;
pub mod path_backend;
@@ -34,13 +33,10 @@ const SUPPORTED_FILE_EXTENSIONS: [&str; 11] = [
];
pub mod types;
pub fn create_backend_constructor<'a, P>(
path: P,
) -> Option<Box<dyn FnOnce() -> Result<Box<dyn VersionBackend + Send + Sync + 'a>>>>
where
P: AsRef<Path>,
{
let path = path.as_ref();
#[allow(clippy::type_complexity)]
pub fn create_backend_constructor<'a>(
path: &Path,
) -> Option<Box<dyn FnOnce() -> Result<Box<dyn VersionBackend + Send + Sync + 'a>>>> {
if !path.exists() {
return None;
}
@@ -53,7 +49,8 @@ where
}));
};
let file_extension = path.extension().map(|v| v.to_str()).flatten()?;
let file_extension = path.extension().and_then(|v| v.to_str())?;
if SUPPORTED_FILE_EXTENSIONS.contains(&file_extension) {
let buf = path.to_path_buf();
return Some(Box::new(move || Ok(Box::new(ZipVersionBackend::new(buf)?))));
-28
View File
@@ -1,28 +0,0 @@
# Generated by Cargo
# will have compiled files and executables
debug/
target/
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
# RustRover
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
# Added by cargo
/target
perf.data
perf.data.old
flamegraph.svg
*.json
.direnv
-75
View File
@@ -1,75 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "droplet_types"
version = "0.1.0"
dependencies = [
"serde",
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
dependencies = [
"proc-macro2",
]
[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_core"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "syn"
version = "2.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
-7
View File
@@ -1,7 +0,0 @@
[package]
name = "droplet_types"
version = "0.1.0"
edition = "2024"
[dependencies]
serde = { version = "*", features = ["derive"] }
-6
View File
@@ -1,6 +0,0 @@
# droplet_types
Shared types between the cross-platform client and the droplet-rs crate.
Split off from droplet-rs because of cross-compiling issues with the desktop client, and there's no need to compile the entirety of droplet-rs if we're only using a few types.
-27
View File
@@ -1,27 +0,0 @@
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Clone)]
pub struct FileEntry {
pub filename: String,
pub start: usize,
pub length: usize, // TODO: Replace with u64 for 32 bit clients
pub permissions: u32,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct ChunkData {
pub files: Vec<FileEntry>,
pub checksum: String,
pub iv: [u8; 16],
}
#[derive(Serialize, Deserialize)]
pub struct Manifest {
pub version: String,
pub chunks: HashMap<String, ChunkData>,
pub size: u64,
pub key: [u8; 16],
}
+2 -2
View File
@@ -3,10 +3,10 @@ name = "libarchive-drop"
version = "0.1.1"
authors = ["Jamie Winsor <reset@chef.io>", "Drop OSS"]
license = "Apache-2.0"
repository = "https://lab.droposs.org/drop-oss/drop/-/tree/develop/libraries/libarchive"
repository = "https://github.com/Drop-OSS/libarchive-rust"
description = "A safe Rust API for authoring and extracting archives with libarchive"
keywords = ["libarchive", "archive", "tar", "zip"]
[dependencies]
libc = ">= 0.2.0"
libarchive3-sys = "0.1.2"
libarchive3-sys = "0.1"
+4 -4
View File
@@ -1,10 +1,10 @@
extern crate libarchive_drop;
extern crate libarchive;
pub mod util;
use libarchive_drop::archive::{self, ReadFilter, ReadFormat};
use libarchive_drop::reader::{self, Reader};
use libarchive_drop::writer;
use libarchive::archive::{self, ReadFilter, ReadFormat};
use libarchive::reader::{self, Reader};
use libarchive::writer;
use std::fs::File;
#[test]
-35
View File
@@ -1,35 +0,0 @@
{
"folders": [
{
"path": "server"
},
{
"path": "torrential"
}
],
"settings": {
"i18n-ally.extract.autoDetect": true,
"i18n-ally.extract.ignored": [
"string >= 14",
"string.alphanumeric >= 5"
],
"i18n-ally.extract.ignoredByFiles": {
"components/NewsArticleCreateButton.vue": [
"[",
"`",
"Enter"
],
"pages/admin/library/sources/index.vue": [
"Filesystem"
],
"server/api/v1/auth/signin/simple.post.ts": [
"boolean | undefined"
]
},
"i18n-ally.keepFulfilled": true,
"i18n-ally.keystyle": "nested",
"i18n-ally.sortKeys": true,
"prisma.pinToPrisma6": false,
"typescript.experimental.useTsgo": false
}
}
-52
View File
@@ -1,52 +0,0 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
// README at: https://github.com/devcontainers/templates/tree/main/src/javascript-node
{
"containerEnv": {
"DATABASE_URL": "postgres://drop:drop@db:5432/drop",
"EXTERNAL_URL": "http://localhost:4000",
"NUXT_PORT": "4000"
},
// Configure tool-specific properties.
"customizations": {
"vscode": {
"extensions": [
"lokalise.i18n-ally",
"esbenp.prettier-vscode",
"Prisma.prisma",
"bradlc.vscode-tailwindcss",
"Vue.volar",
"arktypeio.arkdark",
"EditorConfig.EditorConfig",
"dbaeumer.vscode-eslint"
]
}
},
// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
// "remoteUser": "root"
// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
"dockerComposeFile": "docker-compose.yml",
// Features to add to the dev container. More info: https://containers.dev/features.
"features": {
"ghcr.io/devcontainers-extra/features/protoc:1": {},
"ghcr.io/devcontainers/features/node:1": {},
"ghcr.io/devcontainers/features/rust:1": {
"version": "nightly-2026-03-29"
}
},
"name": "Node.js",
// Use 'forwardPorts' to make a list of ports inside the container available locally.
// "forwardPorts": [],
// Use 'postCreateCommand' to run commands after the container is created.
"postCreateCommand": "git submodule update --init --recursive",
"postStartCommand": "pnpm install && pnpm prisma migrate deploy && cd torrential && cargo build --release && cd ..",
"runServices": ["db", "app"],
"service": "app",
"shutdownAction": "stopCompose",
"workspaceFolder": "/workspaces/drop"
}
-26
View File
@@ -1,26 +0,0 @@
services:
app:
image: mcr.microsoft.com/devcontainers/base:noble
command: sleep infinity
volumes:
- ..:/workspaces/drop:cached
depends_on:
db:
condition: service_healthy
db:
image: postgres:14-alpine
environment:
POSTGRES_USER: drop
POSTGRES_PASSWORD: drop
POSTGRES_DB: drop
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U drop -d drop"]
interval: 5s
timeout: 5s
retries: 20
volumes:
pgdata:
+31 -9
View File
@@ -1116,6 +1116,14 @@ packages:
'@ioredis/commands@1.4.0':
resolution: {integrity: sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==}
'@isaacs/balanced-match@4.0.1':
resolution: {integrity: sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==}
engines: {node: 20 || >=22}
'@isaacs/brace-expansion@5.0.1':
resolution: {integrity: sha512-WMz71T1JS624nWj2n2fnYAuPovhv7EUhk69R6i9dsVyzxt5eM3bjwvgk9L+APE1TRscGysAVMANkB0jh0LQZrQ==}
engines: {node: 20 || >=22}
'@isaacs/cliui@8.0.2':
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
engines: {node: '>=12'}
@@ -2949,8 +2957,8 @@ packages:
boolbase@1.0.0:
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
brace-expansion@5.0.4:
resolution: {integrity: sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==}
brace-expansion@5.0.5:
resolution: {integrity: sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==}
engines: {node: 18 || 20 || >=22}
braces@3.0.3:
@@ -4639,6 +4647,10 @@ packages:
resolution: {integrity: sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg==}
hasBin: true
minimatch@10.0.3:
resolution: {integrity: sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==}
engines: {node: 20 || >=22}
minimatch@10.2.4:
resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==}
engines: {node: 18 || 20 || >=22}
@@ -7032,7 +7044,7 @@ snapshots:
dependencies:
'@eslint/object-schema': 2.1.6
debug: 4.4.3
minimatch: 10.2.4
minimatch: 10.0.3
transitivePeerDependencies:
- supports-color
@@ -7280,6 +7292,12 @@ snapshots:
'@ioredis/commands@1.4.0': {}
'@isaacs/balanced-match@4.0.1': {}
'@isaacs/brace-expansion@5.0.1':
dependencies:
'@isaacs/balanced-match': 4.0.1
'@isaacs/cliui@8.0.2':
dependencies:
string-width: 5.1.2
@@ -8780,7 +8798,7 @@ snapshots:
'@typescript-eslint/types': 8.54.0
'@typescript-eslint/visitor-keys': 8.54.0
debug: 4.4.3
minimatch: 10.2.4
minimatch: 10.0.3
semver: 7.7.4
tinyglobby: 0.2.15
ts-api-utils: 2.4.0(typescript@5.8.3)
@@ -9433,7 +9451,7 @@ snapshots:
boolbase@1.0.0: {}
brace-expansion@5.0.4:
brace-expansion@5.0.5:
dependencies:
balanced-match: 4.0.4
@@ -10253,7 +10271,7 @@ snapshots:
is-glob: 4.0.3
json-stable-stringify-without-jsonify: 1.0.1
lodash.merge: 4.6.2
minimatch: 10.2.4
minimatch: 10.0.3
natural-compare: 1.4.0
optionator: 0.9.4
optionalDependencies:
@@ -10551,7 +10569,7 @@ snapshots:
dependencies:
foreground-child: 3.3.1
jackspeak: 3.4.3
minimatch: 10.2.4
minimatch: 10.0.3
minipass: 7.1.2
package-json-from-dist: 1.0.1
path-scurry: 1.11.1
@@ -11297,9 +11315,13 @@ snapshots:
mini-svg-data-uri@1.4.4: {}
minimatch@10.0.3:
dependencies:
'@isaacs/brace-expansion': 5.0.1
minimatch@10.2.4:
dependencies:
brace-expansion: 5.0.4
brace-expansion: 5.0.5
minimist@1.2.8: {}
@@ -12364,7 +12386,7 @@ snapshots:
readdir-glob@1.1.3:
dependencies:
minimatch: 10.2.4
minimatch: 10.0.3
readdirp@4.1.2: {}
-8
View File
@@ -590,7 +590,6 @@ version = "0.16.3"
dependencies = [
"anyhow",
"async-trait",
"droplet_types",
"dyn-clone",
"futures",
"getrandom 0.3.4",
@@ -610,13 +609,6 @@ dependencies = [
"x509-parser 0.17.0",
]
[[package]]
name = "droplet_types"
version = "0.1.0"
dependencies = [
"serde",
]
[[package]]
name = "dyn-clone"
version = "1.0.20"
+2 -4
View File
@@ -3,11 +3,10 @@ use std::{
sync::{Arc, LazyLock},
};
use droplet_rs::manifest::ManifestWriterFactory;
use log::info;
use protobuf::Message;
use serde_json::json;
use tokio::{io::{BufWriter, SimplexStream}, spawn, sync::Semaphore};
use tokio::{spawn, sync::Semaphore};
use crate::{
proto::{
@@ -63,8 +62,7 @@ pub async fn generate_manifest_rpc(
.await;
});
},
None::<&dyn ManifestWriterFactory<Writer = SimplexStream>>, // Dummy type signature, not actually used
Some(&READER_SEMAPHORE),
Some(READER_SEMAPHORE.clone()),
)
.await?;